From efb5fa7d3844811ac6c813417dc8c10d8567d1f3 Mon Sep 17 00:00:00 2001 From: San Dang Date: Mon, 3 Aug 2026 14:29:31 +0700 Subject: [PATCH 01/49] fix(onboard): grant Jetson nvmap group write access Signed-off-by: San Dang --- docs/reference/troubleshooting.mdx | 62 +++++++++ scripts/setup-jetson.sh | 31 +++++ .../onboard/docker-gpu-jetson-groups.test.ts | 11 ++ src/lib/onboard/docker-gpu-jetson-groups.ts | 6 +- src/lib/onboard/docker-gpu-patch-recreate.ts | 2 +- .../onboard/sandbox-gpu-direct-proof.test.ts | 6 + src/lib/onboard/sandbox-gpu-preflight.ts | 4 +- test/e2e/live/jetson-nvmap-gpu.test.ts | 11 +- test/setup-jetson.test.ts | 124 +++++++++++++++--- 9 files changed, 234 insertions(+), 23 deletions(-) diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index e35128ca0f..b3f65893a7 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -204,6 +204,12 @@ The installer auto-detects NVIDIA Jetson devices (Orin and Thor) and applies req If the Jetson setup step fails, verify that you have `sudo` access and that Docker is installed and running. For JetPack 6 (L4T 36.x), the setup switches iptables to legacy mode and adjusts the Docker daemon configuration. +The setup also grants the existing `/dev/nvmap` owning group read-write access. +It writes the NemoClaw-managed `/etc/udev/rules.d/99-zz-nemoclaw-nvmap.rules` rule with mode `0660`, reloads the udev rules, and updates the live device. +The rule applies the mode when udev recreates `/dev/nvmap`, including after a host reboot. +This change grants write access to every member of the existing device-owning group, but it does not grant access to other users or run the sandbox as root. +If `/dev/nvmap` is absent, the setup warns, skips the rule, and continues installation. +CUDA can still fail until the device exists and you rerun the installer. For JetPack 7 (L4T 38.x / Thor), only bridge netfilter and sysctl settings are applied. For JetPack 7 (L4T 39.x), bridge netfilter is loaded only when the host is missing it. Some R39 images already ship with `br_netfilter` configured and are left untouched. @@ -2510,8 +2516,64 @@ To skip GPU passthrough entirely, rerun with `--no-gpu` or set `NEMOCLAW_SANDBOX Automatic GPU onboarding uses the compatibility path directly; it does not make a native attempt first. The path recreates the OpenShell-managed Docker container with NVIDIA GPU flags and propagates eligible host group IDs for the supported Jetson GPU device nodes. +NemoClaw propagates a device group only when the host node grants that group both read and write access and does not grant both permissions to other users. Use `NEMOCLAW_DOCKER_GPU_PATCH=0` only for troubleshooting because it bypasses that group propagation and CUDA may not initialize. +If CUDA reports `NvRmMemInitNvmap failed with Permission denied` and `cuInit(0)=999`, inspect the host device: + +```bash +stat -c '%A %U:%G %n' /dev/nvmap +``` + +The group permission characters must be `rw`, such as the group field in `crw-rw----`. +On JetPack 6, rerun the NemoClaw installer to apply the persistent udev rule and update the live device: + + +The installer grants write access to every member of the existing `/dev/nvmap` owning group. +It also persists mode `0660` when udev recreates the device. + + +```bash +curl -fsSL https://www.nvidia.com/nemoclaw.sh | bash +``` + +Verify the rule content and the live device permissions: + +```bash +grep -Fx 'KERNEL=="nvmap", MODE="0660"' /etc/udev/rules.d/99-zz-nemoclaw-nvmap.rules +stat -c '%A %U:%G %n' /dev/nvmap +``` + +The first command must print the exact rule. +The group permission characters from the second command must be `rw`. + + +The following temporary remediation grants write access to every member of the existing `/dev/nvmap` owning group. +The permission lasts until udev recreates the device, such as during a host reboot. + + +If you need to diagnose the failure before rerunning the installer, update the live device and verify its group permissions: + +```bash +sudo chmod g+rw /dev/nvmap +stat -c '%A %U:%G %n' /dev/nvmap +``` + +Clean up the failed sandbox with the command from the onboarding output, then rerun onboarding. + +NemoClaw uninstall does not remove the Jetson host rule. +If you no longer need non-root Jetson GPU passthrough, remove the rule and reload udev: + +```bash +sudo rm /etc/udev/rules.d/99-zz-nemoclaw-nvmap.rules +sudo udevadm control --reload-rules +test ! -e /etc/udev/rules.d/99-zz-nemoclaw-nvmap.rules && echo "NemoClaw nvmap rule removed" +``` + +The final command must print `NemoClaw nvmap rule removed`. +Removing the rule does not change the live device mode. +The remaining platform udev rules determine the mode after udev recreates `/dev/nvmap`, such as during a host reboot. + #### Common compatibility-path recovery If the compatibility attempt fails on any host, onboarding leaves the failed sandbox and diagnostic bundle in place so you can inspect the OpenShell and Docker state. diff --git a/scripts/setup-jetson.sh b/scripts/setup-jetson.sh index 6b054780d0..4ee54712cd 100755 --- a/scripts/setup-jetson.sh +++ b/scripts/setup-jetson.sh @@ -7,6 +7,10 @@ set -euo pipefail SUDO=() ((EUID != 0)) && SUDO=(sudo) +NVMAP_DEVICE="/dev/nvmap" +NVMAP_UDEV_RULE="/etc/udev/rules.d/99-zz-nemoclaw-nvmap.rules" +NVMAP_UDEV_RULE_CONTENT='KERNEL=="nvmap", MODE="0660"' + info() { printf "[INFO] %s\n" "$*" } @@ -56,6 +60,32 @@ apply_br_netfilter_setup() { echo "net.bridge.bridge-nf-call-iptables=1" | "${SUDO[@]}" tee /etc/sysctl.d/99-nemoclaw.conf >/dev/null } +configure_nvmap_group_access() { + local device_state device_type verified_state verified_permissions + + if ! device_state="$(LC_ALL=C stat -c '%F|%A' "$NVMAP_DEVICE" 2>/dev/null)"; then + warn "JetPack 6 host setup could not find $NVMAP_DEVICE. Non-root sandbox CUDA can fail until this device exists." + return 0 + fi + + device_type="${device_state%%|*}" + [[ "$device_type" == "character special file" ]] \ + || error "$NVMAP_DEVICE must be a character device before NemoClaw changes its group permissions." + + warn "JetPack 6 host setup grants every member of the existing $NVMAP_DEVICE owning group write access and persists mode 0660 when udev recreates the device." + printf '%s\n' "$NVMAP_UDEV_RULE_CONTENT" | "${SUDO[@]}" tee "$NVMAP_UDEV_RULE" >/dev/null + "${SUDO[@]}" udevadm control --reload-rules + "${SUDO[@]}" chmod g+rw "$NVMAP_DEVICE" + + verified_state="$(LC_ALL=C stat -c '%F|%A' "$NVMAP_DEVICE" 2>/dev/null)" \ + || error "Could not verify $NVMAP_DEVICE after granting group read-write access." + IFS='|' read -r device_type verified_permissions <<<"$verified_state" + [[ "$device_type" == "character special file" && "${verified_permissions:4:2}" == "rw" ]] \ + || error "$NVMAP_DEVICE does not grant its owning group read-write access after host setup." + + info "$NVMAP_DEVICE grants its owning group read-write access. The udev rule $NVMAP_UDEV_RULE preserves this mode after reboot." +} + warn_host_setup_skipped() { warn "Skipped Jetson host setup: iptables legacy mode and the Docker daemon.json adjustment (L4T 36.x only), and br_netfilter with net.bridge.bridge-nf-call-iptables=1 (every release)." warn "Without br_netfilter, k3s inside the OpenShell gateway cannot NAT sandbox pod traffic to ClusterIP services, so sandbox pods cannot reach CoreDNS." @@ -188,6 +218,7 @@ except Exception: os.unlink(tmp) raise PYEOF + configure_nvmap_group_access ;; jp7-r38) # JP7 R38 does not need iptables or Docker daemon.json changes. diff --git a/src/lib/onboard/docker-gpu-jetson-groups.test.ts b/src/lib/onboard/docker-gpu-jetson-groups.test.ts index 90b6dfb544..25d83eb274 100644 --- a/src/lib/onboard/docker-gpu-jetson-groups.test.ts +++ b/src/lib/onboard/docker-gpu-jetson-groups.test.ts @@ -35,6 +35,8 @@ describe("detectTegraDeviceGroupGids", () => { { gid: 0, mode: 0o660 }, { gid: 2_147_483_648, mode: 0o660 }, { gid: 44, mode: 0o600 }, + { gid: 44, mode: 0o440 }, + { gid: 44, mode: 0o620 }, { gid: 104, mode: 0o666 }, ]; let index = 0; @@ -47,6 +49,15 @@ describe("detectTegraDeviceGroupGids", () => { ).toEqual([]); }); + it("does not treat a read-only nvmap group as CUDA access (#7610)", () => { + expect( + detectTegraDeviceGroupGids({ + statDeviceAccess: () => ({ gid: 44, mode: 0o440 }), + listDevicePaths: () => ["/dev/nvmap"], + }), + ).toEqual([]); + }); + it("returns no GIDs when Tegra nodes are missing or unreadable", () => { expect( detectTegraDeviceGroupGids({ diff --git a/src/lib/onboard/docker-gpu-jetson-groups.ts b/src/lib/onboard/docker-gpu-jetson-groups.ts index 4cfcd098f4..e484c19ccc 100644 --- a/src/lib/onboard/docker-gpu-jetson-groups.ts +++ b/src/lib/onboard/docker-gpu-jetson-groups.ts @@ -92,13 +92,15 @@ export function detectTegraDeviceGroupGids( const gid = access?.gid ?? null; const groupAccessBits = access === null ? 0 : (access.mode >> 3) & READ_WRITE_PERMISSION_BITS; const otherAccessBits = access === null ? 0 : access.mode & READ_WRITE_PERMISSION_BITS; - const groupAddsAccess = (groupAccessBits & ~otherAccessBits) !== 0; + const groupAddsReadWriteAccess = + groupAccessBits === READ_WRITE_PERMISSION_BITS && + otherAccessBits !== READ_WRITE_PERMISSION_BITS; if ( gid !== null && Number.isSafeInteger(gid) && gid > 0 && gid <= MAX_DOCKER_SUPPLEMENTARY_GID && - groupAddsAccess + groupAddsReadWriteAccess ) { gids.add(String(gid)); } diff --git a/src/lib/onboard/docker-gpu-patch-recreate.ts b/src/lib/onboard/docker-gpu-patch-recreate.ts index b205c144af..15439eeeb0 100644 --- a/src/lib/onboard/docker-gpu-patch-recreate.ts +++ b/src/lib/onboard/docker-gpu-patch-recreate.ts @@ -273,7 +273,7 @@ export function recreateOpenShellDockerSandboxContainer( ); } else { console.warn( - " ⚠ Could not resolve the group owning Jetson Tegra GPU device nodes (/dev/nvmap); CUDA may fail with NvRmMemInitNvmap permission denied. Confirm /dev/nvmap exists and is group-readable on the host.", + " ⚠ Could not resolve a read-write group for Jetson Tegra GPU device nodes (/dev/nvmap); CUDA may fail with NvRmMemInitNvmap permission denied. Confirm /dev/nvmap exists and grants its owning group read-write access on the host.", ); } } diff --git a/src/lib/onboard/sandbox-gpu-direct-proof.test.ts b/src/lib/onboard/sandbox-gpu-direct-proof.test.ts index a6d55681ee..76052c9ebe 100644 --- a/src/lib/onboard/sandbox-gpu-direct-proof.test.ts +++ b/src/lib/onboard/sandbox-gpu-direct-proof.test.ts @@ -198,6 +198,12 @@ describe("direct sandbox GPU proof", () => { expect(result.detail).toContain("cuInit(0)=999"); const warnings = warnSpy.mock.calls.map((call) => call[0]).join("\n"); expect(warnings).toContain("/dev/nvmap"); + expect(warnings).toContain( + "chmod grants every member of the nvmap owning group write access", + ); + expect(warnings.indexOf("chmod grants every member")).toBeLessThan( + warnings.indexOf("sudo chmod g+rw /dev/nvmap"), + ); } finally { warnSpy.mockRestore(); } diff --git a/src/lib/onboard/sandbox-gpu-preflight.ts b/src/lib/onboard/sandbox-gpu-preflight.ts index 5fbbde0fe0..2d15ab64dc 100644 --- a/src/lib/onboard/sandbox-gpu-preflight.ts +++ b/src/lib/onboard/sandbox-gpu-preflight.ts @@ -65,7 +65,9 @@ export function jetsonGpuProofRemediationLines(): string[] { return [ "Jetson/Tegra CUDA proof did not pass. CUDA needs access to the Tegra device", "nodes; confirm the sandbox propagates them and the agent user's groups:", - " ls -l /dev/nvmap /dev/nvhost-* (must be readable by the sandbox)", + " ls -l /dev/nvmap /dev/nvhost-* (nvmap must grant the sandbox group read-write access)", + " warning: chmod grants every member of the nvmap owning group write access", + " sudo chmod g+rw /dev/nvmap (host remediation until udev recreates the device)", " add the host video/render groups via --group-add when recreating", "Then recreate the sandbox, or force CPU behavior with NEMOCLAW_SANDBOX_GPU=0.", ]; diff --git a/test/e2e/live/jetson-nvmap-gpu.test.ts b/test/e2e/live/jetson-nvmap-gpu.test.ts index b69de1713a..f49aab0d34 100644 --- a/test/e2e/live/jetson-nvmap-gpu.test.ts +++ b/test/e2e/live/jetson-nvmap-gpu.test.ts @@ -196,7 +196,7 @@ exit "$status"`, progress.phase("confirm nvmap and NVIDIA Docker runtime"); const hostNvmap = await hostShell( host, - "ls -l /dev/nvmap && stat -c 'gid=%g group=%G' /dev/nvmap", + "ls -l /dev/nvmap && stat -c 'gid=%g group=%G mode=%a' /dev/nvmap", "phase-0-host-nvmap", ); expect(hostNvmap.exitCode, resultText(hostNvmap)).toBe(0); @@ -264,7 +264,7 @@ exit "$status"`, expect(sandboxId.exitCode, resultText(sandboxId)).toBe(0); expectGroupMembership(resultText(sandboxId), hostNvmapGid); - // A6: /dev/nvmap must be mounted/present inside the sandbox. + // A6: /dev/nvmap must be present and read-write for the sandbox user. const sandboxNvmap = await sandbox.execShell( SANDBOX_NAME, trustedSandboxShellScript("ls -l /dev/nvmap"), @@ -273,6 +273,13 @@ exit "$status"`, expect(sandboxNvmap.exitCode, resultText(sandboxNvmap)).toBe(0); expect(resultText(sandboxNvmap)).toContain("/dev/nvmap"); + const sandboxNvmapAccess = await sandbox.execShell( + SANDBOX_NAME, + trustedSandboxShellScript("test -r /dev/nvmap && test -w /dev/nvmap"), + { artifactName: "phase-3-sandbox-nvmap-access", env: env(), timeoutMs: 60_000 }, + ); + expect(sandboxNvmapAccess.exitCode, resultText(sandboxNvmapAccess)).toBe(0); + // A7: authoritative CUDA usability proof must succeed, not reproduce // NvRmMemInitNvmap permission denial / cuInit(0)=999 from #4231. progress.phase("prove CUDA initialization inside the sandbox"); diff --git a/test/setup-jetson.test.ts b/test/setup-jetson.test.ts index 67ea13c5a0..af9b1c4076 100644 --- a/test/setup-jetson.test.ts +++ b/test/setup-jetson.test.ts @@ -20,9 +20,12 @@ const SCRIPT_PATH = path.join(import.meta.dirname, "..", "scripts", "setup-jetso const HOST_MUTATION_COMMANDS = [ "sudo", + "chmod", "modprobe", + "stat", "sysctl", "tee", + "udevadm", "update-alternatives", "systemctl", "python3", @@ -33,21 +36,56 @@ type SetupJetsonRun = { stdout: string; stderr: string; headArgs: string; + commandLog: string; }; function withJetsonReleaseSandbox( - run: (paths: { headArgsPath: string; releasePath: string; stubDir: string }) => T, + run: (paths: { + commandLogPath: string; + headArgsPath: string; + releasePath: string; + statCountPath: string; + stubDir: string; + }) => T, ): T { const tempDir = mkdtempSync(path.join(tmpdir(), "nemoclaw-jetson-release-")); try { const stubDir = path.join(tempDir, "bin"); + const commandLogPath = path.join(tempDir, "command-log"); const headArgsPath = path.join(tempDir, "head-args"); const releasePath = path.join(tempDir, "nv_tegra_release"); + const statCountPath = path.join(tempDir, "stat-count"); mkdirSync(stubDir); + writeFileSync(commandLogPath, ""); for (const command of HOST_MUTATION_COMMANDS) { const stubPath = path.join(stubDir, command); - writeFileSync(stubPath, "#!/usr/bin/env bash\nexit 0\n"); + writeFileSync( + stubPath, + [ + "#!/usr/bin/env bash", + "set -euo pipefail", + `printf '%s %s\\n' ${JSON.stringify(command)} "$*" >> ${JSON.stringify(commandLogPath)}`, + `if [[ ${JSON.stringify(command)} == "tee" || ( ${JSON.stringify(command)} == "sudo" && "\${1:-}" == "tee" ) ]]; then`, + " input=", + " IFS= read -r input || true", + ` printf 'stdin %s\\n' "$input" >> ${JSON.stringify(commandLogPath)}`, + "fi", + `if [[ ${JSON.stringify(command)} == "stat" ]]; then`, + ` if [[ -f ${JSON.stringify(statCountPath)} ]]; then`, + ' output="${NEMOCLAW_TEST_STAT_OUTPUT_AFTER:-${NEMOCLAW_TEST_STAT_OUTPUT:-}}"', + " else", + ` : > ${JSON.stringify(statCountPath)}`, + ' output="${NEMOCLAW_TEST_STAT_OUTPUT:-}"', + " fi", + ' [[ -n "$output" ]] || exit "${NEMOCLAW_TEST_STAT_STATUS:-1}"', + " printf '%s\\n' \"$output\"", + ' exit "${NEMOCLAW_TEST_STAT_STATUS:-0}"', + "fi", + "exit 0", + "", + ].join("\n"), + ); chmodSync(stubPath, 0o755); } @@ -66,7 +104,7 @@ function withJetsonReleaseSandbox( ); chmodSync(headStubPath, 0o755); - return run({ headArgsPath, releasePath, stubDir }); + return run({ commandLogPath, headArgsPath, releasePath, statCountPath, stubDir }); } finally { rmSync(tempDir, { recursive: true, force: true }); } @@ -75,6 +113,7 @@ function withJetsonReleaseSandbox( function spawnSetupJetson( stubDir: string, headArgsPath: string, + commandLogPath: string, extraEnv: NodeJS.ProcessEnv = {}, ): SetupJetsonRun { const result = spawnSync("bash", [SCRIPT_PATH], { @@ -91,19 +130,20 @@ function spawnSetupJetson( stdout: result.stdout, stderr: result.stderr, headArgs: readFileSync(headArgsPath, "utf-8").trim(), + commandLog: readFileSync(commandLogPath, "utf-8").trim(), }; } function runSetupJetson(releaseLine: string): SetupJetsonRun { - return withJetsonReleaseSandbox(({ headArgsPath, releasePath, stubDir }) => { + return withJetsonReleaseSandbox(({ commandLogPath, headArgsPath, releasePath, stubDir }) => { writeFileSync(releasePath, `${releaseLine}\n`); - return spawnSetupJetson(stubDir, headArgsPath); + return spawnSetupJetson(stubDir, headArgsPath, commandLogPath); }); } function runSetupJetsonWithoutReleaseFile(): SetupJetsonRun { - return withJetsonReleaseSandbox(({ headArgsPath, stubDir }) => - spawnSetupJetson(stubDir, headArgsPath), + return withJetsonReleaseSandbox(({ commandLogPath, headArgsPath, stubDir }) => + spawnSetupJetson(stubDir, headArgsPath, commandLogPath), ); } @@ -298,16 +338,18 @@ describe("setup-jetson host setup on an unrecognized L4T release (#7612)", () => }); it("ignores an inherited test release-path override during normal installation", () => { - const result = withJetsonReleaseSandbox(({ headArgsPath, releasePath, stubDir }) => { - const inheritedOverridePath = path.join(path.dirname(releasePath), "inherited-release"); - writeFileSync( - inheritedOverridePath, - "# R36 (release), REVISION: 5.1, GCID: 12345678, BOARD: t186ref\n", - ); - return spawnSetupJetson(stubDir, headArgsPath, { - NEMOCLAW_TEST_NV_TEGRA_RELEASE_PATH: inheritedOverridePath, - }); - }); + const result = withJetsonReleaseSandbox( + ({ commandLogPath, headArgsPath, releasePath, stubDir }) => { + const inheritedOverridePath = path.join(path.dirname(releasePath), "inherited-release"); + writeFileSync( + inheritedOverridePath, + "# R36 (release), REVISION: 5.1, GCID: 12345678, BOARD: t186ref\n", + ); + return spawnSetupJetson(stubDir, headArgsPath, commandLogPath, { + NEMOCLAW_TEST_NV_TEGRA_RELEASE_PATH: inheritedOverridePath, + }); + }, + ); expect(result.status).toBe(0); expect(result.stdout).toBe(""); @@ -323,3 +365,51 @@ describe("setup-jetson host setup on an unrecognized L4T release (#7612)", () => expect(result.stderr).not.toContain("Skipped Jetson host setup"); }); }); + +describe("setup-jetson JetPack 6 nvmap access", () => { + it("grants the nvmap owning group read-write access and persists the mode on JetPack 6 (#7610)", () => { + const result = withJetsonReleaseSandbox( + ({ commandLogPath, headArgsPath, releasePath, stubDir }) => { + writeFileSync( + releasePath, + "# R36 (release), REVISION: 5.1, GCID: 12345678, BOARD: t186ref\n", + ); + return spawnSetupJetson(stubDir, headArgsPath, commandLogPath, { + NEMOCLAW_TEST_STAT_OUTPUT: "character special file|cr--r-----", + NEMOCLAW_TEST_STAT_OUTPUT_AFTER: "character special file|cr--rw----", + }); + }, + ); + + expect(result.status).toBe(0); + expect(result.commandLog).toContain("tee /etc/udev/rules.d/99-zz-nemoclaw-nvmap.rules"); + expect(result.commandLog).toContain('stdin KERNEL=="nvmap", MODE="0660"'); + expect(result.commandLog).toContain("udevadm control --reload-rules"); + expect(result.commandLog).toContain("chmod g+rw /dev/nvmap"); + expect(result.stdout).toContain("/dev/nvmap grants its owning group read-write access"); + expect(result.stdout).toContain("preserves this mode after reboot"); + expect(result.stderr).toContain( + "grants every member of the existing /dev/nvmap owning group write access", + ); + expect(result.stderr).toContain("persists mode 0660 when udev recreates the device"); + }); + + it("rejects a non-device nvmap path before changing host permissions (#7610)", () => { + const result = withJetsonReleaseSandbox( + ({ commandLogPath, headArgsPath, releasePath, stubDir }) => { + writeFileSync( + releasePath, + "# R36 (release), REVISION: 5.1, GCID: 12345678, BOARD: t186ref\n", + ); + return spawnSetupJetson(stubDir, headArgsPath, commandLogPath, { + NEMOCLAW_TEST_STAT_OUTPUT: "regular file|-rw-r-----", + }); + }, + ); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("/dev/nvmap must be a character device"); + expect(result.commandLog).not.toContain("tee /etc/udev/rules.d/99-zz-nemoclaw-nvmap.rules"); + expect(result.commandLog).not.toContain("chmod g+rw /dev/nvmap"); + }); +}); From 09bef646af8717082e5b888702e180d4856eef4e Mon Sep 17 00:00:00 2001 From: San Dang Date: Mon, 3 Aug 2026 14:46:27 +0700 Subject: [PATCH 02/49] docs(troubleshooting): simplify Jetson nvmap guidance Signed-off-by: San Dang --- docs/reference/troubleshooting.mdx | 50 ++---------------------------- 1 file changed, 3 insertions(+), 47 deletions(-) diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index b3f65893a7..909bd10cc8 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -204,12 +204,6 @@ The installer auto-detects NVIDIA Jetson devices (Orin and Thor) and applies req If the Jetson setup step fails, verify that you have `sudo` access and that Docker is installed and running. For JetPack 6 (L4T 36.x), the setup switches iptables to legacy mode and adjusts the Docker daemon configuration. -The setup also grants the existing `/dev/nvmap` owning group read-write access. -It writes the NemoClaw-managed `/etc/udev/rules.d/99-zz-nemoclaw-nvmap.rules` rule with mode `0660`, reloads the udev rules, and updates the live device. -The rule applies the mode when udev recreates `/dev/nvmap`, including after a host reboot. -This change grants write access to every member of the existing device-owning group, but it does not grant access to other users or run the sandbox as root. -If `/dev/nvmap` is absent, the setup warns, skips the rule, and continues installation. -CUDA can still fail until the device exists and you rerun the installer. For JetPack 7 (L4T 38.x / Thor), only bridge netfilter and sysctl settings are applied. For JetPack 7 (L4T 39.x), bridge netfilter is loaded only when the host is missing it. Some R39 images already ship with `br_netfilter` configured and are left untouched. @@ -2516,63 +2510,25 @@ To skip GPU passthrough entirely, rerun with `--no-gpu` or set `NEMOCLAW_SANDBOX Automatic GPU onboarding uses the compatibility path directly; it does not make a native attempt first. The path recreates the OpenShell-managed Docker container with NVIDIA GPU flags and propagates eligible host group IDs for the supported Jetson GPU device nodes. -NemoClaw propagates a device group only when the host node grants that group both read and write access and does not grant both permissions to other users. Use `NEMOCLAW_DOCKER_GPU_PATCH=0` only for troubleshooting because it bypasses that group propagation and CUDA may not initialize. -If CUDA reports `NvRmMemInitNvmap failed with Permission denied` and `cuInit(0)=999`, inspect the host device: - -```bash -stat -c '%A %U:%G %n' /dev/nvmap -``` - -The group permission characters must be `rw`, such as the group field in `crw-rw----`. -On JetPack 6, rerun the NemoClaw installer to apply the persistent udev rule and update the live device: +If CUDA reports `NvRmMemInitNvmap failed with Permission denied` and `cuInit(0)=999` on JetPack 6, rerun the NemoClaw installer to apply persistent group read-write access to `/dev/nvmap`: The installer grants write access to every member of the existing `/dev/nvmap` owning group. -It also persists mode `0660` when udev recreates the device. ```bash curl -fsSL https://www.nvidia.com/nemoclaw.sh | bash ``` -Verify the rule content and the live device permissions: +Verify the live device permissions: ```bash -grep -Fx 'KERNEL=="nvmap", MODE="0660"' /etc/udev/rules.d/99-zz-nemoclaw-nvmap.rules stat -c '%A %U:%G %n' /dev/nvmap ``` -The first command must print the exact rule. -The group permission characters from the second command must be `rw`. - - -The following temporary remediation grants write access to every member of the existing `/dev/nvmap` owning group. -The permission lasts until udev recreates the device, such as during a host reboot. - - -If you need to diagnose the failure before rerunning the installer, update the live device and verify its group permissions: - -```bash -sudo chmod g+rw /dev/nvmap -stat -c '%A %U:%G %n' /dev/nvmap -``` - -Clean up the failed sandbox with the command from the onboarding output, then rerun onboarding. - -NemoClaw uninstall does not remove the Jetson host rule. -If you no longer need non-root Jetson GPU passthrough, remove the rule and reload udev: - -```bash -sudo rm /etc/udev/rules.d/99-zz-nemoclaw-nvmap.rules -sudo udevadm control --reload-rules -test ! -e /etc/udev/rules.d/99-zz-nemoclaw-nvmap.rules && echo "NemoClaw nvmap rule removed" -``` - -The final command must print `NemoClaw nvmap rule removed`. -Removing the rule does not change the live device mode. -The remaining platform udev rules determine the mode after udev recreates `/dev/nvmap`, such as during a host reboot. +The group permission characters must be `rw`, such as the group field in `crw-rw----`. #### Common compatibility-path recovery From 632a5c9eb44feb8840b0c47a7f15e6af372ab5da Mon Sep 17 00:00:00 2001 From: San Dang Date: Mon, 3 Aug 2026 14:55:57 +0700 Subject: [PATCH 03/49] test(onboard): cover failed nvmap permission verification Signed-off-by: San Dang --- test/setup-jetson.test.ts | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/test/setup-jetson.test.ts b/test/setup-jetson.test.ts index af9b1c4076..484da6ff0a 100644 --- a/test/setup-jetson.test.ts +++ b/test/setup-jetson.test.ts @@ -412,4 +412,25 @@ describe("setup-jetson JetPack 6 nvmap access", () => { expect(result.commandLog).not.toContain("tee /etc/udev/rules.d/99-zz-nemoclaw-nvmap.rules"); expect(result.commandLog).not.toContain("chmod g+rw /dev/nvmap"); }); + + it("fails when nvmap remains read-only after host setup (#7610)", () => { + const result = withJetsonReleaseSandbox( + ({ commandLogPath, headArgsPath, releasePath, stubDir }) => { + writeFileSync( + releasePath, + "# R36 (release), REVISION: 5.1, GCID: 12345678, BOARD: t186ref\n", + ); + return spawnSetupJetson(stubDir, headArgsPath, commandLogPath, { + NEMOCLAW_TEST_STAT_OUTPUT: "character special file|cr--r-----", + NEMOCLAW_TEST_STAT_OUTPUT_AFTER: "character special file|cr--r-----", + }); + }, + ); + + expect(result.status).toBe(1); + expect(result.stderr).toContain( + "/dev/nvmap does not grant its owning group read-write access after host setup", + ); + expect(result.commandLog).toContain("chmod g+rw /dev/nvmap"); + }); }); From 0bb4a74e777c2bbd866b9bbcd960a36f512eb9d5 Mon Sep 17 00:00:00 2001 From: San Dang Date: Mon, 3 Aug 2026 14:58:50 +0700 Subject: [PATCH 04/49] docs(troubleshooting): clarify nvmap rule lifecycle Signed-off-by: San Dang --- docs/reference/troubleshooting.mdx | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 909bd10cc8..e12d04c863 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -2513,6 +2513,7 @@ The path recreates the OpenShell-managed Docker container with NVIDIA GPU flags Use `NEMOCLAW_DOCKER_GPU_PATCH=0` only for troubleshooting because it bypasses that group propagation and CUDA may not initialize. If CUDA reports `NvRmMemInitNvmap failed with Permission denied` and `cuInit(0)=999` on JetPack 6, rerun the NemoClaw installer to apply persistent group read-write access to `/dev/nvmap`: +If the device is absent, setup warns and continues without installing the rule. The installer grants write access to every member of the existing `/dev/nvmap` owning group. @@ -2522,13 +2523,16 @@ The installer grants write access to every member of the existing `/dev/nvmap` o curl -fsSL https://www.nvidia.com/nemoclaw.sh | bash ``` -Verify the live device permissions: +Verify the persistent rule and the live device permissions: ```bash +grep -Fx 'KERNEL=="nvmap", MODE="0660"' /etc/udev/rules.d/99-zz-nemoclaw-nvmap.rules stat -c '%A %U:%G %n' /dev/nvmap ``` -The group permission characters must be `rw`, such as the group field in `crw-rw----`. +The rule must match exactly, and the group permission characters must be `rw`, such as the group field in `crw-rw----`. +The rule reapplies mode `0660` when udev recreates `/dev/nvmap`, including after reboot. +NemoClaw uninstall does not remove the rule; delete it and reload udev to stop future reapplication, while the live mode remains until device recreation. #### Common compatibility-path recovery From ea03eabb72d813ba15b1a50c09687ae5044d5f8d Mon Sep 17 00:00:00 2001 From: San Dang Date: Mon, 3 Aug 2026 15:04:10 +0700 Subject: [PATCH 05/49] test(onboard): cover missing Jetson nvmap device Signed-off-by: San Dang --- test/setup-jetson.test.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/test/setup-jetson.test.ts b/test/setup-jetson.test.ts index 484da6ff0a..34eb97a5b9 100644 --- a/test/setup-jetson.test.ts +++ b/test/setup-jetson.test.ts @@ -413,6 +413,24 @@ describe("setup-jetson JetPack 6 nvmap access", () => { expect(result.commandLog).not.toContain("chmod g+rw /dev/nvmap"); }); + it("skips nvmap host changes when the device is absent (#7610)", () => { + const result = withJetsonReleaseSandbox( + ({ commandLogPath, headArgsPath, releasePath, stubDir }) => { + writeFileSync( + releasePath, + "# R36 (release), REVISION: 5.1, GCID: 12345678, BOARD: t186ref\n", + ); + return spawnSetupJetson(stubDir, headArgsPath, commandLogPath); + }, + ); + + expect(result.status).toBe(0); + expect(result.stderr).toContain("could not find /dev/nvmap"); + expect(result.commandLog).not.toContain("tee /etc/udev/rules.d/99-zz-nemoclaw-nvmap.rules"); + expect(result.commandLog).not.toContain("chmod g+rw /dev/nvmap"); + expect(result.stdout).not.toContain("/dev/nvmap grants its owning group read-write access"); + }); + it("fails when nvmap remains read-only after host setup (#7610)", () => { const result = withJetsonReleaseSandbox( ({ commandLogPath, headArgsPath, releasePath, stubDir }) => { From 29262e9dbbe2d9231041ed22f196cd2256d81a6b Mon Sep 17 00:00:00 2001 From: San Dang Date: Tue, 4 Aug 2026 17:05:55 +0700 Subject: [PATCH 06/49] fix(onboard): preserve Jetson sandbox GPU access Signed-off-by: San Dang --- Dockerfile | 2 + docs/reference/troubleshooting.mdx | 23 +++++++- scripts/jetson-device-group-bootstrap.sh | 53 +++++++++++++++++ src/lib/onboard.ts | 1 + src/lib/onboard/docker-gpu-patch-clone.ts | 49 +++++++++++---- .../onboard/docker-gpu-patch-jetson.test.ts | 59 ++++++++++++++++++- src/lib/onboard/docker-gpu-patch-recreate.ts | 9 ++- src/lib/onboard/docker-gpu-patch-types.ts | 12 ++-- src/lib/onboard/docker-gpu-sandbox-create.ts | 3 + src/lib/onboard/initial-policy.test.ts | 24 ++++++++ src/lib/onboard/initial-policy.ts | 13 ++++ .../onboard/sandbox-create-intent-types.ts | 1 + src/lib/onboard/sandbox-create-intent.ts | 5 ++ .../sandbox-create-plan-materialization.ts | 1 + src/lib/onboard/sandbox-create-plan.test.ts | 11 ++++ src/lib/onboard/sandbox-gpu-create-flow.ts | 1 + .../onboard/sandbox-gpu-create-run-attempt.ts | 1 + src/lib/onboard/sandbox-gpu-create.ts | 1 + test/e2e/live/jetson-nvmap-gpu.test.ts | 6 +- test/openclaw-final-image-layout.test.ts | 1 + 20 files changed, 250 insertions(+), 26 deletions(-) create mode 100755 scripts/jetson-device-group-bootstrap.sh diff --git a/Dockerfile b/Dockerfile index 68f9b9d248..851d8c6855 100644 --- a/Dockerfile +++ b/Dockerfile @@ -96,6 +96,7 @@ COPY scripts/lib/sandbox-rlimits.sh /usr/local/lib/nemoclaw/sandbox-rlimits.sh COPY scripts/lib/openclaw_device_approval_policy.py /usr/local/lib/nemoclaw/openclaw_device_approval_policy.py COPY scripts/lib/clean_runtime_shell_env_shim.py /usr/local/lib/nemoclaw/clean_runtime_shell_env_shim.py COPY scripts/lib/normalize_mutable_config_perms.py /usr/local/lib/nemoclaw/normalize_mutable_config_perms.py +COPY scripts/jetson-device-group-bootstrap.sh /usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh COPY scripts/state-dir-guard.py /usr/local/lib/nemoclaw/state-dir-guard.py COPY scripts/openclaw-config-guard.py /usr/local/lib/nemoclaw/openclaw-config-guard.py COPY scripts/managed-gateway-control.py /usr/local/lib/nemoclaw/managed-gateway-control.py @@ -985,6 +986,7 @@ RUN discovery_contract="$(node /usr/local/lib/nemoclaw/mcp-tool-discovery-runtim # Copy startup script and shared sandbox initialisation library RUN chmod 755 /usr/local/bin/nemoclaw-start /usr/local/bin/nemoclaw-codex-acp \ /usr/local/lib/nemoclaw/sandbox-init.sh \ + /usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh \ /scripts/generate-openclaw-config.mts \ /scripts/validate-openclaw-tool-search.mts \ /src/lib/messaging/applier/build/messaging-build-applier.mts \ diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index dbec0eae18..8351798928 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -2687,8 +2687,15 @@ To skip GPU passthrough entirely, rerun with `--no-gpu` or set `NEMOCLAW_SANDBOX #### Jetson and Tegra compatibility default Automatic GPU onboarding uses the compatibility path directly; it does not make a native attempt first. -The path recreates the OpenShell-managed Docker container with NVIDIA GPU flags and propagates eligible host group IDs for the supported Jetson GPU device nodes. -Use `NEMOCLAW_DOCKER_GPU_PATCH=0` only for troubleshooting because it bypasses that group propagation and CUDA may not initialize. +The path recreates the OpenShell-managed Docker container with NVIDIA GPU flags. + + +It also keeps eligible Jetson GPU device group memberships when OpenShell starts the nonroot sandbox user. +The compatibility policy permits read-only access to the NVIDIA runtime library directory at `/opt/nvidia/l4t-gpu-libs` when that path is present. + + + +Use `NEMOCLAW_DOCKER_GPU_PATCH=0` only for troubleshooting because it bypasses the compatibility patch and CUDA may not initialize. If CUDA reports `NvRmMemInitNvmap failed with Permission denied` and `cuInit(0)=999` on JetPack 6, rerun the NemoClaw installer to apply persistent group read-write access to `/dev/nvmap`: If the device is absent, setup warns and continues without installing the rule. @@ -2712,6 +2719,18 @@ The rule must match exactly, and the group permission characters must be `rw`, s The rule reapplies mode `0660` when udev recreates `/dev/nvmap`, including after reboot. NemoClaw uninstall does not remove the rule; delete it and reload udev to stop future reapplication, while the live mode remains until device recreation. + + +After onboarding recreates the sandbox, verify CUDA as the nonroot sandbox user: + +```bash +$$nemoclaw exec -- python3 -c 'import ctypes; lib=ctypes.CDLL("libcuda.so.1"); print(f"cuInit(0)={lib.cuInit(0)}")' +``` + +The command must print `cuInit(0)=0`. + + + #### Common compatibility-path recovery After compatibility recreation starts, onboarding keeps the pre-patch container as a rollback backup until the replacement passes the Ready, direct GPU, and applicable local-inference checks. diff --git a/scripts/jetson-device-group-bootstrap.sh b/scripts/jetson-device-group-bootstrap.sh new file mode 100755 index 0000000000..3e2d3c3f2a --- /dev/null +++ b/scripts/jetson-device-group-bootstrap.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +readonly GROUP_GIDS="${NEMOCLAW_JETSON_DEVICE_GROUP_GIDS:-}" +readonly SANDBOX_USER="sandbox" + +[[ "$(id -u)" == "0" ]] || { + echo "Jetson device-group bootstrap must run as root before the OpenShell supervisor." >&2 + exit 1 +} +[[ $# -gt 0 ]] || { + echo "Jetson device-group bootstrap requires the OpenShell supervisor command." >&2 + exit 1 +} +id "$SANDBOX_USER" >/dev/null 2>&1 || { + echo "Jetson device-group bootstrap could not resolve the sandbox user." >&2 + exit 1 +} + +IFS=',' read -r -a gids <<<"$GROUP_GIDS" +for gid in "${gids[@]}"; do + [[ "$gid" =~ ^[1-9][0-9]*$ ]] || { + echo "Jetson device-group bootstrap received an invalid group ID." >&2 + exit 1 + } + + group_record="$(getent group "$gid" || true)" + if [[ -n "$group_record" ]]; then + IFS=':' read -r group_name _ <<<"$group_record" + else + group_name="nemoclaw_gpu_$gid" + groupadd --gid "$gid" "$group_name" + fi + [[ -n "$group_name" ]] || { + echo "Jetson device-group bootstrap could not resolve group ID $gid." >&2 + exit 1 + } + usermod --append --groups "$group_name" "$SANDBOX_USER" +done + +# OpenShell calls initgroups() before setgid()/setuid(). The container group +# database must contain every device group before the supervisor starts. +for gid in "${gids[@]}"; do + [[ " $(id -G "$SANDBOX_USER") " == *" $gid "* ]] || { + echo "Jetson device-group bootstrap did not add sandbox to group ID $gid." >&2 + exit 1 + } +done + +exec "$@" diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 13c2dd0e7f..cc35bd8789 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -2714,6 +2714,7 @@ async function createSandboxWithBaseImageResolution( } = await sandboxGpuCreateFlow.runSandboxGpuCreateFlow( { sandboxName, + agentName: agent?.name ?? "openclaw", provider, sandboxGpuConfig: effectiveSandboxGpuConfig, gpuRoutePlan, diff --git a/src/lib/onboard/docker-gpu-patch-clone.ts b/src/lib/onboard/docker-gpu-patch-clone.ts index 828ce0c540..ac9a0481e8 100644 --- a/src/lib/onboard/docker-gpu-patch-clone.ts +++ b/src/lib/onboard/docker-gpu-patch-clone.ts @@ -10,6 +10,8 @@ import type { import { openshellSandboxCommandEnvValue } from "./docker-startup-command-env"; const OPENSHELL_SANDBOX_COMMAND_ENV = "OPENSHELL_SANDBOX_COMMAND"; +const JETSON_DEVICE_GROUP_GIDS_ENV = "NEMOCLAW_JETSON_DEVICE_GROUP_GIDS"; +const JETSON_DEVICE_GROUP_BOOTSTRAP = "/usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh"; const GPU_ENV_KEYS = new Set([ "NVIDIA_VISIBLE_DEVICES", "NVIDIA_DRIVER_CAPABILITIES", @@ -339,6 +341,18 @@ export function buildDockerGpuCloneRunArgs( } const args: string[] = ["--name", containerName, ...mode.args]; const gpuAugment = mode.kind !== "startup-command"; + const extraGroupGids = [...(options.extraGroupGids ?? [])].map((gid) => String(gid).trim()); + if ( + extraGroupGids.some((gid) => { + if (!/^[1-9][0-9]*$/u.test(gid)) return true; + const parsed = Number(gid); + return !Number.isSafeInteger(parsed) || parsed > 2_147_483_647; + }) + ) { + throw new Error("Docker clone received an invalid supplementary group ID."); + } + const preserveJetsonGroups = + options.preserveJetsonDeviceGroupMembership === true && extraGroupGids.length > 0; // Startup-command recreation must retain OpenShell's native CDI attachment. if (!gpuAugment) { @@ -373,6 +387,9 @@ export function buildDockerGpuCloneRunArgs( if (sandboxCommand && !sawSandboxCommand) { args.push("--env", `${OPENSHELL_SANDBOX_COMMAND_ENV}=${sandboxCommand}`); } + if (preserveJetsonGroups) { + args.push("--env", `${JETSON_DEVICE_GROUP_GIDS_ENV}=${extraGroupGids.join(",")}`); + } const labels = config.Labels || {}; for (const key of Object.keys(labels).sort()) { @@ -407,11 +424,10 @@ export function buildDockerGpuCloneRunArgs( for (const hostEntry of stringArray(host.ExtraHosts)) args.push("--add-host", hostEntry); const groupAdds = new Set(stringArray(host.GroupAdd)); for (const group of groupAdds) args.push("--group-add", group); - for (const gid of options.extraGroupGids ?? []) { - const normalized = String(gid).trim(); - if (normalized && !groupAdds.has(normalized)) { - groupAdds.add(normalized); - args.push("--group-add", normalized); + for (const gid of extraGroupGids) { + if (!groupAdds.has(gid)) { + groupAdds.add(gid); + args.push("--group-add", gid); } } for (const ulimit of dockerUlimits(inspect, options.requiredUlimits)) { @@ -444,16 +460,27 @@ export function buildDockerGpuCloneRunArgs( const entrypoint = stringArray(config.Entrypoint); const replacementEntrypoint = String(options.containerEntrypoint ?? "").trim(); - if (replacementEntrypoint) { + if (preserveJetsonGroups && (replacementEntrypoint || options.containerCommand)) { + throw new Error("Jetson device-group bootstrap conflicts with a replacement process."); + } + if (preserveJetsonGroups && entrypoint.length === 0) { + throw new Error("Jetson device-group bootstrap requires the OpenShell supervisor entrypoint."); + } + if (preserveJetsonGroups) { + args.push("--entrypoint", JETSON_DEVICE_GROUP_BOOTSTRAP); + } else if (replacementEntrypoint) { args.push("--entrypoint", replacementEntrypoint); } else if (entrypoint.length > 0) { args.push("--entrypoint", entrypoint[0]); } - const commandArgs = options.containerCommand - ? [...options.containerCommand] - : sandboxCommand - ? [] - : [...entrypoint.slice(1), ...stringArray(config.Cmd)]; + const originalCommandArgs = sandboxCommand + ? [] + : [...entrypoint.slice(1), ...stringArray(config.Cmd)]; + const commandArgs = preserveJetsonGroups + ? [entrypoint[0], ...originalCommandArgs] + : options.containerCommand + ? [...options.containerCommand] + : originalCommandArgs; args.push(image, ...commandArgs); return args; } diff --git a/src/lib/onboard/docker-gpu-patch-jetson.test.ts b/src/lib/onboard/docker-gpu-patch-jetson.test.ts index e091c45a13..84ac3797ea 100644 --- a/src/lib/onboard/docker-gpu-patch-jetson.test.ts +++ b/src/lib/onboard/docker-gpu-patch-jetson.test.ts @@ -34,6 +34,43 @@ describe("Jetson device-node group propagation (#4231, #7610)", () => { expect(args).toEqual(expect.arrayContaining(["--group-add", "110"])); }); + it("adds Jetson device groups to the sandbox user before OpenShell calls initgroups (#7610)", () => { + const inspect = inspectFixture(); + inspect.Config!.Env = ["OPENSHELL_SANDBOX_COMMAND=env nemoclaw-start"]; + const args = buildDockerGpuCloneRunArgs( + inspect, + buildDockerGpuMode("nvidia-runtime", null, { backend: "jetson" }), + { + extraGroupGids: ["44", "995"], + preserveJetsonDeviceGroupMembership: true, + }, + ); + + expect(args).toEqual( + expect.arrayContaining([ + "--env", + "NEMOCLAW_JETSON_DEVICE_GROUP_GIDS=44,995", + "--entrypoint", + "/usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh", + "openshell/sandbox:abc", + "/opt/openshell/bin/openshell-sandbox", + ]), + ); + }); + + it("rejects an invalid Jetson device group before Docker recreation (#7610)", () => { + expect(() => + buildDockerGpuCloneRunArgs( + inspectFixture(), + buildDockerGpuMode("nvidia-runtime", null, { backend: "jetson" }), + { + extraGroupGids: ["44;id"], + preserveJetsonDeviceGroupMembership: true, + }, + ), + ).toThrow("invalid supplementary group ID"); + }); + it("does not add --group-add when extraGroupGids is absent", () => { const inspect = inspectFixture(); inspect.HostConfig!.GroupAdd = []; @@ -41,7 +78,7 @@ describe("Jetson device-node group propagation (#4231, #7610)", () => { expect(args).not.toEqual(expect.arrayContaining(["--group-add"])); }); - it("passes all detected Tegra device GIDs into the Jetson recreate as --group-add", () => { + it("passes all detected Tegra device GIDs into the OpenClaw Jetson bootstrap", () => { const dockerRunDetached = vi.fn(() => ({ status: 0, stdout: "new-container-id\n", @@ -59,7 +96,12 @@ describe("Jetson device-node group propagation (#4231, #7610)", () => { ); recreateOpenShellDockerSandboxWithGpu( - { sandboxName: "alpha", timeoutSecs: 1, backend: "jetson" }, + { + sandboxName: "alpha", + timeoutSecs: 1, + backend: "jetson", + preserveJetsonDeviceGroupMembership: true, + }, { dockerCapture: dockerCaptureFixture(), dockerRun: vi.fn(() => ({ status: 0, stdout: "probe-id\n" })), @@ -77,7 +119,18 @@ describe("Jetson device-node group propagation (#4231, #7610)", () => { expect(detectTegraDeviceGroupGidsStub).toHaveBeenCalled(); expect(dockerRunDetached).toHaveBeenCalledWith( - expect.arrayContaining(["--group-add", "44", "--group-add", "104", "--group-add", "995"]), + expect.arrayContaining([ + "--env", + "NEMOCLAW_JETSON_DEVICE_GROUP_GIDS=44,104,995", + "--group-add", + "44", + "--group-add", + "104", + "--group-add", + "995", + "--entrypoint", + "/usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh", + ]), expect.objectContaining({ ignoreError: true }), ); }); diff --git a/src/lib/onboard/docker-gpu-patch-recreate.ts b/src/lib/onboard/docker-gpu-patch-recreate.ts index 15439eeeb0..e9a9a1b7bc 100644 --- a/src/lib/onboard/docker-gpu-patch-recreate.ts +++ b/src/lib/onboard/docker-gpu-patch-recreate.ts @@ -159,6 +159,7 @@ export function recreateOpenShellDockerSandboxContainer( requiredUlimits?: readonly import("./docker-gpu-patch-types").DockerUlimit[] | null; expectedOldContainerId?: string | null; backend?: "generic" | "jetson"; + preserveJetsonDeviceGroupMembership?: boolean; dockerDesktopWsl?: boolean; modeOverride?: DockerGpuPatchMode; }, @@ -266,10 +267,14 @@ export function recreateOpenShellDockerSandboxContainer( const tegraGroupGids = d.detectTegraDeviceGroupGids(); if (tegraGroupGids.length > 0) { cloneOptions.extraGroupGids = tegraGroupGids; + cloneOptions.preserveJetsonDeviceGroupMembership = + options.preserveJetsonDeviceGroupMembership === true; console.log( - ` ✓ Granting sandbox user the detected Jetson GPU device groups via --group-add ${tegraGroupGids.join( + ` ✓ Preparing detected Jetson GPU device groups ${tegraGroupGids.join( ", ", - )} (so CUDA can initialize as a non-root user)`, + )} for the recreated container${ + options.preserveJetsonDeviceGroupMembership ? " and the OpenShell sandbox user" : "" + }`, ); } else { console.warn( diff --git a/src/lib/onboard/docker-gpu-patch-types.ts b/src/lib/onboard/docker-gpu-patch-types.ts index d72046bd32..366a4ca64f 100644 --- a/src/lib/onboard/docker-gpu-patch-types.ts +++ b/src/lib/onboard/docker-gpu-patch-types.ts @@ -122,12 +122,16 @@ export type DockerGpuCloneRunOptions = { containerName?: string | null; /** * Extra supplementary group IDs to add to the recreated container via - * `--group-add`. On Jetson these are the host group(s) owning the Tegra GPU - * device nodes; granting the sandbox user membership lets CUDA's nvmap init - * open them instead of failing with `NvRmMemInitNvmap ... Permission - * denied` (#4231, #7610). + * `--group-add`. On Jetson these are the host groups that own Tegra GPU + * device nodes. Set `preserveJetsonDeviceGroupMembership` to add them to the + * OpenShell sandbox user's container group database. */ extraGroupGids?: readonly string[] | null; + /** + * Add the detected Jetson device groups to the sandbox user's container group + * database before the OpenShell supervisor calls initgroups(). + */ + preserveJetsonDeviceGroupMembership?: boolean; }; export type DockerGpuPatchDiagnostics = { diff --git a/src/lib/onboard/docker-gpu-sandbox-create.ts b/src/lib/onboard/docker-gpu-sandbox-create.ts index 52c15d0b5b..893db228ac 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create.ts @@ -72,6 +72,7 @@ type DockerGpuSandboxCreatePatchOptions = { requiredUlimits?: Parameters[0]["requiredUlimits"]; timeoutSecs: number; backend?: DockerGpuPatchBackend; + agentName?: string | null; /** * Whether the host is Docker Desktop WSL. Defaults to the cached * `isDockerDesktopWslRuntime()` probe. When true, the GPU patch skips the CDI @@ -167,6 +168,8 @@ export function createDockerGpuSandboxCreatePatch( requiredUlimits: options.requiredUlimits ?? null, timeoutSecs: options.timeoutSecs, backend: options.backend, + preserveJetsonDeviceGroupMembership: + options.backend === "jetson" && options.agentName === "openclaw", dockerDesktopWsl: options.dockerDesktopWsl ?? isDockerDesktopWslRuntime(), }; const recreationEnabled = diff --git a/src/lib/onboard/initial-policy.test.ts b/src/lib/onboard/initial-policy.test.ts index 2cde41037a..1c16171107 100644 --- a/src/lib/onboard/initial-policy.test.ts +++ b/src/lib/onboard/initial-policy.test.ts @@ -369,6 +369,30 @@ network_policies: {} } }); + it("permits the OpenRM driver library path only for an OpenClaw Jetson GPU policy (#7610)", () => { + const genericPolicy = YAML.parse(buildDirectGpuPolicyYaml(BASE_POLICY_FIXTURE)); + const jetsonPolicy = YAML.parse( + buildDirectGpuPolicyYaml(BASE_POLICY_FIXTURE, { jetsonGpu: true }), + ); + + expect(genericPolicy.filesystem_policy.read_only).not.toContain("/opt/nvidia/l4t-gpu-libs"); + expect(jetsonPolicy.filesystem_policy.read_only).toContain("/opt/nvidia/l4t-gpu-libs"); + expect(jetsonPolicy.filesystem_policy.read_write).not.toContain("/opt/nvidia/l4t-gpu-libs"); + }); + + it("threads the OpenClaw Jetson library path through public policy preparation (#7610)", () => { + const basePolicyPath = tmpPolicy(BASE_POLICY_FIXTURE); + const prepared = prepareInitialSandboxCreatePolicy(basePolicyPath, [], { + directGpu: true, + jetsonGpu: true, + stationGb300SysfsReadOnlyPaths: [], + }); + const preparedDoc = YAML.parse(fs.readFileSync(prepared.policyPath, "utf-8")); + + expect(preparedDoc.filesystem_policy.read_only).toContain("/opt/nvidia/l4t-gpu-libs"); + expect(prepared.cleanup?.()).toBe(true); + }); + it("preserves best-effort Landlock for missing Station sysfs paths (#7103)", () => { const sysfsRoot = tmpSysfsRoot(); addPciDevice(sysfsRoot, "0009:06:00.0", "0x10de\n", "0x030200\n"); diff --git a/src/lib/onboard/initial-policy.ts b/src/lib/onboard/initial-policy.ts index 4d56cecb12..e5e8b0ea20 100644 --- a/src/lib/onboard/initial-policy.ts +++ b/src/lib/onboard/initial-policy.ts @@ -46,6 +46,7 @@ export function discloseInitialSandboxPolicy(policy: InitialSandboxPolicy): void const HERMES_MESSAGING_POLICY_KEYS = getMessagingPolicyKeysByChannel({ agent: "hermes" }); const PROC_PATH = "/proc"; +const JETSON_GPU_LIBRARY_PATH = "/opt/nvidia/l4t-gpu-libs"; const PROC_COMM_READ_WRITE_PATHS = ["/proc/self/comm", "/proc/self/task/*/comm"]; const SYSFS_PATH = "/sys"; const PCI_BDF_PATTERN = /^[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]$/iu; @@ -77,6 +78,7 @@ function deduplicateDirectGpuSysfsEntries( type DirectGpuPolicyOptions = { procReadWrite?: boolean; sysfsReadOnlyPaths?: readonly string[]; + jetsonGpu?: boolean; }; export { isStationGb300ProductName }; @@ -223,6 +225,15 @@ export function buildDirectGpuPolicyYaml( } } } + if ( + options.jetsonGpu && + !fsPolicy.read_only.includes(JETSON_GPU_LIBRARY_PATH) && + !fsPolicy.read_write.includes(JETSON_GPU_LIBRARY_PATH) + ) { + // OpenRM runtime injection places libcuda outside the base /usr and /lib + // grants. CUDA cannot load the driver unless Landlock permits this path. + fsPolicy.read_only.push(JETSON_GPU_LIBRARY_PATH); + } if (options.procReadWrite && !fsPolicy.read_write.includes(PROC_PATH)) { // This exists only for the legacy post-create Docker GPU compatibility // path, which recreates the container after `openshell sandbox create` and @@ -381,6 +392,7 @@ export function prepareInitialSandboxCreatePolicy( options: { directGpu?: boolean; dockerGpuPatch?: boolean; + jetsonGpu?: boolean; hostGpuAvailable?: boolean; stationGb300SysfsReadOnlyPaths?: readonly string[]; additionalPresets?: string[]; @@ -392,6 +404,7 @@ export function prepareInitialSandboxCreatePolicy( const directGpuPolicy = options.directGpu ? prepareDirectGpuSandboxPolicy(basePolicyPath, { procReadWrite: options.dockerGpuPatch === true, + jetsonGpu: options.jetsonGpu === true, sysfsReadOnlyPaths: options.stationGb300SysfsReadOnlyPaths ?? discoverHostStationGb300SysfsReadOnlyPaths({ diff --git a/src/lib/onboard/sandbox-create-intent-types.ts b/src/lib/onboard/sandbox-create-intent-types.ts index 5621f6e19c..d1ddce535f 100644 --- a/src/lib/onboard/sandbox-create-intent-types.ts +++ b/src/lib/onboard/sandbox-create-intent-types.ts @@ -24,6 +24,7 @@ export type SandboxCreatePolicyRequest = { readonly activeMessagingChannels: readonly string[]; readonly options: { readonly directGpu: boolean; + readonly jetsonGpu?: boolean; readonly hostGpuAvailable?: boolean; readonly additionalPresets: readonly string[]; readonly agentName?: string | null; diff --git a/src/lib/onboard/sandbox-create-intent.ts b/src/lib/onboard/sandbox-create-intent.ts index ca4dd64adf..9aa1c1a976 100644 --- a/src/lib/onboard/sandbox-create-intent.ts +++ b/src/lib/onboard/sandbox-create-intent.ts @@ -171,6 +171,10 @@ export function resolveSandboxCreateIntent({ ); const normalizedInferenceProvider = inferenceProvider?.trim() || null; + const openclawJetsonGpu = + sandboxGpuConfig.sandboxGpuEnabled && + sandboxGpuConfig.hostGpuPlatform === "jetson" && + (agentName === undefined || agentName === null || agentName === "openclaw"); return { sandboxName, @@ -186,6 +190,7 @@ export function resolveSandboxCreateIntent({ activeMessagingChannels: [...activeMessagingChannels], options: { directGpu: sandboxGpuConfig.sandboxGpuEnabled, + ...(openclawJetsonGpu ? { jetsonGpu: true } : {}), ...(sandboxGpuConfig.hostGpuDetected !== undefined ? { hostGpuAvailable: sandboxGpuConfig.hostGpuDetected } : {}), diff --git a/src/lib/onboard/sandbox-create-plan-materialization.ts b/src/lib/onboard/sandbox-create-plan-materialization.ts index 2524795443..571a5a8205 100644 --- a/src/lib/onboard/sandbox-create-plan-materialization.ts +++ b/src/lib/onboard/sandbox-create-plan-materialization.ts @@ -144,6 +144,7 @@ export function materializeSandboxCreatePlan({ [...intent.policy.activeMessagingChannels], { directGpu: intent.policy.options.directGpu, + jetsonGpu: intent.policy.options.jetsonGpu, hostGpuAvailable: intent.policy.options.hostGpuAvailable, additionalPresets: [...intent.policy.options.additionalPresets], agentName: intent.policy.options.agentName, diff --git a/src/lib/onboard/sandbox-create-plan.test.ts b/src/lib/onboard/sandbox-create-plan.test.ts index 8475c8a318..6c4d99805b 100644 --- a/src/lib/onboard/sandbox-create-plan.test.ts +++ b/src/lib/onboard/sandbox-create-plan.test.ts @@ -184,8 +184,19 @@ describe("resolveSandboxCreateIntent", () => { const first = resolveSandboxCreateIntent(input); const second = resolveSandboxCreateIntent(input); + const openclawJetson = resolveSandboxCreateIntent({ + ...input, + agentName: "openclaw", + sandboxGpuConfig: { ...sandboxGpuConfig, hostGpuPlatform: "jetson" }, + }); + const hermesJetson = resolveSandboxCreateIntent({ + ...input, + sandboxGpuConfig: { ...sandboxGpuConfig, hostGpuPlatform: "jetson" }, + }); expect(first).toEqual(second); + expect(openclawJetson.policy.options.jetsonGpu).toBe(true); + expect(hermesJetson.policy.options.jetsonGpu).toBeUndefined(); expect(first.activeMessagingChannels).toEqual(["telegram", "discord", "whatsapp"]); expect(first.messagingProviderRequests.map(({ name }) => name)).toEqual([ "sandbox-telegram-bridge", diff --git a/src/lib/onboard/sandbox-gpu-create-flow.ts b/src/lib/onboard/sandbox-gpu-create-flow.ts index 46f34359a2..a4ccfbaef6 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.ts @@ -68,6 +68,7 @@ type Sleep = NonNullable; export interface SandboxGpuCreateFlowInput { sandboxName: string; + agentName?: string; provider: string; sandboxGpuConfig: SandboxGpuConfig; gpuRoutePlan: import("./docker-gpu-route").DockerGpuRoutePlan; diff --git a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts index 49c3271ddd..be0c722a77 100644 --- a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts +++ b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts @@ -137,6 +137,7 @@ export function createSandboxGpuCreateAttemptRunner( input.persistStartupCommand === true && (route !== "native" || hasRequiredUlimits), externalRecreation: false, sandboxName: input.sandboxName, + agentName: input.agentName, gpuDevice: input.sandboxGpuConfig.sandboxGpuDevice, openshellSandboxCommand: input.sandboxStartupCommand, requiredUlimits: input.requiredUlimits, diff --git a/src/lib/onboard/sandbox-gpu-create.ts b/src/lib/onboard/sandbox-gpu-create.ts index 56ee5687e8..451b33695b 100644 --- a/src/lib/onboard/sandbox-gpu-create.ts +++ b/src/lib/onboard/sandbox-gpu-create.ts @@ -7,6 +7,7 @@ export type SandboxGpuCreateConfig = { sandboxGpuEnabled: boolean; sandboxGpuDevice?: string | null; hostGpuDetected?: boolean; + hostGpuPlatform?: string | null; }; export function buildSandboxGpuCreateArgs( diff --git a/test/e2e/live/jetson-nvmap-gpu.test.ts b/test/e2e/live/jetson-nvmap-gpu.test.ts index f49aab0d34..71a8f30675 100644 --- a/test/e2e/live/jetson-nvmap-gpu.test.ts +++ b/test/e2e/live/jetson-nvmap-gpu.test.ts @@ -249,10 +249,8 @@ exit "$status"`, expect(installedCli.exitCode, resultText(installedCli)).toBe(0); expect(installedCli.stdout.trim()).not.toBe(""); - // A4: the Jetson recreate must grant Tegra device-node groups via --group-add. - expect(resultText(install)).toContain( - "Granting sandbox user the detected Jetson GPU device groups via --group-add", - ); + // A4: the Jetson recreate must prepare Tegra device-node groups for the sandbox user. + expect(resultText(install)).toContain("Preparing detected Jetson GPU device groups"); // A5: the sandbox user must be in the host /dev/nvmap owning GID. progress.phase("inspect sandbox nvmap access"); diff --git a/test/openclaw-final-image-layout.test.ts b/test/openclaw-final-image-layout.test.ts index 1f2217c46b..633acb2a18 100644 --- a/test/openclaw-final-image-layout.test.ts +++ b/test/openclaw-final-image-layout.test.ts @@ -88,6 +88,7 @@ describe("OpenClaw final image layout", () => { "COPY scripts/lib/openclaw_device_approval_policy.py /usr/local/lib/nemoclaw/openclaw_device_approval_policy.py", "COPY scripts/lib/clean_runtime_shell_env_shim.py /usr/local/lib/nemoclaw/clean_runtime_shell_env_shim.py", "COPY scripts/lib/normalize_mutable_config_perms.py /usr/local/lib/nemoclaw/normalize_mutable_config_perms.py", + "COPY scripts/jetson-device-group-bootstrap.sh /usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh", "COPY scripts/state-dir-guard.py /usr/local/lib/nemoclaw/state-dir-guard.py", "COPY scripts/openclaw-config-guard.py /usr/local/lib/nemoclaw/openclaw-config-guard.py", "COPY scripts/managed-gateway-control.py /usr/local/lib/nemoclaw/managed-gateway-control.py", From c9a80f815a103bf026ed462da503dd0443557ce1 Mon Sep 17 00:00:00 2001 From: San Dang Date: Tue, 4 Aug 2026 17:18:24 +0700 Subject: [PATCH 07/49] fix(onboard): scope Jetson nvmap access to OpenClaw Signed-off-by: San Dang --- docs/reference/troubleshooting.mdx | 8 ++----- scripts/setup-jetson.sh | 4 +++- .../onboard/sandbox-gpu-direct-proof.test.ts | 6 ----- src/lib/onboard/sandbox-gpu-preflight.ts | 4 +--- test/setup-jetson.test.ts | 24 +++++++++++++++++++ 5 files changed, 30 insertions(+), 16 deletions(-) diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 8351798928..68df6df2d4 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -2688,15 +2688,13 @@ To skip GPU passthrough entirely, rerun with `--no-gpu` or set `NEMOCLAW_SANDBOX Automatic GPU onboarding uses the compatibility path directly; it does not make a native attempt first. The path recreates the OpenShell-managed Docker container with NVIDIA GPU flags. +Use `NEMOCLAW_DOCKER_GPU_PATCH=0` only for troubleshooting because it bypasses the compatibility patch and CUDA may not initialize. + It also keeps eligible Jetson GPU device group memberships when OpenShell starts the nonroot sandbox user. The compatibility policy permits read-only access to the NVIDIA runtime library directory at `/opt/nvidia/l4t-gpu-libs` when that path is present. - - -Use `NEMOCLAW_DOCKER_GPU_PATCH=0` only for troubleshooting because it bypasses the compatibility patch and CUDA may not initialize. - If CUDA reports `NvRmMemInitNvmap failed with Permission denied` and `cuInit(0)=999` on JetPack 6, rerun the NemoClaw installer to apply persistent group read-write access to `/dev/nvmap`: If the device is absent, setup warns and continues without installing the rule. @@ -2719,8 +2717,6 @@ The rule must match exactly, and the group permission characters must be `rw`, s The rule reapplies mode `0660` when udev recreates `/dev/nvmap`, including after reboot. NemoClaw uninstall does not remove the rule; delete it and reload udev to stop future reapplication, while the live mode remains until device recreation. - - After onboarding recreates the sandbox, verify CUDA as the nonroot sandbox user: ```bash diff --git a/scripts/setup-jetson.sh b/scripts/setup-jetson.sh index 4ee54712cd..8d69d43108 100755 --- a/scripts/setup-jetson.sh +++ b/scripts/setup-jetson.sh @@ -218,7 +218,9 @@ except Exception: os.unlink(tmp) raise PYEOF - configure_nvmap_group_access + if [[ "${NEMOCLAW_AGENT:-openclaw}" == "openclaw" ]]; then + configure_nvmap_group_access + fi ;; jp7-r38) # JP7 R38 does not need iptables or Docker daemon.json changes. diff --git a/src/lib/onboard/sandbox-gpu-direct-proof.test.ts b/src/lib/onboard/sandbox-gpu-direct-proof.test.ts index 76052c9ebe..a6d55681ee 100644 --- a/src/lib/onboard/sandbox-gpu-direct-proof.test.ts +++ b/src/lib/onboard/sandbox-gpu-direct-proof.test.ts @@ -198,12 +198,6 @@ describe("direct sandbox GPU proof", () => { expect(result.detail).toContain("cuInit(0)=999"); const warnings = warnSpy.mock.calls.map((call) => call[0]).join("\n"); expect(warnings).toContain("/dev/nvmap"); - expect(warnings).toContain( - "chmod grants every member of the nvmap owning group write access", - ); - expect(warnings.indexOf("chmod grants every member")).toBeLessThan( - warnings.indexOf("sudo chmod g+rw /dev/nvmap"), - ); } finally { warnSpy.mockRestore(); } diff --git a/src/lib/onboard/sandbox-gpu-preflight.ts b/src/lib/onboard/sandbox-gpu-preflight.ts index 2d15ab64dc..5fbbde0fe0 100644 --- a/src/lib/onboard/sandbox-gpu-preflight.ts +++ b/src/lib/onboard/sandbox-gpu-preflight.ts @@ -65,9 +65,7 @@ export function jetsonGpuProofRemediationLines(): string[] { return [ "Jetson/Tegra CUDA proof did not pass. CUDA needs access to the Tegra device", "nodes; confirm the sandbox propagates them and the agent user's groups:", - " ls -l /dev/nvmap /dev/nvhost-* (nvmap must grant the sandbox group read-write access)", - " warning: chmod grants every member of the nvmap owning group write access", - " sudo chmod g+rw /dev/nvmap (host remediation until udev recreates the device)", + " ls -l /dev/nvmap /dev/nvhost-* (must be readable by the sandbox)", " add the host video/render groups via --group-add when recreating", "Then recreate the sandbox, or force CPU behavior with NEMOCLAW_SANDBOX_GPU=0.", ]; diff --git a/test/setup-jetson.test.ts b/test/setup-jetson.test.ts index 34eb97a5b9..5260b96eac 100644 --- a/test/setup-jetson.test.ts +++ b/test/setup-jetson.test.ts @@ -451,4 +451,28 @@ describe("setup-jetson JetPack 6 nvmap access", () => { ); expect(result.commandLog).toContain("chmod g+rw /dev/nvmap"); }); + + it.each([ + "hermes", + "langchain-deepagents-code", + ])("does not change nvmap access for the %s agent (#7610)", (agent) => { + const result = withJetsonReleaseSandbox( + ({ commandLogPath, headArgsPath, releasePath, stubDir }) => { + writeFileSync( + releasePath, + "# R36 (release), REVISION: 5.1, GCID: 12345678, BOARD: t186ref\n", + ); + return spawnSetupJetson(stubDir, headArgsPath, commandLogPath, { + NEMOCLAW_AGENT: agent, + NEMOCLAW_TEST_STAT_OUTPUT: "character special file|cr--r-----", + }); + }, + ); + + expect(result.status).toBe(0); + expect(result.commandLog).not.toContain("tee /etc/udev/rules.d/99-zz-nemoclaw-nvmap.rules"); + expect(result.commandLog).not.toContain("chmod g+rw /dev/nvmap"); + expect(result.stdout).not.toContain("/dev/nvmap grants its owning group read-write access"); + expect(result.stderr).not.toContain("/dev/nvmap owning group write access"); + }); }); From 457d1f9a9eeea7dcb52aceb4f69b3e37cfbb13c6 Mon Sep 17 00:00:00 2001 From: San Dang Date: Tue, 4 Aug 2026 17:25:41 +0700 Subject: [PATCH 08/49] refactor(onboard): keep entrypoint line budget neutral Signed-off-by: San Dang --- src/lib/onboard.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index cc35bd8789..976bc1af5a 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -2743,7 +2743,6 @@ async function createSandboxWithBaseImageResolution( if (initialSandboxPolicy.cleanup && initialSandboxPolicy.cleanup()) { process.removeListener("exit", initialSandboxPolicy.cleanup); } - // Clean up build context regardless of outcome. // Use fs.rmSync instead of run() to avoid spawning a shell process. // Only deregister the 'exit' safety net when inline cleanup succeeded; From 9bd5a3a2929f7a375a69327d7a319547f9396c03 Mon Sep 17 00:00:00 2001 From: San Dang Date: Tue, 4 Aug 2026 17:38:11 +0700 Subject: [PATCH 09/49] test(images): stage Jetson bootstrap helper Signed-off-by: San Dang --- test/sandbox-provisioning-helper-permissions.test.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/sandbox-provisioning-helper-permissions.test.ts b/test/sandbox-provisioning-helper-permissions.test.ts index 49038b8414..2f823018d8 100644 --- a/test/sandbox-provisioning-helper-permissions.test.ts +++ b/test/sandbox-provisioning-helper-permissions.test.ts @@ -129,6 +129,7 @@ describe("sandbox provisioning: copied OpenClaw helper permissions (#2861)", () const nestedPluginFile = path.join(nestedPluginDir, "helper.js"); const gatewayControlPath = path.join(localBin, "nemoclaw-gateway-control"); const gatewaySupervisorPath = path.join(localLib, "gateway-supervisor.sh"); + const jetsonGroupBootstrapPath = path.join(localLib, "jetson-device-group-bootstrap.sh"); const stateDirGuardPath = path.join(localLib, "state-dir-guard.py"); const configGuardPath = path.join(localLib, "openclaw-config-guard.py"); const managedGatewayControlPath = path.join(localLib, "managed-gateway-control.py"); @@ -142,6 +143,7 @@ describe("sandbox provisioning: copied OpenClaw helper permissions (#2861)", () path.join(localLib, "sandbox-init.sh"), path.join(localLib, "sandbox-rlimits.sh"), gatewaySupervisorPath, + jetsonGroupBootstrapPath, stateDirGuardPath, configGuardPath, managedGatewayControlPath, @@ -203,6 +205,7 @@ describe("sandbox provisioning: copied OpenClaw helper permissions (#2861)", () expect((fs.statSync(nestedPluginFile).mode & 0o777).toString(8)).toBe("644"); expect((fs.statSync(gatewayControlPath).mode & 0o777).toString(8)).toBe("700"); expect((fs.statSync(gatewaySupervisorPath).mode & 0o777).toString(8)).toBe("444"); + expect((fs.statSync(jetsonGroupBootstrapPath).mode & 0o777).toString(8)).toBe("755"); expect((fs.statSync(stateDirGuardPath).mode & 0o777).toString(8)).toBe("500"); expect((fs.statSync(configGuardPath).mode & 0o777).toString(8)).toBe("500"); expect((fs.statSync(managedGatewayControlPath).mode & 0o777).toString(8)).toBe("500"); From 67a85796caa655a3fdbebeb9da35fa9302fa6ed2 Mon Sep 17 00:00:00 2001 From: San Dang Date: Tue, 4 Aug 2026 19:09:53 +0700 Subject: [PATCH 10/49] fix(images): stage Jetson bootstrap script Signed-off-by: San Dang --- src/lib/sandbox/build-context.ts | 4 ++++ test/sandbox-build-context.test.ts | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/src/lib/sandbox/build-context.ts b/src/lib/sandbox/build-context.ts index 86808b5cc0..aad1b630e5 100644 --- a/src/lib/sandbox/build-context.ts +++ b/src/lib/sandbox/build-context.ts @@ -265,6 +265,10 @@ function stageOptimizedSandboxBuildContext( path.join(rootDir, "scripts", "validate-openclaw-tool-search.mts"), path.join(stagedScriptsDir, "validate-openclaw-tool-search.mts"), ); + fs.copyFileSync( + path.join(rootDir, "scripts", "jetson-device-group-bootstrap.sh"), + path.join(stagedScriptsDir, "jetson-device-group-bootstrap.sh"), + ); // Shared sandbox initialisation library sourced by the entrypoint (#2277) fs.mkdirSync(path.join(stagedScriptsDir, "lib"), { recursive: true }); fs.copyFileSync( diff --git a/test/sandbox-build-context.test.ts b/test/sandbox-build-context.test.ts index be39b47973..d72f968f30 100644 --- a/test/sandbox-build-context.test.ts +++ b/test/sandbox-build-context.test.ts @@ -114,6 +114,7 @@ describe("sandbox build context staging", () => { writeFixture(path.join("scripts", "codex-acp-wrapper.sh")); writeFixture(path.join("scripts", "generate-openclaw-config.mts")); writeFixture(path.join("scripts", "validate-openclaw-tool-search.mts")); + writeFixture(path.join("scripts", "jetson-device-group-bootstrap.sh"), "fixture\n", 0o755); writeFixture( path.join("scripts", "checks", "verify-openshell-policy-boundary-dependencies.mts"), ); @@ -649,6 +650,9 @@ describe("sandbox build context staging", () => { expect( fs.existsSync(path.join(buildCtx, "scripts", "checks", "node-tar-image-scan.mts")), ).toBe(true); + expect( + fs.existsSync(path.join(buildCtx, "scripts", "jetson-device-group-bootstrap.sh")), + ).toBe(true); expect( fs.existsSync(path.join(buildCtx, "scripts", "patch-openclaw-device-self-approval.ts")), ).toBe(false); From ed7f1ec7613023bb97ce8461726aeb3a032f7991 Mon Sep 17 00:00:00 2001 From: San Dang Date: Wed, 5 Aug 2026 00:29:39 +0700 Subject: [PATCH 11/49] fix(onboard): preserve managed Jetson groups Signed-off-by: San Dang --- scripts/managed-bootstrap-trampoline.sh | 28 +++++ src/lib/onboard/docker-gpu-patch-clone.ts | 26 ++--- src/lib/onboard/docker-gpu-patch-recreate.ts | 2 +- .../docker-startup-command-patch.test.ts | 45 ++++++++ .../onboard/docker-startup-command-patch.ts | 2 + ...ker-startup-command-sandbox-create.test.ts | 25 +++++ .../docker-startup-command-sandbox-create.ts | 2 + .../managed-bootstrap/docker-runtime.test.ts | 106 ++++++++++++++++++ .../managed-bootstrap/docker-runtime.ts | 7 +- .../managed-bootstrap/docker-test-fixture.ts | 17 ++- .../onboard/managed-bootstrap/docker.test.ts | 93 ++++++++++++++- src/lib/onboard/managed-bootstrap/docker.ts | 46 +++++++- test/managed-bootstrap-trampoline.test.ts | 22 +++- 13 files changed, 395 insertions(+), 26 deletions(-) diff --git a/scripts/managed-bootstrap-trampoline.sh b/scripts/managed-bootstrap-trampoline.sh index b0319b0f38..68ecd8315d 100644 --- a/scripts/managed-bootstrap-trampoline.sh +++ b/scripts/managed-bootstrap-trampoline.sh @@ -36,6 +36,24 @@ _nemoclaw_supervisor_environment_bytes="$4" [ "${5:-}" = "--" ] || fail "supervisor environment delimiter is missing" shift 5 +# Read the reserved group input from the sealed supervisor environment. +# The native resume path rewinds and validates FD 9 before supervisor exec. +_nemoclaw_jetson_device_group_gids="" +_nemoclaw_jetson_device_group_gids_seen=0 +for ((_nemoclaw_environment_index = 0; _nemoclaw_environment_index < _nemoclaw_supervisor_environment_count; _nemoclaw_environment_index++)); do + IFS= read -r -d '' _nemoclaw_environment_entry <&9 \ + || fail "supervisor environment transport ended before Jetson group validation" + case "$_nemoclaw_environment_entry" in + NEMOCLAW_JETSON_DEVICE_GROUP_GIDS=*) + [ "$_nemoclaw_jetson_device_group_gids_seen" -eq 0 ] \ + || fail "Jetson device-group input is duplicated" + _nemoclaw_jetson_device_group_gids="${_nemoclaw_environment_entry#*=}" + _nemoclaw_jetson_device_group_gids_seen=1 + ;; + esac +done +unset _nemoclaw_environment_entry _nemoclaw_environment_index + if [ "$(/usr/bin/id -u 9<&-)" -ne 0 ] || [ "$(/usr/bin/id -g 9<&-)" -ne 0 ]; then fail "must run as root" fi @@ -89,6 +107,16 @@ fi [ "$_nemoclaw_request" = "/var/lib/nemoclaw-managed-bootstrap-request.json" ] \ || fail "request file path is not the fixed bootstrap path" +if [ "$_nemoclaw_jetson_device_group_gids_seen" -eq 1 ]; then + [ "$_nemoclaw_agent" = "openclaw" ] \ + || fail "Jetson device-group input requires the OpenClaw agent" + [[ "$_nemoclaw_jetson_device_group_gids" =~ ^[1-9][0-9]*(,[1-9][0-9]*)*$ ]] \ + || fail "Jetson device-group input is invalid" + # Update the sandbox account before the OpenShell supervisor calls initgroups(). + NEMOCLAW_JETSON_DEVICE_GROUP_GIDS="$_nemoclaw_jetson_device_group_gids" \ + /usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh /usr/bin/true 9<&- +fi + _nemoclaw_runtime="/usr/local/lib/nemoclaw/managed-startup-image-runtime.cjs" _nemoclaw_request_directory="${_nemoclaw_request%/*}" _nemoclaw_request_basename="${_nemoclaw_request##*/}" diff --git a/src/lib/onboard/docker-gpu-patch-clone.ts b/src/lib/onboard/docker-gpu-patch-clone.ts index ac9a0481e8..b77d196e98 100644 --- a/src/lib/onboard/docker-gpu-patch-clone.ts +++ b/src/lib/onboard/docker-gpu-patch-clone.ts @@ -373,9 +373,10 @@ export function buildDockerGpuCloneRunArgs( const sandboxCommand = openshellSandboxCommandEnvValue(options.openshellSandboxCommand); let sawSandboxCommand = false; - for (const env of stringArray(config.Env).filter( - (entry) => !gpuAugment || !GPU_ENV_KEYS.has(envKey(entry)), - )) { + for (const env of stringArray(config.Env).filter((entry) => { + const key = envKey(entry); + return key !== JETSON_DEVICE_GROUP_GIDS_ENV && (!gpuAugment || !GPU_ENV_KEYS.has(key)); + })) { const key = envKey(env); if (key === OPENSHELL_SANDBOX_COMMAND_ENV && sandboxCommand) { sawSandboxCommand = true; @@ -460,13 +461,11 @@ export function buildDockerGpuCloneRunArgs( const entrypoint = stringArray(config.Entrypoint); const replacementEntrypoint = String(options.containerEntrypoint ?? "").trim(); - if (preserveJetsonGroups && (replacementEntrypoint || options.containerCommand)) { - throw new Error("Jetson device-group bootstrap conflicts with a replacement process."); - } - if (preserveJetsonGroups && entrypoint.length === 0) { + const replacementProcess = Boolean(replacementEntrypoint || options.containerCommand); + if (preserveJetsonGroups && !replacementProcess && entrypoint.length === 0) { throw new Error("Jetson device-group bootstrap requires the OpenShell supervisor entrypoint."); } - if (preserveJetsonGroups) { + if (preserveJetsonGroups && !replacementProcess) { args.push("--entrypoint", JETSON_DEVICE_GROUP_BOOTSTRAP); } else if (replacementEntrypoint) { args.push("--entrypoint", replacementEntrypoint); @@ -476,11 +475,12 @@ export function buildDockerGpuCloneRunArgs( const originalCommandArgs = sandboxCommand ? [] : [...entrypoint.slice(1), ...stringArray(config.Cmd)]; - const commandArgs = preserveJetsonGroups - ? [entrypoint[0], ...originalCommandArgs] - : options.containerCommand - ? [...options.containerCommand] - : originalCommandArgs; + const commandArgs = + preserveJetsonGroups && !replacementProcess + ? [entrypoint[0], ...originalCommandArgs] + : options.containerCommand + ? [...options.containerCommand] + : originalCommandArgs; args.push(image, ...commandArgs); return args; } diff --git a/src/lib/onboard/docker-gpu-patch-recreate.ts b/src/lib/onboard/docker-gpu-patch-recreate.ts index e9a9a1b7bc..a44b1f8ebe 100644 --- a/src/lib/onboard/docker-gpu-patch-recreate.ts +++ b/src/lib/onboard/docker-gpu-patch-recreate.ts @@ -263,7 +263,7 @@ export function recreateOpenShellDockerSandboxContainer( ); } } - if (selection.mode.kind !== "startup-command" && options.backend === "jetson") { + if (options.backend === "jetson") { const tegraGroupGids = d.detectTegraDeviceGroupGids(); if (tegraGroupGids.length > 0) { cloneOptions.extraGroupGids = tegraGroupGids; diff --git a/src/lib/onboard/docker-startup-command-patch.test.ts b/src/lib/onboard/docker-startup-command-patch.test.ts index 71c5c9d12c..05bc4c316a 100644 --- a/src/lib/onboard/docker-startup-command-patch.test.ts +++ b/src/lib/onboard/docker-startup-command-patch.test.ts @@ -137,6 +137,51 @@ describe("Docker startup-command patch", () => { ); }); + it("preserves Jetson device groups during startup-command recreation (#7610)", () => { + const dockerRunDetached = vi.fn((_args: readonly string[]) => ({ + status: 0, + stdout: "new-container-id\n", + })); + + recreateStartupCommandForTest( + { + sandboxName: "alpha", + timeoutSecs: 1, + waitForSupervisor: false, + openshellSandboxCommand: ["env", "nemoclaw-start"], + backend: "jetson", + preserveJetsonDeviceGroupMembership: true, + }, + { + dockerCapture: vi.fn((args: readonly string[]) => + args[0] === "ps" ? "old-container-id\n" : JSON.stringify([inspectFixture()]), + ), + dockerRunDetached, + dockerRename: vi.fn(() => ({ status: 0 })), + dockerStop: vi.fn(() => ({ status: 0 })), + detectTegraDeviceGroupGids: () => ["44", "993"], + sleep: vi.fn(), + now: () => new Date("2026-07-10T00:00:00Z"), + }, + ); + + const cloneArgs = dockerRunDetached.mock.calls[0]?.[0] ?? []; + expect(cloneArgs).toEqual( + expect.arrayContaining([ + "--group-add", + "44", + "--group-add", + "993", + "--env", + "NEMOCLAW_JETSON_DEVICE_GROUP_GIDS=44,993", + "--entrypoint", + "/usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh", + `sha256:${"c".repeat(64)}`, + "/opt/openshell/bin/openshell-sandbox", + ]), + ); + }); + it("rejects an empty restart-persistence command before Docker mutation", () => { expect(() => recreateStartupCommandForTest({ diff --git a/src/lib/onboard/docker-startup-command-patch.ts b/src/lib/onboard/docker-startup-command-patch.ts index 2b5b6f761e..7b972832c1 100644 --- a/src/lib/onboard/docker-startup-command-patch.ts +++ b/src/lib/onboard/docker-startup-command-patch.ts @@ -18,6 +18,8 @@ export function recreateOpenShellDockerSandboxWithStartupCommand( openshellSandboxCommand: readonly string[]; requiredUlimits?: readonly import("./docker-gpu-patch-types").DockerUlimit[] | null; expectedOldContainerId?: string | null; + backend?: "generic" | "jetson"; + preserveJetsonDeviceGroupMembership?: boolean; }, deps: DockerGpuPatchDeps = {}, ): DockerGpuPatchResult { diff --git a/src/lib/onboard/docker-startup-command-sandbox-create.test.ts b/src/lib/onboard/docker-startup-command-sandbox-create.test.ts index 8ccc7295e8..90630503a4 100644 --- a/src/lib/onboard/docker-startup-command-sandbox-create.test.ts +++ b/src/lib/onboard/docker-startup-command-sandbox-create.test.ts @@ -118,6 +118,31 @@ describe("Docker startup-command sandbox creation", () => { expect(patch.selectedMode()?.kind).toBe("startup-command"); }); + it("forwards OpenClaw Jetson group preservation to startup-command recreation (#7610)", async () => { + const recreateStartupPatch = vi.fn(() => startupResult()); + const patch = createDockerGpuSandboxCreatePatch({ + route: "native", + persistStartupCommand: true, + sandboxName: "alpha", + openshellSandboxCommand: ["env", "nemoclaw-start"], + timeoutSecs: 60, + backend: "jetson", + agentName: "openclaw", + deps: makeDeps(), + overrides: { recreateStartupPatch }, + }); + + await patch.ensureApplied(); + + expect(recreateStartupPatch).toHaveBeenCalledWith( + expect.objectContaining({ + backend: "jetson", + preserveJetsonDeviceGroupMembership: true, + }), + expect.any(Object), + ); + }); + it("rolls back startup-command recreation when the supervisor does not reconnect", () => { const deps = makeDeps(); const result = startupResult(); diff --git a/src/lib/onboard/docker-startup-command-sandbox-create.ts b/src/lib/onboard/docker-startup-command-sandbox-create.ts index dd8df2e17b..868db4f238 100644 --- a/src/lib/onboard/docker-startup-command-sandbox-create.ts +++ b/src/lib/onboard/docker-startup-command-sandbox-create.ts @@ -33,6 +33,8 @@ export function createDockerSandboxRecreator(options: { requiredUlimits: options.requiredUlimits, timeoutSecs: options.gpuOptions.timeoutSecs, waitForSupervisor, + backend: options.gpuOptions.backend, + preserveJetsonDeviceGroupMembership: options.gpuOptions.preserveJetsonDeviceGroupMembership, }, deps, ); diff --git a/src/lib/onboard/managed-bootstrap/docker-runtime.test.ts b/src/lib/onboard/managed-bootstrap/docker-runtime.test.ts index 954de3a49c..c8310a1a0b 100644 --- a/src/lib/onboard/managed-bootstrap/docker-runtime.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker-runtime.test.ts @@ -8,6 +8,9 @@ const adapterMocks = vi.hoisted(() => ({ finalize: vi.fn(), prepare: vi.fn(), })); +const jetsonMocks = vi.hoisted(() => ({ + detectDeviceGroupGids: vi.fn<() => string[]>(), +})); vi.mock("./adapter", async (importOriginal) => ({ ...(await importOriginal()), @@ -15,6 +18,28 @@ vi.mock("./adapter", async (importOriginal) => ({ finalizeManagedBootstrapSequence: adapterMocks.finalize, prepareManagedBootstrapSequence: adapterMocks.prepare, })); +vi.mock("../docker-gpu-jetson-groups", async (importOriginal) => ({ + ...(await importOriginal()), + detectTegraDeviceGroupGids: jetsonMocks.detectDeviceGroupGids, +})); +vi.mock("../docker-gpu-patch-mode", async (importOriginal) => { + const original = await importOriginal(); + return { + ...original, + selectDockerGpuPatchMode: (options: { backend?: "generic" | "jetson" }) => ({ + mode: original.buildDockerGpuMode( + options.backend === "jetson" ? "nvidia-runtime" : "gpus", + "all", + { backend: options.backend }, + ), + attempts: [], + }), + }; +}); +vi.mock("../docker-gpu-sandbox-create", async (importOriginal) => ({ + ...(await importOriginal()), + isDockerDesktopWslRuntime: () => false, +})); import type { ManagedBootstrapActivatedTransaction, @@ -26,9 +51,90 @@ import { authority, IDENTITY, NEW_ID, OLD_ID } from "./docker-test-fixture"; beforeEach(() => { vi.clearAllMocks(); + jetsonMocks.detectDeviceGroupGids.mockReturnValue(["44", "993"]); }); +async function replacementOptionsFor( + agent: "openclaw" | "hermes", + hostGpuPlatform: "jetson" | "linux", +) { + const seed = authority(agent); + const prepared = Object.freeze({}) as ManagedBootstrapPreparedTransaction; + const activated = Object.freeze({ + snapshot: { runtimeId: OLD_ID }, + replacement: { replacementRuntimeId: NEW_ID }, + }) as ManagedBootstrapActivatedTransaction; + adapterMocks.prepare.mockImplementationOnce(async (_adapter, input) => { + await input.create.launch({ + heldWorkloadArgv: seed.handle.heldWorkloadArgv, + bootstrapIdentity: IDENTITY, + }); + return prepared; + }); + adapterMocks.activate.mockResolvedValueOnce(activated); + const lifecycle = createDockerManagedBootstrapSurface().createLifecycle({ + providerId: "docker", + bootstrapIdentity: IDENTITY, + request: seed.request, + image: seed.plan.image, + agentIdentity: seed.plan.agentIdentity, + intendedWorkloadArgv: seed.plan.intendedWorkloadArgv, + expectedSupervisorArgv: seed.plan.expectedSupervisorArgv, + launchArgv: ["openshell", "sandbox", "create", "--name", "alpha"], + heldWorkloadArgv: seed.handle.heldWorkloadArgv, + authorityStore: { recordPreparedAuthority: vi.fn() }, + adapterOverride: {} as ManagedBootstrapAdapter, + route: "compatibility", + persistStartupCommand: true, + sandboxName: "alpha", + sandboxGpuConfig: { + mode: "1", + hostGpuDetected: true, + hostGpuPlatform, + sandboxGpuEnabled: true, + sandboxGpuDevice: "all", + errors: [], + }, + requiredLimits: [], + timeoutSecs: 30, + network: { + inferenceProvider: "ollama-local", + gatewayUsesContainerBridge: false, + gatewayPort: 0, + }, + dependencies: {}, + }); + + await lifecycle.runCreate(async () => ({ + value: "launched", + receipt: seed.handle.createReceipt, + })); + const prepareInput = adapterMocks.prepare.mock.calls.at(-1)?.[1]; + expect(prepareInput).toBeDefined(); + return prepareInput!.replacementOptions.values; +} + describe("Docker managed-bootstrap lifecycle composition", () => { + it("preserves detected Jetson groups for OpenClaw compatibility replacement (#7610)", async () => { + const values = await replacementOptionsFor("openclaw", "jetson"); + + expect(values).toMatchObject({ + extraGroupGids: ["44", "993"], + gpuModeKind: "nvidia-runtime", + preserveJetsonDeviceGroupMembership: true, + }); + }); + + it.each([ + ["openclaw", "linux", []], + ["hermes", "jetson", ["44", "993"]], + ] as const)("does not enable Jetson group preservation for %s on %s (#7610)", async (agent, platform, expectedGroupGids) => { + const values = await replacementOptionsFor(agent, platform); + + expect(values.extraGroupGids).toEqual(expectedGroupGids); + expect(values.preserveJetsonDeviceGroupMembership).toBe(false); + }); + it("does not finalize rollback after a claimed commit loses acknowledgement", async () => { const seed = authority("openclaw"); const prepared = Object.freeze({}) as ManagedBootstrapPreparedTransaction; diff --git a/src/lib/onboard/managed-bootstrap/docker-runtime.ts b/src/lib/onboard/managed-bootstrap/docker-runtime.ts index a05a982ec9..ec32a79a26 100644 --- a/src/lib/onboard/managed-bootstrap/docker-runtime.ts +++ b/src/lib/onboard/managed-bootstrap/docker-runtime.ts @@ -43,6 +43,8 @@ function dockerReplacementOptions( input: ManagedBootstrapRuntimeCreateLifecycleInput, ) { const backend = input.sandboxGpuConfig.hostGpuPlatform === "jetson" ? "jetson" : "generic"; + const extraGroupGids = + backend === "jetson" && input.route === "compatibility" ? detectTegraDeviceGroupGids() : []; return { values: { gpuModeArgs: [...mode.args], @@ -52,8 +54,9 @@ function dockerReplacementOptions( requiredUlimits: input.requiredLimits.map( (limit) => `${limit.name}=${limit.soft}:${limit.hard}`, ), - extraGroupGids: - backend === "jetson" && input.route === "compatibility" ? detectTegraDeviceGroupGids() : [], + extraGroupGids, + preserveJetsonDeviceGroupMembership: + extraGroupGids.length > 0 && input.request.agent === "openclaw", }, }; } diff --git a/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts b/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts index 6e52d0c6a4..d305ca1191 100644 --- a/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts +++ b/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts @@ -351,9 +351,13 @@ export function fixture(options: DockerFixtureOptions = {}) { const name = String(args[args.indexOf("--name") + 1] ?? ""); const entrypoint = String(args[args.indexOf("--entrypoint") + 1] ?? ""); const imageIndex = args.indexOf(IMAGE); - const env = args.flatMap((value, index) => - value === "--env" ? [String(args[index + 1] ?? "")] : [], - ); + const flagValues = (flag: string) => + args.flatMap((value, index) => (value === flag ? [String(args[index + 1] ?? "")] : [])); + const env = flagValues("--env"); + const groupAdds = flagValues("--group-add"); + const capAdds = flagValues("--cap-add"); + const securityOptions = flagValues("--security-opt"); + const runtime = flagValues("--runtime").at(-1); replacement = { ...structuredClone(source), Id: NEW_ID, @@ -365,6 +369,13 @@ export function fixture(options: DockerFixtureOptions = {}) { Entrypoint: [entrypoint], Cmd: args.slice(imageIndex + 1), }, + HostConfig: { + ...structuredClone(source.HostConfig), + ...(capAdds.length > 0 ? { CapAdd: capAdds } : {}), + ...(groupAdds.length > 0 ? { GroupAdd: groupAdds } : {}), + ...(runtime ? { Runtime: runtime } : {}), + ...(securityOptions.length > 0 ? { SecurityOpt: securityOptions } : {}), + }, State: { Running: false, Paused: false, Restarting: false, Dead: false }, }; return losesAcknowledgement("container:create") diff --git a/src/lib/onboard/managed-bootstrap/docker.test.ts b/src/lib/onboard/managed-bootstrap/docker.test.ts index a72236b3b4..237604a8d4 100644 --- a/src/lib/onboard/managed-bootstrap/docker.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker.test.ts @@ -7,7 +7,10 @@ import { ManagedBootstrapDurableCommitCleanupPendingError, ManagedBootstrapOwnerCleanupRequiredError, } from "./adapter"; -import { createDockerManagedBootstrapAdapter } from "./docker"; +import { + createDockerManagedBootstrapAdapter, + MANAGED_BOOTSTRAP_TRAMPOLINE_EXECUTABLE, +} from "./docker"; import { normalizeDockerManagedBootstrapLaunchSpec, parseDockerManagedBootstrapLaunchSpec, @@ -31,6 +34,94 @@ function expectEventBefore(events: readonly string[], before: string, after: str } describe("Docker managed bootstrap adapter", () => { + it("passes Jetson groups to Docker and the sandbox-account bootstrap without replacing the managed entrypoint (#7610)", async () => { + const fake = fixture({ agent: "openclaw" }); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, request, snapshot } = authority("openclaw"); + + await adapter.prepareBootstrapReplacement({ + handle, + snapshot, + request, + replacementOptions: { + values: { + gpuModeArgs: [ + "--runtime", + "nvidia", + "--env", + "NVIDIA_VISIBLE_DEVICES=all", + "--env", + "NVIDIA_DRIVER_CAPABILITIES=compute,utility", + ], + gpuModeDevice: "all", + gpuModeKind: "nvidia-runtime", + gpuModeLabel: "--runtime nvidia (NVIDIA_VISIBLE_DEVICES=all)", + extraGroupGids: ["44", "993"], + preserveJetsonDeviceGroupMembership: true, + }, + }, + }); + + const createArgs = vi + .mocked(fake.deps.dockerRun!) + .mock.calls.find(([args]) => args[0] === "create")?.[0]; + expect(createArgs).toEqual( + expect.arrayContaining([ + "--group-add", + "44", + "--group-add", + "993", + "--env", + "NEMOCLAW_JETSON_DEVICE_GROUP_GIDS=44,993", + "--entrypoint", + MANAGED_BOOTSTRAP_TRAMPOLINE_EXECUTABLE, + ]), + ); + expect(fake.replacement?.Config?.Entrypoint).toEqual([MANAGED_BOOTSTRAP_TRAMPOLINE_EXECUTABLE]); + expect(fake.replacement?.Config?.Cmd).toEqual( + expect.arrayContaining(["--agent", "openclaw", "--bootstrap-identity", IDENTITY]), + ); + }); + + it("rejects a non-boolean managed Jetson group-preservation option (#7610)", async () => { + const fake = fixture({ agent: "openclaw" }); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, request, snapshot } = authority("openclaw"); + + await expect( + adapter.prepareBootstrapReplacement({ + handle, + snapshot, + request, + replacementOptions: { + values: { preserveJetsonDeviceGroupMembership: "true" }, + }, + }), + ).rejects.toThrow("Jetson device-group preservation must be a boolean"); + expect(fake.replacement).toBeNull(); + }); + + it("rejects managed Jetson group preservation for a non-OpenClaw agent (#7610)", async () => { + const fake = fixture({ agent: "hermes" }); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, request, snapshot } = authority("hermes"); + + await expect( + adapter.prepareBootstrapReplacement({ + handle, + snapshot, + request, + replacementOptions: { + values: { + extraGroupGids: ["44"], + preserveJetsonDeviceGroupMembership: true, + }, + }, + }), + ).rejects.toThrow("Jetson device-group preservation requires the OpenClaw agent"); + expect(fake.replacement).toBeNull(); + }); + it("publishes durable commit authority before deleting the rollback backup after lost acknowledgements", async () => { const fake = fixture({ lostAcknowledgements: [ diff --git a/src/lib/onboard/managed-bootstrap/docker.ts b/src/lib/onboard/managed-bootstrap/docker.ts index 817263d729..e49f861b80 100644 --- a/src/lib/onboard/managed-bootstrap/docker.ts +++ b/src/lib/onboard/managed-bootstrap/docker.ts @@ -654,6 +654,7 @@ function replacementPlan(options: ManagedBootstrapReplacementOptions): { readonly mode: DockerGpuPatchMode; readonly requiredUlimits: readonly DockerUlimit[]; readonly extraGroupGids: readonly string[]; + readonly preserveJetsonDeviceGroupMembership: boolean; } { const allowed = new Set([ "gpuModeArgs", @@ -661,6 +662,7 @@ function replacementPlan(options: ManagedBootstrapReplacementOptions): { "gpuModeKind", "gpuModeLabel", "extraGroupGids", + "preserveJetsonDeviceGroupMembership", "requiredUlimits", ]); const unknown = Object.keys(options.values).filter((key) => !allowed.has(key)); @@ -674,6 +676,11 @@ function replacementPlan(options: ManagedBootstrapReplacementOptions): { throw new Error(`Managed bootstrap Docker GPU mode '${kind}' is invalid.`); } const args = exactStringArray(options.values.gpuModeArgs ?? [], "GPU mode arguments"); + const preserveJetsonDeviceGroupMembership = + options.values.preserveJetsonDeviceGroupMembership ?? false; + if (typeof preserveJetsonDeviceGroupMembership !== "boolean") { + throw new Error("Managed bootstrap Docker Jetson device-group preservation must be a boolean."); + } return { mode: { kind, @@ -689,6 +696,7 @@ function replacementPlan(options: ManagedBootstrapReplacementOptions): { return value; }, ), + preserveJetsonDeviceGroupMembership, requiredUlimits: parseRequiredUlimits(options.values.requiredUlimits), }; } @@ -797,20 +805,34 @@ function modeEnvironment(mode: DockerGpuPatchMode): string[] { function assertExactEnvironmentDelta( original: Record, replacement: Record, - mode: DockerGpuPatchMode, + plan: { + readonly mode: DockerGpuPatchMode; + readonly extraGroupGids: readonly string[]; + readonly preserveJetsonDeviceGroupMembership: boolean; + }, intendedSandboxCommand: string, ): void { + const { mode } = plan; const gpuAugment = mode.kind !== "startup-command"; const originalEnv = exactStringArray(original.Env ?? [], "original environment"); const expected = [ ...modeEnvironment(mode), ...originalEnv - .filter((entry) => !gpuAugment || !REPLACED_GPU_ENV_KEYS.has(entry.split("=", 1)[0] ?? "")) + .filter((entry) => { + const key = entry.split("=", 1)[0] ?? ""; + return ( + key !== "NEMOCLAW_JETSON_DEVICE_GROUP_GIDS" && + (!gpuAugment || !REPLACED_GPU_ENV_KEYS.has(key)) + ); + }) .map((entry) => entry.startsWith("OPENSHELL_SANDBOX_COMMAND=") ? `OPENSHELL_SANDBOX_COMMAND=${intendedSandboxCommand}` : entry, ), + ...(plan.preserveJetsonDeviceGroupMembership && plan.extraGroupGids.length > 0 + ? [`NEMOCLAW_JETSON_DEVICE_GROUP_GIDS=${plan.extraGroupGids.join(",")}`] + : []), ]; const observed = exactStringArray(replacement.Env ?? [], "replacement environment"); if (!exactArrayEqual(observed, expected)) { @@ -929,7 +951,7 @@ function scrubVerifiedReplacementDeltas(canonicalJson: string): string { config.Entrypoint = [""]; config.Cmd = [""]; config.Env = ""; - for (const key of [ + const verifiedHostKeys = [ "CapAdd", "DeviceRequests", "Devices", @@ -937,7 +959,9 @@ function scrubVerifiedReplacementDeltas(canonicalJson: string): string { "Runtime", "SecurityOpt", "Ulimits", - ]) { + ] as const; + for (const key of verifiedHostKeys) delete host[key]; + for (const key of verifiedHostKeys) { host[key] = ``; } return JSON.stringify(root); @@ -951,6 +975,7 @@ function assertReplacementMatchesIntent( readonly mode: DockerGpuPatchMode; readonly requiredUlimits: readonly DockerUlimit[]; readonly extraGroupGids: readonly string[]; + readonly preserveJetsonDeviceGroupMembership: boolean; }, intendedSandboxCommand: string, ): string { @@ -967,7 +992,7 @@ function assertReplacementMatchesIntent( const observedConfig = objectField(observedInspect, "Config"); const observedHost = objectField(observedInspect, "HostConfig"); const gpuAugment = plan.mode.kind !== "startup-command"; - assertExactEnvironmentDelta(originalConfig, observedConfig, plan.mode, intendedSandboxCommand); + assertExactEnvironmentDelta(originalConfig, observedConfig, plan, intendedSandboxCommand); assertExactStringSet( observedHost.CapAdd, [ @@ -3171,6 +3196,16 @@ export function createDockerManagedBootstrapAdapter( ); } const plan = replacementPlan(replacementOptions); + if ( + plan.preserveJetsonDeviceGroupMembership && + (request.agent !== "openclaw" || plan.extraGroupGids.length === 0) + ) { + throw new Error( + request.agent !== "openclaw" + ? "Managed bootstrap Docker Jetson device-group preservation requires the OpenClaw agent." + : "Managed bootstrap Docker Jetson device-group preservation requires device groups.", + ); + } const originalName = dockerContainerName(parsed.inspect); const backupContainerName = backupName(originalName, handle.bootstrapIdentity); const stagingName = replacementStagingName(originalName, handle.bootstrapIdentity); @@ -3189,6 +3224,7 @@ export function createDockerManagedBootstrapAdapter( openshellSandboxCommand: handle.intendedWorkloadArgv, requiredUlimits: plan.requiredUlimits, extraGroupGids: plan.extraGroupGids, + preserveJetsonDeviceGroupMembership: plan.preserveJetsonDeviceGroupMembership, containerEntrypoint: MANAGED_BOOTSTRAP_TRAMPOLINE_EXECUTABLE, containerCommand: trampolineCommand, containerName: stagingName, diff --git a/test/managed-bootstrap-trampoline.test.ts b/test/managed-bootstrap-trampoline.test.ts index 5da0253256..b5d09347a4 100644 --- a/test/managed-bootstrap-trampoline.test.ts +++ b/test/managed-bootstrap-trampoline.test.ts @@ -603,7 +603,7 @@ exec /usr/bin/env -i NEMOCLAW_MANAGED_BOOTSTRAP_RESUME=1 ${JSON.stringify( it.each( MANAGED_STARTUP_AGENTS, - )("consumes the protected %s request or recovered claim before exact supervisor exec and drops bootstrap variables", (agent) => { + )("consumes the protected %s request and applies eligible Jetson groups before exact supervisor exec (#7610)", (agent) => { const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-bootstrap-trampoline-")); try { const request = path.join(directory, "request.json"); @@ -615,6 +615,7 @@ exec /usr/bin/env -i NEMOCLAW_MANAGED_BOOTSTRAP_RESUME=1 ${JSON.stringify( const trace = path.join(directory, "trace"); const script = path.join(directory, "trampoline.sh"); const supervisor = path.join(directory, "supervisor"); + const jetsonGroupBootstrap = path.join(directory, "jetson-device-group-bootstrap"); const injection = path.join(directory, "injection"); const attackerFunction = path.join(directory, "attacker-function-ran"); fs.mkdirSync(sandbox); @@ -642,6 +643,14 @@ esac path.join(directory, "rm"), '#!/bin/sh\ntest ! -e /proc/self/fd/9\nexec /bin/rm "$@"\n', ); + executable( + jetsonGroupBootstrap, + `#!/bin/sh +test ! -e /proc/self/fd/9 +printf 'groups:%s\\n' "$NEMOCLAW_JETSON_DEVICE_GROUP_GIDS" >>${JSON.stringify(trace)} +exec "$@" +`, + ); executable( path.join(directory, "node"), `#!/bin/sh @@ -679,6 +688,11 @@ test -e ${JSON.stringify(attackerFunction)} test -z "\${NEMOCLAW_MANAGED_BOOTSTRAP_ENTRYPOINT+x}" test -z "\${NEMOCLAW_MANAGED_BOOTSTRAP_RESUME+x}" test -z "\${NEMOCLAW_MANAGED_BOOTSTRAP_RESUME_EXECUTABLE+x}" +${ + agent === "openclaw" + ? 'test "$NEMOCLAW_JETSON_DEVICE_GROUP_GIDS" = "44,993"' + : 'test -z "${NEMOCLAW_JETSON_DEVICE_GROUP_GIDS+x}"' +} printf 'supervisor:%s|%s|%s:identity=%s:request=%s:home=%s:path=%s:lang=%s:capability=%s:bash-env=%s\\n' "$1" "$2" "$3" "\${_nemoclaw_bootstrap_identity-unset}" "\${_nemoclaw_request-unset}" "$HOME" "$PATH" "$LANG" "\${NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION-unset}" "\${BASH_ENV+x}" >>"$TRACE" `, ); @@ -693,6 +707,10 @@ printf 'supervisor:%s|%s|%s:identity=%s:request=%s:home=%s:path=%s:lang=%s:capab ) .replaceAll("/var/lib/nemoclaw-managed-bootstrap-request.json", request) .replaceAll("/sandbox", sandbox) + .replaceAll( + "/usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh", + jetsonGroupBootstrap, + ) .replaceAll("/usr/local/bin/node", path.join(directory, "node")); fs.writeFileSync(script, source, { mode: 0o644 }); fs.chmodSync(script, 0o644); @@ -740,6 +758,7 @@ printf 'supervisor:%s|%s|%s:identity=%s:request=%s:home=%s:path=%s:lang=%s:capab LD_PRELOAD: loader.library, DYLD_INSERT_LIBRARIES: loader.library, LD_AUDIT: loader.library, + ...(agent === "openclaw" ? { NEMOCLAW_JETSON_DEVICE_GROUP_GIDS: "44,993" } : {}), }; execFileSync(entrypoint, argv, { env: environment }); @@ -750,6 +769,7 @@ printf 'supervisor:%s|%s|%s:identity=%s:request=%s:home=%s:path=%s:lang=%s:capab expect(fs.existsSync(loader.earlyTrace)).toBe(false); expect(fs.existsSync(loader.afterTrace)).toBe(true); expect(fs.readFileSync(trace, "utf8").trim().split("\n")).toEqual([ + ...(agent === "openclaw" ? ["groups:44,993"] : []), `node:${runtime} --recover-bootstrap-claim --agent ${agent} --profile-fingerprint ${fingerprint} --bootstrap-identity ${identity}:home=/root:path=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:lang=C.UTF-8:capability=1`, `node:${runtime} --apply-bootstrap-file --agent ${agent} --profile-fingerprint ${fingerprint} --bootstrap-identity ${identity}:home=/root:path=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:lang=C.UTF-8:capability=1`, `node:${runtime} --verify-bootstrap-completion --agent ${agent} --profile-fingerprint ${fingerprint} --bootstrap-identity ${identity}:home=/root:path=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:lang=C.UTF-8:capability=1`, From 70600b307230d05fd6ff911366239b1b5f47a744 Mon Sep 17 00:00:00 2001 From: San Dang Date: Wed, 5 Aug 2026 22:55:07 +0700 Subject: [PATCH 12/49] fix(installer): configure Jetson nvmap before L4T classification Signed-off-by: San Dang --- docs/reference/troubleshooting.mdx | 12 +++++++-- scripts/setup-jetson.sh | 43 +++++++++++++++++++----------- test/setup-jetson.test.ts | 31 ++++++++++++++++++++- 3 files changed, 68 insertions(+), 18 deletions(-) diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index ecbd4d3fa7..07ce17601b 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -211,7 +211,13 @@ Some R39 images already ship with `br_netfilter` configured and are left untouch On affected R39 hosts, the installer prints `loading br_netfilter (required by k3s inside the OpenShell gateway)`. Without this fix, sandbox pods fail DNS resolution against the in-cluster service and the onboard `Setting up OpenClaw inside sandbox` step times out. -If the L4T version is not recognized, the setup step is skipped and the installer continues normally. +If the L4T version is not recognized, the installer skips the version-specific iptables, Docker, and `br_netfilter` setup and continues in an untested configuration. + + + +The OpenClaw installer still configures a real `/dev/nvmap` character device independently of the parsed L4T release. + + ### DNS resolution from inside docker fails (corporate firewall) @@ -2736,8 +2742,10 @@ Use `NEMOCLAW_DOCKER_GPU_PATCH=0` only for troubleshooting because it bypasses t It also keeps eligible Jetson GPU device group memberships when OpenShell starts the nonroot sandbox user. The compatibility policy permits read-only access to the NVIDIA runtime library directory at `/opt/nvidia/l4t-gpu-libs` when that path is present. -If CUDA reports `NvRmMemInitNvmap failed with Permission denied` and `cuInit(0)=999` on JetPack 6, rerun the NemoClaw installer to apply persistent group read-write access to `/dev/nvmap`: +On an OpenClaw Jetson host, the installer applies persistent group read-write access when `/dev/nvmap` is a real character device. If the device is absent, setup warns and continues without installing the rule. +This device permission setup runs even when the L4T release line is unrecognized, but it does not make that release tested or supported. +If CUDA reports `NvRmMemInitNvmap failed with Permission denied` and `cuInit(0)=999`, rerun the NemoClaw installer to apply the permission setup: The installer grants write access to every member of the existing `/dev/nvmap` owning group. diff --git a/scripts/setup-jetson.sh b/scripts/setup-jetson.sh index 8d69d43108..3486f55308 100755 --- a/scripts/setup-jetson.sh +++ b/scripts/setup-jetson.sh @@ -6,6 +6,7 @@ set -euo pipefail SUDO=() ((EUID != 0)) && SUDO=(sudo) +JETSON_HOST_SUDO_READY=0 NVMAP_DEVICE="/dev/nvmap" NVMAP_UDEV_RULE="/etc/udev/rules.d/99-zz-nemoclaw-nvmap.rules" @@ -24,6 +25,16 @@ error() { exit 1 } +ensure_jetson_host_sudo() { + if ((EUID == 0)) || [[ "$JETSON_HOST_SUDO_READY" == "1" ]]; then + return 0 + fi + + info "Jetson host configuration requires sudo. You may be prompted for your password." + "${SUDO[@]}" true >/dev/null || error "Sudo is required to apply Jetson host configuration." + JETSON_HOST_SUDO_READY=1 +} + # Returns 0 only when both the live kernel state AND our persistent # drop-ins are in place: # - runtime: bridge-nf-call-iptables sysctl reads back as 1 @@ -64,7 +75,7 @@ configure_nvmap_group_access() { local device_state device_type verified_state verified_permissions if ! device_state="$(LC_ALL=C stat -c '%F|%A' "$NVMAP_DEVICE" 2>/dev/null)"; then - warn "JetPack 6 host setup could not find $NVMAP_DEVICE. Non-root sandbox CUDA can fail until this device exists." + warn "Jetson host setup could not find $NVMAP_DEVICE. Non-root sandbox CUDA can fail until this device exists." return 0 fi @@ -72,7 +83,8 @@ configure_nvmap_group_access() { [[ "$device_type" == "character special file" ]] \ || error "$NVMAP_DEVICE must be a character device before NemoClaw changes its group permissions." - warn "JetPack 6 host setup grants every member of the existing $NVMAP_DEVICE owning group write access and persists mode 0660 when udev recreates the device." + ensure_jetson_host_sudo + warn "Jetson host setup grants every member of the existing $NVMAP_DEVICE owning group write access and persists mode 0660 when udev recreates the device." printf '%s\n' "$NVMAP_UDEV_RULE_CONTENT" | "${SUDO[@]}" tee "$NVMAP_UDEV_RULE" >/dev/null "${SUDO[@]}" udevadm control --reload-rules "${SUDO[@]}" chmod g+rw "$NVMAP_DEVICE" @@ -94,10 +106,7 @@ warn_host_setup_skipped() { } get_jetpack_version() { - local release_line release revision l4t_version - - release_line="$(head -n1 /etc/nv_tegra_release 2>/dev/null || true)" - [[ -n "$release_line" ]] || return 0 + local release_line="$1" release revision l4t_version release="$(printf '%s\n' "$release_line" | sed -n 's/^# R\([0-9][0-9]*\) (release).*/\1/p')" revision="$(printf '%s\n' "$release_line" | sed -n 's/^.*REVISION: \([0-9][0-9]*\)\..*$/\1/p')" @@ -155,10 +164,7 @@ get_jetpack_version() { configure_jetson_host() { local jetpack_version="$1" - if ((EUID != 0)); then - info "Jetson host configuration requires sudo. You may be prompted for your password." - "${SUDO[@]}" true >/dev/null || error "Sudo is required to apply Jetson host configuration." - fi + ensure_jetson_host_sudo case "$jetpack_version" in jp6) @@ -218,9 +224,6 @@ except Exception: os.unlink(tmp) raise PYEOF - if [[ "${NEMOCLAW_AGENT:-openclaw}" == "openclaw" ]]; then - configure_nvmap_group_access - fi ;; jp7-r38) # JP7 R38 does not need iptables or Docker daemon.json changes. @@ -238,8 +241,18 @@ PYEOF } main() { - local jetpack_version - jetpack_version="$(get_jetpack_version)" + local jetpack_version release_line + + release_line="$(head -n1 /etc/nv_tegra_release 2>/dev/null || true)" + [[ -n "$release_line" ]] || exit 0 + + # nvmap permissions follow the detected device, not the L4T version parser. + # Version-specific networking changes remain gated below. + if [[ "${NEMOCLAW_AGENT:-openclaw}" == "openclaw" ]]; then + configure_nvmap_group_access + fi + + jetpack_version="$(get_jetpack_version "$release_line")" [[ -n "$jetpack_version" ]] || exit 0 info "Jetson detected ($jetpack_version) — applying required host configuration" diff --git a/test/setup-jetson.test.ts b/test/setup-jetson.test.ts index 5260b96eac..5f1a02069c 100644 --- a/test/setup-jetson.test.ts +++ b/test/setup-jetson.test.ts @@ -366,7 +366,7 @@ describe("setup-jetson host setup on an unrecognized L4T release (#7612)", () => }); }); -describe("setup-jetson JetPack 6 nvmap access", () => { +describe("setup-jetson OpenClaw nvmap access", () => { it("grants the nvmap owning group read-write access and persists the mode on JetPack 6 (#7610)", () => { const result = withJetsonReleaseSandbox( ({ commandLogPath, headArgsPath, releasePath, stubDir }) => { @@ -452,6 +452,35 @@ describe("setup-jetson JetPack 6 nvmap access", () => { expect(result.commandLog).toContain("chmod g+rw /dev/nvmap"); }); + it("configures nvmap before skipping version-specific setup for an unrecognized L4T release (#7610)", () => { + const result = withJetsonReleaseSandbox( + ({ commandLogPath, headArgsPath, releasePath, stubDir }) => { + writeFileSync( + releasePath, + "# R00 (release), REVISION: 0.0, GCID: 46579312, BOARD: generic\n", + ); + return spawnSetupJetson(stubDir, headArgsPath, commandLogPath, { + NEMOCLAW_TEST_STAT_OUTPUT: "character special file|cr--r-----", + NEMOCLAW_TEST_STAT_OUTPUT_AFTER: "character special file|cr--rw----", + }); + }, + ); + + expect(result.status).toBe(0); + expect(result.commandLog).toContain("tee /etc/udev/rules.d/99-zz-nemoclaw-nvmap.rules"); + expect(result.commandLog).toContain('stdin KERNEL=="nvmap", MODE="0660"'); + expect(result.commandLog).toContain("chmod g+rw /dev/nvmap"); + expect(result.commandLog).not.toContain("update-alternatives"); + expect(result.commandLog).not.toContain("modprobe br_netfilter"); + expect(result.commandLog).not.toContain("systemctl restart docker"); + expect(result.stdout).toContain("/dev/nvmap grants its owning group read-write access"); + expect(result.stderr).toContain( + "Jetson detected (L4T 00.0) but this L4T release is not recognized.", + ); + expect(result.stderr).toContain("Skipped Jetson host setup"); + expect(result.stderr).toContain("Installation continues in an untested configuration."); + }); + it.each([ "hermes", "langchain-deepagents-code", From 25d2c8b7d6727ff0f5067948b10712fd45e90bc5 Mon Sep 17 00:00:00 2001 From: San Dang Date: Wed, 5 Aug 2026 23:42:55 +0700 Subject: [PATCH 13/49] fix(onboard): verify Jetson nvmap access before sandbox creation Signed-off-by: San Dang --- docs/reference/troubleshooting.mdx | 14 ++- scripts/setup-jetson.sh | 13 ++- .../onboard/docker-gpu-jetson-groups.test.ts | 66 ++++++++++++- src/lib/onboard/docker-gpu-jetson-groups.ts | 95 +++++++++++++++++-- src/lib/onboard/docker-gpu-patch-types.ts | 7 +- .../onboard/sandbox-gpu-create-flow.test.ts | 53 +++++++++++ src/lib/onboard/sandbox-gpu-create-flow.ts | 9 ++ .../sandbox-gpu-preflight-routing.test.ts | 11 +++ src/lib/onboard/sandbox-gpu-preflight.ts | 15 +-- test/setup-jetson.test.ts | 24 ++++- 10 files changed, 277 insertions(+), 30 deletions(-) diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 07ce17601b..69858b628f 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -2742,18 +2742,16 @@ Use `NEMOCLAW_DOCKER_GPU_PATCH=0` only for troubleshooting because it bypasses t It also keeps eligible Jetson GPU device group memberships when OpenShell starts the nonroot sandbox user. The compatibility policy permits read-only access to the NVIDIA runtime library directory at `/opt/nvidia/l4t-gpu-libs` when that path is present. -On an OpenClaw Jetson host, the installer applies persistent group read-write access when `/dev/nvmap` is a real character device. -If the device is absent, setup warns and continues without installing the rule. -This device permission setup runs even when the L4T release line is unrecognized, but it does not make that release tested or supported. -If CUDA reports `NvRmMemInitNvmap failed with Permission denied` and `cuInit(0)=999`, rerun the NemoClaw installer to apply the permission setup: +Before each OpenClaw Jetson GPU sandbox creation, including `$$nemoclaw onboard --resume`, NemoClaw verifies that `/dev/nvmap` is a real character device whose owning group has read-write access. -The installer grants write access to every member of the existing `/dev/nvmap` owning group. +When repair is required, NemoClaw grants write access to every member of the existing `/dev/nvmap` owning group. -```bash -curl -fsSL https://www.nvidia.com/nemoclaw.sh | bash -``` +If the initial verification fails, NemoClaw runs the same host permission setup as the installer. +After setup exits successfully, NemoClaw verifies the device again. +If setup fails or the device still lacks verified group read-write access, onboarding stops before sandbox creation. +This device permission setup runs even when the L4T release line is unrecognized, but it does not make that release tested or supported. Verify the persistent rule and the live device permissions: diff --git a/scripts/setup-jetson.sh b/scripts/setup-jetson.sh index 3486f55308..35a9d2a595 100755 --- a/scripts/setup-jetson.sh +++ b/scripts/setup-jetson.sh @@ -241,7 +241,18 @@ PYEOF } main() { - local jetpack_version release_line + local jetpack_version release_line mode="${1:-}" + + if [[ "$#" -gt 1 || (-n "$mode" && "$mode" != "--nvmap-only") ]]; then + error "Usage: setup-jetson.sh [--nvmap-only]" + fi + + if [[ "$mode" == "--nvmap-only" ]]; then + [[ "${NEMOCLAW_AGENT:-openclaw}" == "openclaw" ]] \ + || error "Jetson nvmap-only setup is available only for OpenClaw." + configure_nvmap_group_access + return 0 + fi release_line="$(head -n1 /etc/nv_tegra_release 2>/dev/null || true)" [[ -n "$release_line" ]] || exit 0 diff --git a/src/lib/onboard/docker-gpu-jetson-groups.test.ts b/src/lib/onboard/docker-gpu-jetson-groups.test.ts index 25d83eb274..8d4ae1f232 100644 --- a/src/lib/onboard/docker-gpu-jetson-groups.test.ts +++ b/src/lib/onboard/docker-gpu-jetson-groups.test.ts @@ -5,7 +5,17 @@ import fs from "node:fs"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { detectTegraDeviceGroupGids } from "./docker-gpu-jetson-groups"; +import { + detectTegraDeviceGroupGids, + ensureJetsonNvmapGroupAccess, +} from "./docker-gpu-jetson-groups"; + +const READ_ONLY_NVMAP = { + isCharacterDevice: true, + isSymbolicLink: false, + mode: 0o440, +}; +const READ_WRITE_NVMAP = { ...READ_ONLY_NVMAP, mode: 0o660 }; describe("detectTegraDeviceGroupGids", () => { afterEach(() => { @@ -142,3 +152,57 @@ describe("detectTegraDeviceGroupGids", () => { } }); }); + +describe("ensureJetsonNvmapGroupAccess", () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("keeps verified group read-write access without running host setup (#7610)", () => { + const runSetup = vi.fn(); + + ensureJetsonNvmapGroupAccess({ statDevice: () => READ_WRITE_NVMAP, runSetup }); + + expect(runSetup).not.toHaveBeenCalled(); + }); + + it("repairs group-read-only nvmap access and verifies the result (#7610)", () => { + const statDevice = vi + .fn() + .mockReturnValueOnce(READ_ONLY_NVMAP) + .mockReturnValueOnce(READ_WRITE_NVMAP); + const runSetup = vi.fn(() => ({ status: 0 })); + vi.spyOn(console, "log").mockImplementation(() => {}); + + ensureJetsonNvmapGroupAccess({ + statDevice, + runSetup, + setupScriptPath: "/package/scripts/setup-jetson.sh", + }); + + expect(runSetup).toHaveBeenCalledWith("/package/scripts/setup-jetson.sh"); + expect(statDevice).toHaveBeenCalledTimes(2); + }); + + it("stops before sandbox creation when host setup fails (#7610)", () => { + vi.spyOn(console, "log").mockImplementation(() => {}); + + expect(() => + ensureJetsonNvmapGroupAccess({ + statDevice: () => READ_ONLY_NVMAP, + runSetup: () => ({ status: 1 }), + }), + ).toThrow("Jetson /dev/nvmap group setup failed before sandbox creation (exit status 1)"); + }); + + it("rejects an unverified post-setup device state (#7610)", () => { + vi.spyOn(console, "log").mockImplementation(() => {}); + + expect(() => + ensureJetsonNvmapGroupAccess({ + statDevice: () => READ_ONLY_NVMAP, + runSetup: () => ({ status: 0 }), + }), + ).toThrow("still does not grant its owning group read-write access"); + }); +}); diff --git a/src/lib/onboard/docker-gpu-jetson-groups.ts b/src/lib/onboard/docker-gpu-jetson-groups.ts index e484c19ccc..4b8ffbbf32 100644 --- a/src/lib/onboard/docker-gpu-jetson-groups.ts +++ b/src/lib/onboard/docker-gpu-jetson-groups.ts @@ -1,7 +1,9 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { spawnSync } from "node:child_process"; import fs from "node:fs"; +import path from "node:path"; const TEGRA_GPU_DEVICE_NODES = [ "/dev/nvmap", @@ -18,12 +20,87 @@ const TEGRA_GPU_DEVICE_NODES = [ ] as const; const READ_WRITE_PERMISSION_BITS = 0o6; const MAX_DOCKER_SUPPLEMENTARY_GID = 2_147_483_647; +const NVMAP_DEVICE = "/dev/nvmap"; type DeviceGroupAccess = { gid: number; mode: number; }; +type NvmapDeviceAccess = { + isCharacterDevice: boolean; + isSymbolicLink: boolean; + mode: number; +}; + +export interface EnsureJetsonNvmapGroupAccessDeps { + statDevice?: () => NvmapDeviceAccess | null; + runSetup?: (scriptPath: string) => { status: number | null; error?: Error }; + setupScriptPath?: string; +} + +function defaultStatNvmapDevice(): NvmapDeviceAccess | null { + try { + const stat = fs.lstatSync(NVMAP_DEVICE); + return { + isCharacterDevice: stat.isCharacterDevice(), + isSymbolicLink: stat.isSymbolicLink(), + mode: stat.mode, + }; + } catch { + return null; + } +} + +function hasGroupReadWriteAccess(access: NvmapDeviceAccess | null): boolean { + return ( + access !== null && + access.isCharacterDevice && + !access.isSymbolicLink && + ((access.mode >> 3) & READ_WRITE_PERMISSION_BITS) === READ_WRITE_PERMISSION_BITS + ); +} + +function runJetsonNvmapSetup(scriptPath: string): { status: number | null; error?: Error } { + const result = spawnSync("bash", [scriptPath, "--nvmap-only"], { + env: { ...process.env, NEMOCLAW_AGENT: "openclaw" }, + stdio: "inherit", + }); + return { status: result.status, ...(result.error ? { error: result.error } : {}) }; +} + +function setupFailureDetail(result: { status: number | null; error?: Error }): string { + if (result.error?.message) return result.error.message; + return `exit status ${result.status === null ? "unknown" : result.status}`; +} + +/** + * Verify the host permission that makes the detected nvmap group useful to the + * nonroot OpenShell sandbox user. Installer setup is not sufficient at this + * boundary because onboarding can run directly or after host device state + * changes. Repair before any sandbox create or replacement begins. + */ +export function ensureJetsonNvmapGroupAccess(deps: EnsureJetsonNvmapGroupAccessDeps = {}): void { + const statDevice = deps.statDevice ?? defaultStatNvmapDevice; + if (hasGroupReadWriteAccess(statDevice())) return; + + const setupScriptPath = + deps.setupScriptPath ?? path.resolve(__dirname, "../../../scripts/setup-jetson.sh"); + const runSetup = deps.runSetup ?? runJetsonNvmapSetup; + console.log(" Preparing Jetson /dev/nvmap group access before sandbox creation..."); + const result = runSetup(setupScriptPath); + if (result.status !== 0) { + throw new Error( + `Jetson /dev/nvmap group setup failed before sandbox creation (${setupFailureDetail(result)}).`, + ); + } + if (!hasGroupReadWriteAccess(statDevice())) { + throw new Error( + "Jetson /dev/nvmap still does not grant its owning group read-write access after host setup; refusing sandbox creation.", + ); + } +} + /** * Find real DRI render character devices without following symlinks or * scanning other DRI device families. @@ -54,16 +131,16 @@ function listTegraGpuDevicePaths(): string[] { /** * Source-of-truth boundary for Jetson/Tegra supplementary device groups: * - * - Invalid state: the non-root sandbox user can see `/dev/nvmap` and `/dev/nvhost-*` but cannot - * open them because Docker did not copy their host-owned supplementary GIDs into the container. - * - Source boundary: host device-node ownership is authoritative; NemoClaw only carries each - * bounded, non-root numeric GID with effective group read/write permission into the Jetson - * compatibility recreation via `--group-add`. - * - Source-fix constraint: changing host udev ownership or image-local groups cannot reliably fix - * device nodes whose ownership is assigned by the Jetson host at runtime. + * - Invalid state: `/dev/nvmap` lacks owning-group read/write access, or the non-root sandbox user + * loses the matching host device GID when the OpenShell supervisor calls `initgroups()`. + * - Source boundary: NemoClaw verifies and persists the host nvmap mode, carries each bounded + * numeric device GID into the Jetson recreation via `--group-add`, and records matching sandbox + * account membership before the supervisor starts. + * - Source-fix constraint: `--group-add` alone does not survive the supervisor's account-group + * initialization, and image-local group names can differ from the host's numeric device GIDs. * - Regression coverage: docker-gpu-jetson-groups.test.ts covers discovery and hostile numeric - * values; docker-gpu-patch-jetson.test.ts covers clone-envelope propagation and generic-host - * exclusion. + * values; setup-jetson.test.ts covers the host mode; docker-gpu-patch-jetson.test.ts covers + * clone-envelope and sandbox-account propagation plus generic-host exclusion. * - Removal condition: remove this probe when the minimum supported native OpenShell Jetson path * propagates the host device groups without compatibility container recreation. */ diff --git a/src/lib/onboard/docker-gpu-patch-types.ts b/src/lib/onboard/docker-gpu-patch-types.ts index f3c18711a4..fa9c17a0f6 100644 --- a/src/lib/onboard/docker-gpu-patch-types.ts +++ b/src/lib/onboard/docker-gpu-patch-types.ts @@ -42,9 +42,10 @@ export type DockerGpuPatchDeps = { /** * Resolve the host group ID(s) that own the Jetson/Tegra GPU device nodes * (`/dev/nvmap`, `/dev/nvhost-*`, and `/dev/dri/renderD*`). Used by the - * Jetson recreate to grant the sandbox user matching `--group-add` - * membership so CUDA can open them (#4231, #7610). Injectable so the Jetson - * permission path is testable without Tegra hardware. + * Jetson recreate for Docker `--group-add` and matching sandbox-account + * membership before the OpenShell supervisor calls `initgroups()`, so CUDA + * can open them (#4231, #7610). Injectable so the Jetson permission path is + * testable without Tegra hardware. */ detectTegraDeviceGroupGids?: () => string[]; /** Injectable directory lister for unit testing CDI spec discovery. */ diff --git a/src/lib/onboard/sandbox-gpu-create-flow.test.ts b/src/lib/onboard/sandbox-gpu-create-flow.test.ts index f60f0a59ab..c5e3b791c5 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.test.ts @@ -169,6 +169,59 @@ function createSourceInput(): SandboxGpuCreateFlowInput { beforeEach(() => setupGpuFlowMocks(mocks)); afterEach(resetGpuFlowMocks); +describe("runSandboxGpuCreateFlow Jetson host access", () => { + it("repairs OpenClaw nvmap access before starting a sandbox attempt (#7610)", async () => { + const input = createInput(); + input.agentName = "openclaw"; + input.sandboxGpuConfig.hostGpuPlatform = "jetson"; + const deps = createDeps(); + deps.ensureJetsonNvmapGroupAccess = vi.fn(); + + await expect(runSandboxGpuCreateFlow(input, deps)).resolves.toMatchObject({ route: "native" }); + + expect(deps.ensureJetsonNvmapGroupAccess).toHaveBeenCalledOnce(); + expect(vi.mocked(deps.ensureJetsonNvmapGroupAccess).mock.invocationCallOrder[0]).toBeLessThan( + mocks.streamSandboxCreate.mock.invocationCallOrder[0]!, + ); + }); + + it("stops before sandbox creation when OpenClaw nvmap access cannot be repaired (#7610)", async () => { + const input = createInput(); + input.agentName = "openclaw"; + input.sandboxGpuConfig.hostGpuPlatform = "jetson"; + const deps = createDeps(); + deps.ensureJetsonNvmapGroupAccess = vi.fn(() => { + throw new Error("nvmap host repair failed"); + }); + + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow("nvmap host repair failed"); + + expect(mocks.streamSandboxCreate).not.toHaveBeenCalled(); + expect(mocks.createDockerGpuSandboxCreatePatch).not.toHaveBeenCalled(); + }); + + it.each([ + { agentName: "hermes", hostGpuPlatform: "jetson", sandboxGpuEnabled: true }, + { agentName: "openclaw", hostGpuPlatform: null, sandboxGpuEnabled: true }, + { agentName: "openclaw", hostGpuPlatform: "jetson", sandboxGpuEnabled: false }, + ] as const)("does not change nvmap access for agent=$agentName platform=$hostGpuPlatform enabled=$sandboxGpuEnabled (#7610)", async ({ + agentName, + hostGpuPlatform, + sandboxGpuEnabled, + }) => { + const input = createInput(); + input.agentName = agentName; + input.sandboxGpuConfig.hostGpuPlatform = hostGpuPlatform; + input.sandboxGpuConfig.sandboxGpuEnabled = sandboxGpuEnabled; + const deps = createDeps(); + deps.ensureJetsonNvmapGroupAccess = vi.fn(); + + await runSandboxGpuCreateFlow(input, deps); + + expect(deps.ensureJetsonNvmapGroupAccess).not.toHaveBeenCalled(); + }); +}); + describe("runSandboxGpuCreateFlow provider-owned managed create", () => { it("recovers before an MXC-style create without a Docker branch in central orchestration", async () => { const input = createInput(); diff --git a/src/lib/onboard/sandbox-gpu-create-flow.ts b/src/lib/onboard/sandbox-gpu-create-flow.ts index a4ccfbaef6..58d7782dfd 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.ts @@ -4,6 +4,7 @@ import type { StreamSandboxCreateResult } from "../sandbox/create-stream"; import { redactFull } from "../security/redact"; import type { SandboxGpuProofResult } from "../state/registry"; +import { ensureJetsonNvmapGroupAccess } from "./docker-gpu-jetson-groups"; import * as dockerGpuLocalInference from "./docker-gpu-local-inference"; import { collectDockerGpuPatchDiagnostics } from "./docker-gpu-patch"; import type { DockerGpuPatchDeps, DockerUlimit } from "./docker-gpu-patch-types"; @@ -105,6 +106,7 @@ export interface SandboxGpuCreateFlowDeps { sleep: Sleep; openshellArgv(args: string[]): string[]; verifyDirectSandboxGpu(sandboxName: string): SandboxGpuProofResult; + ensureJetsonNvmapGroupAccess?: () => void; /** Production callers omit this factory and use the runtime provider's adapter. */ createManagedBootstrapAdapter?: () => ManagedBootstrapAdapter; } @@ -135,6 +137,13 @@ export async function runSandboxGpuCreateFlow( deps: SandboxGpuCreateFlowDeps, ): Promise { let registryImageRef: string | null = input.prebuild.imageRef; + if ( + input.sandboxGpuConfig.sandboxGpuEnabled && + input.sandboxGpuConfig.hostGpuPlatform === "jetson" && + (input.agentName ?? "openclaw") === "openclaw" + ) { + (deps.ensureJetsonNvmapGroupAccess ?? ensureJetsonNvmapGroupAccess)(); + } const attemptRunner = createSandboxGpuCreateAttemptRunner(input, deps); const gpuCreateOutcome = await sandboxGpuCreateAttempt .executeSandboxGpuCreatePlan(input.gpuRoutePlan, { diff --git a/src/lib/onboard/sandbox-gpu-preflight-routing.test.ts b/src/lib/onboard/sandbox-gpu-preflight-routing.test.ts index 9a1e70c3f2..8b9b002f05 100644 --- a/src/lib/onboard/sandbox-gpu-preflight-routing.test.ts +++ b/src/lib/onboard/sandbox-gpu-preflight-routing.test.ts @@ -11,6 +11,7 @@ import type { SandboxGpuConfig } from "./sandbox-gpu-mode"; import { dockerNvidiaRuntimeAvailable, formatSandboxGpuPassthroughNote, + jetsonGpuProofRemediationLines, parseDockerRuntimeNames, sandboxGpuRemediationLines, validateSandboxGpuPreflight, @@ -28,6 +29,16 @@ function sandboxGpuConfig(overrides: Partial = {}): SandboxGpu }; } describe("sandbox GPU preflight routing", () => { + it("describes every Jetson device-access boundary after CUDA proof failure (#7610)", () => { + const remediation = jetsonGpuProofRemediationLines().join("\n"); + + expect(remediation).toContain("owning-group read-write access"); + expect(remediation).toContain("propagate the device GIDs through Docker"); + expect(remediation).toContain("before OpenShell calls initgroups()"); + expect(remediation).not.toContain("add the host video/render groups via --group-add"); + expect(remediation).not.toContain("must be readable by the sandbox"); + }); + it("formats Jetson sandbox GPU notes around the NVIDIA runtime backend", () => { expect(formatSandboxGpuPassthroughNote({ hostGpuPlatform: "jetson" })).toContain( "Docker NVIDIA runtime", diff --git a/src/lib/onboard/sandbox-gpu-preflight.ts b/src/lib/onboard/sandbox-gpu-preflight.ts index 5fbbde0fe0..ed176c3e81 100644 --- a/src/lib/onboard/sandbox-gpu-preflight.ts +++ b/src/lib/onboard/sandbox-gpu-preflight.ts @@ -58,16 +58,17 @@ export function resolveSandboxGpuFlagFromOptions(opts: SandboxGpuFlagOptions): S // Jetson/Tegra CUDA failures are usually device/group permission issues rather // than CDI/runtime misconfiguration: the sandbox sees the GPU but the agent -// user lacks access to the Tegra device nodes. Surface the concrete devices and -// groups so the user can fix the recreate rather than seeing a bare "enabled" -// status that hides an unusable GPU (#4231). +// user lacks access to the Tegra device nodes. Surface the complete host-mode, +// Docker-group, and sandbox-account boundary rather than an incomplete manual +// --group-add workaround or a bare "enabled" status (#4231, #7610). export function jetsonGpuProofRemediationLines(): string[] { return [ "Jetson/Tegra CUDA proof did not pass. CUDA needs access to the Tegra device", - "nodes; confirm the sandbox propagates them and the agent user's groups:", - " ls -l /dev/nvmap /dev/nvhost-* (must be readable by the sandbox)", - " add the host video/render groups via --group-add when recreating", - "Then recreate the sandbox, or force CPU behavior with NEMOCLAW_SANDBOX_GPU=0.", + "nodes. NemoClaw must verify host /dev/nvmap owning-group read-write access,", + "propagate the device GIDs through Docker, and add matching membership to the", + "sandbox account before OpenShell calls initgroups(). Review the onboarding output", + "and saved diagnostics to identify the failing boundary, then retry onboarding;", + "or use NEMOCLAW_SANDBOX_GPU=0 for CPU.", ]; } diff --git a/test/setup-jetson.test.ts b/test/setup-jetson.test.ts index 5f1a02069c..67aeacfe99 100644 --- a/test/setup-jetson.test.ts +++ b/test/setup-jetson.test.ts @@ -58,6 +58,7 @@ function withJetsonReleaseSandbox( const statCountPath = path.join(tempDir, "stat-count"); mkdirSync(stubDir); writeFileSync(commandLogPath, ""); + writeFileSync(headArgsPath, ""); for (const command of HOST_MUTATION_COMMANDS) { const stubPath = path.join(stubDir, command); writeFileSync( @@ -115,8 +116,9 @@ function spawnSetupJetson( headArgsPath: string, commandLogPath: string, extraEnv: NodeJS.ProcessEnv = {}, + scriptArgs: string[] = [], ): SetupJetsonRun { - const result = spawnSync("bash", [SCRIPT_PATH], { + const result = spawnSync("bash", [SCRIPT_PATH, ...scriptArgs], { encoding: "utf-8", env: { ...process.env, @@ -367,6 +369,26 @@ describe("setup-jetson host setup on an unrecognized L4T release (#7612)", () => }); describe("setup-jetson OpenClaw nvmap access", () => { + it("repairs nvmap without requiring an L4T release in nvmap-only mode (#7610)", () => { + const result = withJetsonReleaseSandbox(({ commandLogPath, headArgsPath, stubDir }) => + spawnSetupJetson( + stubDir, + headArgsPath, + commandLogPath, + { + NEMOCLAW_TEST_STAT_OUTPUT: "character special file|cr--r-----", + NEMOCLAW_TEST_STAT_OUTPUT_AFTER: "character special file|cr--rw----", + }, + ["--nvmap-only"], + ), + ); + + expect(result.status).toBe(0); + expect(result.headArgs).toBe(""); + expect(result.commandLog).toContain("chmod g+rw /dev/nvmap"); + expect(result.stdout).toContain("/dev/nvmap grants its owning group read-write access"); + }); + it("grants the nvmap owning group read-write access and persists the mode on JetPack 6 (#7610)", () => { const result = withJetsonReleaseSandbox( ({ commandLogPath, headArgsPath, releasePath, stubDir }) => { From e9198bc3d2d024cc3a5c41c74668a442a58a243c Mon Sep 17 00:00:00 2001 From: San Dang Date: Thu, 6 Aug 2026 00:13:38 +0700 Subject: [PATCH 14/49] test(jetson): add nvmap boundary proof Signed-off-by: San Dang --- scripts/jetson-nvmap-poc.sh | 201 ++++++++++++++++++++++++++++++++++++ 1 file changed, 201 insertions(+) create mode 100755 scripts/jetson-nvmap-poc.sh diff --git a/scripts/jetson-nvmap-poc.sh b/scripts/jetson-nvmap-poc.sh new file mode 100755 index 0000000000..6bb5aec5ee --- /dev/null +++ b/scripts/jetson-nvmap-poc.sh @@ -0,0 +1,201 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -uo pipefail + +readonly CUDA_PROBE='import ctypes; lib=ctypes.CDLL("libcuda.so.1"); rc=lib.cuInit(0); print(f"cuInit(0)={rc}"); raise SystemExit(rc != 0)' + +run_sandbox_probe() { + local cuda_status nvmap_status + + id + grep '^Groups:' /proc/self/status || true + if stat -Lc 'nvmap type=%F mode=%a uid=%u gid=%g' /dev/nvmap 2>/dev/null; then + if [[ -r /dev/nvmap && -w /dev/nvmap ]]; then + echo 'nvmap_access=read-write' + nvmap_status=0 + else + echo 'nvmap_access=denied' + nvmap_status=1 + fi + else + echo 'nvmap_access=missing' + nvmap_status=1 + fi + + python3 -c "$CUDA_PROBE" + cuda_status=$? + [[ "$nvmap_status" == "0" && "$cuda_status" == "0" ]] +} + +run_stage() { + local label="$1" + shift + + printf '\n=== %s ===\n' "$label" + "$@" + local status=$? + printf 'stage_status=%s\n' "$status" + return "$status" +} + +if [[ "${1:-}" == "--inside-sandbox" ]]; then + run_sandbox_probe + exit $? +fi + +if [[ "${1:-}" == "--inside-root" ]]; then + echo "sandbox_account=$(id sandbox 2>&1 || true)" + for gid in ${NEMOCLAW_JETSON_DEVICE_GROUP_GIDS//,/ }; do + getent group "$gid" || true + done + + sandbox_uid="$(id -u sandbox)" + sandbox_gid="$(id -g sandbox)" + if command -v setpriv >/dev/null 2>&1; then + exec setpriv --reuid="$sandbox_uid" --regid="$sandbox_gid" --init-groups \ + /bin/bash /tmp/nemoclaw-jetson-nvmap-poc --inside-sandbox + fi + if command -v runuser >/dev/null 2>&1; then + exec runuser -u sandbox -- /bin/bash /tmp/nemoclaw-jetson-nvmap-poc --inside-sandbox + fi + echo 'Neither setpriv nor runuser is available to exercise initgroups().' >&2 + exit 1 +fi + +sandbox_name="${1:-tm}" +command -v docker >/dev/null 2>&1 || { + echo 'docker is required.' >&2 + exit 1 +} +command -v openshell >/dev/null 2>&1 || { + echo 'openshell is required.' >&2 + exit 1 +} + +mapfile -t sandbox_container_ids < <( + docker ps -q \ + --filter 'label=openshell.ai/managed-by=openshell' \ + --filter "label=openshell.ai/sandbox-name=${sandbox_name}" +) +if [[ "${#sandbox_container_ids[@]}" != "1" ]]; then + echo "Expected one running OpenShell Docker container for sandbox '${sandbox_name}', found ${#sandbox_container_ids[@]}." >&2 + exit 1 +fi + +readonly sandbox_container_id="${sandbox_container_ids[0]}" +readonly sandbox_image_id +sandbox_image_id="$(docker inspect --format '{{.Image}}' "$sandbox_container_id")" +readonly script_path +script_path="$(readlink -f "$0")" + +printf 'sandbox=%s\ncontainer=%s\nimage=%s\n' \ + "$sandbox_name" "$sandbox_container_id" "$sandbox_image_id" + +printf '\n=== host device ===\n' +host_status=0 +if stat -Lc 'type=%F mode=%a uid=%u gid=%g group=%G path=%n' /dev/nvmap 2>/dev/null; then + host_mode="$(stat -Lc '%a' /dev/nvmap)" + host_mode_value=$((8#${host_mode: -3})) + if ((((host_mode_value >> 3) & 6) != 6)); then + echo 'host_nvmap_group_access=not-read-write' + host_status=1 + else + echo 'host_nvmap_group_access=read-write' + fi +else + echo 'host_nvmap=missing' + host_status=1 +fi +if [[ -f /etc/udev/rules.d/99-zz-nemoclaw-nvmap.rules ]]; then + echo 'host_nvmap_udev_rule=present' +else + echo 'host_nvmap_udev_rule=missing' +fi +printf 'stage_status=%s\n' "$host_status" + +mapfile -t tegra_group_gids < <( + for device in \ + /dev/nvmap \ + /dev/nvhost-ctrl \ + /dev/nvhost-ctrl-gpu \ + /dev/nvhost-gpu \ + /dev/nvhost-as-gpu \ + /dev/nvhost-prof-gpu \ + /dev/nvhost-dbg-gpu \ + /dev/nvhost-tsg-gpu \ + /dev/nvgpu/igpu0/ctrl \ + /dev/nvgpu/igpu0/as \ + /dev/nvgpu/igpu0/prof \ + /dev/dri/renderD*; do + [[ -c "$device" && ! -L "$device" ]] || continue + read -r gid mode < <(stat -Lc '%g %a' "$device") + mode_value=$((8#${mode: -3})) + group_bits=$(((mode_value >> 3) & 6)) + other_bits=$((mode_value & 6)) + if ((gid > 0 && group_bits == 6 && other_bits != 6)); then + printf '%s\n' "$gid" + fi + done | sort -nu +) +readonly tegra_gids_csv +tegra_gids_csv="$( + IFS=, + echo "${tegra_group_gids[*]}" +)" +echo "detected_tegra_group_gids=${tegra_gids_csv:-none}" + +probe_command="$(declare -f run_sandbox_probe); CUDA_PROBE=$(printf '%q' "$CUDA_PROBE"); run_sandbox_probe" + +run_stage 'outer container as root' \ + docker exec --user 0 "$sandbox_container_id" /bin/bash -lc "$probe_command" +outer_root_status=$? + +run_stage 'outer container as sandbox' \ + docker exec --user sandbox "$sandbox_container_id" /bin/bash -lc "$probe_command" +outer_sandbox_status=$? + +run_stage 'OpenShell sandbox execution' \ + openshell sandbox exec -n "$sandbox_name" -- /bin/bash -lc "$probe_command" +openshell_status=$? + +isolated_status=1 +printf '\n=== isolated Docker bootstrap POC ===\n' +if [[ "$host_status" != "0" ]]; then + echo 'skipped: host /dev/nvmap does not grant its owning group read-write access' +elif [[ "${#tegra_group_gids[@]}" == "0" ]]; then + echo 'skipped: no usable Tegra device groups were detected' +else + docker_args=( + run --rm + --runtime nvidia + --env NVIDIA_VISIBLE_DEVICES=all + --env 'NVIDIA_DRIVER_CAPABILITIES=compute,utility' + --env "NEMOCLAW_JETSON_DEVICE_GROUP_GIDS=$tegra_gids_csv" + --volume "$script_path:/tmp/nemoclaw-jetson-nvmap-poc:ro" + --entrypoint /usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh + ) + for gid in "${tegra_group_gids[@]}"; do + docker_args+=(--group-add "$gid") + done + docker_args+=("$sandbox_image_id" /bin/bash /tmp/nemoclaw-jetson-nvmap-poc --inside-root) + docker "${docker_args[@]}" + isolated_status=$? +fi +printf 'stage_status=%s\n' "$isolated_status" + +printf '\n=== verdict ===\n' +if [[ "$host_status" != "0" ]]; then + echo 'FAIL boundary=host-nvmap-mode' +elif [[ "$isolated_status" != "0" ]]; then + echo 'FAIL boundary=docker-runtime-or-bootstrap' +elif [[ "$outer_sandbox_status" != "0" ]]; then + echo 'FAIL boundary=recreated-container-bootstrap' +elif [[ "$openshell_status" != "0" ]]; then + echo 'FAIL boundary=openshell-sandbox-execution' +else + echo 'PASS boundary=end-to-end' +fi + +echo "evidence host=$host_status isolated=$isolated_status outer_root=$outer_root_status outer_sandbox=$outer_sandbox_status openshell=$openshell_status" From c7d6e059e41479337f3cbb9cc5131721c5f47271 Mon Sep 17 00:00:00 2001 From: San Dang Date: Thu, 6 Aug 2026 00:25:40 +0700 Subject: [PATCH 15/49] fix(test): initialize Jetson POC readonly values Signed-off-by: San Dang --- scripts/jetson-nvmap-poc.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/jetson-nvmap-poc.sh b/scripts/jetson-nvmap-poc.sh index 6bb5aec5ee..0c714b1956 100755 --- a/scripts/jetson-nvmap-poc.sh +++ b/scripts/jetson-nvmap-poc.sh @@ -85,10 +85,10 @@ if [[ "${#sandbox_container_ids[@]}" != "1" ]]; then fi readonly sandbox_container_id="${sandbox_container_ids[0]}" -readonly sandbox_image_id sandbox_image_id="$(docker inspect --format '{{.Image}}' "$sandbox_container_id")" -readonly script_path +readonly sandbox_image_id script_path="$(readlink -f "$0")" +readonly script_path printf 'sandbox=%s\ncontainer=%s\nimage=%s\n' \ "$sandbox_name" "$sandbox_container_id" "$sandbox_image_id" @@ -139,11 +139,11 @@ mapfile -t tegra_group_gids < <( fi done | sort -nu ) -readonly tegra_gids_csv tegra_gids_csv="$( IFS=, echo "${tegra_group_gids[*]}" )" +readonly tegra_gids_csv echo "detected_tegra_group_gids=${tegra_gids_csv:-none}" probe_command="$(declare -f run_sandbox_probe); CUDA_PROBE=$(printf '%q' "$CUDA_PROBE"); run_sandbox_probe" From f27f7245a0e5b55b97413ccd21e962ead9574b59 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Wed, 5 Aug 2026 10:24:11 -0700 Subject: [PATCH 16/49] fix(test): make Jetson nvmap proof fail closed Signed-off-by: Apurv Kumaria --- scripts/jetson-nvmap-poc.sh | 6 +++ test/jetson-nvmap-poc.test.ts | 85 +++++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 test/jetson-nvmap-poc.test.ts diff --git a/scripts/jetson-nvmap-poc.sh b/scripts/jetson-nvmap-poc.sh index 0c714b1956..b75d9e1621 100755 --- a/scripts/jetson-nvmap-poc.sh +++ b/scripts/jetson-nvmap-poc.sh @@ -186,16 +186,22 @@ fi printf 'stage_status=%s\n' "$isolated_status" printf '\n=== verdict ===\n' +verdict_status=0 if [[ "$host_status" != "0" ]]; then echo 'FAIL boundary=host-nvmap-mode' + verdict_status=1 elif [[ "$isolated_status" != "0" ]]; then echo 'FAIL boundary=docker-runtime-or-bootstrap' + verdict_status=1 elif [[ "$outer_sandbox_status" != "0" ]]; then echo 'FAIL boundary=recreated-container-bootstrap' + verdict_status=1 elif [[ "$openshell_status" != "0" ]]; then echo 'FAIL boundary=openshell-sandbox-execution' + verdict_status=1 else echo 'PASS boundary=end-to-end' fi echo "evidence host=$host_status isolated=$isolated_status outer_root=$outer_root_status outer_sandbox=$outer_sandbox_status openshell=$openshell_status" +exit "$verdict_status" diff --git a/test/jetson-nvmap-poc.test.ts b/test/jetson-nvmap-poc.test.ts new file mode 100644 index 0000000000..6cd29acef1 --- /dev/null +++ b/test/jetson-nvmap-poc.test.ts @@ -0,0 +1,85 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +const SCRIPT_PATH = path.join(import.meta.dirname, "..", "scripts", "jetson-nvmap-poc.sh"); + +describe("Jetson nvmap boundary proof", () => { + it("reaches the final verdict and exits nonzero when a required boundary fails (#7610)", () => { + const tempDir = mkdtempSync(path.join(tmpdir(), "nemoclaw-jetson-nvmap-poc-")); + + try { + const stubDir = path.join(tempDir, "bin"); + const commandLog = path.join(tempDir, "commands.log"); + mkdirSync(stubDir); + writeFileSync(commandLog, ""); + + const dockerStub = path.join(stubDir, "docker"); + writeFileSync( + dockerStub, + [ + "#!/usr/bin/env bash", + "set -euo pipefail", + `printf 'docker %s\\n' "$*" >> ${JSON.stringify(commandLog)}`, + 'case "${1:-}" in', + " ps) echo container-123 ;;&", + " inspect) echo sha256:image-123 ;;&", + " exec) exit 0 ;;&", + "esac", + "", + ].join("\n"), + ); + chmodSync(dockerStub, 0o755); + + const openshellStub = path.join(stubDir, "openshell"); + writeFileSync( + openshellStub, + [ + "#!/usr/bin/env bash", + "set -euo pipefail", + `printf 'openshell %s\\n' "$*" >> ${JSON.stringify(commandLog)}`, + "exit 0", + "", + ].join("\n"), + ); + chmodSync(openshellStub, 0o755); + + const statStub = path.join(stubDir, "stat"); + writeFileSync( + statStub, + [ + "#!/usr/bin/env bash", + "set -euo pipefail", + 'if [[ "$*" == *"%a"* && "$*" != *"type="* ]]; then', + " echo 660", + "else", + " echo 'type=character special file mode=660 uid=0 gid=44 group=video path=/dev/nvmap'", + "fi", + "", + ].join("\n"), + ); + chmodSync(statStub, 0o755); + + const result = spawnSync("bash", [SCRIPT_PATH, "test-sandbox"], { + encoding: "utf8", + env: { ...process.env, PATH: `${stubDir}${path.delimiter}${process.env.PATH ?? ""}` }, + }); + + expect(result.status).toBe(1); + expect(result.stderr).not.toContain("readonly variable"); + expect(result.stdout).toContain("sandbox=test-sandbox"); + expect(result.stdout).toContain("image=sha256:image-123"); + expect(result.stdout).toContain("FAIL boundary=docker-runtime-or-bootstrap"); + expect(result.stdout).toContain("evidence host=0 isolated=1"); + expect(readFileSync(commandLog, "utf8")).toContain("docker exec --user sandbox"); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); +}); From bf27905f5bb091a7f0e59d7ef1ec8c261b2c7d91 Mon Sep 17 00:00:00 2001 From: San Dang Date: Thu, 6 Aug 2026 00:43:10 +0700 Subject: [PATCH 17/49] fix(onboard): grant Jetson devices in sandbox policy Signed-off-by: San Dang --- docs/reference/troubleshooting.mdx | 8 +- scripts/jetson-nvmap-poc.sh | 207 ------------------ .../onboard/docker-gpu-jetson-groups.test.ts | 23 ++ src/lib/onboard/docker-gpu-jetson-groups.ts | 54 ++++- src/lib/onboard/docker-gpu-patch-types.ts | 6 +- src/lib/onboard/initial-policy.test.ts | 52 +++++ src/lib/onboard/initial-policy.ts | 22 ++ .../sandbox-gpu-preflight-routing.test.ts | 5 +- src/lib/onboard/sandbox-gpu-preflight.ts | 13 +- test/jetson-nvmap-poc.test.ts | 85 ------- 10 files changed, 160 insertions(+), 315 deletions(-) delete mode 100755 scripts/jetson-nvmap-poc.sh delete mode 100644 test/jetson-nvmap-poc.test.ts diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 69858b628f..0e63aed8a9 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -813,7 +813,7 @@ NVIDIA NIM and GPU-backed sandbox setup require a real NVIDIA GPU. If NemoClaw rejects the detected GPU name during preflight, select a CPU or remote inference provider, or move the setup to a host with a supported NVIDIA GPU and current drivers. Jetson/Tegra hosts support sandbox GPU passthrough through the compatibility route. -Onboarding detects those hosts separately and propagates eligible host group IDs for selected `/dev/nvmap`, `/dev/nvhost-*`, and `/dev/nvgpu/igpu0/*` nodes plus real `/dev/dri/renderD*` character devices. +Onboarding detects those hosts separately and prepares access to selected `/dev/nvmap`, `/dev/nvhost-*`, and `/dev/nvgpu/igpu0/*` nodes plus real `/dev/dri/renderD*` character devices. If that path fails, follow the Jetson/Tegra compatibility guidance below instead of treating a missing `nvidia-smi` result as a placeholder adapter. ### Colima socket not detected (macOS) @@ -2739,8 +2739,10 @@ Use `NEMOCLAW_DOCKER_GPU_PATCH=0` only for troubleshooting because it bypasses t -It also keeps eligible Jetson GPU device group memberships when OpenShell starts the nonroot sandbox user. -The compatibility policy permits read-only access to the NVIDIA runtime library directory at `/opt/nvidia/l4t-gpu-libs` when that path is present. +The compatibility path keeps eligible Jetson GPU device group memberships when OpenShell starts the nonroot sandbox user. +Group membership alone does not grant device access when the sandbox filesystem policy omits a required path. +The route-specific policy grants read-write access only to selected paths that exist as character devices and are not symbolic links. +It also permits read-only access to the NVIDIA runtime library directory at `/opt/nvidia/l4t-gpu-libs` when that path is present. Before each OpenClaw Jetson GPU sandbox creation, including `$$nemoclaw onboard --resume`, NemoClaw verifies that `/dev/nvmap` is a real character device whose owning group has read-write access. diff --git a/scripts/jetson-nvmap-poc.sh b/scripts/jetson-nvmap-poc.sh deleted file mode 100755 index b75d9e1621..0000000000 --- a/scripts/jetson-nvmap-poc.sh +++ /dev/null @@ -1,207 +0,0 @@ -#!/usr/bin/env bash -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -set -uo pipefail - -readonly CUDA_PROBE='import ctypes; lib=ctypes.CDLL("libcuda.so.1"); rc=lib.cuInit(0); print(f"cuInit(0)={rc}"); raise SystemExit(rc != 0)' - -run_sandbox_probe() { - local cuda_status nvmap_status - - id - grep '^Groups:' /proc/self/status || true - if stat -Lc 'nvmap type=%F mode=%a uid=%u gid=%g' /dev/nvmap 2>/dev/null; then - if [[ -r /dev/nvmap && -w /dev/nvmap ]]; then - echo 'nvmap_access=read-write' - nvmap_status=0 - else - echo 'nvmap_access=denied' - nvmap_status=1 - fi - else - echo 'nvmap_access=missing' - nvmap_status=1 - fi - - python3 -c "$CUDA_PROBE" - cuda_status=$? - [[ "$nvmap_status" == "0" && "$cuda_status" == "0" ]] -} - -run_stage() { - local label="$1" - shift - - printf '\n=== %s ===\n' "$label" - "$@" - local status=$? - printf 'stage_status=%s\n' "$status" - return "$status" -} - -if [[ "${1:-}" == "--inside-sandbox" ]]; then - run_sandbox_probe - exit $? -fi - -if [[ "${1:-}" == "--inside-root" ]]; then - echo "sandbox_account=$(id sandbox 2>&1 || true)" - for gid in ${NEMOCLAW_JETSON_DEVICE_GROUP_GIDS//,/ }; do - getent group "$gid" || true - done - - sandbox_uid="$(id -u sandbox)" - sandbox_gid="$(id -g sandbox)" - if command -v setpriv >/dev/null 2>&1; then - exec setpriv --reuid="$sandbox_uid" --regid="$sandbox_gid" --init-groups \ - /bin/bash /tmp/nemoclaw-jetson-nvmap-poc --inside-sandbox - fi - if command -v runuser >/dev/null 2>&1; then - exec runuser -u sandbox -- /bin/bash /tmp/nemoclaw-jetson-nvmap-poc --inside-sandbox - fi - echo 'Neither setpriv nor runuser is available to exercise initgroups().' >&2 - exit 1 -fi - -sandbox_name="${1:-tm}" -command -v docker >/dev/null 2>&1 || { - echo 'docker is required.' >&2 - exit 1 -} -command -v openshell >/dev/null 2>&1 || { - echo 'openshell is required.' >&2 - exit 1 -} - -mapfile -t sandbox_container_ids < <( - docker ps -q \ - --filter 'label=openshell.ai/managed-by=openshell' \ - --filter "label=openshell.ai/sandbox-name=${sandbox_name}" -) -if [[ "${#sandbox_container_ids[@]}" != "1" ]]; then - echo "Expected one running OpenShell Docker container for sandbox '${sandbox_name}', found ${#sandbox_container_ids[@]}." >&2 - exit 1 -fi - -readonly sandbox_container_id="${sandbox_container_ids[0]}" -sandbox_image_id="$(docker inspect --format '{{.Image}}' "$sandbox_container_id")" -readonly sandbox_image_id -script_path="$(readlink -f "$0")" -readonly script_path - -printf 'sandbox=%s\ncontainer=%s\nimage=%s\n' \ - "$sandbox_name" "$sandbox_container_id" "$sandbox_image_id" - -printf '\n=== host device ===\n' -host_status=0 -if stat -Lc 'type=%F mode=%a uid=%u gid=%g group=%G path=%n' /dev/nvmap 2>/dev/null; then - host_mode="$(stat -Lc '%a' /dev/nvmap)" - host_mode_value=$((8#${host_mode: -3})) - if ((((host_mode_value >> 3) & 6) != 6)); then - echo 'host_nvmap_group_access=not-read-write' - host_status=1 - else - echo 'host_nvmap_group_access=read-write' - fi -else - echo 'host_nvmap=missing' - host_status=1 -fi -if [[ -f /etc/udev/rules.d/99-zz-nemoclaw-nvmap.rules ]]; then - echo 'host_nvmap_udev_rule=present' -else - echo 'host_nvmap_udev_rule=missing' -fi -printf 'stage_status=%s\n' "$host_status" - -mapfile -t tegra_group_gids < <( - for device in \ - /dev/nvmap \ - /dev/nvhost-ctrl \ - /dev/nvhost-ctrl-gpu \ - /dev/nvhost-gpu \ - /dev/nvhost-as-gpu \ - /dev/nvhost-prof-gpu \ - /dev/nvhost-dbg-gpu \ - /dev/nvhost-tsg-gpu \ - /dev/nvgpu/igpu0/ctrl \ - /dev/nvgpu/igpu0/as \ - /dev/nvgpu/igpu0/prof \ - /dev/dri/renderD*; do - [[ -c "$device" && ! -L "$device" ]] || continue - read -r gid mode < <(stat -Lc '%g %a' "$device") - mode_value=$((8#${mode: -3})) - group_bits=$(((mode_value >> 3) & 6)) - other_bits=$((mode_value & 6)) - if ((gid > 0 && group_bits == 6 && other_bits != 6)); then - printf '%s\n' "$gid" - fi - done | sort -nu -) -tegra_gids_csv="$( - IFS=, - echo "${tegra_group_gids[*]}" -)" -readonly tegra_gids_csv -echo "detected_tegra_group_gids=${tegra_gids_csv:-none}" - -probe_command="$(declare -f run_sandbox_probe); CUDA_PROBE=$(printf '%q' "$CUDA_PROBE"); run_sandbox_probe" - -run_stage 'outer container as root' \ - docker exec --user 0 "$sandbox_container_id" /bin/bash -lc "$probe_command" -outer_root_status=$? - -run_stage 'outer container as sandbox' \ - docker exec --user sandbox "$sandbox_container_id" /bin/bash -lc "$probe_command" -outer_sandbox_status=$? - -run_stage 'OpenShell sandbox execution' \ - openshell sandbox exec -n "$sandbox_name" -- /bin/bash -lc "$probe_command" -openshell_status=$? - -isolated_status=1 -printf '\n=== isolated Docker bootstrap POC ===\n' -if [[ "$host_status" != "0" ]]; then - echo 'skipped: host /dev/nvmap does not grant its owning group read-write access' -elif [[ "${#tegra_group_gids[@]}" == "0" ]]; then - echo 'skipped: no usable Tegra device groups were detected' -else - docker_args=( - run --rm - --runtime nvidia - --env NVIDIA_VISIBLE_DEVICES=all - --env 'NVIDIA_DRIVER_CAPABILITIES=compute,utility' - --env "NEMOCLAW_JETSON_DEVICE_GROUP_GIDS=$tegra_gids_csv" - --volume "$script_path:/tmp/nemoclaw-jetson-nvmap-poc:ro" - --entrypoint /usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh - ) - for gid in "${tegra_group_gids[@]}"; do - docker_args+=(--group-add "$gid") - done - docker_args+=("$sandbox_image_id" /bin/bash /tmp/nemoclaw-jetson-nvmap-poc --inside-root) - docker "${docker_args[@]}" - isolated_status=$? -fi -printf 'stage_status=%s\n' "$isolated_status" - -printf '\n=== verdict ===\n' -verdict_status=0 -if [[ "$host_status" != "0" ]]; then - echo 'FAIL boundary=host-nvmap-mode' - verdict_status=1 -elif [[ "$isolated_status" != "0" ]]; then - echo 'FAIL boundary=docker-runtime-or-bootstrap' - verdict_status=1 -elif [[ "$outer_sandbox_status" != "0" ]]; then - echo 'FAIL boundary=recreated-container-bootstrap' - verdict_status=1 -elif [[ "$openshell_status" != "0" ]]; then - echo 'FAIL boundary=openshell-sandbox-execution' - verdict_status=1 -else - echo 'PASS boundary=end-to-end' -fi - -echo "evidence host=$host_status isolated=$isolated_status outer_root=$outer_root_status outer_sandbox=$outer_sandbox_status openshell=$openshell_status" -exit "$verdict_status" diff --git a/src/lib/onboard/docker-gpu-jetson-groups.test.ts b/src/lib/onboard/docker-gpu-jetson-groups.test.ts index 8d4ae1f232..72db1729e1 100644 --- a/src/lib/onboard/docker-gpu-jetson-groups.test.ts +++ b/src/lib/onboard/docker-gpu-jetson-groups.test.ts @@ -7,6 +7,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { detectTegraDeviceGroupGids, + detectTegraGpuDevicePaths, ensureJetsonNvmapGroupAccess, } from "./docker-gpu-jetson-groups"; @@ -17,6 +18,28 @@ const READ_ONLY_NVMAP = { }; const READ_WRITE_NVMAP = { ...READ_ONLY_NVMAP, mode: 0o660 }; +describe("detectTegraGpuDevicePaths", () => { + it("returns only existing non-symlink character devices for Landlock (#7610)", () => { + const pathAccess = new Map([ + ["/dev/nvmap", { isCharacterDevice: true, isSymbolicLink: false }], + ["/dev/nvhost-gpu", { isCharacterDevice: false, isSymbolicLink: false }], + ["/dev/dri/renderD128", { isCharacterDevice: true, isSymbolicLink: true }], + ]); + + expect( + detectTegraGpuDevicePaths({ + listDevicePaths: () => [ + "/dev/nvmap", + "/dev/nvhost-gpu", + "/dev/dri/renderD128", + "/dev/missing", + ], + statDevicePath: (devicePath) => pathAccess.get(devicePath) ?? null, + }), + ).toEqual(["/dev/nvmap"]); + }); +}); + describe("detectTegraDeviceGroupGids", () => { afterEach(() => { vi.restoreAllMocks(); diff --git a/src/lib/onboard/docker-gpu-jetson-groups.ts b/src/lib/onboard/docker-gpu-jetson-groups.ts index 4b8ffbbf32..95a9cb3afd 100644 --- a/src/lib/onboard/docker-gpu-jetson-groups.ts +++ b/src/lib/onboard/docker-gpu-jetson-groups.ts @@ -27,6 +27,11 @@ type DeviceGroupAccess = { mode: number; }; +type DevicePathAccess = { + isCharacterDevice: boolean; + isSymbolicLink: boolean; +}; + type NvmapDeviceAccess = { isCharacterDevice: boolean; isSymbolicLink: boolean; @@ -128,19 +133,52 @@ function listTegraGpuDevicePaths(): string[] { return [...TEGRA_GPU_DEVICE_NODES, ...discoverTegraRenderDevicePaths()]; } +/** + * Return only existing Jetson GPU character devices that can be granted at + * the OpenShell Landlock boundary. The fixed candidates keep the policy from + * widening to unrelated host devices, and lstat prevents symlink traversal. + */ +export function detectTegraGpuDevicePaths( + deps: { + statDevicePath?: (path: string) => DevicePathAccess | null; + listDevicePaths?: () => string[]; + } = {}, +): string[] { + const devicePaths = deps.listDevicePaths?.() ?? listTegraGpuDevicePaths(); + const statPath = + deps.statDevicePath ?? + ((devicePath: string): DevicePathAccess | null => { + try { + const stat = fs.lstatSync(devicePath); + return { + isCharacterDevice: stat.isCharacterDevice(), + isSymbolicLink: stat.isSymbolicLink(), + }; + } catch { + return null; + } + }); + return devicePaths.filter((devicePath) => { + const access = statPath(devicePath); + return access?.isCharacterDevice === true && access.isSymbolicLink === false; + }); +} + /** * Source-of-truth boundary for Jetson/Tegra supplementary device groups: * - * - Invalid state: `/dev/nvmap` lacks owning-group read/write access, or the non-root sandbox user - * loses the matching host device GID when the OpenShell supervisor calls `initgroups()`. - * - Source boundary: NemoClaw verifies and persists the host nvmap mode, carries each bounded - * numeric device GID into the Jetson recreation via `--group-add`, and records matching sandbox - * account membership before the supervisor starts. + * - Invalid state: `/dev/nvmap` lacks owning-group read/write access, Landlock omits an injected + * Tegra character device, or the non-root sandbox user loses the matching host device GID. + * - Source boundary: NemoClaw verifies and persists the host nvmap mode, grants exact detected + * character-device paths in the route policy, carries each bounded numeric device GID into the + * Jetson recreation via `--group-add`, and records matching sandbox account membership before + * the supervisor starts. * - Source-fix constraint: `--group-add` alone does not survive the supervisor's account-group * initialization, and image-local group names can differ from the host's numeric device GIDs. - * - Regression coverage: docker-gpu-jetson-groups.test.ts covers discovery and hostile numeric - * values; setup-jetson.test.ts covers the host mode; docker-gpu-patch-jetson.test.ts covers - * clone-envelope and sandbox-account propagation plus generic-host exclusion. + * - Regression coverage: docker-gpu-jetson-groups.test.ts covers path/GID discovery and hostile + * numeric values; initial-policy.test.ts covers Landlock grants; setup-jetson.test.ts covers the + * host mode; docker-gpu-patch-jetson.test.ts covers clone-envelope and sandbox-account + * propagation plus generic-host exclusion. * - Removal condition: remove this probe when the minimum supported native OpenShell Jetson path * propagates the host device groups without compatibility container recreation. */ diff --git a/src/lib/onboard/docker-gpu-patch-types.ts b/src/lib/onboard/docker-gpu-patch-types.ts index fa9c17a0f6..de1c5bdf3a 100644 --- a/src/lib/onboard/docker-gpu-patch-types.ts +++ b/src/lib/onboard/docker-gpu-patch-types.ts @@ -43,9 +43,9 @@ export type DockerGpuPatchDeps = { * Resolve the host group ID(s) that own the Jetson/Tegra GPU device nodes * (`/dev/nvmap`, `/dev/nvhost-*`, and `/dev/dri/renderD*`). Used by the * Jetson recreate for Docker `--group-add` and matching sandbox-account - * membership before the OpenShell supervisor calls `initgroups()`, so CUDA - * can open them (#4231, #7610). Injectable so the Jetson permission path is - * testable without Tegra hardware. + * membership. The route-specific policy separately grants the exact device + * paths through Landlock so CUDA can open them (#4231, #7610). Injectable so + * the Jetson permission path is testable without Tegra hardware. */ detectTegraDeviceGroupGids?: () => string[]; /** Injectable directory lister for unit testing CDI spec discovery. */ diff --git a/src/lib/onboard/initial-policy.test.ts b/src/lib/onboard/initial-policy.test.ts index 1c16171107..95d98c0646 100644 --- a/src/lib/onboard/initial-policy.test.ts +++ b/src/lib/onboard/initial-policy.test.ts @@ -380,6 +380,42 @@ network_policies: {} expect(jetsonPolicy.filesystem_policy.read_write).not.toContain("/opt/nvidia/l4t-gpu-libs"); }); + it("grants exact detected Jetson character devices read-write for Landlock (#7610)", () => { + const gpuPolicy = buildDirectGpuPolicyYaml( + ` +version: 1 +filesystem_policy: + read_only: + - /usr + - /dev/nvmap + read_write: + - /tmp + - /dev/nvhost-gpu +network_policies: {} +`, + { + jetsonGpu: true, + jetsonGpuDevicePaths: ["/dev/nvmap", "/dev/nvhost-gpu", "/dev/nvmap"], + }, + ); + const gpuDoc = YAML.parse(gpuPolicy); + + expect(gpuDoc.filesystem_policy.read_only).not.toContain("/dev/nvmap"); + expectSingleOccurrence(gpuDoc.filesystem_policy.read_write, "/dev/nvmap"); + expectSingleOccurrence(gpuDoc.filesystem_policy.read_write, "/dev/nvhost-gpu"); + }); + + it("does not grant Jetson character devices to a generic GPU policy (#7610)", () => { + const gpuPolicy = YAML.parse( + buildDirectGpuPolicyYaml(BASE_POLICY_FIXTURE, { + jetsonGpuDevicePaths: ["/dev/nvmap", "/dev/nvhost-gpu"], + }), + ); + + expect(gpuPolicy.filesystem_policy.read_write).not.toContain("/dev/nvmap"); + expect(gpuPolicy.filesystem_policy.read_write).not.toContain("/dev/nvhost-gpu"); + }); + it("threads the OpenClaw Jetson library path through public policy preparation (#7610)", () => { const basePolicyPath = tmpPolicy(BASE_POLICY_FIXTURE); const prepared = prepareInitialSandboxCreatePolicy(basePolicyPath, [], { @@ -393,6 +429,22 @@ network_policies: {} expect(prepared.cleanup?.()).toBe(true); }); + it("threads detected Jetson character devices through public policy preparation (#7610)", () => { + const basePolicyPath = tmpPolicy(BASE_POLICY_FIXTURE); + const prepared = prepareInitialSandboxCreatePolicy(basePolicyPath, [], { + directGpu: true, + jetsonGpu: true, + jetsonGpuDevicePaths: ["/dev/nvmap", "/dev/nvhost-ctrl", "/dev/nvhost-gpu"], + stationGb300SysfsReadOnlyPaths: [], + }); + const preparedDoc = YAML.parse(fs.readFileSync(prepared.policyPath, "utf-8")); + + expect(preparedDoc.filesystem_policy.read_write).toEqual( + expect.arrayContaining(["/dev/nvmap", "/dev/nvhost-ctrl", "/dev/nvhost-gpu"]), + ); + expect(prepared.cleanup?.()).toBe(true); + }); + it("preserves best-effort Landlock for missing Station sysfs paths (#7103)", () => { const sysfsRoot = tmpSysfsRoot(); addPciDevice(sysfsRoot, "0009:06:00.0", "0x10de\n", "0x030200\n"); diff --git a/src/lib/onboard/initial-policy.ts b/src/lib/onboard/initial-policy.ts index e5e8b0ea20..9475aade34 100644 --- a/src/lib/onboard/initial-policy.ts +++ b/src/lib/onboard/initial-policy.ts @@ -23,6 +23,7 @@ import { isStationGb300ProductName, type StationProfile, } from "../readiness/station-qualification"; +import { detectTegraGpuDevicePaths } from "./docker-gpu-jetson-groups"; import { allMessagingChannelPolicyPresets, requiredMessagingChannelPolicyPresets, @@ -79,6 +80,7 @@ type DirectGpuPolicyOptions = { procReadWrite?: boolean; sysfsReadOnlyPaths?: readonly string[]; jetsonGpu?: boolean; + jetsonGpuDevicePaths?: readonly string[]; }; export { isStationGb300ProductName }; @@ -234,6 +236,22 @@ export function buildDirectGpuPolicyYaml( // grants. CUDA cannot load the driver unless Landlock permits this path. fsPolicy.read_only.push(JETSON_GPU_LIBRARY_PATH); } + if (options.jetsonGpu) { + // OpenShell v0.0.85 enriches Landlock for /dev/nvidia* and /dev/dxg but + // does not recognize Jetson /dev/nvmap or /dev/nvhost-* devices. The + // compatibility route therefore grants only measured host character + // devices; group membership alone cannot bypass Landlock. Remove this + // grant when the minimum supported OpenShell release enriches Jetson + // devices before applying the sandbox filesystem policy (#7610). + const jetsonGpuDevicePaths = [...new Set(options.jetsonGpuDevicePaths ?? [])]; + const jetsonGpuDevicePathSet = new Set(jetsonGpuDevicePaths); + fsPolicy.read_only = fsPolicy.read_only.filter( + (entry: string) => !jetsonGpuDevicePathSet.has(entry), + ); + for (const devicePath of jetsonGpuDevicePaths) { + if (!fsPolicy.read_write.includes(devicePath)) fsPolicy.read_write.push(devicePath); + } + } if (options.procReadWrite && !fsPolicy.read_write.includes(PROC_PATH)) { // This exists only for the legacy post-create Docker GPU compatibility // path, which recreates the container after `openshell sandbox create` and @@ -393,6 +411,7 @@ export function prepareInitialSandboxCreatePolicy( directGpu?: boolean; dockerGpuPatch?: boolean; jetsonGpu?: boolean; + jetsonGpuDevicePaths?: readonly string[]; hostGpuAvailable?: boolean; stationGb300SysfsReadOnlyPaths?: readonly string[]; additionalPresets?: string[]; @@ -405,6 +424,9 @@ export function prepareInitialSandboxCreatePolicy( ? prepareDirectGpuSandboxPolicy(basePolicyPath, { procReadWrite: options.dockerGpuPatch === true, jetsonGpu: options.jetsonGpu === true, + jetsonGpuDevicePaths: + options.jetsonGpuDevicePaths ?? + (options.jetsonGpu === true ? detectTegraGpuDevicePaths() : []), sysfsReadOnlyPaths: options.stationGb300SysfsReadOnlyPaths ?? discoverHostStationGb300SysfsReadOnlyPaths({ diff --git a/src/lib/onboard/sandbox-gpu-preflight-routing.test.ts b/src/lib/onboard/sandbox-gpu-preflight-routing.test.ts index 8b9b002f05..ccc749f8f3 100644 --- a/src/lib/onboard/sandbox-gpu-preflight-routing.test.ts +++ b/src/lib/onboard/sandbox-gpu-preflight-routing.test.ts @@ -33,8 +33,9 @@ describe("sandbox GPU preflight routing", () => { const remediation = jetsonGpuProofRemediationLines().join("\n"); expect(remediation).toContain("owning-group read-write access"); - expect(remediation).toContain("propagate the device GIDs through Docker"); - expect(remediation).toContain("before OpenShell calls initgroups()"); + expect(remediation).toContain("OpenShell Landlock policy"); + expect(remediation).toContain("their GIDs through Docker"); + expect(remediation).toContain("preserve sandbox-account membership"); expect(remediation).not.toContain("add the host video/render groups via --group-add"); expect(remediation).not.toContain("must be readable by the sandbox"); }); diff --git a/src/lib/onboard/sandbox-gpu-preflight.ts b/src/lib/onboard/sandbox-gpu-preflight.ts index ed176c3e81..da44bc4efe 100644 --- a/src/lib/onboard/sandbox-gpu-preflight.ts +++ b/src/lib/onboard/sandbox-gpu-preflight.ts @@ -56,18 +56,17 @@ export function resolveSandboxGpuFlagFromOptions(opts: SandboxGpuFlagOptions): S return null; } -// Jetson/Tegra CUDA failures are usually device/group permission issues rather -// than CDI/runtime misconfiguration: the sandbox sees the GPU but the agent -// user lacks access to the Tegra device nodes. Surface the complete host-mode, -// Docker-group, and sandbox-account boundary rather than an incomplete manual +// Jetson/Tegra CUDA failures are usually device-access issues rather than +// CDI/runtime selection failures. Surface the complete host mode, Landlock, +// Docker group, and sandbox-account boundary rather than an incomplete manual // --group-add workaround or a bare "enabled" status (#4231, #7610). export function jetsonGpuProofRemediationLines(): string[] { return [ "Jetson/Tegra CUDA proof did not pass. CUDA needs access to the Tegra device", "nodes. NemoClaw must verify host /dev/nvmap owning-group read-write access,", - "propagate the device GIDs through Docker, and add matching membership to the", - "sandbox account before OpenShell calls initgroups(). Review the onboarding output", - "and saved diagnostics to identify the failing boundary, then retry onboarding;", + "grant the detected character devices in the OpenShell Landlock policy, propagate", + "their GIDs through Docker, and preserve sandbox-account membership. Review the", + "onboarding output and saved diagnostics, then retry onboarding;", "or use NEMOCLAW_SANDBOX_GPU=0 for CPU.", ]; } diff --git a/test/jetson-nvmap-poc.test.ts b/test/jetson-nvmap-poc.test.ts deleted file mode 100644 index 6cd29acef1..0000000000 --- a/test/jetson-nvmap-poc.test.ts +++ /dev/null @@ -1,85 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { spawnSync } from "node:child_process"; -import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; -import path from "node:path"; - -import { describe, expect, it } from "vitest"; - -const SCRIPT_PATH = path.join(import.meta.dirname, "..", "scripts", "jetson-nvmap-poc.sh"); - -describe("Jetson nvmap boundary proof", () => { - it("reaches the final verdict and exits nonzero when a required boundary fails (#7610)", () => { - const tempDir = mkdtempSync(path.join(tmpdir(), "nemoclaw-jetson-nvmap-poc-")); - - try { - const stubDir = path.join(tempDir, "bin"); - const commandLog = path.join(tempDir, "commands.log"); - mkdirSync(stubDir); - writeFileSync(commandLog, ""); - - const dockerStub = path.join(stubDir, "docker"); - writeFileSync( - dockerStub, - [ - "#!/usr/bin/env bash", - "set -euo pipefail", - `printf 'docker %s\\n' "$*" >> ${JSON.stringify(commandLog)}`, - 'case "${1:-}" in', - " ps) echo container-123 ;;&", - " inspect) echo sha256:image-123 ;;&", - " exec) exit 0 ;;&", - "esac", - "", - ].join("\n"), - ); - chmodSync(dockerStub, 0o755); - - const openshellStub = path.join(stubDir, "openshell"); - writeFileSync( - openshellStub, - [ - "#!/usr/bin/env bash", - "set -euo pipefail", - `printf 'openshell %s\\n' "$*" >> ${JSON.stringify(commandLog)}`, - "exit 0", - "", - ].join("\n"), - ); - chmodSync(openshellStub, 0o755); - - const statStub = path.join(stubDir, "stat"); - writeFileSync( - statStub, - [ - "#!/usr/bin/env bash", - "set -euo pipefail", - 'if [[ "$*" == *"%a"* && "$*" != *"type="* ]]; then', - " echo 660", - "else", - " echo 'type=character special file mode=660 uid=0 gid=44 group=video path=/dev/nvmap'", - "fi", - "", - ].join("\n"), - ); - chmodSync(statStub, 0o755); - - const result = spawnSync("bash", [SCRIPT_PATH, "test-sandbox"], { - encoding: "utf8", - env: { ...process.env, PATH: `${stubDir}${path.delimiter}${process.env.PATH ?? ""}` }, - }); - - expect(result.status).toBe(1); - expect(result.stderr).not.toContain("readonly variable"); - expect(result.stdout).toContain("sandbox=test-sandbox"); - expect(result.stdout).toContain("image=sha256:image-123"); - expect(result.stdout).toContain("FAIL boundary=docker-runtime-or-bootstrap"); - expect(result.stdout).toContain("evidence host=0 isolated=1"); - expect(readFileSync(commandLog, "utf8")).toContain("docker exec --user sandbox"); - } finally { - rmSync(tempDir, { recursive: true, force: true }); - } - }); -}); From 98e392e96d0f39df422b173e772f0e075197bb1b Mon Sep 17 00:00:00 2001 From: San Dang Date: Thu, 6 Aug 2026 01:06:27 +0700 Subject: [PATCH 18/49] test(jetson): add CUDA boundary probe Signed-off-by: San Dang --- scripts/jetson-cuda-boundary-probe.sh | 161 ++++++++++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100755 scripts/jetson-cuda-boundary-probe.sh diff --git a/scripts/jetson-cuda-boundary-probe.sh b/scripts/jetson-cuda-boundary-probe.sh new file mode 100755 index 0000000000..847ed82aa2 --- /dev/null +++ b/scripts/jetson-cuda-boundary-probe.sh @@ -0,0 +1,161 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Non-configuring boundary probe for NemoClaw/OpenShell Jetson CUDA failures. +# This script does not recreate containers, change permissions, or edit policy. + +set -uo pipefail + +sandbox_name="${1:-tm}" +cuda_probe='import ctypes; lib=ctypes.CDLL("libcuda.so.1"); rc=lib.cuInit(0); print(f"cuInit(0)={rc}"); raise SystemExit(rc != 0)' + +section() { + printf '\n=== %s ===\n' "$1" +} + +run_stage() { + local label="$1" + shift + + printf '%s\n' "--- ${label} ---" + "$@" + local status=$? + printf 'stage_status=%s\n' "$status" + return 0 +} + +container_context() { + local docker_user="$1" + + docker exec --user "$docker_user" "$container_id" sh -c ' + id + grep -E "^(NoNewPrivs|Seccomp|Seccomp_filters):" /proc/self/status 2>/dev/null || true + for device_path in \ + /dev/nvmap \ + /dev/nvhost-* \ + /dev/nvgpu/igpu0/* \ + /dev/dri/renderD* \ + /dev/nvsciipc*; do + [ -e "$device_path" ] || continue + stat -Lc "device type=%F mode=%a uid=%u gid=%g path=%n" "$device_path" 2>/dev/null || true + if [ -r "$device_path" ]; then device_read=yes; else device_read=no; fi + if [ -w "$device_path" ]; then device_write=yes; else device_write=no; fi + printf "device_permission_check read=%s write=%s path=%s\n" "$device_read" "$device_write" "$device_path" + done + for candidate_path in \ + /proc/device-tree \ + /sys/firmware/devicetree/base \ + /sys/devices/platform \ + /sys/class/devfreq \ + /sys/module; do + if [ ! -e "$candidate_path" ]; then + printf "path_access exists=no path=%s\n" "$candidate_path" + elif ls -A "$candidate_path" >/dev/null 2>&1; then + printf "path_access exists=yes list=yes path=%s\n" "$candidate_path" + else + printf "path_access exists=yes list=no path=%s\n" "$candidate_path" + fi + done + ' +} + +openshell_context() { + # Variables in the next single-quoted command belong to the in-sandbox shell. + # shellcheck disable=SC2016 + openshell sandbox exec -n "$sandbox_name" -- sh -c ' + id + grep -E "^(NoNewPrivs|Seccomp|Seccomp_filters):" /proc/self/status 2>/dev/null || true + for device_path in \ + /dev/nvmap \ + /dev/nvhost-* \ + /dev/nvgpu/igpu0/* \ + /dev/dri/renderD* \ + /dev/nvsciipc*; do + [ -e "$device_path" ] || continue + stat -Lc "device type=%F mode=%a uid=%u gid=%g path=%n" "$device_path" 2>/dev/null || true + if [ -r "$device_path" ]; then device_read=yes; else device_read=no; fi + if [ -w "$device_path" ]; then device_write=yes; else device_write=no; fi + printf "device_permission_check read=%s write=%s path=%s\n" "$device_read" "$device_write" "$device_path" + done + for candidate_path in \ + /proc/device-tree \ + /sys/firmware/devicetree/base \ + /sys/devices/platform \ + /sys/class/devfreq \ + /sys/module; do + if [ ! -e "$candidate_path" ]; then + printf "path_access exists=no path=%s\n" "$candidate_path" + elif ls -A "$candidate_path" >/dev/null 2>&1; then + printf "path_access exists=yes list=yes path=%s\n" "$candidate_path" + else + printf "path_access exists=yes list=no path=%s\n" "$candidate_path" + fi + done + ' +} + +selected_nvidia_environment() { + docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' "$container_id" \ + | grep -E '^NVIDIA_(VISIBLE_DEVICES|DRIVER_CAPABILITIES)=' +} + +base_policy_gpu_paths() { + openshell policy get --base "$sandbox_name" 2>&1 \ + | grep -E 'filesystem_policy:|read_only:|read_write:|/dev/(nv|dri)|/opt/nvidia|/proc|/sys' +} + +section "host and tool versions" +printf 'sandbox=%s\n' "$sandbox_name" +run_stage "git head" git rev-parse HEAD +run_stage "nemoclaw version" nemoclaw --version +run_stage "openshell version" openshell --version +run_stage "kernel" uname -a +run_stage "host nvmap" stat -Lc 'type=%F mode=%a uid=%u gid=%g group=%G path=%n' /dev/nvmap + +container_id="$({ + docker ps -q \ + --filter 'label=openshell.ai/managed-by=openshell' \ + --filter "label=openshell.ai/sandbox-name=${sandbox_name}" +} | head -n 1)" + +if [ -z "$container_id" ]; then + printf 'ERROR: no running OpenShell-managed container found for sandbox %s\n' "$sandbox_name" >&2 + exit 2 +fi + +section "managed container" +printf 'container=%s\n' "$container_id" +run_stage "container configuration" docker inspect --format \ + 'runtime={{.HostConfig.Runtime}} user={{json .Config.User}} status={{.State.Status}} health={{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}} group_add={{json .HostConfig.GroupAdd}}' \ + "$container_id" +run_stage "selected NVIDIA environment" selected_nvidia_environment + +section "effective filesystem policy entries" +run_stage "OpenShell base policy GPU paths" base_policy_gpu_paths + +section "Docker exec as root" +run_stage "identity, devices, sysfs, and seccomp" container_context 0 +run_stage "cuInit baseline" docker exec --user 0 "$container_id" python3 -c "$cuda_probe" +run_stage "cuInit without eager JIT preload" docker exec --user 0 "$container_id" env \ + CUDA_FORCE_PRELOAD_LIBRARIES=0 python3 -c "$cuda_probe" + +section "Docker exec as sandbox" +run_stage "identity, devices, sysfs, and seccomp" container_context sandbox +run_stage "cuInit baseline" docker exec --user sandbox "$container_id" python3 -c "$cuda_probe" +run_stage "cuInit without eager JIT preload" docker exec --user sandbox "$container_id" env \ + CUDA_FORCE_PRELOAD_LIBRARIES=0 python3 -c "$cuda_probe" + +section "OpenShell sandbox execution" +run_stage "identity, devices, sysfs, and seccomp" openshell_context +run_stage "cuInit baseline" openshell sandbox exec -n "$sandbox_name" -- python3 -c "$cuda_probe" +run_stage "cuInit without eager JIT preload" openshell sandbox exec -n "$sandbox_name" -- env \ + CUDA_FORCE_PRELOAD_LIBRARIES=0 python3 -c "$cuda_probe" + +section "interpretation" +printf '%s\n' \ + 'Docker sandbox-user cuInit(0) succeeds while OpenShell cuInit(0) fails: suggests an OpenShell confinement difference.' \ + 'Docker root cuInit(0) succeeds while Docker sandbox-user cuInit(0) fails: suggests an account or device-permission difference.' \ + 'Docker root cuInit(0) fails: suggests a recreated-container or NVIDIA runtime difference.' \ + 'Baseline cuInit(0) fails while the no-preload probe succeeds: suggests a CUDA eager JIT library preload difference.' From 06d74079b8731c960db85c979303782b20e6938e Mon Sep 17 00:00:00 2001 From: San Dang Date: Thu, 6 Aug 2026 17:41:46 +0700 Subject: [PATCH 19/49] chore(onboard): revert prior Jetson GPU changes Signed-off-by: San Dang --- Dockerfile | 2 - docs/reference/troubleshooting.mdx | 53 +--- scripts/jetson-cuda-boundary-probe.sh | 161 ------------ scripts/jetson-device-group-bootstrap.sh | 53 ---- scripts/managed-bootstrap-trampoline.sh | 28 -- scripts/setup-jetson.sh | 77 +----- src/lib/onboard.ts | 2 +- .../onboard/docker-gpu-jetson-groups.test.ts | 100 +------- src/lib/onboard/docker-gpu-jetson-groups.ts | 141 +--------- src/lib/onboard/docker-gpu-patch-clone.ts | 55 +--- .../onboard/docker-gpu-patch-jetson.test.ts | 59 +---- src/lib/onboard/docker-gpu-patch-recreate.ts | 13 +- src/lib/onboard/docker-gpu-patch-types.ts | 19 +- src/lib/onboard/docker-gpu-sandbox-create.ts | 3 - .../docker-startup-command-patch.test.ts | 45 ---- .../onboard/docker-startup-command-patch.ts | 2 - ...ker-startup-command-sandbox-create.test.ts | 25 -- .../docker-startup-command-sandbox-create.ts | 2 - .../portable-host-preparation.test.ts | 25 ++ .../experimental/portable-host-preparation.ts | 8 + src/lib/onboard/initial-policy.test.ts | 76 ------ src/lib/onboard/initial-policy.ts | 35 --- .../managed-bootstrap/docker-runtime.test.ts | 106 -------- .../managed-bootstrap/docker-runtime.ts | 7 +- .../managed-bootstrap/docker-test-fixture.ts | 17 +- .../onboard/managed-bootstrap/docker.test.ts | 93 +------ src/lib/onboard/managed-bootstrap/docker.ts | 46 +--- .../onboard/sandbox-create-intent-types.ts | 1 - src/lib/onboard/sandbox-create-intent.ts | 5 - .../sandbox-create-plan-materialization.ts | 1 - src/lib/onboard/sandbox-create-plan.test.ts | 11 - .../onboard/sandbox-gpu-create-flow.test.ts | 53 ---- src/lib/onboard/sandbox-gpu-create-flow.ts | 10 - .../onboard/sandbox-gpu-create-run-attempt.ts | 1 - src/lib/onboard/sandbox-gpu-create.ts | 1 - .../sandbox-gpu-preflight-routing.test.ts | 12 - src/lib/onboard/sandbox-gpu-preflight.ts | 18 +- src/lib/sandbox/build-context.ts | 4 - test/e2e/live/jetson-nvmap-gpu.test.ts | 17 +- test/managed-bootstrap-trampoline.test.ts | 22 +- test/openclaw-final-image-layout.test.ts | 1 - test/sandbox-build-context.test.ts | 4 - ...ox-provisioning-helper-permissions.test.ts | 3 - test/setup-jetson.test.ts | 240 ++---------------- 44 files changed, 134 insertions(+), 1523 deletions(-) delete mode 100755 scripts/jetson-cuda-boundary-probe.sh delete mode 100755 scripts/jetson-device-group-bootstrap.sh diff --git a/Dockerfile b/Dockerfile index 417388a91b..fede87d692 100644 --- a/Dockerfile +++ b/Dockerfile @@ -168,7 +168,6 @@ COPY scripts/lib/sandbox-rlimits.sh /usr/local/lib/nemoclaw/sandbox-rlimits.sh COPY scripts/lib/openclaw_device_approval_policy.py /usr/local/lib/nemoclaw/openclaw_device_approval_policy.py COPY scripts/lib/clean_runtime_shell_env_shim.py /usr/local/lib/nemoclaw/clean_runtime_shell_env_shim.py COPY scripts/lib/normalize_mutable_config_perms.py /usr/local/lib/nemoclaw/normalize_mutable_config_perms.py -COPY scripts/jetson-device-group-bootstrap.sh /usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh COPY scripts/state-dir-guard.py /usr/local/lib/nemoclaw/state-dir-guard.py COPY agents/openclaw/state-lock-plan.json /usr/local/share/nemoclaw/state-lock-plan.json COPY scripts/openclaw-config-guard.py /usr/local/lib/nemoclaw/openclaw-config-guard.py @@ -1102,7 +1101,6 @@ RUN chmod 755 /usr/local/bin/nemoclaw-start /usr/local/bin/nemoclaw-codex-acp \ /usr/local/bin/nemoclaw-managed-bootstrap \ /usr/local/bin/nemoclaw-managed-startup-hold \ /usr/local/lib/nemoclaw/sandbox-init.sh \ - /usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh \ /scripts/generate-openclaw-config.mts \ /scripts/validate-openclaw-tool-search.mts \ /src/lib/messaging/applier/build/messaging-build-applier.mts \ diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 0e63aed8a9..09ab640bd8 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -211,13 +211,7 @@ Some R39 images already ship with `br_netfilter` configured and are left untouch On affected R39 hosts, the installer prints `loading br_netfilter (required by k3s inside the OpenShell gateway)`. Without this fix, sandbox pods fail DNS resolution against the in-cluster service and the onboard `Setting up OpenClaw inside sandbox` step times out. -If the L4T version is not recognized, the installer skips the version-specific iptables, Docker, and `br_netfilter` setup and continues in an untested configuration. - - - -The OpenClaw installer still configures a real `/dev/nvmap` character device independently of the parsed L4T release. - - +If the L4T version is not recognized, the setup step is skipped and the installer continues normally. ### DNS resolution from inside docker fails (corporate firewall) @@ -813,7 +807,7 @@ NVIDIA NIM and GPU-backed sandbox setup require a real NVIDIA GPU. If NemoClaw rejects the detected GPU name during preflight, select a CPU or remote inference provider, or move the setup to a host with a supported NVIDIA GPU and current drivers. Jetson/Tegra hosts support sandbox GPU passthrough through the compatibility route. -Onboarding detects those hosts separately and prepares access to selected `/dev/nvmap`, `/dev/nvhost-*`, and `/dev/nvgpu/igpu0/*` nodes plus real `/dev/dri/renderD*` character devices. +Onboarding detects those hosts separately and propagates eligible host group IDs for selected `/dev/nvmap`, `/dev/nvhost-*`, and `/dev/nvgpu/igpu0/*` nodes plus real `/dev/dri/renderD*` character devices. If that path fails, follow the Jetson/Tegra compatibility guidance below instead of treating a missing `nvidia-smi` result as a placeholder adapter. ### Colima socket not detected (macOS) @@ -2734,47 +2728,8 @@ To skip GPU passthrough entirely, rerun with `--no-gpu` or set `NEMOCLAW_SANDBOX #### Jetson and Tegra compatibility default Automatic GPU onboarding uses the compatibility path directly; it does not make a native attempt first. -The path recreates the OpenShell-managed Docker container with NVIDIA GPU flags. -Use `NEMOCLAW_DOCKER_GPU_PATCH=0` only for troubleshooting because it bypasses the compatibility patch and CUDA may not initialize. - - - -The compatibility path keeps eligible Jetson GPU device group memberships when OpenShell starts the nonroot sandbox user. -Group membership alone does not grant device access when the sandbox filesystem policy omits a required path. -The route-specific policy grants read-write access only to selected paths that exist as character devices and are not symbolic links. -It also permits read-only access to the NVIDIA runtime library directory at `/opt/nvidia/l4t-gpu-libs` when that path is present. - -Before each OpenClaw Jetson GPU sandbox creation, including `$$nemoclaw onboard --resume`, NemoClaw verifies that `/dev/nvmap` is a real character device whose owning group has read-write access. - - -When repair is required, NemoClaw grants write access to every member of the existing `/dev/nvmap` owning group. - - -If the initial verification fails, NemoClaw runs the same host permission setup as the installer. -After setup exits successfully, NemoClaw verifies the device again. -If setup fails or the device still lacks verified group read-write access, onboarding stops before sandbox creation. -This device permission setup runs even when the L4T release line is unrecognized, but it does not make that release tested or supported. - -Verify the persistent rule and the live device permissions: - -```bash -grep -Fx 'KERNEL=="nvmap", MODE="0660"' /etc/udev/rules.d/99-zz-nemoclaw-nvmap.rules -stat -c '%A %U:%G %n' /dev/nvmap -``` - -The rule must match exactly, and the group permission characters must be `rw`, such as the group field in `crw-rw----`. -The rule reapplies mode `0660` when udev recreates `/dev/nvmap`, including after reboot. -NemoClaw uninstall does not remove the rule; delete it and reload udev to stop future reapplication, while the live mode remains until device recreation. - -After onboarding recreates the sandbox, verify CUDA as the nonroot sandbox user: - -```bash -$$nemoclaw exec -- python3 -c 'import ctypes; lib=ctypes.CDLL("libcuda.so.1"); print(f"cuInit(0)={lib.cuInit(0)}")' -``` - -The command must print `cuInit(0)=0`. - - +The path recreates the OpenShell-managed Docker container with NVIDIA GPU flags and propagates eligible host group IDs for the supported Jetson GPU device nodes. +Use `NEMOCLAW_DOCKER_GPU_PATCH=0` only for troubleshooting because it bypasses that group propagation and CUDA may not initialize. #### Common compatibility-path recovery diff --git a/scripts/jetson-cuda-boundary-probe.sh b/scripts/jetson-cuda-boundary-probe.sh deleted file mode 100755 index 847ed82aa2..0000000000 --- a/scripts/jetson-cuda-boundary-probe.sh +++ /dev/null @@ -1,161 +0,0 @@ -#!/usr/bin/env bash - -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# Non-configuring boundary probe for NemoClaw/OpenShell Jetson CUDA failures. -# This script does not recreate containers, change permissions, or edit policy. - -set -uo pipefail - -sandbox_name="${1:-tm}" -cuda_probe='import ctypes; lib=ctypes.CDLL("libcuda.so.1"); rc=lib.cuInit(0); print(f"cuInit(0)={rc}"); raise SystemExit(rc != 0)' - -section() { - printf '\n=== %s ===\n' "$1" -} - -run_stage() { - local label="$1" - shift - - printf '%s\n' "--- ${label} ---" - "$@" - local status=$? - printf 'stage_status=%s\n' "$status" - return 0 -} - -container_context() { - local docker_user="$1" - - docker exec --user "$docker_user" "$container_id" sh -c ' - id - grep -E "^(NoNewPrivs|Seccomp|Seccomp_filters):" /proc/self/status 2>/dev/null || true - for device_path in \ - /dev/nvmap \ - /dev/nvhost-* \ - /dev/nvgpu/igpu0/* \ - /dev/dri/renderD* \ - /dev/nvsciipc*; do - [ -e "$device_path" ] || continue - stat -Lc "device type=%F mode=%a uid=%u gid=%g path=%n" "$device_path" 2>/dev/null || true - if [ -r "$device_path" ]; then device_read=yes; else device_read=no; fi - if [ -w "$device_path" ]; then device_write=yes; else device_write=no; fi - printf "device_permission_check read=%s write=%s path=%s\n" "$device_read" "$device_write" "$device_path" - done - for candidate_path in \ - /proc/device-tree \ - /sys/firmware/devicetree/base \ - /sys/devices/platform \ - /sys/class/devfreq \ - /sys/module; do - if [ ! -e "$candidate_path" ]; then - printf "path_access exists=no path=%s\n" "$candidate_path" - elif ls -A "$candidate_path" >/dev/null 2>&1; then - printf "path_access exists=yes list=yes path=%s\n" "$candidate_path" - else - printf "path_access exists=yes list=no path=%s\n" "$candidate_path" - fi - done - ' -} - -openshell_context() { - # Variables in the next single-quoted command belong to the in-sandbox shell. - # shellcheck disable=SC2016 - openshell sandbox exec -n "$sandbox_name" -- sh -c ' - id - grep -E "^(NoNewPrivs|Seccomp|Seccomp_filters):" /proc/self/status 2>/dev/null || true - for device_path in \ - /dev/nvmap \ - /dev/nvhost-* \ - /dev/nvgpu/igpu0/* \ - /dev/dri/renderD* \ - /dev/nvsciipc*; do - [ -e "$device_path" ] || continue - stat -Lc "device type=%F mode=%a uid=%u gid=%g path=%n" "$device_path" 2>/dev/null || true - if [ -r "$device_path" ]; then device_read=yes; else device_read=no; fi - if [ -w "$device_path" ]; then device_write=yes; else device_write=no; fi - printf "device_permission_check read=%s write=%s path=%s\n" "$device_read" "$device_write" "$device_path" - done - for candidate_path in \ - /proc/device-tree \ - /sys/firmware/devicetree/base \ - /sys/devices/platform \ - /sys/class/devfreq \ - /sys/module; do - if [ ! -e "$candidate_path" ]; then - printf "path_access exists=no path=%s\n" "$candidate_path" - elif ls -A "$candidate_path" >/dev/null 2>&1; then - printf "path_access exists=yes list=yes path=%s\n" "$candidate_path" - else - printf "path_access exists=yes list=no path=%s\n" "$candidate_path" - fi - done - ' -} - -selected_nvidia_environment() { - docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' "$container_id" \ - | grep -E '^NVIDIA_(VISIBLE_DEVICES|DRIVER_CAPABILITIES)=' -} - -base_policy_gpu_paths() { - openshell policy get --base "$sandbox_name" 2>&1 \ - | grep -E 'filesystem_policy:|read_only:|read_write:|/dev/(nv|dri)|/opt/nvidia|/proc|/sys' -} - -section "host and tool versions" -printf 'sandbox=%s\n' "$sandbox_name" -run_stage "git head" git rev-parse HEAD -run_stage "nemoclaw version" nemoclaw --version -run_stage "openshell version" openshell --version -run_stage "kernel" uname -a -run_stage "host nvmap" stat -Lc 'type=%F mode=%a uid=%u gid=%g group=%G path=%n' /dev/nvmap - -container_id="$({ - docker ps -q \ - --filter 'label=openshell.ai/managed-by=openshell' \ - --filter "label=openshell.ai/sandbox-name=${sandbox_name}" -} | head -n 1)" - -if [ -z "$container_id" ]; then - printf 'ERROR: no running OpenShell-managed container found for sandbox %s\n' "$sandbox_name" >&2 - exit 2 -fi - -section "managed container" -printf 'container=%s\n' "$container_id" -run_stage "container configuration" docker inspect --format \ - 'runtime={{.HostConfig.Runtime}} user={{json .Config.User}} status={{.State.Status}} health={{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}} group_add={{json .HostConfig.GroupAdd}}' \ - "$container_id" -run_stage "selected NVIDIA environment" selected_nvidia_environment - -section "effective filesystem policy entries" -run_stage "OpenShell base policy GPU paths" base_policy_gpu_paths - -section "Docker exec as root" -run_stage "identity, devices, sysfs, and seccomp" container_context 0 -run_stage "cuInit baseline" docker exec --user 0 "$container_id" python3 -c "$cuda_probe" -run_stage "cuInit without eager JIT preload" docker exec --user 0 "$container_id" env \ - CUDA_FORCE_PRELOAD_LIBRARIES=0 python3 -c "$cuda_probe" - -section "Docker exec as sandbox" -run_stage "identity, devices, sysfs, and seccomp" container_context sandbox -run_stage "cuInit baseline" docker exec --user sandbox "$container_id" python3 -c "$cuda_probe" -run_stage "cuInit without eager JIT preload" docker exec --user sandbox "$container_id" env \ - CUDA_FORCE_PRELOAD_LIBRARIES=0 python3 -c "$cuda_probe" - -section "OpenShell sandbox execution" -run_stage "identity, devices, sysfs, and seccomp" openshell_context -run_stage "cuInit baseline" openshell sandbox exec -n "$sandbox_name" -- python3 -c "$cuda_probe" -run_stage "cuInit without eager JIT preload" openshell sandbox exec -n "$sandbox_name" -- env \ - CUDA_FORCE_PRELOAD_LIBRARIES=0 python3 -c "$cuda_probe" - -section "interpretation" -printf '%s\n' \ - 'Docker sandbox-user cuInit(0) succeeds while OpenShell cuInit(0) fails: suggests an OpenShell confinement difference.' \ - 'Docker root cuInit(0) succeeds while Docker sandbox-user cuInit(0) fails: suggests an account or device-permission difference.' \ - 'Docker root cuInit(0) fails: suggests a recreated-container or NVIDIA runtime difference.' \ - 'Baseline cuInit(0) fails while the no-preload probe succeeds: suggests a CUDA eager JIT library preload difference.' diff --git a/scripts/jetson-device-group-bootstrap.sh b/scripts/jetson-device-group-bootstrap.sh deleted file mode 100755 index 3e2d3c3f2a..0000000000 --- a/scripts/jetson-device-group-bootstrap.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env bash -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -set -euo pipefail - -readonly GROUP_GIDS="${NEMOCLAW_JETSON_DEVICE_GROUP_GIDS:-}" -readonly SANDBOX_USER="sandbox" - -[[ "$(id -u)" == "0" ]] || { - echo "Jetson device-group bootstrap must run as root before the OpenShell supervisor." >&2 - exit 1 -} -[[ $# -gt 0 ]] || { - echo "Jetson device-group bootstrap requires the OpenShell supervisor command." >&2 - exit 1 -} -id "$SANDBOX_USER" >/dev/null 2>&1 || { - echo "Jetson device-group bootstrap could not resolve the sandbox user." >&2 - exit 1 -} - -IFS=',' read -r -a gids <<<"$GROUP_GIDS" -for gid in "${gids[@]}"; do - [[ "$gid" =~ ^[1-9][0-9]*$ ]] || { - echo "Jetson device-group bootstrap received an invalid group ID." >&2 - exit 1 - } - - group_record="$(getent group "$gid" || true)" - if [[ -n "$group_record" ]]; then - IFS=':' read -r group_name _ <<<"$group_record" - else - group_name="nemoclaw_gpu_$gid" - groupadd --gid "$gid" "$group_name" - fi - [[ -n "$group_name" ]] || { - echo "Jetson device-group bootstrap could not resolve group ID $gid." >&2 - exit 1 - } - usermod --append --groups "$group_name" "$SANDBOX_USER" -done - -# OpenShell calls initgroups() before setgid()/setuid(). The container group -# database must contain every device group before the supervisor starts. -for gid in "${gids[@]}"; do - [[ " $(id -G "$SANDBOX_USER") " == *" $gid "* ]] || { - echo "Jetson device-group bootstrap did not add sandbox to group ID $gid." >&2 - exit 1 - } -done - -exec "$@" diff --git a/scripts/managed-bootstrap-trampoline.sh b/scripts/managed-bootstrap-trampoline.sh index 68ecd8315d..b0319b0f38 100644 --- a/scripts/managed-bootstrap-trampoline.sh +++ b/scripts/managed-bootstrap-trampoline.sh @@ -36,24 +36,6 @@ _nemoclaw_supervisor_environment_bytes="$4" [ "${5:-}" = "--" ] || fail "supervisor environment delimiter is missing" shift 5 -# Read the reserved group input from the sealed supervisor environment. -# The native resume path rewinds and validates FD 9 before supervisor exec. -_nemoclaw_jetson_device_group_gids="" -_nemoclaw_jetson_device_group_gids_seen=0 -for ((_nemoclaw_environment_index = 0; _nemoclaw_environment_index < _nemoclaw_supervisor_environment_count; _nemoclaw_environment_index++)); do - IFS= read -r -d '' _nemoclaw_environment_entry <&9 \ - || fail "supervisor environment transport ended before Jetson group validation" - case "$_nemoclaw_environment_entry" in - NEMOCLAW_JETSON_DEVICE_GROUP_GIDS=*) - [ "$_nemoclaw_jetson_device_group_gids_seen" -eq 0 ] \ - || fail "Jetson device-group input is duplicated" - _nemoclaw_jetson_device_group_gids="${_nemoclaw_environment_entry#*=}" - _nemoclaw_jetson_device_group_gids_seen=1 - ;; - esac -done -unset _nemoclaw_environment_entry _nemoclaw_environment_index - if [ "$(/usr/bin/id -u 9<&-)" -ne 0 ] || [ "$(/usr/bin/id -g 9<&-)" -ne 0 ]; then fail "must run as root" fi @@ -107,16 +89,6 @@ fi [ "$_nemoclaw_request" = "/var/lib/nemoclaw-managed-bootstrap-request.json" ] \ || fail "request file path is not the fixed bootstrap path" -if [ "$_nemoclaw_jetson_device_group_gids_seen" -eq 1 ]; then - [ "$_nemoclaw_agent" = "openclaw" ] \ - || fail "Jetson device-group input requires the OpenClaw agent" - [[ "$_nemoclaw_jetson_device_group_gids" =~ ^[1-9][0-9]*(,[1-9][0-9]*)*$ ]] \ - || fail "Jetson device-group input is invalid" - # Update the sandbox account before the OpenShell supervisor calls initgroups(). - NEMOCLAW_JETSON_DEVICE_GROUP_GIDS="$_nemoclaw_jetson_device_group_gids" \ - /usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh /usr/bin/true 9<&- -fi - _nemoclaw_runtime="/usr/local/lib/nemoclaw/managed-startup-image-runtime.cjs" _nemoclaw_request_directory="${_nemoclaw_request%/*}" _nemoclaw_request_basename="${_nemoclaw_request##*/}" diff --git a/scripts/setup-jetson.sh b/scripts/setup-jetson.sh index 35a9d2a595..6b054780d0 100755 --- a/scripts/setup-jetson.sh +++ b/scripts/setup-jetson.sh @@ -6,11 +6,6 @@ set -euo pipefail SUDO=() ((EUID != 0)) && SUDO=(sudo) -JETSON_HOST_SUDO_READY=0 - -NVMAP_DEVICE="/dev/nvmap" -NVMAP_UDEV_RULE="/etc/udev/rules.d/99-zz-nemoclaw-nvmap.rules" -NVMAP_UDEV_RULE_CONTENT='KERNEL=="nvmap", MODE="0660"' info() { printf "[INFO] %s\n" "$*" @@ -25,16 +20,6 @@ error() { exit 1 } -ensure_jetson_host_sudo() { - if ((EUID == 0)) || [[ "$JETSON_HOST_SUDO_READY" == "1" ]]; then - return 0 - fi - - info "Jetson host configuration requires sudo. You may be prompted for your password." - "${SUDO[@]}" true >/dev/null || error "Sudo is required to apply Jetson host configuration." - JETSON_HOST_SUDO_READY=1 -} - # Returns 0 only when both the live kernel state AND our persistent # drop-ins are in place: # - runtime: bridge-nf-call-iptables sysctl reads back as 1 @@ -71,33 +56,6 @@ apply_br_netfilter_setup() { echo "net.bridge.bridge-nf-call-iptables=1" | "${SUDO[@]}" tee /etc/sysctl.d/99-nemoclaw.conf >/dev/null } -configure_nvmap_group_access() { - local device_state device_type verified_state verified_permissions - - if ! device_state="$(LC_ALL=C stat -c '%F|%A' "$NVMAP_DEVICE" 2>/dev/null)"; then - warn "Jetson host setup could not find $NVMAP_DEVICE. Non-root sandbox CUDA can fail until this device exists." - return 0 - fi - - device_type="${device_state%%|*}" - [[ "$device_type" == "character special file" ]] \ - || error "$NVMAP_DEVICE must be a character device before NemoClaw changes its group permissions." - - ensure_jetson_host_sudo - warn "Jetson host setup grants every member of the existing $NVMAP_DEVICE owning group write access and persists mode 0660 when udev recreates the device." - printf '%s\n' "$NVMAP_UDEV_RULE_CONTENT" | "${SUDO[@]}" tee "$NVMAP_UDEV_RULE" >/dev/null - "${SUDO[@]}" udevadm control --reload-rules - "${SUDO[@]}" chmod g+rw "$NVMAP_DEVICE" - - verified_state="$(LC_ALL=C stat -c '%F|%A' "$NVMAP_DEVICE" 2>/dev/null)" \ - || error "Could not verify $NVMAP_DEVICE after granting group read-write access." - IFS='|' read -r device_type verified_permissions <<<"$verified_state" - [[ "$device_type" == "character special file" && "${verified_permissions:4:2}" == "rw" ]] \ - || error "$NVMAP_DEVICE does not grant its owning group read-write access after host setup." - - info "$NVMAP_DEVICE grants its owning group read-write access. The udev rule $NVMAP_UDEV_RULE preserves this mode after reboot." -} - warn_host_setup_skipped() { warn "Skipped Jetson host setup: iptables legacy mode and the Docker daemon.json adjustment (L4T 36.x only), and br_netfilter with net.bridge.bridge-nf-call-iptables=1 (every release)." warn "Without br_netfilter, k3s inside the OpenShell gateway cannot NAT sandbox pod traffic to ClusterIP services, so sandbox pods cannot reach CoreDNS." @@ -106,7 +64,10 @@ warn_host_setup_skipped() { } get_jetpack_version() { - local release_line="$1" release revision l4t_version + local release_line release revision l4t_version + + release_line="$(head -n1 /etc/nv_tegra_release 2>/dev/null || true)" + [[ -n "$release_line" ]] || return 0 release="$(printf '%s\n' "$release_line" | sed -n 's/^# R\([0-9][0-9]*\) (release).*/\1/p')" revision="$(printf '%s\n' "$release_line" | sed -n 's/^.*REVISION: \([0-9][0-9]*\)\..*$/\1/p')" @@ -164,7 +125,10 @@ get_jetpack_version() { configure_jetson_host() { local jetpack_version="$1" - ensure_jetson_host_sudo + if ((EUID != 0)); then + info "Jetson host configuration requires sudo. You may be prompted for your password." + "${SUDO[@]}" true >/dev/null || error "Sudo is required to apply Jetson host configuration." + fi case "$jetpack_version" in jp6) @@ -241,29 +205,8 @@ PYEOF } main() { - local jetpack_version release_line mode="${1:-}" - - if [[ "$#" -gt 1 || (-n "$mode" && "$mode" != "--nvmap-only") ]]; then - error "Usage: setup-jetson.sh [--nvmap-only]" - fi - - if [[ "$mode" == "--nvmap-only" ]]; then - [[ "${NEMOCLAW_AGENT:-openclaw}" == "openclaw" ]] \ - || error "Jetson nvmap-only setup is available only for OpenClaw." - configure_nvmap_group_access - return 0 - fi - - release_line="$(head -n1 /etc/nv_tegra_release 2>/dev/null || true)" - [[ -n "$release_line" ]] || exit 0 - - # nvmap permissions follow the detected device, not the L4T version parser. - # Version-specific networking changes remain gated below. - if [[ "${NEMOCLAW_AGENT:-openclaw}" == "openclaw" ]]; then - configure_nvmap_group_access - fi - - jetpack_version="$(get_jetpack_version "$release_line")" + local jetpack_version + jetpack_version="$(get_jetpack_version)" [[ -n "$jetpack_version" ]] || exit 0 info "Jetson detected ($jetpack_version) — applying required host configuration" diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index abb1a9b709..499cb51b88 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -2641,7 +2641,6 @@ async function createSandboxWithBaseImageResolution( } = await sandboxGpuCreateFlow.runSandboxGpuCreateFlow( { sandboxName, - agentName: agent?.name ?? "openclaw", provider, sandboxGpuConfig: effectiveSandboxGpuConfig, gpuRoutePlan, @@ -2671,6 +2670,7 @@ async function createSandboxWithBaseImageResolution( if (initialSandboxPolicy.cleanup && initialSandboxPolicy.cleanup()) { process.removeListener("exit", initialSandboxPolicy.cleanup); } + // Clean up build context regardless of outcome. // Use fs.rmSync instead of run() to avoid spawning a shell process. // Only deregister the 'exit' safety net when inline cleanup succeeded; diff --git a/src/lib/onboard/docker-gpu-jetson-groups.test.ts b/src/lib/onboard/docker-gpu-jetson-groups.test.ts index 72db1729e1..90b6dfb544 100644 --- a/src/lib/onboard/docker-gpu-jetson-groups.test.ts +++ b/src/lib/onboard/docker-gpu-jetson-groups.test.ts @@ -5,40 +5,7 @@ import fs from "node:fs"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { - detectTegraDeviceGroupGids, - detectTegraGpuDevicePaths, - ensureJetsonNvmapGroupAccess, -} from "./docker-gpu-jetson-groups"; - -const READ_ONLY_NVMAP = { - isCharacterDevice: true, - isSymbolicLink: false, - mode: 0o440, -}; -const READ_WRITE_NVMAP = { ...READ_ONLY_NVMAP, mode: 0o660 }; - -describe("detectTegraGpuDevicePaths", () => { - it("returns only existing non-symlink character devices for Landlock (#7610)", () => { - const pathAccess = new Map([ - ["/dev/nvmap", { isCharacterDevice: true, isSymbolicLink: false }], - ["/dev/nvhost-gpu", { isCharacterDevice: false, isSymbolicLink: false }], - ["/dev/dri/renderD128", { isCharacterDevice: true, isSymbolicLink: true }], - ]); - - expect( - detectTegraGpuDevicePaths({ - listDevicePaths: () => [ - "/dev/nvmap", - "/dev/nvhost-gpu", - "/dev/dri/renderD128", - "/dev/missing", - ], - statDevicePath: (devicePath) => pathAccess.get(devicePath) ?? null, - }), - ).toEqual(["/dev/nvmap"]); - }); -}); +import { detectTegraDeviceGroupGids } from "./docker-gpu-jetson-groups"; describe("detectTegraDeviceGroupGids", () => { afterEach(() => { @@ -68,8 +35,6 @@ describe("detectTegraDeviceGroupGids", () => { { gid: 0, mode: 0o660 }, { gid: 2_147_483_648, mode: 0o660 }, { gid: 44, mode: 0o600 }, - { gid: 44, mode: 0o440 }, - { gid: 44, mode: 0o620 }, { gid: 104, mode: 0o666 }, ]; let index = 0; @@ -82,15 +47,6 @@ describe("detectTegraDeviceGroupGids", () => { ).toEqual([]); }); - it("does not treat a read-only nvmap group as CUDA access (#7610)", () => { - expect( - detectTegraDeviceGroupGids({ - statDeviceAccess: () => ({ gid: 44, mode: 0o440 }), - listDevicePaths: () => ["/dev/nvmap"], - }), - ).toEqual([]); - }); - it("returns no GIDs when Tegra nodes are missing or unreadable", () => { expect( detectTegraDeviceGroupGids({ @@ -175,57 +131,3 @@ describe("detectTegraDeviceGroupGids", () => { } }); }); - -describe("ensureJetsonNvmapGroupAccess", () => { - afterEach(() => { - vi.restoreAllMocks(); - }); - - it("keeps verified group read-write access without running host setup (#7610)", () => { - const runSetup = vi.fn(); - - ensureJetsonNvmapGroupAccess({ statDevice: () => READ_WRITE_NVMAP, runSetup }); - - expect(runSetup).not.toHaveBeenCalled(); - }); - - it("repairs group-read-only nvmap access and verifies the result (#7610)", () => { - const statDevice = vi - .fn() - .mockReturnValueOnce(READ_ONLY_NVMAP) - .mockReturnValueOnce(READ_WRITE_NVMAP); - const runSetup = vi.fn(() => ({ status: 0 })); - vi.spyOn(console, "log").mockImplementation(() => {}); - - ensureJetsonNvmapGroupAccess({ - statDevice, - runSetup, - setupScriptPath: "/package/scripts/setup-jetson.sh", - }); - - expect(runSetup).toHaveBeenCalledWith("/package/scripts/setup-jetson.sh"); - expect(statDevice).toHaveBeenCalledTimes(2); - }); - - it("stops before sandbox creation when host setup fails (#7610)", () => { - vi.spyOn(console, "log").mockImplementation(() => {}); - - expect(() => - ensureJetsonNvmapGroupAccess({ - statDevice: () => READ_ONLY_NVMAP, - runSetup: () => ({ status: 1 }), - }), - ).toThrow("Jetson /dev/nvmap group setup failed before sandbox creation (exit status 1)"); - }); - - it("rejects an unverified post-setup device state (#7610)", () => { - vi.spyOn(console, "log").mockImplementation(() => {}); - - expect(() => - ensureJetsonNvmapGroupAccess({ - statDevice: () => READ_ONLY_NVMAP, - runSetup: () => ({ status: 0 }), - }), - ).toThrow("still does not grant its owning group read-write access"); - }); -}); diff --git a/src/lib/onboard/docker-gpu-jetson-groups.ts b/src/lib/onboard/docker-gpu-jetson-groups.ts index 95a9cb3afd..4cfcd098f4 100644 --- a/src/lib/onboard/docker-gpu-jetson-groups.ts +++ b/src/lib/onboard/docker-gpu-jetson-groups.ts @@ -1,9 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { spawnSync } from "node:child_process"; import fs from "node:fs"; -import path from "node:path"; const TEGRA_GPU_DEVICE_NODES = [ "/dev/nvmap", @@ -20,92 +18,12 @@ const TEGRA_GPU_DEVICE_NODES = [ ] as const; const READ_WRITE_PERMISSION_BITS = 0o6; const MAX_DOCKER_SUPPLEMENTARY_GID = 2_147_483_647; -const NVMAP_DEVICE = "/dev/nvmap"; type DeviceGroupAccess = { gid: number; mode: number; }; -type DevicePathAccess = { - isCharacterDevice: boolean; - isSymbolicLink: boolean; -}; - -type NvmapDeviceAccess = { - isCharacterDevice: boolean; - isSymbolicLink: boolean; - mode: number; -}; - -export interface EnsureJetsonNvmapGroupAccessDeps { - statDevice?: () => NvmapDeviceAccess | null; - runSetup?: (scriptPath: string) => { status: number | null; error?: Error }; - setupScriptPath?: string; -} - -function defaultStatNvmapDevice(): NvmapDeviceAccess | null { - try { - const stat = fs.lstatSync(NVMAP_DEVICE); - return { - isCharacterDevice: stat.isCharacterDevice(), - isSymbolicLink: stat.isSymbolicLink(), - mode: stat.mode, - }; - } catch { - return null; - } -} - -function hasGroupReadWriteAccess(access: NvmapDeviceAccess | null): boolean { - return ( - access !== null && - access.isCharacterDevice && - !access.isSymbolicLink && - ((access.mode >> 3) & READ_WRITE_PERMISSION_BITS) === READ_WRITE_PERMISSION_BITS - ); -} - -function runJetsonNvmapSetup(scriptPath: string): { status: number | null; error?: Error } { - const result = spawnSync("bash", [scriptPath, "--nvmap-only"], { - env: { ...process.env, NEMOCLAW_AGENT: "openclaw" }, - stdio: "inherit", - }); - return { status: result.status, ...(result.error ? { error: result.error } : {}) }; -} - -function setupFailureDetail(result: { status: number | null; error?: Error }): string { - if (result.error?.message) return result.error.message; - return `exit status ${result.status === null ? "unknown" : result.status}`; -} - -/** - * Verify the host permission that makes the detected nvmap group useful to the - * nonroot OpenShell sandbox user. Installer setup is not sufficient at this - * boundary because onboarding can run directly or after host device state - * changes. Repair before any sandbox create or replacement begins. - */ -export function ensureJetsonNvmapGroupAccess(deps: EnsureJetsonNvmapGroupAccessDeps = {}): void { - const statDevice = deps.statDevice ?? defaultStatNvmapDevice; - if (hasGroupReadWriteAccess(statDevice())) return; - - const setupScriptPath = - deps.setupScriptPath ?? path.resolve(__dirname, "../../../scripts/setup-jetson.sh"); - const runSetup = deps.runSetup ?? runJetsonNvmapSetup; - console.log(" Preparing Jetson /dev/nvmap group access before sandbox creation..."); - const result = runSetup(setupScriptPath); - if (result.status !== 0) { - throw new Error( - `Jetson /dev/nvmap group setup failed before sandbox creation (${setupFailureDetail(result)}).`, - ); - } - if (!hasGroupReadWriteAccess(statDevice())) { - throw new Error( - "Jetson /dev/nvmap still does not grant its owning group read-write access after host setup; refusing sandbox creation.", - ); - } -} - /** * Find real DRI render character devices without following symlinks or * scanning other DRI device families. @@ -133,52 +51,19 @@ function listTegraGpuDevicePaths(): string[] { return [...TEGRA_GPU_DEVICE_NODES, ...discoverTegraRenderDevicePaths()]; } -/** - * Return only existing Jetson GPU character devices that can be granted at - * the OpenShell Landlock boundary. The fixed candidates keep the policy from - * widening to unrelated host devices, and lstat prevents symlink traversal. - */ -export function detectTegraGpuDevicePaths( - deps: { - statDevicePath?: (path: string) => DevicePathAccess | null; - listDevicePaths?: () => string[]; - } = {}, -): string[] { - const devicePaths = deps.listDevicePaths?.() ?? listTegraGpuDevicePaths(); - const statPath = - deps.statDevicePath ?? - ((devicePath: string): DevicePathAccess | null => { - try { - const stat = fs.lstatSync(devicePath); - return { - isCharacterDevice: stat.isCharacterDevice(), - isSymbolicLink: stat.isSymbolicLink(), - }; - } catch { - return null; - } - }); - return devicePaths.filter((devicePath) => { - const access = statPath(devicePath); - return access?.isCharacterDevice === true && access.isSymbolicLink === false; - }); -} - /** * Source-of-truth boundary for Jetson/Tegra supplementary device groups: * - * - Invalid state: `/dev/nvmap` lacks owning-group read/write access, Landlock omits an injected - * Tegra character device, or the non-root sandbox user loses the matching host device GID. - * - Source boundary: NemoClaw verifies and persists the host nvmap mode, grants exact detected - * character-device paths in the route policy, carries each bounded numeric device GID into the - * Jetson recreation via `--group-add`, and records matching sandbox account membership before - * the supervisor starts. - * - Source-fix constraint: `--group-add` alone does not survive the supervisor's account-group - * initialization, and image-local group names can differ from the host's numeric device GIDs. - * - Regression coverage: docker-gpu-jetson-groups.test.ts covers path/GID discovery and hostile - * numeric values; initial-policy.test.ts covers Landlock grants; setup-jetson.test.ts covers the - * host mode; docker-gpu-patch-jetson.test.ts covers clone-envelope and sandbox-account - * propagation plus generic-host exclusion. + * - Invalid state: the non-root sandbox user can see `/dev/nvmap` and `/dev/nvhost-*` but cannot + * open them because Docker did not copy their host-owned supplementary GIDs into the container. + * - Source boundary: host device-node ownership is authoritative; NemoClaw only carries each + * bounded, non-root numeric GID with effective group read/write permission into the Jetson + * compatibility recreation via `--group-add`. + * - Source-fix constraint: changing host udev ownership or image-local groups cannot reliably fix + * device nodes whose ownership is assigned by the Jetson host at runtime. + * - Regression coverage: docker-gpu-jetson-groups.test.ts covers discovery and hostile numeric + * values; docker-gpu-patch-jetson.test.ts covers clone-envelope propagation and generic-host + * exclusion. * - Removal condition: remove this probe when the minimum supported native OpenShell Jetson path * propagates the host device groups without compatibility container recreation. */ @@ -207,15 +92,13 @@ export function detectTegraDeviceGroupGids( const gid = access?.gid ?? null; const groupAccessBits = access === null ? 0 : (access.mode >> 3) & READ_WRITE_PERMISSION_BITS; const otherAccessBits = access === null ? 0 : access.mode & READ_WRITE_PERMISSION_BITS; - const groupAddsReadWriteAccess = - groupAccessBits === READ_WRITE_PERMISSION_BITS && - otherAccessBits !== READ_WRITE_PERMISSION_BITS; + const groupAddsAccess = (groupAccessBits & ~otherAccessBits) !== 0; if ( gid !== null && Number.isSafeInteger(gid) && gid > 0 && gid <= MAX_DOCKER_SUPPLEMENTARY_GID && - groupAddsReadWriteAccess + groupAddsAccess ) { gids.add(String(gid)); } diff --git a/src/lib/onboard/docker-gpu-patch-clone.ts b/src/lib/onboard/docker-gpu-patch-clone.ts index 9c3138fa92..2adb44f47d 100644 --- a/src/lib/onboard/docker-gpu-patch-clone.ts +++ b/src/lib/onboard/docker-gpu-patch-clone.ts @@ -10,8 +10,6 @@ import type { import { openshellSandboxCommandEnvValue } from "./docker-startup-command-env"; const OPENSHELL_SANDBOX_COMMAND_ENV = "OPENSHELL_SANDBOX_COMMAND"; -const JETSON_DEVICE_GROUP_GIDS_ENV = "NEMOCLAW_JETSON_DEVICE_GROUP_GIDS"; -const JETSON_DEVICE_GROUP_BOOTSTRAP = "/usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh"; const GPU_ENV_KEYS = new Set([ "NVIDIA_VISIBLE_DEVICES", "NVIDIA_DRIVER_CAPABILITIES", @@ -355,18 +353,6 @@ export function buildDockerGpuCloneRunArgs( } const args: string[] = ["--name", containerName, ...mode.args]; const gpuAugment = mode.kind !== "startup-command"; - const extraGroupGids = [...(options.extraGroupGids ?? [])].map((gid) => String(gid).trim()); - if ( - extraGroupGids.some((gid) => { - if (!/^[1-9][0-9]*$/u.test(gid)) return true; - const parsed = Number(gid); - return !Number.isSafeInteger(parsed) || parsed > 2_147_483_647; - }) - ) { - throw new Error("Docker clone received an invalid supplementary group ID."); - } - const preserveJetsonGroups = - options.preserveJetsonDeviceGroupMembership === true && extraGroupGids.length > 0; // Startup-command recreation must retain OpenShell's native CDI attachment. if (!gpuAugment) { @@ -394,10 +380,9 @@ export function buildDockerGpuCloneRunArgs( const sandboxCommand = openshellSandboxCommandEnvValue(options.openshellSandboxCommand); let sawSandboxCommand = false; - for (const env of stringArray(config.Env).filter((entry) => { - const key = envKey(entry); - return key !== JETSON_DEVICE_GROUP_GIDS_ENV && (!gpuAugment || !GPU_ENV_KEYS.has(key)); - })) { + for (const env of stringArray(config.Env).filter( + (entry) => !gpuAugment || !GPU_ENV_KEYS.has(envKey(entry)), + )) { const key = envKey(env); if (key === OPENSHELL_SANDBOX_COMMAND_ENV && sandboxCommand) { sawSandboxCommand = true; @@ -409,9 +394,6 @@ export function buildDockerGpuCloneRunArgs( if (sandboxCommand && !sawSandboxCommand) { args.push("--env", `${OPENSHELL_SANDBOX_COMMAND_ENV}=${sandboxCommand}`); } - if (preserveJetsonGroups) { - args.push("--env", `${JETSON_DEVICE_GROUP_GIDS_ENV}=${extraGroupGids.join(",")}`); - } const labels = config.Labels || {}; for (const key of Object.keys(labels).sort()) { @@ -446,10 +428,11 @@ export function buildDockerGpuCloneRunArgs( for (const hostEntry of stringArray(host.ExtraHosts)) args.push("--add-host", hostEntry); const groupAdds = new Set(stringArray(host.GroupAdd)); for (const group of groupAdds) args.push("--group-add", group); - for (const gid of extraGroupGids) { - if (!groupAdds.has(gid)) { - groupAdds.add(gid); - args.push("--group-add", gid); + for (const gid of options.extraGroupGids ?? []) { + const normalized = String(gid).trim(); + if (normalized && !groupAdds.has(normalized)) { + groupAdds.add(normalized); + args.push("--group-add", normalized); } } for (const ulimit of dockerUlimits(inspect, options.requiredUlimits)) { @@ -483,26 +466,16 @@ export function buildDockerGpuCloneRunArgs( const entrypoint = stringArray(config.Entrypoint); const replacementEntrypoint = String(options.containerEntrypoint ?? "").trim(); - const replacementProcess = Boolean(replacementEntrypoint || options.containerCommand); - if (preserveJetsonGroups && !replacementProcess && entrypoint.length === 0) { - throw new Error("Jetson device-group bootstrap requires the OpenShell supervisor entrypoint."); - } - if (preserveJetsonGroups && !replacementProcess) { - args.push("--entrypoint", JETSON_DEVICE_GROUP_BOOTSTRAP); - } else if (replacementEntrypoint) { + if (replacementEntrypoint) { args.push("--entrypoint", replacementEntrypoint); } else if (entrypoint.length > 0) { args.push("--entrypoint", entrypoint[0]); } - const originalCommandArgs = sandboxCommand - ? [] - : [...entrypoint.slice(1), ...stringArray(config.Cmd)]; - const commandArgs = - preserveJetsonGroups && !replacementProcess - ? [entrypoint[0], ...originalCommandArgs] - : options.containerCommand - ? [...options.containerCommand] - : originalCommandArgs; + const commandArgs = options.containerCommand + ? [...options.containerCommand] + : sandboxCommand + ? [] + : [...entrypoint.slice(1), ...stringArray(config.Cmd)]; args.push(image, ...commandArgs); return args; } diff --git a/src/lib/onboard/docker-gpu-patch-jetson.test.ts b/src/lib/onboard/docker-gpu-patch-jetson.test.ts index 84ac3797ea..e091c45a13 100644 --- a/src/lib/onboard/docker-gpu-patch-jetson.test.ts +++ b/src/lib/onboard/docker-gpu-patch-jetson.test.ts @@ -34,43 +34,6 @@ describe("Jetson device-node group propagation (#4231, #7610)", () => { expect(args).toEqual(expect.arrayContaining(["--group-add", "110"])); }); - it("adds Jetson device groups to the sandbox user before OpenShell calls initgroups (#7610)", () => { - const inspect = inspectFixture(); - inspect.Config!.Env = ["OPENSHELL_SANDBOX_COMMAND=env nemoclaw-start"]; - const args = buildDockerGpuCloneRunArgs( - inspect, - buildDockerGpuMode("nvidia-runtime", null, { backend: "jetson" }), - { - extraGroupGids: ["44", "995"], - preserveJetsonDeviceGroupMembership: true, - }, - ); - - expect(args).toEqual( - expect.arrayContaining([ - "--env", - "NEMOCLAW_JETSON_DEVICE_GROUP_GIDS=44,995", - "--entrypoint", - "/usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh", - "openshell/sandbox:abc", - "/opt/openshell/bin/openshell-sandbox", - ]), - ); - }); - - it("rejects an invalid Jetson device group before Docker recreation (#7610)", () => { - expect(() => - buildDockerGpuCloneRunArgs( - inspectFixture(), - buildDockerGpuMode("nvidia-runtime", null, { backend: "jetson" }), - { - extraGroupGids: ["44;id"], - preserveJetsonDeviceGroupMembership: true, - }, - ), - ).toThrow("invalid supplementary group ID"); - }); - it("does not add --group-add when extraGroupGids is absent", () => { const inspect = inspectFixture(); inspect.HostConfig!.GroupAdd = []; @@ -78,7 +41,7 @@ describe("Jetson device-node group propagation (#4231, #7610)", () => { expect(args).not.toEqual(expect.arrayContaining(["--group-add"])); }); - it("passes all detected Tegra device GIDs into the OpenClaw Jetson bootstrap", () => { + it("passes all detected Tegra device GIDs into the Jetson recreate as --group-add", () => { const dockerRunDetached = vi.fn(() => ({ status: 0, stdout: "new-container-id\n", @@ -96,12 +59,7 @@ describe("Jetson device-node group propagation (#4231, #7610)", () => { ); recreateOpenShellDockerSandboxWithGpu( - { - sandboxName: "alpha", - timeoutSecs: 1, - backend: "jetson", - preserveJetsonDeviceGroupMembership: true, - }, + { sandboxName: "alpha", timeoutSecs: 1, backend: "jetson" }, { dockerCapture: dockerCaptureFixture(), dockerRun: vi.fn(() => ({ status: 0, stdout: "probe-id\n" })), @@ -119,18 +77,7 @@ describe("Jetson device-node group propagation (#4231, #7610)", () => { expect(detectTegraDeviceGroupGidsStub).toHaveBeenCalled(); expect(dockerRunDetached).toHaveBeenCalledWith( - expect.arrayContaining([ - "--env", - "NEMOCLAW_JETSON_DEVICE_GROUP_GIDS=44,104,995", - "--group-add", - "44", - "--group-add", - "104", - "--group-add", - "995", - "--entrypoint", - "/usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh", - ]), + expect.arrayContaining(["--group-add", "44", "--group-add", "104", "--group-add", "995"]), expect.objectContaining({ ignoreError: true }), ); }); diff --git a/src/lib/onboard/docker-gpu-patch-recreate.ts b/src/lib/onboard/docker-gpu-patch-recreate.ts index a2981bd4e7..4eb1c0ad8d 100644 --- a/src/lib/onboard/docker-gpu-patch-recreate.ts +++ b/src/lib/onboard/docker-gpu-patch-recreate.ts @@ -159,7 +159,6 @@ export function recreateOpenShellDockerSandboxContainer( requiredUlimits?: readonly import("./docker-gpu-patch-types").DockerUlimit[] | null; expectedOldContainerId?: string | null; backend?: "generic" | "jetson"; - preserveJetsonDeviceGroupMembership?: boolean; dockerDesktopWsl?: boolean; modeOverride?: DockerGpuPatchMode; }, @@ -263,22 +262,18 @@ export function recreateOpenShellDockerSandboxContainer( ); } } - if (options.backend === "jetson") { + if (selection.mode.kind !== "startup-command" && options.backend === "jetson") { const tegraGroupGids = d.detectTegraDeviceGroupGids(); if (tegraGroupGids.length > 0) { cloneOptions.extraGroupGids = tegraGroupGids; - cloneOptions.preserveJetsonDeviceGroupMembership = - options.preserveJetsonDeviceGroupMembership === true; console.log( - ` ✓ Preparing detected Jetson GPU device groups ${tegraGroupGids.join( + ` ✓ Granting sandbox user the detected Jetson GPU device groups via --group-add ${tegraGroupGids.join( ", ", - )} for the recreated container${ - options.preserveJetsonDeviceGroupMembership ? " and the OpenShell sandbox user" : "" - }`, + )} (so CUDA can initialize as a non-root user)`, ); } else { console.warn( - " ⚠ Could not resolve a read-write group for Jetson Tegra GPU device nodes (/dev/nvmap); CUDA may fail with NvRmMemInitNvmap permission denied. Confirm /dev/nvmap exists and grants its owning group read-write access on the host.", + " ⚠ Could not resolve the group owning Jetson Tegra GPU device nodes (/dev/nvmap); CUDA may fail with NvRmMemInitNvmap permission denied. Confirm /dev/nvmap exists and is group-readable on the host.", ); } } diff --git a/src/lib/onboard/docker-gpu-patch-types.ts b/src/lib/onboard/docker-gpu-patch-types.ts index f7e83cdb94..f367df0ab2 100644 --- a/src/lib/onboard/docker-gpu-patch-types.ts +++ b/src/lib/onboard/docker-gpu-patch-types.ts @@ -42,10 +42,9 @@ export type DockerGpuPatchDeps = { /** * Resolve the host group ID(s) that own the Jetson/Tegra GPU device nodes * (`/dev/nvmap`, `/dev/nvhost-*`, and `/dev/dri/renderD*`). Used by the - * Jetson recreate for Docker `--group-add` and matching sandbox-account - * membership. The route-specific policy separately grants the exact device - * paths through Landlock so CUDA can open them (#4231, #7610). Injectable so - * the Jetson permission path is testable without Tegra hardware. + * Jetson recreate to grant the sandbox user matching `--group-add` + * membership so CUDA can open them (#4231, #7610). Injectable so the Jetson + * permission path is testable without Tegra hardware. */ detectTegraDeviceGroupGids?: () => string[]; /** Injectable directory lister for unit testing CDI spec discovery. */ @@ -126,16 +125,12 @@ export type DockerGpuCloneRunOptions = { containerName?: string | null; /** * Extra supplementary group IDs to add to the recreated container via - * `--group-add`. On Jetson these are the host groups that own Tegra GPU - * device nodes. Set `preserveJetsonDeviceGroupMembership` to add them to the - * OpenShell sandbox user's container group database. + * `--group-add`. On Jetson these are the host group(s) owning the Tegra GPU + * device nodes; granting the sandbox user membership lets CUDA's nvmap init + * open them instead of failing with `NvRmMemInitNvmap ... Permission + * denied` (#4231, #7610). */ extraGroupGids?: readonly string[] | null; - /** - * Add the detected Jetson device groups to the sandbox user's container group - * database before the OpenShell supervisor calls initgroups(). - */ - preserveJetsonDeviceGroupMembership?: boolean; }; export type DockerGpuPatchDiagnostics = { diff --git a/src/lib/onboard/docker-gpu-sandbox-create.ts b/src/lib/onboard/docker-gpu-sandbox-create.ts index 104dfbd80f..d4356c79d0 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create.ts @@ -84,7 +84,6 @@ type DockerGpuSandboxCreatePatchOptions = { requiredUlimits?: Parameters[0]["requiredUlimits"]; timeoutSecs: number; backend?: DockerGpuPatchBackend; - agentName?: string | null; /** * Whether the host is Docker Desktop WSL. Defaults to the cached * `isDockerDesktopWslRuntime()` probe. When true, the GPU patch skips the CDI @@ -188,8 +187,6 @@ export function createDockerGpuSandboxCreatePatch( requiredUlimits: options.requiredUlimits ?? null, timeoutSecs: options.timeoutSecs, backend: options.backend, - preserveJetsonDeviceGroupMembership: - options.backend === "jetson" && options.agentName === "openclaw", dockerDesktopWsl: options.dockerDesktopWsl ?? isDockerDesktopWslRuntime(), }; const recreationEnabled = diff --git a/src/lib/onboard/docker-startup-command-patch.test.ts b/src/lib/onboard/docker-startup-command-patch.test.ts index 05bc4c316a..71c5c9d12c 100644 --- a/src/lib/onboard/docker-startup-command-patch.test.ts +++ b/src/lib/onboard/docker-startup-command-patch.test.ts @@ -137,51 +137,6 @@ describe("Docker startup-command patch", () => { ); }); - it("preserves Jetson device groups during startup-command recreation (#7610)", () => { - const dockerRunDetached = vi.fn((_args: readonly string[]) => ({ - status: 0, - stdout: "new-container-id\n", - })); - - recreateStartupCommandForTest( - { - sandboxName: "alpha", - timeoutSecs: 1, - waitForSupervisor: false, - openshellSandboxCommand: ["env", "nemoclaw-start"], - backend: "jetson", - preserveJetsonDeviceGroupMembership: true, - }, - { - dockerCapture: vi.fn((args: readonly string[]) => - args[0] === "ps" ? "old-container-id\n" : JSON.stringify([inspectFixture()]), - ), - dockerRunDetached, - dockerRename: vi.fn(() => ({ status: 0 })), - dockerStop: vi.fn(() => ({ status: 0 })), - detectTegraDeviceGroupGids: () => ["44", "993"], - sleep: vi.fn(), - now: () => new Date("2026-07-10T00:00:00Z"), - }, - ); - - const cloneArgs = dockerRunDetached.mock.calls[0]?.[0] ?? []; - expect(cloneArgs).toEqual( - expect.arrayContaining([ - "--group-add", - "44", - "--group-add", - "993", - "--env", - "NEMOCLAW_JETSON_DEVICE_GROUP_GIDS=44,993", - "--entrypoint", - "/usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh", - `sha256:${"c".repeat(64)}`, - "/opt/openshell/bin/openshell-sandbox", - ]), - ); - }); - it("rejects an empty restart-persistence command before Docker mutation", () => { expect(() => recreateStartupCommandForTest({ diff --git a/src/lib/onboard/docker-startup-command-patch.ts b/src/lib/onboard/docker-startup-command-patch.ts index 7b972832c1..2b5b6f761e 100644 --- a/src/lib/onboard/docker-startup-command-patch.ts +++ b/src/lib/onboard/docker-startup-command-patch.ts @@ -18,8 +18,6 @@ export function recreateOpenShellDockerSandboxWithStartupCommand( openshellSandboxCommand: readonly string[]; requiredUlimits?: readonly import("./docker-gpu-patch-types").DockerUlimit[] | null; expectedOldContainerId?: string | null; - backend?: "generic" | "jetson"; - preserveJetsonDeviceGroupMembership?: boolean; }, deps: DockerGpuPatchDeps = {}, ): DockerGpuPatchResult { diff --git a/src/lib/onboard/docker-startup-command-sandbox-create.test.ts b/src/lib/onboard/docker-startup-command-sandbox-create.test.ts index 90630503a4..8ccc7295e8 100644 --- a/src/lib/onboard/docker-startup-command-sandbox-create.test.ts +++ b/src/lib/onboard/docker-startup-command-sandbox-create.test.ts @@ -118,31 +118,6 @@ describe("Docker startup-command sandbox creation", () => { expect(patch.selectedMode()?.kind).toBe("startup-command"); }); - it("forwards OpenClaw Jetson group preservation to startup-command recreation (#7610)", async () => { - const recreateStartupPatch = vi.fn(() => startupResult()); - const patch = createDockerGpuSandboxCreatePatch({ - route: "native", - persistStartupCommand: true, - sandboxName: "alpha", - openshellSandboxCommand: ["env", "nemoclaw-start"], - timeoutSecs: 60, - backend: "jetson", - agentName: "openclaw", - deps: makeDeps(), - overrides: { recreateStartupPatch }, - }); - - await patch.ensureApplied(); - - expect(recreateStartupPatch).toHaveBeenCalledWith( - expect.objectContaining({ - backend: "jetson", - preserveJetsonDeviceGroupMembership: true, - }), - expect.any(Object), - ); - }); - it("rolls back startup-command recreation when the supervisor does not reconnect", () => { const deps = makeDeps(); const result = startupResult(); diff --git a/src/lib/onboard/docker-startup-command-sandbox-create.ts b/src/lib/onboard/docker-startup-command-sandbox-create.ts index 868db4f238..dd8df2e17b 100644 --- a/src/lib/onboard/docker-startup-command-sandbox-create.ts +++ b/src/lib/onboard/docker-startup-command-sandbox-create.ts @@ -33,8 +33,6 @@ export function createDockerSandboxRecreator(options: { requiredUlimits: options.requiredUlimits, timeoutSecs: options.gpuOptions.timeoutSecs, waitForSupervisor, - backend: options.gpuOptions.backend, - preserveJetsonDeviceGroupMembership: options.gpuOptions.preserveJetsonDeviceGroupMembership, }, deps, ); diff --git a/src/lib/onboard/experimental/portable-host-preparation.test.ts b/src/lib/onboard/experimental/portable-host-preparation.test.ts index 3216de5771..4acbe18246 100644 --- a/src/lib/onboard/experimental/portable-host-preparation.test.ts +++ b/src/lib/onboard/experimental/portable-host-preparation.test.ts @@ -101,6 +101,31 @@ describe("preparePortableExperimentalHost", () => { expect(fs.statSync(containersConf).mode & 0o777).toBe(0o600); }); + it("keeps the portable firewall driver in the Podman default search path (#8441)", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-portable-")); + tempDirs.push(home); + const systemctl = vi.fn<(args: readonly string[], env: NodeJS.ProcessEnv) => SpawnResult>(() => + result(), + ); + const docker = vi + .fn<(args: readonly string[], env: NodeJS.ProcessEnv) => SpawnResult>() + .mockReturnValueOnce(result(1)) + .mockReturnValueOnce(result()); + const podman = vi.fn(() => result(0, "/run/user/1001/podman/podman.sock\n")); + + preparePortableExperimentalHost( + { NEMOCLAW_EXPERIMENTAL_PROFILE: "portable" }, + { platform: "linux", home, uid: 1001, systemctl, podman, docker }, + ); + + const dropIn = path.join( + home, + ".config/containers/containers.conf.d/99-nemoclaw-portable.conf", + ); + expect(fs.readFileSync(dropIn, "utf-8")).toContain('firewall_driver = "iptables"'); + expect(fs.statSync(dropIn).mode & 0o777).toBe(0o600); + }); + it("refuses to replace an unmanaged registry container", () => { const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-portable-")); tempDirs.push(home); diff --git a/src/lib/onboard/experimental/portable-host-preparation.ts b/src/lib/onboard/experimental/portable-host-preparation.ts index 58307c5dc1..e3f46df3c8 100644 --- a/src/lib/onboard/experimental/portable-host-preparation.ts +++ b/src/lib/onboard/experimental/portable-host-preparation.ts @@ -87,6 +87,14 @@ function writePortableRuntimeConfig(home: string, env: NodeJS.ProcessEnv): strin path.join(configHome, "containers", "registries.conf.d", "99-nemoclaw-portable.conf"), REGISTRY_FRAGMENT, ); + // Podman reads `containers.conf.d` drop-ins from its own default search path, so this drop-in + // keeps `firewall_driver` in effect for a shell that starts without CONTAINERS_CONF. The + // `systemctl --user set-environment` values below last only until the user manager restarts. + writePrivateConfig( + path.join(configHome, "containers", "containers.conf.d", "99-nemoclaw-portable.conf"), + PORTABLE_CONTAINERS_CONF, + ); + // The OpenShell gateway service and sandbox prebuild read this file through CONTAINERS_CONF. const containersConf = path.join(configHome, "nemoclaw", "portable", "containers.conf"); writePrivateConfig(containersConf, PORTABLE_CONTAINERS_CONF); return containersConf; diff --git a/src/lib/onboard/initial-policy.test.ts b/src/lib/onboard/initial-policy.test.ts index 95d98c0646..2cde41037a 100644 --- a/src/lib/onboard/initial-policy.test.ts +++ b/src/lib/onboard/initial-policy.test.ts @@ -369,82 +369,6 @@ network_policies: {} } }); - it("permits the OpenRM driver library path only for an OpenClaw Jetson GPU policy (#7610)", () => { - const genericPolicy = YAML.parse(buildDirectGpuPolicyYaml(BASE_POLICY_FIXTURE)); - const jetsonPolicy = YAML.parse( - buildDirectGpuPolicyYaml(BASE_POLICY_FIXTURE, { jetsonGpu: true }), - ); - - expect(genericPolicy.filesystem_policy.read_only).not.toContain("/opt/nvidia/l4t-gpu-libs"); - expect(jetsonPolicy.filesystem_policy.read_only).toContain("/opt/nvidia/l4t-gpu-libs"); - expect(jetsonPolicy.filesystem_policy.read_write).not.toContain("/opt/nvidia/l4t-gpu-libs"); - }); - - it("grants exact detected Jetson character devices read-write for Landlock (#7610)", () => { - const gpuPolicy = buildDirectGpuPolicyYaml( - ` -version: 1 -filesystem_policy: - read_only: - - /usr - - /dev/nvmap - read_write: - - /tmp - - /dev/nvhost-gpu -network_policies: {} -`, - { - jetsonGpu: true, - jetsonGpuDevicePaths: ["/dev/nvmap", "/dev/nvhost-gpu", "/dev/nvmap"], - }, - ); - const gpuDoc = YAML.parse(gpuPolicy); - - expect(gpuDoc.filesystem_policy.read_only).not.toContain("/dev/nvmap"); - expectSingleOccurrence(gpuDoc.filesystem_policy.read_write, "/dev/nvmap"); - expectSingleOccurrence(gpuDoc.filesystem_policy.read_write, "/dev/nvhost-gpu"); - }); - - it("does not grant Jetson character devices to a generic GPU policy (#7610)", () => { - const gpuPolicy = YAML.parse( - buildDirectGpuPolicyYaml(BASE_POLICY_FIXTURE, { - jetsonGpuDevicePaths: ["/dev/nvmap", "/dev/nvhost-gpu"], - }), - ); - - expect(gpuPolicy.filesystem_policy.read_write).not.toContain("/dev/nvmap"); - expect(gpuPolicy.filesystem_policy.read_write).not.toContain("/dev/nvhost-gpu"); - }); - - it("threads the OpenClaw Jetson library path through public policy preparation (#7610)", () => { - const basePolicyPath = tmpPolicy(BASE_POLICY_FIXTURE); - const prepared = prepareInitialSandboxCreatePolicy(basePolicyPath, [], { - directGpu: true, - jetsonGpu: true, - stationGb300SysfsReadOnlyPaths: [], - }); - const preparedDoc = YAML.parse(fs.readFileSync(prepared.policyPath, "utf-8")); - - expect(preparedDoc.filesystem_policy.read_only).toContain("/opt/nvidia/l4t-gpu-libs"); - expect(prepared.cleanup?.()).toBe(true); - }); - - it("threads detected Jetson character devices through public policy preparation (#7610)", () => { - const basePolicyPath = tmpPolicy(BASE_POLICY_FIXTURE); - const prepared = prepareInitialSandboxCreatePolicy(basePolicyPath, [], { - directGpu: true, - jetsonGpu: true, - jetsonGpuDevicePaths: ["/dev/nvmap", "/dev/nvhost-ctrl", "/dev/nvhost-gpu"], - stationGb300SysfsReadOnlyPaths: [], - }); - const preparedDoc = YAML.parse(fs.readFileSync(prepared.policyPath, "utf-8")); - - expect(preparedDoc.filesystem_policy.read_write).toEqual( - expect.arrayContaining(["/dev/nvmap", "/dev/nvhost-ctrl", "/dev/nvhost-gpu"]), - ); - expect(prepared.cleanup?.()).toBe(true); - }); - it("preserves best-effort Landlock for missing Station sysfs paths (#7103)", () => { const sysfsRoot = tmpSysfsRoot(); addPciDevice(sysfsRoot, "0009:06:00.0", "0x10de\n", "0x030200\n"); diff --git a/src/lib/onboard/initial-policy.ts b/src/lib/onboard/initial-policy.ts index 9475aade34..4d56cecb12 100644 --- a/src/lib/onboard/initial-policy.ts +++ b/src/lib/onboard/initial-policy.ts @@ -23,7 +23,6 @@ import { isStationGb300ProductName, type StationProfile, } from "../readiness/station-qualification"; -import { detectTegraGpuDevicePaths } from "./docker-gpu-jetson-groups"; import { allMessagingChannelPolicyPresets, requiredMessagingChannelPolicyPresets, @@ -47,7 +46,6 @@ export function discloseInitialSandboxPolicy(policy: InitialSandboxPolicy): void const HERMES_MESSAGING_POLICY_KEYS = getMessagingPolicyKeysByChannel({ agent: "hermes" }); const PROC_PATH = "/proc"; -const JETSON_GPU_LIBRARY_PATH = "/opt/nvidia/l4t-gpu-libs"; const PROC_COMM_READ_WRITE_PATHS = ["/proc/self/comm", "/proc/self/task/*/comm"]; const SYSFS_PATH = "/sys"; const PCI_BDF_PATTERN = /^[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]$/iu; @@ -79,8 +77,6 @@ function deduplicateDirectGpuSysfsEntries( type DirectGpuPolicyOptions = { procReadWrite?: boolean; sysfsReadOnlyPaths?: readonly string[]; - jetsonGpu?: boolean; - jetsonGpuDevicePaths?: readonly string[]; }; export { isStationGb300ProductName }; @@ -227,31 +223,6 @@ export function buildDirectGpuPolicyYaml( } } } - if ( - options.jetsonGpu && - !fsPolicy.read_only.includes(JETSON_GPU_LIBRARY_PATH) && - !fsPolicy.read_write.includes(JETSON_GPU_LIBRARY_PATH) - ) { - // OpenRM runtime injection places libcuda outside the base /usr and /lib - // grants. CUDA cannot load the driver unless Landlock permits this path. - fsPolicy.read_only.push(JETSON_GPU_LIBRARY_PATH); - } - if (options.jetsonGpu) { - // OpenShell v0.0.85 enriches Landlock for /dev/nvidia* and /dev/dxg but - // does not recognize Jetson /dev/nvmap or /dev/nvhost-* devices. The - // compatibility route therefore grants only measured host character - // devices; group membership alone cannot bypass Landlock. Remove this - // grant when the minimum supported OpenShell release enriches Jetson - // devices before applying the sandbox filesystem policy (#7610). - const jetsonGpuDevicePaths = [...new Set(options.jetsonGpuDevicePaths ?? [])]; - const jetsonGpuDevicePathSet = new Set(jetsonGpuDevicePaths); - fsPolicy.read_only = fsPolicy.read_only.filter( - (entry: string) => !jetsonGpuDevicePathSet.has(entry), - ); - for (const devicePath of jetsonGpuDevicePaths) { - if (!fsPolicy.read_write.includes(devicePath)) fsPolicy.read_write.push(devicePath); - } - } if (options.procReadWrite && !fsPolicy.read_write.includes(PROC_PATH)) { // This exists only for the legacy post-create Docker GPU compatibility // path, which recreates the container after `openshell sandbox create` and @@ -410,8 +381,6 @@ export function prepareInitialSandboxCreatePolicy( options: { directGpu?: boolean; dockerGpuPatch?: boolean; - jetsonGpu?: boolean; - jetsonGpuDevicePaths?: readonly string[]; hostGpuAvailable?: boolean; stationGb300SysfsReadOnlyPaths?: readonly string[]; additionalPresets?: string[]; @@ -423,10 +392,6 @@ export function prepareInitialSandboxCreatePolicy( const directGpuPolicy = options.directGpu ? prepareDirectGpuSandboxPolicy(basePolicyPath, { procReadWrite: options.dockerGpuPatch === true, - jetsonGpu: options.jetsonGpu === true, - jetsonGpuDevicePaths: - options.jetsonGpuDevicePaths ?? - (options.jetsonGpu === true ? detectTegraGpuDevicePaths() : []), sysfsReadOnlyPaths: options.stationGb300SysfsReadOnlyPaths ?? discoverHostStationGb300SysfsReadOnlyPaths({ diff --git a/src/lib/onboard/managed-bootstrap/docker-runtime.test.ts b/src/lib/onboard/managed-bootstrap/docker-runtime.test.ts index c9fc5041a8..81d57f8f63 100644 --- a/src/lib/onboard/managed-bootstrap/docker-runtime.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker-runtime.test.ts @@ -12,9 +12,6 @@ const adapterMocks = vi.hoisted(() => ({ finalize: vi.fn(), prepare: vi.fn(), })); -const jetsonMocks = vi.hoisted(() => ({ - detectDeviceGroupGids: vi.fn<() => string[]>(), -})); vi.mock("./adapter", async (importOriginal) => ({ ...(await importOriginal()), @@ -22,28 +19,6 @@ vi.mock("./adapter", async (importOriginal) => ({ finalizeManagedBootstrapSequence: adapterMocks.finalize, prepareManagedBootstrapSequence: adapterMocks.prepare, })); -vi.mock("../docker-gpu-jetson-groups", async (importOriginal) => ({ - ...(await importOriginal()), - detectTegraDeviceGroupGids: jetsonMocks.detectDeviceGroupGids, -})); -vi.mock("../docker-gpu-patch-mode", async (importOriginal) => { - const original = await importOriginal(); - return { - ...original, - selectDockerGpuPatchMode: (options: { backend?: "generic" | "jetson" }) => ({ - mode: original.buildDockerGpuMode( - options.backend === "jetson" ? "nvidia-runtime" : "gpus", - "all", - { backend: options.backend }, - ), - attempts: [], - }), - }; -}); -vi.mock("../docker-gpu-sandbox-create", async (importOriginal) => ({ - ...(await importOriginal()), - isDockerDesktopWslRuntime: () => false, -})); import type { ManagedBootstrapActivatedTransaction, @@ -54,90 +29,9 @@ import { authority, IDENTITY, NEW_ID, OLD_ID } from "./docker-test-fixture"; beforeEach(() => { vi.clearAllMocks(); - jetsonMocks.detectDeviceGroupGids.mockReturnValue(["44", "993"]); }); -async function replacementOptionsFor( - agent: "openclaw" | "hermes", - hostGpuPlatform: "jetson" | "linux", -) { - const seed = authority(agent); - const prepared = Object.freeze({}) as ManagedBootstrapPreparedTransaction; - const activated = Object.freeze({ - snapshot: { runtimeId: OLD_ID }, - replacement: { replacementRuntimeId: NEW_ID }, - }) as ManagedBootstrapActivatedTransaction; - adapterMocks.prepare.mockImplementationOnce(async (_adapter, input) => { - await input.create.launch({ - heldWorkloadArgv: seed.handle.heldWorkloadArgv, - bootstrapIdentity: IDENTITY, - }); - return prepared; - }); - adapterMocks.activate.mockResolvedValueOnce(activated); - const lifecycle = createDockerManagedBootstrapSurface().createLifecycle({ - providerId: "docker", - bootstrapIdentity: IDENTITY, - request: seed.request, - image: seed.plan.image, - agentIdentity: seed.plan.agentIdentity, - intendedWorkloadArgv: seed.plan.intendedWorkloadArgv, - expectedSupervisorArgv: seed.plan.expectedSupervisorArgv, - launchArgv: ["openshell", "sandbox", "create", "--name", "alpha"], - heldWorkloadArgv: seed.handle.heldWorkloadArgv, - authorityStore: { recordPreparedAuthority: vi.fn() }, - adapterOverride: {} as ManagedBootstrapAdapter, - route: "compatibility", - persistStartupCommand: true, - sandboxName: "alpha", - sandboxGpuConfig: { - mode: "1", - hostGpuDetected: true, - hostGpuPlatform, - sandboxGpuEnabled: true, - sandboxGpuDevice: "all", - errors: [], - }, - requiredLimits: [], - timeoutSecs: 30, - network: { - inferenceProvider: "ollama-local", - gatewayUsesContainerBridge: false, - gatewayPort: 0, - }, - dependencies: {}, - }); - - await lifecycle.runCreate(async () => ({ - value: "launched", - receipt: seed.handle.createReceipt, - })); - const prepareInput = adapterMocks.prepare.mock.calls.at(-1)?.[1]; - expect(prepareInput).toBeDefined(); - return prepareInput!.replacementOptions.values; -} - describe("Docker managed-bootstrap lifecycle composition", () => { - it("preserves detected Jetson groups for OpenClaw compatibility replacement (#7610)", async () => { - const values = await replacementOptionsFor("openclaw", "jetson"); - - expect(values).toMatchObject({ - extraGroupGids: ["44", "993"], - gpuModeKind: "nvidia-runtime", - preserveJetsonDeviceGroupMembership: true, - }); - }); - - it.each([ - ["openclaw", "linux", []], - ["hermes", "jetson", ["44", "993"]], - ] as const)("does not enable Jetson group preservation for %s on %s (#7610)", async (agent, platform, expectedGroupGids) => { - const values = await replacementOptionsFor(agent, platform); - - expect(values.extraGroupGids).toEqual(expectedGroupGids); - expect(values.preserveJetsonDeviceGroupMembership).toBe(false); - }); - it("does not finalize rollback after a claimed commit loses acknowledgement", async () => { const stateRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-runtime-")); const seed = authority("openclaw"); diff --git a/src/lib/onboard/managed-bootstrap/docker-runtime.ts b/src/lib/onboard/managed-bootstrap/docker-runtime.ts index 059b4cd5b5..8fdfb1ecc3 100644 --- a/src/lib/onboard/managed-bootstrap/docker-runtime.ts +++ b/src/lib/onboard/managed-bootstrap/docker-runtime.ts @@ -44,8 +44,6 @@ function dockerReplacementOptions( input: ManagedBootstrapRuntimeCreateLifecycleInput, ) { const backend = input.sandboxGpuConfig.hostGpuPlatform === "jetson" ? "jetson" : "generic"; - const extraGroupGids = - backend === "jetson" && input.route === "compatibility" ? detectTegraDeviceGroupGids() : []; return { values: { gpuModeArgs: [...mode.args], @@ -55,9 +53,8 @@ function dockerReplacementOptions( requiredUlimits: input.requiredLimits.map( (limit) => `${limit.name}=${limit.soft}:${limit.hard}`, ), - extraGroupGids, - preserveJetsonDeviceGroupMembership: - extraGroupGids.length > 0 && input.request.agent === "openclaw", + extraGroupGids: + backend === "jetson" && input.route === "compatibility" ? detectTegraDeviceGroupGids() : [], }, }; } diff --git a/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts b/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts index 47d84319ce..4cb3672d02 100644 --- a/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts +++ b/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts @@ -354,13 +354,9 @@ export function fixture(options: DockerFixtureOptions = {}) { const name = String(args[args.indexOf("--name") + 1] ?? ""); const entrypoint = String(args[args.indexOf("--entrypoint") + 1] ?? ""); const imageIndex = args.indexOf(IMAGE); - const flagValues = (flag: string) => - args.flatMap((value, index) => (value === flag ? [String(args[index + 1] ?? "")] : [])); - const env = flagValues("--env"); - const groupAdds = flagValues("--group-add"); - const capAdds = flagValues("--cap-add"); - const securityOptions = flagValues("--security-opt"); - const runtime = flagValues("--runtime").at(-1); + const env = args.flatMap((value, index) => + value === "--env" ? [String(args[index + 1] ?? "")] : [], + ); replacement = { ...structuredClone(source), Id: NEW_ID, @@ -372,13 +368,6 @@ export function fixture(options: DockerFixtureOptions = {}) { Entrypoint: [entrypoint], Cmd: args.slice(imageIndex + 1), }, - HostConfig: { - ...structuredClone(source.HostConfig), - ...(capAdds.length > 0 ? { CapAdd: capAdds } : {}), - ...(groupAdds.length > 0 ? { GroupAdd: groupAdds } : {}), - ...(runtime ? { Runtime: runtime } : {}), - ...(securityOptions.length > 0 ? { SecurityOpt: securityOptions } : {}), - }, State: { Running: false, Paused: false, Restarting: false, Dead: false }, }; return losesAcknowledgement("container:create") diff --git a/src/lib/onboard/managed-bootstrap/docker.test.ts b/src/lib/onboard/managed-bootstrap/docker.test.ts index b6627ec173..3a3d6f7c6a 100644 --- a/src/lib/onboard/managed-bootstrap/docker.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker.test.ts @@ -8,10 +8,7 @@ import { ManagedBootstrapDurableCommitCleanupPendingError, ManagedBootstrapOwnerCleanupRequiredError, } from "./adapter"; -import { - createDockerManagedBootstrapAdapter, - MANAGED_BOOTSTRAP_TRAMPOLINE_EXECUTABLE, -} from "./docker"; +import { createDockerManagedBootstrapAdapter } from "./docker"; import { normalizeDockerManagedBootstrapLaunchSpec, parseDockerManagedBootstrapLaunchSpec, @@ -35,94 +32,6 @@ function expectEventBefore(events: readonly string[], before: string, after: str } describe("Docker managed bootstrap adapter", () => { - it("passes Jetson groups to Docker and the sandbox-account bootstrap without replacing the managed entrypoint (#7610)", async () => { - const fake = fixture({ agent: "openclaw" }); - const adapter = createDockerManagedBootstrapAdapter(fake.deps); - const { handle, request, snapshot } = authority("openclaw"); - - await adapter.prepareBootstrapReplacement({ - handle, - snapshot, - request, - replacementOptions: { - values: { - gpuModeArgs: [ - "--runtime", - "nvidia", - "--env", - "NVIDIA_VISIBLE_DEVICES=all", - "--env", - "NVIDIA_DRIVER_CAPABILITIES=compute,utility", - ], - gpuModeDevice: "all", - gpuModeKind: "nvidia-runtime", - gpuModeLabel: "--runtime nvidia (NVIDIA_VISIBLE_DEVICES=all)", - extraGroupGids: ["44", "993"], - preserveJetsonDeviceGroupMembership: true, - }, - }, - }); - - const createArgs = vi - .mocked(fake.deps.dockerRun!) - .mock.calls.find(([args]) => args[0] === "create")?.[0]; - expect(createArgs).toEqual( - expect.arrayContaining([ - "--group-add", - "44", - "--group-add", - "993", - "--env", - "NEMOCLAW_JETSON_DEVICE_GROUP_GIDS=44,993", - "--entrypoint", - MANAGED_BOOTSTRAP_TRAMPOLINE_EXECUTABLE, - ]), - ); - expect(fake.replacement?.Config?.Entrypoint).toEqual([MANAGED_BOOTSTRAP_TRAMPOLINE_EXECUTABLE]); - expect(fake.replacement?.Config?.Cmd).toEqual( - expect.arrayContaining(["--agent", "openclaw", "--bootstrap-identity", IDENTITY]), - ); - }); - - it("rejects a non-boolean managed Jetson group-preservation option (#7610)", async () => { - const fake = fixture({ agent: "openclaw" }); - const adapter = createDockerManagedBootstrapAdapter(fake.deps); - const { handle, request, snapshot } = authority("openclaw"); - - await expect( - adapter.prepareBootstrapReplacement({ - handle, - snapshot, - request, - replacementOptions: { - values: { preserveJetsonDeviceGroupMembership: "true" }, - }, - }), - ).rejects.toThrow("Jetson device-group preservation must be a boolean"); - expect(fake.replacement).toBeNull(); - }); - - it("rejects managed Jetson group preservation for a non-OpenClaw agent (#7610)", async () => { - const fake = fixture({ agent: "hermes" }); - const adapter = createDockerManagedBootstrapAdapter(fake.deps); - const { handle, request, snapshot } = authority("hermes"); - - await expect( - adapter.prepareBootstrapReplacement({ - handle, - snapshot, - request, - replacementOptions: { - values: { - extraGroupGids: ["44"], - preserveJetsonDeviceGroupMembership: true, - }, - }, - }), - ).rejects.toThrow("Jetson device-group preservation requires the OpenClaw agent"); - expect(fake.replacement).toBeNull(); - }); - it("captures the live OpenShell idle supervisor with a separately persisted bootstrap identity", async () => { const fake = fixture(); const adapter = createDockerManagedBootstrapAdapter(fake.deps); diff --git a/src/lib/onboard/managed-bootstrap/docker.ts b/src/lib/onboard/managed-bootstrap/docker.ts index ad307346cd..7405fbd511 100644 --- a/src/lib/onboard/managed-bootstrap/docker.ts +++ b/src/lib/onboard/managed-bootstrap/docker.ts @@ -650,7 +650,6 @@ function replacementPlan(options: ManagedBootstrapReplacementOptions): { readonly mode: DockerGpuPatchMode; readonly requiredUlimits: readonly DockerUlimit[]; readonly extraGroupGids: readonly string[]; - readonly preserveJetsonDeviceGroupMembership: boolean; } { const allowed = new Set([ "gpuModeArgs", @@ -658,7 +657,6 @@ function replacementPlan(options: ManagedBootstrapReplacementOptions): { "gpuModeKind", "gpuModeLabel", "extraGroupGids", - "preserveJetsonDeviceGroupMembership", "requiredUlimits", ]); const unknown = Object.keys(options.values).filter((key) => !allowed.has(key)); @@ -672,11 +670,6 @@ function replacementPlan(options: ManagedBootstrapReplacementOptions): { throw new Error(`Managed bootstrap Docker GPU mode '${kind}' is invalid.`); } const args = exactStringArray(options.values.gpuModeArgs ?? [], "GPU mode arguments"); - const preserveJetsonDeviceGroupMembership = - options.values.preserveJetsonDeviceGroupMembership ?? false; - if (typeof preserveJetsonDeviceGroupMembership !== "boolean") { - throw new Error("Managed bootstrap Docker Jetson device-group preservation must be a boolean."); - } return { mode: { kind, @@ -692,7 +685,6 @@ function replacementPlan(options: ManagedBootstrapReplacementOptions): { return value; }, ), - preserveJetsonDeviceGroupMembership, requiredUlimits: parseRequiredUlimits(options.values.requiredUlimits), }; } @@ -837,34 +829,20 @@ function canonicalEnvironmentBindings(value: unknown, label: string): string[] { function assertExactEnvironmentDelta( original: Record, replacement: Record, - plan: { - readonly mode: DockerGpuPatchMode; - readonly extraGroupGids: readonly string[]; - readonly preserveJetsonDeviceGroupMembership: boolean; - }, + mode: DockerGpuPatchMode, intendedSandboxCommand: string, ): void { - const { mode } = plan; const gpuAugment = mode.kind !== "startup-command"; const originalEnv = exactStringArray(original.Env ?? [], "original environment"); const expected = [ ...modeEnvironment(mode), ...originalEnv - .filter((entry) => { - const key = entry.split("=", 1)[0] ?? ""; - return ( - key !== "NEMOCLAW_JETSON_DEVICE_GROUP_GIDS" && - (!gpuAugment || !REPLACED_GPU_ENV_KEYS.has(key)) - ); - }) + .filter((entry) => !gpuAugment || !REPLACED_GPU_ENV_KEYS.has(entry.split("=", 1)[0] ?? "")) .map((entry) => entry.startsWith("OPENSHELL_SANDBOX_COMMAND=") ? `OPENSHELL_SANDBOX_COMMAND=${intendedSandboxCommand}` : entry, ), - ...(plan.preserveJetsonDeviceGroupMembership && plan.extraGroupGids.length > 0 - ? [`NEMOCLAW_JETSON_DEVICE_GROUP_GIDS=${plan.extraGroupGids.join(",")}`] - : []), ]; const observed = canonicalEnvironmentBindings(replacement.Env ?? [], "replacement environment"); if ( @@ -988,7 +966,7 @@ function scrubVerifiedReplacementDeltas(canonicalJson: string): string { config.Entrypoint = [""]; config.Cmd = [""]; config.Env = ""; - const verifiedHostKeys = [ + for (const key of [ "CapAdd", "DeviceRequests", "Devices", @@ -996,9 +974,7 @@ function scrubVerifiedReplacementDeltas(canonicalJson: string): string { "Runtime", "SecurityOpt", "Ulimits", - ] as const; - for (const key of verifiedHostKeys) delete host[key]; - for (const key of verifiedHostKeys) { + ]) { host[key] = ``; } return JSON.stringify(root); @@ -1038,7 +1014,6 @@ function assertReplacementMatchesIntent( readonly mode: DockerGpuPatchMode; readonly requiredUlimits: readonly DockerUlimit[]; readonly extraGroupGids: readonly string[]; - readonly preserveJetsonDeviceGroupMembership: boolean; }, intendedSandboxCommand: string, ): string { @@ -1055,7 +1030,7 @@ function assertReplacementMatchesIntent( const observedConfig = objectField(observedInspect, "Config"); const observedHost = objectField(observedInspect, "HostConfig"); const gpuAugment = plan.mode.kind !== "startup-command"; - assertExactEnvironmentDelta(originalConfig, observedConfig, plan, intendedSandboxCommand); + assertExactEnvironmentDelta(originalConfig, observedConfig, plan.mode, intendedSandboxCommand); const originalCapabilities = capabilitySet(originalHost.CapAdd, "original capability additions"); assertExactCapabilitySet( observedHost.CapAdd, @@ -3266,16 +3241,6 @@ export function createDockerManagedBootstrapAdapter( ); } const plan = replacementPlan(replacementOptions); - if ( - plan.preserveJetsonDeviceGroupMembership && - (request.agent !== "openclaw" || plan.extraGroupGids.length === 0) - ) { - throw new Error( - request.agent !== "openclaw" - ? "Managed bootstrap Docker Jetson device-group preservation requires the OpenClaw agent." - : "Managed bootstrap Docker Jetson device-group preservation requires device groups.", - ); - } const originalName = dockerContainerName(parsed.inspect); const backupContainerName = backupName(originalName, handle.bootstrapIdentity); const stagingName = replacementStagingName(originalName, handle.bootstrapIdentity); @@ -3294,7 +3259,6 @@ export function createDockerManagedBootstrapAdapter( openshellSandboxCommand: handle.intendedWorkloadArgv, requiredUlimits: plan.requiredUlimits, extraGroupGids: plan.extraGroupGids, - preserveJetsonDeviceGroupMembership: plan.preserveJetsonDeviceGroupMembership, containerEntrypoint: MANAGED_BOOTSTRAP_TRAMPOLINE_EXECUTABLE, containerCommand: trampolineCommand, containerName: stagingName, diff --git a/src/lib/onboard/sandbox-create-intent-types.ts b/src/lib/onboard/sandbox-create-intent-types.ts index c0e60709ca..4f875420f0 100644 --- a/src/lib/onboard/sandbox-create-intent-types.ts +++ b/src/lib/onboard/sandbox-create-intent-types.ts @@ -24,7 +24,6 @@ export type SandboxCreatePolicyRequest = { readonly activeMessagingChannels: readonly string[]; readonly options: { readonly directGpu: boolean; - readonly jetsonGpu?: boolean; readonly hostGpuAvailable?: boolean; readonly additionalPresets: readonly string[]; readonly agentName?: string | null; diff --git a/src/lib/onboard/sandbox-create-intent.ts b/src/lib/onboard/sandbox-create-intent.ts index 9aa1c1a976..ca4dd64adf 100644 --- a/src/lib/onboard/sandbox-create-intent.ts +++ b/src/lib/onboard/sandbox-create-intent.ts @@ -171,10 +171,6 @@ export function resolveSandboxCreateIntent({ ); const normalizedInferenceProvider = inferenceProvider?.trim() || null; - const openclawJetsonGpu = - sandboxGpuConfig.sandboxGpuEnabled && - sandboxGpuConfig.hostGpuPlatform === "jetson" && - (agentName === undefined || agentName === null || agentName === "openclaw"); return { sandboxName, @@ -190,7 +186,6 @@ export function resolveSandboxCreateIntent({ activeMessagingChannels: [...activeMessagingChannels], options: { directGpu: sandboxGpuConfig.sandboxGpuEnabled, - ...(openclawJetsonGpu ? { jetsonGpu: true } : {}), ...(sandboxGpuConfig.hostGpuDetected !== undefined ? { hostGpuAvailable: sandboxGpuConfig.hostGpuDetected } : {}), diff --git a/src/lib/onboard/sandbox-create-plan-materialization.ts b/src/lib/onboard/sandbox-create-plan-materialization.ts index 3fc28743c2..5722290dd0 100644 --- a/src/lib/onboard/sandbox-create-plan-materialization.ts +++ b/src/lib/onboard/sandbox-create-plan-materialization.ts @@ -144,7 +144,6 @@ export function materializeSandboxCreatePlan({ [...intent.policy.activeMessagingChannels], { directGpu: intent.policy.options.directGpu, - jetsonGpu: intent.policy.options.jetsonGpu, hostGpuAvailable: intent.policy.options.hostGpuAvailable, additionalPresets: [...intent.policy.options.additionalPresets], agentName: intent.policy.options.agentName, diff --git a/src/lib/onboard/sandbox-create-plan.test.ts b/src/lib/onboard/sandbox-create-plan.test.ts index 8a12b95ab8..cd14649c0a 100644 --- a/src/lib/onboard/sandbox-create-plan.test.ts +++ b/src/lib/onboard/sandbox-create-plan.test.ts @@ -193,19 +193,8 @@ describe("resolveSandboxCreateIntent", () => { const first = resolveSandboxCreateIntent(input); const second = resolveSandboxCreateIntent(input); - const openclawJetson = resolveSandboxCreateIntent({ - ...input, - agentName: "openclaw", - sandboxGpuConfig: { ...sandboxGpuConfig, hostGpuPlatform: "jetson" }, - }); - const hermesJetson = resolveSandboxCreateIntent({ - ...input, - sandboxGpuConfig: { ...sandboxGpuConfig, hostGpuPlatform: "jetson" }, - }); expect(first).toEqual(second); - expect(openclawJetson.policy.options.jetsonGpu).toBe(true); - expect(hermesJetson.policy.options.jetsonGpu).toBeUndefined(); expect(first.activeMessagingChannels).toEqual(["telegram", "discord", "whatsapp"]); expect(first.messagingProviderRequests.map(({ name }) => name)).toEqual([ "sandbox-telegram-bridge", diff --git a/src/lib/onboard/sandbox-gpu-create-flow.test.ts b/src/lib/onboard/sandbox-gpu-create-flow.test.ts index 275333d778..fca980de2e 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.test.ts @@ -169,59 +169,6 @@ function createSourceInput(): SandboxGpuCreateFlowInput { beforeEach(() => setupGpuFlowMocks(mocks)); afterEach(resetGpuFlowMocks); -describe("runSandboxGpuCreateFlow Jetson host access", () => { - it("repairs OpenClaw nvmap access before starting a sandbox attempt (#7610)", async () => { - const input = createInput(); - input.agentName = "openclaw"; - input.sandboxGpuConfig.hostGpuPlatform = "jetson"; - const deps = createDeps(); - deps.ensureJetsonNvmapGroupAccess = vi.fn(); - - await expect(runSandboxGpuCreateFlow(input, deps)).resolves.toMatchObject({ route: "native" }); - - expect(deps.ensureJetsonNvmapGroupAccess).toHaveBeenCalledOnce(); - expect(vi.mocked(deps.ensureJetsonNvmapGroupAccess).mock.invocationCallOrder[0]).toBeLessThan( - mocks.streamSandboxCreate.mock.invocationCallOrder[0]!, - ); - }); - - it("stops before sandbox creation when OpenClaw nvmap access cannot be repaired (#7610)", async () => { - const input = createInput(); - input.agentName = "openclaw"; - input.sandboxGpuConfig.hostGpuPlatform = "jetson"; - const deps = createDeps(); - deps.ensureJetsonNvmapGroupAccess = vi.fn(() => { - throw new Error("nvmap host repair failed"); - }); - - await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow("nvmap host repair failed"); - - expect(mocks.streamSandboxCreate).not.toHaveBeenCalled(); - expect(mocks.createDockerGpuSandboxCreatePatch).not.toHaveBeenCalled(); - }); - - it.each([ - { agentName: "hermes", hostGpuPlatform: "jetson", sandboxGpuEnabled: true }, - { agentName: "openclaw", hostGpuPlatform: null, sandboxGpuEnabled: true }, - { agentName: "openclaw", hostGpuPlatform: "jetson", sandboxGpuEnabled: false }, - ] as const)("does not change nvmap access for agent=$agentName platform=$hostGpuPlatform enabled=$sandboxGpuEnabled (#7610)", async ({ - agentName, - hostGpuPlatform, - sandboxGpuEnabled, - }) => { - const input = createInput(); - input.agentName = agentName; - input.sandboxGpuConfig.hostGpuPlatform = hostGpuPlatform; - input.sandboxGpuConfig.sandboxGpuEnabled = sandboxGpuEnabled; - const deps = createDeps(); - deps.ensureJetsonNvmapGroupAccess = vi.fn(); - - await runSandboxGpuCreateFlow(input, deps); - - expect(deps.ensureJetsonNvmapGroupAccess).not.toHaveBeenCalled(); - }); -}); - describe("runSandboxGpuCreateFlow provider-owned managed create", () => { it("recovers before an MXC-style create without a Docker branch in central orchestration", async () => { const input = createInput(); diff --git a/src/lib/onboard/sandbox-gpu-create-flow.ts b/src/lib/onboard/sandbox-gpu-create-flow.ts index d002a30055..a46fce580e 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.ts @@ -4,7 +4,6 @@ import type { StreamSandboxCreateResult } from "../sandbox/create-stream"; import { redactFull } from "../security/redact"; import type { SandboxGpuProofResult } from "../state/registry"; -import { ensureJetsonNvmapGroupAccess } from "./docker-gpu-jetson-groups"; import * as dockerGpuLocalInference from "./docker-gpu-local-inference"; import { collectDockerGpuPatchDiagnostics } from "./docker-gpu-patch"; import type { DockerGpuPatchDeps, DockerUlimit } from "./docker-gpu-patch-types"; @@ -69,7 +68,6 @@ type Sleep = NonNullable; export interface SandboxGpuCreateFlowInput { sandboxName: string; - agentName?: string; provider: string; sandboxGpuConfig: SandboxGpuConfig; gpuRoutePlan: import("./docker-gpu-route").DockerGpuRoutePlan; @@ -107,7 +105,6 @@ export interface SandboxGpuCreateFlowDeps { sleep: Sleep; openshellArgv(args: string[]): string[]; verifyDirectSandboxGpu(sandboxName: string): SandboxGpuProofResult; - ensureJetsonNvmapGroupAccess?: () => void; /** Production callers omit this factory and use the runtime provider's adapter. */ createManagedBootstrapAdapter?: () => ManagedBootstrapAdapter; } @@ -138,13 +135,6 @@ export async function runSandboxGpuCreateFlow( deps: SandboxGpuCreateFlowDeps, ): Promise { let registryImageRef: string | null = input.prebuild.imageRef; - if ( - input.sandboxGpuConfig.sandboxGpuEnabled && - input.sandboxGpuConfig.hostGpuPlatform === "jetson" && - (input.agentName ?? "openclaw") === "openclaw" - ) { - (deps.ensureJetsonNvmapGroupAccess ?? ensureJetsonNvmapGroupAccess)(); - } const attemptRunner = createSandboxGpuCreateAttemptRunner(input, deps); const gpuCreateOutcome = await sandboxGpuCreateAttempt .executeSandboxGpuCreatePlan(input.gpuRoutePlan, { diff --git a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts index 3112e5b8bc..4d0f046e8f 100644 --- a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts +++ b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts @@ -138,7 +138,6 @@ export function createSandboxGpuCreateAttemptRunner( input.persistStartupCommand === true && (route !== "native" || hasRequiredUlimits), externalRecreation: false, sandboxName: input.sandboxName, - agentName: input.agentName, gpuDevice: input.sandboxGpuConfig.sandboxGpuDevice, openshellSandboxCommand: input.sandboxStartupCommand, requiredUlimits: input.requiredUlimits, diff --git a/src/lib/onboard/sandbox-gpu-create.ts b/src/lib/onboard/sandbox-gpu-create.ts index 451b33695b..56ee5687e8 100644 --- a/src/lib/onboard/sandbox-gpu-create.ts +++ b/src/lib/onboard/sandbox-gpu-create.ts @@ -7,7 +7,6 @@ export type SandboxGpuCreateConfig = { sandboxGpuEnabled: boolean; sandboxGpuDevice?: string | null; hostGpuDetected?: boolean; - hostGpuPlatform?: string | null; }; export function buildSandboxGpuCreateArgs( diff --git a/src/lib/onboard/sandbox-gpu-preflight-routing.test.ts b/src/lib/onboard/sandbox-gpu-preflight-routing.test.ts index ccc749f8f3..9a1e70c3f2 100644 --- a/src/lib/onboard/sandbox-gpu-preflight-routing.test.ts +++ b/src/lib/onboard/sandbox-gpu-preflight-routing.test.ts @@ -11,7 +11,6 @@ import type { SandboxGpuConfig } from "./sandbox-gpu-mode"; import { dockerNvidiaRuntimeAvailable, formatSandboxGpuPassthroughNote, - jetsonGpuProofRemediationLines, parseDockerRuntimeNames, sandboxGpuRemediationLines, validateSandboxGpuPreflight, @@ -29,17 +28,6 @@ function sandboxGpuConfig(overrides: Partial = {}): SandboxGpu }; } describe("sandbox GPU preflight routing", () => { - it("describes every Jetson device-access boundary after CUDA proof failure (#7610)", () => { - const remediation = jetsonGpuProofRemediationLines().join("\n"); - - expect(remediation).toContain("owning-group read-write access"); - expect(remediation).toContain("OpenShell Landlock policy"); - expect(remediation).toContain("their GIDs through Docker"); - expect(remediation).toContain("preserve sandbox-account membership"); - expect(remediation).not.toContain("add the host video/render groups via --group-add"); - expect(remediation).not.toContain("must be readable by the sandbox"); - }); - it("formats Jetson sandbox GPU notes around the NVIDIA runtime backend", () => { expect(formatSandboxGpuPassthroughNote({ hostGpuPlatform: "jetson" })).toContain( "Docker NVIDIA runtime", diff --git a/src/lib/onboard/sandbox-gpu-preflight.ts b/src/lib/onboard/sandbox-gpu-preflight.ts index da44bc4efe..5fbbde0fe0 100644 --- a/src/lib/onboard/sandbox-gpu-preflight.ts +++ b/src/lib/onboard/sandbox-gpu-preflight.ts @@ -56,18 +56,18 @@ export function resolveSandboxGpuFlagFromOptions(opts: SandboxGpuFlagOptions): S return null; } -// Jetson/Tegra CUDA failures are usually device-access issues rather than -// CDI/runtime selection failures. Surface the complete host mode, Landlock, -// Docker group, and sandbox-account boundary rather than an incomplete manual -// --group-add workaround or a bare "enabled" status (#4231, #7610). +// Jetson/Tegra CUDA failures are usually device/group permission issues rather +// than CDI/runtime misconfiguration: the sandbox sees the GPU but the agent +// user lacks access to the Tegra device nodes. Surface the concrete devices and +// groups so the user can fix the recreate rather than seeing a bare "enabled" +// status that hides an unusable GPU (#4231). export function jetsonGpuProofRemediationLines(): string[] { return [ "Jetson/Tegra CUDA proof did not pass. CUDA needs access to the Tegra device", - "nodes. NemoClaw must verify host /dev/nvmap owning-group read-write access,", - "grant the detected character devices in the OpenShell Landlock policy, propagate", - "their GIDs through Docker, and preserve sandbox-account membership. Review the", - "onboarding output and saved diagnostics, then retry onboarding;", - "or use NEMOCLAW_SANDBOX_GPU=0 for CPU.", + "nodes; confirm the sandbox propagates them and the agent user's groups:", + " ls -l /dev/nvmap /dev/nvhost-* (must be readable by the sandbox)", + " add the host video/render groups via --group-add when recreating", + "Then recreate the sandbox, or force CPU behavior with NEMOCLAW_SANDBOX_GPU=0.", ]; } diff --git a/src/lib/sandbox/build-context.ts b/src/lib/sandbox/build-context.ts index ecf4a6a6a2..ee01a48420 100644 --- a/src/lib/sandbox/build-context.ts +++ b/src/lib/sandbox/build-context.ts @@ -273,10 +273,6 @@ function stageOptimizedSandboxBuildContext( path.join(rootDir, "scripts", "validate-openclaw-tool-search.mts"), path.join(stagedScriptsDir, "validate-openclaw-tool-search.mts"), ); - fs.copyFileSync( - path.join(rootDir, "scripts", "jetson-device-group-bootstrap.sh"), - path.join(stagedScriptsDir, "jetson-device-group-bootstrap.sh"), - ); // Shared sandbox initialisation library sourced by the entrypoint (#2277) fs.mkdirSync(path.join(stagedScriptsDir, "lib"), { recursive: true }); fs.copyFileSync( diff --git a/test/e2e/live/jetson-nvmap-gpu.test.ts b/test/e2e/live/jetson-nvmap-gpu.test.ts index 71a8f30675..b69de1713a 100644 --- a/test/e2e/live/jetson-nvmap-gpu.test.ts +++ b/test/e2e/live/jetson-nvmap-gpu.test.ts @@ -196,7 +196,7 @@ exit "$status"`, progress.phase("confirm nvmap and NVIDIA Docker runtime"); const hostNvmap = await hostShell( host, - "ls -l /dev/nvmap && stat -c 'gid=%g group=%G mode=%a' /dev/nvmap", + "ls -l /dev/nvmap && stat -c 'gid=%g group=%G' /dev/nvmap", "phase-0-host-nvmap", ); expect(hostNvmap.exitCode, resultText(hostNvmap)).toBe(0); @@ -249,8 +249,10 @@ exit "$status"`, expect(installedCli.exitCode, resultText(installedCli)).toBe(0); expect(installedCli.stdout.trim()).not.toBe(""); - // A4: the Jetson recreate must prepare Tegra device-node groups for the sandbox user. - expect(resultText(install)).toContain("Preparing detected Jetson GPU device groups"); + // A4: the Jetson recreate must grant Tegra device-node groups via --group-add. + expect(resultText(install)).toContain( + "Granting sandbox user the detected Jetson GPU device groups via --group-add", + ); // A5: the sandbox user must be in the host /dev/nvmap owning GID. progress.phase("inspect sandbox nvmap access"); @@ -262,7 +264,7 @@ exit "$status"`, expect(sandboxId.exitCode, resultText(sandboxId)).toBe(0); expectGroupMembership(resultText(sandboxId), hostNvmapGid); - // A6: /dev/nvmap must be present and read-write for the sandbox user. + // A6: /dev/nvmap must be mounted/present inside the sandbox. const sandboxNvmap = await sandbox.execShell( SANDBOX_NAME, trustedSandboxShellScript("ls -l /dev/nvmap"), @@ -271,13 +273,6 @@ exit "$status"`, expect(sandboxNvmap.exitCode, resultText(sandboxNvmap)).toBe(0); expect(resultText(sandboxNvmap)).toContain("/dev/nvmap"); - const sandboxNvmapAccess = await sandbox.execShell( - SANDBOX_NAME, - trustedSandboxShellScript("test -r /dev/nvmap && test -w /dev/nvmap"), - { artifactName: "phase-3-sandbox-nvmap-access", env: env(), timeoutMs: 60_000 }, - ); - expect(sandboxNvmapAccess.exitCode, resultText(sandboxNvmapAccess)).toBe(0); - // A7: authoritative CUDA usability proof must succeed, not reproduce // NvRmMemInitNvmap permission denial / cuInit(0)=999 from #4231. progress.phase("prove CUDA initialization inside the sandbox"); diff --git a/test/managed-bootstrap-trampoline.test.ts b/test/managed-bootstrap-trampoline.test.ts index b5d09347a4..5da0253256 100644 --- a/test/managed-bootstrap-trampoline.test.ts +++ b/test/managed-bootstrap-trampoline.test.ts @@ -603,7 +603,7 @@ exec /usr/bin/env -i NEMOCLAW_MANAGED_BOOTSTRAP_RESUME=1 ${JSON.stringify( it.each( MANAGED_STARTUP_AGENTS, - )("consumes the protected %s request and applies eligible Jetson groups before exact supervisor exec (#7610)", (agent) => { + )("consumes the protected %s request or recovered claim before exact supervisor exec and drops bootstrap variables", (agent) => { const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-bootstrap-trampoline-")); try { const request = path.join(directory, "request.json"); @@ -615,7 +615,6 @@ exec /usr/bin/env -i NEMOCLAW_MANAGED_BOOTSTRAP_RESUME=1 ${JSON.stringify( const trace = path.join(directory, "trace"); const script = path.join(directory, "trampoline.sh"); const supervisor = path.join(directory, "supervisor"); - const jetsonGroupBootstrap = path.join(directory, "jetson-device-group-bootstrap"); const injection = path.join(directory, "injection"); const attackerFunction = path.join(directory, "attacker-function-ran"); fs.mkdirSync(sandbox); @@ -643,14 +642,6 @@ esac path.join(directory, "rm"), '#!/bin/sh\ntest ! -e /proc/self/fd/9\nexec /bin/rm "$@"\n', ); - executable( - jetsonGroupBootstrap, - `#!/bin/sh -test ! -e /proc/self/fd/9 -printf 'groups:%s\\n' "$NEMOCLAW_JETSON_DEVICE_GROUP_GIDS" >>${JSON.stringify(trace)} -exec "$@" -`, - ); executable( path.join(directory, "node"), `#!/bin/sh @@ -688,11 +679,6 @@ test -e ${JSON.stringify(attackerFunction)} test -z "\${NEMOCLAW_MANAGED_BOOTSTRAP_ENTRYPOINT+x}" test -z "\${NEMOCLAW_MANAGED_BOOTSTRAP_RESUME+x}" test -z "\${NEMOCLAW_MANAGED_BOOTSTRAP_RESUME_EXECUTABLE+x}" -${ - agent === "openclaw" - ? 'test "$NEMOCLAW_JETSON_DEVICE_GROUP_GIDS" = "44,993"' - : 'test -z "${NEMOCLAW_JETSON_DEVICE_GROUP_GIDS+x}"' -} printf 'supervisor:%s|%s|%s:identity=%s:request=%s:home=%s:path=%s:lang=%s:capability=%s:bash-env=%s\\n' "$1" "$2" "$3" "\${_nemoclaw_bootstrap_identity-unset}" "\${_nemoclaw_request-unset}" "$HOME" "$PATH" "$LANG" "\${NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION-unset}" "\${BASH_ENV+x}" >>"$TRACE" `, ); @@ -707,10 +693,6 @@ printf 'supervisor:%s|%s|%s:identity=%s:request=%s:home=%s:path=%s:lang=%s:capab ) .replaceAll("/var/lib/nemoclaw-managed-bootstrap-request.json", request) .replaceAll("/sandbox", sandbox) - .replaceAll( - "/usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh", - jetsonGroupBootstrap, - ) .replaceAll("/usr/local/bin/node", path.join(directory, "node")); fs.writeFileSync(script, source, { mode: 0o644 }); fs.chmodSync(script, 0o644); @@ -758,7 +740,6 @@ printf 'supervisor:%s|%s|%s:identity=%s:request=%s:home=%s:path=%s:lang=%s:capab LD_PRELOAD: loader.library, DYLD_INSERT_LIBRARIES: loader.library, LD_AUDIT: loader.library, - ...(agent === "openclaw" ? { NEMOCLAW_JETSON_DEVICE_GROUP_GIDS: "44,993" } : {}), }; execFileSync(entrypoint, argv, { env: environment }); @@ -769,7 +750,6 @@ printf 'supervisor:%s|%s|%s:identity=%s:request=%s:home=%s:path=%s:lang=%s:capab expect(fs.existsSync(loader.earlyTrace)).toBe(false); expect(fs.existsSync(loader.afterTrace)).toBe(true); expect(fs.readFileSync(trace, "utf8").trim().split("\n")).toEqual([ - ...(agent === "openclaw" ? ["groups:44,993"] : []), `node:${runtime} --recover-bootstrap-claim --agent ${agent} --profile-fingerprint ${fingerprint} --bootstrap-identity ${identity}:home=/root:path=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:lang=C.UTF-8:capability=1`, `node:${runtime} --apply-bootstrap-file --agent ${agent} --profile-fingerprint ${fingerprint} --bootstrap-identity ${identity}:home=/root:path=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:lang=C.UTF-8:capability=1`, `node:${runtime} --verify-bootstrap-completion --agent ${agent} --profile-fingerprint ${fingerprint} --bootstrap-identity ${identity}:home=/root:path=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:lang=C.UTF-8:capability=1`, diff --git a/test/openclaw-final-image-layout.test.ts b/test/openclaw-final-image-layout.test.ts index 9bf6856ad4..7dae1c0a0e 100644 --- a/test/openclaw-final-image-layout.test.ts +++ b/test/openclaw-final-image-layout.test.ts @@ -93,7 +93,6 @@ describe("OpenClaw final image layout", () => { "COPY scripts/lib/openclaw_device_approval_policy.py /usr/local/lib/nemoclaw/openclaw_device_approval_policy.py", "COPY scripts/lib/clean_runtime_shell_env_shim.py /usr/local/lib/nemoclaw/clean_runtime_shell_env_shim.py", "COPY scripts/lib/normalize_mutable_config_perms.py /usr/local/lib/nemoclaw/normalize_mutable_config_perms.py", - "COPY scripts/jetson-device-group-bootstrap.sh /usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh", "COPY scripts/state-dir-guard.py /usr/local/lib/nemoclaw/state-dir-guard.py", "COPY agents/openclaw/state-lock-plan.json /usr/local/share/nemoclaw/state-lock-plan.json", "COPY scripts/openclaw-config-guard.py /usr/local/lib/nemoclaw/openclaw-config-guard.py", diff --git a/test/sandbox-build-context.test.ts b/test/sandbox-build-context.test.ts index 5946340e88..b02bedee35 100644 --- a/test/sandbox-build-context.test.ts +++ b/test/sandbox-build-context.test.ts @@ -116,7 +116,6 @@ describe("sandbox build context staging", () => { writeFixture(path.join("scripts", "codex-acp-wrapper.sh")); writeFixture(path.join("scripts", "generate-openclaw-config.mts")); writeFixture(path.join("scripts", "validate-openclaw-tool-search.mts")); - writeFixture(path.join("scripts", "jetson-device-group-bootstrap.sh"), "fixture\n", 0o755); writeFixture( path.join("scripts", "checks", "verify-openshell-policy-boundary-dependencies.mts"), ); @@ -666,9 +665,6 @@ describe("sandbox build context staging", () => { expect( fs.existsSync(path.join(buildCtx, "scripts", "checks", "node-tar-image-scan.mts")), ).toBe(true); - expect( - fs.existsSync(path.join(buildCtx, "scripts", "jetson-device-group-bootstrap.sh")), - ).toBe(true); expect( fs.existsSync(path.join(buildCtx, "scripts", "patch-openclaw-device-self-approval.ts")), ).toBe(false); diff --git a/test/sandbox-provisioning-helper-permissions.test.ts b/test/sandbox-provisioning-helper-permissions.test.ts index 3a761509c9..5f8a36a7a7 100644 --- a/test/sandbox-provisioning-helper-permissions.test.ts +++ b/test/sandbox-provisioning-helper-permissions.test.ts @@ -129,7 +129,6 @@ describe("sandbox provisioning: copied OpenClaw helper permissions (#2861)", () const nestedPluginFile = path.join(nestedPluginDir, "helper.js"); const gatewayControlPath = path.join(localBin, "nemoclaw-gateway-control"); const gatewaySupervisorPath = path.join(localLib, "gateway-supervisor.sh"); - const jetsonGroupBootstrapPath = path.join(localLib, "jetson-device-group-bootstrap.sh"); const stateDirGuardPath = path.join(localLib, "state-dir-guard.py"); const stateLockPlanPath = path.join(localShare, "state-lock-plan.json"); const configGuardPath = path.join(localLib, "openclaw-config-guard.py"); @@ -144,7 +143,6 @@ describe("sandbox provisioning: copied OpenClaw helper permissions (#2861)", () path.join(localLib, "sandbox-init.sh"), path.join(localLib, "sandbox-rlimits.sh"), gatewaySupervisorPath, - jetsonGroupBootstrapPath, stateDirGuardPath, stateLockPlanPath, configGuardPath, @@ -207,7 +205,6 @@ describe("sandbox provisioning: copied OpenClaw helper permissions (#2861)", () expect((fs.statSync(nestedPluginFile).mode & 0o777).toString(8)).toBe("644"); expect((fs.statSync(gatewayControlPath).mode & 0o777).toString(8)).toBe("700"); expect((fs.statSync(gatewaySupervisorPath).mode & 0o777).toString(8)).toBe("444"); - expect((fs.statSync(jetsonGroupBootstrapPath).mode & 0o777).toString(8)).toBe("755"); expect((fs.statSync(stateDirGuardPath).mode & 0o777).toString(8)).toBe("500"); expect((fs.statSync(stateLockPlanPath).mode & 0o777).toString(8)).toBe("444"); expect((fs.statSync(configGuardPath).mode & 0o777).toString(8)).toBe("500"); diff --git a/test/setup-jetson.test.ts b/test/setup-jetson.test.ts index 67aeacfe99..67ea13c5a0 100644 --- a/test/setup-jetson.test.ts +++ b/test/setup-jetson.test.ts @@ -20,12 +20,9 @@ const SCRIPT_PATH = path.join(import.meta.dirname, "..", "scripts", "setup-jetso const HOST_MUTATION_COMMANDS = [ "sudo", - "chmod", "modprobe", - "stat", "sysctl", "tee", - "udevadm", "update-alternatives", "systemctl", "python3", @@ -36,57 +33,21 @@ type SetupJetsonRun = { stdout: string; stderr: string; headArgs: string; - commandLog: string; }; function withJetsonReleaseSandbox( - run: (paths: { - commandLogPath: string; - headArgsPath: string; - releasePath: string; - statCountPath: string; - stubDir: string; - }) => T, + run: (paths: { headArgsPath: string; releasePath: string; stubDir: string }) => T, ): T { const tempDir = mkdtempSync(path.join(tmpdir(), "nemoclaw-jetson-release-")); try { const stubDir = path.join(tempDir, "bin"); - const commandLogPath = path.join(tempDir, "command-log"); const headArgsPath = path.join(tempDir, "head-args"); const releasePath = path.join(tempDir, "nv_tegra_release"); - const statCountPath = path.join(tempDir, "stat-count"); mkdirSync(stubDir); - writeFileSync(commandLogPath, ""); - writeFileSync(headArgsPath, ""); for (const command of HOST_MUTATION_COMMANDS) { const stubPath = path.join(stubDir, command); - writeFileSync( - stubPath, - [ - "#!/usr/bin/env bash", - "set -euo pipefail", - `printf '%s %s\\n' ${JSON.stringify(command)} "$*" >> ${JSON.stringify(commandLogPath)}`, - `if [[ ${JSON.stringify(command)} == "tee" || ( ${JSON.stringify(command)} == "sudo" && "\${1:-}" == "tee" ) ]]; then`, - " input=", - " IFS= read -r input || true", - ` printf 'stdin %s\\n' "$input" >> ${JSON.stringify(commandLogPath)}`, - "fi", - `if [[ ${JSON.stringify(command)} == "stat" ]]; then`, - ` if [[ -f ${JSON.stringify(statCountPath)} ]]; then`, - ' output="${NEMOCLAW_TEST_STAT_OUTPUT_AFTER:-${NEMOCLAW_TEST_STAT_OUTPUT:-}}"', - " else", - ` : > ${JSON.stringify(statCountPath)}`, - ' output="${NEMOCLAW_TEST_STAT_OUTPUT:-}"', - " fi", - ' [[ -n "$output" ]] || exit "${NEMOCLAW_TEST_STAT_STATUS:-1}"', - " printf '%s\\n' \"$output\"", - ' exit "${NEMOCLAW_TEST_STAT_STATUS:-0}"', - "fi", - "exit 0", - "", - ].join("\n"), - ); + writeFileSync(stubPath, "#!/usr/bin/env bash\nexit 0\n"); chmodSync(stubPath, 0o755); } @@ -105,7 +66,7 @@ function withJetsonReleaseSandbox( ); chmodSync(headStubPath, 0o755); - return run({ commandLogPath, headArgsPath, releasePath, statCountPath, stubDir }); + return run({ headArgsPath, releasePath, stubDir }); } finally { rmSync(tempDir, { recursive: true, force: true }); } @@ -114,11 +75,9 @@ function withJetsonReleaseSandbox( function spawnSetupJetson( stubDir: string, headArgsPath: string, - commandLogPath: string, extraEnv: NodeJS.ProcessEnv = {}, - scriptArgs: string[] = [], ): SetupJetsonRun { - const result = spawnSync("bash", [SCRIPT_PATH, ...scriptArgs], { + const result = spawnSync("bash", [SCRIPT_PATH], { encoding: "utf-8", env: { ...process.env, @@ -132,20 +91,19 @@ function spawnSetupJetson( stdout: result.stdout, stderr: result.stderr, headArgs: readFileSync(headArgsPath, "utf-8").trim(), - commandLog: readFileSync(commandLogPath, "utf-8").trim(), }; } function runSetupJetson(releaseLine: string): SetupJetsonRun { - return withJetsonReleaseSandbox(({ commandLogPath, headArgsPath, releasePath, stubDir }) => { + return withJetsonReleaseSandbox(({ headArgsPath, releasePath, stubDir }) => { writeFileSync(releasePath, `${releaseLine}\n`); - return spawnSetupJetson(stubDir, headArgsPath, commandLogPath); + return spawnSetupJetson(stubDir, headArgsPath); }); } function runSetupJetsonWithoutReleaseFile(): SetupJetsonRun { - return withJetsonReleaseSandbox(({ commandLogPath, headArgsPath, stubDir }) => - spawnSetupJetson(stubDir, headArgsPath, commandLogPath), + return withJetsonReleaseSandbox(({ headArgsPath, stubDir }) => + spawnSetupJetson(stubDir, headArgsPath), ); } @@ -340,18 +298,16 @@ describe("setup-jetson host setup on an unrecognized L4T release (#7612)", () => }); it("ignores an inherited test release-path override during normal installation", () => { - const result = withJetsonReleaseSandbox( - ({ commandLogPath, headArgsPath, releasePath, stubDir }) => { - const inheritedOverridePath = path.join(path.dirname(releasePath), "inherited-release"); - writeFileSync( - inheritedOverridePath, - "# R36 (release), REVISION: 5.1, GCID: 12345678, BOARD: t186ref\n", - ); - return spawnSetupJetson(stubDir, headArgsPath, commandLogPath, { - NEMOCLAW_TEST_NV_TEGRA_RELEASE_PATH: inheritedOverridePath, - }); - }, - ); + const result = withJetsonReleaseSandbox(({ headArgsPath, releasePath, stubDir }) => { + const inheritedOverridePath = path.join(path.dirname(releasePath), "inherited-release"); + writeFileSync( + inheritedOverridePath, + "# R36 (release), REVISION: 5.1, GCID: 12345678, BOARD: t186ref\n", + ); + return spawnSetupJetson(stubDir, headArgsPath, { + NEMOCLAW_TEST_NV_TEGRA_RELEASE_PATH: inheritedOverridePath, + }); + }); expect(result.status).toBe(0); expect(result.stdout).toBe(""); @@ -367,163 +323,3 @@ describe("setup-jetson host setup on an unrecognized L4T release (#7612)", () => expect(result.stderr).not.toContain("Skipped Jetson host setup"); }); }); - -describe("setup-jetson OpenClaw nvmap access", () => { - it("repairs nvmap without requiring an L4T release in nvmap-only mode (#7610)", () => { - const result = withJetsonReleaseSandbox(({ commandLogPath, headArgsPath, stubDir }) => - spawnSetupJetson( - stubDir, - headArgsPath, - commandLogPath, - { - NEMOCLAW_TEST_STAT_OUTPUT: "character special file|cr--r-----", - NEMOCLAW_TEST_STAT_OUTPUT_AFTER: "character special file|cr--rw----", - }, - ["--nvmap-only"], - ), - ); - - expect(result.status).toBe(0); - expect(result.headArgs).toBe(""); - expect(result.commandLog).toContain("chmod g+rw /dev/nvmap"); - expect(result.stdout).toContain("/dev/nvmap grants its owning group read-write access"); - }); - - it("grants the nvmap owning group read-write access and persists the mode on JetPack 6 (#7610)", () => { - const result = withJetsonReleaseSandbox( - ({ commandLogPath, headArgsPath, releasePath, stubDir }) => { - writeFileSync( - releasePath, - "# R36 (release), REVISION: 5.1, GCID: 12345678, BOARD: t186ref\n", - ); - return spawnSetupJetson(stubDir, headArgsPath, commandLogPath, { - NEMOCLAW_TEST_STAT_OUTPUT: "character special file|cr--r-----", - NEMOCLAW_TEST_STAT_OUTPUT_AFTER: "character special file|cr--rw----", - }); - }, - ); - - expect(result.status).toBe(0); - expect(result.commandLog).toContain("tee /etc/udev/rules.d/99-zz-nemoclaw-nvmap.rules"); - expect(result.commandLog).toContain('stdin KERNEL=="nvmap", MODE="0660"'); - expect(result.commandLog).toContain("udevadm control --reload-rules"); - expect(result.commandLog).toContain("chmod g+rw /dev/nvmap"); - expect(result.stdout).toContain("/dev/nvmap grants its owning group read-write access"); - expect(result.stdout).toContain("preserves this mode after reboot"); - expect(result.stderr).toContain( - "grants every member of the existing /dev/nvmap owning group write access", - ); - expect(result.stderr).toContain("persists mode 0660 when udev recreates the device"); - }); - - it("rejects a non-device nvmap path before changing host permissions (#7610)", () => { - const result = withJetsonReleaseSandbox( - ({ commandLogPath, headArgsPath, releasePath, stubDir }) => { - writeFileSync( - releasePath, - "# R36 (release), REVISION: 5.1, GCID: 12345678, BOARD: t186ref\n", - ); - return spawnSetupJetson(stubDir, headArgsPath, commandLogPath, { - NEMOCLAW_TEST_STAT_OUTPUT: "regular file|-rw-r-----", - }); - }, - ); - - expect(result.status).toBe(1); - expect(result.stderr).toContain("/dev/nvmap must be a character device"); - expect(result.commandLog).not.toContain("tee /etc/udev/rules.d/99-zz-nemoclaw-nvmap.rules"); - expect(result.commandLog).not.toContain("chmod g+rw /dev/nvmap"); - }); - - it("skips nvmap host changes when the device is absent (#7610)", () => { - const result = withJetsonReleaseSandbox( - ({ commandLogPath, headArgsPath, releasePath, stubDir }) => { - writeFileSync( - releasePath, - "# R36 (release), REVISION: 5.1, GCID: 12345678, BOARD: t186ref\n", - ); - return spawnSetupJetson(stubDir, headArgsPath, commandLogPath); - }, - ); - - expect(result.status).toBe(0); - expect(result.stderr).toContain("could not find /dev/nvmap"); - expect(result.commandLog).not.toContain("tee /etc/udev/rules.d/99-zz-nemoclaw-nvmap.rules"); - expect(result.commandLog).not.toContain("chmod g+rw /dev/nvmap"); - expect(result.stdout).not.toContain("/dev/nvmap grants its owning group read-write access"); - }); - - it("fails when nvmap remains read-only after host setup (#7610)", () => { - const result = withJetsonReleaseSandbox( - ({ commandLogPath, headArgsPath, releasePath, stubDir }) => { - writeFileSync( - releasePath, - "# R36 (release), REVISION: 5.1, GCID: 12345678, BOARD: t186ref\n", - ); - return spawnSetupJetson(stubDir, headArgsPath, commandLogPath, { - NEMOCLAW_TEST_STAT_OUTPUT: "character special file|cr--r-----", - NEMOCLAW_TEST_STAT_OUTPUT_AFTER: "character special file|cr--r-----", - }); - }, - ); - - expect(result.status).toBe(1); - expect(result.stderr).toContain( - "/dev/nvmap does not grant its owning group read-write access after host setup", - ); - expect(result.commandLog).toContain("chmod g+rw /dev/nvmap"); - }); - - it("configures nvmap before skipping version-specific setup for an unrecognized L4T release (#7610)", () => { - const result = withJetsonReleaseSandbox( - ({ commandLogPath, headArgsPath, releasePath, stubDir }) => { - writeFileSync( - releasePath, - "# R00 (release), REVISION: 0.0, GCID: 46579312, BOARD: generic\n", - ); - return spawnSetupJetson(stubDir, headArgsPath, commandLogPath, { - NEMOCLAW_TEST_STAT_OUTPUT: "character special file|cr--r-----", - NEMOCLAW_TEST_STAT_OUTPUT_AFTER: "character special file|cr--rw----", - }); - }, - ); - - expect(result.status).toBe(0); - expect(result.commandLog).toContain("tee /etc/udev/rules.d/99-zz-nemoclaw-nvmap.rules"); - expect(result.commandLog).toContain('stdin KERNEL=="nvmap", MODE="0660"'); - expect(result.commandLog).toContain("chmod g+rw /dev/nvmap"); - expect(result.commandLog).not.toContain("update-alternatives"); - expect(result.commandLog).not.toContain("modprobe br_netfilter"); - expect(result.commandLog).not.toContain("systemctl restart docker"); - expect(result.stdout).toContain("/dev/nvmap grants its owning group read-write access"); - expect(result.stderr).toContain( - "Jetson detected (L4T 00.0) but this L4T release is not recognized.", - ); - expect(result.stderr).toContain("Skipped Jetson host setup"); - expect(result.stderr).toContain("Installation continues in an untested configuration."); - }); - - it.each([ - "hermes", - "langchain-deepagents-code", - ])("does not change nvmap access for the %s agent (#7610)", (agent) => { - const result = withJetsonReleaseSandbox( - ({ commandLogPath, headArgsPath, releasePath, stubDir }) => { - writeFileSync( - releasePath, - "# R36 (release), REVISION: 5.1, GCID: 12345678, BOARD: t186ref\n", - ); - return spawnSetupJetson(stubDir, headArgsPath, commandLogPath, { - NEMOCLAW_AGENT: agent, - NEMOCLAW_TEST_STAT_OUTPUT: "character special file|cr--r-----", - }); - }, - ); - - expect(result.status).toBe(0); - expect(result.commandLog).not.toContain("tee /etc/udev/rules.d/99-zz-nemoclaw-nvmap.rules"); - expect(result.commandLog).not.toContain("chmod g+rw /dev/nvmap"); - expect(result.stdout).not.toContain("/dev/nvmap grants its owning group read-write access"); - expect(result.stderr).not.toContain("/dev/nvmap owning group write access"); - }); -}); From 000e88ecf9dc9874f27b7bae38b84bac2f7823be Mon Sep 17 00:00:00 2001 From: San Dang Date: Thu, 6 Aug 2026 17:32:40 +0700 Subject: [PATCH 20/49] fix(onboard): permit Jetson GPU policy paths Signed-off-by: San Dang --- docs/reference/troubleshooting.mdx | 8 +++ .../onboard/docker-gpu-jetson-groups.test.ts | 32 +++++++++++- src/lib/onboard/docker-gpu-jetson-groups.ts | 38 ++++++++++++++ src/lib/onboard/initial-policy.test.ts | 51 +++++++++++++++++++ src/lib/onboard/initial-policy.ts | 25 +++++++++ 5 files changed, 153 insertions(+), 1 deletion(-) diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 09ab640bd8..8d6406834c 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -2729,6 +2729,14 @@ To skip GPU passthrough entirely, rerun with `--no-gpu` or set `NEMOCLAW_SANDBOX Automatic GPU onboarding uses the compatibility path directly; it does not make a native attempt first. The path recreates the OpenShell-managed Docker container with NVIDIA GPU flags and propagates eligible host group IDs for the supported Jetson GPU device nodes. + + +For OpenClaw direct GPU onboarding, the creation-time filesystem policy adds no Jetson entries unless `/dev/nvmap` is an existing, non-symlink character device. +When that condition is met, the policy adds `/opt/nvidia` as read-only. +It adds each existing, non-symlink character device on the eligible GPU path list as read-write. +Generic GPU and CPU-only policies do not receive these Jetson entries. + + Use `NEMOCLAW_DOCKER_GPU_PATCH=0` only for troubleshooting because it bypasses that group propagation and CUDA may not initialize. #### Common compatibility-path recovery diff --git a/src/lib/onboard/docker-gpu-jetson-groups.test.ts b/src/lib/onboard/docker-gpu-jetson-groups.test.ts index 90b6dfb544..c6cacdd6fe 100644 --- a/src/lib/onboard/docker-gpu-jetson-groups.test.ts +++ b/src/lib/onboard/docker-gpu-jetson-groups.test.ts @@ -5,7 +5,37 @@ import fs from "node:fs"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { detectTegraDeviceGroupGids } from "./docker-gpu-jetson-groups"; +import { detectTegraDeviceGroupGids, detectTegraGpuDevicePaths } from "./docker-gpu-jetson-groups"; + +describe("detectTegraGpuDevicePaths", () => { + it("returns only existing character devices without following symlinks (#7610)", () => { + const paths = ["/dev/nvmap", "/dev/nvhost-gpu", "/dev/nvgpu/igpu0/link"]; + + expect( + detectTegraGpuDevicePaths({ + listDevicePaths: () => paths, + statDevicePath: (devicePath) => + devicePath === "/dev/nvmap" + ? { isCharacterDevice: true, isSymbolicLink: false } + : devicePath === "/dev/nvgpu/igpu0/link" + ? { isCharacterDevice: true, isSymbolicLink: true } + : null, + }), + ).toEqual(["/dev/nvmap"]); + }); + + it("requires a character device at /dev/nvmap before returning DRI render devices (#7610)", () => { + expect( + detectTegraGpuDevicePaths({ + listDevicePaths: () => ["/dev/nvmap", "/dev/dri/renderD128"], + statDevicePath: (devicePath) => ({ + isCharacterDevice: devicePath === "/dev/dri/renderD128", + isSymbolicLink: false, + }), + }), + ).toEqual([]); + }); +}); describe("detectTegraDeviceGroupGids", () => { afterEach(() => { diff --git a/src/lib/onboard/docker-gpu-jetson-groups.ts b/src/lib/onboard/docker-gpu-jetson-groups.ts index 4cfcd098f4..b6a8c7265f 100644 --- a/src/lib/onboard/docker-gpu-jetson-groups.ts +++ b/src/lib/onboard/docker-gpu-jetson-groups.ts @@ -16,6 +16,7 @@ const TEGRA_GPU_DEVICE_NODES = [ "/dev/nvgpu/igpu0/as", "/dev/nvgpu/igpu0/prof", ] as const; +const NVMAP_DEVICE = "/dev/nvmap"; const READ_WRITE_PERMISSION_BITS = 0o6; const MAX_DOCKER_SUPPLEMENTARY_GID = 2_147_483_647; @@ -24,6 +25,11 @@ type DeviceGroupAccess = { mode: number; }; +type DevicePathAccess = { + isCharacterDevice: boolean; + isSymbolicLink: boolean; +}; + /** * Find real DRI render character devices without following symlinks or * scanning other DRI device families. @@ -51,6 +57,38 @@ function listTegraGpuDevicePaths(): string[] { return [...TEGRA_GPU_DEVICE_NODES, ...discoverTegraRenderDevicePaths()]; } +/** + * Require a non-symlink /dev/nvmap character device before returning the + * curated paths for the OpenShell filesystem policy. + */ +export function detectTegraGpuDevicePaths( + deps: { + statDevicePath?: (path: string) => DevicePathAccess | null; + listDevicePaths?: () => string[]; + } = {}, +): string[] { + const devicePaths = deps.listDevicePaths?.() ?? listTegraGpuDevicePaths(); + const statDevicePath = + deps.statDevicePath ?? + ((devicePath: string): DevicePathAccess | null => { + try { + const stat = fs.lstatSync(devicePath); + return { + isCharacterDevice: stat.isCharacterDevice(), + isSymbolicLink: stat.isSymbolicLink(), + }; + } catch { + return null; + } + }); + + const detectedPaths = devicePaths.filter((devicePath) => { + const access = statDevicePath(devicePath); + return access?.isCharacterDevice === true && access.isSymbolicLink === false; + }); + return detectedPaths.includes(NVMAP_DEVICE) ? detectedPaths : []; +} + /** * Source-of-truth boundary for Jetson/Tegra supplementary device groups: * diff --git a/src/lib/onboard/initial-policy.test.ts b/src/lib/onboard/initial-policy.test.ts index 2cde41037a..968cbdc8af 100644 --- a/src/lib/onboard/initial-policy.test.ts +++ b/src/lib/onboard/initial-policy.test.ts @@ -369,6 +369,57 @@ network_policies: {} } }); + it("moves detected Jetson device paths to read-write and adds /opt/nvidia as read-only (#7610)", () => { + const gpuPolicy = buildDirectGpuPolicyYaml( + ` +version: 1 +filesystem_policy: + read_only: + - /usr + - /dev/nvmap + read_write: + - /tmp + - /dev/nvhost-gpu +network_policies: {} +`, + { + jetsonGpuDevicePaths: ["/dev/nvmap", "/dev/nvhost-gpu", "/dev/nvmap"], + }, + ); + const gpuDoc = YAML.parse(gpuPolicy); + + expect(gpuDoc.filesystem_policy.read_only).toContain("/opt/nvidia"); + expect(gpuDoc.filesystem_policy.read_only).not.toContain("/dev/nvmap"); + expectSingleOccurrence(gpuDoc.filesystem_policy.read_write, "/dev/nvmap"); + expectSingleOccurrence(gpuDoc.filesystem_policy.read_write, "/dev/nvhost-gpu"); + }); + + it("keeps Jetson filesystem grants scoped to OpenClaw direct GPU policy (#7610)", () => { + const basePolicyPath = tmpPolicy(BASE_POLICY_FIXTURE); + const devicePaths = ["/dev/nvmap", "/dev/nvhost-gpu"]; + const openclaw = prepareInitialSandboxCreatePolicy(basePolicyPath, [], { + directGpu: true, + agentName: "openclaw", + jetsonGpuDevicePaths: devicePaths, + stationGb300SysfsReadOnlyPaths: [], + }); + const hermes = prepareInitialSandboxCreatePolicy(basePolicyPath, [], { + directGpu: true, + agentName: "hermes", + jetsonGpuDevicePaths: devicePaths, + stationGb300SysfsReadOnlyPaths: [], + }); + const openclawDoc = YAML.parse(fs.readFileSync(openclaw.policyPath, "utf-8")); + const hermesDoc = YAML.parse(fs.readFileSync(hermes.policyPath, "utf-8")); + + expect(openclawDoc.filesystem_policy.read_only).toContain("/opt/nvidia"); + expect(openclawDoc.filesystem_policy.read_write).toEqual(expect.arrayContaining(devicePaths)); + expect(hermesDoc.filesystem_policy.read_only).not.toContain("/opt/nvidia"); + expect(hermesDoc.filesystem_policy.read_write).not.toEqual(expect.arrayContaining(devicePaths)); + expect(openclaw.cleanup?.()).toBe(true); + expect(hermes.cleanup?.()).toBe(true); + }); + it("preserves best-effort Landlock for missing Station sysfs paths (#7103)", () => { const sysfsRoot = tmpSysfsRoot(); addPciDevice(sysfsRoot, "0009:06:00.0", "0x10de\n", "0x030200\n"); diff --git a/src/lib/onboard/initial-policy.ts b/src/lib/onboard/initial-policy.ts index 4d56cecb12..7bdc3b555f 100644 --- a/src/lib/onboard/initial-policy.ts +++ b/src/lib/onboard/initial-policy.ts @@ -23,6 +23,7 @@ import { isStationGb300ProductName, type StationProfile, } from "../readiness/station-qualification"; +import { detectTegraGpuDevicePaths } from "./docker-gpu-jetson-groups"; import { allMessagingChannelPolicyPresets, requiredMessagingChannelPolicyPresets, @@ -46,6 +47,7 @@ export function discloseInitialSandboxPolicy(policy: InitialSandboxPolicy): void const HERMES_MESSAGING_POLICY_KEYS = getMessagingPolicyKeysByChannel({ agent: "hermes" }); const PROC_PATH = "/proc"; +const JETSON_GPU_LIBRARY_ROOT = "/opt/nvidia"; const PROC_COMM_READ_WRITE_PATHS = ["/proc/self/comm", "/proc/self/task/*/comm"]; const SYSFS_PATH = "/sys"; const PCI_BDF_PATTERN = /^[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]$/iu; @@ -77,6 +79,7 @@ function deduplicateDirectGpuSysfsEntries( type DirectGpuPolicyOptions = { procReadWrite?: boolean; sysfsReadOnlyPaths?: readonly string[]; + jetsonGpuDevicePaths?: readonly string[]; }; export { isStationGb300ProductName }; @@ -223,6 +226,23 @@ export function buildDirectGpuPolicyYaml( } } } + const jetsonGpuDevicePaths = [...new Set(options.jetsonGpuDevicePaths ?? [])]; + if (jetsonGpuDevicePaths.length > 0) { + if ( + !fsPolicy.read_only.includes(JETSON_GPU_LIBRARY_ROOT) && + !fsPolicy.read_write.includes(JETSON_GPU_LIBRARY_ROOT) + ) { + fsPolicy.read_only.push(JETSON_GPU_LIBRARY_ROOT); + } + + const jetsonGpuDevicePathSet = new Set(jetsonGpuDevicePaths); + fsPolicy.read_only = fsPolicy.read_only.filter( + (entry: string) => !jetsonGpuDevicePathSet.has(entry), + ); + for (const devicePath of jetsonGpuDevicePaths) { + if (!fsPolicy.read_write.includes(devicePath)) fsPolicy.read_write.push(devicePath); + } + } if (options.procReadWrite && !fsPolicy.read_write.includes(PROC_PATH)) { // This exists only for the legacy post-create Docker GPU compatibility // path, which recreates the container after `openshell sandbox create` and @@ -383,6 +403,7 @@ export function prepareInitialSandboxCreatePolicy( dockerGpuPatch?: boolean; hostGpuAvailable?: boolean; stationGb300SysfsReadOnlyPaths?: readonly string[]; + jetsonGpuDevicePaths?: readonly string[]; additionalPresets?: string[]; agentName?: string | null; policyTier?: string | null; @@ -397,6 +418,10 @@ export function prepareInitialSandboxCreatePolicy( discoverHostStationGb300SysfsReadOnlyPaths({ hasNvidiaGpu: options.hostGpuAvailable, }), + jetsonGpuDevicePaths: + options.agentName === "openclaw" + ? (options.jetsonGpuDevicePaths ?? detectTegraGpuDevicePaths()) + : [], }) : null; let effectiveBasePolicyPath = directGpuPolicy?.policyPath || basePolicyPath; From 48338b7e67203e0de99b6a81437de74ac0498a75 Mon Sep 17 00:00:00 2001 From: San Dang Date: Thu, 6 Aug 2026 17:52:27 +0700 Subject: [PATCH 21/49] fix(onboard): apply Jetson policy to default OpenClaw Signed-off-by: San Dang --- src/lib/onboard/initial-policy.test.ts | 11 +++++++++++ src/lib/onboard/initial-policy.ts | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/lib/onboard/initial-policy.test.ts b/src/lib/onboard/initial-policy.test.ts index 968cbdc8af..b12909739b 100644 --- a/src/lib/onboard/initial-policy.test.ts +++ b/src/lib/onboard/initial-policy.test.ts @@ -397,6 +397,11 @@ network_policies: {} it("keeps Jetson filesystem grants scoped to OpenClaw direct GPU policy (#7610)", () => { const basePolicyPath = tmpPolicy(BASE_POLICY_FIXTURE); const devicePaths = ["/dev/nvmap", "/dev/nvhost-gpu"]; + const defaultOpenclaw = prepareInitialSandboxCreatePolicy(basePolicyPath, [], { + directGpu: true, + jetsonGpuDevicePaths: devicePaths, + stationGb300SysfsReadOnlyPaths: [], + }); const openclaw = prepareInitialSandboxCreatePolicy(basePolicyPath, [], { directGpu: true, agentName: "openclaw", @@ -409,13 +414,19 @@ network_policies: {} jetsonGpuDevicePaths: devicePaths, stationGb300SysfsReadOnlyPaths: [], }); + const defaultOpenclawDoc = YAML.parse(fs.readFileSync(defaultOpenclaw.policyPath, "utf-8")); const openclawDoc = YAML.parse(fs.readFileSync(openclaw.policyPath, "utf-8")); const hermesDoc = YAML.parse(fs.readFileSync(hermes.policyPath, "utf-8")); + expect(defaultOpenclawDoc.filesystem_policy.read_only).toContain("/opt/nvidia"); + expect(defaultOpenclawDoc.filesystem_policy.read_write).toEqual( + expect.arrayContaining(devicePaths), + ); expect(openclawDoc.filesystem_policy.read_only).toContain("/opt/nvidia"); expect(openclawDoc.filesystem_policy.read_write).toEqual(expect.arrayContaining(devicePaths)); expect(hermesDoc.filesystem_policy.read_only).not.toContain("/opt/nvidia"); expect(hermesDoc.filesystem_policy.read_write).not.toEqual(expect.arrayContaining(devicePaths)); + expect(defaultOpenclaw.cleanup?.()).toBe(true); expect(openclaw.cleanup?.()).toBe(true); expect(hermes.cleanup?.()).toBe(true); }); diff --git a/src/lib/onboard/initial-policy.ts b/src/lib/onboard/initial-policy.ts index 7bdc3b555f..17680d8e88 100644 --- a/src/lib/onboard/initial-policy.ts +++ b/src/lib/onboard/initial-policy.ts @@ -419,7 +419,7 @@ export function prepareInitialSandboxCreatePolicy( hasNvidiaGpu: options.hostGpuAvailable, }), jetsonGpuDevicePaths: - options.agentName === "openclaw" + (options.agentName ?? "openclaw") === "openclaw" ? (options.jetsonGpuDevicePaths ?? detectTegraGpuDevicePaths()) : [], }) From deee7ef1f20831dff5680321d29b5f18d6d3fdd8 Mon Sep 17 00:00:00 2001 From: San Dang Date: Thu, 6 Aug 2026 18:36:07 +0700 Subject: [PATCH 22/49] fix(onboard): preserve Jetson device groups Signed-off-by: San Dang --- Dockerfile | 7 ++- docs/reference/troubleshooting.mdx | 15 ++++- scripts/jetson-device-group-bootstrap.sh | 51 ++++++++++++++++ src/lib/onboard/docker-gpu-jetson-groups.ts | 12 ++-- src/lib/onboard/docker-gpu-patch-clone.ts | 55 ++++++++++++++---- .../onboard/docker-gpu-patch-jetson.test.ts | 58 +++++++++++++++++++ src/lib/onboard/docker-gpu-patch-types.ts | 13 +++-- .../managed-bootstrap/docker-test-fixture.ts | 27 +++++++++ .../onboard/managed-bootstrap/docker.test.ts | 33 +++++++++++ src/lib/onboard/managed-bootstrap/docker.ts | 52 +++++++++++++++-- .../sandbox-gpu-preflight-routing.test.ts | 8 +++ src/lib/onboard/sandbox-gpu-preflight.ts | 4 +- test/openclaw-final-image-layout.test.ts | 2 + 13 files changed, 308 insertions(+), 29 deletions(-) create mode 100755 scripts/jetson-device-group-bootstrap.sh diff --git a/Dockerfile b/Dockerfile index fede87d692..42a1d8a688 100644 --- a/Dockerfile +++ b/Dockerfile @@ -165,6 +165,7 @@ COPY scripts/lib/sandbox-init.sh /usr/local/lib/nemoclaw/sandbox-init.sh COPY scripts/lib/entrypoint-env-wrapper.sh /usr/local/lib/nemoclaw/entrypoint-env-wrapper.sh COPY scripts/lib/gateway-supervisor.sh /usr/local/lib/nemoclaw/gateway-supervisor.sh COPY scripts/lib/sandbox-rlimits.sh /usr/local/lib/nemoclaw/sandbox-rlimits.sh +COPY scripts/jetson-device-group-bootstrap.sh /usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh COPY scripts/lib/openclaw_device_approval_policy.py /usr/local/lib/nemoclaw/openclaw_device_approval_policy.py COPY scripts/lib/clean_runtime_shell_env_shim.py /usr/local/lib/nemoclaw/clean_runtime_shell_env_shim.py COPY scripts/lib/normalize_mutable_config_perms.py /usr/local/lib/nemoclaw/normalize_mutable_config_perms.py @@ -1109,12 +1110,14 @@ RUN chmod 755 /usr/local/bin/nemoclaw-start /usr/local/bin/nemoclaw-codex-acp \ && chmod -R a+rX /src/lib/messaging \ && chown root:root /usr/local/bin/nemoclaw-gateway-control \ /usr/local/lib/nemoclaw/gateway-supervisor.sh \ + /usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh \ /usr/local/lib/nemoclaw/state-dir-guard.py \ /usr/local/share/nemoclaw/state-lock-plan.json \ /usr/local/lib/nemoclaw/openclaw-config-guard.py \ /usr/local/lib/nemoclaw/managed-gateway-control.py \ && chmod 700 /usr/local/bin/nemoclaw-gateway-control \ - && chmod 500 /usr/local/lib/nemoclaw/state-dir-guard.py \ + && chmod 500 /usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh \ + /usr/local/lib/nemoclaw/state-dir-guard.py \ /usr/local/lib/nemoclaw/openclaw-config-guard.py \ /usr/local/lib/nemoclaw/managed-gateway-control.py \ && chmod 444 /usr/local/share/nemoclaw/state-lock-plan.json \ @@ -1837,6 +1840,8 @@ RUN check_metadata() { \ && check_metadata /usr/local/bin/nemoclaw-managed-bootstrap 'root:root:755' \ && test ! -L /usr/local/lib/nemoclaw/managed-bootstrap-trampoline.sh \ && check_metadata /usr/local/lib/nemoclaw/managed-bootstrap-trampoline.sh 'root:root:444' \ + && test ! -L /usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh \ + && check_metadata /usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh 'root:root:500' \ && check_metadata /usr/local/bin/nemoclaw-gateway-control 'root:root:700' \ && check_metadata /usr/local/lib/nemoclaw/state-dir-guard.py 'root:root:500' \ && check_metadata /usr/local/share/nemoclaw/state-lock-plan.json 'root:root:444' \ diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 8d6406834c..c578c0ce54 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -2731,10 +2731,21 @@ Automatic GPU onboarding uses the compatibility path directly; it does not make The path recreates the OpenShell-managed Docker container with NVIDIA GPU flags and propagates eligible host group IDs for the supported Jetson GPU device nodes. -For OpenClaw direct GPU onboarding, the creation-time filesystem policy adds no Jetson entries unless `/dev/nvmap` is an existing, non-symlink character device. +For OpenClaw, Docker compatibility recreation passes those group IDs with `--group-add`. +OpenShell 0.0.85 calls `initgroups()` from the container group database before it starts the sandbox account. +That call replaces the inherited supplementary groups, so `--group-add` alone does not preserve Jetson device access. +The managed OpenClaw Jetson replacement container runs a bounded, root-owned wrapper from the sandbox image before the OpenShell supervisor starts. +The wrapper is `root:root` with mode `0500`. +The wrapper can hand off only to `/usr/local/bin/nemoclaw-managed-bootstrap`. +That entrypoint resumes the fixed `/opt/openshell/bin/openshell-sandbox` supervisor. +The wrapper adds only the validated Jetson device GIDs that onboarding detected to the existing sandbox account in `/etc/group`. +It verifies the resulting membership before handoff. +Legacy and custom image recreation, generic GPU, CPU-only, Hermes, and Deep Agents paths do not use this wrapper. + +The creation-time filesystem policy adds no Jetson entries unless `/dev/nvmap` is an existing, non-symlink character device. When that condition is met, the policy adds `/opt/nvidia` as read-only. It adds each existing, non-symlink character device on the eligible GPU path list as read-write. -Generic GPU and CPU-only policies do not receive these Jetson entries. +Generic GPU, CPU-only, Hermes, and Deep Agents policies do not receive these Jetson entries. Use `NEMOCLAW_DOCKER_GPU_PATCH=0` only for troubleshooting because it bypasses that group propagation and CUDA may not initialize. diff --git a/scripts/jetson-device-group-bootstrap.sh b/scripts/jetson-device-group-bootstrap.sh new file mode 100755 index 0000000000..00fe4a1422 --- /dev/null +++ b/scripts/jetson-device-group-bootstrap.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +fail() { + printf 'Jetson device-group bootstrap: %s\n' "$*" >&2 + exit 1 +} + +[ "$(/usr/bin/id -u)" -eq 0 ] || fail "must run as root" +[ "${1:-}" = "--device-group-gids" ] || fail "device group argument is missing" +group_gids="${2:-}" +[ "${3:-}" = "--" ] || fail "supervisor delimiter is missing" +shift 3 +[ "${1:-}" = "/usr/local/bin/nemoclaw-managed-bootstrap" ] \ + || fail "managed bootstrap entrypoint is invalid" +/usr/bin/id sandbox >/dev/null 2>&1 || fail "sandbox user is missing" +[ -f /etc/group ] && [ ! -L /etc/group ] || fail "container group database is invalid" + +IFS=',' read -r -a gids <<<"$group_gids" +[ "${#gids[@]}" -gt 0 ] && [ "${#gids[@]}" -le 16 ] \ + || fail "device group count is invalid" + +declare -A seen=() +for gid in "${gids[@]}"; do + [[ "$gid" =~ ^[1-9][0-9]{0,9}$ ]] || fail "device group ID is invalid" + [ "$gid" -le 2147483647 ] || fail "device group ID is out of range" + [ -z "${seen[$gid]:-}" ] || fail "device group ID is duplicated" + seen[$gid]=1 + + group_record="$(/usr/bin/getent group "$gid" || true)" + if [ -z "$group_record" ]; then + group_name="nemoclaw_gpu_$gid" + /usr/sbin/groupadd --gid "$gid" "$group_name" + else + IFS=':' read -r group_name _ resolved_gid _ <<<"$group_record" + [ -n "$group_name" ] && [ "$resolved_gid" = "$gid" ] \ + || fail "device group record is invalid" + fi + /usr/sbin/usermod --append --groups "$group_name" sandbox +done + +sandbox_groups=" $(/usr/bin/id -G sandbox) " +for gid in "${gids[@]}"; do + [[ "$sandbox_groups" == *" $gid "* ]] \ + || fail "sandbox membership verification failed" +done + +exec "$@" diff --git a/src/lib/onboard/docker-gpu-jetson-groups.ts b/src/lib/onboard/docker-gpu-jetson-groups.ts index b6a8c7265f..c8e6decf2d 100644 --- a/src/lib/onboard/docker-gpu-jetson-groups.ts +++ b/src/lib/onboard/docker-gpu-jetson-groups.ts @@ -92,13 +92,15 @@ export function detectTegraGpuDevicePaths( /** * Source-of-truth boundary for Jetson/Tegra supplementary device groups: * - * - Invalid state: the non-root sandbox user can see `/dev/nvmap` and `/dev/nvhost-*` but cannot - * open them because Docker did not copy their host-owned supplementary GIDs into the container. + * - Invalid state: the non-root sandbox user can see `/dev/nvmap` and `/dev/nvhost-*` but loses + * access when OpenShell rebuilds supplementary groups from the container group database. * - Source boundary: host device-node ownership is authoritative; NemoClaw only carries each * bounded, non-root numeric GID with effective group read/write permission into the Jetson - * compatibility recreation via `--group-add`. - * - Source-fix constraint: changing host udev ownership or image-local groups cannot reliably fix - * device nodes whose ownership is assigned by the Jetson host at runtime. + * compatibility recreation via `--group-add`. OpenClaw also records those same GIDs in the + * replacement container's sandbox account before OpenShell rebuilds its supplementary group + * list. + * - Source-fix constraint: the replacement container membership must be derived from the current + * host device nodes; a static group name or GID cannot represent Jetson hosts reliably. * - Regression coverage: docker-gpu-jetson-groups.test.ts covers discovery and hostile numeric * values; docker-gpu-patch-jetson.test.ts covers clone-envelope propagation and generic-host * exclusion. diff --git a/src/lib/onboard/docker-gpu-patch-clone.ts b/src/lib/onboard/docker-gpu-patch-clone.ts index 2adb44f47d..c49f3c3e1f 100644 --- a/src/lib/onboard/docker-gpu-patch-clone.ts +++ b/src/lib/onboard/docker-gpu-patch-clone.ts @@ -10,6 +10,9 @@ import type { import { openshellSandboxCommandEnvValue } from "./docker-startup-command-env"; const OPENSHELL_SANDBOX_COMMAND_ENV = "OPENSHELL_SANDBOX_COMMAND"; +const MANAGED_BOOTSTRAP_ENTRYPOINT = "/usr/local/bin/nemoclaw-managed-bootstrap"; +const JETSON_DEVICE_GROUP_BOOTSTRAP = "/usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh"; +const MAX_JETSON_DEVICE_GROUPS = 16; const GPU_ENV_KEYS = new Set([ "NVIDIA_VISIBLE_DEVICES", "NVIDIA_DRIVER_CAPABILITIES", @@ -353,6 +356,21 @@ export function buildDockerGpuCloneRunArgs( } const args: string[] = ["--name", containerName, ...mode.args]; const gpuAugment = mode.kind !== "startup-command"; + const extraGroupGids = [ + ...new Set((options.extraGroupGids ?? []).map((gid) => String(gid).trim())), + ]; + if ( + extraGroupGids.length > MAX_JETSON_DEVICE_GROUPS || + extraGroupGids.some((gid) => { + if (!/^[1-9][0-9]*$/u.test(gid)) return true; + const parsed = Number(gid); + return !Number.isSafeInteger(parsed) || parsed > 2_147_483_647; + }) + ) { + throw new Error("Docker clone received invalid or excessive supplementary group IDs."); + } + const preserveJetsonGroups = + options.preserveJetsonDeviceGroupMembership === true && extraGroupGids.length > 0; // Startup-command recreation must retain OpenShell's native CDI attachment. if (!gpuAugment) { @@ -428,11 +446,10 @@ export function buildDockerGpuCloneRunArgs( for (const hostEntry of stringArray(host.ExtraHosts)) args.push("--add-host", hostEntry); const groupAdds = new Set(stringArray(host.GroupAdd)); for (const group of groupAdds) args.push("--group-add", group); - for (const gid of options.extraGroupGids ?? []) { - const normalized = String(gid).trim(); - if (normalized && !groupAdds.has(normalized)) { - groupAdds.add(normalized); - args.push("--group-add", normalized); + for (const gid of extraGroupGids) { + if (!groupAdds.has(gid)) { + groupAdds.add(gid); + args.push("--group-add", gid); } } for (const ulimit of dockerUlimits(inspect, options.requiredUlimits)) { @@ -466,16 +483,34 @@ export function buildDockerGpuCloneRunArgs( const entrypoint = stringArray(config.Entrypoint); const replacementEntrypoint = String(options.containerEntrypoint ?? "").trim(); - if (replacementEntrypoint) { + const managedBootstrapTarget = replacementEntrypoint === MANAGED_BOOTSTRAP_ENTRYPOINT; + if (preserveJetsonGroups && (!managedBootstrapTarget || !options.containerCommand?.length)) { + throw new Error("Jetson device-group bootstrap requires the managed OpenClaw entrypoint."); + } + if (preserveJetsonGroups) { + args.push("--entrypoint", JETSON_DEVICE_GROUP_BOOTSTRAP); + } else if (replacementEntrypoint) { args.push("--entrypoint", replacementEntrypoint); } else if (entrypoint.length > 0) { args.push("--entrypoint", entrypoint[0]); } - const commandArgs = options.containerCommand + const originalCommandArgs = sandboxCommand + ? [] + : [...entrypoint.slice(1), ...stringArray(config.Cmd)]; + const targetCommandArgs = options.containerCommand ? [...options.containerCommand] - : sandboxCommand - ? [] - : [...entrypoint.slice(1), ...stringArray(config.Cmd)]; + : originalCommandArgs; + const commandArgs = preserveJetsonGroups + ? [ + "--device-group-gids", + extraGroupGids.join(","), + "--", + MANAGED_BOOTSTRAP_ENTRYPOINT, + ...targetCommandArgs, + ] + : options.containerCommand + ? [...options.containerCommand] + : originalCommandArgs; args.push(image, ...commandArgs); return args; } diff --git a/src/lib/onboard/docker-gpu-patch-jetson.test.ts b/src/lib/onboard/docker-gpu-patch-jetson.test.ts index e091c45a13..1aae15c626 100644 --- a/src/lib/onboard/docker-gpu-patch-jetson.test.ts +++ b/src/lib/onboard/docker-gpu-patch-jetson.test.ts @@ -32,6 +32,7 @@ describe("Jetson device-node group propagation (#4231, #7610)", () => { args.filter((arg, index) => args[index - 1] === "--group-add" && arg === "44").length, ).toBe(1); expect(args).toEqual(expect.arrayContaining(["--group-add", "110"])); + expect(args).not.toContain("/usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh"); }); it("does not add --group-add when extraGroupGids is absent", () => { @@ -41,6 +42,63 @@ describe("Jetson device-node group propagation (#4231, #7610)", () => { expect(args).not.toEqual(expect.arrayContaining(["--group-add"])); }); + it("runs the managed bootstrap through the Jetson group bootstrap (#7610)", () => { + const args = buildDockerGpuCloneRunArgs( + inspectFixture(), + buildDockerGpuMode("nvidia-runtime", null, { backend: "jetson" }), + { + extraGroupGids: ["44"], + preserveJetsonDeviceGroupMembership: true, + containerEntrypoint: "/usr/local/bin/nemoclaw-managed-bootstrap", + containerCommand: ["--request", "/run/nemoclaw/bootstrap-request.json"], + }, + ); + + expect(args.slice(args.indexOf("openshell/sandbox:abc"))).toEqual([ + "openshell/sandbox:abc", + "--device-group-gids", + "44", + "--", + "/usr/local/bin/nemoclaw-managed-bootstrap", + "--request", + "/run/nemoclaw/bootstrap-request.json", + ]); + }); + + it("rejects Jetson group preservation outside the managed image boundary (#7610)", () => { + expect(() => + buildDockerGpuCloneRunArgs( + inspectFixture(), + buildDockerGpuMode("nvidia-runtime", null, { backend: "jetson" }), + { + extraGroupGids: ["44"], + preserveJetsonDeviceGroupMembership: true, + }, + ), + ).toThrow("Jetson device-group bootstrap requires the managed OpenClaw entrypoint."); + }); + + it("rejects invalid or excessive supplementary group IDs before clone creation (#7610)", () => { + const options = { + containerEntrypoint: "/usr/local/bin/nemoclaw-managed-bootstrap", + containerCommand: ["--request", "/run/nemoclaw/bootstrap-request.json"], + preserveJetsonDeviceGroupMembership: true, + } as const; + const build = (extraGroupGids: readonly string[]) => + buildDockerGpuCloneRunArgs( + inspectFixture(), + buildDockerGpuMode("nvidia-runtime", null, { backend: "jetson" }), + { ...options, extraGroupGids }, + ); + + expect(() => build(["0"])).toThrow( + "Docker clone received invalid or excessive supplementary group IDs.", + ); + expect(() => build(Array.from({ length: 17 }, (_, index) => String(index + 1)))).toThrow( + "Docker clone received invalid or excessive supplementary group IDs.", + ); + }); + it("passes all detected Tegra device GIDs into the Jetson recreate as --group-add", () => { const dockerRunDetached = vi.fn(() => ({ status: 0, diff --git a/src/lib/onboard/docker-gpu-patch-types.ts b/src/lib/onboard/docker-gpu-patch-types.ts index f367df0ab2..2b184d27f3 100644 --- a/src/lib/onboard/docker-gpu-patch-types.ts +++ b/src/lib/onboard/docker-gpu-patch-types.ts @@ -125,12 +125,17 @@ export type DockerGpuCloneRunOptions = { containerName?: string | null; /** * Extra supplementary group IDs to add to the recreated container via - * `--group-add`. On Jetson these are the host group(s) owning the Tegra GPU - * device nodes; granting the sandbox user membership lets CUDA's nvmap init - * open them instead of failing with `NvRmMemInitNvmap ... Permission - * denied` (#4231, #7610). + * `--group-add`. The OpenClaw Jetson path also records these validated GIDs + * in the replacement container's group database before OpenShell calls + * initgroups() (#7610). */ extraGroupGids?: readonly string[] | null; + /** + * Add the detected Jetson device GIDs to the replacement container's sandbox + * account before OpenShell rebuilds supplementary groups with initgroups() + * (#7610). + */ + preserveJetsonDeviceGroupMembership?: boolean; }; export type DockerGpuPatchDiagnostics = { diff --git a/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts b/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts index 4cb3672d02..dd232e46f1 100644 --- a/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts +++ b/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts @@ -147,9 +147,17 @@ function originalInspect(inputs = agentInputs()): DockerContainerInspect { Binds: ["/host/workspace:/sandbox:rw"], NetworkMode: "openshell", RestartPolicy: { Name: "unless-stopped" }, + CapAdd: null, CapDrop: ["NET_RAW"], + DeviceRequests: null, + Devices: null, + GroupAdd: null, + Runtime: null, SecurityOpt: ["no-new-privileges"], Ulimits: [{ Name: "nofile", Soft: 65_536, Hard: 65_536 }], + } as NonNullable & { + Devices?: unknown; + Runtime?: string | null; }, NetworkSettings: { Networks: { openshell: { Aliases: ["openshell-alpha"] } } }, }; @@ -357,6 +365,16 @@ export function fixture(options: DockerFixtureOptions = {}) { const env = args.flatMap((value, index) => value === "--env" ? [String(args[index + 1] ?? "")] : [], ); + const valuesAfter = (flag: string) => + args.flatMap((value, index) => (value === flag ? [String(args[index + 1] ?? "")] : [])); + const runtimeIndex = args.indexOf("--runtime"); + const sourceRuntime = ( + source.HostConfig as + | (NonNullable & { + Runtime?: string | null; + }) + | null + )?.Runtime; replacement = { ...structuredClone(source), Id: NEW_ID, @@ -368,6 +386,15 @@ export function fixture(options: DockerFixtureOptions = {}) { Entrypoint: [entrypoint], Cmd: args.slice(imageIndex + 1), }, + HostConfig: { + ...structuredClone(source.HostConfig), + CapAdd: valuesAfter("--cap-add"), + GroupAdd: valuesAfter("--group-add"), + Runtime: runtimeIndex >= 0 ? String(args[runtimeIndex + 1] ?? "") : sourceRuntime, + SecurityOpt: valuesAfter("--security-opt"), + } as NonNullable & { + Runtime?: string | null; + }, State: { Running: false, Paused: false, Restarting: false, Dead: false }, }; return losesAcknowledgement("container:create") diff --git a/src/lib/onboard/managed-bootstrap/docker.test.ts b/src/lib/onboard/managed-bootstrap/docker.test.ts index 3a3d6f7c6a..82c5508a98 100644 --- a/src/lib/onboard/managed-bootstrap/docker.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker.test.ts @@ -32,6 +32,39 @@ function expectEventBefore(events: readonly string[], before: string, after: str } describe("Docker managed bootstrap adapter", () => { + it("preserves OpenClaw Jetson groups across the managed bootstrap boundary (#7610)", async () => { + const fake = fixture({ agent: "openclaw" }); + const adapter = createDockerManagedBootstrapAdapter(fake.deps); + const { handle, request, snapshot } = authority("openclaw"); + + await expect( + adapter.prepareBootstrapReplacement({ + handle, + snapshot, + request, + replacementOptions: { + values: { + gpuModeArgs: ["--runtime", "nvidia"], + gpuModeKind: "nvidia-runtime", + gpuModeLabel: "Jetson NVIDIA runtime", + extraGroupGids: ["44", "993"], + }, + }, + }), + ).resolves.toMatchObject({ preparedRuntimeId: NEW_ID }); + + expect(fake.replacement?.HostConfig?.GroupAdd).toEqual(["44", "993"]); + expect(fake.replacement?.Config?.Entrypoint).toEqual([ + "/usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh", + ]); + expect(fake.replacement?.Config?.Cmd?.slice(0, 4)).toEqual([ + "--device-group-gids", + "44,993", + "--", + "/usr/local/bin/nemoclaw-managed-bootstrap", + ]); + }); + it("captures the live OpenShell idle supervisor with a separately persisted bootstrap identity", async () => { const fake = fixture(); const adapter = createDockerManagedBootstrapAdapter(fake.deps); diff --git a/src/lib/onboard/managed-bootstrap/docker.ts b/src/lib/onboard/managed-bootstrap/docker.ts index 7405fbd511..f6866c110f 100644 --- a/src/lib/onboard/managed-bootstrap/docker.ts +++ b/src/lib/onboard/managed-bootstrap/docker.ts @@ -113,6 +113,8 @@ const MAX_RECOVERY_FAILURE_DETAIL_BYTES = 8 * 1024; const OPENSHELL_DRIVER_IDLE_COMMAND = "sleep infinity"; export const MANAGED_BOOTSTRAP_TRAMPOLINE_EXECUTABLE = "/usr/local/bin/nemoclaw-managed-bootstrap"; +const JETSON_DEVICE_GROUP_BOOTSTRAP_EXECUTABLE = + "/usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh"; function boundedRecoveryFailureDetail(error: unknown): string { const raw = (error instanceof Error ? error.message : String(error)).replaceAll("\0", "�"); @@ -717,13 +719,44 @@ function assertReplacementBoundary( inspect: DockerContainerInspect, handle: ManagedBootstrapHeldWorkloadHandle, snapshot: ManagedBootstrapObservedSnapshot, + expectedJetsonGroupGids?: readonly string[], ): void { const entrypoint = exactStringArray(inspect.Config?.Entrypoint, "replacement entrypoint"); const command = exactStringArray(inspect.Config?.Cmd, "replacement command"); - if ( - !exactArrayEqual(entrypoint, [MANAGED_BOOTSTRAP_TRAMPOLINE_EXECUTABLE]) || - !exactArrayEqual(command, replacementCommand(handle, snapshot)) - ) { + const managedCommand = replacementCommand(handle, snapshot); + const wrappedCommandPrefix = [ + "--device-group-gids", + command[1] ?? "", + "--", + MANAGED_BOOTSTRAP_TRAMPOLINE_EXECUTABLE, + ]; + const wrappedGids = String(command[1] ?? "").split(","); + const expectedGroupSet = new Set(expectedJetsonGroupGids); + const replacementGroupAdds = exactStringArray( + inspect.HostConfig?.GroupAdd, + "replacement supplementary groups", + ); + const wrappedBoundary = + handle.plan.profile.agent === "openclaw" && + exactArrayEqual(entrypoint, [JETSON_DEVICE_GROUP_BOOTSTRAP_EXECUTABLE]) && + wrappedGids.length > 0 && + wrappedGids.length <= 16 && + wrappedGids.every( + (gid, index) => + /^[1-9][0-9]*$/u.test(gid) && + Number(gid) <= 2_147_483_647 && + wrappedGids.indexOf(gid) === index && + (expectedJetsonGroupGids === undefined || expectedGroupSet.has(gid)), + ) && + (expectedJetsonGroupGids === undefined || + wrappedGids.length === expectedJetsonGroupGids.length) && + wrappedGids.every((gid) => replacementGroupAdds.includes(gid)) && + exactArrayEqual(command, [...wrappedCommandPrefix, ...managedCommand]); + const directBoundary = + expectedJetsonGroupGids === undefined && + exactArrayEqual(entrypoint, [MANAGED_BOOTSTRAP_TRAMPOLINE_EXECUTABLE]) && + exactArrayEqual(command, managedCommand); + if (!directBoundary && !wrappedBoundary) { throw new Error("Managed bootstrap Docker replacement process boundary changed."); } const intended = openshellSandboxCommandEnvValue(handle.intendedWorkloadArgv); @@ -3259,6 +3292,8 @@ export function createDockerManagedBootstrapAdapter( openshellSandboxCommand: handle.intendedWorkloadArgv, requiredUlimits: plan.requiredUlimits, extraGroupGids: plan.extraGroupGids, + preserveJetsonDeviceGroupMembership: + handle.plan.profile.agent === "openclaw" && plan.extraGroupGids.length > 0, containerEntrypoint: MANAGED_BOOTSTRAP_TRAMPOLINE_EXECUTABLE, containerCommand: trampolineCommand, containerName: stagingName, @@ -3316,7 +3351,14 @@ export function createDockerManagedBootstrapAdapter( "Managed bootstrap Docker replacement requires one bounded intended workload argv.", ); } - assertReplacementBoundary(createdInspect, handle, snapshot); + assertReplacementBoundary( + createdInspect, + handle, + snapshot, + handle.plan.profile.agent === "openclaw" && plan.extraGroupGids.length > 0 + ? plan.extraGroupGids + : undefined, + ); const expectedActivatedSpecHash = assertReplacementMatchesIntent( snapshot.specCanonicalJson, createdInspect, diff --git a/src/lib/onboard/sandbox-gpu-preflight-routing.test.ts b/src/lib/onboard/sandbox-gpu-preflight-routing.test.ts index 9a1e70c3f2..8407fa39eb 100644 --- a/src/lib/onboard/sandbox-gpu-preflight-routing.test.ts +++ b/src/lib/onboard/sandbox-gpu-preflight-routing.test.ts @@ -11,6 +11,7 @@ import type { SandboxGpuConfig } from "./sandbox-gpu-mode"; import { dockerNvidiaRuntimeAvailable, formatSandboxGpuPassthroughNote, + jetsonGpuProofRemediationLines, parseDockerRuntimeNames, sandboxGpuRemediationLines, validateSandboxGpuPreflight, @@ -28,6 +29,13 @@ function sandboxGpuConfig(overrides: Partial = {}): SandboxGpu }; } describe("sandbox GPU preflight routing", () => { + it("checks retained Jetson group membership instead of repeating --group-add advice (#7610)", () => { + const remediation = jetsonGpuProofRemediationLines().join("\n"); + + expect(remediation).toContain("id (must include the groups that own those device nodes)"); + expect(remediation).not.toContain("via --group-add"); + }); + it("formats Jetson sandbox GPU notes around the NVIDIA runtime backend", () => { expect(formatSandboxGpuPassthroughNote({ hostGpuPlatform: "jetson" })).toContain( "Docker NVIDIA runtime", diff --git a/src/lib/onboard/sandbox-gpu-preflight.ts b/src/lib/onboard/sandbox-gpu-preflight.ts index 5fbbde0fe0..ad606c3ebb 100644 --- a/src/lib/onboard/sandbox-gpu-preflight.ts +++ b/src/lib/onboard/sandbox-gpu-preflight.ts @@ -64,9 +64,9 @@ export function resolveSandboxGpuFlagFromOptions(opts: SandboxGpuFlagOptions): S export function jetsonGpuProofRemediationLines(): string[] { return [ "Jetson/Tegra CUDA proof did not pass. CUDA needs access to the Tegra device", - "nodes; confirm the sandbox propagates them and the agent user's groups:", + "nodes; confirm the sandbox receives them and keeps the device group memberships:", " ls -l /dev/nvmap /dev/nvhost-* (must be readable by the sandbox)", - " add the host video/render groups via --group-add when recreating", + " id (must include the groups that own those device nodes)", "Then recreate the sandbox, or force CPU behavior with NEMOCLAW_SANDBOX_GPU=0.", ]; } diff --git a/test/openclaw-final-image-layout.test.ts b/test/openclaw-final-image-layout.test.ts index 7dae1c0a0e..83c92e899d 100644 --- a/test/openclaw-final-image-layout.test.ts +++ b/test/openclaw-final-image-layout.test.ts @@ -90,6 +90,7 @@ describe("OpenClaw final image layout", () => { "COPY scripts/lib/entrypoint-env-wrapper.sh /usr/local/lib/nemoclaw/entrypoint-env-wrapper.sh", "COPY scripts/lib/gateway-supervisor.sh /usr/local/lib/nemoclaw/gateway-supervisor.sh", "COPY scripts/lib/sandbox-rlimits.sh /usr/local/lib/nemoclaw/sandbox-rlimits.sh", + "COPY scripts/jetson-device-group-bootstrap.sh /usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh", "COPY scripts/lib/openclaw_device_approval_policy.py /usr/local/lib/nemoclaw/openclaw_device_approval_policy.py", "COPY scripts/lib/clean_runtime_shell_env_shim.py /usr/local/lib/nemoclaw/clean_runtime_shell_env_shim.py", "COPY scripts/lib/normalize_mutable_config_perms.py /usr/local/lib/nemoclaw/normalize_mutable_config_perms.py", @@ -154,6 +155,7 @@ describe("OpenClaw final image layout", () => { "/usr/local/lib/nemoclaw/patch-openclaw-tool-catalog.mts 'root:root:755'", "/usr/local/bin/nemoclaw-managed-bootstrap 'root:root:755'", "/usr/local/lib/nemoclaw/managed-bootstrap-trampoline.sh 'root:root:444'", + "/usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh 'root:root:500'", "/usr/local/bin/nemoclaw-gateway-control 'root:root:700'", "/usr/local/lib/nemoclaw/state-dir-guard.py 'root:root:500'", "/usr/local/share/nemoclaw/state-lock-plan.json 'root:root:444'", From a6bf7b54b93d5b5647a65107d4191ff56fa3af4d Mon Sep 17 00:00:00 2001 From: San Dang Date: Thu, 6 Aug 2026 18:46:29 +0700 Subject: [PATCH 23/49] fix(onboard): stage Jetson group bootstrap Signed-off-by: San Dang --- src/lib/sandbox/build-context.ts | 4 ++++ test/sandbox-build-context.test.ts | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/src/lib/sandbox/build-context.ts b/src/lib/sandbox/build-context.ts index ee01a48420..01a5586c73 100644 --- a/src/lib/sandbox/build-context.ts +++ b/src/lib/sandbox/build-context.ts @@ -245,6 +245,10 @@ function stageOptimizedSandboxBuildContext( path.join(rootDir, "scripts", "managed-bootstrap-trampoline.sh"), path.join(stagedScriptsDir, "managed-bootstrap-trampoline.sh"), ); + fs.copyFileSync( + path.join(rootDir, "scripts", "jetson-device-group-bootstrap.sh"), + path.join(stagedScriptsDir, "jetson-device-group-bootstrap.sh"), + ); fs.copyFileSync( path.join(rootDir, "scripts", "gateway-control.sh"), path.join(stagedScriptsDir, "gateway-control.sh"), diff --git a/test/sandbox-build-context.test.ts b/test/sandbox-build-context.test.ts index b02bedee35..e9e7d66805 100644 --- a/test/sandbox-build-context.test.ts +++ b/test/sandbox-build-context.test.ts @@ -109,6 +109,7 @@ describe("sandbox build context staging", () => { writeFixture(path.join("scripts", "managed-startup-hold.sh")); writeFixture(path.join("scripts", "managed-bootstrap-entrypoint.c")); writeFixture(path.join("scripts", "managed-bootstrap-trampoline.sh")); + writeFixture(path.join("scripts", "jetson-device-group-bootstrap.sh")); writeFixture(path.join("scripts", "gateway-control.sh")); writeFixture(path.join("scripts", "managed-gateway-control.py")); writeFixture(path.join("scripts", "state-dir-guard.py")); @@ -580,6 +581,9 @@ describe("sandbox build context staging", () => { expect(fs.existsSync(path.join(buildCtx, "scripts", "managed-bootstrap-trampoline.sh"))).toBe( true, ); + expect( + fs.existsSync(path.join(buildCtx, "scripts", "jetson-device-group-bootstrap.sh")), + ).toBe(true); expect(fs.existsSync(path.join(buildCtx, "scripts", "gateway-control.sh"))).toBe(true); expect(fs.existsSync(path.join(buildCtx, "scripts", "managed-gateway-control.py"))).toBe( true, From 82551188fbd95c4ce5de76d2098eaf0b281938f0 Mon Sep 17 00:00:00 2001 From: San Dang Date: Thu, 6 Aug 2026 19:03:49 +0700 Subject: [PATCH 24/49] chore(onboard): add Jetson GPU diagnostic Signed-off-by: San Dang --- scripts/diagnose-jetson-gpu.sh | 211 +++++++++++++++++++++++++++++++++ 1 file changed, 211 insertions(+) create mode 100755 scripts/diagnose-jetson-gpu.sh diff --git a/scripts/diagnose-jetson-gpu.sh b/scripts/diagnose-jetson-gpu.sh new file mode 100755 index 0000000000..3acb03718d --- /dev/null +++ b/scripts/diagnose-jetson-gpu.sh @@ -0,0 +1,211 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +sandbox_name="${1:-tm}" + +if [[ ! "$sandbox_name" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]*$ ]]; then + printf 'Invalid sandbox name: %s\n' "$sandbox_name" >&2 + exit 2 +fi + +section() { + printf '\n=== %s ===\n' "$1" +} + +run() { + printf '$' + printf ' %q' "$@" + printf '\n' + if "$@" 2>&1; then + return 0 + else + local rc=$? + printf '[exit=%d]\n' "$rc" + return 0 + fi +} + +run_labeled() { + local label="$1" + shift + printf '$ %s\n' "$label" + if "$@" 2>&1; then + return 0 + else + local rc=$? + printf '[exit=%d]\n' "$rc" + return 0 + fi +} + +print_host_device_nodes() { + local path + shopt -s nullglob + for path in /dev/nvmap /dev/nvhost-* /dev/nvgpu/* /dev/nvgpu/*/*; do + stat -Lc 'type=%F mode=%a uid=%u gid=%g major=%t minor=%T path=%n' "$path" 2>&1 || true + done + shopt -u nullglob +} + +collect_device_gids() { + local path gid + shopt -s nullglob + for path in /dev/nvmap /dev/nvhost-* /dev/nvgpu/* /dev/nvgpu/*/*; do + [[ -c "$path" ]] || continue + gid="$(stat -Lc '%g' "$path")" + [[ "$gid" =~ ^[0-9]+$ ]] || continue + printf '%s\n' "$gid" + done + shopt -u nullglob +} + +section "Host" +run date -u '+%Y-%m-%dT%H:%M:%SZ' +run uname -a +run id +if [[ -r /etc/nv_tegra_release ]]; then + run sed -n '1,3p' /etc/nv_tegra_release +fi +print_host_device_nodes + +section "OpenShell" +run openshell --version +run openshell sandbox list +printf '$ openshell policy get --base %q | filter Jetson paths\n' "$sandbox_name" +policy_output="$(openshell policy get --base "$sandbox_name" 2>&1)" || policy_rc=$? +printf '%s\n' "$policy_output" \ + | grep -E 'read_only:|read_write:|/opt/nvidia|/dev/nvmap|/dev/nvhost|/dev/nvgpu' \ + || true +if [[ -n "${policy_rc:-}" ]]; then + printf '[exit=%d]\n' "$policy_rc" +fi + +section "Matching Docker containers" +docker_rows="$(docker ps -a --no-trunc --format '{{.ID}}\t{{.Names}}\t{{.Status}}' 2>&1)" \ + || { + printf '%s\n' "$docker_rows" + exit 1 + } +printf '%s\n' "$docker_rows" \ + | awk -F '\t' -v prefix="openshell-${sandbox_name}-" 'index($2, prefix) == 1 { print }' + +candidate_ids=() +while IFS=$'\t' read -r short_id container_name; do + if [[ "$container_name" == "openshell-${sandbox_name}-"* ]] \ + && [[ "$container_name" != *-nemoclaw-gpu-backup-* ]]; then + candidate_ids+=("$short_id") + fi +done < <(docker ps --format '{{.ID}}\t{{.Names}}') + +if ((${#candidate_ids[@]} == 0)); then + printf 'No running non-backup Docker container found for sandbox %s.\n' "$sandbox_name" >&2 + exit 1 +fi +if ((${#candidate_ids[@]} > 1)); then + printf 'Warning: found %d running containers; using Docker newest-first candidate %s.\n' \ + "${#candidate_ids[@]}" "${candidate_ids[0]}" +fi + +cid="$(docker inspect --format '{{.Id}}' "${candidate_ids[0]}")" + +section "Active container configuration" +run docker inspect --format 'id={{.Id}} name={{.Name}} created={{.Created}} status={{.State.Status}} pid={{.State.Pid}} health={{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$cid" +run docker inspect --format 'image={{.Config.Image}} runtime={{.HostConfig.Runtime}} user={{json .Config.User}}' "$cid" +run docker inspect --format 'entrypoint={{json .Config.Entrypoint}}' "$cid" +run docker inspect --format 'cmd={{json .Config.Cmd}}' "$cid" +run docker inspect --format 'group_add={{json .HostConfig.GroupAdd}}' "$cid" +run docker inspect --format 'devices={{json .HostConfig.Devices}}' "$cid" +run docker inspect --format 'device_requests={{json .HostConfig.DeviceRequests}}' "$cid" +run docker inspect --format 'security_opt={{json .HostConfig.SecurityOpt}}' "$cid" +printf '$ docker inspect NVIDIA_* environment names\n' +docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' "$cid" \ + | sed -n -E 's/^(NVIDIA_[A-Z0-9_]+)=.*/\1=/p' + +container_pid="$(docker inspect --format '{{.State.Pid}}' "$cid")" +if [[ "$container_pid" =~ ^[1-9][0-9]*$ ]] && [[ -r "/proc/$container_pid/status" ]]; then + section "Container init process on host" + run sed -n -E '/^(Name|Pid|PPid|Uid|Gid|Groups):/p' "/proc/$container_pid/status" +fi + +section "Container process identities" +run docker top "$cid" -eo pid,ppid,user,group,comm +run docker exec -u 0 "$cid" sh -c 'printf "pid1_cmd="; tr "\\000" " " Date: Thu, 6 Aug 2026 19:13:52 +0700 Subject: [PATCH 25/49] chore(onboard): add Jetson device-group proof Signed-off-by: San Dang --- scripts/prove-jetson-device-groups.sh | 211 ++++++++++++++++++++++++++ 1 file changed, 211 insertions(+) create mode 100755 scripts/prove-jetson-device-groups.sh diff --git a/scripts/prove-jetson-device-groups.sh b/scripts/prove-jetson-device-groups.sh new file mode 100755 index 0000000000..b0b17bf36d --- /dev/null +++ b/scripts/prove-jetson-device-groups.sh @@ -0,0 +1,211 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +sandbox_name="${1:-tm}" + +if [[ ! "$sandbox_name" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]*$ ]]; then + printf 'Invalid sandbox name: %s\n' "$sandbox_name" >&2 + exit 2 +fi + +collect_device_gids() { + local path gid mode file_type group_digit other_digit group_access other_access + local paths=( + /dev/nvmap + /dev/nvhost-ctrl + /dev/nvhost-ctrl-gpu + /dev/nvhost-gpu + /dev/nvhost-as-gpu + /dev/nvhost-prof-gpu + /dev/nvhost-dbg-gpu + /dev/nvhost-tsg-gpu + /dev/nvgpu/igpu0/ctrl + /dev/nvgpu/igpu0/as + /dev/nvgpu/igpu0/prof + /dev/dri/renderD* + ) + shopt -s nullglob + for path in "${paths[@]}"; do + read -r gid mode file_type < <(stat -Lc '%g %a %F' "$path" 2>/dev/null) || continue + [[ "$file_type" == "character special file" ]] || continue + [[ "$gid" =~ ^[1-9][0-9]{0,9}$ ]] || continue + ((gid <= 2147483647)) || continue + group_digit=$(((10#$mode / 10) % 10)) + other_digit=$((10#$mode % 10)) + group_access=$((group_digit & 6)) + other_access=$((other_digit & 6)) + if (((group_access & ~other_access) != 0)); then + printf '%s\n' "$gid" + fi + done + shopt -u nullglob +} + +candidate_ids=() +while IFS=$'\t' read -r short_id container_name; do + if [[ "$container_name" == "openshell-${sandbox_name}-"* ]] \ + && [[ "$container_name" != *-nemoclaw-gpu-backup-* ]]; then + candidate_ids+=("$short_id") + fi +done < <(docker ps --format '{{.ID}}\t{{.Names}}') + +if ((${#candidate_ids[@]} != 1)); then + printf 'Expected one running non-backup Docker container for sandbox %s; found %d.\n' \ + "$sandbox_name" "${#candidate_ids[@]}" >&2 + exit 1 +fi + +container_id="$(docker inspect --format '{{.Id}}' "${candidate_ids[0]}")" +image_id="$(docker inspect --format '{{.Image}}' "$container_id")" +if [[ ! "$image_id" =~ ^sha256:[0-9a-f]{64}$ ]]; then + printf 'Could not resolve the immutable sandbox image ID.\n' >&2 + exit 1 +fi + +gids=() +while IFS= read -r gid; do + [[ -n "$gid" ]] && gids+=("$gid") +done < <(collect_device_gids | sort -nu) +if ((${#gids[@]} == 0 || ${#gids[@]} > 16)); then + printf 'Detected an invalid Jetson device group count: %d.\n' "${#gids[@]}" >&2 + exit 1 +fi + +saved_ifs="$IFS" +IFS=, +gid_csv="${gids[*]}" +IFS="$saved_ifs" + +container_probe="$({ + cat <<'PROBE' +set -euo pipefail + +mode="$1" +group_gids="$2" + +if [[ "$mode" == "proposed" ]]; then + [[ -f /etc/group && ! -L /etc/group ]] + IFS=',' read -r -a gids <<<"$group_gids" + for gid in "${gids[@]}"; do + group_record="$(getent group "$gid" || true)" + if [[ -z "$group_record" ]]; then + group_name="nemoclaw_gpu_$gid" + groupadd --gid "$gid" "$group_name" + else + IFS=':' read -r group_name _ resolved_gid _ <<<"$group_record" + [[ -n "$group_name" && "$resolved_gid" == "$gid" ]] + fi + usermod --append --groups "$group_name" sandbox + done +fi + +printf 'group_database_identity: ' +id sandbox + +python3 - <<'PY' +import ctypes +import os +import pwd +import stat +import sys + +account = pwd.getpwnam("sandbox") +os.initgroups(account.pw_name, account.pw_gid) +os.setgid(account.pw_gid) +os.setuid(account.pw_uid) +print(f"post_initgroups_identity=uid={os.getuid()} gid={os.getgid()} groups={os.getgroups()}") + +path = "/dev/nvmap" +try: + info = os.stat(path) + node_type = "char" if stat.S_ISCHR(info.st_mode) else "directory" if stat.S_ISDIR(info.st_mode) else "other" + print( + f"nvmap_stat=type={node_type} mode={info.st_mode & 0o777:o} " + f"uid={info.st_uid} gid={info.st_gid}" + ) +except Exception as error: + print(f"nvmap_stat_error={type(error).__name__}: {error}") + +try: + fd = os.open(path, os.O_RDWR) + os.close(fd) + print("nvmap_open_read_write=ok") +except Exception as error: + print(f"nvmap_open_read_write={type(error).__name__}: {error}") + +try: + cuda = ctypes.CDLL("libcuda.so.1") + cuda.cuInit.argtypes = [ctypes.c_uint] + cuda.cuInit.restype = ctypes.c_int + print("libcuda_load=ok") + result = cuda.cuInit(0) + print(f"cuInit(0)={result}") + raise SystemExit(0 if result == 0 else 10) +except OSError as error: + print(f"libcuda_error=OSError: {error}") + raise SystemExit(11) +PY +PROBE +})" + +docker_args=( + run + --rm + --network none + --user 0 + --runtime nvidia + --env NVIDIA_VISIBLE_DEVICES=all + --env "NVIDIA_DRIVER_CAPABILITIES=compute,utility" + --cap-add SYS_PTRACE + --security-opt apparmor=unconfined +) +for gid in "${gids[@]}"; do + docker_args+=(--group-add "$gid") +done +docker_args+=(--entrypoint /bin/bash "$image_id" -c "$container_probe" --) + +run_case() { + local mode="$1" + if case_output="$(docker "${docker_args[@]}" "$mode" "$gid_csv" 2>&1)"; then + case_rc=0 + else + case_rc=$? + fi +} + +printf 'sandbox=%s\ncontainer=%s\nimage=%s\ndetected_device_gids=%s\n' \ + "$sandbox_name" "$container_id" "$image_id" "$gid_csv" +printf 'The proof uses disposable --rm containers and does not modify the OpenShell sandbox.\n' + +printf '\n=== Baseline: Docker groups followed by unchanged initgroups() ===\n' +run_case baseline +baseline_output="$case_output" +baseline_rc="$case_rc" +printf '%s\ncase_exit=%d\n' "$baseline_output" "$baseline_rc" + +printf '\n=== Proposed: update /etc/group before the same initgroups() ===\n' +run_case proposed +proposed_output="$case_output" +proposed_rc="$case_rc" +printf '%s\ncase_exit=%d\n' "$proposed_output" "$proposed_rc" + +baseline_cuda="$(printf '%s\n' "$baseline_output" | sed -n -E 's/^cuInit\(0\)=([0-9]+)$/\1/p' | tail -1)" +proposed_cuda="$(printf '%s\n' "$proposed_output" | sed -n -E 's/^cuInit\(0\)=([0-9]+)$/\1/p' | tail -1)" + +if [[ "$baseline_rc" -ne 0 && + -n "$baseline_cuda" && + "$baseline_cuda" -ne 0 && + "$baseline_output" == *"libcuda_load=ok"* && + "$baseline_output" == *"nvmap_open_read_write=PermissionError"* && + "$proposed_rc" -eq 0 && + "$proposed_cuda" == "0" && + "$proposed_output" == *"nvmap_open_read_write=ok"* ]]; then + printf '\nPROVEN: rebuilding sandbox supplementary groups from the unchanged container group database causes the failure; recording the device groups before initgroups fixes both nvmap access and CUDA initialization.\n' + exit 0 +fi + +printf '\nINCONCLUSIVE: the A/B result did not isolate device-group persistence. Do not implement the proposed change from this result.\n' >&2 +exit 1 From 0a65abe8629e63fab71e0431cd515f50956f4c6e Mon Sep 17 00:00:00 2001 From: San Dang Date: Thu, 6 Aug 2026 19:34:59 +0700 Subject: [PATCH 26/49] fix(onboard): preserve Jetson groups in compatibility path Signed-off-by: San Dang --- docs/reference/troubleshooting.mdx | 22 +- .../checks/run-managed-image-openshell-e2e.ts | 1 + scripts/diagnose-jetson-gpu.sh | 211 ------------------ scripts/jetson-device-group-bootstrap.sh | 4 +- scripts/prove-jetson-device-groups.sh | 211 ------------------ src/lib/onboard.ts | 1 + .../sandbox-gpu-create-flow.ts | 1 + src/lib/onboard/docker-gpu-patch-clone.ts | 13 +- .../onboard/docker-gpu-patch-jetson.test.ts | 73 ++++-- src/lib/onboard/docker-gpu-patch-recreate.ts | 36 ++- ...ocker-gpu-sandbox-create-lifecycle.test.ts | 6 +- src/lib/onboard/docker-gpu-sandbox-create.ts | 2 + .../managed-bootstrap/docker-test-fixture.ts | 27 --- .../onboard/managed-bootstrap/docker.test.ts | 33 --- src/lib/onboard/managed-bootstrap/docker.ts | 52 +---- .../onboard/sandbox-gpu-create-flow.test.ts | 19 +- src/lib/onboard/sandbox-gpu-create-flow.ts | 1 + .../onboard/sandbox-gpu-create-run-attempt.ts | 1 + 18 files changed, 147 insertions(+), 567 deletions(-) delete mode 100755 scripts/diagnose-jetson-gpu.sh delete mode 100755 scripts/prove-jetson-device-groups.sh diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index c578c0ce54..d699add4f1 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -2725,31 +2725,33 @@ The path creates the sandbox and then recreates the OpenShell-managed Docker con `NEMOCLAW_DOCKER_GPU_PATCH=0` is ignored because this runtime requires the compatibility patch for GPU passthrough, and onboarding logs a warning when it is set. To skip GPU passthrough entirely, rerun with `--no-gpu` or set `NEMOCLAW_SANDBOX_GPU=0`. + + #### Jetson and Tegra compatibility default Automatic GPU onboarding uses the compatibility path directly; it does not make a native attempt first. The path recreates the OpenShell-managed Docker container with NVIDIA GPU flags and propagates eligible host group IDs for the supported Jetson GPU device nodes. - -For OpenClaw, Docker compatibility recreation passes those group IDs with `--group-add`. -OpenShell 0.0.85 calls `initgroups()` from the container group database before it starts the sandbox account. +For legacy OpenClaw Jetson compatibility recreation, Docker passes those group IDs with `--group-add`. +OpenShell 0.0.85 calls `initgroups()` from the unchanged container group database before it starts the sandbox account. That call replaces the inherited supplementary groups, so `--group-add` alone does not preserve Jetson device access. -The managed OpenClaw Jetson replacement container runs a bounded, root-owned wrapper from the sandbox image before the OpenShell supervisor starts. -The wrapper is `root:root` with mode `0500`. -The wrapper can hand off only to `/usr/local/bin/nemoclaw-managed-bootstrap`. -That entrypoint resumes the fixed `/opt/openshell/bin/openshell-sandbox` supervisor. +Before the fixed OpenShell supervisor starts, NemoClaw runs a bounded wrapper from the sandbox image as root. +The image owns the wrapper as `root:root` with mode `0500`. +The wrapper can hand off only to `/opt/openshell/bin/openshell-sandbox`. The wrapper adds only the validated Jetson device GIDs that onboarding detected to the existing sandbox account in `/etc/group`. It verifies the resulting membership before handoff. -Legacy and custom image recreation, generic GPU, CPU-only, Hermes, and Deep Agents paths do not use this wrapper. +OpenShell then rebuilds the account's group list from the updated database, preserving access to the detected device nodes. +This wrapper runs only when legacy OpenClaw Jetson recreation preserves the fixed supervisor entrypoint. The creation-time filesystem policy adds no Jetson entries unless `/dev/nvmap` is an existing, non-symlink character device. When that condition is met, the policy adds `/opt/nvidia` as read-only. It adds each existing, non-symlink character device on the eligible GPU path list as read-write. -Generic GPU, CPU-only, Hermes, and Deep Agents policies do not receive these Jetson entries. +Generic GPU and CPU-only policies do not receive these Jetson entries. - Use `NEMOCLAW_DOCKER_GPU_PATCH=0` only for troubleshooting because it bypasses that group propagation and CUDA may not initialize. + + #### Common compatibility-path recovery After compatibility recreation starts, onboarding keeps the pre-patch container as a rollback backup until the replacement passes the Ready, direct GPU, and applicable local-inference checks. diff --git a/scripts/checks/run-managed-image-openshell-e2e.ts b/scripts/checks/run-managed-image-openshell-e2e.ts index c948c763dc..5a60457217 100644 --- a/scripts/checks/run-managed-image-openshell-e2e.ts +++ b/scripts/checks/run-managed-image-openshell-e2e.ts @@ -786,6 +786,7 @@ async function run(input: Inputs): Promise { flow = await runSandboxGpuCreateFlow( { sandboxName: input.sandbox, + agentName: input.agent, provider: input.localProvider ? resolveManagedImageLocalInferenceRoute(input.localProvider).providerName : "nvidia", diff --git a/scripts/diagnose-jetson-gpu.sh b/scripts/diagnose-jetson-gpu.sh deleted file mode 100755 index 3acb03718d..0000000000 --- a/scripts/diagnose-jetson-gpu.sh +++ /dev/null @@ -1,211 +0,0 @@ -#!/usr/bin/env bash -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -set -euo pipefail - -sandbox_name="${1:-tm}" - -if [[ ! "$sandbox_name" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]*$ ]]; then - printf 'Invalid sandbox name: %s\n' "$sandbox_name" >&2 - exit 2 -fi - -section() { - printf '\n=== %s ===\n' "$1" -} - -run() { - printf '$' - printf ' %q' "$@" - printf '\n' - if "$@" 2>&1; then - return 0 - else - local rc=$? - printf '[exit=%d]\n' "$rc" - return 0 - fi -} - -run_labeled() { - local label="$1" - shift - printf '$ %s\n' "$label" - if "$@" 2>&1; then - return 0 - else - local rc=$? - printf '[exit=%d]\n' "$rc" - return 0 - fi -} - -print_host_device_nodes() { - local path - shopt -s nullglob - for path in /dev/nvmap /dev/nvhost-* /dev/nvgpu/* /dev/nvgpu/*/*; do - stat -Lc 'type=%F mode=%a uid=%u gid=%g major=%t minor=%T path=%n' "$path" 2>&1 || true - done - shopt -u nullglob -} - -collect_device_gids() { - local path gid - shopt -s nullglob - for path in /dev/nvmap /dev/nvhost-* /dev/nvgpu/* /dev/nvgpu/*/*; do - [[ -c "$path" ]] || continue - gid="$(stat -Lc '%g' "$path")" - [[ "$gid" =~ ^[0-9]+$ ]] || continue - printf '%s\n' "$gid" - done - shopt -u nullglob -} - -section "Host" -run date -u '+%Y-%m-%dT%H:%M:%SZ' -run uname -a -run id -if [[ -r /etc/nv_tegra_release ]]; then - run sed -n '1,3p' /etc/nv_tegra_release -fi -print_host_device_nodes - -section "OpenShell" -run openshell --version -run openshell sandbox list -printf '$ openshell policy get --base %q | filter Jetson paths\n' "$sandbox_name" -policy_output="$(openshell policy get --base "$sandbox_name" 2>&1)" || policy_rc=$? -printf '%s\n' "$policy_output" \ - | grep -E 'read_only:|read_write:|/opt/nvidia|/dev/nvmap|/dev/nvhost|/dev/nvgpu' \ - || true -if [[ -n "${policy_rc:-}" ]]; then - printf '[exit=%d]\n' "$policy_rc" -fi - -section "Matching Docker containers" -docker_rows="$(docker ps -a --no-trunc --format '{{.ID}}\t{{.Names}}\t{{.Status}}' 2>&1)" \ - || { - printf '%s\n' "$docker_rows" - exit 1 - } -printf '%s\n' "$docker_rows" \ - | awk -F '\t' -v prefix="openshell-${sandbox_name}-" 'index($2, prefix) == 1 { print }' - -candidate_ids=() -while IFS=$'\t' read -r short_id container_name; do - if [[ "$container_name" == "openshell-${sandbox_name}-"* ]] \ - && [[ "$container_name" != *-nemoclaw-gpu-backup-* ]]; then - candidate_ids+=("$short_id") - fi -done < <(docker ps --format '{{.ID}}\t{{.Names}}') - -if ((${#candidate_ids[@]} == 0)); then - printf 'No running non-backup Docker container found for sandbox %s.\n' "$sandbox_name" >&2 - exit 1 -fi -if ((${#candidate_ids[@]} > 1)); then - printf 'Warning: found %d running containers; using Docker newest-first candidate %s.\n' \ - "${#candidate_ids[@]}" "${candidate_ids[0]}" -fi - -cid="$(docker inspect --format '{{.Id}}' "${candidate_ids[0]}")" - -section "Active container configuration" -run docker inspect --format 'id={{.Id}} name={{.Name}} created={{.Created}} status={{.State.Status}} pid={{.State.Pid}} health={{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$cid" -run docker inspect --format 'image={{.Config.Image}} runtime={{.HostConfig.Runtime}} user={{json .Config.User}}' "$cid" -run docker inspect --format 'entrypoint={{json .Config.Entrypoint}}' "$cid" -run docker inspect --format 'cmd={{json .Config.Cmd}}' "$cid" -run docker inspect --format 'group_add={{json .HostConfig.GroupAdd}}' "$cid" -run docker inspect --format 'devices={{json .HostConfig.Devices}}' "$cid" -run docker inspect --format 'device_requests={{json .HostConfig.DeviceRequests}}' "$cid" -run docker inspect --format 'security_opt={{json .HostConfig.SecurityOpt}}' "$cid" -printf '$ docker inspect NVIDIA_* environment names\n' -docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' "$cid" \ - | sed -n -E 's/^(NVIDIA_[A-Z0-9_]+)=.*/\1=/p' - -container_pid="$(docker inspect --format '{{.State.Pid}}' "$cid")" -if [[ "$container_pid" =~ ^[1-9][0-9]*$ ]] && [[ -r "/proc/$container_pid/status" ]]; then - section "Container init process on host" - run sed -n -E '/^(Name|Pid|PPid|Uid|Gid|Groups):/p' "/proc/$container_pid/status" -fi - -section "Container process identities" -run docker top "$cid" -eo pid,ppid,user,group,comm -run docker exec -u 0 "$cid" sh -c 'printf "pid1_cmd="; tr "\\000" " " /dev/null 2>&1 || fail "sandbox user is missing" [ -f /etc/group ] && [ ! -L /etc/group ] || fail "container group database is invalid" diff --git a/scripts/prove-jetson-device-groups.sh b/scripts/prove-jetson-device-groups.sh deleted file mode 100755 index b0b17bf36d..0000000000 --- a/scripts/prove-jetson-device-groups.sh +++ /dev/null @@ -1,211 +0,0 @@ -#!/usr/bin/env bash -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -set -euo pipefail - -sandbox_name="${1:-tm}" - -if [[ ! "$sandbox_name" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]*$ ]]; then - printf 'Invalid sandbox name: %s\n' "$sandbox_name" >&2 - exit 2 -fi - -collect_device_gids() { - local path gid mode file_type group_digit other_digit group_access other_access - local paths=( - /dev/nvmap - /dev/nvhost-ctrl - /dev/nvhost-ctrl-gpu - /dev/nvhost-gpu - /dev/nvhost-as-gpu - /dev/nvhost-prof-gpu - /dev/nvhost-dbg-gpu - /dev/nvhost-tsg-gpu - /dev/nvgpu/igpu0/ctrl - /dev/nvgpu/igpu0/as - /dev/nvgpu/igpu0/prof - /dev/dri/renderD* - ) - shopt -s nullglob - for path in "${paths[@]}"; do - read -r gid mode file_type < <(stat -Lc '%g %a %F' "$path" 2>/dev/null) || continue - [[ "$file_type" == "character special file" ]] || continue - [[ "$gid" =~ ^[1-9][0-9]{0,9}$ ]] || continue - ((gid <= 2147483647)) || continue - group_digit=$(((10#$mode / 10) % 10)) - other_digit=$((10#$mode % 10)) - group_access=$((group_digit & 6)) - other_access=$((other_digit & 6)) - if (((group_access & ~other_access) != 0)); then - printf '%s\n' "$gid" - fi - done - shopt -u nullglob -} - -candidate_ids=() -while IFS=$'\t' read -r short_id container_name; do - if [[ "$container_name" == "openshell-${sandbox_name}-"* ]] \ - && [[ "$container_name" != *-nemoclaw-gpu-backup-* ]]; then - candidate_ids+=("$short_id") - fi -done < <(docker ps --format '{{.ID}}\t{{.Names}}') - -if ((${#candidate_ids[@]} != 1)); then - printf 'Expected one running non-backup Docker container for sandbox %s; found %d.\n' \ - "$sandbox_name" "${#candidate_ids[@]}" >&2 - exit 1 -fi - -container_id="$(docker inspect --format '{{.Id}}' "${candidate_ids[0]}")" -image_id="$(docker inspect --format '{{.Image}}' "$container_id")" -if [[ ! "$image_id" =~ ^sha256:[0-9a-f]{64}$ ]]; then - printf 'Could not resolve the immutable sandbox image ID.\n' >&2 - exit 1 -fi - -gids=() -while IFS= read -r gid; do - [[ -n "$gid" ]] && gids+=("$gid") -done < <(collect_device_gids | sort -nu) -if ((${#gids[@]} == 0 || ${#gids[@]} > 16)); then - printf 'Detected an invalid Jetson device group count: %d.\n' "${#gids[@]}" >&2 - exit 1 -fi - -saved_ifs="$IFS" -IFS=, -gid_csv="${gids[*]}" -IFS="$saved_ifs" - -container_probe="$({ - cat <<'PROBE' -set -euo pipefail - -mode="$1" -group_gids="$2" - -if [[ "$mode" == "proposed" ]]; then - [[ -f /etc/group && ! -L /etc/group ]] - IFS=',' read -r -a gids <<<"$group_gids" - for gid in "${gids[@]}"; do - group_record="$(getent group "$gid" || true)" - if [[ -z "$group_record" ]]; then - group_name="nemoclaw_gpu_$gid" - groupadd --gid "$gid" "$group_name" - else - IFS=':' read -r group_name _ resolved_gid _ <<<"$group_record" - [[ -n "$group_name" && "$resolved_gid" == "$gid" ]] - fi - usermod --append --groups "$group_name" sandbox - done -fi - -printf 'group_database_identity: ' -id sandbox - -python3 - <<'PY' -import ctypes -import os -import pwd -import stat -import sys - -account = pwd.getpwnam("sandbox") -os.initgroups(account.pw_name, account.pw_gid) -os.setgid(account.pw_gid) -os.setuid(account.pw_uid) -print(f"post_initgroups_identity=uid={os.getuid()} gid={os.getgid()} groups={os.getgroups()}") - -path = "/dev/nvmap" -try: - info = os.stat(path) - node_type = "char" if stat.S_ISCHR(info.st_mode) else "directory" if stat.S_ISDIR(info.st_mode) else "other" - print( - f"nvmap_stat=type={node_type} mode={info.st_mode & 0o777:o} " - f"uid={info.st_uid} gid={info.st_gid}" - ) -except Exception as error: - print(f"nvmap_stat_error={type(error).__name__}: {error}") - -try: - fd = os.open(path, os.O_RDWR) - os.close(fd) - print("nvmap_open_read_write=ok") -except Exception as error: - print(f"nvmap_open_read_write={type(error).__name__}: {error}") - -try: - cuda = ctypes.CDLL("libcuda.so.1") - cuda.cuInit.argtypes = [ctypes.c_uint] - cuda.cuInit.restype = ctypes.c_int - print("libcuda_load=ok") - result = cuda.cuInit(0) - print(f"cuInit(0)={result}") - raise SystemExit(0 if result == 0 else 10) -except OSError as error: - print(f"libcuda_error=OSError: {error}") - raise SystemExit(11) -PY -PROBE -})" - -docker_args=( - run - --rm - --network none - --user 0 - --runtime nvidia - --env NVIDIA_VISIBLE_DEVICES=all - --env "NVIDIA_DRIVER_CAPABILITIES=compute,utility" - --cap-add SYS_PTRACE - --security-opt apparmor=unconfined -) -for gid in "${gids[@]}"; do - docker_args+=(--group-add "$gid") -done -docker_args+=(--entrypoint /bin/bash "$image_id" -c "$container_probe" --) - -run_case() { - local mode="$1" - if case_output="$(docker "${docker_args[@]}" "$mode" "$gid_csv" 2>&1)"; then - case_rc=0 - else - case_rc=$? - fi -} - -printf 'sandbox=%s\ncontainer=%s\nimage=%s\ndetected_device_gids=%s\n' \ - "$sandbox_name" "$container_id" "$image_id" "$gid_csv" -printf 'The proof uses disposable --rm containers and does not modify the OpenShell sandbox.\n' - -printf '\n=== Baseline: Docker groups followed by unchanged initgroups() ===\n' -run_case baseline -baseline_output="$case_output" -baseline_rc="$case_rc" -printf '%s\ncase_exit=%d\n' "$baseline_output" "$baseline_rc" - -printf '\n=== Proposed: update /etc/group before the same initgroups() ===\n' -run_case proposed -proposed_output="$case_output" -proposed_rc="$case_rc" -printf '%s\ncase_exit=%d\n' "$proposed_output" "$proposed_rc" - -baseline_cuda="$(printf '%s\n' "$baseline_output" | sed -n -E 's/^cuInit\(0\)=([0-9]+)$/\1/p' | tail -1)" -proposed_cuda="$(printf '%s\n' "$proposed_output" | sed -n -E 's/^cuInit\(0\)=([0-9]+)$/\1/p' | tail -1)" - -if [[ "$baseline_rc" -ne 0 && - -n "$baseline_cuda" && - "$baseline_cuda" -ne 0 && - "$baseline_output" == *"libcuda_load=ok"* && - "$baseline_output" == *"nvmap_open_read_write=PermissionError"* && - "$proposed_rc" -eq 0 && - "$proposed_cuda" == "0" && - "$proposed_output" == *"nvmap_open_read_write=ok"* ]]; then - printf '\nPROVEN: rebuilding sandbox supplementary groups from the unchanged container group database causes the failure; recording the device groups before initgroups fixes both nvmap access and CUDA initialization.\n' - exit 0 -fi - -printf '\nINCONCLUSIVE: the A/B result did not isolate device-group persistence. Do not implement the proposed change from this result.\n' >&2 -exit 1 diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 499cb51b88..ce9055d2cd 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -2641,6 +2641,7 @@ async function createSandboxWithBaseImageResolution( } = await sandboxGpuCreateFlow.runSandboxGpuCreateFlow( { sandboxName, + agentName: agent?.name ?? "openclaw", provider, sandboxGpuConfig: effectiveSandboxGpuConfig, gpuRoutePlan, diff --git a/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts b/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts index eebacbacd3..c65ff161d3 100644 --- a/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts @@ -21,6 +21,7 @@ export const GPU_IMAGE_ID = `sha256:${"a".repeat(64)}`; export function createGpuFlowInput(): SandboxGpuCreateFlowInput { return { sandboxName: "alpha", + agentName: "openclaw", provider: "nim", sandboxGpuConfig: { mode: "1", diff --git a/src/lib/onboard/docker-gpu-patch-clone.ts b/src/lib/onboard/docker-gpu-patch-clone.ts index c49f3c3e1f..216e48e378 100644 --- a/src/lib/onboard/docker-gpu-patch-clone.ts +++ b/src/lib/onboard/docker-gpu-patch-clone.ts @@ -10,8 +10,9 @@ import type { import { openshellSandboxCommandEnvValue } from "./docker-startup-command-env"; const OPENSHELL_SANDBOX_COMMAND_ENV = "OPENSHELL_SANDBOX_COMMAND"; -const MANAGED_BOOTSTRAP_ENTRYPOINT = "/usr/local/bin/nemoclaw-managed-bootstrap"; -const JETSON_DEVICE_GROUP_BOOTSTRAP = "/usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh"; +const OPENSHELL_SANDBOX_ENTRYPOINT = "/opt/openshell/bin/openshell-sandbox"; +export const JETSON_DEVICE_GROUP_BOOTSTRAP = + "/usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh"; const MAX_JETSON_DEVICE_GROUPS = 16; const GPU_ENV_KEYS = new Set([ "NVIDIA_VISIBLE_DEVICES", @@ -483,9 +484,9 @@ export function buildDockerGpuCloneRunArgs( const entrypoint = stringArray(config.Entrypoint); const replacementEntrypoint = String(options.containerEntrypoint ?? "").trim(); - const managedBootstrapTarget = replacementEntrypoint === MANAGED_BOOTSTRAP_ENTRYPOINT; - if (preserveJetsonGroups && (!managedBootstrapTarget || !options.containerCommand?.length)) { - throw new Error("Jetson device-group bootstrap requires the managed OpenClaw entrypoint."); + const groupBootstrapTarget = replacementEntrypoint || entrypoint[0] || ""; + if (preserveJetsonGroups && groupBootstrapTarget !== OPENSHELL_SANDBOX_ENTRYPOINT) { + throw new Error("Jetson device-group bootstrap requires the OpenShell supervisor entrypoint."); } if (preserveJetsonGroups) { args.push("--entrypoint", JETSON_DEVICE_GROUP_BOOTSTRAP); @@ -505,7 +506,7 @@ export function buildDockerGpuCloneRunArgs( "--device-group-gids", extraGroupGids.join(","), "--", - MANAGED_BOOTSTRAP_ENTRYPOINT, + groupBootstrapTarget, ...targetCommandArgs, ] : options.containerCommand diff --git a/src/lib/onboard/docker-gpu-patch-jetson.test.ts b/src/lib/onboard/docker-gpu-patch-jetson.test.ts index 1aae15c626..55d516dbee 100644 --- a/src/lib/onboard/docker-gpu-patch-jetson.test.ts +++ b/src/lib/onboard/docker-gpu-patch-jetson.test.ts @@ -42,15 +42,13 @@ describe("Jetson device-node group propagation (#4231, #7610)", () => { expect(args).not.toEqual(expect.arrayContaining(["--group-add"])); }); - it("runs the managed bootstrap through the Jetson group bootstrap (#7610)", () => { + it("runs the OpenShell supervisor through the Jetson group bootstrap (#7610)", () => { const args = buildDockerGpuCloneRunArgs( inspectFixture(), buildDockerGpuMode("nvidia-runtime", null, { backend: "jetson" }), { extraGroupGids: ["44"], preserveJetsonDeviceGroupMembership: true, - containerEntrypoint: "/usr/local/bin/nemoclaw-managed-bootstrap", - containerCommand: ["--request", "/run/nemoclaw/bootstrap-request.json"], }, ); @@ -59,29 +57,27 @@ describe("Jetson device-node group propagation (#4231, #7610)", () => { "--device-group-gids", "44", "--", - "/usr/local/bin/nemoclaw-managed-bootstrap", - "--request", - "/run/nemoclaw/bootstrap-request.json", + "/opt/openshell/bin/openshell-sandbox", ]); }); - it("rejects Jetson group preservation outside the managed image boundary (#7610)", () => { + it("rejects Jetson group preservation outside the OpenShell supervisor boundary (#7610)", () => { + const inspect = inspectFixture(); + inspect.Config!.Entrypoint = ["/custom/entrypoint"]; expect(() => buildDockerGpuCloneRunArgs( - inspectFixture(), + inspect, buildDockerGpuMode("nvidia-runtime", null, { backend: "jetson" }), { extraGroupGids: ["44"], preserveJetsonDeviceGroupMembership: true, }, ), - ).toThrow("Jetson device-group bootstrap requires the managed OpenClaw entrypoint."); + ).toThrow("Jetson device-group bootstrap requires the OpenShell supervisor entrypoint."); }); it("rejects invalid or excessive supplementary group IDs before clone creation (#7610)", () => { const options = { - containerEntrypoint: "/usr/local/bin/nemoclaw-managed-bootstrap", - containerCommand: ["--request", "/run/nemoclaw/bootstrap-request.json"], preserveJetsonDeviceGroupMembership: true, } as const; const build = (extraGroupGids: readonly string[]) => @@ -100,7 +96,7 @@ describe("Jetson device-node group propagation (#4231, #7610)", () => { }); it("passes all detected Tegra device GIDs into the Jetson recreate as --group-add", () => { - const dockerRunDetached = vi.fn(() => ({ + const dockerRunDetached = vi.fn((_args: readonly string[], _options?: unknown) => ({ status: 0, stdout: "new-container-id\n", })); @@ -117,7 +113,12 @@ describe("Jetson device-node group propagation (#4231, #7610)", () => { ); recreateOpenShellDockerSandboxWithGpu( - { sandboxName: "alpha", timeoutSecs: 1, backend: "jetson" }, + { + sandboxName: "alpha", + timeoutSecs: 1, + backend: "jetson", + preserveJetsonDeviceGroupMembership: true, + }, { dockerCapture: dockerCaptureFixture(), dockerRun: vi.fn(() => ({ status: 0, stdout: "probe-id\n" })), @@ -138,6 +139,20 @@ describe("Jetson device-node group propagation (#4231, #7610)", () => { expect.arrayContaining(["--group-add", "44", "--group-add", "104", "--group-add", "995"]), expect.objectContaining({ ignoreError: true }), ); + const createArgs = dockerRunDetached.mock.calls[0]?.[0] ?? []; + expect(createArgs).toEqual( + expect.arrayContaining([ + "--entrypoint", + "/usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh", + ]), + ); + expect(createArgs.slice(createArgs.indexOf("openshell/sandbox:abc"))).toEqual([ + "openshell/sandbox:abc", + "--device-group-gids", + "44,104,995", + "--", + "/opt/openshell/bin/openshell-sandbox", + ]); }); it("does not add Tegra device GIDs for the generic (non-Jetson) backend", () => { @@ -170,4 +185,36 @@ describe("Jetson device-node group propagation (#4231, #7610)", () => { expect.anything(), ); }); + + it("refuses to stop the original container when the OpenClaw wrapper is missing (#7610)", () => { + const dockerStop = vi.fn(() => ({ status: 0 })); + + expect(() => + recreateOpenShellDockerSandboxWithGpu( + { + sandboxName: "alpha", + timeoutSecs: 1, + backend: "jetson", + preserveJetsonDeviceGroupMembership: true, + }, + { + dockerCapture: dockerCaptureFixture(), + dockerRun: vi.fn((args: readonly string[]) => ({ + status: args[0] === "exec" ? 1 : 0, + stdout: "probe-id\n", + })), + dockerRunDetached: vi.fn(() => ({ status: 0, stdout: "new-container-id\n" })), + dockerRename: vi.fn(() => ({ status: 0 })), + dockerStop, + dockerRm: vi.fn(() => ({ status: 0 })), + runOpenshell: vi.fn(() => ({ status: 0 })), + sleep: vi.fn(), + now: () => new Date("2026-05-15T00:00:00Z"), + detectSandboxFallbackDns: () => null, + detectTegraDeviceGroupGids: () => ["44"], + }, + ), + ).toThrow("OpenClaw sandbox image is missing executable"); + expect(dockerStop).not.toHaveBeenCalled(); + }); }); diff --git a/src/lib/onboard/docker-gpu-patch-recreate.ts b/src/lib/onboard/docker-gpu-patch-recreate.ts index 4eb1c0ad8d..03657c740e 100644 --- a/src/lib/onboard/docker-gpu-patch-recreate.ts +++ b/src/lib/onboard/docker-gpu-patch-recreate.ts @@ -18,6 +18,7 @@ import { buildDockerGpuCloneRunOptions, dockerContainerName, getDockerGpuCloneFallbackDns, + JETSON_DEVICE_GROUP_BOOTSTRAP, parseDockerInspectJson, sameContainerId, validateRequiredDockerUlimits, @@ -159,6 +160,7 @@ export function recreateOpenShellDockerSandboxContainer( requiredUlimits?: readonly import("./docker-gpu-patch-types").DockerUlimit[] | null; expectedOldContainerId?: string | null; backend?: "generic" | "jetson"; + preserveJetsonDeviceGroupMembership?: boolean; dockerDesktopWsl?: boolean; modeOverride?: DockerGpuPatchMode; }, @@ -266,11 +268,35 @@ export function recreateOpenShellDockerSandboxContainer( const tegraGroupGids = d.detectTegraDeviceGroupGids(); if (tegraGroupGids.length > 0) { cloneOptions.extraGroupGids = tegraGroupGids; - console.log( - ` ✓ Granting sandbox user the detected Jetson GPU device groups via --group-add ${tegraGroupGids.join( - ", ", - )} (so CUDA can initialize as a non-root user)`, - ); + if (options.preserveJetsonDeviceGroupMembership === true) { + const wrapperProbe = d.dockerRun( + [ + "exec", + "--user", + "0", + oldContainerId, + "/usr/bin/test", + "-x", + JETSON_DEVICE_GROUP_BOOTSTRAP, + ], + { ignoreError: true, suppressOutput: true, timeout: DOCKER_GPU_PATCH_TIMEOUT_MS }, + ); + if (!hasZeroDockerExitStatus(wrapperProbe)) { + throw new Error( + `OpenClaw sandbox image is missing executable ${JETSON_DEVICE_GROUP_BOOTSTRAP}.`, + ); + } + cloneOptions.preserveJetsonDeviceGroupMembership = true; + console.log( + ` ✓ Preserving the detected Jetson GPU device groups through OpenShell startup: ${tegraGroupGids.join(", ")}`, + ); + } else { + console.log( + ` ✓ Granting sandbox user the detected Jetson GPU device groups via --group-add ${tegraGroupGids.join( + ", ", + )} (so CUDA can initialize as a non-root user)`, + ); + } } else { console.warn( " ⚠ Could not resolve the group owning Jetson Tegra GPU device nodes (/dev/nvmap); CUDA may fail with NvRmMemInitNvmap permission denied. Confirm /dev/nvmap exists and is group-readable on the host.", diff --git a/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts b/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts index 0206ce9a80..0296b58835 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts @@ -59,6 +59,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { route: "compatibility", sandboxName: "alpha", timeoutSecs: 60, + preserveJetsonDeviceGroupMembership: true, deps, overrides: { findContainerIds, @@ -72,7 +73,10 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { patch.maybeApplyDuringCreate(); expect(recreatePatch).toHaveBeenCalledWith( - expect.objectContaining({ waitForSupervisor: false }), + expect.objectContaining({ + waitForSupervisor: false, + preserveJetsonDeviceGroupMembership: true, + }), expect.objectContaining({ runCaptureOpenshell: deps.runCaptureOpenshell, }), diff --git a/src/lib/onboard/docker-gpu-sandbox-create.ts b/src/lib/onboard/docker-gpu-sandbox-create.ts index d4356c79d0..8ccb34f2b2 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create.ts @@ -84,6 +84,7 @@ type DockerGpuSandboxCreatePatchOptions = { requiredUlimits?: Parameters[0]["requiredUlimits"]; timeoutSecs: number; backend?: DockerGpuPatchBackend; + preserveJetsonDeviceGroupMembership?: boolean; /** * Whether the host is Docker Desktop WSL. Defaults to the cached * `isDockerDesktopWslRuntime()` probe. When true, the GPU patch skips the CDI @@ -187,6 +188,7 @@ export function createDockerGpuSandboxCreatePatch( requiredUlimits: options.requiredUlimits ?? null, timeoutSecs: options.timeoutSecs, backend: options.backend, + preserveJetsonDeviceGroupMembership: options.preserveJetsonDeviceGroupMembership, dockerDesktopWsl: options.dockerDesktopWsl ?? isDockerDesktopWslRuntime(), }; const recreationEnabled = diff --git a/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts b/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts index dd232e46f1..4cb3672d02 100644 --- a/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts +++ b/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts @@ -147,17 +147,9 @@ function originalInspect(inputs = agentInputs()): DockerContainerInspect { Binds: ["/host/workspace:/sandbox:rw"], NetworkMode: "openshell", RestartPolicy: { Name: "unless-stopped" }, - CapAdd: null, CapDrop: ["NET_RAW"], - DeviceRequests: null, - Devices: null, - GroupAdd: null, - Runtime: null, SecurityOpt: ["no-new-privileges"], Ulimits: [{ Name: "nofile", Soft: 65_536, Hard: 65_536 }], - } as NonNullable & { - Devices?: unknown; - Runtime?: string | null; }, NetworkSettings: { Networks: { openshell: { Aliases: ["openshell-alpha"] } } }, }; @@ -365,16 +357,6 @@ export function fixture(options: DockerFixtureOptions = {}) { const env = args.flatMap((value, index) => value === "--env" ? [String(args[index + 1] ?? "")] : [], ); - const valuesAfter = (flag: string) => - args.flatMap((value, index) => (value === flag ? [String(args[index + 1] ?? "")] : [])); - const runtimeIndex = args.indexOf("--runtime"); - const sourceRuntime = ( - source.HostConfig as - | (NonNullable & { - Runtime?: string | null; - }) - | null - )?.Runtime; replacement = { ...structuredClone(source), Id: NEW_ID, @@ -386,15 +368,6 @@ export function fixture(options: DockerFixtureOptions = {}) { Entrypoint: [entrypoint], Cmd: args.slice(imageIndex + 1), }, - HostConfig: { - ...structuredClone(source.HostConfig), - CapAdd: valuesAfter("--cap-add"), - GroupAdd: valuesAfter("--group-add"), - Runtime: runtimeIndex >= 0 ? String(args[runtimeIndex + 1] ?? "") : sourceRuntime, - SecurityOpt: valuesAfter("--security-opt"), - } as NonNullable & { - Runtime?: string | null; - }, State: { Running: false, Paused: false, Restarting: false, Dead: false }, }; return losesAcknowledgement("container:create") diff --git a/src/lib/onboard/managed-bootstrap/docker.test.ts b/src/lib/onboard/managed-bootstrap/docker.test.ts index 82c5508a98..3a3d6f7c6a 100644 --- a/src/lib/onboard/managed-bootstrap/docker.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker.test.ts @@ -32,39 +32,6 @@ function expectEventBefore(events: readonly string[], before: string, after: str } describe("Docker managed bootstrap adapter", () => { - it("preserves OpenClaw Jetson groups across the managed bootstrap boundary (#7610)", async () => { - const fake = fixture({ agent: "openclaw" }); - const adapter = createDockerManagedBootstrapAdapter(fake.deps); - const { handle, request, snapshot } = authority("openclaw"); - - await expect( - adapter.prepareBootstrapReplacement({ - handle, - snapshot, - request, - replacementOptions: { - values: { - gpuModeArgs: ["--runtime", "nvidia"], - gpuModeKind: "nvidia-runtime", - gpuModeLabel: "Jetson NVIDIA runtime", - extraGroupGids: ["44", "993"], - }, - }, - }), - ).resolves.toMatchObject({ preparedRuntimeId: NEW_ID }); - - expect(fake.replacement?.HostConfig?.GroupAdd).toEqual(["44", "993"]); - expect(fake.replacement?.Config?.Entrypoint).toEqual([ - "/usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh", - ]); - expect(fake.replacement?.Config?.Cmd?.slice(0, 4)).toEqual([ - "--device-group-gids", - "44,993", - "--", - "/usr/local/bin/nemoclaw-managed-bootstrap", - ]); - }); - it("captures the live OpenShell idle supervisor with a separately persisted bootstrap identity", async () => { const fake = fixture(); const adapter = createDockerManagedBootstrapAdapter(fake.deps); diff --git a/src/lib/onboard/managed-bootstrap/docker.ts b/src/lib/onboard/managed-bootstrap/docker.ts index f6866c110f..7405fbd511 100644 --- a/src/lib/onboard/managed-bootstrap/docker.ts +++ b/src/lib/onboard/managed-bootstrap/docker.ts @@ -113,8 +113,6 @@ const MAX_RECOVERY_FAILURE_DETAIL_BYTES = 8 * 1024; const OPENSHELL_DRIVER_IDLE_COMMAND = "sleep infinity"; export const MANAGED_BOOTSTRAP_TRAMPOLINE_EXECUTABLE = "/usr/local/bin/nemoclaw-managed-bootstrap"; -const JETSON_DEVICE_GROUP_BOOTSTRAP_EXECUTABLE = - "/usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh"; function boundedRecoveryFailureDetail(error: unknown): string { const raw = (error instanceof Error ? error.message : String(error)).replaceAll("\0", "�"); @@ -719,44 +717,13 @@ function assertReplacementBoundary( inspect: DockerContainerInspect, handle: ManagedBootstrapHeldWorkloadHandle, snapshot: ManagedBootstrapObservedSnapshot, - expectedJetsonGroupGids?: readonly string[], ): void { const entrypoint = exactStringArray(inspect.Config?.Entrypoint, "replacement entrypoint"); const command = exactStringArray(inspect.Config?.Cmd, "replacement command"); - const managedCommand = replacementCommand(handle, snapshot); - const wrappedCommandPrefix = [ - "--device-group-gids", - command[1] ?? "", - "--", - MANAGED_BOOTSTRAP_TRAMPOLINE_EXECUTABLE, - ]; - const wrappedGids = String(command[1] ?? "").split(","); - const expectedGroupSet = new Set(expectedJetsonGroupGids); - const replacementGroupAdds = exactStringArray( - inspect.HostConfig?.GroupAdd, - "replacement supplementary groups", - ); - const wrappedBoundary = - handle.plan.profile.agent === "openclaw" && - exactArrayEqual(entrypoint, [JETSON_DEVICE_GROUP_BOOTSTRAP_EXECUTABLE]) && - wrappedGids.length > 0 && - wrappedGids.length <= 16 && - wrappedGids.every( - (gid, index) => - /^[1-9][0-9]*$/u.test(gid) && - Number(gid) <= 2_147_483_647 && - wrappedGids.indexOf(gid) === index && - (expectedJetsonGroupGids === undefined || expectedGroupSet.has(gid)), - ) && - (expectedJetsonGroupGids === undefined || - wrappedGids.length === expectedJetsonGroupGids.length) && - wrappedGids.every((gid) => replacementGroupAdds.includes(gid)) && - exactArrayEqual(command, [...wrappedCommandPrefix, ...managedCommand]); - const directBoundary = - expectedJetsonGroupGids === undefined && - exactArrayEqual(entrypoint, [MANAGED_BOOTSTRAP_TRAMPOLINE_EXECUTABLE]) && - exactArrayEqual(command, managedCommand); - if (!directBoundary && !wrappedBoundary) { + if ( + !exactArrayEqual(entrypoint, [MANAGED_BOOTSTRAP_TRAMPOLINE_EXECUTABLE]) || + !exactArrayEqual(command, replacementCommand(handle, snapshot)) + ) { throw new Error("Managed bootstrap Docker replacement process boundary changed."); } const intended = openshellSandboxCommandEnvValue(handle.intendedWorkloadArgv); @@ -3292,8 +3259,6 @@ export function createDockerManagedBootstrapAdapter( openshellSandboxCommand: handle.intendedWorkloadArgv, requiredUlimits: plan.requiredUlimits, extraGroupGids: plan.extraGroupGids, - preserveJetsonDeviceGroupMembership: - handle.plan.profile.agent === "openclaw" && plan.extraGroupGids.length > 0, containerEntrypoint: MANAGED_BOOTSTRAP_TRAMPOLINE_EXECUTABLE, containerCommand: trampolineCommand, containerName: stagingName, @@ -3351,14 +3316,7 @@ export function createDockerManagedBootstrapAdapter( "Managed bootstrap Docker replacement requires one bounded intended workload argv.", ); } - assertReplacementBoundary( - createdInspect, - handle, - snapshot, - handle.plan.profile.agent === "openclaw" && plan.extraGroupGids.length > 0 - ? plan.extraGroupGids - : undefined, - ); + assertReplacementBoundary(createdInspect, handle, snapshot); const expectedActivatedSpecHash = assertReplacementMatchesIntent( snapshot.specCanonicalJson, createdInspect, diff --git a/src/lib/onboard/sandbox-gpu-create-flow.test.ts b/src/lib/onboard/sandbox-gpu-create-flow.test.ts index fca980de2e..e2f800c830 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.test.ts @@ -535,7 +535,24 @@ describe("runSandboxGpuCreateFlow native failure and readiness", () => { }); expect(mocks.createDockerGpuSandboxCreatePatch).toHaveBeenCalledWith( - expect.objectContaining({ route: "native", persistStartupCommand: false }), + expect.objectContaining({ + route: "native", + persistStartupCommand: false, + preserveJetsonDeviceGroupMembership: true, + }), + ); + }); + + it("does not apply the OpenClaw Jetson group bootstrap to Hermes (#7610)", async () => { + const input = createInput(); + input.agentName = "hermes"; + + await expect(runSandboxGpuCreateFlow(input, createDeps())).resolves.toMatchObject({ + route: "native", + }); + + expect(mocks.createDockerGpuSandboxCreatePatch).toHaveBeenCalledWith( + expect.objectContaining({ preserveJetsonDeviceGroupMembership: false }), ); }); diff --git a/src/lib/onboard/sandbox-gpu-create-flow.ts b/src/lib/onboard/sandbox-gpu-create-flow.ts index a46fce580e..8be9ec81be 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.ts @@ -68,6 +68,7 @@ type Sleep = NonNullable; export interface SandboxGpuCreateFlowInput { sandboxName: string; + agentName: string; provider: string; sandboxGpuConfig: SandboxGpuConfig; gpuRoutePlan: import("./docker-gpu-route").DockerGpuRoutePlan; diff --git a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts index 4d0f046e8f..4c16561d04 100644 --- a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts +++ b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts @@ -143,6 +143,7 @@ export function createSandboxGpuCreateAttemptRunner( requiredUlimits: input.requiredUlimits, timeoutSecs: input.sandboxReadyTimeoutSecs, backend: input.sandboxGpuConfig.hostGpuPlatform === "jetson" ? "jetson" : "generic", + preserveJetsonDeviceGroupMembership: input.agentName === "openclaw", deps, }); const recovery = await managedLifecycle?.recoverUnfinished(); From d6cfa87f7c4a8f0789f840d700b89275cdff12bb Mon Sep 17 00:00:00 2001 From: San Dang Date: Thu, 6 Aug 2026 19:48:28 +0700 Subject: [PATCH 27/49] refactor(onboard): carry agent scope with patch options Signed-off-by: San Dang --- scripts/checks/run-managed-image-openshell-e2e.ts | 1 - src/lib/onboard.ts | 1 - src/lib/onboard/docker-startup-command-agent.ts | 6 ++++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/checks/run-managed-image-openshell-e2e.ts b/scripts/checks/run-managed-image-openshell-e2e.ts index 5a60457217..c948c763dc 100644 --- a/scripts/checks/run-managed-image-openshell-e2e.ts +++ b/scripts/checks/run-managed-image-openshell-e2e.ts @@ -786,7 +786,6 @@ async function run(input: Inputs): Promise { flow = await runSandboxGpuCreateFlow( { sandboxName: input.sandbox, - agentName: input.agent, provider: input.localProvider ? resolveManagedImageLocalInferenceRoute(input.localProvider).providerName : "nvidia", diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index ce9055d2cd..499cb51b88 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -2641,7 +2641,6 @@ async function createSandboxWithBaseImageResolution( } = await sandboxGpuCreateFlow.runSandboxGpuCreateFlow( { sandboxName, - agentName: agent?.name ?? "openclaw", provider, sandboxGpuConfig: effectiveSandboxGpuConfig, gpuRoutePlan, diff --git a/src/lib/onboard/docker-startup-command-agent.ts b/src/lib/onboard/docker-startup-command-agent.ts index de483bda8d..fdb60a6318 100644 --- a/src/lib/onboard/docker-startup-command-agent.ts +++ b/src/lib/onboard/docker-startup-command-agent.ts @@ -18,14 +18,16 @@ export function resolveDockerStartupCommandPatch( agent: AgentDefinition | null | undefined, dockerDriverGateway: boolean | null | undefined, ): { + agentName: string; persistStartupCommand: boolean; requiredUlimits: readonly DockerUlimit[] | null; } { + const agentName = agent?.name ?? "openclaw"; if (dockerDriverGateway !== true) { - return { persistStartupCommand: false, requiredUlimits: null }; + return { agentName, persistStartupCommand: false, requiredUlimits: null }; } - const agentName = agent?.name ?? "openclaw"; return { + agentName, persistStartupCommand: agentName === "openclaw" || agentName === "hermes" || agentName === DCODE_AGENT_NAME, requiredUlimits: agentName === DCODE_AGENT_NAME ? DCODE_DOCKER_ULIMITS : null, From 03529c586648a9d398b47ef7e621a4038e98de0c Mon Sep 17 00:00:00 2001 From: San Dang Date: Thu, 6 Aug 2026 20:02:56 +0700 Subject: [PATCH 28/49] fix(images): audit Jetson wrapper metadata check Signed-off-by: San Dang --- src/lib/onboard/dockerfile-remote-dashboard-bind-contract.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/onboard/dockerfile-remote-dashboard-bind-contract.ts b/src/lib/onboard/dockerfile-remote-dashboard-bind-contract.ts index eeaf3d8975..6c2e770b8c 100644 --- a/src/lib/onboard/dockerfile-remote-dashboard-bind-contract.ts +++ b/src/lib/onboard/dockerfile-remote-dashboard-bind-contract.ts @@ -60,6 +60,7 @@ const CANONICAL_POST_GENERATOR_RUN_SHA256 = new Set([ "1197b99bdb996b37a3e4e386a507dfabcdfb2c26a40b015d617f97208668187d", "c0b409e1bf4d33a9e44f407c6bd9b0445b2ffd0b796823fe3cfa5989314d6603", "9fcc674a44a152707380cdb09a67f8594f568288406c96f5354f1c87f5b939a6", + "a91a2cb7531542b50169301d172079cc63132bd6d5f9e986fe0e125722a055fe", "83567d1fa0e73bef6a3333383c13ace05e26704964ae6a7a76ee24a2f2be3d7e", "ca1f7b1cb9dd5d467f806792c4072a84ef1e6402c3e8650b6325b95cc186ccdf", "4165899eb1f0f948f8883eddf4136136caac21cee1df39b12afea7672b23a378", From 77576759d32051f7bc8116a997d59ffc4a01b91a Mon Sep 17 00:00:00 2001 From: San Dang Date: Thu, 6 Aug 2026 20:23:06 +0700 Subject: [PATCH 29/49] chore(onboard): add cuInit boundary diagnostic Signed-off-by: San Dang --- scripts/diagnose-jetson-cuinit-boundary.sh | 164 +++++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100755 scripts/diagnose-jetson-cuinit-boundary.sh diff --git a/scripts/diagnose-jetson-cuinit-boundary.sh b/scripts/diagnose-jetson-cuinit-boundary.sh new file mode 100755 index 0000000000..0677b93947 --- /dev/null +++ b/scripts/diagnose-jetson-cuinit-boundary.sh @@ -0,0 +1,164 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -uo pipefail + +sandbox_name="${1:-tm}" +if [[ ! "$sandbox_name" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]*$ ]]; then + printf 'Invalid sandbox name: %s\n' "$sandbox_name" >&2 + exit 2 +fi + +container_ids=() +while IFS=$'\t' read -r container_id container_name; do + if [[ "$container_name" == "openshell-${sandbox_name}-"* ]] \ + && [[ "$container_name" != *-nemoclaw-gpu-backup-* ]]; then + container_ids+=("$container_id") + fi +done < <(docker ps --no-trunc --format '{{.ID}}\t{{.Names}}') + +if ((${#container_ids[@]} != 1)); then + printf 'Expected one running non-backup Docker container for sandbox %s; found %d.\n' \ + "$sandbox_name" "${#container_ids[@]}" >&2 + docker ps --no-trunc --format 'ID={{.ID}} NAME={{.Names}} STATUS={{.Status}}' >&2 + exit 1 +fi + +container_id="${container_ids[0]}" + +cuda_probe='import ctypes +import glob +import os +import stat + +print(f"identity=uid={os.getuid()} gid={os.getgid()} groups={os.getgroups()}") +for key in ("LD_LIBRARY_PATH", "NVIDIA_VISIBLE_DEVICES", "NVIDIA_DRIVER_CAPABILITIES"): + print("env_{}={}".format(key, os.environ.get(key, ""))) + +patterns = ( + "/dev/nvidia*", + "/dev/nvhost-*", + "/dev/nvmap", + "/dev/nvgpu/igpu0/*", + "/dev/dri/renderD*", +) +paths = sorted({path for pattern in patterns for path in glob.glob(pattern)}) +for path in paths: + try: + info = os.stat(path) + except Exception as error: + print(f"device_stat={path} {type(error).__name__}: {error}") + continue + if not stat.S_ISCHR(info.st_mode): + continue + access = [] + for label, flags in (("r", os.O_RDONLY), ("rw", os.O_RDWR)): + try: + fd = os.open(path, flags) + os.close(fd) + access.append(f"{label}=ok") + except Exception as error: + error_number = getattr(error, "errno", "") + access.append(f"{label}={type(error).__name__}:{error_number}") + print( + f"device={path} mode={info.st_mode & 0o777:o} uid={info.st_uid} gid={info.st_gid} " + + " ".join(access) + ) + +try: + cuda = ctypes.CDLL("libcuda.so.1") +except OSError as error: + print(f"libcuda_load=OSError: {error}") + raise SystemExit(11) + +cuda.cuInit.argtypes = [ctypes.c_uint] +cuda.cuInit.restype = ctypes.c_int +result = cuda.cuInit(0) +print("libcuda_load=ok") +try: + names = sorted( + { + line.split()[-1] + for line in open("/proc/self/maps", encoding="utf-8") + if "libcuda.so" in line and line.split()[-1].startswith("/") + } + ) + print(f"libcuda_maps={names}") +except Exception as error: + print(f"libcuda_maps={type(error).__name__}: {error}") + +error_name = ctypes.c_char_p() +try: + cuda.cuGetErrorName.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_char_p)] + cuda.cuGetErrorName.restype = ctypes.c_int + name_result = cuda.cuGetErrorName(result, ctypes.byref(error_name)) + decoded_name = error_name.value.decode() if name_result == 0 and error_name.value else "unknown" +except Exception as error: + decoded_name = f"unavailable:{type(error).__name__}" +print(f"cuInit(0)={result} name={decoded_name}") +raise SystemExit(0 if result == 0 else 10)' + +run_probe() { + local label="$1" + shift + local output status + printf '\n=== %s ===\n' "$label" + if output="$("$@" 2>&1)"; then + status=0 + else + status=$? + fi + printf '%s\nprobe_exit=%d\n' "$output" "$status" + probe_output="$output" +} + +extract_cuinit() { + sed -n -E 's/^cuInit\(0\)=([0-9]+).*$/\1/p' <<<"$1" | tail -1 +} + +printf 'sandbox=%s\ncontainer=%s\n' "$sandbox_name" "$container_id" +printf 'This diagnostic is read-only. It does not create, restart, rename, or remove a sandbox or container.\n' +printf 'git_head=%s\n' "$(git rev-parse HEAD 2>/dev/null || printf unknown)" +printf 'openshell_version=%s\n' "$(openshell --version 2>&1 || printf unknown)" + +printf '\n=== OpenShell policy paths ===\n' +openshell policy get --base "$sandbox_name" 2>&1 \ + | grep -E 'read_only:|read_write:|/opt/nvidia|/dev/nv|/dev/dri' || true + +printf '\n=== Active container configuration ===\n' +docker inspect --format \ + 'image={{.Image}} runtime={{.HostConfig.Runtime}} user={{json .Config.User}} group_add={{json .HostConfig.GroupAdd}} entrypoint={{json .Config.Entrypoint}} cmd={{json .Config.Cmd}} devices={{json .HostConfig.Devices}} device_requests={{json .HostConfig.DeviceRequests}}' \ + "$container_id" 2>&1 || true +docker exec --user 0 "$container_id" /usr/bin/id sandbox 2>&1 || true +docker exec --user 0 "$container_id" /usr/bin/stat -Lc \ + 'wrapper=type=%F mode=%a uid=%u gid=%g path=%n' \ + /usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh 2>&1 || true + +run_probe "Host account" python3 -c "$cuda_probe" +host_output="$probe_output" +run_probe "Direct Docker as root" docker exec --user 0 "$container_id" python3 -c "$cuda_probe" +docker_root_output="$probe_output" +run_probe "Direct Docker as sandbox" docker exec --user sandbox "$container_id" python3 -c "$cuda_probe" +docker_sandbox_output="$probe_output" +run_probe "OpenShell sandbox execution" openshell sandbox exec -n "$sandbox_name" -- python3 -c "$cuda_probe" +openshell_output="$probe_output" + +host_cuinit="$(extract_cuinit "$host_output")" +docker_root_cuinit="$(extract_cuinit "$docker_root_output")" +docker_sandbox_cuinit="$(extract_cuinit "$docker_sandbox_output")" +openshell_cuinit="$(extract_cuinit "$openshell_output")" + +printf '\n=== Boundary result ===\n' +printf 'host_cuInit=%s docker_root_cuInit=%s docker_sandbox_cuInit=%s openshell_cuInit=%s\n' \ + "${host_cuinit:-missing}" "${docker_root_cuinit:-missing}" \ + "${docker_sandbox_cuinit:-missing}" "${openshell_cuinit:-missing}" +if [[ "$docker_sandbox_cuinit" == "0" && "$openshell_cuinit" != "0" ]]; then + printf 'ISOLATED: CUDA works in the running container as sandbox but fails through OpenShell execution. Investigate the OpenShell filesystem/device policy boundary.\n' +elif [[ "$docker_root_cuinit" == "0" && "$docker_sandbox_cuinit" != "0" ]]; then + printf 'ISOLATED: CUDA works as root in the running container but fails as sandbox. Investigate identity, group, or device permission differences.\n' +elif [[ "$docker_root_cuinit" != "0" && "$docker_sandbox_cuinit" != "0" ]]; then + printf 'ISOLATED: CUDA already fails in direct Docker execution. Investigate the recreated container runtime, injected driver libraries, and device set before changing OpenShell policy.\n' +else + printf 'INCONCLUSIVE: preserve this output; the four boundaries did not produce a single failing transition.\n' +fi From f0a67207d7a2cba15b1925d2268943e74a1c9563 Mon Sep 17 00:00:00 2001 From: San Dang Date: Thu, 6 Aug 2026 20:36:43 +0700 Subject: [PATCH 30/49] chore(onboard): prove OpenRM policy boundary Signed-off-by: San Dang --- .../prove-jetson-openrm-policy-boundary.sh | 248 ++++++++++++++++++ 1 file changed, 248 insertions(+) create mode 100755 scripts/prove-jetson-openrm-policy-boundary.sh diff --git a/scripts/prove-jetson-openrm-policy-boundary.sh b/scripts/prove-jetson-openrm-policy-boundary.sh new file mode 100755 index 0000000000..fd3ca530c1 --- /dev/null +++ b/scripts/prove-jetson-openrm-policy-boundary.sh @@ -0,0 +1,248 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +sandbox_name="${1:-tm}" +if [[ ! "$sandbox_name" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]*$ ]]; then + printf 'Invalid sandbox name: %s\n' "$sandbox_name" >&2 + exit 2 +fi + +for command_name in docker nemoclaw openshell python3; do + if ! command -v "$command_name" >/dev/null 2>&1; then + printf 'Required command not found: %s\n' "$command_name" >&2 + exit 2 + fi +done + +candidate_path=/dev/nvidia-caps/nvidia-cap2 +if [[ ! -c "$candidate_path" ]]; then + printf 'Candidate OpenRM capability device is not a character device: %s\n' \ + "$candidate_path" >&2 + exit 2 +fi + +temporary_dir="$(mktemp -d)" +baseline_policy="$temporary_dir/baseline-policy.yaml" +candidate_policy="$temporary_dir/candidate-policy.yaml" +onboard_log="$temporary_dir/onboard.log" +onboard_pid="" +policy_changed=0 + +# shellcheck disable=SC2329 # Invoked by trap. +cleanup() { + local status=$? + trap - EXIT INT TERM + if [[ "$policy_changed" == "1" && -s "$baseline_policy" ]]; then + printf '\nRestoring the original OpenShell policy...\n' >&2 + openshell policy set --policy "$baseline_policy" --wait "$sandbox_name" >/dev/null 2>&1 || true + fi + if [[ -n "$onboard_pid" ]]; then + kill -CONT "$onboard_pid" >/dev/null 2>&1 || true + fi + rm -rf -- "$temporary_dir" + exit "$status" +} +trap cleanup EXIT INT TERM + +nemoclaw "$sandbox_name" policy get >"$baseline_policy" +if [[ ! -s "$baseline_policy" ]]; then + printf 'Could not export the base policy for sandbox %s.\n' "$sandbox_name" >&2 + exit 1 +fi +if grep -Fxq " - $candidate_path" "$baseline_policy"; then + printf 'The baseline policy already grants %s; this A/B test requires it to be absent.\n' \ + "$candidate_path" >&2 + exit 2 +fi + +awk -v candidate="$candidate_path" ' + /^ read_only:$/ && !inserted { + print + print " - " candidate + inserted = 1 + next + } + { print } + END { + if (!inserted) exit 42 + } +' "$baseline_policy" >"$candidate_policy" || { + printf 'Could not add %s to filesystem_policy.read_only.\n' "$candidate_path" >&2 + exit 1 +} + +cuda_probe='import ctypes +import os +import stat + +print(f"identity=uid={os.getuid()} gid={os.getgid()} groups={os.getgroups()}") +for path in ("/dev/nvidia-caps/nvidia-cap1", "/dev/nvidia-caps/nvidia-cap2"): + try: + info = os.stat(path) + node_type = "char" if stat.S_ISCHR(info.st_mode) else "other" + print(f"path={path} type={node_type} mode={info.st_mode & 0o777:o} uid={info.st_uid} gid={info.st_gid}") + try: + fd = os.open(path, os.O_RDONLY) + os.close(fd) + print(f"open_read={path}:ok") + except Exception as error: + error_number = getattr(error, "errno", "") + print(f"open_read={path}:{type(error).__name__}:{error_number}") + except Exception as error: + error_number = getattr(error, "errno", "") + print(f"stat={path}:{type(error).__name__}:{error_number}") + +try: + cuda = ctypes.CDLL("libcuda.so.1") +except OSError as error: + print(f"libcuda_load=OSError: {error}") + raise SystemExit(11) + +cuda.cuInit.argtypes = [ctypes.c_uint] +cuda.cuInit.restype = ctypes.c_int +result = cuda.cuInit(0) +error_name = ctypes.c_char_p() +try: + cuda.cuGetErrorName.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_char_p)] + cuda.cuGetErrorName.restype = ctypes.c_int + name_result = cuda.cuGetErrorName(result, ctypes.byref(error_name)) + name = error_name.value.decode() if name_result == 0 and error_name.value else "unknown" +except Exception as error: + name = f"unavailable:{type(error).__name__}" +print("libcuda_load=ok") +print(f"cuInit(0)={result} name={name}") +raise SystemExit(0 if result == 0 else 10)' + +run_probe() { + local label="$1" + shift + local output status + printf '\n=== %s ===\n' "$label" + if output="$("$@" 2>&1)"; then + status=0 + else + status=$? + fi + printf '%s\nprobe_exit=%d\n' "$output" "$status" + probe_output="$output" +} + +extract_cuinit() { + sed -n -E 's/^cuInit\(0\)=([0-9]+).*$/\1/p' <<<"$1" | tail -1 +} + +find_replacement() { + local container_id entrypoint + while IFS= read -r container_id; do + [[ -n "$container_id" ]] || continue + entrypoint="$(docker inspect --format '{{index .Config.Entrypoint 0}}' "$container_id" 2>/dev/null || true)" + if [[ "$entrypoint" == "/usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh" ]]; then + printf '%s\n' "$container_id" + return 0 + fi + done < <( + docker ps --no-trunc \ + --filter "label=openshell.ai/sandbox-name=$sandbox_name" \ + --format '{{.ID}}' + ) + return 1 +} + +printf 'sandbox=%s\n' "$sandbox_name" +printf 'candidate_read_only=%s\n' "$candidate_path" +printf 'This proof runs the real nemoclaw onboard --resume recreation, pauses the CLI while its replacement container is live, restores the original policy, and then allows normal rollback.\n' + +nemoclaw onboard --resume >"$onboard_log" 2>&1 & +onboard_pid=$! + +replacement_id="" +for _ in {1..360}; do + if replacement_id="$(find_replacement)"; then + break + fi + if ! kill -0 "$onboard_pid" 2>/dev/null; then + printf 'Onboarding exited before a Jetson replacement container appeared.\n' >&2 + sed -n '1,240p' "$onboard_log" >&2 + exit 1 + fi + sleep 0.25 +done +if [[ -z "$replacement_id" ]]; then + printf 'Timed out waiting for the Jetson replacement container.\n' >&2 + exit 1 +fi + +kill -STOP "$onboard_pid" +printf 'replacement=%s\n' "$replacement_id" +docker inspect --format \ + 'runtime={{.HostConfig.Runtime}} group_add={{json .HostConfig.GroupAdd}} entrypoint={{json .Config.Entrypoint}} cmd={{json .Config.Cmd}}' \ + "$replacement_id" + +for _ in {1..240}; do + container_status="$(docker inspect --format '{{.State.Status}}' "$replacement_id" 2>/dev/null || true)" + health_status="$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$replacement_id" 2>/dev/null || true)" + if [[ "$container_status" == "running" && "$health_status" == "healthy" ]]; then + break + fi + if [[ "$container_status" == "exited" || "$container_status" == "dead" ]]; then + printf 'Replacement container stopped before the A/B probe.\n' >&2 + docker logs --tail 160 "$replacement_id" >&2 || true + exit 1 + fi + sleep 0.25 +done +if [[ "$container_status" != "running" || "$health_status" != "healthy" ]]; then + printf 'Replacement did not become healthy: status=%s health=%s\n' \ + "$container_status" "$health_status" >&2 + exit 1 +fi + +run_probe "Direct Docker as sandbox with the recreated container" \ + docker exec --user sandbox "$replacement_id" python3 -c "$cuda_probe" +direct_output="$probe_output" + +run_probe "OpenShell with the unchanged baseline policy" \ + openshell sandbox exec -n "$sandbox_name" -- python3 -c "$cuda_probe" +baseline_output="$probe_output" + +printf '\nApplying the one-path candidate policy...\n' +policy_changed=1 +openshell policy set --policy "$candidate_policy" --wait "$sandbox_name" + +run_probe "OpenShell with nvidia-cap2 read-only" \ + openshell sandbox exec -n "$sandbox_name" -- python3 -c "$cuda_probe" +candidate_output="$probe_output" + +printf '\nRestoring the unchanged baseline policy...\n' +openshell policy set --policy "$baseline_policy" --wait "$sandbox_name" +policy_changed=0 + +direct_cuinit="$(extract_cuinit "$direct_output")" +baseline_cuinit="$(extract_cuinit "$baseline_output")" +candidate_cuinit="$(extract_cuinit "$candidate_output")" + +printf '\n=== A/B result ===\n' +printf 'direct_docker_cuInit=%s baseline_openshell_cuInit=%s candidate_openshell_cuInit=%s\n' \ + "${direct_cuinit:-missing}" "${baseline_cuinit:-missing}" "${candidate_cuinit:-missing}" + +kill -CONT "$onboard_pid" +if wait "$onboard_pid"; then + onboard_status=0 +else + onboard_status=$? +fi +onboard_pid="" +printf 'onboard_exit_after_baseline_restore=%d\n' "$onboard_status" + +if [[ "$direct_cuinit" == "0" && "$baseline_cuinit" == "801" && "$candidate_cuinit" == "0" ]]; then + printf '\nPROVEN: the replacement and sandbox identity can initialize CUDA, but OpenShell denies the world-readable OpenRM capability device. Granting only %s read-only changes cuInit from 801 to 0.\n' \ + "$candidate_path" + exit 0 +fi + +printf '\nINCONCLUSIVE: this A/B did not isolate %s. Do not implement that policy change from this result.\n' \ + "$candidate_path" >&2 +exit 1 From 19648709a112464bd39409b844c2519013e9d34c Mon Sep 17 00:00:00 2001 From: San Dang Date: Thu, 6 Aug 2026 20:44:11 +0700 Subject: [PATCH 31/49] fix(onboard): make OpenRM proof non-interactive Signed-off-by: San Dang --- .../prove-jetson-openrm-policy-boundary.sh | 57 ++++++++++--------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/scripts/prove-jetson-openrm-policy-boundary.sh b/scripts/prove-jetson-openrm-policy-boundary.sh index fd3ca530c1..bbd564b8e1 100755 --- a/scripts/prove-jetson-openrm-policy-boundary.sh +++ b/scripts/prove-jetson-openrm-policy-boundary.sh @@ -47,33 +47,6 @@ cleanup() { } trap cleanup EXIT INT TERM -nemoclaw "$sandbox_name" policy get >"$baseline_policy" -if [[ ! -s "$baseline_policy" ]]; then - printf 'Could not export the base policy for sandbox %s.\n' "$sandbox_name" >&2 - exit 1 -fi -if grep -Fxq " - $candidate_path" "$baseline_policy"; then - printf 'The baseline policy already grants %s; this A/B test requires it to be absent.\n' \ - "$candidate_path" >&2 - exit 2 -fi - -awk -v candidate="$candidate_path" ' - /^ read_only:$/ && !inserted { - print - print " - " candidate - inserted = 1 - next - } - { print } - END { - if (!inserted) exit 42 - } -' "$baseline_policy" >"$candidate_policy" || { - printf 'Could not add %s to filesystem_policy.read_only.\n' "$candidate_path" >&2 - exit 1 -} - cuda_probe='import ctypes import os import stat @@ -155,7 +128,8 @@ printf 'sandbox=%s\n' "$sandbox_name" printf 'candidate_read_only=%s\n' "$candidate_path" printf 'This proof runs the real nemoclaw onboard --resume recreation, pauses the CLI while its replacement container is live, restores the original policy, and then allows normal rollback.\n' -nemoclaw onboard --resume >"$onboard_log" 2>&1 & +NEMOCLAW_POLICY_TIER=balanced \ + nemoclaw onboard --resume --non-interactive "$onboard_log" 2>&1 & onboard_pid=$! replacement_id="" @@ -200,6 +174,33 @@ if [[ "$container_status" != "running" || "$health_status" != "healthy" ]]; then exit 1 fi +nemoclaw "$sandbox_name" policy get >"$baseline_policy" +if [[ ! -s "$baseline_policy" ]]; then + printf 'Could not export the live base policy for sandbox %s.\n' "$sandbox_name" >&2 + exit 1 +fi +if grep -Fxq " - $candidate_path" "$baseline_policy"; then + printf 'The live baseline policy already grants %s; this A/B test requires it to be absent.\n' \ + "$candidate_path" >&2 + exit 2 +fi + +awk -v candidate="$candidate_path" ' + /^ read_only:$/ && !inserted { + print + print " - " candidate + inserted = 1 + next + } + { print } + END { + if (!inserted) exit 42 + } +' "$baseline_policy" >"$candidate_policy" || { + printf 'Could not add %s to filesystem_policy.read_only.\n' "$candidate_path" >&2 + exit 1 +} + run_probe "Direct Docker as sandbox with the recreated container" \ docker exec --user sandbox "$replacement_id" python3 -c "$cuda_probe" direct_output="$probe_output" From 2b4a89a314a29f604738470c651e327521f19d88 Mon Sep 17 00:00:00 2001 From: San Dang Date: Thu, 6 Aug 2026 21:06:54 +0700 Subject: [PATCH 32/49] fix(onboard): run OpenRM proof before rollback Signed-off-by: San Dang --- ci/env-var-doc-allowlist.json | 4 + .../checks/openshell-policy-mutation-read.mts | 4 + .../prove-jetson-openrm-policy-boundary.sh | 242 +----------------- .../diagnostics/jetson-openrm-proof.test.ts | 156 +++++++++++ .../diagnostics/jetson-openrm-proof.ts | 177 +++++++++++++ src/lib/onboard/docker-gpu-patch.ts | 3 +- src/lib/onboard/docker-gpu-sandbox-create.ts | 16 ++ 7 files changed, 366 insertions(+), 236 deletions(-) create mode 100644 src/lib/onboard/diagnostics/jetson-openrm-proof.test.ts create mode 100644 src/lib/onboard/diagnostics/jetson-openrm-proof.ts diff --git a/ci/env-var-doc-allowlist.json b/ci/env-var-doc-allowlist.json index 53729dedb4..de8f33f781 100644 --- a/ci/env-var-doc-allowlist.json +++ b/ci/env-var-doc-allowlist.json @@ -1,4 +1,8 @@ [ + { + "name": "NEMOCLAW_DIAGNOSE_JETSON_OPENRM_POLICY", + "reason": "Temporary maintainer-only Jetson hardware A/B for issue #7610; remove after the exact policy boundary is proven." + }, { "name": "NEMOCLAW_DISABLE_AUTO_DISPATCH", "reason": "Test harness sentinel set to '1' so test files can import src/nemoclaw.ts without triggering main(). Never user-set in production." diff --git a/scripts/checks/openshell-policy-mutation-read.mts b/scripts/checks/openshell-policy-mutation-read.mts index e194357854..f9a1359b62 100644 --- a/scripts/checks/openshell-policy-mutation-read.mts +++ b/scripts/checks/openshell-policy-mutation-read.mts @@ -73,6 +73,10 @@ export const MUTATION_READS: readonly AuditedPolicyReadFile[] = [ preservingBase("getPresetContentGatewayState/readPolicy"), ], }, + { + relativePath: "src/lib/onboard/diagnostics/jetson-openrm-proof.ts", + expectedReads: [preservingBase("maybeRunJetsonOpenRmPolicyProof")], + }, { relativePath: "nemoclaw/src/blueprint/runner.ts", expectedReads: [unclassifiedBase("actionApply")], diff --git a/scripts/prove-jetson-openrm-policy-boundary.sh b/scripts/prove-jetson-openrm-policy-boundary.sh index bbd564b8e1..d83f6fe65f 100755 --- a/scripts/prove-jetson-openrm-policy-boundary.sh +++ b/scripts/prove-jetson-openrm-policy-boundary.sh @@ -10,240 +10,12 @@ if [[ ! "$sandbox_name" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]*$ ]]; then exit 2 fi -for command_name in docker nemoclaw openshell python3; do - if ! command -v "$command_name" >/dev/null 2>&1; then - printf 'Required command not found: %s\n' "$command_name" >&2 - exit 2 - fi -done +printf 'The proof is armed for sandbox %s. Complete the normal onboarding prompts.\n' \ + "$sandbox_name" +printf 'When the live replacement returns cuInit(0)=801, NemoClaw will run the one-path A/B before its normal rollback.\n' -candidate_path=/dev/nvidia-caps/nvidia-cap2 -if [[ ! -c "$candidate_path" ]]; then - printf 'Candidate OpenRM capability device is not a character device: %s\n' \ - "$candidate_path" >&2 - exit 2 -fi - -temporary_dir="$(mktemp -d)" -baseline_policy="$temporary_dir/baseline-policy.yaml" -candidate_policy="$temporary_dir/candidate-policy.yaml" -onboard_log="$temporary_dir/onboard.log" -onboard_pid="" -policy_changed=0 - -# shellcheck disable=SC2329 # Invoked by trap. -cleanup() { - local status=$? - trap - EXIT INT TERM - if [[ "$policy_changed" == "1" && -s "$baseline_policy" ]]; then - printf '\nRestoring the original OpenShell policy...\n' >&2 - openshell policy set --policy "$baseline_policy" --wait "$sandbox_name" >/dev/null 2>&1 || true - fi - if [[ -n "$onboard_pid" ]]; then - kill -CONT "$onboard_pid" >/dev/null 2>&1 || true - fi - rm -rf -- "$temporary_dir" - exit "$status" -} -trap cleanup EXIT INT TERM - -cuda_probe='import ctypes -import os -import stat - -print(f"identity=uid={os.getuid()} gid={os.getgid()} groups={os.getgroups()}") -for path in ("/dev/nvidia-caps/nvidia-cap1", "/dev/nvidia-caps/nvidia-cap2"): - try: - info = os.stat(path) - node_type = "char" if stat.S_ISCHR(info.st_mode) else "other" - print(f"path={path} type={node_type} mode={info.st_mode & 0o777:o} uid={info.st_uid} gid={info.st_gid}") - try: - fd = os.open(path, os.O_RDONLY) - os.close(fd) - print(f"open_read={path}:ok") - except Exception as error: - error_number = getattr(error, "errno", "") - print(f"open_read={path}:{type(error).__name__}:{error_number}") - except Exception as error: - error_number = getattr(error, "errno", "") - print(f"stat={path}:{type(error).__name__}:{error_number}") - -try: - cuda = ctypes.CDLL("libcuda.so.1") -except OSError as error: - print(f"libcuda_load=OSError: {error}") - raise SystemExit(11) - -cuda.cuInit.argtypes = [ctypes.c_uint] -cuda.cuInit.restype = ctypes.c_int -result = cuda.cuInit(0) -error_name = ctypes.c_char_p() -try: - cuda.cuGetErrorName.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_char_p)] - cuda.cuGetErrorName.restype = ctypes.c_int - name_result = cuda.cuGetErrorName(result, ctypes.byref(error_name)) - name = error_name.value.decode() if name_result == 0 and error_name.value else "unknown" -except Exception as error: - name = f"unavailable:{type(error).__name__}" -print("libcuda_load=ok") -print(f"cuInit(0)={result} name={name}") -raise SystemExit(0 if result == 0 else 10)' - -run_probe() { - local label="$1" - shift - local output status - printf '\n=== %s ===\n' "$label" - if output="$("$@" 2>&1)"; then - status=0 - else - status=$? - fi - printf '%s\nprobe_exit=%d\n' "$output" "$status" - probe_output="$output" -} - -extract_cuinit() { - sed -n -E 's/^cuInit\(0\)=([0-9]+).*$/\1/p' <<<"$1" | tail -1 -} - -find_replacement() { - local container_id entrypoint - while IFS= read -r container_id; do - [[ -n "$container_id" ]] || continue - entrypoint="$(docker inspect --format '{{index .Config.Entrypoint 0}}' "$container_id" 2>/dev/null || true)" - if [[ "$entrypoint" == "/usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh" ]]; then - printf '%s\n' "$container_id" - return 0 - fi - done < <( - docker ps --no-trunc \ - --filter "label=openshell.ai/sandbox-name=$sandbox_name" \ - --format '{{.ID}}' - ) - return 1 -} - -printf 'sandbox=%s\n' "$sandbox_name" -printf 'candidate_read_only=%s\n' "$candidate_path" -printf 'This proof runs the real nemoclaw onboard --resume recreation, pauses the CLI while its replacement container is live, restores the original policy, and then allows normal rollback.\n' - -NEMOCLAW_POLICY_TIER=balanced \ - nemoclaw onboard --resume --non-interactive "$onboard_log" 2>&1 & -onboard_pid=$! - -replacement_id="" -for _ in {1..360}; do - if replacement_id="$(find_replacement)"; then - break - fi - if ! kill -0 "$onboard_pid" 2>/dev/null; then - printf 'Onboarding exited before a Jetson replacement container appeared.\n' >&2 - sed -n '1,240p' "$onboard_log" >&2 - exit 1 - fi - sleep 0.25 -done -if [[ -z "$replacement_id" ]]; then - printf 'Timed out waiting for the Jetson replacement container.\n' >&2 - exit 1 -fi - -kill -STOP "$onboard_pid" -printf 'replacement=%s\n' "$replacement_id" -docker inspect --format \ - 'runtime={{.HostConfig.Runtime}} group_add={{json .HostConfig.GroupAdd}} entrypoint={{json .Config.Entrypoint}} cmd={{json .Config.Cmd}}' \ - "$replacement_id" - -for _ in {1..240}; do - container_status="$(docker inspect --format '{{.State.Status}}' "$replacement_id" 2>/dev/null || true)" - health_status="$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$replacement_id" 2>/dev/null || true)" - if [[ "$container_status" == "running" && "$health_status" == "healthy" ]]; then - break - fi - if [[ "$container_status" == "exited" || "$container_status" == "dead" ]]; then - printf 'Replacement container stopped before the A/B probe.\n' >&2 - docker logs --tail 160 "$replacement_id" >&2 || true - exit 1 - fi - sleep 0.25 -done -if [[ "$container_status" != "running" || "$health_status" != "healthy" ]]; then - printf 'Replacement did not become healthy: status=%s health=%s\n' \ - "$container_status" "$health_status" >&2 - exit 1 -fi - -nemoclaw "$sandbox_name" policy get >"$baseline_policy" -if [[ ! -s "$baseline_policy" ]]; then - printf 'Could not export the live base policy for sandbox %s.\n' "$sandbox_name" >&2 - exit 1 -fi -if grep -Fxq " - $candidate_path" "$baseline_policy"; then - printf 'The live baseline policy already grants %s; this A/B test requires it to be absent.\n' \ - "$candidate_path" >&2 - exit 2 -fi - -awk -v candidate="$candidate_path" ' - /^ read_only:$/ && !inserted { - print - print " - " candidate - inserted = 1 - next - } - { print } - END { - if (!inserted) exit 42 - } -' "$baseline_policy" >"$candidate_policy" || { - printf 'Could not add %s to filesystem_policy.read_only.\n' "$candidate_path" >&2 - exit 1 -} - -run_probe "Direct Docker as sandbox with the recreated container" \ - docker exec --user sandbox "$replacement_id" python3 -c "$cuda_probe" -direct_output="$probe_output" - -run_probe "OpenShell with the unchanged baseline policy" \ - openshell sandbox exec -n "$sandbox_name" -- python3 -c "$cuda_probe" -baseline_output="$probe_output" - -printf '\nApplying the one-path candidate policy...\n' -policy_changed=1 -openshell policy set --policy "$candidate_policy" --wait "$sandbox_name" - -run_probe "OpenShell with nvidia-cap2 read-only" \ - openshell sandbox exec -n "$sandbox_name" -- python3 -c "$cuda_probe" -candidate_output="$probe_output" - -printf '\nRestoring the unchanged baseline policy...\n' -openshell policy set --policy "$baseline_policy" --wait "$sandbox_name" -policy_changed=0 - -direct_cuinit="$(extract_cuinit "$direct_output")" -baseline_cuinit="$(extract_cuinit "$baseline_output")" -candidate_cuinit="$(extract_cuinit "$candidate_output")" - -printf '\n=== A/B result ===\n' -printf 'direct_docker_cuInit=%s baseline_openshell_cuInit=%s candidate_openshell_cuInit=%s\n' \ - "${direct_cuinit:-missing}" "${baseline_cuinit:-missing}" "${candidate_cuinit:-missing}" - -kill -CONT "$onboard_pid" -if wait "$onboard_pid"; then - onboard_status=0 -else - onboard_status=$? -fi -onboard_pid="" -printf 'onboard_exit_after_baseline_restore=%d\n' "$onboard_status" - -if [[ "$direct_cuinit" == "0" && "$baseline_cuinit" == "801" && "$candidate_cuinit" == "0" ]]; then - printf '\nPROVEN: the replacement and sandbox identity can initialize CUDA, but OpenShell denies the world-readable OpenRM capability device. Granting only %s read-only changes cuInit from 801 to 0.\n' \ - "$candidate_path" - exit 0 -fi +npm run build:cli -printf '\nINCONCLUSIVE: this A/B did not isolate %s. Do not implement that policy change from this result.\n' \ - "$candidate_path" >&2 -exit 1 +export NEMOCLAW_SANDBOX_NAME="$sandbox_name" +export NEMOCLAW_DIAGNOSE_JETSON_OPENRM_POLICY=1 +exec node bin/nemoclaw.js onboard --resume diff --git a/src/lib/onboard/diagnostics/jetson-openrm-proof.test.ts b/src/lib/onboard/diagnostics/jetson-openrm-proof.test.ts new file mode 100644 index 0000000000..42d93770b1 --- /dev/null +++ b/src/lib/onboard/diagnostics/jetson-openrm-proof.test.ts @@ -0,0 +1,156 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { DockerGpuPatchResult } from "../docker-gpu-patch-types"; +import { maybeRunJetsonOpenRmPolicyProof } from "./jetson-openrm-proof"; + +const BASE_POLICY = `Version: 2 +Hash: fixture +--- +version: 1 +filesystem_policy: + read_only: + - /opt/nvidia + read_write: + - /dev/nvmap +network_policies: {} +`; + +function result(): DockerGpuPatchResult { + return { + applied: true, + oldContainerId: "a".repeat(64), + newContainerId: "b".repeat(64), + originalName: "openshell-alpha-fixture", + backupContainerName: "openshell-alpha-fixture-backup", + mode: { + kind: "nvidia-runtime", + label: "--runtime nvidia", + device: "all", + args: ["--runtime", "nvidia"], + }, + backupRemoved: false, + }; +} + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe("Jetson OpenRM policy proof", () => { + it("proves the exact one-path A/B and restores the baseline policy", () => { + const appliedPolicies: string[] = []; + const runOpenshell = vi.fn((args: string[]) => { + appliedPolicies.push(fs.readFileSync(args[3] ?? "", "utf8")); + return { status: 0 }; + }); + const log = vi.spyOn(console, "log").mockImplementation(() => undefined); + + maybeRunJetsonOpenRmPolicyProof({ + backend: "jetson", + enabled: true, + failure: new Error("cuInit(0)=801"), + preserveJetsonDeviceGroupMembership: true, + result: result(), + sandboxName: "alpha", + verifyDirectSandboxGpu: vi.fn(() => ({ + status: "verified" as const, + cudaVerified: true, + at: "2026-08-06T00:00:00.000Z", + })), + deps: { + dockerRun: vi.fn(() => ({ status: 0, stdout: "cuInit(0)=0", stderr: "" })), + isCharacterDevice: () => true, + runCaptureOpenshell: vi.fn(() => BASE_POLICY), + runOpenshell, + }, + }); + + expect(appliedPolicies).toHaveLength(2); + expect(appliedPolicies[0]).toContain("/dev/nvidia-caps/nvidia-cap2"); + expect(appliedPolicies[1]).not.toContain("/dev/nvidia-caps/nvidia-cap2"); + expect(log).toHaveBeenCalledWith( + expect.stringContaining("direct_docker_cuInit=0 baseline_openshell_cuInit=801"), + ); + expect(log).toHaveBeenCalledWith(expect.stringContaining("PROVEN:")); + }); + + it("restores the baseline when the candidate CUDA proof throws", () => { + const appliedPolicies: string[] = []; + const runOpenshell = vi.fn((args: string[]) => { + appliedPolicies.push(fs.readFileSync(args[3] ?? "", "utf8")); + return { status: 0 }; + }); + + expect(() => + maybeRunJetsonOpenRmPolicyProof({ + backend: "jetson", + enabled: true, + failure: new Error("cuInit(0)=801"), + preserveJetsonDeviceGroupMembership: true, + result: result(), + sandboxName: "alpha", + verifyDirectSandboxGpu: vi.fn(() => { + throw new Error("candidate probe failed"); + }), + deps: { + dockerRun: vi.fn(() => ({ status: 0, stdout: "cuInit(0)=0", stderr: "" })), + isCharacterDevice: () => true, + runCaptureOpenshell: vi.fn(() => BASE_POLICY), + runOpenshell, + }, + }), + ).toThrow("candidate probe failed"); + + expect(appliedPolicies).toHaveLength(2); + expect(appliedPolicies[0]).toContain("/dev/nvidia-caps/nvidia-cap2"); + expect(appliedPolicies[1]).not.toContain("/dev/nvidia-caps/nvidia-cap2"); + }); + + it("attempts baseline restoration when candidate policy application reports failure", () => { + const appliedPolicies: string[] = []; + const runOpenshell = vi.fn((args: string[]) => { + appliedPolicies.push(fs.readFileSync(args[3] ?? "", "utf8")); + return { status: appliedPolicies.length === 1 ? 1 : 0 }; + }); + + expect(() => + maybeRunJetsonOpenRmPolicyProof({ + backend: "jetson", + enabled: true, + failure: new Error("cuInit(0)=801"), + preserveJetsonDeviceGroupMembership: true, + result: result(), + sandboxName: "alpha", + verifyDirectSandboxGpu: vi.fn(), + deps: { + dockerRun: vi.fn(() => ({ status: 0, stdout: "cuInit(0)=0", stderr: "" })), + isCharacterDevice: () => true, + runCaptureOpenshell: vi.fn(() => BASE_POLICY), + runOpenshell, + }, + }), + ).toThrow("candidate.yaml"); + + expect(appliedPolicies).toHaveLength(2); + expect(appliedPolicies[0]).toContain("/dev/nvidia-caps/nvidia-cap2"); + expect(appliedPolicies[1]).not.toContain("/dev/nvidia-caps/nvidia-cap2"); + }); + + it("does nothing outside the exact Jetson cuInit 801 failure", () => { + const dockerRun = vi.fn(); + maybeRunJetsonOpenRmPolicyProof({ + backend: "jetson", + enabled: true, + failure: new Error("cuInit(0)=100"), + preserveJetsonDeviceGroupMembership: true, + result: result(), + sandboxName: "alpha", + verifyDirectSandboxGpu: vi.fn(), + deps: { dockerRun }, + }); + expect(dockerRun).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/onboard/diagnostics/jetson-openrm-proof.ts b/src/lib/onboard/diagnostics/jetson-openrm-proof.ts new file mode 100644 index 0000000000..13b6789e2d --- /dev/null +++ b/src/lib/onboard/diagnostics/jetson-openrm-proof.ts @@ -0,0 +1,177 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import YAML from "yaml"; +import { parseOpenShellPolicy } from "../../policy/merge"; +import type { SandboxGpuProofResult } from "../../state/registry"; +import { dockerRun as defaultDockerRun } from "../docker-gpu-patch"; +import type { + DockerGpuPatchBackend, + DockerGpuPatchDeps, + DockerGpuPatchResult, +} from "../docker-gpu-patch-types"; + +const OPENRM_CAPABILITY_DEVICE = "/dev/nvidia-caps/nvidia-cap2"; +const CUDA_RESULT_PATTERN = /cuInit\(0\)=(-?\d+)/u; +const PROOF_TIMEOUT_MS = 30_000; +const CUDA_PROBE = [ + "import ctypes", + 'lib = ctypes.CDLL("libcuda.so.1")', + "lib.cuInit.argtypes = [ctypes.c_uint]", + "lib.cuInit.restype = ctypes.c_int", + "rc = lib.cuInit(0)", + 'print(f"cuInit(0)={rc}")', + "raise SystemExit(0 if rc == 0 else 1)", +].join("; "); + +type OpenRmProofDeps = Pick< + DockerGpuPatchDeps, + "dockerRun" | "runCaptureOpenshell" | "runOpenshell" +> & { + isCharacterDevice?: (devicePath: string) => boolean; +}; + +type OpenRmProofOptions = { + backend?: DockerGpuPatchBackend; + enabled?: boolean; + failure: Error; + preserveJetsonDeviceGroupMembership?: boolean; + result: DockerGpuPatchResult | null; + sandboxName: string; + verifyDirectSandboxGpu: (sandboxName: string) => SandboxGpuProofResult; + deps: OpenRmProofDeps; +}; + +type PolicyDocument = { + filesystem_policy?: { + read_only?: unknown; + }; +}; + +function cudaResult(value: string): string { + return value.match(CUDA_RESULT_PATTERN)?.[1] ?? "missing"; +} + +function proofCudaResult(proof: SandboxGpuProofResult): string { + if (proof.status === "verified" && proof.cudaVerified) return "0"; + return cudaResult(proof.detail ?? ""); +} + +function hasCharacterDevice(devicePath: string): boolean { + try { + return fs.statSync(devicePath).isCharacterDevice(); + } catch { + return false; + } +} + +function setPolicy( + sandboxName: string, + policyPath: string, + runOpenshell: NonNullable, +): void { + const result = runOpenshell(["policy", "set", "--policy", policyPath, "--wait", sandboxName], { + ignoreError: true, + suppressOutput: true, + timeout: PROOF_TIMEOUT_MS, + }); + if (result.status !== 0) { + throw new Error(`OpenShell rejected diagnostic policy file ${policyPath}.`); + } +} + +function candidatePolicy(policyYaml: string): string { + const policy = YAML.parse(policyYaml) as PolicyDocument | null; + const filesystemPolicy = policy?.filesystem_policy; + if (!filesystemPolicy || typeof filesystemPolicy !== "object") { + throw new Error("OpenShell base policy has no filesystem_policy mapping."); + } + if (!Array.isArray(filesystemPolicy.read_only)) { + throw new Error("OpenShell base policy filesystem_policy.read_only is not a list."); + } + if (filesystemPolicy.read_only.includes(OPENRM_CAPABILITY_DEVICE)) { + throw new Error(`${OPENRM_CAPABILITY_DEVICE} is already present in the baseline policy.`); + } + filesystemPolicy.read_only.push(OPENRM_CAPABILITY_DEVICE); + return YAML.stringify(policy); +} + +/** + * Maintainer-only hardware A/B for issue #7610. The caller invokes this after + * the live OpenShell CUDA proof returns 801 and before it rolls the exact + * replacement container back. The baseline policy is restored in `finally`. + */ +export function maybeRunJetsonOpenRmPolicyProof(options: OpenRmProofOptions): void { + const enabled = options.enabled ?? process.env.NEMOCLAW_DIAGNOSE_JETSON_OPENRM_POLICY === "1"; + if ( + !enabled || + options.backend !== "jetson" || + options.preserveJetsonDeviceGroupMembership !== true || + options.result?.mode.kind !== "nvidia-runtime" || + !/cuInit\(0\)=801/u.test(options.failure.message) + ) { + return; + } + + const dockerRun = options.deps.dockerRun ?? defaultDockerRun; + const { runCaptureOpenshell: captureOpenshell, runOpenshell } = options.deps; + if (!captureOpenshell || !runOpenshell) { + console.error(" OpenRM A/B inconclusive: required OpenShell adapters are unavailable."); + return; + } + const isCharacterDevice = options.deps.isCharacterDevice ?? hasCharacterDevice; + if (!isCharacterDevice(OPENRM_CAPABILITY_DEVICE)) { + console.error( + ` OpenRM A/B inconclusive: ${OPENRM_CAPABILITY_DEVICE} is not a host character device.`, + ); + return; + } + + const direct = dockerRun( + ["exec", "--user", "sandbox", options.result.newContainerId, "python3", "-c", CUDA_PROBE], + { ignoreError: true, suppressOutput: true, timeout: PROOF_TIMEOUT_MS }, + ); + const directOutput = `${direct.stderr ?? ""}\n${direct.stdout ?? ""}`; + const directResult = cudaResult(directOutput); + const rawPolicy = captureOpenshell(["policy", "get", "--base", options.sandboxName], { + ignoreError: false, + timeout: PROOF_TIMEOUT_MS, + }); + const baselinePolicy = parseOpenShellPolicy(rawPolicy).yamlBody; + if (!baselinePolicy) throw new Error("OpenShell returned no round-trippable base policy."); + const candidate = candidatePolicy(baselinePolicy); + const temporaryDirectory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-openrm-proof-")); + const baselinePath = path.join(temporaryDirectory, "baseline.yaml"); + const candidatePath = path.join(temporaryDirectory, "candidate.yaml"); + fs.writeFileSync(baselinePath, baselinePolicy, { encoding: "utf8", mode: 0o600 }); + fs.writeFileSync(candidatePath, candidate, { encoding: "utf8", mode: 0o600 }); + + let candidateResult = "missing"; + let candidateApplied = false; + try { + candidateApplied = true; + setPolicy(options.sandboxName, candidatePath, runOpenshell); + candidateResult = proofCudaResult(options.verifyDirectSandboxGpu(options.sandboxName)); + } finally { + if (candidateApplied) setPolicy(options.sandboxName, baselinePath, runOpenshell); + fs.rmSync(temporaryDirectory, { recursive: true, force: true }); + } + + console.log(""); + console.log(" === Jetson OpenRM policy A/B ==="); + console.log( + ` direct_docker_cuInit=${directResult} baseline_openshell_cuInit=801 candidate_openshell_cuInit=${candidateResult}`, + ); + if (directResult === "0" && candidateResult === "0") { + console.log( + ` PROVEN: granting only ${OPENRM_CAPABILITY_DEVICE} read-only changes OpenShell cuInit from 801 to 0.`, + ); + } else { + console.error( + ` INCONCLUSIVE: the A/B did not isolate ${OPENRM_CAPABILITY_DEVICE}; no production policy change is justified.`, + ); + } +} diff --git a/src/lib/onboard/docker-gpu-patch.ts b/src/lib/onboard/docker-gpu-patch.ts index 4331d2f56f..74d64efd08 100644 --- a/src/lib/onboard/docker-gpu-patch.ts +++ b/src/lib/onboard/docker-gpu-patch.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { dockerCapture } from "../adapters/docker"; +import { dockerCapture, dockerRun } from "../adapters/docker"; import { parseLiveSandboxEntries } from "../runtime-recovery"; import { createDockerGpuDiagnosticRedactor } from "./docker-gpu-diagnostic-redaction"; import { DOCKER_GPU_PATCH_TIMEOUT_MS } from "./docker-gpu-patch-constants"; @@ -27,6 +27,7 @@ export { getDockerGpuPatchNetworkMode, parseDockerInspectJson, } from "./docker-gpu-patch-clone"; +export { dockerRun }; import { collectDockerGpuPatchDiagnostics } from "./docker-gpu-patch-diagnostics"; import { diff --git a/src/lib/onboard/docker-gpu-sandbox-create.ts b/src/lib/onboard/docker-gpu-sandbox-create.ts index 8ccb34f2b2..fd246adb54 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create.ts @@ -3,6 +3,7 @@ import { getSandboxFailurePhase } from "../state/gateway"; import type { SandboxGpuProofResult } from "../state/registry"; +import { maybeRunJetsonOpenRmPolicyProof } from "./diagnostics/jetson-openrm-proof"; import { getDockerGpuSupervisorReconnectTimeoutSecs, printDockerGpuPatchFailureAndExit, @@ -557,6 +558,21 @@ export function createDockerGpuSandboxCreatePatch( return proof; } catch (error) { const failure = error instanceof Error ? error : new Error(String(error)); + try { + maybeRunJetsonOpenRmPolicyProof({ + backend: options.backend, + failure, + preserveJetsonDeviceGroupMembership: options.preserveJetsonDeviceGroupMembership, + result, + sandboxName, + verifyDirectSandboxGpu, + deps: options.deps, + }); + } catch (diagnosticError) { + console.error( + ` OpenRM A/B inconclusive: ${diagnosticError instanceof Error ? diagnosticError.message : String(diagnosticError)}`, + ); + } printDockerGpuProofFailure(sandboxName, failure, selectedMode(), { runCaptureOpenshell: options.deps.runCaptureOpenshell, dockerCapture: options.deps.dockerCapture, From f06f36537751532d6164579fa267018e905511d8 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Thu, 6 Aug 2026 09:41:48 -0700 Subject: [PATCH 33/49] fix(onboard): resolve Jetson review findings Signed-off-by: Apurv Kumaria --- scripts/jetson-device-group-bootstrap.sh | 14 +++++++++----- src/lib/onboard/diagnostics/jetson-openrm-proof.ts | 4 +--- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/scripts/jetson-device-group-bootstrap.sh b/scripts/jetson-device-group-bootstrap.sh index 0d533c82ff..c814d1015a 100755 --- a/scripts/jetson-device-group-bootstrap.sh +++ b/scripts/jetson-device-group-bootstrap.sh @@ -17,11 +17,14 @@ shift 3 [ "${1:-}" = "/opt/openshell/bin/openshell-sandbox" ] \ || fail "OpenShell supervisor entrypoint is invalid" /usr/bin/id sandbox >/dev/null 2>&1 || fail "sandbox user is missing" -[ -f /etc/group ] && [ ! -L /etc/group ] || fail "container group database is invalid" +if [ ! -f /etc/group ] || [ -L /etc/group ]; then + fail "container group database is invalid" +fi IFS=',' read -r -a gids <<<"$group_gids" -[ "${#gids[@]}" -gt 0 ] && [ "${#gids[@]}" -le 16 ] \ - || fail "device group count is invalid" +if [ "${#gids[@]}" -eq 0 ] || [ "${#gids[@]}" -gt 16 ]; then + fail "device group count is invalid" +fi declare -A seen=() for gid in "${gids[@]}"; do @@ -36,8 +39,9 @@ for gid in "${gids[@]}"; do /usr/sbin/groupadd --gid "$gid" "$group_name" else IFS=':' read -r group_name _ resolved_gid _ <<<"$group_record" - [ -n "$group_name" ] && [ "$resolved_gid" = "$gid" ] \ - || fail "device group record is invalid" + if [ -z "$group_name" ] || [ "$resolved_gid" != "$gid" ]; then + fail "device group record is invalid" + fi fi /usr/sbin/usermod --append --groups "$group_name" sandbox done diff --git a/src/lib/onboard/diagnostics/jetson-openrm-proof.ts b/src/lib/onboard/diagnostics/jetson-openrm-proof.ts index 13b6789e2d..bcd59835cc 100644 --- a/src/lib/onboard/diagnostics/jetson-openrm-proof.ts +++ b/src/lib/onboard/diagnostics/jetson-openrm-proof.ts @@ -150,13 +150,11 @@ export function maybeRunJetsonOpenRmPolicyProof(options: OpenRmProofOptions): vo fs.writeFileSync(candidatePath, candidate, { encoding: "utf8", mode: 0o600 }); let candidateResult = "missing"; - let candidateApplied = false; try { - candidateApplied = true; setPolicy(options.sandboxName, candidatePath, runOpenshell); candidateResult = proofCudaResult(options.verifyDirectSandboxGpu(options.sandboxName)); } finally { - if (candidateApplied) setPolicy(options.sandboxName, baselinePath, runOpenshell); + setPolicy(options.sandboxName, baselinePath, runOpenshell); fs.rmSync(temporaryDirectory, { recursive: true, force: true }); } From cd42a42447e9e97a3224a2418230d686c7f72702 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Thu, 6 Aug 2026 09:45:12 -0700 Subject: [PATCH 34/49] test(images): cover Jetson helper permissions Signed-off-by: Apurv Kumaria --- test/sandbox-provisioning-helper-permissions.test.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/sandbox-provisioning-helper-permissions.test.ts b/test/sandbox-provisioning-helper-permissions.test.ts index 5f8a36a7a7..57c9275b1d 100644 --- a/test/sandbox-provisioning-helper-permissions.test.ts +++ b/test/sandbox-provisioning-helper-permissions.test.ts @@ -129,6 +129,7 @@ describe("sandbox provisioning: copied OpenClaw helper permissions (#2861)", () const nestedPluginFile = path.join(nestedPluginDir, "helper.js"); const gatewayControlPath = path.join(localBin, "nemoclaw-gateway-control"); const gatewaySupervisorPath = path.join(localLib, "gateway-supervisor.sh"); + const jetsonDeviceGroupBootstrapPath = path.join(localLib, "jetson-device-group-bootstrap.sh"); const stateDirGuardPath = path.join(localLib, "state-dir-guard.py"); const stateLockPlanPath = path.join(localShare, "state-lock-plan.json"); const configGuardPath = path.join(localLib, "openclaw-config-guard.py"); @@ -143,6 +144,7 @@ describe("sandbox provisioning: copied OpenClaw helper permissions (#2861)", () path.join(localLib, "sandbox-init.sh"), path.join(localLib, "sandbox-rlimits.sh"), gatewaySupervisorPath, + jetsonDeviceGroupBootstrapPath, stateDirGuardPath, stateLockPlanPath, configGuardPath, @@ -205,6 +207,7 @@ describe("sandbox provisioning: copied OpenClaw helper permissions (#2861)", () expect((fs.statSync(nestedPluginFile).mode & 0o777).toString(8)).toBe("644"); expect((fs.statSync(gatewayControlPath).mode & 0o777).toString(8)).toBe("700"); expect((fs.statSync(gatewaySupervisorPath).mode & 0o777).toString(8)).toBe("444"); + expect((fs.statSync(jetsonDeviceGroupBootstrapPath).mode & 0o777).toString(8)).toBe("500"); expect((fs.statSync(stateDirGuardPath).mode & 0o777).toString(8)).toBe("500"); expect((fs.statSync(stateLockPlanPath).mode & 0o777).toString(8)).toBe("444"); expect((fs.statSync(configGuardPath).mode & 0o777).toString(8)).toBe("500"); From 0a0bc1328ab077cf06004acec7a867d64e427ad4 Mon Sep 17 00:00:00 2001 From: San Dang Date: Thu, 6 Aug 2026 23:56:03 +0700 Subject: [PATCH 35/49] fix(onboard): recreate sandbox for OpenRM proof Signed-off-by: San Dang --- scripts/prove-jetson-openrm-policy-boundary.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/prove-jetson-openrm-policy-boundary.sh b/scripts/prove-jetson-openrm-policy-boundary.sh index d83f6fe65f..2678b55c2c 100755 --- a/scripts/prove-jetson-openrm-policy-boundary.sh +++ b/scripts/prove-jetson-openrm-policy-boundary.sh @@ -12,10 +12,11 @@ fi printf 'The proof is armed for sandbox %s. Complete the normal onboarding prompts.\n' \ "$sandbox_name" +printf 'The sandbox will be rebuilt with the current checkout before the proof runs.\n' printf 'When the live replacement returns cuInit(0)=801, NemoClaw will run the one-path A/B before its normal rollback.\n' npm run build:cli export NEMOCLAW_SANDBOX_NAME="$sandbox_name" export NEMOCLAW_DIAGNOSE_JETSON_OPENRM_POLICY=1 -exec node bin/nemoclaw.js onboard --resume +exec node bin/nemoclaw.js onboard --resume --recreate-sandbox From 6e1cb7664580b9a1df7d7e47f478ebb7007bb9b5 Mon Sep 17 00:00:00 2001 From: San Dang Date: Fri, 7 Aug 2026 00:17:40 +0700 Subject: [PATCH 36/49] fix(onboard): honor explicit sandbox recreation Signed-off-by: San Dang --- .../sandbox-checkpoint-crash-recovery.test.ts | 22 +++++++++++++++++++ src/lib/onboard/machine/handlers/sandbox.ts | 1 + 2 files changed, 23 insertions(+) diff --git a/src/lib/onboard/machine/handlers/sandbox-checkpoint-crash-recovery.test.ts b/src/lib/onboard/machine/handlers/sandbox-checkpoint-crash-recovery.test.ts index 9b54893fb2..520af17538 100644 --- a/src/lib/onboard/machine/handlers/sandbox-checkpoint-crash-recovery.test.ts +++ b/src/lib/onboard/machine/handlers/sandbox-checkpoint-crash-recovery.test.ts @@ -883,6 +883,28 @@ describe("sandbox crash-recovery replay (#5961, #6228)", () => { expect(calls.error.mock.calls.flat().join("\n")).toContain("--recreate-sandbox"); }); + it("recreates after build or policy drift when explicitly requested (#7022)", async () => { + const session = sessionWithCheckpoint( + crashedCheckpoint({ + effectGroups: { + sandbox_create: { completedAt: "2026-01-01T00:00:00.000Z", fingerprint: "stale-build" }, + }, + }), + ); + session.machine.state = "openclaw"; + const { deps, calls } = createDeps({ getSandboxReuseState: () => "ready" }, session); + + await handleSandboxState({ + ...baseOptions(deps, session), + resume: true, + sandboxName: "my-assistant", + recreateSandbox: () => true, + }); + + expect(calls.createSandbox).toHaveBeenCalledOnce(); + expect(calls.error).not.toHaveBeenCalled(); + }); + it("rejects reuse when a resolved policy or package input drifted despite an unchanged build version and policy tier (#7022)", async () => { const { deps, calls } = createDeps({ getSandboxReuseState: () => "ready" }); const session = sessionWithCheckpoint(crashedCheckpoint()); diff --git a/src/lib/onboard/machine/handlers/sandbox.ts b/src/lib/onboard/machine/handlers/sandbox.ts index f318a1aa5c..51b3157453 100644 --- a/src/lib/onboard/machine/handlers/sandbox.ts +++ b/src/lib/onboard/machine/handlers/sandbox.ts @@ -781,6 +781,7 @@ class SandboxStateFlow< sandboxName: string, createIntent: ResolvedSandboxCreateIntent, ): void { + if (this.options.recreateSandbox(false)) return; const recordedFingerprint = state.session?.checkpoint?.effectGroups.sandbox_create?.fingerprint; if (!recordedFingerprint) return; if (recordedFingerprint !== this.currentSandboxCreateFingerprint(sandboxName, createIntent)) { From 5a0fa51eed8400716e7c34968ae72f1b104811e0 Mon Sep 17 00:00:00 2001 From: San Dang Date: Fri, 7 Aug 2026 00:36:34 +0700 Subject: [PATCH 37/49] chore(onboard): expand Jetson OpenRM proof Signed-off-by: San Dang --- .../diagnostics/jetson-openrm-proof.test.ts | 66 ++++-- .../diagnostics/jetson-openrm-proof.ts | 195 ++++++++++++++---- 2 files changed, 206 insertions(+), 55 deletions(-) diff --git a/src/lib/onboard/diagnostics/jetson-openrm-proof.test.ts b/src/lib/onboard/diagnostics/jetson-openrm-proof.test.ts index 42d93770b1..5dd321f06f 100644 --- a/src/lib/onboard/diagnostics/jetson-openrm-proof.test.ts +++ b/src/lib/onboard/diagnostics/jetson-openrm-proof.test.ts @@ -35,18 +35,50 @@ function result(): DockerGpuPatchResult { }; } +function dockerRunForBoundaryProof() { + return vi.fn((args: readonly string[]) => + args.includes("0") + ? { + status: 0, + stdout: ["/dev/nvidia-caps/nvidia-cap2", "/dev/nvhost-ctrl-pva0", "/dev/nvmap"].join( + "\n", + ), + stderr: "", + } + : { status: 0, stdout: "cuInit(0)=0", stderr: "" }, + ); +} + afterEach(() => { vi.restoreAllMocks(); }); describe("Jetson OpenRM policy proof", () => { - it("proves the exact one-path A/B and restores the baseline policy", () => { + it("isolates missing injected devices from sysfs and restores the baseline policy", () => { const appliedPolicies: string[] = []; const runOpenshell = vi.fn((args: string[]) => { appliedPolicies.push(fs.readFileSync(args[3] ?? "", "utf8")); return { status: 0 }; }); const log = vi.spyOn(console, "log").mockImplementation(() => undefined); + const verifyDirectSandboxGpu = vi + .fn() + .mockReturnValueOnce({ + status: "verified" as const, + cudaVerified: true, + at: "2026-08-06T00:00:00.000Z", + }) + .mockReturnValueOnce({ + status: "failed" as const, + cudaVerified: false, + detail: "cuInit(0)=801", + at: "2026-08-06T00:00:00.000Z", + }) + .mockReturnValueOnce({ + status: "verified" as const, + cudaVerified: true, + at: "2026-08-06T00:00:00.000Z", + }); maybeRunJetsonOpenRmPolicyProof({ backend: "jetson", @@ -55,26 +87,26 @@ describe("Jetson OpenRM policy proof", () => { preserveJetsonDeviceGroupMembership: true, result: result(), sandboxName: "alpha", - verifyDirectSandboxGpu: vi.fn(() => ({ - status: "verified" as const, - cudaVerified: true, - at: "2026-08-06T00:00:00.000Z", - })), + verifyDirectSandboxGpu, deps: { - dockerRun: vi.fn(() => ({ status: 0, stdout: "cuInit(0)=0", stderr: "" })), - isCharacterDevice: () => true, + dockerRun: dockerRunForBoundaryProof(), runCaptureOpenshell: vi.fn(() => BASE_POLICY), runOpenshell, }, }); - expect(appliedPolicies).toHaveLength(2); + expect(appliedPolicies).toHaveLength(4); expect(appliedPolicies[0]).toContain("/dev/nvidia-caps/nvidia-cap2"); + expect(appliedPolicies[0]).toContain("/dev/nvhost-ctrl-pva0"); + expect(appliedPolicies[0]).not.toContain("- /sys"); + expect(appliedPolicies[1]).toContain("- /sys"); expect(appliedPolicies[1]).not.toContain("/dev/nvidia-caps/nvidia-cap2"); - expect(log).toHaveBeenCalledWith( - expect.stringContaining("direct_docker_cuInit=0 baseline_openshell_cuInit=801"), - ); - expect(log).toHaveBeenCalledWith(expect.stringContaining("PROVEN:")); + expect(appliedPolicies[2]).toContain("/dev/nvidia-caps/nvidia-cap2"); + expect(appliedPolicies[2]).toContain("- /sys"); + expect(appliedPolicies[3]).not.toContain("/dev/nvidia-caps/nvidia-cap2"); + expect(appliedPolicies[3]).not.toContain("- /sys"); + expect(log).toHaveBeenCalledWith(expect.stringContaining("devices_cuInit=0 sysfs_cuInit=801")); + expect(log).toHaveBeenCalledWith(expect.stringContaining("ISOLATED:")); }); it("restores the baseline when the candidate CUDA proof throws", () => { @@ -96,8 +128,7 @@ describe("Jetson OpenRM policy proof", () => { throw new Error("candidate probe failed"); }), deps: { - dockerRun: vi.fn(() => ({ status: 0, stdout: "cuInit(0)=0", stderr: "" })), - isCharacterDevice: () => true, + dockerRun: dockerRunForBoundaryProof(), runCaptureOpenshell: vi.fn(() => BASE_POLICY), runOpenshell, }, @@ -126,13 +157,12 @@ describe("Jetson OpenRM policy proof", () => { sandboxName: "alpha", verifyDirectSandboxGpu: vi.fn(), deps: { - dockerRun: vi.fn(() => ({ status: 0, stdout: "cuInit(0)=0", stderr: "" })), - isCharacterDevice: () => true, + dockerRun: dockerRunForBoundaryProof(), runCaptureOpenshell: vi.fn(() => BASE_POLICY), runOpenshell, }, }), - ).toThrow("candidate.yaml"); + ).toThrow("devices.yaml"); expect(appliedPolicies).toHaveLength(2); expect(appliedPolicies[0]).toContain("/dev/nvidia-caps/nvidia-cap2"); diff --git a/src/lib/onboard/diagnostics/jetson-openrm-proof.ts b/src/lib/onboard/diagnostics/jetson-openrm-proof.ts index bcd59835cc..eb97e75abf 100644 --- a/src/lib/onboard/diagnostics/jetson-openrm-proof.ts +++ b/src/lib/onboard/diagnostics/jetson-openrm-proof.ts @@ -14,9 +14,10 @@ import type { DockerGpuPatchResult, } from "../docker-gpu-patch-types"; -const OPENRM_CAPABILITY_DEVICE = "/dev/nvidia-caps/nvidia-cap2"; const CUDA_RESULT_PATTERN = /cuInit\(0\)=(-?\d+)/u; const PROOF_TIMEOUT_MS = 30_000; +const SYSFS_ROOT = "/sys"; +const MAX_GPU_DEVICE_PATHS = 128; const CUDA_PROBE = [ "import ctypes", 'lib = ctypes.CDLL("libcuda.so.1")', @@ -26,13 +27,28 @@ const CUDA_PROBE = [ 'print(f"cuInit(0)={rc}")', "raise SystemExit(0 if rc == 0 else 1)", ].join("; "); +const GPU_DEVICE_DISCOVERY_PROBE = [ + "import os, stat", + "prefixes = ('/dev/nvidia', '/dev/nvhost', '/dev/nvgpu', '/dev/tegra')", + "for root, dirs, files in os.walk('/dev'):", + " for name in files:", + " device = os.path.join(root, name)", + " relevant = device == '/dev/nvmap' or device.startswith(prefixes) or (device.startswith('/dev/dri/') and (name.startswith('renderD') or name.startswith('card')))", + " if relevant:", + " try:", + " if stat.S_ISCHR(os.lstat(device).st_mode): print(device)", + " except OSError:", + " pass", +].join("\n"); +const PROCESS_STATUS_PROBE = + "grep -E '^(Uid|Gid|Groups|CapInh|CapPrm|CapEff|CapBnd|CapAmb|NoNewPrivs|Seccomp|Seccomp_filters):' /proc/self/status"; +const GPU_DEVICE_PATH_PATTERN = + /^\/dev\/(?:nvidia[A-Za-z0-9._/-]*|nvhost[A-Za-z0-9._/-]*|nvgpu(?:\/[A-Za-z0-9._-]+)*|tegra[A-Za-z0-9._/-]*|nvmap|dri\/(?:renderD|card)\d+)$/u; type OpenRmProofDeps = Pick< DockerGpuPatchDeps, "dockerRun" | "runCaptureOpenshell" | "runOpenshell" -> & { - isCharacterDevice?: (devicePath: string) => boolean; -}; +>; type OpenRmProofOptions = { backend?: DockerGpuPatchBackend; @@ -48,9 +64,16 @@ type OpenRmProofOptions = { type PolicyDocument = { filesystem_policy?: { read_only?: unknown; + read_write?: unknown; }; }; +type PolicyCandidate = { + name: string; + readOnly: string[]; + readWrite: string[]; +}; + function cudaResult(value: string): string { return value.match(CUDA_RESULT_PATTERN)?.[1] ?? "missing"; } @@ -60,14 +83,6 @@ function proofCudaResult(proof: SandboxGpuProofResult): string { return cudaResult(proof.detail ?? ""); } -function hasCharacterDevice(devicePath: string): boolean { - try { - return fs.statSync(devicePath).isCharacterDevice(); - } catch { - return false; - } -} - function setPolicy( sandboxName: string, policyPath: string, @@ -83,22 +98,71 @@ function setPolicy( } } -function candidatePolicy(policyYaml: string): string { +function parseFilesystemPolicy(policyYaml: string): { + policy: PolicyDocument; + readOnly: string[]; + readWrite: string[]; +} { const policy = YAML.parse(policyYaml) as PolicyDocument | null; const filesystemPolicy = policy?.filesystem_policy; if (!filesystemPolicy || typeof filesystemPolicy !== "object") { throw new Error("OpenShell base policy has no filesystem_policy mapping."); } - if (!Array.isArray(filesystemPolicy.read_only)) { - throw new Error("OpenShell base policy filesystem_policy.read_only is not a list."); + if (!Array.isArray(filesystemPolicy.read_only) || !Array.isArray(filesystemPolicy.read_write)) { + throw new Error("OpenShell base policy filesystem policy paths are not lists."); } - if (filesystemPolicy.read_only.includes(OPENRM_CAPABILITY_DEVICE)) { - throw new Error(`${OPENRM_CAPABILITY_DEVICE} is already present in the baseline policy.`); + return { + policy, + readOnly: filesystemPolicy.read_only.map(String), + readWrite: filesystemPolicy.read_write.map(String), + }; +} + +function candidatePolicy(policyYaml: string, candidate: PolicyCandidate): string { + const { policy, readOnly, readWrite } = parseFilesystemPolicy(policyYaml); + const filesystemPolicy = policy.filesystem_policy; + if (!filesystemPolicy) throw new Error("OpenShell base policy has no filesystem_policy mapping."); + const readWriteSet = new Set(readWrite); + for (const devicePath of candidate.readWrite) readWriteSet.add(devicePath); + const readOnlySet = new Set(readOnly.filter((policyPath) => !readWriteSet.has(policyPath))); + for (const policyPath of candidate.readOnly) { + if (!readWriteSet.has(policyPath)) readOnlySet.add(policyPath); } - filesystemPolicy.read_only.push(OPENRM_CAPABILITY_DEVICE); + filesystemPolicy.read_only = [...readOnlySet]; + filesystemPolicy.read_write = [...readWriteSet]; return YAML.stringify(policy); } +function discoverInjectedGpuDevicePaths( + containerId: string, + dockerRun: NonNullable, +): string[] { + const result = dockerRun( + ["exec", "--user", "0", containerId, "python3", "-c", GPU_DEVICE_DISCOVERY_PROBE], + { ignoreError: true, suppressOutput: true, timeout: PROOF_TIMEOUT_MS }, + ); + if (result.status !== 0) throw new Error("Could not enumerate injected GPU character devices."); + const devicePaths = [ + ...new Set( + String(result.stdout ?? "") + .split(/\r?\n/u) + .map((devicePath) => devicePath.trim()) + .filter((devicePath) => GPU_DEVICE_PATH_PATTERN.test(devicePath)), + ), + ].sort(); + if (devicePaths.length === 0 || devicePaths.length > MAX_GPU_DEVICE_PATHS) { + throw new Error("Injected GPU character-device enumeration is empty or excessive."); + } + return devicePaths; +} + +function compactProcessStatus(value: string | Buffer | null | undefined): string { + return String(value ?? "") + .trim() + .split(/\r?\n/u) + .join("; "); +} + /** * Maintainer-only hardware A/B for issue #7610. The caller invokes this after * the live OpenShell CUDA proof returns 801 and before it rolls the exact @@ -122,54 +186,111 @@ export function maybeRunJetsonOpenRmPolicyProof(options: OpenRmProofOptions): vo console.error(" OpenRM A/B inconclusive: required OpenShell adapters are unavailable."); return; } - const isCharacterDevice = options.deps.isCharacterDevice ?? hasCharacterDevice; - if (!isCharacterDevice(OPENRM_CAPABILITY_DEVICE)) { - console.error( - ` OpenRM A/B inconclusive: ${OPENRM_CAPABILITY_DEVICE} is not a host character device.`, - ); - return; - } - const direct = dockerRun( ["exec", "--user", "sandbox", options.result.newContainerId, "python3", "-c", CUDA_PROBE], { ignoreError: true, suppressOutput: true, timeout: PROOF_TIMEOUT_MS }, ); const directOutput = `${direct.stderr ?? ""}\n${direct.stdout ?? ""}`; const directResult = cudaResult(directOutput); + const injectedDevicePaths = discoverInjectedGpuDevicePaths( + options.result.newContainerId, + dockerRun, + ); const rawPolicy = captureOpenshell(["policy", "get", "--base", options.sandboxName], { ignoreError: false, timeout: PROOF_TIMEOUT_MS, }); const baselinePolicy = parseOpenShellPolicy(rawPolicy).yamlBody; if (!baselinePolicy) throw new Error("OpenShell returned no round-trippable base policy."); - const candidate = candidatePolicy(baselinePolicy); + const baselineFilesystemPolicy = parseFilesystemPolicy(baselinePolicy); + const baselineReadWrite = new Set(baselineFilesystemPolicy.readWrite); + const missingDevicePaths = injectedDevicePaths.filter( + (devicePath) => !baselineReadWrite.has(devicePath), + ); + const sysfsMissing = + !baselineFilesystemPolicy.readOnly.includes(SYSFS_ROOT) && + !baselineFilesystemPolicy.readWrite.includes(SYSFS_ROOT); + const candidates: PolicyCandidate[] = []; + if (missingDevicePaths.length > 0) { + candidates.push({ name: "devices", readOnly: [], readWrite: missingDevicePaths }); + } + if (sysfsMissing) { + candidates.push({ name: "sysfs", readOnly: [SYSFS_ROOT], readWrite: [] }); + } + if (missingDevicePaths.length > 0 && sysfsMissing) { + candidates.push({ + name: "devices-plus-sysfs", + readOnly: [SYSFS_ROOT], + readWrite: missingDevicePaths, + }); + } const temporaryDirectory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-openrm-proof-")); const baselinePath = path.join(temporaryDirectory, "baseline.yaml"); - const candidatePath = path.join(temporaryDirectory, "candidate.yaml"); fs.writeFileSync(baselinePath, baselinePolicy, { encoding: "utf8", mode: 0o600 }); - fs.writeFileSync(candidatePath, candidate, { encoding: "utf8", mode: 0o600 }); - let candidateResult = "missing"; + const candidateResults = new Map(); try { - setPolicy(options.sandboxName, candidatePath, runOpenshell); - candidateResult = proofCudaResult(options.verifyDirectSandboxGpu(options.sandboxName)); + for (const candidate of candidates) { + const candidatePath = path.join(temporaryDirectory, `${candidate.name}.yaml`); + fs.writeFileSync(candidatePath, candidatePolicy(baselinePolicy, candidate), { + encoding: "utf8", + mode: 0o600, + }); + setPolicy(options.sandboxName, candidatePath, runOpenshell); + candidateResults.set( + candidate.name, + proofCudaResult(options.verifyDirectSandboxGpu(options.sandboxName)), + ); + } } finally { setPolicy(options.sandboxName, baselinePath, runOpenshell); fs.rmSync(temporaryDirectory, { recursive: true, force: true }); } + const deviceResult = candidateResults.get("devices") ?? "not-tested"; + const sysfsResult = candidateResults.get("sysfs") ?? "not-tested"; + const combinedResult = candidateResults.get("devices-plus-sysfs") ?? "not-tested"; + console.log(""); - console.log(" === Jetson OpenRM policy A/B ==="); + console.log(" === Jetson OpenRM policy boundary matrix ==="); + console.log(` injected_gpu_devices=${injectedDevicePaths.join(",")}`); + console.log(` policy_missing_gpu_devices=${missingDevicePaths.join(",") || "none"}`); console.log( - ` direct_docker_cuInit=${directResult} baseline_openshell_cuInit=801 candidate_openshell_cuInit=${candidateResult}`, + ` direct_docker_cuInit=${directResult} baseline_openshell_cuInit=801 devices_cuInit=${deviceResult} sysfs_cuInit=${sysfsResult} devices_plus_sysfs_cuInit=${combinedResult}`, ); - if (directResult === "0" && candidateResult === "0") { + if (directResult === "0" && deviceResult === "0" && sysfsResult !== "0") { + console.log( + " ISOLATED: OpenShell policy is missing one or more NVIDIA/Tegra character devices; no sysfs grant is required.", + ); + } else if (directResult === "0" && sysfsResult === "0" && deviceResult !== "0") { console.log( - ` PROVEN: granting only ${OPENRM_CAPABILITY_DEVICE} read-only changes OpenShell cuInit from 801 to 0.`, + " ISOLATED: OpenShell policy is missing CUDA-required sysfs visibility; exact sysfs paths still need narrowing.", + ); + } else if (directResult === "0" && combinedResult === "0") { + console.log( + " ISOLATED: CUDA requires both the missing GPU devices and sysfs visibility through OpenShell.", ); } else { console.error( - ` INCONCLUSIVE: the A/B did not isolate ${OPENRM_CAPABILITY_DEVICE}; no production policy change is justified.`, + " INCONCLUSIVE: the filesystem-policy matrix did not restore CUDA; no production policy change is justified.", + ); + const directStatus = dockerRun( + [ + "exec", + "--user", + "sandbox", + options.result.newContainerId, + "sh", + "-lc", + PROCESS_STATUS_PROBE, + ], + { ignoreError: true, suppressOutput: true, timeout: PROOF_TIMEOUT_MS }, + ); + const openshellStatus = runOpenshell( + ["sandbox", "exec", "-n", options.sandboxName, "--", "sh", "-lc", PROCESS_STATUS_PROBE], + { ignoreError: true, suppressOutput: true, timeout: PROOF_TIMEOUT_MS }, ); + console.log(` direct_process_status=${compactProcessStatus(directStatus.stdout)}`); + console.log(` openshell_process_status=${compactProcessStatus(openshellStatus.stdout)}`); } } From d9cbe35d30a35136dbbfa758a0a91a7c5994baf5 Mon Sep 17 00:00:00 2001 From: San Dang Date: Fri, 7 Aug 2026 00:38:23 +0700 Subject: [PATCH 38/49] docs(onboard): describe OpenRM proof matrix Signed-off-by: San Dang --- scripts/prove-jetson-openrm-policy-boundary.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/prove-jetson-openrm-policy-boundary.sh b/scripts/prove-jetson-openrm-policy-boundary.sh index 2678b55c2c..023ea1e106 100755 --- a/scripts/prove-jetson-openrm-policy-boundary.sh +++ b/scripts/prove-jetson-openrm-policy-boundary.sh @@ -13,7 +13,7 @@ fi printf 'The proof is armed for sandbox %s. Complete the normal onboarding prompts.\n' \ "$sandbox_name" printf 'The sandbox will be rebuilt with the current checkout before the proof runs.\n' -printf 'When the live replacement returns cuInit(0)=801, NemoClaw will run the one-path A/B before its normal rollback.\n' +printf 'When the live replacement returns cuInit(0)=801, NemoClaw will run the device/sysfs policy matrix before its normal rollback.\n' npm run build:cli From f25c2b5c5a656ea55e9008146159301eda48db6d Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Thu, 6 Aug 2026 15:23:55 -0700 Subject: [PATCH 39/49] fix(onboard): validate Jetson groups before mutation Signed-off-by: Apurv Kumaria --- docs/reference/troubleshooting.mdx | 1 + scripts/jetson-device-group-bootstrap.sh | 5 + test/jetson-device-group-bootstrap.test.ts | 390 +++++++++++++++++++++ 3 files changed, 396 insertions(+) create mode 100644 test/jetson-device-group-bootstrap.test.ts diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index d699add4f1..ab27d6330c 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -2738,6 +2738,7 @@ That call replaces the inherited supplementary groups, so `--group-add` alone do Before the fixed OpenShell supervisor starts, NemoClaw runs a bounded wrapper from the sandbox image as root. The image owns the wrapper as `root:root` with mode `0500`. The wrapper can hand off only to `/opt/openshell/bin/openshell-sandbox`. +The wrapper validates the complete supplied list for count, group-ID format, range, and duplicates before it changes the container account. The wrapper adds only the validated Jetson device GIDs that onboarding detected to the existing sandbox account in `/etc/group`. It verifies the resulting membership before handoff. OpenShell then rebuilds the account's group list from the updated database, preserving access to the detected device nodes. diff --git a/scripts/jetson-device-group-bootstrap.sh b/scripts/jetson-device-group-bootstrap.sh index c814d1015a..049fe17c56 100755 --- a/scripts/jetson-device-group-bootstrap.sh +++ b/scripts/jetson-device-group-bootstrap.sh @@ -2,6 +2,9 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +# Compatibility bridge for #7610. Remove when the minimum supported OpenShell +# release natively preserves Jetson device groups across the sandbox-user handoff. + set -euo pipefail fail() { @@ -32,7 +35,9 @@ for gid in "${gids[@]}"; do [ "$gid" -le 2147483647 ] || fail "device group ID is out of range" [ -z "${seen[$gid]:-}" ] || fail "device group ID is duplicated" seen[$gid]=1 +done +for gid in "${gids[@]}"; do group_record="$(/usr/bin/getent group "$gid" || true)" if [ -z "$group_record" ]; then group_name="nemoclaw_gpu_$gid" diff --git a/test/jetson-device-group-bootstrap.test.ts b/test/jetson-device-group-bootstrap.test.ts new file mode 100644 index 0000000000..5ec6714738 --- /dev/null +++ b/test/jetson-device-group-bootstrap.test.ts @@ -0,0 +1,390 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterAll, beforeAll, describe, expect, it } from "vitest"; + +import { dockerSpawnSync } from "../src/lib/adapters/docker/exec"; + +const REPO_ROOT = path.join(import.meta.dirname, ".."); +const BOOTSTRAP_SCRIPT = path.join(REPO_ROOT, "scripts", "jetson-device-group-bootstrap.sh"); +const FIXTURE_BASE_IMAGE = + "ghcr.io/nvidia/nemoclaw/sandbox-base@sha256:16b01f6d7e0b843a72331538f3bf690b6112840064e7d8b81e361a61277bebf7"; +const SUPERVISOR = "/opt/openshell/bin/openshell-sandbox"; +const CONTAINER_TIMEOUT_MS = 20_000; + +type BootstrapRunOptions = { + environment?: Record; +}; + +type BootstrapRun = { + after: FixtureState; + before: FixtureState; + status: number | null; + stderr: string; + stdout: string; +}; + +type FixtureState = { + groupAddLog: string; + groupMap: string; + memberships: string; + supervisorArgv: Buffer | null; + usermodLog: string; +}; + +const fixtureParent = process.platform === "darwin" ? "/private/tmp" : os.tmpdir(); +const fixtureImage = `nemoclaw-jetson-bootstrap-test:${String(process.pid)}-${String(Date.now())}`; +let containerFixtureRoot = ""; + +function writeExecutable(filePath: string, source: string): void { + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, source, { mode: 0o755 }); +} + +const INITIAL_STATE: FixtureState = { + groupAddLog: "", + groupMap: "44:video\n", + memberships: "1000 44\n", + supervisorArgv: null, + usermodLog: "", +}; + +function createContainerFixture(): string { + const root = fs.mkdtempSync(path.join(fixtureParent, "nemoclaw-jetson-bootstrap-")); + const usrBin = path.join(root, "usr-bin"); + const usrSbin = path.join(root, "usr-sbin"); + const supervisorDir = path.join(root, "supervisor"); + + writeExecutable( + path.join(usrBin, "id"), + `#!/bin/sh +set -eu +if [ "\${1:-}" = "-u" ]; then + printf '%s\n' "\${TEST_ID_UID:-0}" + exit 0 +fi +if [ "\${1:-}" = "sandbox" ]; then + [ "\${TEST_SANDBOX_MISSING:-0}" != "1" ] || exit 1 + printf 'uid=1000(sandbox) gid=1000(sandbox) groups=%s\n' "$(cat /test-state/memberships)" + exit 0 +fi +if [ "\${1:-}" = "-G" ] && [ "\${2:-}" = "sandbox" ]; then + cat /test-state/memberships + exit 0 +fi +exit 2 +`, + ); + writeExecutable( + path.join(usrBin, "getent"), + `#!/bin/sh +set -eu +[ "\${1:-}" = "group" ] || exit 2 +gid="\${2:-}" +if [ "\${TEST_GETENT_MALFORMED_GID:-}" = "$gid" ]; then + printf ':x:999:\n' + exit 0 +fi +record="$(awk -F: -v gid="$gid" '$1 == gid { print; exit }' /test-state/group-map)" +[ -n "$record" ] || exit 2 +name="\${record#*:}" +printf '%s:x:%s:\n' "$name" "$gid" +`, + ); + writeExecutable( + path.join(usrSbin, "groupadd"), + `#!/bin/sh +set -eu +[ "$#" -eq 3 ] && [ "$1" = "--gid" ] +printf '%s\n' "$*" >>/test-state/groupadd.log +printf '%s:%s\n' "$2" "$3" >>/test-state/group-map +`, + ); + writeExecutable( + path.join(usrSbin, "usermod"), + `#!/bin/sh +set -eu +[ "$#" -eq 4 ] && [ "$1" = "--append" ] && [ "$2" = "--groups" ] && [ "$4" = "sandbox" ] +printf '%s\n' "$*" >>/test-state/usermod.log +[ "\${TEST_USERMOD_NOOP:-0}" != "1" ] || exit 0 +gid="$(awk -F: -v name="$3" '$2 == name { print $1; exit }' /test-state/group-map)" +[ -n "$gid" ] +memberships="$(cat /test-state/memberships)" +case " $memberships " in + *" $gid "*) ;; + *) printf '%s %s\n' "$memberships" "$gid" >/test-state/memberships ;; +esac +`, + ); + writeExecutable( + path.join(supervisorDir, "openshell-sandbox"), + `#!/bin/sh +set -eu +printf '%s\\0' "$@" >/test-state/supervisor.argv +printf 'SUPERVISOR_EXECUTED\n' +`, + ); + writeExecutable( + path.join(root, "fixture-runner"), + `#!/bin/bash +set -uo pipefail +printf '1000 44\n' >/test-state/memberships +printf '44:video\n' >/test-state/group-map +set +e +/usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh "$@" +status=$? +set -e +printf 'NEMOCLAW_TEST_STATE_BEGIN\n' +for file in groupadd.log group-map memberships supervisor.argv usermod.log; do + printf '%s=' "$file" + if [ -f "/test-state/$file" ]; then + base64 "/test-state/$file" | tr -d '\n' + fi + printf '\n' +done +printf 'NEMOCLAW_TEST_STATE_END\n' +exit "$status" +`, + ); + fs.copyFileSync(BOOTSTRAP_SCRIPT, path.join(root, "jetson-device-group-bootstrap.sh")); + fs.writeFileSync( + path.join(root, "Dockerfile"), + `FROM ${FIXTURE_BASE_IMAGE} +COPY jetson-device-group-bootstrap.sh /usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh +COPY fixture-runner /test-fixture/run +COPY supervisor/ /opt/openshell/bin/ +COPY usr-bin/id /usr/bin/id +COPY usr-bin/getent /usr/bin/getent +COPY usr-sbin/ /usr/sbin/ +`, + ); + return root; +} + +function parseFixtureState(stdout: string): { state: FixtureState; stdout: string } { + const startMarker = "NEMOCLAW_TEST_STATE_BEGIN\n"; + const endMarker = "NEMOCLAW_TEST_STATE_END\n"; + const start = stdout.indexOf(startMarker); + const end = stdout.indexOf(endMarker, start + startMarker.length); + expect(start, "fixture state start marker is missing").toBeGreaterThanOrEqual(0); + expect(end, "fixture state end marker is missing").toBeGreaterThan(start); + const encoded = new Map( + stdout + .slice(start + startMarker.length, end) + .trimEnd() + .split("\n") + .map((line) => { + const separator = line.indexOf("="); + return [line.slice(0, separator), line.slice(separator + 1)] as const; + }), + ); + const decode = (name: string): Buffer | null => { + const value = encoded.get(name); + return value ? Buffer.from(value, "base64") : null; + }; + return { + state: { + groupAddLog: decode("groupadd.log")?.toString("utf8") ?? "", + groupMap: decode("group-map")?.toString("utf8") ?? "", + memberships: decode("memberships")?.toString("utf8") ?? "", + supervisorArgv: decode("supervisor.argv"), + usermodLog: decode("usermod.log")?.toString("utf8") ?? "", + }, + stdout: stdout.slice(0, start), + }; +} + +function runBootstrap(args: readonly string[], options: BootstrapRunOptions = {}): BootstrapRun { + const dockerArgs = [ + "run", + "--rm", + "--network", + "none", + "--read-only", + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges", + "--tmpfs", + "/tmp:rw,nosuid,nodev,noexec,size=1m", + "--tmpfs", + "/test-state:rw,nosuid,nodev,noexec,size=1m", + ...Object.entries(options.environment ?? {}).flatMap(([key, value]) => [ + "--env", + `${key}=${value}`, + ]), + "--entrypoint", + "/test-fixture/run", + fixtureImage, + ...args, + ]; + const result = dockerSpawnSync(dockerArgs, { + encoding: "utf8", + killSignal: "SIGKILL", + timeout: CONTAINER_TIMEOUT_MS, + }); + expect(result.error, result.error?.message).toBeUndefined(); + const parsed = parseFixtureState(String(result.stdout)); + return { + after: parsed.state, + before: INITIAL_STATE, + status: result.status, + stderr: String(result.stderr), + stdout: parsed.stdout, + }; +} + +function expectNoMutation(run: BootstrapRun): void { + expect(run.after).toEqual(run.before); +} + +const dockerProbe = dockerSpawnSync(["info", "--format", "{{.ServerVersion}}"], { + encoding: "utf8", + killSignal: "SIGKILL", + timeout: 5_000, +}); +const suite = dockerProbe.status === 0 || process.platform === "linux" ? describe : describe.skip; + +suite("Jetson device-group bootstrap", () => { + beforeAll(() => { + expect( + dockerProbe.status, + `Docker is required for the Linux bootstrap security boundary: ${String(dockerProbe.stderr)}`, + ).toBe(0); + containerFixtureRoot = createContainerFixture(); + const build = dockerSpawnSync( + ["build", "--network", "none", "--tag", fixtureImage, containerFixtureRoot], + { + encoding: "utf8", + killSignal: "SIGKILL", + timeout: 60_000, + }, + ); + expect(build.error, build.error?.message).toBeUndefined(); + expect(build.status, `${String(build.stderr)}\n${String(build.stdout)}`).toBe(0); + }, 65_000); + + afterAll(() => { + if (dockerProbe.status === 0) { + dockerSpawnSync(["image", "rm", "--force", fixtureImage], { + encoding: "utf8", + killSignal: "SIGKILL", + timeout: 10_000, + }); + } + if (containerFixtureRoot) { + fs.rmSync(containerFixtureRoot, { force: true, recursive: true }); + } + }); + + it("adds existing and new device groups before the fixed supervisor handoff (#8099)", () => { + const run = runBootstrap([ + "--device-group-gids", + "44,110", + "--", + SUPERVISOR, + "--ready", + "value with space", + ]); + + expect(run.status, run.stderr).toBe(0); + expect(run.stdout).toBe("SUPERVISOR_EXECUTED\n"); + expect(run.after.groupAddLog).toBe("--gid 110 nemoclaw_gpu_110\n"); + expect(run.after.usermodLog).toBe( + "--append --groups video sandbox\n--append --groups nemoclaw_gpu_110 sandbox\n", + ); + expect(run.after.memberships.trim().split(/\s+/)).toEqual(["1000", "44", "110"]); + expect(run.after.supervisorArgv).toEqual(Buffer.from("--ready\0value with space\0", "utf8")); + }); + + it.each([ + { + args: ["--device-group-gids", "44,invalid", "--", SUPERVISOR], + error: "device group ID is invalid", + title: "a later invalid group ID", + }, + { + args: ["--device-group-gids", "44,44", "--", SUPERVISOR], + error: "device group ID is duplicated", + title: "a later duplicate group ID", + }, + { + args: ["--device-group-gids", "2147483648", "--", SUPERVISOR], + error: "device group ID is out of range", + title: "an out-of-range group ID", + }, + { + args: [ + "--device-group-gids", + Array.from({ length: 17 }, (_, index) => String(index + 1)).join(","), + "--", + SUPERVISOR, + ], + error: "device group count is invalid", + title: "more than 16 group IDs", + }, + { + args: ["--device-group-gids", "44", "not-a-delimiter", SUPERVISOR], + error: "supervisor delimiter is missing", + title: "an invalid supervisor delimiter", + }, + { + args: ["--device-group-gids", "44", "--", "/tmp/openshell-sandbox"], + error: "OpenShell supervisor entrypoint is invalid", + title: "a different supervisor entrypoint", + }, + ])("rejects $title before account mutation (#8099)", ({ args, error }) => { + const run = runBootstrap(args); + + expect(run.status).toBe(1); + expect(run.stderr).toContain(`Jetson device-group bootstrap: ${error}`); + expectNoMutation(run); + }); + + it("rejects a missing sandbox account before group mutation (#8099)", () => { + const run = runBootstrap(["--device-group-gids", "44", "--", SUPERVISOR], { + environment: { TEST_SANDBOX_MISSING: "1" }, + }); + + expect(run.status).toBe(1); + expect(run.stderr).toContain("Jetson device-group bootstrap: sandbox user is missing"); + expectNoMutation(run); + }); + + it("rejects a non-root caller before group mutation (#8099)", () => { + const run = runBootstrap(["--device-group-gids", "44", "--", SUPERVISOR], { + environment: { TEST_ID_UID: "1000" }, + }); + + expect(run.status).toBe(1); + expect(run.stderr).toContain("Jetson device-group bootstrap: must run as root"); + expectNoMutation(run); + }); + + it("rejects a malformed existing group before account mutation (#8099)", () => { + const run = runBootstrap(["--device-group-gids", "44", "--", SUPERVISOR], { + environment: { TEST_GETENT_MALFORMED_GID: "44" }, + }); + + expect(run.status).toBe(1); + expect(run.stderr).toContain("Jetson device-group bootstrap: device group record is invalid"); + expectNoMutation(run); + }); + + it("stops before supervisor handoff when membership verification fails (#8099)", () => { + const run = runBootstrap(["--device-group-gids", "110", "--", SUPERVISOR], { + environment: { TEST_USERMOD_NOOP: "1" }, + }); + + expect(run.status).toBe(1); + expect(run.stderr).toContain( + "Jetson device-group bootstrap: sandbox membership verification failed", + ); + expect(run.after.supervisorArgv).toBeNull(); + expect(run.after.memberships).toBe(run.before.memberships); + }); +}); From 7b70ce2cf7ec4635e2b5616a653fc10ae2eae2ba Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Thu, 6 Aug 2026 16:34:20 -0700 Subject: [PATCH 40/49] fix(onboard): fail closed on OpenRM restore Signed-off-by: Apurv Kumaria --- .../diagnostics/jetson-openrm-proof.test.ts | 49 +++++++++- .../diagnostics/jetson-openrm-proof.ts | 45 ++++++++- ...ocker-gpu-sandbox-create-lifecycle.test.ts | 91 +++++++++++++++++++ src/lib/onboard/docker-gpu-sandbox-create.ts | 11 ++- 4 files changed, 193 insertions(+), 3 deletions(-) diff --git a/src/lib/onboard/diagnostics/jetson-openrm-proof.test.ts b/src/lib/onboard/diagnostics/jetson-openrm-proof.test.ts index 5dd321f06f..ab91735151 100644 --- a/src/lib/onboard/diagnostics/jetson-openrm-proof.test.ts +++ b/src/lib/onboard/diagnostics/jetson-openrm-proof.test.ts @@ -2,9 +2,13 @@ // SPDX-License-Identifier: Apache-2.0 import fs from "node:fs"; +import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import type { DockerGpuPatchResult } from "../docker-gpu-patch-types"; -import { maybeRunJetsonOpenRmPolicyProof } from "./jetson-openrm-proof"; +import { + JetsonOpenRmPolicyRestorationError, + maybeRunJetsonOpenRmPolicyProof, +} from "./jetson-openrm-proof"; const BASE_POLICY = `Version: 2 Hash: fixture @@ -169,6 +173,49 @@ describe("Jetson OpenRM policy proof", () => { expect(appliedPolicies[1]).not.toContain("/dev/nvidia-caps/nvidia-cap2"); }); + it("preserves a candidate failure and cleans temporary files when baseline restoration fails", () => { + const candidateError = new Error("candidate probe failed"); + let temporaryDirectory = ""; + let policySetCount = 0; + const runOpenshell = vi.fn((args: string[]) => { + const policyPath = args[3] ?? ""; + temporaryDirectory = path.dirname(policyPath); + policySetCount += 1; + return { status: policySetCount === 1 ? 0 : 1 }; + }); + + let failure: unknown; + try { + maybeRunJetsonOpenRmPolicyProof({ + backend: "jetson", + enabled: true, + failure: new Error("cuInit(0)=801"), + preserveJetsonDeviceGroupMembership: true, + result: result(), + sandboxName: "alpha", + verifyDirectSandboxGpu: vi.fn(() => { + throw candidateError; + }), + deps: { + dockerRun: dockerRunForBoundaryProof(), + runCaptureOpenshell: vi.fn(() => BASE_POLICY), + runOpenshell, + }, + }); + } catch (error) { + failure = error; + } + + expect(failure).toBeInstanceOf(JetsonOpenRmPolicyRestorationError); + expect((failure as JetsonOpenRmPolicyRestorationError).candidateError).toBe(candidateError); + expect((failure as JetsonOpenRmPolicyRestorationError).restorationError).toEqual( + expect.objectContaining({ message: expect.stringContaining("baseline.yaml") }), + ); + expect((failure as JetsonOpenRmPolicyRestorationError).cleanupError).toBeNull(); + expect(temporaryDirectory).not.toBe(""); + expect(fs.existsSync(temporaryDirectory)).toBe(false); + }); + it("does nothing outside the exact Jetson cuInit 801 failure", () => { const dockerRun = vi.fn(); maybeRunJetsonOpenRmPolicyProof({ diff --git a/src/lib/onboard/diagnostics/jetson-openrm-proof.ts b/src/lib/onboard/diagnostics/jetson-openrm-proof.ts index eb97e75abf..ee1a55f757 100644 --- a/src/lib/onboard/diagnostics/jetson-openrm-proof.ts +++ b/src/lib/onboard/diagnostics/jetson-openrm-proof.ts @@ -74,6 +74,28 @@ type PolicyCandidate = { readWrite: string[]; }; +export class JetsonOpenRmPolicyRestorationError extends Error { + readonly candidateError: unknown | null; + readonly restorationError: unknown; + readonly cleanupError: unknown | null; + + constructor(options: { + candidateError: unknown | null; + restorationError: unknown; + cleanupError: unknown | null; + }) { + const detail = + options.restorationError instanceof Error + ? options.restorationError.message + : String(options.restorationError); + super(`NemoClaw could not confirm that OpenShell restored the baseline policy: ${detail}`); + this.name = "JetsonOpenRmPolicyRestorationError"; + this.candidateError = options.candidateError; + this.restorationError = options.restorationError; + this.cleanupError = options.cleanupError; + } +} + function cudaResult(value: string): string { return value.match(CUDA_RESULT_PATTERN)?.[1] ?? "missing"; } @@ -229,6 +251,7 @@ export function maybeRunJetsonOpenRmPolicyProof(options: OpenRmProofOptions): vo fs.writeFileSync(baselinePath, baselinePolicy, { encoding: "utf8", mode: 0o600 }); const candidateResults = new Map(); + let candidateFailure: { readonly error: unknown } | null = null; try { for (const candidate of candidates) { const candidatePath = path.join(temporaryDirectory, `${candidate.name}.yaml`); @@ -242,10 +265,30 @@ export function maybeRunJetsonOpenRmPolicyProof(options: OpenRmProofOptions): vo proofCudaResult(options.verifyDirectSandboxGpu(options.sandboxName)), ); } - } finally { + } catch (error) { + candidateFailure = { error }; + } + let restorationFailure: { readonly error: unknown } | null = null; + try { setPolicy(options.sandboxName, baselinePath, runOpenshell); + } catch (error) { + restorationFailure = { error }; + } + let cleanupFailure: { readonly error: unknown } | null = null; + try { fs.rmSync(temporaryDirectory, { recursive: true, force: true }); + } catch (error) { + cleanupFailure = { error }; + } + if (restorationFailure !== null) { + throw new JetsonOpenRmPolicyRestorationError({ + candidateError: candidateFailure?.error ?? null, + restorationError: restorationFailure.error, + cleanupError: cleanupFailure?.error ?? null, + }); } + if (candidateFailure !== null) throw candidateFailure.error; + if (cleanupFailure !== null) throw cleanupFailure.error; const deviceResult = candidateResults.get("devices") ?? "not-tested"; const sysfsResult = candidateResults.get("sysfs") ?? "not-tested"; diff --git a/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts b/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts index 0296b58835..4b04239788 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts @@ -1,11 +1,27 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import fs from "node:fs"; +import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { JetsonOpenRmPolicyRestorationError } from "./diagnostics/jetson-openrm-proof"; import type { DockerGpuPatchFailureContext, DockerGpuPatchResult } from "./docker-gpu-patch"; import { createDockerGpuSandboxCreatePatch } from "./docker-gpu-sandbox-create"; +const OPENRM_BASE_POLICY = `Version: 2 +Hash: fixture +--- +version: 1 +filesystem_policy: + read_only: + - /opt/nvidia + - /sys + read_write: + - /dev/nvmap +network_policies: {} +`; + function deferredCreateResult(): DockerGpuPatchResult { return { applied: true, @@ -415,4 +431,79 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { expect.stringContaining("pre-patch container was not restored"), ); }); + + it("blocks lifecycle rollback when OpenShell cannot confirm baseline policy restoration", async () => { + vi.stubEnv("NEMOCLAW_DIAGNOSE_JETSON_OPENRM_POLICY", "1"); + let policySetCount = 0; + let temporaryDirectory = ""; + const deps = { + ...makeDeps(), + runCaptureOpenshell: vi.fn((args: string[]) => + args[0] === "policy" ? OPENRM_BASE_POLICY : "", + ), + runOpenshell: vi.fn((args: string[]) => { + const policyPath = args[3] ?? ""; + temporaryDirectory = path.dirname(policyPath); + policySetCount += 1; + return { status: policySetCount === 1 ? 0 : 1 }; + }), + dockerRun: vi.fn((args: readonly string[]) => + args.includes("0") + ? { + status: 0, + stdout: "/dev/nvidia-caps/nvidia-cap2\n/dev/nvmap", + stderr: "", + } + : { status: 0, stdout: "cuInit(0)=0", stderr: "" }, + ), + }; + const result = { + ...deferredCreateResult(), + mode: { + kind: "nvidia-runtime" as const, + label: "--runtime nvidia", + device: "all", + args: ["--runtime", "nvidia"], + }, + }; + const finalizeBackup = vi.fn(() => ({ backupRemoved: false, rolledBack: true })); + const patch = createDockerGpuSandboxCreatePatch({ + route: "compatibility", + sandboxName: "alpha", + timeoutSecs: 60, + backend: "jetson", + preserveJetsonDeviceGroupMembership: true, + deps, + overrides: { + findContainerIds: vi.fn(() => ["existing-container"]), + recreatePatch: vi.fn(() => result), + waitForSupervisor: vi.fn(() => true), + finalizeBackup, + }, + }); + const verifyDirectSandboxGpu = vi + .fn() + .mockImplementationOnce(() => { + throw new Error("cuInit(0)=801"); + }) + .mockReturnValue({ + status: "verified" as const, + cudaVerified: true, + at: "2026-08-06T00:00:00.000Z", + }); + + patch.maybeApplyDuringCreate(); + patch.waitForSupervisorReconnectIfNeeded(); + + await expect(patch.verifyGpuOrExit(verifyDirectSandboxGpu)).rejects.toBeInstanceOf( + JetsonOpenRmPolicyRestorationError, + ); + await expect(patch.rollbackManagedStartupAfterCreateFailure()).rejects.toBeInstanceOf( + JetsonOpenRmPolicyRestorationError, + ); + + expect(finalizeBackup).not.toHaveBeenCalled(); + expect(temporaryDirectory).not.toBe(""); + expect(fs.existsSync(temporaryDirectory)).toBe(false); + }); }); diff --git a/src/lib/onboard/docker-gpu-sandbox-create.ts b/src/lib/onboard/docker-gpu-sandbox-create.ts index fd246adb54..fcc8a0c771 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create.ts @@ -3,7 +3,10 @@ import { getSandboxFailurePhase } from "../state/gateway"; import type { SandboxGpuProofResult } from "../state/registry"; -import { maybeRunJetsonOpenRmPolicyProof } from "./diagnostics/jetson-openrm-proof"; +import { + JetsonOpenRmPolicyRestorationError, + maybeRunJetsonOpenRmPolicyProof, +} from "./diagnostics/jetson-openrm-proof"; import { getDockerGpuSupervisorReconnectTimeoutSecs, printDockerGpuPatchFailureAndExit, @@ -162,6 +165,7 @@ export function createDockerGpuSandboxCreatePatch( let cutoverFinalization: Promise | null = null; let cutoverFinalizationOutcome: "commit" | "rollback" | null = null; let cutoverFinalizationFailure: Error | null = null; + let policyRestorationFailure: JetsonOpenRmPolicyRestorationError | null = null; const findContainerIds = options.overrides?.findContainerIds ?? findOpenShellDockerSandboxContainerIds; @@ -316,6 +320,7 @@ export function createDockerGpuSandboxCreatePatch( }, async rollbackManagedStartupAfterCreateFailure() { + if (policyRestorationFailure) throw policyRestorationFailure; const rollbackError = await rollbackAfterFailure(); if (!rollbackError) return; onPatchFailureExit(options.sandboxName, rollbackError, { @@ -572,6 +577,10 @@ export function createDockerGpuSandboxCreatePatch( console.error( ` OpenRM A/B inconclusive: ${diagnosticError instanceof Error ? diagnosticError.message : String(diagnosticError)}`, ); + if (diagnosticError instanceof JetsonOpenRmPolicyRestorationError) { + policyRestorationFailure = diagnosticError; + throw diagnosticError; + } } printDockerGpuProofFailure(sandboxName, failure, selectedMode(), { runCaptureOpenshell: options.deps.runCaptureOpenshell, From 8fde81f0f31673576e1e3195e79faac1eab6a407 Mon Sep 17 00:00:00 2001 From: San Dang Date: Fri, 7 Aug 2026 12:33:47 +0700 Subject: [PATCH 41/49] test(onboard): make Jetson policy proof standalone Signed-off-by: San Dang --- .../prove-jetson-openrm-policy-boundary.sh | 26 +++- .../jetson-openrm-standalone.test.ts | 94 +++++++++++++ .../diagnostics/jetson-openrm-standalone.ts | 129 ++++++++++++++++++ 3 files changed, 242 insertions(+), 7 deletions(-) create mode 100644 src/lib/onboard/diagnostics/jetson-openrm-standalone.test.ts create mode 100644 src/lib/onboard/diagnostics/jetson-openrm-standalone.ts diff --git a/scripts/prove-jetson-openrm-policy-boundary.sh b/scripts/prove-jetson-openrm-policy-boundary.sh index 023ea1e106..34af804903 100755 --- a/scripts/prove-jetson-openrm-policy-boundary.sh +++ b/scripts/prove-jetson-openrm-policy-boundary.sh @@ -10,13 +10,25 @@ if [[ ! "$sandbox_name" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]*$ ]]; then exit 2 fi -printf 'The proof is armed for sandbox %s. Complete the normal onboarding prompts.\n' \ - "$sandbox_name" -printf 'The sandbox will be rebuilt with the current checkout before the proof runs.\n' -printf 'When the live replacement returns cuInit(0)=801, NemoClaw will run the device/sysfs policy matrix before its normal rollback.\n' +printf 'Running the standalone Jetson OpenRM policy proof for sandbox %s.\n' "$sandbox_name" +printf 'This bypasses onboarding and its resume checkpoints.\n' +printf 'The current container is preserved as a rollback backup before the production recreation and policy matrix run.\n' npm run build:cli +exec node - "$sandbox_name" <<'NODE' +const { + createDockerGpuDiagnosticRedactor, +} = require("./dist/lib/onboard/docker-gpu-diagnostic-redaction"); +const { + runStandaloneJetsonOpenRmPolicyProof, +} = require("./dist/lib/onboard/diagnostics/jetson-openrm-standalone"); -export NEMOCLAW_SANDBOX_NAME="$sandbox_name" -export NEMOCLAW_DIAGNOSE_JETSON_OPENRM_POLICY=1 -exec node bin/nemoclaw.js onboard --resume --recreate-sandbox +runStandaloneJetsonOpenRmPolicyProof(process.argv[2]).catch((error) => { + const message = error instanceof Error ? error.message : String(error); + const redacted = createDockerGpuDiagnosticRedactor() + .redactText(message) + .replace(/[\r\n]+/gu, " "); + console.error(`Error: ${redacted}`); + process.exitCode = 1; +}); +NODE diff --git a/src/lib/onboard/diagnostics/jetson-openrm-standalone.test.ts b/src/lib/onboard/diagnostics/jetson-openrm-standalone.test.ts new file mode 100644 index 0000000000..4a0187a114 --- /dev/null +++ b/src/lib/onboard/diagnostics/jetson-openrm-standalone.test.ts @@ -0,0 +1,94 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; +import { runStandaloneJetsonOpenRmPolicyProof } from "./jetson-openrm-standalone"; + +function verifierFactory() { + return vi.fn(() => vi.fn()); +} + +describe("standalone Jetson OpenRM policy proof", () => { + it("runs the production recreation boundary and restores it after cuInit 801 (#7610)", async () => { + const rollback = vi.fn(async () => undefined); + const createPatch = vi.fn(() => ({ + ensureApplied: vi.fn(async () => undefined), + waitForSupervisorReconnectIfNeeded: vi.fn(), + verifyGpuOrExit: vi.fn(async () => { + throw new Error("Sandbox GPU proof returned failed status (cuInit(0)=801)"); + }), + rollbackManagedStartupAfterCreateFailure: rollback, + })); + + await runStandaloneJetsonOpenRmPolicyProof("alpha", { + createPatch, + createVerifier: verifierFactory(), + runCaptureOpenshell: vi.fn(), + runOpenshell: vi.fn(), + }); + + expect(createPatch).toHaveBeenCalledWith( + expect.objectContaining({ + route: "compatibility", + sandboxName: "alpha", + backend: "jetson", + preserveJetsonDeviceGroupMembership: true, + }), + ); + expect(rollback).toHaveBeenCalledOnce(); + }); + + it("restores the original container after an unexpected proof failure (#7610)", async () => { + const rollback = vi.fn(async () => undefined); + const createPatch = vi.fn(() => ({ + ensureApplied: vi.fn(async () => undefined), + waitForSupervisorReconnectIfNeeded: vi.fn(), + verifyGpuOrExit: vi.fn(async () => { + throw new Error("cuInit(0)=100"); + }), + rollbackManagedStartupAfterCreateFailure: rollback, + })); + + await expect( + runStandaloneJetsonOpenRmPolicyProof("alpha", { + createPatch, + createVerifier: verifierFactory(), + runCaptureOpenshell: vi.fn(), + runOpenshell: vi.fn(), + }), + ).rejects.toThrow("cuInit(0)=100"); + expect(rollback).toHaveBeenCalledOnce(); + }); + + it("does not accept cuInit 801 when the production rollback failed (#7610)", async () => { + const rollbackError = new Error("pre-patch container was not restored"); + const proofError = Object.assign(new Error("cuInit(0)=801"), { + managedBootstrapRollbackError: rollbackError, + }); + const createPatch = vi.fn(() => ({ + ensureApplied: vi.fn(async () => undefined), + waitForSupervisorReconnectIfNeeded: vi.fn(), + verifyGpuOrExit: vi.fn(async () => { + throw proofError; + }), + rollbackManagedStartupAfterCreateFailure: vi.fn(async () => undefined), + })); + + await expect( + runStandaloneJetsonOpenRmPolicyProof("alpha", { + createPatch, + createVerifier: verifierFactory(), + runCaptureOpenshell: vi.fn(), + runOpenshell: vi.fn(), + }), + ).rejects.toThrow("cuInit(0)=801"); + }); + + it("rejects an invalid sandbox name before creating a patch (#7610)", async () => { + const createPatch = vi.fn(); + await expect( + runStandaloneJetsonOpenRmPolicyProof("alpha;docker ps", { createPatch }), + ).rejects.toThrow("Invalid sandbox name"); + expect(createPatch).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/onboard/diagnostics/jetson-openrm-standalone.ts b/src/lib/onboard/diagnostics/jetson-openrm-standalone.ts new file mode 100644 index 0000000000..ac1b1d65bf --- /dev/null +++ b/src/lib/onboard/diagnostics/jetson-openrm-standalone.ts @@ -0,0 +1,129 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { SandboxGpuProofResult } from "../../state/registry"; +import { createDockerGpuDiagnosticRedactor } from "../docker-gpu-diagnostic-redaction"; +import type { DockerGpuPatchDeps } from "../docker-gpu-patch-types"; +import { createDockerGpuSandboxCreatePatch } from "../docker-gpu-sandbox-create"; +import { createOpenshellCliHelpers } from "../openshell-cli"; +import { createDirectSandboxGpuVerifier } from "../sandbox-gpu-preflight"; + +const SANDBOX_NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_.-]*$/u; +const CUINIT_801_PATTERN = /cuInit\(0\)=801/u; +const RECREATE_TIMEOUT_SECS = 180; + +type SandboxPatch = Pick< + ReturnType, + | "ensureApplied" + | "rollbackManagedStartupAfterCreateFailure" + | "verifyGpuOrExit" + | "waitForSupervisorReconnectIfNeeded" +>; + +type StandaloneProofDeps = { + createPatch?: (options: Parameters[0]) => SandboxPatch; + createVerifier?: typeof createDirectSandboxGpuVerifier; + runCaptureOpenshell?: NonNullable; + runOpenshell?: NonNullable; +}; + +function compactText(value: string): string { + return String(value).replace(/\s+/gu, " ").trim(); +} + +function liveOpenShellRunners(): Pick< + ReturnType, + "runCaptureOpenshell" | "runOpenshell" +> { + let cachedBinary: string | null = null; + const helpers = createOpenshellCliHelpers({ + getCachedBinary: () => cachedBinary, + setCachedBinary: (binary) => { + cachedBinary = binary; + }, + getGatewayPort: () => 0, + getDockerDriverGatewayEndpoint: () => "", + }); + return { + runCaptureOpenshell: helpers.runCaptureOpenshell, + runOpenshell: helpers.runOpenshell, + }; +} + +/** + * Run the production Jetson Docker recreation, OpenShell CUDA proof, and + * OpenRM policy matrix without entering the onboarding state machine. The + * production rollback path restores the original container on every exit. + */ +export async function runStandaloneJetsonOpenRmPolicyProof( + sandboxName: string, + deps: StandaloneProofDeps = {}, +): Promise { + if (!SANDBOX_NAME_PATTERN.test(sandboxName)) { + throw new Error(`Invalid sandbox name: ${sandboxName}`); + } + + const liveRunners = deps.runOpenshell && deps.runCaptureOpenshell ? null : liveOpenShellRunners(); + const run = deps.runOpenshell ?? liveRunners?.runOpenshell; + const capture = deps.runCaptureOpenshell ?? liveRunners?.runCaptureOpenshell; + if (!run || !capture) throw new Error("OpenShell command runners are unavailable."); + const createVerifier = deps.createVerifier ?? createDirectSandboxGpuVerifier; + const redactor = createDockerGpuDiagnosticRedactor(); + const verifyGpu = createVerifier({ + runOpenshell: run, + compactText, + redact: (value) => redactor.redactText(String(value ?? "")), + detectNvidiaPlatform: () => "jetson", + }); + const createPatch = deps.createPatch ?? createDockerGpuSandboxCreatePatch; + const patch = createPatch({ + route: "compatibility", + sandboxName, + timeoutSecs: RECREATE_TIMEOUT_SECS, + backend: "jetson", + preserveJetsonDeviceGroupMembership: true, + deps: { + runCaptureOpenshell: capture, + runOpenshell: run, + }, + }); + + let proof: SandboxGpuProofResult | null = null; + let expectedBoundaryFailure = false; + let rollbackFailure: unknown = null; + try { + await patch.ensureApplied(); + patch.waitForSupervisorReconnectIfNeeded(); + try { + proof = await patch.verifyGpuOrExit(verifyGpu); + } catch (error) { + rollbackFailure = + error && typeof error === "object" + ? (error as { managedBootstrapRollbackError?: unknown }).managedBootstrapRollbackError + : null; + if (rollbackFailure) throw error; + const message = error instanceof Error ? error.message : String(error); + if (!CUINIT_801_PATTERN.test(message)) throw error; + expectedBoundaryFailure = true; + } + } finally { + await patch.rollbackManagedStartupAfterCreateFailure(); + if (!rollbackFailure) { + console.log(" ✓ Original sandbox container restored after the standalone proof."); + } + } + + if (expectedBoundaryFailure) { + console.log( + " ✓ Reproduced the cuInit(0)=801 OpenShell boundary; use the policy matrix above as the result.", + ); + return; + } + if (proof?.status === "verified" && proof.cudaVerified) { + console.log( + " ✓ CUDA already passes through OpenShell; the cuInit(0)=801 boundary did not reproduce.", + ); + return; + } + throw new Error("The standalone run did not execute a conclusive CUDA proof."); +} From 72dd2f3855350b905d4da4182fb05d12e338704c Mon Sep 17 00:00:00 2001 From: San Dang Date: Fri, 7 Aug 2026 13:11:13 +0700 Subject: [PATCH 42/49] test(onboard): enable standalone OpenRM matrix Signed-off-by: San Dang --- .../onboard/diagnostics/jetson-openrm-standalone.test.ts | 6 ++++++ src/lib/onboard/diagnostics/jetson-openrm-standalone.ts | 8 ++++++++ 2 files changed, 14 insertions(+) diff --git a/src/lib/onboard/diagnostics/jetson-openrm-standalone.test.ts b/src/lib/onboard/diagnostics/jetson-openrm-standalone.test.ts index 4a0187a114..c2fa5950f4 100644 --- a/src/lib/onboard/diagnostics/jetson-openrm-standalone.test.ts +++ b/src/lib/onboard/diagnostics/jetson-openrm-standalone.test.ts @@ -10,11 +10,15 @@ function verifierFactory() { describe("standalone Jetson OpenRM policy proof", () => { it("runs the production recreation boundary and restores it after cuInit 801 (#7610)", async () => { + const policyProofEnv = "NEMOCLAW_DIAGNOSE_JETSON_OPENRM_POLICY"; + const previousPolicyProofSetting = process.env[policyProofEnv]; + let observedPolicyProofSetting: string | undefined; const rollback = vi.fn(async () => undefined); const createPatch = vi.fn(() => ({ ensureApplied: vi.fn(async () => undefined), waitForSupervisorReconnectIfNeeded: vi.fn(), verifyGpuOrExit: vi.fn(async () => { + observedPolicyProofSetting = process.env[policyProofEnv]; throw new Error("Sandbox GPU proof returned failed status (cuInit(0)=801)"); }), rollbackManagedStartupAfterCreateFailure: rollback, @@ -35,6 +39,8 @@ describe("standalone Jetson OpenRM policy proof", () => { preserveJetsonDeviceGroupMembership: true, }), ); + expect(observedPolicyProofSetting).toBe("1"); + expect(process.env[policyProofEnv]).toBe(previousPolicyProofSetting); expect(rollback).toHaveBeenCalledOnce(); }); diff --git a/src/lib/onboard/diagnostics/jetson-openrm-standalone.ts b/src/lib/onboard/diagnostics/jetson-openrm-standalone.ts index ac1b1d65bf..07d96b6f60 100644 --- a/src/lib/onboard/diagnostics/jetson-openrm-standalone.ts +++ b/src/lib/onboard/diagnostics/jetson-openrm-standalone.ts @@ -11,6 +11,7 @@ import { createDirectSandboxGpuVerifier } from "../sandbox-gpu-preflight"; const SANDBOX_NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_.-]*$/u; const CUINIT_801_PATTERN = /cuInit\(0\)=801/u; const RECREATE_TIMEOUT_SECS = 180; +const OPENRM_POLICY_PROOF_ENV = "NEMOCLAW_DIAGNOSE_JETSON_OPENRM_POLICY"; type SandboxPatch = Pick< ReturnType, @@ -91,9 +92,11 @@ export async function runStandaloneJetsonOpenRmPolicyProof( let proof: SandboxGpuProofResult | null = null; let expectedBoundaryFailure = false; let rollbackFailure: unknown = null; + const previousPolicyProofSetting = process.env[OPENRM_POLICY_PROOF_ENV]; try { await patch.ensureApplied(); patch.waitForSupervisorReconnectIfNeeded(); + process.env[OPENRM_POLICY_PROOF_ENV] = "1"; try { proof = await patch.verifyGpuOrExit(verifyGpu); } catch (error) { @@ -107,6 +110,11 @@ export async function runStandaloneJetsonOpenRmPolicyProof( expectedBoundaryFailure = true; } } finally { + if (previousPolicyProofSetting === undefined) { + delete process.env[OPENRM_POLICY_PROOF_ENV]; + } else { + process.env[OPENRM_POLICY_PROOF_ENV] = previousPolicyProofSetting; + } await patch.rollbackManagedStartupAfterCreateFailure(); if (!rollbackFailure) { console.log(" ✓ Original sandbox container restored after the standalone proof."); From 21f21eee75b2e8722dfffa21baca38a737dfddd2 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Fri, 7 Aug 2026 00:20:48 -0700 Subject: [PATCH 43/49] test(onboard): cover invalid group database Signed-off-by: Apurv Kumaria --- test/jetson-device-group-bootstrap.test.ts | 57 ++++++++++++++++------ 1 file changed, 41 insertions(+), 16 deletions(-) diff --git a/test/jetson-device-group-bootstrap.test.ts b/test/jetson-device-group-bootstrap.test.ts index 5ec6714738..11accf58c6 100644 --- a/test/jetson-device-group-bootstrap.test.ts +++ b/test/jetson-device-group-bootstrap.test.ts @@ -18,6 +18,7 @@ const CONTAINER_TIMEOUT_MS = 20_000; type BootstrapRunOptions = { environment?: Record; + groupDatabase?: "regular" | "symlink"; }; type BootstrapRun = { @@ -37,8 +38,17 @@ type FixtureState = { }; const fixtureParent = process.platform === "darwin" ? "/private/tmp" : os.tmpdir(); -const fixtureImage = `nemoclaw-jetson-bootstrap-test:${String(process.pid)}-${String(Date.now())}`; -let containerFixtureRoot = ""; +const fixtureId = `${String(process.pid)}-${String(Date.now())}`; +const fixtureImage = `nemoclaw-jetson-bootstrap-test:${fixtureId}`; +let containerFixtureRoot = path.join( + fixtureParent, + `nemoclaw-jetson-bootstrap-not-created-${fixtureId}`, +); + +const GROUP_DATABASE_DOCKER_ARGS = { + regular: [], + symlink: ["--tmpfs", "/etc:rw,nosuid,nodev,noexec,size=1m"], +} as const satisfies Record, readonly string[]>; function writeExecutable(filePath: string, source: string): void { fs.mkdirSync(path.dirname(filePath), { recursive: true }); @@ -134,6 +144,11 @@ printf 'SUPERVISOR_EXECUTED\n' set -uo pipefail printf '1000 44\n' >/test-state/memberships printf '44:video\n' >/test-state/group-map +case "\${TEST_GROUP_DATABASE:-regular}" in + regular) ;; + symlink) /bin/ln -s /tmp/nemoclaw-missing-group /etc/group ;; + *) exit 2 ;; +esac set +e /usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh "$@" status=$? @@ -199,6 +214,7 @@ function parseFixtureState(stdout: string): { state: FixtureState; stdout: strin } function runBootstrap(args: readonly string[], options: BootstrapRunOptions = {}): BootstrapRun { + const groupDatabase = options.groupDatabase ?? "regular"; const dockerArgs = [ "run", "--rm", @@ -213,10 +229,11 @@ function runBootstrap(args: readonly string[], options: BootstrapRunOptions = {} "/tmp:rw,nosuid,nodev,noexec,size=1m", "--tmpfs", "/test-state:rw,nosuid,nodev,noexec,size=1m", - ...Object.entries(options.environment ?? {}).flatMap(([key, value]) => [ - "--env", - `${key}=${value}`, - ]), + ...GROUP_DATABASE_DOCKER_ARGS[groupDatabase], + ...Object.entries({ + ...options.environment, + TEST_GROUP_DATABASE: groupDatabase, + }).flatMap(([key, value]) => ["--env", `${key}=${value}`]), "--entrypoint", "/test-fixture/run", fixtureImage, @@ -269,16 +286,12 @@ suite("Jetson device-group bootstrap", () => { }, 65_000); afterAll(() => { - if (dockerProbe.status === 0) { - dockerSpawnSync(["image", "rm", "--force", fixtureImage], { - encoding: "utf8", - killSignal: "SIGKILL", - timeout: 10_000, - }); - } - if (containerFixtureRoot) { - fs.rmSync(containerFixtureRoot, { force: true, recursive: true }); - } + dockerSpawnSync(["image", "rm", "--force", fixtureImage], { + encoding: "utf8", + killSignal: "SIGKILL", + timeout: 10_000, + }); + fs.rmSync(containerFixtureRoot, { force: true, recursive: true }); }); it("adds existing and new device groups before the fixed supervisor handoff (#8099)", () => { @@ -375,6 +388,18 @@ suite("Jetson device-group bootstrap", () => { expectNoMutation(run); }); + it("rejects a symlinked group database before account mutation (#8099)", () => { + const run = runBootstrap(["--device-group-gids", "44", "--", SUPERVISOR], { + groupDatabase: "symlink", + }); + + expect(run.status).toBe(1); + expect(run.stderr).toContain( + "Jetson device-group bootstrap: container group database is invalid", + ); + expectNoMutation(run); + }); + it("stops before supervisor handoff when membership verification fails (#8099)", () => { const run = runBootstrap(["--device-group-gids", "110", "--", SUPERVISOR], { environment: { TEST_USERMOD_NOOP: "1" }, From ceff582b542753ae8a5552dd3ed738a5665c2070 Mon Sep 17 00:00:00 2001 From: San Dang Date: Fri, 7 Aug 2026 16:14:22 +0700 Subject: [PATCH 44/49] test(onboard): isolate Jetson process confinement Signed-off-by: San Dang --- .../jetson-openrm-process-proof.test.ts | 66 ++++ .../jetson-openrm-process-proof.ts | 325 ++++++++++++++++++ .../diagnostics/jetson-openrm-proof.ts | 2 + 3 files changed, 393 insertions(+) create mode 100644 src/lib/onboard/diagnostics/jetson-openrm-process-proof.test.ts create mode 100644 src/lib/onboard/diagnostics/jetson-openrm-process-proof.ts diff --git a/src/lib/onboard/diagnostics/jetson-openrm-process-proof.test.ts b/src/lib/onboard/diagnostics/jetson-openrm-process-proof.test.ts new file mode 100644 index 0000000000..02969321c3 --- /dev/null +++ b/src/lib/onboard/diagnostics/jetson-openrm-process-proof.test.ts @@ -0,0 +1,66 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; +import { runJetsonOpenRmProcessProof } from "./jetson-openrm-process-proof"; + +function modeFromArgs(args: readonly string[]): string { + return args.at(-1) ?? ""; +} + +describe("Jetson OpenRM process proof", () => { + it("isolates a non-seccomp process control before syscall probes (#7610)", () => { + const dockerRun = vi.fn((args: readonly string[]) => ({ + status: 0, + stdout: `cuInit(0)=${modeFromArgs(args) === "nondumpable" ? "801" : "0"}`, + })); + const log = vi.spyOn(console, "log").mockImplementation(() => undefined); + + runJetsonOpenRmProcessProof("a".repeat(64), dockerRun); + + expect(dockerRun).toHaveBeenCalledTimes(7); + expect(log).toHaveBeenCalledWith(expect.stringContaining("nondumpable")); + }); + + it("isolates one OpenShell blocked syscall after the fixed process cases pass (#7610)", () => { + const dockerRun = vi.fn((args: readonly string[]) => ({ + status: 0, + stdout: `cuInit(0)=${["openshell-seccomp", "deny-process_vm_readv"].includes(modeFromArgs(args)) ? "801" : "0"}`, + })); + const log = vi.spyOn(console, "log").mockImplementation(() => undefined); + + runJetsonOpenRmProcessProof("b".repeat(64), dockerRun); + + expect(log).toHaveBeenCalledWith( + expect.stringContaining("CUDA-required rule(s): process_vm_readv"), + ); + expect(dockerRun.mock.calls.some(([args]) => modeFromArgs(args) === "deny-clone3")).toBe(true); + expect( + dockerRun.mock.calls.some(([args]) => modeFromArgs(args) === "deny-socket-netlink-non-route"), + ).toBe(true); + }); + + it("isolates an interaction with the complete OpenShell seccomp filter (#7610)", () => { + const dockerRun = vi.fn((args: readonly string[]) => ({ + status: 0, + stdout: `cuInit(0)=${modeFromArgs(args).includes("nondumpable-plus") || modeFromArgs(args) === "hardening-plus-openshell-seccomp" ? "801" : "0"}`, + })); + const log = vi.spyOn(console, "log").mockImplementation(() => undefined); + + runJetsonOpenRmProcessProof("c".repeat(64), dockerRun); + + expect(log).toHaveBeenCalledWith( + expect.stringContaining("seccomp is combined with nondumpable"), + ); + }); + + it("reports an invalid direct-Docker baseline without testing syscall denials (#7610)", () => { + const dockerRun = vi.fn(() => ({ status: 1, stdout: "cuInit(0)=801" })); + const error = vi.spyOn(console, "error").mockImplementation(() => undefined); + + runJetsonOpenRmProcessProof("d".repeat(64), dockerRun); + + expect(dockerRun).toHaveBeenCalledTimes(7); + expect(error).toHaveBeenCalledWith(expect.stringContaining("baseline did not pass")); + }); +}); diff --git a/src/lib/onboard/diagnostics/jetson-openrm-process-proof.ts b/src/lib/onboard/diagnostics/jetson-openrm-process-proof.ts new file mode 100644 index 0000000000..5b6b478576 --- /dev/null +++ b/src/lib/onboard/diagnostics/jetson-openrm-process-proof.ts @@ -0,0 +1,325 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { DockerGpuPatchDeps } from "../docker-gpu-patch-types"; + +const PROOF_TIMEOUT_MS = 30_000; +const CUDA_RESULT_PATTERN = /cuInit\(0\)=(-?\d+)/u; + +// OpenShell v0.0.85 blocks these syscalls unconditionally in its inherited +// supervisor prelude and runtime seccomp filters on aarch64. clone3 uses ENOSYS +// so libc can fall back to clone; the other denials use EPERM. +const OPEN_SHELL_BLOCKED_SYSCALLS = [ + ["umount2", 39, 1], + ["mount", 40, 1], + ["pivot_root", 41, 1], + ["kexec_load", 104, 1], + ["init_module", 105, 1], + ["delete_module", 106, 1], + ["ptrace", 117, 1], + ["perf_event_open", 241, 1], + ["setns", 268, 1], + ["process_vm_readv", 270, 1], + ["process_vm_writev", 271, 1], + ["finit_module", 273, 1], + ["memfd_create", 279, 1], + ["bpf", 280, 1], + ["userfaultfd", 282, 1], + ["kexec_file_load", 294, 1], + ["open_tree", 428, 1], + ["move_mount", 429, 1], + ["fsopen", 430, 1], + ["fsconfig", 431, 1], + ["fsmount", 432, 1], + ["fspick", 433, 1], + ["pidfd_open", 434, 1], + ["clone3", 435, 38], + ["pidfd_getfd", 438, 1], + ["pidfd_send_signal", 424, 1], + ["io_uring_setup", 425, 1], +] as const; + +const OPEN_SHELL_CONDITIONAL_RULES = [ + "socket-af-packet", + "socket-af-bluetooth", + "socket-af-vsock", + "socket-netlink-non-route", + "execveat-empty-path", + "unshare-newuser", + "clone-newuser", + "seccomp-set-filter", +] as const; + +const PROCESS_BOUNDARY_PROBE = String.raw` +import ctypes +import errno +import glob +import os +import pwd +import resource +import stat +import sys + +mode = sys.argv[1] +blocked = { +${OPEN_SHELL_BLOCKED_SYSCALLS.map(([name, number, error]) => ` ${JSON.stringify(name)}: (${String(number)}, ${String(error)}),`).join("\n")} +} +conditional = { + "socket-af-packet": ("eq", 198, 0, 17), + "socket-af-bluetooth": ("eq", 198, 0, 31), + "socket-af-vsock": ("eq", 198, 0, 40), + "socket-netlink-non-route": ("netlink", 198, 0, 16), + "execveat-empty-path": ("masked", 281, 4, 0x1000), + "unshare-newuser": ("masked", 97, 0, 0x10000000), + "clone-newuser": ("masked", 220, 0, 0x10000000), + "seccomp-set-filter": ("eq", 277, 0, 1), +} + +libc = ctypes.CDLL(None, use_errno=True) + +def prctl(option, arg2=0, arg3=0, arg4=0, arg5=0, allow_einval=False): + ctypes.set_errno(0) + rc = libc.prctl( + ctypes.c_int(option), + ctypes.c_ulong(arg2), + ctypes.c_ulong(arg3), + ctypes.c_ulong(arg4), + ctypes.c_ulong(arg5), + ) + error = ctypes.get_errno() + if rc != 0 and not (allow_einval and error == errno.EINVAL): + raise OSError(error, os.strerror(error)) + return rc + +def prctl_get(option): + ctypes.set_errno(0) + rc = libc.prctl(ctypes.c_int(option), 0, 0, 0, 0) + if rc < 0: + error = ctypes.get_errno() + raise OSError(error, os.strerror(error)) + return rc + +def drop_bounding_set(): + for capability in range(64): + prctl(24, capability, allow_einval=True) + +def drop_to_sandbox(): + account = pwd.getpwnam("sandbox") + groups = {account.pw_gid} + for pattern in ("/dev/nvmap", "/dev/nvhost-*", "/dev/dri/renderD*", "/dev/dri/card*"): + for device in glob.glob(pattern): + try: + device_stat = os.stat(device) + except OSError: + continue + if stat.S_ISCHR(device_stat.st_mode) and device_stat.st_gid > 0: + groups.add(device_stat.st_gid) + os.setgroups(sorted(groups)) + os.setgid(account.pw_gid) + os.setuid(account.pw_uid) + +class SockFilter(ctypes.Structure): + _fields_ = [ + ("code", ctypes.c_ushort), + ("jt", ctypes.c_ubyte), + ("jf", ctypes.c_ubyte), + ("k", ctypes.c_uint), + ] + +class SockFprog(ctypes.Structure): + _fields_ = [ + ("length", ctypes.c_ushort), + ("filters", ctypes.POINTER(SockFilter)), + ] + +def install_filter(denials, conditional_denials): + instructions = [(0x20, 0, 0, 0)] + for syscall_number, syscall_errno in denials: + instructions.append((0x15, 0, 1, syscall_number)) + instructions.append((0x06, 0, 0, 0x00050000 | syscall_errno)) + for kind, syscall_number, arg_index, value in conditional_denials: + instructions.append((0x20, 0, 0, 0)) + if kind == "masked": + instructions.append((0x15, 0, 4, syscall_number)) + instructions.append((0x20, 0, 0, 16 + (arg_index * 8))) + instructions.append((0x54, 0, 0, value)) + instructions.append((0x15, 0, 1, value)) + elif kind == "netlink": + instructions.append((0x15, 0, 5, syscall_number)) + instructions.append((0x20, 0, 0, 16)) + instructions.append((0x15, 0, 3, value)) + instructions.append((0x20, 0, 0, 32)) + instructions.append((0x15, 1, 0, 0)) + else: + instructions.append((0x15, 0, 3, syscall_number)) + instructions.append((0x20, 0, 0, 16 + (arg_index * 8))) + instructions.append((0x15, 0, 1, value)) + instructions.append((0x06, 0, 0, 0x00050001)) + instructions.append((0x06, 0, 0, 0x7fff0000)) + filters = (SockFilter * len(instructions))( + *(SockFilter(*instruction) for instruction in instructions) + ) + program = SockFprog(len(instructions), filters) + prctl(38, 1) + prctl(22, 2, ctypes.addressof(program)) + +drop_caps = mode in ( + "empty-capability-bounding", + "openshell-hardening", + "hardening-plus-openshell-seccomp", +) or mode == "empty-capability-bounding-plus-openshell-seccomp" +if drop_caps: + drop_bounding_set() +drop_to_sandbox() +prctl(4, 1) + +if mode in ("core-zero", "openshell-hardening", "hardening-plus-openshell-seccomp", "core-zero-plus-openshell-seccomp"): + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) +if mode in ("nondumpable", "openshell-hardening", "hardening-plus-openshell-seccomp", "nondumpable-plus-openshell-seccomp"): + prctl(4, 0) +if mode in ("no-new-privs", "openshell-hardening", "hardening-plus-openshell-seccomp", "no-new-privs-plus-openshell-seccomp"): + prctl(38, 1) + +denials = [] +conditional_denials = [] +if mode.startswith("deny-"): + rule = mode.removeprefix("deny-") + if rule in blocked: + denials = [blocked[rule]] + else: + conditional_denials = [conditional[rule]] +elif mode == "openshell-seccomp" or mode.endswith("-plus-openshell-seccomp"): + denials = list(blocked.values()) + conditional_denials = list(conditional.values()) +if mode == "allow-all-seccomp" or denials or conditional_denials: + install_filter(denials, conditional_denials) + +status = {} +with open("/proc/self/status", encoding="utf8") as status_file: + for line in status_file: + key, _, value = line.partition(":") + if key in ("Uid", "Gid", "Groups", "CapBnd", "NoNewPrivs", "Seccomp", "Seccomp_filters"): + status[key] = value.strip() +print("process_status=" + "; ".join(f"{key}={value}" for key, value in status.items())) +print(f"dumpable={prctl_get(3)} core_limit={resource.getrlimit(resource.RLIMIT_CORE)[0]}") + +cuda = ctypes.CDLL("libcuda.so.1") +cuda.cuInit.argtypes = [ctypes.c_uint] +cuda.cuInit.restype = ctypes.c_int +result = cuda.cuInit(0) +print(f"cuInit(0)={result}") +raise SystemExit(0 if result == 0 else 1) +`.trim(); + +type DockerRun = NonNullable; + +function cudaResult(value: string): string { + return value.match(CUDA_RESULT_PATTERN)?.[1] ?? "missing"; +} + +function runCase(containerId: string, mode: string, dockerRun: DockerRun): string { + const result = dockerRun( + ["exec", "--user", "0", containerId, "python3", "-c", PROCESS_BOUNDARY_PROBE, mode], + { ignoreError: true, suppressOutput: true, timeout: PROOF_TIMEOUT_MS }, + ); + const output = `${result.stderr ?? ""}\n${result.stdout ?? ""}`; + const cuda = cudaResult(output); + if (cuda === "missing") { + console.error( + ` process_case_error[${mode}]=${output.trim().replaceAll(/\s+/gu, " ").slice(0, 500) || "no output"}`, + ); + } + return cuda; +} + +/** Isolate the process controls that differ between direct Docker and OpenShell. */ +export function runJetsonOpenRmProcessProof(containerId: string, dockerRun: DockerRun): void { + const fixedCases = [ + "baseline", + "no-new-privs", + "nondumpable", + "core-zero", + "empty-capability-bounding", + "allow-all-seccomp", + "openshell-hardening", + ]; + const results = new Map(fixedCases.map((mode) => [mode, runCase(containerId, mode, dockerRun)])); + + console.log(""); + console.log(" === Jetson OpenRM process boundary matrix ==="); + console.log( + fixedCases.map((mode) => `${mode.replaceAll("-", "_")}_cuInit=${results.get(mode)}`).join(" "), + ); + if (results.get("baseline") !== "0") { + console.error(" INCONCLUSIVE: the direct-Docker process probe baseline did not pass."); + return; + } + const isolatedHardening = fixedCases.slice(1, 5).filter((mode) => results.get(mode) === "801"); + if (isolatedHardening.length > 0) { + console.log(` ISOLATED: cuInit fails after ${isolatedHardening.join(", ")}.`); + return; + } + if (results.get("allow-all-seccomp") === "801") { + console.log(" ISOLATED: cuInit fails when any additional seccomp filter is installed."); + return; + } + if (results.get("openshell-hardening") === "801") { + console.log( + " ISOLATED: cuInit fails only when the non-seccomp process controls are combined.", + ); + return; + } + + const exactSeccomp = runCase(containerId, "openshell-seccomp", dockerRun); + console.log(` openshell_seccomp_cuInit=${exactSeccomp}`); + if (exactSeccomp === "801") { + const rules = [ + ...OPEN_SHELL_BLOCKED_SYSCALLS.map(([name]) => name), + ...OPEN_SHELL_CONDITIONAL_RULES, + ]; + const ruleResults = new Map( + rules.map((name) => [name, runCase(containerId, `deny-${name}`, dockerRun)]), + ); + console.log( + ` seccomp_rule_cuInit=${[...ruleResults].map(([name, result]) => `${name}:${result}`).join(",")}`, + ); + const isolatedRules = [...ruleResults] + .filter(([, result]) => result === "801") + .map(([name]) => name); + if (isolatedRules.length > 0) { + console.log( + ` ISOLATED: OpenShell blocks CUDA-required rule(s): ${isolatedRules.join(", ")}.`, + ); + } else { + console.log(" ISOLATED: CUDA requires a combination of OpenShell seccomp rules."); + } + return; + } + + const full = runCase(containerId, "hardening-plus-openshell-seccomp", dockerRun); + console.log(` hardening_plus_openshell_seccomp_cuInit=${full}`); + if (full === "801") { + const hardening = ["no-new-privs", "nondumpable", "core-zero", "empty-capability-bounding"]; + const interactionResults = new Map( + hardening.map((name) => [ + name, + runCase(containerId, `${name}-plus-openshell-seccomp`, dockerRun), + ]), + ); + console.log( + ` hardening_seccomp_interaction_cuInit=${[...interactionResults].map(([name, result]) => `${name}:${result}`).join(",")}`, + ); + const isolatedInteractions = [...interactionResults] + .filter(([, result]) => result === "801") + .map(([name]) => name); + console.log( + isolatedInteractions.length > 0 + ? ` ISOLATED: CUDA fails when OpenShell seccomp is combined with ${isolatedInteractions.join(", ")}.` + : " ISOLATED: CUDA requires the combined OpenShell hardening and seccomp state.", + ); + return; + } + console.error( + " INCONCLUSIVE: the complete OpenShell process-control model passes; Landlock or an unmodeled launch difference remains.", + ); +} diff --git a/src/lib/onboard/diagnostics/jetson-openrm-proof.ts b/src/lib/onboard/diagnostics/jetson-openrm-proof.ts index ee1a55f757..d6ee600d02 100644 --- a/src/lib/onboard/diagnostics/jetson-openrm-proof.ts +++ b/src/lib/onboard/diagnostics/jetson-openrm-proof.ts @@ -13,6 +13,7 @@ import type { DockerGpuPatchDeps, DockerGpuPatchResult, } from "../docker-gpu-patch-types"; +import { runJetsonOpenRmProcessProof } from "./jetson-openrm-process-proof"; const CUDA_RESULT_PATTERN = /cuInit\(0\)=(-?\d+)/u; const PROOF_TIMEOUT_MS = 30_000; @@ -335,5 +336,6 @@ export function maybeRunJetsonOpenRmPolicyProof(options: OpenRmProofOptions): vo ); console.log(` direct_process_status=${compactProcessStatus(directStatus.stdout)}`); console.log(` openshell_process_status=${compactProcessStatus(openshellStatus.stdout)}`); + runJetsonOpenRmProcessProof(options.result.newContainerId, dockerRun); } } From 3c8113467718ed2b014339979384addc224c176f Mon Sep 17 00:00:00 2001 From: San Dang Date: Fri, 7 Aug 2026 16:45:01 +0700 Subject: [PATCH 45/49] test(onboard): isolate remaining Jetson boundary Signed-off-by: San Dang --- .../prove-jetson-openrm-policy-boundary.sh | 1 + .../jetson-openrm-namespace-proof.test.ts | 32 ++++ .../jetson-openrm-namespace-proof.ts | 155 +++++++++++++++ .../jetson-openrm-process-proof.ts | 15 +- .../diagnostics/jetson-openrm-proof.test.ts | 74 +++++++- .../diagnostics/jetson-openrm-proof.ts | 176 +++++++++++++++--- 6 files changed, 417 insertions(+), 36 deletions(-) create mode 100644 src/lib/onboard/diagnostics/jetson-openrm-namespace-proof.test.ts create mode 100644 src/lib/onboard/diagnostics/jetson-openrm-namespace-proof.ts diff --git a/scripts/prove-jetson-openrm-policy-boundary.sh b/scripts/prove-jetson-openrm-policy-boundary.sh index 34af804903..478d07ad8f 100755 --- a/scripts/prove-jetson-openrm-policy-boundary.sh +++ b/scripts/prove-jetson-openrm-policy-boundary.sh @@ -13,6 +13,7 @@ fi printf 'Running the standalone Jetson OpenRM policy proof for sandbox %s.\n' "$sandbox_name" printf 'This bypasses onboarding and its resume checkpoints.\n' printf 'The current container is preserved as a rollback backup before the production recreation and policy matrix run.\n' +printf 'The matrix briefly widens only the replacement sandbox policy, restores the baseline policy, then restores the original container.\n' npm run build:cli exec node - "$sandbox_name" <<'NODE' diff --git a/src/lib/onboard/diagnostics/jetson-openrm-namespace-proof.test.ts b/src/lib/onboard/diagnostics/jetson-openrm-namespace-proof.test.ts new file mode 100644 index 0000000000..4481dca9d6 --- /dev/null +++ b/src/lib/onboard/diagnostics/jetson-openrm-namespace-proof.test.ts @@ -0,0 +1,32 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; +import { runJetsonOpenRmNamespaceProof } from "./jetson-openrm-namespace-proof"; + +describe("Jetson OpenRM namespace proof", () => { + it("isolates the OpenShell network namespace from direct Docker execution (#7610)", () => { + const dockerRun = vi.fn((args: readonly string[]) => ({ + status: 0, + stdout: `cuInit(0)=${args.at(-1) === "net-namespace" ? "801" : "0"}`, + })); + const log = vi.spyOn(console, "log").mockImplementation(() => undefined); + + runJetsonOpenRmNamespaceProof("a".repeat(64), dockerRun); + + expect(dockerRun).toHaveBeenCalledTimes(8); + expect(log).toHaveBeenCalledWith(expect.stringContaining("net-namespace")); + }); + + it("reports missing namespace probes with their exact mode (#7610)", () => { + const dockerRun = vi.fn(() => ({ status: 1, stderr: "setns denied" })); + const error = vi.spyOn(console, "error").mockImplementation(() => undefined); + + runJetsonOpenRmNamespaceProof("b".repeat(64), dockerRun); + + expect(error).toHaveBeenCalledWith( + expect.stringContaining("namespace_case_error[net-namespace]"), + ); + expect(error).toHaveBeenCalledWith(expect.stringContaining("baseline did not pass")); + }); +}); diff --git a/src/lib/onboard/diagnostics/jetson-openrm-namespace-proof.ts b/src/lib/onboard/diagnostics/jetson-openrm-namespace-proof.ts new file mode 100644 index 0000000000..cc1337565e --- /dev/null +++ b/src/lib/onboard/diagnostics/jetson-openrm-namespace-proof.ts @@ -0,0 +1,155 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { DockerGpuPatchDeps } from "../docker-gpu-patch-types"; + +const PROOF_TIMEOUT_MS = 30_000; +const CUDA_RESULT_PATTERN = /cuInit\(0\)=(-?\d+)/u; + +const NAMESPACE_BOUNDARY_PROBE = String.raw` +import ctypes +import errno +import glob +import os +import pwd +import resource +import stat +import sys + +mode = sys.argv[1] +libc = ctypes.CDLL(None, use_errno=True) + +def checked_call(name, *args): + ctypes.set_errno(0) + rc = getattr(libc, name)(*args) + if rc != 0: + error = ctypes.get_errno() + raise OSError(error, os.strerror(error)) + +def drop_bounding_set(): + for capability in range(64): + ctypes.set_errno(0) + rc = libc.prctl(24, capability, 0, 0, 0) + error = ctypes.get_errno() + if rc != 0 and error != errno.EINVAL: + raise OSError(error, os.strerror(error)) + +def find_openshell_workload(): + self_net = os.stat("/proc/self/ns/net").st_ino + candidates = [] + for entry in os.scandir("/proc"): + if not entry.name.isdigit(): + continue + pid = int(entry.name) + try: + with open(f"/proc/{pid}/status", encoding="utf8") as status_file: + uid_line = next(line for line in status_file if line.startswith("Uid:")) + if int(uid_line.split()[1]) != 998: + continue + net_inode = os.stat(f"/proc/{pid}/ns/net").st_ino + if net_inode == self_net: + continue + with open(f"/proc/{pid}/cmdline", "rb") as command_file: + command = command_file.read().replace(b"\\0", b" ").decode("utf8", "replace") + priority = 0 if "openclaw" in command or "node" in command else 1 + candidates.append((priority, pid, net_inode, command[:160])) + except (OSError, StopIteration, ValueError): + continue + if not candidates: + raise RuntimeError("no sandbox workload in a distinct network namespace") + return sorted(candidates)[0] + +target = None +if "namespace" in mode: + target = find_openshell_workload() + _, target_pid, target_net, target_command = target + print(f"namespace_target=pid:{target_pid} net:{target_net} command:{target_command}") + if "net" in mode: + with open(f"/proc/{target_pid}/ns/net", "rb", buffering=0) as namespace: + checked_call("setns", namespace.fileno(), 0x40000000) + if "mount" in mode: + with open(f"/proc/{target_pid}/ns/mnt", "rb", buffering=0) as namespace: + checked_call("setns", namespace.fileno(), 0x00020000) + +if "process-group" in mode: + os.setpgid(0, 0) +if "hardening" in mode: + drop_bounding_set() + +account = pwd.getpwnam("sandbox") +groups = {account.pw_gid} +for pattern in ("/dev/nvmap", "/dev/nvhost-*", "/dev/dri/renderD*", "/dev/dri/card*"): + for device in glob.glob(pattern): + try: + device_stat = os.stat(device) + except OSError: + continue + if stat.S_ISCHR(device_stat.st_mode) and device_stat.st_gid > 0: + groups.add(device_stat.st_gid) +os.setgroups(sorted(groups)) +os.setgid(account.pw_gid) +os.setuid(account.pw_uid) +checked_call("prctl", 4, 1, 0, 0, 0) + +if "hardening" in mode: + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + checked_call("prctl", 4, 0, 0, 0, 0) + checked_call("prctl", 38, 1, 0, 0, 0) + +cuda = ctypes.CDLL("libcuda.so.1") +cuda.cuInit.argtypes = [ctypes.c_uint] +cuda.cuInit.restype = ctypes.c_int +result = cuda.cuInit(0) +print(f"cuInit(0)={result}") +raise SystemExit(0 if result == 0 else 1) +`.trim(); + +type DockerRun = NonNullable; + +function runCase(containerId: string, mode: string, dockerRun: DockerRun): string { + const result = dockerRun( + ["exec", "--user", "0", containerId, "python3", "-c", NAMESPACE_BOUNDARY_PROBE, mode], + { ignoreError: true, suppressOutput: true, timeout: PROOF_TIMEOUT_MS }, + ); + const output = `${result.stderr ?? ""}\n${result.stdout ?? ""}`; + const cuda = output.match(CUDA_RESULT_PATTERN)?.[1] ?? "missing"; + if (cuda === "missing") { + console.error( + ` namespace_case_error[${mode}]=${output.trim().replaceAll(/\s+/gu, " ").slice(0, 500) || "no output"}`, + ); + } + return cuda; +} + +/** Compare direct Docker execution with the namespaces used by OpenShell workloads. */ +export function runJetsonOpenRmNamespaceProof(containerId: string, dockerRun: DockerRun): void { + const cases = [ + "baseline", + "process-group", + "net-namespace", + "mount-namespace", + "net-mount-namespace", + "hardening-net-namespace", + "hardening-mount-namespace", + "hardening-net-mount-namespace", + ]; + const results = new Map(cases.map((mode) => [mode, runCase(containerId, mode, dockerRun)])); + + console.log(""); + console.log(" === Jetson OpenRM namespace boundary matrix ==="); + console.log( + cases.map((mode) => `${mode.replaceAll("-", "_")}_cuInit=${results.get(mode)}`).join(" "), + ); + if (results.get("baseline") !== "0") { + console.error(" INCONCLUSIVE: the direct-Docker namespace baseline did not pass."); + return; + } + const isolated = cases.slice(1).filter((mode) => results.get(mode) === "801"); + if (isolated.length > 0) { + console.log(` ISOLATED: cuInit fails in OpenShell launch context(s): ${isolated.join(", ")}.`); + return; + } + console.error( + " INCONCLUSIVE: filesystem access, modeled process controls, and workload namespaces all pass outside OpenShell.", + ); +} diff --git a/src/lib/onboard/diagnostics/jetson-openrm-process-proof.ts b/src/lib/onboard/diagnostics/jetson-openrm-process-proof.ts index 5b6b478576..15d7f69957 100644 --- a/src/lib/onboard/diagnostics/jetson-openrm-process-proof.ts +++ b/src/lib/onboard/diagnostics/jetson-openrm-process-proof.ts @@ -233,7 +233,7 @@ function runCase(containerId: string, mode: string, dockerRun: DockerRun): strin } /** Isolate the process controls that differ between direct Docker and OpenShell. */ -export function runJetsonOpenRmProcessProof(containerId: string, dockerRun: DockerRun): void { +export function runJetsonOpenRmProcessProof(containerId: string, dockerRun: DockerRun): boolean { const fixedCases = [ "baseline", "no-new-privs", @@ -252,22 +252,22 @@ export function runJetsonOpenRmProcessProof(containerId: string, dockerRun: Dock ); if (results.get("baseline") !== "0") { console.error(" INCONCLUSIVE: the direct-Docker process probe baseline did not pass."); - return; + return false; } const isolatedHardening = fixedCases.slice(1, 5).filter((mode) => results.get(mode) === "801"); if (isolatedHardening.length > 0) { console.log(` ISOLATED: cuInit fails after ${isolatedHardening.join(", ")}.`); - return; + return false; } if (results.get("allow-all-seccomp") === "801") { console.log(" ISOLATED: cuInit fails when any additional seccomp filter is installed."); - return; + return false; } if (results.get("openshell-hardening") === "801") { console.log( " ISOLATED: cuInit fails only when the non-seccomp process controls are combined.", ); - return; + return false; } const exactSeccomp = runCase(containerId, "openshell-seccomp", dockerRun); @@ -293,7 +293,7 @@ export function runJetsonOpenRmProcessProof(containerId: string, dockerRun: Dock } else { console.log(" ISOLATED: CUDA requires a combination of OpenShell seccomp rules."); } - return; + return false; } const full = runCase(containerId, "hardening-plus-openshell-seccomp", dockerRun); @@ -317,9 +317,10 @@ export function runJetsonOpenRmProcessProof(containerId: string, dockerRun: Dock ? ` ISOLATED: CUDA fails when OpenShell seccomp is combined with ${isolatedInteractions.join(", ")}.` : " ISOLATED: CUDA requires the combined OpenShell hardening and seccomp state.", ); - return; + return false; } console.error( " INCONCLUSIVE: the complete OpenShell process-control model passes; Landlock or an unmodeled launch difference remains.", ); + return true; } diff --git a/src/lib/onboard/diagnostics/jetson-openrm-proof.test.ts b/src/lib/onboard/diagnostics/jetson-openrm-proof.test.ts index ab91735151..5c4373903a 100644 --- a/src/lib/onboard/diagnostics/jetson-openrm-proof.test.ts +++ b/src/lib/onboard/diagnostics/jetson-openrm-proof.test.ts @@ -82,6 +82,17 @@ describe("Jetson OpenRM policy proof", () => { status: "verified" as const, cudaVerified: true, at: "2026-08-06T00:00:00.000Z", + }) + .mockReturnValueOnce({ + status: "verified" as const, + cudaVerified: true, + at: "2026-08-06T00:00:00.000Z", + }) + .mockReturnValueOnce({ + status: "failed" as const, + cudaVerified: false, + detail: "cuInit(0)=801", + at: "2026-08-06T00:00:00.000Z", }); maybeRunJetsonOpenRmPolicyProof({ @@ -99,7 +110,7 @@ describe("Jetson OpenRM policy proof", () => { }, }); - expect(appliedPolicies).toHaveLength(4); + expect(appliedPolicies).toHaveLength(6); expect(appliedPolicies[0]).toContain("/dev/nvidia-caps/nvidia-cap2"); expect(appliedPolicies[0]).toContain("/dev/nvhost-ctrl-pva0"); expect(appliedPolicies[0]).not.toContain("- /sys"); @@ -107,10 +118,69 @@ describe("Jetson OpenRM policy proof", () => { expect(appliedPolicies[1]).not.toContain("/dev/nvidia-caps/nvidia-cap2"); expect(appliedPolicies[2]).toContain("/dev/nvidia-caps/nvidia-cap2"); expect(appliedPolicies[2]).toContain("- /sys"); + expect(appliedPolicies[3]).toContain("/dev/nvhost-ctrl-pva0"); expect(appliedPolicies[3]).not.toContain("/dev/nvidia-caps/nvidia-cap2"); - expect(appliedPolicies[3]).not.toContain("- /sys"); + expect(appliedPolicies[4]).toContain("/dev/nvidia-caps/nvidia-cap2"); + expect(appliedPolicies[4]).not.toContain("/dev/nvhost-ctrl-pva0"); + expect(appliedPolicies[5]).not.toContain("/dev/nvidia-caps/nvidia-cap2"); + expect(appliedPolicies[5]).not.toContain("- /sys"); expect(log).toHaveBeenCalledWith(expect.stringContaining("devices_cuInit=0 sysfs_cuInit=801")); expect(log).toHaveBeenCalledWith(expect.stringContaining("ISOLATED:")); + expect(log).toHaveBeenCalledWith(expect.stringContaining("/dev/nvhost-ctrl-pva0")); + }); + + it("isolates a CUDA-required character device outside the known GPU name families (#7610)", () => { + const appliedPolicies: string[] = []; + const runOpenshell = vi.fn((args: string[]) => { + appliedPolicies.push(fs.readFileSync(args[3] ?? "", "utf8")); + return { status: 0 }; + }); + const verifyDirectSandboxGpu = vi + .fn() + .mockReturnValueOnce({ + status: "failed" as const, + cudaVerified: false, + detail: "cuInit(0)=801", + at: "2026-08-06T00:00:00.000Z", + }) + .mockReturnValueOnce({ + status: "verified" as const, + cudaVerified: true, + at: "2026-08-06T00:00:00.000Z", + }) + .mockReturnValueOnce({ + status: "verified" as const, + cudaVerified: true, + at: "2026-08-06T00:00:00.000Z", + }); + const log = vi.spyOn(console, "log").mockImplementation(() => undefined); + const dockerRun = vi.fn((args: readonly string[]) => + args.includes("0") + ? { status: 0, stdout: "/dev/nvmap\n/dev/special-gpu\n", stderr: "" } + : { status: 0, stdout: "cuInit(0)=0", stderr: "" }, + ); + + maybeRunJetsonOpenRmPolicyProof({ + backend: "jetson", + enabled: true, + failure: new Error("cuInit(0)=801"), + preserveJetsonDeviceGroupMembership: true, + result: result(), + sandboxName: "alpha", + verifyDirectSandboxGpu, + deps: { + dockerRun, + runCaptureOpenshell: vi.fn(() => BASE_POLICY), + runOpenshell, + }, + }); + + expect(appliedPolicies).toHaveLength(4); + expect(appliedPolicies[0]).toContain("- /sys"); + expect(appliedPolicies[1]).toContain("/dev/special-gpu"); + expect(appliedPolicies[2]).toContain("/dev/special-gpu"); + expect(appliedPolicies[3]).not.toContain("/dev/special-gpu"); + expect(log).toHaveBeenCalledWith(expect.stringContaining("/dev/special-gpu")); }); it("restores the baseline when the candidate CUDA proof throws", () => { diff --git a/src/lib/onboard/diagnostics/jetson-openrm-proof.ts b/src/lib/onboard/diagnostics/jetson-openrm-proof.ts index d6ee600d02..06b393172f 100644 --- a/src/lib/onboard/diagnostics/jetson-openrm-proof.ts +++ b/src/lib/onboard/diagnostics/jetson-openrm-proof.ts @@ -13,12 +13,13 @@ import type { DockerGpuPatchDeps, DockerGpuPatchResult, } from "../docker-gpu-patch-types"; +import { runJetsonOpenRmNamespaceProof } from "./jetson-openrm-namespace-proof"; import { runJetsonOpenRmProcessProof } from "./jetson-openrm-process-proof"; const CUDA_RESULT_PATTERN = /cuInit\(0\)=(-?\d+)/u; const PROOF_TIMEOUT_MS = 30_000; const SYSFS_ROOT = "/sys"; -const MAX_GPU_DEVICE_PATHS = 128; +const MAX_CHARACTER_DEVICE_PATHS = 256; const CUDA_PROBE = [ "import ctypes", 'lib = ctypes.CDLL("libcuda.so.1")', @@ -28,23 +29,34 @@ const CUDA_PROBE = [ 'print(f"cuInit(0)={rc}")', "raise SystemExit(0 if rc == 0 else 1)", ].join("; "); -const GPU_DEVICE_DISCOVERY_PROBE = [ +const CHARACTER_DEVICE_DISCOVERY_PROBE = [ "import os, stat", - "prefixes = ('/dev/nvidia', '/dev/nvhost', '/dev/nvgpu', '/dev/tegra')", "for root, dirs, files in os.walk('/dev'):", " for name in files:", " device = os.path.join(root, name)", - " relevant = device == '/dev/nvmap' or device.startswith(prefixes) or (device.startswith('/dev/dri/') and (name.startswith('renderD') or name.startswith('card')))", - " if relevant:", - " try:", - " if stat.S_ISCHR(os.lstat(device).st_mode): print(device)", - " except OSError:", - " pass", + " try:", + " if stat.S_ISCHR(os.lstat(device).st_mode): print(device)", + " except OSError:", + " pass", ].join("\n"); const PROCESS_STATUS_PROBE = "grep -E '^(Uid|Gid|Groups|CapInh|CapPrm|CapEff|CapBnd|CapAmb|NoNewPrivs|Seccomp|Seccomp_filters):' /proc/self/status"; +const CHARACTER_DEVICE_PATH_PATTERN = /^\/dev(?:\/[A-Za-z0-9._-]+)+$/u; const GPU_DEVICE_PATH_PATTERN = - /^\/dev\/(?:nvidia[A-Za-z0-9._/-]*|nvhost[A-Za-z0-9._/-]*|nvgpu(?:\/[A-Za-z0-9._-]+)*|tegra[A-Za-z0-9._/-]*|nvmap|dri\/(?:renderD|card)\d+)$/u; + /^\/dev\/(?:nvidia[A-Za-z0-9._/-]*|nvhost[A-Za-z0-9._/-]*|nvgpu(?:\/[A-Za-z0-9._-]+)*|nvsci[A-Za-z0-9._/-]*|tegra[A-Za-z0-9._/-]*|nvmap|dri\/(?:renderD|card)\d+)$/u; +const FILESYSTEM_PATH_CANDIDATES: readonly PolicyCandidate[] = [ + { name: "dev-shm-read", readOnly: ["/dev/shm"], readWrite: [] }, + { name: "dev-shm-read-write", readOnly: [], readWrite: ["/dev/shm"] }, + { name: "run-read", readOnly: ["/run"], readWrite: [] }, + { name: "run-read-write", readOnly: [], readWrite: ["/run"] }, + { name: "var-read", readOnly: ["/var"], readWrite: [] }, + { name: "var-read-write", readOnly: [], readWrite: ["/var"] }, + { name: "home-read", readOnly: ["/home"], readWrite: [] }, + { name: "opt-read", readOnly: ["/opt"], readWrite: [] }, + { name: "mnt-read", readOnly: ["/mnt"], readWrite: [] }, + { name: "media-read", readOnly: ["/media"], readWrite: [] }, + { name: "srv-read", readOnly: ["/srv"], readWrite: [] }, +]; type OpenRmProofDeps = Pick< DockerGpuPatchDeps, @@ -156,29 +168,33 @@ function candidatePolicy(policyYaml: string, candidate: PolicyCandidate): string return YAML.stringify(policy); } -function discoverInjectedGpuDevicePaths( +function discoverCharacterDevicePaths( containerId: string, dockerRun: NonNullable, ): string[] { const result = dockerRun( - ["exec", "--user", "0", containerId, "python3", "-c", GPU_DEVICE_DISCOVERY_PROBE], + ["exec", "--user", "0", containerId, "python3", "-c", CHARACTER_DEVICE_DISCOVERY_PROBE], { ignoreError: true, suppressOutput: true, timeout: PROOF_TIMEOUT_MS }, ); - if (result.status !== 0) throw new Error("Could not enumerate injected GPU character devices."); + if (result.status !== 0) throw new Error("Could not enumerate container character devices."); const devicePaths = [ ...new Set( String(result.stdout ?? "") .split(/\r?\n/u) .map((devicePath) => devicePath.trim()) - .filter((devicePath) => GPU_DEVICE_PATH_PATTERN.test(devicePath)), + .filter((devicePath) => CHARACTER_DEVICE_PATH_PATTERN.test(devicePath)), ), ].sort(); - if (devicePaths.length === 0 || devicePaths.length > MAX_GPU_DEVICE_PATHS) { - throw new Error("Injected GPU character-device enumeration is empty or excessive."); + if (devicePaths.length === 0 || devicePaths.length > MAX_CHARACTER_DEVICE_PATHS) { + throw new Error("Container character-device enumeration is empty or excessive."); } return devicePaths; } +function policyPathCovers(policyPath: string, targetPath: string): boolean { + return policyPath === "/" || targetPath === policyPath || targetPath.startsWith(`${policyPath}/`); +} + function compactProcessStatus(value: string | Buffer | null | undefined): string { return String(value ?? "") .trim() @@ -215,10 +231,13 @@ export function maybeRunJetsonOpenRmPolicyProof(options: OpenRmProofOptions): vo ); const directOutput = `${direct.stderr ?? ""}\n${direct.stdout ?? ""}`; const directResult = cudaResult(directOutput); - const injectedDevicePaths = discoverInjectedGpuDevicePaths( + const characterDevicePaths = discoverCharacterDevicePaths( options.result.newContainerId, dockerRun, ); + const injectedDevicePaths = characterDevicePaths.filter((devicePath) => + GPU_DEVICE_PATH_PATTERN.test(devicePath), + ); const rawPolicy = captureOpenshell(["policy", "get", "--base", options.sandboxName], { ignoreError: false, timeout: PROOF_TIMEOUT_MS, @@ -226,9 +245,17 @@ export function maybeRunJetsonOpenRmPolicyProof(options: OpenRmProofOptions): vo const baselinePolicy = parseOpenShellPolicy(rawPolicy).yamlBody; if (!baselinePolicy) throw new Error("OpenShell returned no round-trippable base policy."); const baselineFilesystemPolicy = parseFilesystemPolicy(baselinePolicy); - const baselineReadWrite = new Set(baselineFilesystemPolicy.readWrite); const missingDevicePaths = injectedDevicePaths.filter( - (devicePath) => !baselineReadWrite.has(devicePath), + (devicePath) => + !baselineFilesystemPolicy.readWrite.some((policyPath) => + policyPathCovers(policyPath, devicePath), + ), + ); + const missingCharacterDevicePaths = characterDevicePaths.filter( + (devicePath) => + !baselineFilesystemPolicy.readWrite.some((policyPath) => + policyPathCovers(policyPath, devicePath), + ), ); const sysfsMissing = !baselineFilesystemPolicy.readOnly.includes(SYSFS_ROOT) && @@ -252,19 +279,65 @@ export function maybeRunJetsonOpenRmPolicyProof(options: OpenRmProofOptions): vo fs.writeFileSync(baselinePath, baselinePolicy, { encoding: "utf8", mode: 0o600 }); const candidateResults = new Map(); + const individualCharacterDeviceResults = new Map(); let candidateFailure: { readonly error: unknown } | null = null; try { - for (const candidate of candidates) { + const applyCandidate = (candidate: PolicyCandidate): string => { const candidatePath = path.join(temporaryDirectory, `${candidate.name}.yaml`); fs.writeFileSync(candidatePath, candidatePolicy(baselinePolicy, candidate), { encoding: "utf8", mode: 0o600, }); setPolicy(options.sandboxName, candidatePath, runOpenshell); - candidateResults.set( - candidate.name, - proofCudaResult(options.verifyDirectSandboxGpu(options.sandboxName)), - ); + const result = proofCudaResult(options.verifyDirectSandboxGpu(options.sandboxName)); + candidateResults.set(candidate.name, result); + return result; + }; + for (const candidate of candidates) applyCandidate(candidate); + + if (candidateResults.get("devices") === "0") { + const onlyPath = missingDevicePaths.length === 1 ? missingDevicePaths[0] : null; + if (onlyPath) { + individualCharacterDeviceResults.set(onlyPath, "0"); + } else { + for (const [index, devicePath] of missingDevicePaths.entries()) { + const result = applyCandidate({ + name: `gpu-device-${String(index)}`, + readOnly: [], + readWrite: [devicePath], + }); + individualCharacterDeviceResults.set(devicePath, result); + } + } + } + + if (![...candidateResults.values()].includes("0") && missingCharacterDevicePaths.length > 0) { + const allCharacterDevicesResult = applyCandidate({ + name: "all-character-devices", + readOnly: [], + readWrite: missingCharacterDevicePaths, + }); + if (allCharacterDevicesResult === "0") { + for (const [index, devicePath] of missingCharacterDevicePaths.entries()) { + const result = applyCandidate({ + name: `character-device-${String(index)}`, + readOnly: [], + readWrite: [devicePath], + }); + individualCharacterDeviceResults.set(devicePath, result); + } + } + } + + if (![...candidateResults.values()].includes("0")) { + for (const candidate of FILESYSTEM_PATH_CANDIDATES) applyCandidate(candidate); + } + + if (![...candidateResults.values()].includes("0")) { + applyCandidate({ name: "read-root", readOnly: ["/"], readWrite: [] }); + if (candidateResults.get("read-root") !== "0") { + applyCandidate({ name: "read-write-root", readOnly: [], readWrite: ["/"] }); + } } } catch (error) { candidateFailure = { error }; @@ -294,15 +367,46 @@ export function maybeRunJetsonOpenRmPolicyProof(options: OpenRmProofOptions): vo const deviceResult = candidateResults.get("devices") ?? "not-tested"; const sysfsResult = candidateResults.get("sysfs") ?? "not-tested"; const combinedResult = candidateResults.get("devices-plus-sysfs") ?? "not-tested"; + const allCharacterDevicesResult = candidateResults.get("all-character-devices") ?? "not-tested"; + const readRootResult = candidateResults.get("read-root") ?? "not-tested"; + const readWriteRootResult = candidateResults.get("read-write-root") ?? "not-tested"; + const isolatedCharacterDevices = [...individualCharacterDeviceResults] + .filter(([, result]) => result === "0") + .map(([devicePath]) => devicePath); + const passingFilesystemCandidates = FILESYSTEM_PATH_CANDIDATES.filter( + (candidate) => candidateResults.get(candidate.name) === "0", + ); + const isolatedFilesystemCandidates = passingFilesystemCandidates.filter((candidate) => { + const writePath = candidate.readWrite[0]; + return ( + !writePath || !passingFilesystemCandidates.some((other) => other.readOnly.includes(writePath)) + ); + }); console.log(""); console.log(" === Jetson OpenRM policy boundary matrix ==="); console.log(` injected_gpu_devices=${injectedDevicePaths.join(",")}`); console.log(` policy_missing_gpu_devices=${missingDevicePaths.join(",") || "none"}`); console.log( - ` direct_docker_cuInit=${directResult} baseline_openshell_cuInit=801 devices_cuInit=${deviceResult} sysfs_cuInit=${sysfsResult} devices_plus_sysfs_cuInit=${combinedResult}`, + ` policy_missing_character_devices=${missingCharacterDevicePaths.join(",") || "none"}`, + ); + console.log( + ` direct_docker_cuInit=${directResult} baseline_openshell_cuInit=801 devices_cuInit=${deviceResult} sysfs_cuInit=${sysfsResult} devices_plus_sysfs_cuInit=${combinedResult} all_character_devices_cuInit=${allCharacterDevicesResult} read_root_cuInit=${readRootResult} read_write_root_cuInit=${readWriteRootResult}`, ); - if (directResult === "0" && deviceResult === "0" && sysfsResult !== "0") { + if (FILESYSTEM_PATH_CANDIDATES.some((candidate) => candidateResults.has(candidate.name))) { + console.log( + ` filesystem_candidate_cuInit=${FILESYSTEM_PATH_CANDIDATES.map((candidate) => `${candidate.name}:${candidateResults.get(candidate.name) ?? "not-tested"}`).join(",")}`, + ); + } + if (directResult === "0" && isolatedCharacterDevices.length > 0) { + console.log( + ` ISOLATED: OpenShell policy is missing CUDA-required character device(s): ${isolatedCharacterDevices.join(",")}.`, + ); + } else if (directResult === "0" && allCharacterDevicesResult === "0") { + console.log( + " ISOLATED: CUDA requires a combination of character devices currently missing from OpenShell policy.", + ); + } else if (directResult === "0" && deviceResult === "0" && sysfsResult !== "0") { console.log( " ISOLATED: OpenShell policy is missing one or more NVIDIA/Tegra character devices; no sysfs grant is required.", ); @@ -314,6 +418,18 @@ export function maybeRunJetsonOpenRmPolicyProof(options: OpenRmProofOptions): vo console.log( " ISOLATED: CUDA requires both the missing GPU devices and sysfs visibility through OpenShell.", ); + } else if (directResult === "0" && isolatedFilesystemCandidates.length > 0) { + console.log( + ` ISOLATED: CUDA requires additional Landlock path access: ${isolatedFilesystemCandidates.map((candidate) => `${candidate.readWrite.length > 0 ? "read-write" : "read-only"}:${candidate.readWrite[0] ?? candidate.readOnly[0]}`).join(",")}.`, + ); + } else if (directResult === "0" && readRootResult === "0") { + console.log( + " ISOLATED: Landlock is missing CUDA-required read access outside the tested GPU devices and /sys.", + ); + } else if (directResult === "0" && readWriteRootResult === "0") { + console.log( + " ISOLATED: Landlock is missing CUDA-required write access outside the tested character devices.", + ); } else { console.error( " INCONCLUSIVE: the filesystem-policy matrix did not restore CUDA; no production policy change is justified.", @@ -336,6 +452,12 @@ export function maybeRunJetsonOpenRmPolicyProof(options: OpenRmProofOptions): vo ); console.log(` direct_process_status=${compactProcessStatus(directStatus.stdout)}`); console.log(` openshell_process_status=${compactProcessStatus(openshellStatus.stdout)}`); - runJetsonOpenRmProcessProof(options.result.newContainerId, dockerRun); + const processModelPasses = runJetsonOpenRmProcessProof( + options.result.newContainerId, + dockerRun, + ); + if (processModelPasses && readWriteRootResult !== "0") { + runJetsonOpenRmNamespaceProof(options.result.newContainerId, dockerRun); + } } } From 60339b69ed808b31c4cbb326eb19180c5fe3774f Mon Sep 17 00:00:00 2001 From: San Dang Date: Fri, 7 Aug 2026 17:39:54 +0700 Subject: [PATCH 46/49] fix(onboard): allow Jetson NvSci IPC device Signed-off-by: San Dang --- docs/reference/troubleshooting.mdx | 3 ++- .../onboard/docker-gpu-jetson-groups.test.ts | 22 +++++++++++++++++++ src/lib/onboard/docker-gpu-jetson-groups.ts | 1 + src/lib/onboard/docker-gpu-patch-types.ts | 2 +- src/lib/onboard/initial-policy.test.ts | 3 ++- 5 files changed, 28 insertions(+), 3 deletions(-) diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 9762ada50c..1684ba4420 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -836,7 +836,7 @@ NVIDIA NIM and GPU-backed sandbox setup require a real NVIDIA GPU. If NemoClaw rejects the detected GPU name during preflight, select a CPU or remote inference provider, or move the setup to a host with a supported NVIDIA GPU and current drivers. Jetson/Tegra hosts support sandbox GPU passthrough through the compatibility route. -Onboarding detects those hosts separately and propagates eligible host group IDs for selected `/dev/nvmap`, `/dev/nvhost-*`, and `/dev/nvgpu/igpu0/*` nodes plus real `/dev/dri/renderD*` character devices. +Onboarding detects those hosts separately and propagates eligible host group IDs for supported device nodes, including selected `/dev/nvmap`, `/dev/nvhost-*`, and `/dev/nvgpu/igpu0/*` nodes plus real `/dev/dri/renderD*` character devices. If that path fails, follow the Jetson/Tegra compatibility guidance below instead of treating a missing `nvidia-smi` result as a placeholder adapter. ### Colima socket not detected (macOS) @@ -2786,6 +2786,7 @@ This wrapper runs only when legacy OpenClaw Jetson recreation preserves the fixe The creation-time filesystem policy adds no Jetson entries unless `/dev/nvmap` is an existing, non-symlink character device. When that condition is met, the policy adds `/opt/nvidia` as read-only. It adds each existing, non-symlink character device on the eligible GPU path list as read-write. +The eligible GPU path list includes `/dev/nvsciipc` for the read-write policy and device-group detection. Generic GPU and CPU-only policies do not receive these Jetson entries. Use `NEMOCLAW_DOCKER_GPU_PATCH=0` only for troubleshooting because it bypasses that group propagation and CUDA may not initialize. diff --git a/src/lib/onboard/docker-gpu-jetson-groups.test.ts b/src/lib/onboard/docker-gpu-jetson-groups.test.ts index c6cacdd6fe..c943ed638b 100644 --- a/src/lib/onboard/docker-gpu-jetson-groups.test.ts +++ b/src/lib/onboard/docker-gpu-jetson-groups.test.ts @@ -24,6 +24,28 @@ describe("detectTegraGpuDevicePaths", () => { ).toEqual(["/dev/nvmap"]); }); + it("includes nvSci IPC in default policy paths and group discovery (#7610)", () => { + expect( + detectTegraGpuDevicePaths({ + statDevicePath: (devicePath) => + devicePath === "/dev/nvmap" || devicePath === "/dev/nvsciipc" + ? { isCharacterDevice: true, isSymbolicLink: false } + : null, + }), + ).toEqual(["/dev/nvmap", "/dev/nvsciipc"]); + + expect( + detectTegraDeviceGroupGids({ + statDeviceAccess: (devicePath) => + devicePath === "/dev/nvmap" + ? { gid: 44, mode: 0o660 } + : devicePath === "/dev/nvsciipc" + ? { gid: 995, mode: 0o660 } + : null, + }), + ).toEqual(["44", "995"]); + }); + it("requires a character device at /dev/nvmap before returning DRI render devices (#7610)", () => { expect( detectTegraGpuDevicePaths({ diff --git a/src/lib/onboard/docker-gpu-jetson-groups.ts b/src/lib/onboard/docker-gpu-jetson-groups.ts index c8e6decf2d..224fff3f73 100644 --- a/src/lib/onboard/docker-gpu-jetson-groups.ts +++ b/src/lib/onboard/docker-gpu-jetson-groups.ts @@ -5,6 +5,7 @@ import fs from "node:fs"; const TEGRA_GPU_DEVICE_NODES = [ "/dev/nvmap", + "/dev/nvsciipc", "/dev/nvhost-ctrl", "/dev/nvhost-ctrl-gpu", "/dev/nvhost-gpu", diff --git a/src/lib/onboard/docker-gpu-patch-types.ts b/src/lib/onboard/docker-gpu-patch-types.ts index 2b184d27f3..10dc970cc2 100644 --- a/src/lib/onboard/docker-gpu-patch-types.ts +++ b/src/lib/onboard/docker-gpu-patch-types.ts @@ -41,7 +41,7 @@ export type DockerGpuPatchDeps = { probeContainerDns?: ContainerDnsProbeFn; /** * Resolve the host group ID(s) that own the Jetson/Tegra GPU device nodes - * (`/dev/nvmap`, `/dev/nvhost-*`, and `/dev/dri/renderD*`). Used by the + * (`/dev/nvmap`, `/dev/nvsciipc`, `/dev/nvhost-*`, and `/dev/dri/renderD*`). Used by the * Jetson recreate to grant the sandbox user matching `--group-add` * membership so CUDA can open them (#4231, #7610). Injectable so the Jetson * permission path is testable without Tegra hardware. diff --git a/src/lib/onboard/initial-policy.test.ts b/src/lib/onboard/initial-policy.test.ts index b12909739b..cb57a891ff 100644 --- a/src/lib/onboard/initial-policy.test.ts +++ b/src/lib/onboard/initial-policy.test.ts @@ -383,7 +383,7 @@ filesystem_policy: network_policies: {} `, { - jetsonGpuDevicePaths: ["/dev/nvmap", "/dev/nvhost-gpu", "/dev/nvmap"], + jetsonGpuDevicePaths: ["/dev/nvmap", "/dev/nvsciipc", "/dev/nvhost-gpu", "/dev/nvmap"], }, ); const gpuDoc = YAML.parse(gpuPolicy); @@ -391,6 +391,7 @@ network_policies: {} expect(gpuDoc.filesystem_policy.read_only).toContain("/opt/nvidia"); expect(gpuDoc.filesystem_policy.read_only).not.toContain("/dev/nvmap"); expectSingleOccurrence(gpuDoc.filesystem_policy.read_write, "/dev/nvmap"); + expectSingleOccurrence(gpuDoc.filesystem_policy.read_write, "/dev/nvsciipc"); expectSingleOccurrence(gpuDoc.filesystem_policy.read_write, "/dev/nvhost-gpu"); }); From 250b7b45917d24da0df14bcb3584eaf0d04a7e09 Mon Sep 17 00:00:00 2001 From: San Dang Date: Fri, 7 Aug 2026 18:03:20 +0700 Subject: [PATCH 47/49] fix(onboard): remove unproven NvSci policy Signed-off-by: San Dang --- docs/reference/troubleshooting.mdx | 3 +-- .../onboard/docker-gpu-jetson-groups.test.ts | 22 ------------------- src/lib/onboard/docker-gpu-jetson-groups.ts | 1 - src/lib/onboard/docker-gpu-patch-types.ts | 2 +- src/lib/onboard/initial-policy.test.ts | 3 +-- 5 files changed, 3 insertions(+), 28 deletions(-) diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 1684ba4420..9762ada50c 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -836,7 +836,7 @@ NVIDIA NIM and GPU-backed sandbox setup require a real NVIDIA GPU. If NemoClaw rejects the detected GPU name during preflight, select a CPU or remote inference provider, or move the setup to a host with a supported NVIDIA GPU and current drivers. Jetson/Tegra hosts support sandbox GPU passthrough through the compatibility route. -Onboarding detects those hosts separately and propagates eligible host group IDs for supported device nodes, including selected `/dev/nvmap`, `/dev/nvhost-*`, and `/dev/nvgpu/igpu0/*` nodes plus real `/dev/dri/renderD*` character devices. +Onboarding detects those hosts separately and propagates eligible host group IDs for selected `/dev/nvmap`, `/dev/nvhost-*`, and `/dev/nvgpu/igpu0/*` nodes plus real `/dev/dri/renderD*` character devices. If that path fails, follow the Jetson/Tegra compatibility guidance below instead of treating a missing `nvidia-smi` result as a placeholder adapter. ### Colima socket not detected (macOS) @@ -2786,7 +2786,6 @@ This wrapper runs only when legacy OpenClaw Jetson recreation preserves the fixe The creation-time filesystem policy adds no Jetson entries unless `/dev/nvmap` is an existing, non-symlink character device. When that condition is met, the policy adds `/opt/nvidia` as read-only. It adds each existing, non-symlink character device on the eligible GPU path list as read-write. -The eligible GPU path list includes `/dev/nvsciipc` for the read-write policy and device-group detection. Generic GPU and CPU-only policies do not receive these Jetson entries. Use `NEMOCLAW_DOCKER_GPU_PATCH=0` only for troubleshooting because it bypasses that group propagation and CUDA may not initialize. diff --git a/src/lib/onboard/docker-gpu-jetson-groups.test.ts b/src/lib/onboard/docker-gpu-jetson-groups.test.ts index c943ed638b..c6cacdd6fe 100644 --- a/src/lib/onboard/docker-gpu-jetson-groups.test.ts +++ b/src/lib/onboard/docker-gpu-jetson-groups.test.ts @@ -24,28 +24,6 @@ describe("detectTegraGpuDevicePaths", () => { ).toEqual(["/dev/nvmap"]); }); - it("includes nvSci IPC in default policy paths and group discovery (#7610)", () => { - expect( - detectTegraGpuDevicePaths({ - statDevicePath: (devicePath) => - devicePath === "/dev/nvmap" || devicePath === "/dev/nvsciipc" - ? { isCharacterDevice: true, isSymbolicLink: false } - : null, - }), - ).toEqual(["/dev/nvmap", "/dev/nvsciipc"]); - - expect( - detectTegraDeviceGroupGids({ - statDeviceAccess: (devicePath) => - devicePath === "/dev/nvmap" - ? { gid: 44, mode: 0o660 } - : devicePath === "/dev/nvsciipc" - ? { gid: 995, mode: 0o660 } - : null, - }), - ).toEqual(["44", "995"]); - }); - it("requires a character device at /dev/nvmap before returning DRI render devices (#7610)", () => { expect( detectTegraGpuDevicePaths({ diff --git a/src/lib/onboard/docker-gpu-jetson-groups.ts b/src/lib/onboard/docker-gpu-jetson-groups.ts index 224fff3f73..c8e6decf2d 100644 --- a/src/lib/onboard/docker-gpu-jetson-groups.ts +++ b/src/lib/onboard/docker-gpu-jetson-groups.ts @@ -5,7 +5,6 @@ import fs from "node:fs"; const TEGRA_GPU_DEVICE_NODES = [ "/dev/nvmap", - "/dev/nvsciipc", "/dev/nvhost-ctrl", "/dev/nvhost-ctrl-gpu", "/dev/nvhost-gpu", diff --git a/src/lib/onboard/docker-gpu-patch-types.ts b/src/lib/onboard/docker-gpu-patch-types.ts index 10dc970cc2..2b184d27f3 100644 --- a/src/lib/onboard/docker-gpu-patch-types.ts +++ b/src/lib/onboard/docker-gpu-patch-types.ts @@ -41,7 +41,7 @@ export type DockerGpuPatchDeps = { probeContainerDns?: ContainerDnsProbeFn; /** * Resolve the host group ID(s) that own the Jetson/Tegra GPU device nodes - * (`/dev/nvmap`, `/dev/nvsciipc`, `/dev/nvhost-*`, and `/dev/dri/renderD*`). Used by the + * (`/dev/nvmap`, `/dev/nvhost-*`, and `/dev/dri/renderD*`). Used by the * Jetson recreate to grant the sandbox user matching `--group-add` * membership so CUDA can open them (#4231, #7610). Injectable so the Jetson * permission path is testable without Tegra hardware. diff --git a/src/lib/onboard/initial-policy.test.ts b/src/lib/onboard/initial-policy.test.ts index cb57a891ff..b12909739b 100644 --- a/src/lib/onboard/initial-policy.test.ts +++ b/src/lib/onboard/initial-policy.test.ts @@ -383,7 +383,7 @@ filesystem_policy: network_policies: {} `, { - jetsonGpuDevicePaths: ["/dev/nvmap", "/dev/nvsciipc", "/dev/nvhost-gpu", "/dev/nvmap"], + jetsonGpuDevicePaths: ["/dev/nvmap", "/dev/nvhost-gpu", "/dev/nvmap"], }, ); const gpuDoc = YAML.parse(gpuPolicy); @@ -391,7 +391,6 @@ network_policies: {} expect(gpuDoc.filesystem_policy.read_only).toContain("/opt/nvidia"); expect(gpuDoc.filesystem_policy.read_only).not.toContain("/dev/nvmap"); expectSingleOccurrence(gpuDoc.filesystem_policy.read_write, "/dev/nvmap"); - expectSingleOccurrence(gpuDoc.filesystem_policy.read_write, "/dev/nvsciipc"); expectSingleOccurrence(gpuDoc.filesystem_policy.read_write, "/dev/nvhost-gpu"); }); From 1c723ba5cfaa9622b393a4892d588a48fdc0a9b6 Mon Sep 17 00:00:00 2001 From: San Dang Date: Tue, 11 Aug 2026 20:22:45 +0530 Subject: [PATCH 48/49] test(onboard): expand Jetson CUDA diagnostics Signed-off-by: San Dang --- src/lib/onboard/initial-policy.test.ts | 17 ++++- src/lib/onboard/initial-policy.ts | 67 +++++++++++++++---- .../onboard/sandbox-gpu-direct-proof.test.ts | 57 +++++++++++++++- src/lib/onboard/sandbox-gpu-preflight.ts | 12 +++- 4 files changed, 136 insertions(+), 17 deletions(-) diff --git a/src/lib/onboard/initial-policy.test.ts b/src/lib/onboard/initial-policy.test.ts index b12909739b..62014661e2 100644 --- a/src/lib/onboard/initial-policy.test.ts +++ b/src/lib/onboard/initial-policy.test.ts @@ -481,7 +481,9 @@ network_policies: {} }); it("builds direct sandbox GPU proof commands", () => { - const commands = buildDirectSandboxGpuProofCommands("alpha"); + const commands = buildDirectSandboxGpuProofCommands("alpha", { + includeCudaDriverDiagnostics: true, + }); expect(commands.map((entry) => entry.label)).toEqual([ "nvidia-smi when available", "/proc//task//comm write", @@ -506,7 +508,18 @@ network_policies: {} ]); expect(commands[1].args.join(" ")).toContain("/proc/self/comm"); expect(commands[1].args.join(" ")).not.toContain("ls /proc/self/task"); - expect(commands[2].args.join(" ")).toContain("cuInit(0)"); + const cudaProbe = commands[2].args.join(" "); + expect(cudaProbe).toContain("cuInit(0)"); + expect(cudaProbe).toContain("cuDriverGetVersion()"); + expect(cudaProbe).toContain("cuDeviceGetCount()"); + expect(cudaProbe).toContain("cuDeviceGet(0)"); + expect(cudaProbe).toContain("cuDeviceGetName(0)"); + expect(cudaProbe).toContain("raise SystemExit(0 if init_rc == 0 else 1)"); + expect(cudaProbe).not.toContain("cuCtxCreate"); + expect(cudaProbe).not.toContain("cuMemAlloc"); + const genericCudaProbe = buildDirectSandboxGpuProofCommands("alpha")[2].args.join(" "); + expect(genericCudaProbe).toContain("cuInit(0)"); + expect(genericCudaProbe).not.toContain("cuDeviceGetCount()"); for (const command of commands) { for (const arg of command.args) { expect(arg).not.toMatch(/[\r\n]/); diff --git a/src/lib/onboard/initial-policy.ts b/src/lib/onboard/initial-policy.ts index 17680d8e88..3cd203d626 100644 --- a/src/lib/onboard/initial-policy.ts +++ b/src/lib/onboard/initial-policy.ts @@ -265,17 +265,48 @@ const PROC_COMM_WRITE_PROBE = [ "fi", ].join(" "); -const CUDA_INIT_PROBE = [ - "python3", - "-c", - [ - "'import ctypes;", - 'lib = ctypes.CDLL("libcuda.so.1");', - "rc = lib.cuInit(0);", - 'print(f"cuInit(0)={rc}");', - "raise SystemExit(0 if rc == 0 else 1)'", - ].join(" "), -].join(" "); +function buildCudaInitProbe(diagnosticStatements: readonly string[] = []): string { + return [ + "python3", + "-c", + [ + "'import ctypes;", + 'lib = ctypes.CDLL("libcuda.so.1");', + "lib.cuInit.argtypes = [ctypes.c_uint];", + "lib.cuInit.restype = ctypes.c_int;", + "init_rc = lib.cuInit(0);", + 'print(f"cuInit(0)={init_rc}");', + ...diagnosticStatements, + "raise SystemExit(0 if init_rc == 0 else 1)'", + ].join(" "), + ].join(" "); +} + +const CUDA_INIT_PROBE = buildCudaInitProbe(); +const JETSON_CUDA_DRIVER_DIAGNOSTIC_PROBE = buildCudaInitProbe([ + "driver_version = ctypes.c_int();", + "lib.cuDriverGetVersion.argtypes = [ctypes.POINTER(ctypes.c_int)];", + "lib.cuDriverGetVersion.restype = ctypes.c_int;", + "driver_version_rc = lib.cuDriverGetVersion(ctypes.byref(driver_version));", + 'print(f"cuDriverGetVersion()={driver_version_rc} version={driver_version.value}");', + "device_count = ctypes.c_int();", + "lib.cuDeviceGetCount.argtypes = [ctypes.POINTER(ctypes.c_int)];", + "lib.cuDeviceGetCount.restype = ctypes.c_int;", + "device_count_rc = lib.cuDeviceGetCount(ctypes.byref(device_count));", + 'print(f"cuDeviceGetCount()={device_count_rc} count={device_count.value}");', + "device = ctypes.c_int();", + "lib.cuDeviceGet.argtypes = [ctypes.POINTER(ctypes.c_int), ctypes.c_int];", + "lib.cuDeviceGet.restype = ctypes.c_int;", + "device_rc = lib.cuDeviceGet(ctypes.byref(device), 0);", + 'print(f"cuDeviceGet(0)={device_rc} device={device.value}");', + "device_name = ctypes.create_string_buffer(128);", + "lib.cuDeviceGetName.argtypes = [ctypes.c_void_p, ctypes.c_int, ctypes.c_int];", + "lib.cuDeviceGetName.restype = ctypes.c_int;", + "device_name_rc = lib.cuDeviceGetName(ctypes.cast(device_name, ctypes.c_void_p), len(device_name), device.value) if device_rc == 0 else None;", + 'device_name_status = str(device_name_rc) if device_name_rc is not None else "skipped";', + 'device_name_value = device_name.value.decode("utf-8", errors="replace") if device_name_rc == 0 else "unavailable";', + 'print(f"cuDeviceGetName(0)={device_name_status} name={device_name_value}");', +]); const NVIDIA_SMI_OPTIONAL_PROBE = [ "set -eu;", @@ -294,6 +325,7 @@ export type DirectSandboxGpuProofCommand = { export function buildDirectSandboxGpuProofCommands( sandboxName: string, + options: { includeCudaDriverDiagnostics?: boolean } = {}, ): DirectSandboxGpuProofCommand[] { return [ { @@ -311,7 +343,18 @@ export function buildDirectSandboxGpuProofCommands( id: "cuda-init", label: "cuInit(0) via libcuda.so.1", optional: true, - args: ["sandbox", "exec", "-n", sandboxName, "--", "sh", "-lc", CUDA_INIT_PROBE], + args: [ + "sandbox", + "exec", + "-n", + sandboxName, + "--", + "sh", + "-lc", + options.includeCudaDriverDiagnostics + ? JETSON_CUDA_DRIVER_DIAGNOSTIC_PROBE + : CUDA_INIT_PROBE, + ], }, ]; } diff --git a/src/lib/onboard/sandbox-gpu-direct-proof.test.ts b/src/lib/onboard/sandbox-gpu-direct-proof.test.ts index a6d55681ee..e2a615b455 100644 --- a/src/lib/onboard/sandbox-gpu-direct-proof.test.ts +++ b/src/lib/onboard/sandbox-gpu-direct-proof.test.ts @@ -168,7 +168,17 @@ describe("direct sandbox GPU proof", () => { const verifier = createDirectSandboxGpuVerifier({ runOpenshell: vi.fn((args: string[]) => { if (args.includes("cuda-init-cmd")) { - return { status: 1, stdout: "cuInit(0)=999", stderr: "" }; + return { + status: 1, + stdout: [ + "cuInit(0)=999", + "cuDriverGetVersion()=3 version=0", + "cuDeviceGetCount()=3 count=0", + "cuDeviceGet(0)=3 device=0", + "cuDeviceGetName(0)=skipped name=unavailable", + ].join("\n"), + stderr: "", + }; } return { status: 0, stdout: "", stderr: "" }; }), @@ -196,6 +206,9 @@ describe("direct sandbox GPU proof", () => { expect(result.status).toBe("failed"); expect(result.cudaVerified).toBe(false); expect(result.detail).toContain("cuInit(0)=999"); + expect(result.detail).toContain("cuDriverGetVersion()=3 version=0"); + expect(result.detail).toContain("cuDeviceGetCount()=3 count=0"); + expect(result.detail).toContain("cuDeviceGetName(0)=skipped name=unavailable"); const warnings = warnSpy.mock.calls.map((call) => call[0]).join("\n"); expect(warnings).toContain("/dev/nvmap"); } finally { @@ -224,6 +237,48 @@ describe("direct sandbox GPU proof", () => { expect(result.cudaVerified).toBe(true); }); + it("reports Driver API diagnostics without changing the cuInit verification gate (#7610)", () => { + const diagnostic = [ + "cuInit(0)=0", + "cuDriverGetVersion()=0 version=12080", + "cuDeviceGetCount()=801 count=0", + "cuDeviceGet(0)=801 device=0", + "cuDeviceGetName(0)=skipped name=unavailable", + ].join("\n"); + const buildProofCommands = vi.fn(() => [ + { + id: "cuda-init", + args: ["sandbox", "exec", "demo", "--", "cuda"], + label: "cuInit(0)", + optional: true, + }, + ]); + const verifier = createDirectSandboxGpuVerifier({ + runOpenshell: vi.fn(() => ({ status: 0, stdout: diagnostic, stderr: "" })), + detectNvidiaPlatform: () => "jetson", + buildDirectSandboxGpuProofCommands: buildProofCommands, + compactText: (value) => value.trim().replace(/\s+/gu, " "), + redact: (value) => String(value), + }); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined); + + try { + const result = verifier("demo"); + const output = logSpy.mock.calls.flat().join("\n"); + + expect(result.status).toBe("verified"); + expect(result.cudaVerified).toBe(true); + expect(buildProofCommands).toHaveBeenCalledWith("demo", { + includeCudaDriverDiagnostics: true, + }); + expect(output).toContain("cuDriverGetVersion()=0 version=12080"); + expect(output).toContain("cuDeviceGetCount()=801 count=0"); + expect(output).toContain("cuDeviceGetName(0)=skipped name=unavailable"); + } finally { + logSpy.mockRestore(); + } + }); + it("does not report verified when cuda-init exits 0 without the cuInit marker", () => { // A zero exit that never printed `cuInit(0)=` (e.g. a wrapper that swallowed // the real exit code) must not be trusted as CUDA-verified. diff --git a/src/lib/onboard/sandbox-gpu-preflight.ts b/src/lib/onboard/sandbox-gpu-preflight.ts index 0416b60d44..a16e3160ad 100644 --- a/src/lib/onboard/sandbox-gpu-preflight.ts +++ b/src/lib/onboard/sandbox-gpu-preflight.ts @@ -161,7 +161,10 @@ export interface DirectSandboxGpuVerifierDeps extends WslDockerDesktopDetectionD args: string[], opts?: Record, ): { status?: number | null; stdout?: unknown; stderr?: unknown }; - buildDirectSandboxGpuProofCommands?: (sandboxName: string) => Array<{ + buildDirectSandboxGpuProofCommands?: ( + sandboxName: string, + options?: { includeCudaDriverDiagnostics?: boolean }, + ) => Array<{ id?: string; args: string[]; label: string; @@ -236,7 +239,9 @@ export function createDirectSandboxGpuVerifier( // could not run at all). Records the proof that determines "failed" status. let cudaFailure: { label: string; detail: string } | null = null; let explicitNvidiaSmiFailure: { label: string; detail: string } | null = null; - for (const proof of buildProofCommands(sandboxName)) { + for (const proof of buildProofCommands(sandboxName, { + includeCudaDriverDiagnostics: resolvedPlatform === "jetson", + })) { const result = deps.runOpenshell(proof.args, { ignoreError: true, suppressOutput: true, @@ -257,6 +262,9 @@ export function createDirectSandboxGpuVerifier( const diagnostic = deps.compactText(rawOutput).slice(0, 300); if (result.status === 0) { console.log(` ✓ GPU proof passed: ${proof.label}`); + if (proof.id === CUDA_USABILITY_PROOF_ID && resolvedPlatform === "jetson" && diagnostic) { + console.log(` ${diagnostic}`); + } if (proof.id === CUDA_USABILITY_PROOF_ID && cudaInitRan) { // Require the cuInit(0)=0 marker on success too, symmetric with the // failure path: a zero exit without driver initialization, or a From 5e2459ad92d7f4bb988306618a15b06465adc517 Mon Sep 17 00:00:00 2001 From: San Dang Date: Tue, 11 Aug 2026 21:32:57 +0530 Subject: [PATCH 49/49] Revert "test(onboard): expand Jetson CUDA diagnostics" This reverts commit 1c723ba5cfaa9622b393a4892d588a48fdc0a9b6. Signed-off-by: San Dang --- src/lib/onboard/initial-policy.test.ts | 17 +---- src/lib/onboard/initial-policy.ts | 67 ++++--------------- .../onboard/sandbox-gpu-direct-proof.test.ts | 57 +--------------- src/lib/onboard/sandbox-gpu-preflight.ts | 12 +--- 4 files changed, 17 insertions(+), 136 deletions(-) diff --git a/src/lib/onboard/initial-policy.test.ts b/src/lib/onboard/initial-policy.test.ts index 62014661e2..b12909739b 100644 --- a/src/lib/onboard/initial-policy.test.ts +++ b/src/lib/onboard/initial-policy.test.ts @@ -481,9 +481,7 @@ network_policies: {} }); it("builds direct sandbox GPU proof commands", () => { - const commands = buildDirectSandboxGpuProofCommands("alpha", { - includeCudaDriverDiagnostics: true, - }); + const commands = buildDirectSandboxGpuProofCommands("alpha"); expect(commands.map((entry) => entry.label)).toEqual([ "nvidia-smi when available", "/proc//task//comm write", @@ -508,18 +506,7 @@ network_policies: {} ]); expect(commands[1].args.join(" ")).toContain("/proc/self/comm"); expect(commands[1].args.join(" ")).not.toContain("ls /proc/self/task"); - const cudaProbe = commands[2].args.join(" "); - expect(cudaProbe).toContain("cuInit(0)"); - expect(cudaProbe).toContain("cuDriverGetVersion()"); - expect(cudaProbe).toContain("cuDeviceGetCount()"); - expect(cudaProbe).toContain("cuDeviceGet(0)"); - expect(cudaProbe).toContain("cuDeviceGetName(0)"); - expect(cudaProbe).toContain("raise SystemExit(0 if init_rc == 0 else 1)"); - expect(cudaProbe).not.toContain("cuCtxCreate"); - expect(cudaProbe).not.toContain("cuMemAlloc"); - const genericCudaProbe = buildDirectSandboxGpuProofCommands("alpha")[2].args.join(" "); - expect(genericCudaProbe).toContain("cuInit(0)"); - expect(genericCudaProbe).not.toContain("cuDeviceGetCount()"); + expect(commands[2].args.join(" ")).toContain("cuInit(0)"); for (const command of commands) { for (const arg of command.args) { expect(arg).not.toMatch(/[\r\n]/); diff --git a/src/lib/onboard/initial-policy.ts b/src/lib/onboard/initial-policy.ts index 3cd203d626..17680d8e88 100644 --- a/src/lib/onboard/initial-policy.ts +++ b/src/lib/onboard/initial-policy.ts @@ -265,48 +265,17 @@ const PROC_COMM_WRITE_PROBE = [ "fi", ].join(" "); -function buildCudaInitProbe(diagnosticStatements: readonly string[] = []): string { - return [ - "python3", - "-c", - [ - "'import ctypes;", - 'lib = ctypes.CDLL("libcuda.so.1");', - "lib.cuInit.argtypes = [ctypes.c_uint];", - "lib.cuInit.restype = ctypes.c_int;", - "init_rc = lib.cuInit(0);", - 'print(f"cuInit(0)={init_rc}");', - ...diagnosticStatements, - "raise SystemExit(0 if init_rc == 0 else 1)'", - ].join(" "), - ].join(" "); -} - -const CUDA_INIT_PROBE = buildCudaInitProbe(); -const JETSON_CUDA_DRIVER_DIAGNOSTIC_PROBE = buildCudaInitProbe([ - "driver_version = ctypes.c_int();", - "lib.cuDriverGetVersion.argtypes = [ctypes.POINTER(ctypes.c_int)];", - "lib.cuDriverGetVersion.restype = ctypes.c_int;", - "driver_version_rc = lib.cuDriverGetVersion(ctypes.byref(driver_version));", - 'print(f"cuDriverGetVersion()={driver_version_rc} version={driver_version.value}");', - "device_count = ctypes.c_int();", - "lib.cuDeviceGetCount.argtypes = [ctypes.POINTER(ctypes.c_int)];", - "lib.cuDeviceGetCount.restype = ctypes.c_int;", - "device_count_rc = lib.cuDeviceGetCount(ctypes.byref(device_count));", - 'print(f"cuDeviceGetCount()={device_count_rc} count={device_count.value}");', - "device = ctypes.c_int();", - "lib.cuDeviceGet.argtypes = [ctypes.POINTER(ctypes.c_int), ctypes.c_int];", - "lib.cuDeviceGet.restype = ctypes.c_int;", - "device_rc = lib.cuDeviceGet(ctypes.byref(device), 0);", - 'print(f"cuDeviceGet(0)={device_rc} device={device.value}");', - "device_name = ctypes.create_string_buffer(128);", - "lib.cuDeviceGetName.argtypes = [ctypes.c_void_p, ctypes.c_int, ctypes.c_int];", - "lib.cuDeviceGetName.restype = ctypes.c_int;", - "device_name_rc = lib.cuDeviceGetName(ctypes.cast(device_name, ctypes.c_void_p), len(device_name), device.value) if device_rc == 0 else None;", - 'device_name_status = str(device_name_rc) if device_name_rc is not None else "skipped";', - 'device_name_value = device_name.value.decode("utf-8", errors="replace") if device_name_rc == 0 else "unavailable";', - 'print(f"cuDeviceGetName(0)={device_name_status} name={device_name_value}");', -]); +const CUDA_INIT_PROBE = [ + "python3", + "-c", + [ + "'import ctypes;", + 'lib = ctypes.CDLL("libcuda.so.1");', + "rc = lib.cuInit(0);", + 'print(f"cuInit(0)={rc}");', + "raise SystemExit(0 if rc == 0 else 1)'", + ].join(" "), +].join(" "); const NVIDIA_SMI_OPTIONAL_PROBE = [ "set -eu;", @@ -325,7 +294,6 @@ export type DirectSandboxGpuProofCommand = { export function buildDirectSandboxGpuProofCommands( sandboxName: string, - options: { includeCudaDriverDiagnostics?: boolean } = {}, ): DirectSandboxGpuProofCommand[] { return [ { @@ -343,18 +311,7 @@ export function buildDirectSandboxGpuProofCommands( id: "cuda-init", label: "cuInit(0) via libcuda.so.1", optional: true, - args: [ - "sandbox", - "exec", - "-n", - sandboxName, - "--", - "sh", - "-lc", - options.includeCudaDriverDiagnostics - ? JETSON_CUDA_DRIVER_DIAGNOSTIC_PROBE - : CUDA_INIT_PROBE, - ], + args: ["sandbox", "exec", "-n", sandboxName, "--", "sh", "-lc", CUDA_INIT_PROBE], }, ]; } diff --git a/src/lib/onboard/sandbox-gpu-direct-proof.test.ts b/src/lib/onboard/sandbox-gpu-direct-proof.test.ts index e2a615b455..a6d55681ee 100644 --- a/src/lib/onboard/sandbox-gpu-direct-proof.test.ts +++ b/src/lib/onboard/sandbox-gpu-direct-proof.test.ts @@ -168,17 +168,7 @@ describe("direct sandbox GPU proof", () => { const verifier = createDirectSandboxGpuVerifier({ runOpenshell: vi.fn((args: string[]) => { if (args.includes("cuda-init-cmd")) { - return { - status: 1, - stdout: [ - "cuInit(0)=999", - "cuDriverGetVersion()=3 version=0", - "cuDeviceGetCount()=3 count=0", - "cuDeviceGet(0)=3 device=0", - "cuDeviceGetName(0)=skipped name=unavailable", - ].join("\n"), - stderr: "", - }; + return { status: 1, stdout: "cuInit(0)=999", stderr: "" }; } return { status: 0, stdout: "", stderr: "" }; }), @@ -206,9 +196,6 @@ describe("direct sandbox GPU proof", () => { expect(result.status).toBe("failed"); expect(result.cudaVerified).toBe(false); expect(result.detail).toContain("cuInit(0)=999"); - expect(result.detail).toContain("cuDriverGetVersion()=3 version=0"); - expect(result.detail).toContain("cuDeviceGetCount()=3 count=0"); - expect(result.detail).toContain("cuDeviceGetName(0)=skipped name=unavailable"); const warnings = warnSpy.mock.calls.map((call) => call[0]).join("\n"); expect(warnings).toContain("/dev/nvmap"); } finally { @@ -237,48 +224,6 @@ describe("direct sandbox GPU proof", () => { expect(result.cudaVerified).toBe(true); }); - it("reports Driver API diagnostics without changing the cuInit verification gate (#7610)", () => { - const diagnostic = [ - "cuInit(0)=0", - "cuDriverGetVersion()=0 version=12080", - "cuDeviceGetCount()=801 count=0", - "cuDeviceGet(0)=801 device=0", - "cuDeviceGetName(0)=skipped name=unavailable", - ].join("\n"); - const buildProofCommands = vi.fn(() => [ - { - id: "cuda-init", - args: ["sandbox", "exec", "demo", "--", "cuda"], - label: "cuInit(0)", - optional: true, - }, - ]); - const verifier = createDirectSandboxGpuVerifier({ - runOpenshell: vi.fn(() => ({ status: 0, stdout: diagnostic, stderr: "" })), - detectNvidiaPlatform: () => "jetson", - buildDirectSandboxGpuProofCommands: buildProofCommands, - compactText: (value) => value.trim().replace(/\s+/gu, " "), - redact: (value) => String(value), - }); - const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined); - - try { - const result = verifier("demo"); - const output = logSpy.mock.calls.flat().join("\n"); - - expect(result.status).toBe("verified"); - expect(result.cudaVerified).toBe(true); - expect(buildProofCommands).toHaveBeenCalledWith("demo", { - includeCudaDriverDiagnostics: true, - }); - expect(output).toContain("cuDriverGetVersion()=0 version=12080"); - expect(output).toContain("cuDeviceGetCount()=801 count=0"); - expect(output).toContain("cuDeviceGetName(0)=skipped name=unavailable"); - } finally { - logSpy.mockRestore(); - } - }); - it("does not report verified when cuda-init exits 0 without the cuInit marker", () => { // A zero exit that never printed `cuInit(0)=` (e.g. a wrapper that swallowed // the real exit code) must not be trusted as CUDA-verified. diff --git a/src/lib/onboard/sandbox-gpu-preflight.ts b/src/lib/onboard/sandbox-gpu-preflight.ts index a16e3160ad..0416b60d44 100644 --- a/src/lib/onboard/sandbox-gpu-preflight.ts +++ b/src/lib/onboard/sandbox-gpu-preflight.ts @@ -161,10 +161,7 @@ export interface DirectSandboxGpuVerifierDeps extends WslDockerDesktopDetectionD args: string[], opts?: Record, ): { status?: number | null; stdout?: unknown; stderr?: unknown }; - buildDirectSandboxGpuProofCommands?: ( - sandboxName: string, - options?: { includeCudaDriverDiagnostics?: boolean }, - ) => Array<{ + buildDirectSandboxGpuProofCommands?: (sandboxName: string) => Array<{ id?: string; args: string[]; label: string; @@ -239,9 +236,7 @@ export function createDirectSandboxGpuVerifier( // could not run at all). Records the proof that determines "failed" status. let cudaFailure: { label: string; detail: string } | null = null; let explicitNvidiaSmiFailure: { label: string; detail: string } | null = null; - for (const proof of buildProofCommands(sandboxName, { - includeCudaDriverDiagnostics: resolvedPlatform === "jetson", - })) { + for (const proof of buildProofCommands(sandboxName)) { const result = deps.runOpenshell(proof.args, { ignoreError: true, suppressOutput: true, @@ -262,9 +257,6 @@ export function createDirectSandboxGpuVerifier( const diagnostic = deps.compactText(rawOutput).slice(0, 300); if (result.status === 0) { console.log(` ✓ GPU proof passed: ${proof.label}`); - if (proof.id === CUDA_USABILITY_PROOF_ID && resolvedPlatform === "jetson" && diagnostic) { - console.log(` ${diagnostic}`); - } if (proof.id === CUDA_USABILITY_PROOF_ID && cudaInitRan) { // Require the cuInit(0)=0 marker on success too, symmetric with the // failure path: a zero exit without driver initialization, or a