Skip to content
Closed
Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
19c04c3
feat(onboard): preserve Jetson GPU device groups
cjagwani Aug 12, 2026
b0ec392
Merge branch 'main' into codex/jetson-gpu-followup-7610
prekshivyas Aug 13, 2026
de82709
Merge remote-tracking branch 'origin/main' into pr-8910
prekshivyas Aug 13, 2026
0985303
docs(jetson): define native managed replacement
prekshivyas Aug 13, 2026
fff5a76
Merge remote-tracking branch 'origin/main' into pr-8910
prekshivyas Aug 13, 2026
3a87992
merge: reconcile Jetson group preservation with main
prekshivyas Aug 17, 2026
15ee273
Merge remote-tracking branch 'origin/main' into codex/jetson-gpu-foll…
prekshivyas Aug 17, 2026
91a3a9e
test(onboard): keep Jetson cases linear
prekshivyas Aug 17, 2026
15330cd
merge: refresh Jetson follow-up with main
prekshivyas Aug 17, 2026
8dfde23
merge: refresh Jetson follow-up with main
cv Aug 23, 2026
94e6e89
Merge branch 'main' into codex/jetson-gpu-followup-7610
cv Aug 24, 2026
84bfafe
Merge branch 'main' into codex/jetson-gpu-followup-7610
cv Aug 24, 2026
317fa99
Merge branch 'main' into codex/jetson-gpu-followup-7610
cv Aug 24, 2026
821529a
Merge branch 'main' into codex/jetson-gpu-followup-7610
cv Aug 24, 2026
9e7cb05
Merge branch 'main' into codex/jetson-gpu-followup-7610
cv Aug 24, 2026
aa0a5a2
Merge branch 'main' into codex/jetson-gpu-followup-7610
cv Aug 24, 2026
558e8eb
Merge branch 'main' into codex/jetson-gpu-followup-7610
cv Aug 24, 2026
c0c92f5
Merge branch 'main' into codex/jetson-gpu-followup-7610
cv Aug 24, 2026
fe2f324
merge(onboard): sync Jetson GPU follow-up with main
prekshivyas Aug 29, 2026
6462718
Merge remote-tracking branch 'origin/main' into codex/jetson-gpu-foll…
prekshivyas Aug 29, 2026
c23e69e
fix(onboard): install Jetson helper atomically
prekshivyas Aug 29, 2026
2d63cc8
Merge remote-tracking branch 'origin/main' into codex/jetson-gpu-foll…
prekshivyas Aug 29, 2026
4214192
fix(onboard): constrain Jetson group handoff
prekshivyas Aug 29, 2026
7f4f93c
Merge remote-tracking branch 'origin/main' into codex/jetson-gpu-foll…
prekshivyas Aug 29, 2026
5ade992
fix(onboard): make Jetson bootstrap retry-safe
prekshivyas Aug 29, 2026
c3b8e34
docs(e2e): name Jetson bootstrap CI owner
prekshivyas Aug 29, 2026
08935f2
merge: resolve conflicts with main
github-actions[bot] Sep 2, 2026
da9ecdf
merge: resolve conflicts with main
github-actions[bot] Sep 3, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions .github/workflows/platform-vitest-main.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -326,7 +326,7 @@ jobs:
"@
Invoke-WslScript -Distro $env:WSL_DISTRO -User $env:WSL_TEST_USER -Script $script

- name: Run full Vitest suite in WSL
- name: Run Vitest suite in WSL
shell: powershell
run: |
. "$env:TRUSTED_WSL_HELPER"
Expand All @@ -338,7 +338,8 @@ jobs:
cd $workdir
export NEMOCLAW_EXEC_TIMEOUT=60000
export NEMOCLAW_TEST_TIMEOUT=60000
npx vitest run --testTimeout 60000 --shard='${{ matrix.shard }}/4'
npx vitest run --testTimeout 60000 --shard='${{ matrix.shard }}/4' \
--exclude test/jetson-device-group-bootstrap.test.ts
"@
Invoke-WslScript -Distro $env:WSL_DISTRO -User $env:WSL_TEST_USER -Script $script

Expand Down
7 changes: 6 additions & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -566,6 +566,7 @@ COPY --chmod=0444 scripts/lib/corporate-ca-runtime.sh /usr/local/lib/nemoclaw/co
COPY scripts/lib/entrypoint-env-wrapper.sh /usr/local/lib/nemoclaw/entrypoint-env-wrapper.sh
COPY scripts/lib/gateway-supervisor.sh /usr/local/lib/nemoclaw/gateway-supervisor.sh
COPY scripts/lib/sandbox-rlimits.sh /usr/local/lib/nemoclaw/sandbox-rlimits.sh
COPY scripts/jetson-device-group-bootstrap.sh /usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh
COPY scripts/lib/openclaw_device_approval_policy.py /usr/local/lib/nemoclaw/openclaw_device_approval_policy.py
COPY scripts/lib/clean_runtime_shell_env_shim.py /usr/local/lib/nemoclaw/clean_runtime_shell_env_shim.py
COPY scripts/lib/normalize_mutable_config_perms.py /usr/local/lib/nemoclaw/normalize_mutable_config_perms.py
Expand Down Expand Up @@ -1572,12 +1573,14 @@ RUN chmod 755 /usr/local/bin/nemoclaw-start /usr/local/bin/nemoclaw-codex-acp \
&& chmod -R a+rX /src/lib/messaging \
&& chown root:root /usr/local/bin/nemoclaw-gateway-control \
/usr/local/lib/nemoclaw/gateway-supervisor.sh \
/usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh \
/usr/local/lib/nemoclaw/state-dir-guard.py \
/usr/local/share/nemoclaw/state-lock-plan.json \
/usr/local/lib/nemoclaw/openclaw-config-guard.py \
/usr/local/lib/nemoclaw/managed-gateway-control.py \
&& chmod 700 /usr/local/bin/nemoclaw-gateway-control \
&& chmod 500 /usr/local/lib/nemoclaw/state-dir-guard.py \
&& chmod 500 /usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh \
/usr/local/lib/nemoclaw/state-dir-guard.py \
/usr/local/lib/nemoclaw/openclaw-config-guard.py \
/usr/local/lib/nemoclaw/managed-gateway-control.py \
&& chmod 444 /usr/local/share/nemoclaw/state-lock-plan.json \
Expand Down Expand Up @@ -2326,6 +2329,8 @@ RUN check_metadata() { \
&& check_metadata /usr/local/bin/nemoclaw-managed-bootstrap 'root:root:755' \
&& test ! -L /usr/local/lib/nemoclaw/managed-bootstrap-trampoline.sh \
&& check_metadata /usr/local/lib/nemoclaw/managed-bootstrap-trampoline.sh 'root:root:444' \
&& test ! -L /usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh \
&& check_metadata /usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh 'root:root:500' \
&& check_metadata /usr/local/bin/nemoclaw-gateway-control 'root:root:700' \
&& check_metadata /usr/local/lib/nemoclaw/state-dir-guard.py 'root:root:500' \
&& check_metadata /usr/local/share/nemoclaw/state-lock-plan.json 'root:root:444' \
Expand Down
28 changes: 26 additions & 2 deletions docs/reference/troubleshooting.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -3124,11 +3124,35 @@ The path creates the sandbox and then recreates the OpenShell-managed Docker con
`NEMOCLAW_DOCKER_GPU_PATCH=0` is ignored because this runtime requires the compatibility patch for GPU passthrough, and onboarding logs a warning when it is set.
To skip GPU passthrough entirely, rerun with `--no-gpu` or set `NEMOCLAW_SANDBOX_GPU=0`.

<AgentOnly variant="openclaw">

#### Jetson and Tegra compatibility default

Automatic GPU onboarding uses the compatibility path directly; it does not make a native attempt first.
The path recreates the OpenShell-managed Docker container with NVIDIA GPU flags and propagates eligible host group IDs for the supported Jetson GPU device nodes.
Use `NEMOCLAW_DOCKER_GPU_PATCH=0` only for troubleshooting because it bypasses that group propagation and CUDA may not initialize.
The compatibility path recreates the OpenShell-managed Docker container with NVIDIA GPU flags.
For OpenClaw, both that compatibility recreation and a managed-bootstrap replacement on the native route propagate eligible host group IDs for the supported Jetson GPU device nodes.
Docker passes those group IDs through `--group-add` in both replacements.
OpenShell calls `initgroups()` before it starts processes as the sandbox account.
That call reloads supplementary groups from the container group database and can replace the inherited Docker memberships.

Before either replacement starts the OpenShell supervisor, NemoClaw runs a root-owned helper from the OpenClaw sandbox image.
The helper validates the complete detected group-ID list before it changes the sandbox account's group database.
It then verifies the resulting memberships and can hand off only to `/opt/openshell/bin/openshell-sandbox`.
If validation or membership verification fails, the replacement container stops.

The initial filesystem policy adds Jetson paths only when `/dev/nvmap` is an existing, non-symlink character device.
It grants read-only access to the injected NVIDIA libraries under `/opt/nvidia`.
It grants read-write access only to existing, non-symlink character devices from the eligible Tegra GPU path list.
Generic GPU and CPU-only policies do not receive these Jetson entries.

`NEMOCLAW_DOCKER_GPU_PATCH=0` selects the native route instead of the default Jetson compatibility path.
Use it only for troubleshooting because preserving the device groups does not establish that the native route initializes CUDA.

These controls address NVIDIA library visibility, eligible device access, and supplementary group preservation.
They do not establish successful `cuInit(0)` on IGX Orin with JetPack 6.2.
Use `NEMOCLAW_SANDBOX_GPU=0` for CPU-only sandbox behavior while [issue #7610](https://github.com/NVIDIA/NemoClaw/issues/7610) remains open.

</AgentOnly>

#### Common compatibility-path recovery

Expand Down
60 changes: 60 additions & 0 deletions scripts/jetson-device-group-bootstrap.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
#!/usr/bin/env bash
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

# Compatibility bridge for #7610. Remove when the minimum supported OpenShell
# release natively preserves Jetson device groups across the sandbox-user handoff.

set -euo pipefail

fail() {
printf 'Jetson device-group bootstrap: %s\n' "$*" >&2
exit 1
}

[ "$(/usr/bin/id -u)" -eq 0 ] || fail "must run as root"
[ "${1:-}" = "--device-group-gids" ] || fail "device group argument is missing"
group_gids="${2:-}"
[ "${3:-}" = "--" ] || fail "supervisor delimiter is missing"
shift 3
[ "${1:-}" = "/opt/openshell/bin/openshell-sandbox" ] \
|| fail "OpenShell supervisor entrypoint is invalid"
/usr/bin/id sandbox >/dev/null 2>&1 || fail "sandbox user is missing"
if [ ! -f /etc/group ] || [ -L /etc/group ]; then
fail "container group database is invalid"
fi

IFS=',' read -r -a gids <<<"$group_gids"
if [ "${#gids[@]}" -eq 0 ] || [ "${#gids[@]}" -gt 16 ]; then
fail "device group count is invalid"
fi

declare -A seen=()
for gid in "${gids[@]}"; do
[[ "$gid" =~ ^[1-9][0-9]{0,9}$ ]] || fail "device group ID is invalid"
[ "$gid" -le 2147483647 ] || fail "device group ID is out of range"
[ -z "${seen[$gid]:-}" ] || fail "device group ID is duplicated"
seen[$gid]=1
done

for gid in "${gids[@]}"; do
group_record="$(/usr/bin/getent group "$gid" || true)"
if [ -z "$group_record" ]; then
group_name="nemoclaw_gpu_$gid"
/usr/sbin/groupadd --gid "$gid" "$group_name"
else
IFS=':' read -r group_name _ resolved_gid _ <<<"$group_record"
if [ -z "$group_name" ] || [ "$resolved_gid" != "$gid" ]; then
fail "device group record is invalid"
fi
fi
/usr/sbin/usermod --append --groups "$group_name" sandbox
done

sandbox_groups=" $(/usr/bin/id -G sandbox) "
for gid in "${gids[@]}"; do
[[ "$sandbox_groups" == *" $gid "* ]] \
|| fail "sandbox membership verification failed"
done

exec "$@"
36 changes: 35 additions & 1 deletion src/lib/onboard/docker-gpu-jetson-groups.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,41 @@ import fs from "node:fs";

import { afterEach, describe, expect, it, vi } from "vitest";

import { detectTegraDeviceGroupGids } from "./docker-gpu-jetson-groups";
import { detectTegraDeviceGroupGids, detectTegraGpuDevicePaths } from "./docker-gpu-jetson-groups";

describe("detectTegraGpuDevicePaths", () => {
it("returns existing non-symlink character devices when nvmap anchors detection (#7610)", () => {
const paths = ["/dev/nvmap", "/dev/nvhost-gpu", "/dev/nvgpu/igpu0/link"];

expect(
detectTegraGpuDevicePaths({
listDevicePaths: () => paths,
statDevicePath: (devicePath) =>
devicePath === "/dev/nvmap"
? { isCharacterDevice: true, isSymbolicLink: false }
: devicePath === "/dev/nvgpu/igpu0/link"
? { isCharacterDevice: true, isSymbolicLink: true }
: null,
}),
).toEqual(["/dev/nvmap"]);
});

it.each([
null,
{ isCharacterDevice: true, isSymbolicLink: true },
{ isCharacterDevice: false, isSymbolicLink: false },
])("returns no paths for invalid nvmap access case %# (#7610)", (nvmapAccess) => {
expect(
detectTegraGpuDevicePaths({
listDevicePaths: () => ["/dev/nvmap", "/dev/dri/renderD128"],
statDevicePath: (devicePath) =>
devicePath === "/dev/nvmap"
? nvmapAccess
: { isCharacterDevice: true, isSymbolicLink: false },
}),
).toEqual([]);
});
});

describe("detectTegraDeviceGroupGids", () => {
afterEach(() => {
Expand Down
39 changes: 39 additions & 0 deletions src/lib/onboard/docker-gpu-jetson-groups.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ const TEGRA_GPU_DEVICE_NODES = [
"/dev/nvgpu/igpu0/as",
"/dev/nvgpu/igpu0/prof",
] as const;
const NVMAP_DEVICE = "/dev/nvmap";
const READ_WRITE_PERMISSION_BITS = 0o6;
const MAX_DOCKER_SUPPLEMENTARY_GID = 2_147_483_647;

Expand All @@ -24,6 +25,11 @@ type DeviceGroupAccess = {
mode: number;
};

type DevicePathAccess = {
isCharacterDevice: boolean;
isSymbolicLink: boolean;
};

/**
* Find real DRI render character devices without following symlinks or
* scanning other DRI device families.
Expand Down Expand Up @@ -51,6 +57,39 @@ function listTegraGpuDevicePaths(): string[] {
return [...TEGRA_GPU_DEVICE_NODES, ...discoverTegraRenderDevicePaths()];
}

/**
* Return the fixed Jetson GPU device paths that are safe to add to an
* OpenShell filesystem policy. `/dev/nvmap` anchors detection so a host with
* only an unrelated DRI render device does not receive Jetson grants.
*/
export function detectTegraGpuDevicePaths(
deps: {
statDevicePath?: (path: string) => DevicePathAccess | null;
listDevicePaths?: () => string[];
} = {},
): string[] {
const devicePaths = deps.listDevicePaths?.() ?? listTegraGpuDevicePaths();
const statDevicePath =
deps.statDevicePath ??
((devicePath: string): DevicePathAccess | null => {
try {
const stat = fs.lstatSync(devicePath);
return {
isCharacterDevice: stat.isCharacterDevice(),
isSymbolicLink: stat.isSymbolicLink(),
};
} catch {
return null;
}
});

const detectedPaths = devicePaths.filter((devicePath) => {
const access = statDevicePath(devicePath);
return access?.isCharacterDevice === true && access.isSymbolicLink === false;
});
return detectedPaths.includes(NVMAP_DEVICE) ? detectedPaths : [];
}

/**
* Source-of-truth boundary for Jetson/Tegra supplementary device groups:
*
Expand Down
45 changes: 38 additions & 7 deletions src/lib/onboard/docker-gpu-patch-clone.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@ import { openshellSandboxCommandEnvValue } from "./docker-startup-command-env";

const OPENSHELL_SANDBOX_COMMAND_ENV = "OPENSHELL_SANDBOX_COMMAND";
const OPENSHELL_SANDBOX_ENTRYPOINT = "/opt/openshell/bin/openshell-sandbox";
export const JETSON_DEVICE_GROUP_BOOTSTRAP =
"/usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh";
const MAX_JETSON_DEVICE_GROUPS = 16;
const OPENSHELL_V0_0_99_WORKDIR_COMMAND = ["--workdir", "/sandbox"] as const;
const OPENSHELL_OCI_IMAGE_USER_ENV = "OPENSHELL_OCI_IMAGE_USER";
const OPENSHELL_SANDBOX_UID_ENV = "OPENSHELL_SANDBOX_UID";
Expand Down Expand Up @@ -465,6 +468,20 @@ export function buildDockerGpuCloneRunArgs(
}
const args: string[] = ["--name", containerName, ...mode.args];
const gpuAugment = mode.kind !== "startup-command";
const extraGroupGids = (options.extraGroupGids ?? []).map((gid) => String(gid).trim());
if (
extraGroupGids.length > MAX_JETSON_DEVICE_GROUPS ||
new Set(extraGroupGids).size !== extraGroupGids.length ||
extraGroupGids.some((gid) => {
if (!/^[1-9][0-9]*$/u.test(gid)) return true;
const parsed = Number(gid);
return !Number.isSafeInteger(parsed) || parsed > 2_147_483_647;
})
) {
throw new Error("Docker clone received invalid or excessive supplementary group IDs.");
}
const preserveJetsonGroups =
options.preserveJetsonDeviceGroupMembership === true && extraGroupGids.length > 0;

// Startup-command recreation must retain OpenShell's native CDI attachment.
if (!gpuAugment) {
Expand Down Expand Up @@ -546,11 +563,10 @@ export function buildDockerGpuCloneRunArgs(
for (const hostEntry of stringArray(host.ExtraHosts)) args.push("--add-host", hostEntry);
const groupAdds = new Set(stringArray(host.GroupAdd));
for (const group of groupAdds) args.push("--group-add", group);
for (const gid of options.extraGroupGids ?? []) {
const normalized = String(gid).trim();
if (normalized && !groupAdds.has(normalized)) {
groupAdds.add(normalized);
args.push("--group-add", normalized);
for (const gid of extraGroupGids) {
if (!groupAdds.has(gid)) {
groupAdds.add(gid);
args.push("--group-add", gid);
}
}
for (const ulimit of dockerUlimits(inspect, options.requiredUlimits)) {
Expand Down Expand Up @@ -584,17 +600,32 @@ export function buildDockerGpuCloneRunArgs(

const entrypoint = stringArray(config.Entrypoint);
const replacementEntrypoint = String(options.containerEntrypoint ?? "").trim();
if (replacementEntrypoint) {
const groupBootstrapTarget = replacementEntrypoint || entrypoint[0] || "";
if (preserveJetsonGroups && groupBootstrapTarget !== OPENSHELL_SANDBOX_ENTRYPOINT) {
throw new Error("Jetson device-group bootstrap requires the OpenShell supervisor entrypoint.");
}
if (preserveJetsonGroups) {
args.push("--entrypoint", JETSON_DEVICE_GROUP_BOOTSTRAP);
} else if (replacementEntrypoint) {
args.push("--entrypoint", replacementEntrypoint);
} else if (entrypoint.length > 0) {
args.push("--entrypoint", entrypoint[0]);
}
const commandArgs = dockerContainerCommandArgs(
const targetCommandArgs = dockerContainerCommandArgs(
entrypoint,
stringArray(config.Cmd),
sandboxCommand,
options.containerCommand,
);
const commandArgs = preserveJetsonGroups
? [
"--device-group-gids",
extraGroupGids.join(","),
"--",
groupBootstrapTarget,
...targetCommandArgs,
]
: targetCommandArgs;
args.push(image, ...commandArgs);
return args;
}
Expand Down
Loading
Loading