diff --git a/Dockerfile b/Dockerfile index d4b5bfc0a08..e2fc88d8906 100644 --- a/Dockerfile +++ b/Dockerfile @@ -564,6 +564,7 @@ COPY scripts/lib/sandbox-init.sh /usr/local/lib/nemoclaw/sandbox-init.sh COPY scripts/lib/entrypoint-env-wrapper.sh /usr/local/lib/nemoclaw/entrypoint-env-wrapper.sh COPY scripts/lib/gateway-supervisor.sh /usr/local/lib/nemoclaw/gateway-supervisor.sh COPY scripts/lib/sandbox-rlimits.sh /usr/local/lib/nemoclaw/sandbox-rlimits.sh +COPY scripts/jetson-device-group-bootstrap.sh /usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh COPY scripts/lib/openclaw_device_approval_policy.py /usr/local/lib/nemoclaw/openclaw_device_approval_policy.py COPY scripts/lib/clean_runtime_shell_env_shim.py /usr/local/lib/nemoclaw/clean_runtime_shell_env_shim.py COPY scripts/lib/normalize_mutable_config_perms.py /usr/local/lib/nemoclaw/normalize_mutable_config_perms.py @@ -1553,12 +1554,14 @@ RUN chmod 755 /usr/local/bin/nemoclaw-start /usr/local/bin/nemoclaw-codex-acp \ && chmod -R a+rX /src/lib/messaging \ && chown root:root /usr/local/bin/nemoclaw-gateway-control \ /usr/local/lib/nemoclaw/gateway-supervisor.sh \ + /usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh \ /usr/local/lib/nemoclaw/state-dir-guard.py \ /usr/local/share/nemoclaw/state-lock-plan.json \ /usr/local/lib/nemoclaw/openclaw-config-guard.py \ /usr/local/lib/nemoclaw/managed-gateway-control.py \ && chmod 700 /usr/local/bin/nemoclaw-gateway-control \ - && chmod 500 /usr/local/lib/nemoclaw/state-dir-guard.py \ + && chmod 500 /usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh \ + /usr/local/lib/nemoclaw/state-dir-guard.py \ /usr/local/lib/nemoclaw/openclaw-config-guard.py \ /usr/local/lib/nemoclaw/managed-gateway-control.py \ && chmod 444 /usr/local/share/nemoclaw/state-lock-plan.json \ @@ -2306,6 +2309,8 @@ RUN check_metadata() { \ && check_metadata /usr/local/bin/nemoclaw-managed-bootstrap 'root:root:755' \ && test ! -L /usr/local/lib/nemoclaw/managed-bootstrap-trampoline.sh \ && check_metadata /usr/local/lib/nemoclaw/managed-bootstrap-trampoline.sh 'root:root:444' \ + && test ! -L /usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh \ + && check_metadata /usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh 'root:root:500' \ && check_metadata /usr/local/bin/nemoclaw-gateway-control 'root:root:700' \ && check_metadata /usr/local/lib/nemoclaw/state-dir-guard.py 'root:root:500' \ && check_metadata /usr/local/share/nemoclaw/state-lock-plan.json 'root:root:444' \ diff --git a/ci/env-var-doc-allowlist.json b/ci/env-var-doc-allowlist.json index 53729dedb47..de8f33f7812 100644 --- a/ci/env-var-doc-allowlist.json +++ b/ci/env-var-doc-allowlist.json @@ -1,4 +1,8 @@ [ + { + "name": "NEMOCLAW_DIAGNOSE_JETSON_OPENRM_POLICY", + "reason": "Temporary maintainer-only Jetson hardware A/B for issue #7610; remove after the exact policy boundary is proven." + }, { "name": "NEMOCLAW_DISABLE_AUTO_DISPATCH", "reason": "Test harness sentinel set to '1' so test files can import src/nemoclaw.ts without triggering main(). Never user-set in production." diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 57d25be7323..c5d8c196311 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -2832,12 +2832,34 @@ The path creates the sandbox and then recreates the OpenShell-managed Docker con `NEMOCLAW_DOCKER_GPU_PATCH=0` is ignored because this runtime requires the compatibility patch for GPU passthrough, and onboarding logs a warning when it is set. To skip GPU passthrough entirely, rerun with `--no-gpu` or set `NEMOCLAW_SANDBOX_GPU=0`. + + #### Jetson and Tegra compatibility default Automatic GPU onboarding uses the compatibility path directly; it does not make a native attempt first. The path recreates the OpenShell-managed Docker container with NVIDIA GPU flags and propagates eligible host group IDs for the supported Jetson GPU device nodes. + +For legacy OpenClaw Jetson compatibility recreation, Docker passes those group IDs with `--group-add`. +OpenShell 0.0.85 calls `initgroups()` from the unchanged container group database before it starts the sandbox account. +That call replaces the inherited supplementary groups, so `--group-add` alone does not preserve Jetson device access. +Before the fixed OpenShell supervisor starts, NemoClaw runs a bounded wrapper from the sandbox image as root. +The image owns the wrapper as `root:root` with mode `0500`. +The wrapper can hand off only to `/opt/openshell/bin/openshell-sandbox`. +The wrapper validates the complete supplied list for count, group-ID format, range, and duplicates before it changes the container account. +The wrapper adds only the validated Jetson device GIDs that onboarding detected to the existing sandbox account in `/etc/group`. +It verifies the resulting membership before handoff. +OpenShell then rebuilds the account's group list from the updated database, preserving access to the detected device nodes. +This wrapper runs only when legacy OpenClaw Jetson recreation preserves the fixed supervisor entrypoint. + +The creation-time filesystem policy adds no Jetson entries unless `/dev/nvmap` is an existing, non-symlink character device. +When that condition is met, the policy adds `/opt/nvidia` as read-only. +It adds each existing, non-symlink character device on the eligible GPU path list as read-write. +Generic GPU and CPU-only policies do not receive these Jetson entries. + Use `NEMOCLAW_DOCKER_GPU_PATCH=0` only for troubleshooting because it bypasses that group propagation and CUDA may not initialize. + + #### Common compatibility-path recovery After compatibility recreation starts, onboarding keeps the pre-patch container as a rollback backup until the replacement passes the Ready, direct GPU, and applicable local-inference checks. diff --git a/scripts/checks/openshell-policy-mutation-read.mts b/scripts/checks/openshell-policy-mutation-read.mts index e1943578549..f9a1359b628 100644 --- a/scripts/checks/openshell-policy-mutation-read.mts +++ b/scripts/checks/openshell-policy-mutation-read.mts @@ -73,6 +73,10 @@ export const MUTATION_READS: readonly AuditedPolicyReadFile[] = [ preservingBase("getPresetContentGatewayState/readPolicy"), ], }, + { + relativePath: "src/lib/onboard/diagnostics/jetson-openrm-proof.ts", + expectedReads: [preservingBase("maybeRunJetsonOpenRmPolicyProof")], + }, { relativePath: "nemoclaw/src/blueprint/runner.ts", expectedReads: [unclassifiedBase("actionApply")], diff --git a/scripts/diagnose-jetson-cuinit-boundary.sh b/scripts/diagnose-jetson-cuinit-boundary.sh new file mode 100755 index 00000000000..0677b939475 --- /dev/null +++ b/scripts/diagnose-jetson-cuinit-boundary.sh @@ -0,0 +1,164 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -uo pipefail + +sandbox_name="${1:-tm}" +if [[ ! "$sandbox_name" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]*$ ]]; then + printf 'Invalid sandbox name: %s\n' "$sandbox_name" >&2 + exit 2 +fi + +container_ids=() +while IFS=$'\t' read -r container_id container_name; do + if [[ "$container_name" == "openshell-${sandbox_name}-"* ]] \ + && [[ "$container_name" != *-nemoclaw-gpu-backup-* ]]; then + container_ids+=("$container_id") + fi +done < <(docker ps --no-trunc --format '{{.ID}}\t{{.Names}}') + +if ((${#container_ids[@]} != 1)); then + printf 'Expected one running non-backup Docker container for sandbox %s; found %d.\n' \ + "$sandbox_name" "${#container_ids[@]}" >&2 + docker ps --no-trunc --format 'ID={{.ID}} NAME={{.Names}} STATUS={{.Status}}' >&2 + exit 1 +fi + +container_id="${container_ids[0]}" + +cuda_probe='import ctypes +import glob +import os +import stat + +print(f"identity=uid={os.getuid()} gid={os.getgid()} groups={os.getgroups()}") +for key in ("LD_LIBRARY_PATH", "NVIDIA_VISIBLE_DEVICES", "NVIDIA_DRIVER_CAPABILITIES"): + print("env_{}={}".format(key, os.environ.get(key, ""))) + +patterns = ( + "/dev/nvidia*", + "/dev/nvhost-*", + "/dev/nvmap", + "/dev/nvgpu/igpu0/*", + "/dev/dri/renderD*", +) +paths = sorted({path for pattern in patterns for path in glob.glob(pattern)}) +for path in paths: + try: + info = os.stat(path) + except Exception as error: + print(f"device_stat={path} {type(error).__name__}: {error}") + continue + if not stat.S_ISCHR(info.st_mode): + continue + access = [] + for label, flags in (("r", os.O_RDONLY), ("rw", os.O_RDWR)): + try: + fd = os.open(path, flags) + os.close(fd) + access.append(f"{label}=ok") + except Exception as error: + error_number = getattr(error, "errno", "") + access.append(f"{label}={type(error).__name__}:{error_number}") + print( + f"device={path} mode={info.st_mode & 0o777:o} uid={info.st_uid} gid={info.st_gid} " + + " ".join(access) + ) + +try: + cuda = ctypes.CDLL("libcuda.so.1") +except OSError as error: + print(f"libcuda_load=OSError: {error}") + raise SystemExit(11) + +cuda.cuInit.argtypes = [ctypes.c_uint] +cuda.cuInit.restype = ctypes.c_int +result = cuda.cuInit(0) +print("libcuda_load=ok") +try: + names = sorted( + { + line.split()[-1] + for line in open("/proc/self/maps", encoding="utf-8") + if "libcuda.so" in line and line.split()[-1].startswith("/") + } + ) + print(f"libcuda_maps={names}") +except Exception as error: + print(f"libcuda_maps={type(error).__name__}: {error}") + +error_name = ctypes.c_char_p() +try: + cuda.cuGetErrorName.argtypes = [ctypes.c_int, ctypes.POINTER(ctypes.c_char_p)] + cuda.cuGetErrorName.restype = ctypes.c_int + name_result = cuda.cuGetErrorName(result, ctypes.byref(error_name)) + decoded_name = error_name.value.decode() if name_result == 0 and error_name.value else "unknown" +except Exception as error: + decoded_name = f"unavailable:{type(error).__name__}" +print(f"cuInit(0)={result} name={decoded_name}") +raise SystemExit(0 if result == 0 else 10)' + +run_probe() { + local label="$1" + shift + local output status + printf '\n=== %s ===\n' "$label" + if output="$("$@" 2>&1)"; then + status=0 + else + status=$? + fi + printf '%s\nprobe_exit=%d\n' "$output" "$status" + probe_output="$output" +} + +extract_cuinit() { + sed -n -E 's/^cuInit\(0\)=([0-9]+).*$/\1/p' <<<"$1" | tail -1 +} + +printf 'sandbox=%s\ncontainer=%s\n' "$sandbox_name" "$container_id" +printf 'This diagnostic is read-only. It does not create, restart, rename, or remove a sandbox or container.\n' +printf 'git_head=%s\n' "$(git rev-parse HEAD 2>/dev/null || printf unknown)" +printf 'openshell_version=%s\n' "$(openshell --version 2>&1 || printf unknown)" + +printf '\n=== OpenShell policy paths ===\n' +openshell policy get --base "$sandbox_name" 2>&1 \ + | grep -E 'read_only:|read_write:|/opt/nvidia|/dev/nv|/dev/dri' || true + +printf '\n=== Active container configuration ===\n' +docker inspect --format \ + 'image={{.Image}} runtime={{.HostConfig.Runtime}} user={{json .Config.User}} group_add={{json .HostConfig.GroupAdd}} entrypoint={{json .Config.Entrypoint}} cmd={{json .Config.Cmd}} devices={{json .HostConfig.Devices}} device_requests={{json .HostConfig.DeviceRequests}}' \ + "$container_id" 2>&1 || true +docker exec --user 0 "$container_id" /usr/bin/id sandbox 2>&1 || true +docker exec --user 0 "$container_id" /usr/bin/stat -Lc \ + 'wrapper=type=%F mode=%a uid=%u gid=%g path=%n' \ + /usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh 2>&1 || true + +run_probe "Host account" python3 -c "$cuda_probe" +host_output="$probe_output" +run_probe "Direct Docker as root" docker exec --user 0 "$container_id" python3 -c "$cuda_probe" +docker_root_output="$probe_output" +run_probe "Direct Docker as sandbox" docker exec --user sandbox "$container_id" python3 -c "$cuda_probe" +docker_sandbox_output="$probe_output" +run_probe "OpenShell sandbox execution" openshell sandbox exec -n "$sandbox_name" -- python3 -c "$cuda_probe" +openshell_output="$probe_output" + +host_cuinit="$(extract_cuinit "$host_output")" +docker_root_cuinit="$(extract_cuinit "$docker_root_output")" +docker_sandbox_cuinit="$(extract_cuinit "$docker_sandbox_output")" +openshell_cuinit="$(extract_cuinit "$openshell_output")" + +printf '\n=== Boundary result ===\n' +printf 'host_cuInit=%s docker_root_cuInit=%s docker_sandbox_cuInit=%s openshell_cuInit=%s\n' \ + "${host_cuinit:-missing}" "${docker_root_cuinit:-missing}" \ + "${docker_sandbox_cuinit:-missing}" "${openshell_cuinit:-missing}" +if [[ "$docker_sandbox_cuinit" == "0" && "$openshell_cuinit" != "0" ]]; then + printf 'ISOLATED: CUDA works in the running container as sandbox but fails through OpenShell execution. Investigate the OpenShell filesystem/device policy boundary.\n' +elif [[ "$docker_root_cuinit" == "0" && "$docker_sandbox_cuinit" != "0" ]]; then + printf 'ISOLATED: CUDA works as root in the running container but fails as sandbox. Investigate identity, group, or device permission differences.\n' +elif [[ "$docker_root_cuinit" != "0" && "$docker_sandbox_cuinit" != "0" ]]; then + printf 'ISOLATED: CUDA already fails in direct Docker execution. Investigate the recreated container runtime, injected driver libraries, and device set before changing OpenShell policy.\n' +else + printf 'INCONCLUSIVE: preserve this output; the four boundaries did not produce a single failing transition.\n' +fi diff --git a/scripts/jetson-device-group-bootstrap.sh b/scripts/jetson-device-group-bootstrap.sh new file mode 100755 index 00000000000..049fe17c568 --- /dev/null +++ b/scripts/jetson-device-group-bootstrap.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Compatibility bridge for #7610. Remove when the minimum supported OpenShell +# release natively preserves Jetson device groups across the sandbox-user handoff. + +set -euo pipefail + +fail() { + printf 'Jetson device-group bootstrap: %s\n' "$*" >&2 + exit 1 +} + +[ "$(/usr/bin/id -u)" -eq 0 ] || fail "must run as root" +[ "${1:-}" = "--device-group-gids" ] || fail "device group argument is missing" +group_gids="${2:-}" +[ "${3:-}" = "--" ] || fail "supervisor delimiter is missing" +shift 3 +[ "${1:-}" = "/opt/openshell/bin/openshell-sandbox" ] \ + || fail "OpenShell supervisor entrypoint is invalid" +/usr/bin/id sandbox >/dev/null 2>&1 || fail "sandbox user is missing" +if [ ! -f /etc/group ] || [ -L /etc/group ]; then + fail "container group database is invalid" +fi + +IFS=',' read -r -a gids <<<"$group_gids" +if [ "${#gids[@]}" -eq 0 ] || [ "${#gids[@]}" -gt 16 ]; then + fail "device group count is invalid" +fi + +declare -A seen=() +for gid in "${gids[@]}"; do + [[ "$gid" =~ ^[1-9][0-9]{0,9}$ ]] || fail "device group ID is invalid" + [ "$gid" -le 2147483647 ] || fail "device group ID is out of range" + [ -z "${seen[$gid]:-}" ] || fail "device group ID is duplicated" + seen[$gid]=1 +done + +for gid in "${gids[@]}"; do + group_record="$(/usr/bin/getent group "$gid" || true)" + if [ -z "$group_record" ]; then + group_name="nemoclaw_gpu_$gid" + /usr/sbin/groupadd --gid "$gid" "$group_name" + else + IFS=':' read -r group_name _ resolved_gid _ <<<"$group_record" + if [ -z "$group_name" ] || [ "$resolved_gid" != "$gid" ]; then + fail "device group record is invalid" + fi + fi + /usr/sbin/usermod --append --groups "$group_name" sandbox +done + +sandbox_groups=" $(/usr/bin/id -G sandbox) " +for gid in "${gids[@]}"; do + [[ "$sandbox_groups" == *" $gid "* ]] \ + || fail "sandbox membership verification failed" +done + +exec "$@" diff --git a/scripts/prove-jetson-openrm-policy-boundary.sh b/scripts/prove-jetson-openrm-policy-boundary.sh new file mode 100755 index 00000000000..478d07ad8f4 --- /dev/null +++ b/scripts/prove-jetson-openrm-policy-boundary.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +sandbox_name="${1:-tm}" +if [[ ! "$sandbox_name" =~ ^[A-Za-z0-9][A-Za-z0-9_.-]*$ ]]; then + printf 'Invalid sandbox name: %s\n' "$sandbox_name" >&2 + exit 2 +fi + +printf 'Running the standalone Jetson OpenRM policy proof for sandbox %s.\n' "$sandbox_name" +printf 'This bypasses onboarding and its resume checkpoints.\n' +printf 'The current container is preserved as a rollback backup before the production recreation and policy matrix run.\n' +printf 'The matrix briefly widens only the replacement sandbox policy, restores the baseline policy, then restores the original container.\n' + +npm run build:cli +exec node - "$sandbox_name" <<'NODE' +const { + createDockerGpuDiagnosticRedactor, +} = require("./dist/lib/onboard/docker-gpu-diagnostic-redaction"); +const { + runStandaloneJetsonOpenRmPolicyProof, +} = require("./dist/lib/onboard/diagnostics/jetson-openrm-standalone"); + +runStandaloneJetsonOpenRmPolicyProof(process.argv[2]).catch((error) => { + const message = error instanceof Error ? error.message : String(error); + const redacted = createDockerGpuDiagnosticRedactor() + .redactText(message) + .replace(/[\r\n]+/gu, " "); + console.error(`Error: ${redacted}`); + process.exitCode = 1; +}); +NODE diff --git a/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts b/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts index eebacbacd3d..c65ff161d3b 100644 --- a/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts @@ -21,6 +21,7 @@ export const GPU_IMAGE_ID = `sha256:${"a".repeat(64)}`; export function createGpuFlowInput(): SandboxGpuCreateFlowInput { return { sandboxName: "alpha", + agentName: "openclaw", provider: "nim", sandboxGpuConfig: { mode: "1", diff --git a/src/lib/onboard/diagnostics/jetson-openrm-namespace-proof.test.ts b/src/lib/onboard/diagnostics/jetson-openrm-namespace-proof.test.ts new file mode 100644 index 00000000000..4481dca9d6c --- /dev/null +++ b/src/lib/onboard/diagnostics/jetson-openrm-namespace-proof.test.ts @@ -0,0 +1,32 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; +import { runJetsonOpenRmNamespaceProof } from "./jetson-openrm-namespace-proof"; + +describe("Jetson OpenRM namespace proof", () => { + it("isolates the OpenShell network namespace from direct Docker execution (#7610)", () => { + const dockerRun = vi.fn((args: readonly string[]) => ({ + status: 0, + stdout: `cuInit(0)=${args.at(-1) === "net-namespace" ? "801" : "0"}`, + })); + const log = vi.spyOn(console, "log").mockImplementation(() => undefined); + + runJetsonOpenRmNamespaceProof("a".repeat(64), dockerRun); + + expect(dockerRun).toHaveBeenCalledTimes(8); + expect(log).toHaveBeenCalledWith(expect.stringContaining("net-namespace")); + }); + + it("reports missing namespace probes with their exact mode (#7610)", () => { + const dockerRun = vi.fn(() => ({ status: 1, stderr: "setns denied" })); + const error = vi.spyOn(console, "error").mockImplementation(() => undefined); + + runJetsonOpenRmNamespaceProof("b".repeat(64), dockerRun); + + expect(error).toHaveBeenCalledWith( + expect.stringContaining("namespace_case_error[net-namespace]"), + ); + expect(error).toHaveBeenCalledWith(expect.stringContaining("baseline did not pass")); + }); +}); diff --git a/src/lib/onboard/diagnostics/jetson-openrm-namespace-proof.ts b/src/lib/onboard/diagnostics/jetson-openrm-namespace-proof.ts new file mode 100644 index 00000000000..cc1337565e3 --- /dev/null +++ b/src/lib/onboard/diagnostics/jetson-openrm-namespace-proof.ts @@ -0,0 +1,155 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { DockerGpuPatchDeps } from "../docker-gpu-patch-types"; + +const PROOF_TIMEOUT_MS = 30_000; +const CUDA_RESULT_PATTERN = /cuInit\(0\)=(-?\d+)/u; + +const NAMESPACE_BOUNDARY_PROBE = String.raw` +import ctypes +import errno +import glob +import os +import pwd +import resource +import stat +import sys + +mode = sys.argv[1] +libc = ctypes.CDLL(None, use_errno=True) + +def checked_call(name, *args): + ctypes.set_errno(0) + rc = getattr(libc, name)(*args) + if rc != 0: + error = ctypes.get_errno() + raise OSError(error, os.strerror(error)) + +def drop_bounding_set(): + for capability in range(64): + ctypes.set_errno(0) + rc = libc.prctl(24, capability, 0, 0, 0) + error = ctypes.get_errno() + if rc != 0 and error != errno.EINVAL: + raise OSError(error, os.strerror(error)) + +def find_openshell_workload(): + self_net = os.stat("/proc/self/ns/net").st_ino + candidates = [] + for entry in os.scandir("/proc"): + if not entry.name.isdigit(): + continue + pid = int(entry.name) + try: + with open(f"/proc/{pid}/status", encoding="utf8") as status_file: + uid_line = next(line for line in status_file if line.startswith("Uid:")) + if int(uid_line.split()[1]) != 998: + continue + net_inode = os.stat(f"/proc/{pid}/ns/net").st_ino + if net_inode == self_net: + continue + with open(f"/proc/{pid}/cmdline", "rb") as command_file: + command = command_file.read().replace(b"\\0", b" ").decode("utf8", "replace") + priority = 0 if "openclaw" in command or "node" in command else 1 + candidates.append((priority, pid, net_inode, command[:160])) + except (OSError, StopIteration, ValueError): + continue + if not candidates: + raise RuntimeError("no sandbox workload in a distinct network namespace") + return sorted(candidates)[0] + +target = None +if "namespace" in mode: + target = find_openshell_workload() + _, target_pid, target_net, target_command = target + print(f"namespace_target=pid:{target_pid} net:{target_net} command:{target_command}") + if "net" in mode: + with open(f"/proc/{target_pid}/ns/net", "rb", buffering=0) as namespace: + checked_call("setns", namespace.fileno(), 0x40000000) + if "mount" in mode: + with open(f"/proc/{target_pid}/ns/mnt", "rb", buffering=0) as namespace: + checked_call("setns", namespace.fileno(), 0x00020000) + +if "process-group" in mode: + os.setpgid(0, 0) +if "hardening" in mode: + drop_bounding_set() + +account = pwd.getpwnam("sandbox") +groups = {account.pw_gid} +for pattern in ("/dev/nvmap", "/dev/nvhost-*", "/dev/dri/renderD*", "/dev/dri/card*"): + for device in glob.glob(pattern): + try: + device_stat = os.stat(device) + except OSError: + continue + if stat.S_ISCHR(device_stat.st_mode) and device_stat.st_gid > 0: + groups.add(device_stat.st_gid) +os.setgroups(sorted(groups)) +os.setgid(account.pw_gid) +os.setuid(account.pw_uid) +checked_call("prctl", 4, 1, 0, 0, 0) + +if "hardening" in mode: + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + checked_call("prctl", 4, 0, 0, 0, 0) + checked_call("prctl", 38, 1, 0, 0, 0) + +cuda = ctypes.CDLL("libcuda.so.1") +cuda.cuInit.argtypes = [ctypes.c_uint] +cuda.cuInit.restype = ctypes.c_int +result = cuda.cuInit(0) +print(f"cuInit(0)={result}") +raise SystemExit(0 if result == 0 else 1) +`.trim(); + +type DockerRun = NonNullable; + +function runCase(containerId: string, mode: string, dockerRun: DockerRun): string { + const result = dockerRun( + ["exec", "--user", "0", containerId, "python3", "-c", NAMESPACE_BOUNDARY_PROBE, mode], + { ignoreError: true, suppressOutput: true, timeout: PROOF_TIMEOUT_MS }, + ); + const output = `${result.stderr ?? ""}\n${result.stdout ?? ""}`; + const cuda = output.match(CUDA_RESULT_PATTERN)?.[1] ?? "missing"; + if (cuda === "missing") { + console.error( + ` namespace_case_error[${mode}]=${output.trim().replaceAll(/\s+/gu, " ").slice(0, 500) || "no output"}`, + ); + } + return cuda; +} + +/** Compare direct Docker execution with the namespaces used by OpenShell workloads. */ +export function runJetsonOpenRmNamespaceProof(containerId: string, dockerRun: DockerRun): void { + const cases = [ + "baseline", + "process-group", + "net-namespace", + "mount-namespace", + "net-mount-namespace", + "hardening-net-namespace", + "hardening-mount-namespace", + "hardening-net-mount-namespace", + ]; + const results = new Map(cases.map((mode) => [mode, runCase(containerId, mode, dockerRun)])); + + console.log(""); + console.log(" === Jetson OpenRM namespace boundary matrix ==="); + console.log( + cases.map((mode) => `${mode.replaceAll("-", "_")}_cuInit=${results.get(mode)}`).join(" "), + ); + if (results.get("baseline") !== "0") { + console.error(" INCONCLUSIVE: the direct-Docker namespace baseline did not pass."); + return; + } + const isolated = cases.slice(1).filter((mode) => results.get(mode) === "801"); + if (isolated.length > 0) { + console.log(` ISOLATED: cuInit fails in OpenShell launch context(s): ${isolated.join(", ")}.`); + return; + } + console.error( + " INCONCLUSIVE: filesystem access, modeled process controls, and workload namespaces all pass outside OpenShell.", + ); +} diff --git a/src/lib/onboard/diagnostics/jetson-openrm-process-proof.test.ts b/src/lib/onboard/diagnostics/jetson-openrm-process-proof.test.ts new file mode 100644 index 00000000000..02969321c35 --- /dev/null +++ b/src/lib/onboard/diagnostics/jetson-openrm-process-proof.test.ts @@ -0,0 +1,66 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; +import { runJetsonOpenRmProcessProof } from "./jetson-openrm-process-proof"; + +function modeFromArgs(args: readonly string[]): string { + return args.at(-1) ?? ""; +} + +describe("Jetson OpenRM process proof", () => { + it("isolates a non-seccomp process control before syscall probes (#7610)", () => { + const dockerRun = vi.fn((args: readonly string[]) => ({ + status: 0, + stdout: `cuInit(0)=${modeFromArgs(args) === "nondumpable" ? "801" : "0"}`, + })); + const log = vi.spyOn(console, "log").mockImplementation(() => undefined); + + runJetsonOpenRmProcessProof("a".repeat(64), dockerRun); + + expect(dockerRun).toHaveBeenCalledTimes(7); + expect(log).toHaveBeenCalledWith(expect.stringContaining("nondumpable")); + }); + + it("isolates one OpenShell blocked syscall after the fixed process cases pass (#7610)", () => { + const dockerRun = vi.fn((args: readonly string[]) => ({ + status: 0, + stdout: `cuInit(0)=${["openshell-seccomp", "deny-process_vm_readv"].includes(modeFromArgs(args)) ? "801" : "0"}`, + })); + const log = vi.spyOn(console, "log").mockImplementation(() => undefined); + + runJetsonOpenRmProcessProof("b".repeat(64), dockerRun); + + expect(log).toHaveBeenCalledWith( + expect.stringContaining("CUDA-required rule(s): process_vm_readv"), + ); + expect(dockerRun.mock.calls.some(([args]) => modeFromArgs(args) === "deny-clone3")).toBe(true); + expect( + dockerRun.mock.calls.some(([args]) => modeFromArgs(args) === "deny-socket-netlink-non-route"), + ).toBe(true); + }); + + it("isolates an interaction with the complete OpenShell seccomp filter (#7610)", () => { + const dockerRun = vi.fn((args: readonly string[]) => ({ + status: 0, + stdout: `cuInit(0)=${modeFromArgs(args).includes("nondumpable-plus") || modeFromArgs(args) === "hardening-plus-openshell-seccomp" ? "801" : "0"}`, + })); + const log = vi.spyOn(console, "log").mockImplementation(() => undefined); + + runJetsonOpenRmProcessProof("c".repeat(64), dockerRun); + + expect(log).toHaveBeenCalledWith( + expect.stringContaining("seccomp is combined with nondumpable"), + ); + }); + + it("reports an invalid direct-Docker baseline without testing syscall denials (#7610)", () => { + const dockerRun = vi.fn(() => ({ status: 1, stdout: "cuInit(0)=801" })); + const error = vi.spyOn(console, "error").mockImplementation(() => undefined); + + runJetsonOpenRmProcessProof("d".repeat(64), dockerRun); + + expect(dockerRun).toHaveBeenCalledTimes(7); + expect(error).toHaveBeenCalledWith(expect.stringContaining("baseline did not pass")); + }); +}); diff --git a/src/lib/onboard/diagnostics/jetson-openrm-process-proof.ts b/src/lib/onboard/diagnostics/jetson-openrm-process-proof.ts new file mode 100644 index 00000000000..15d7f699577 --- /dev/null +++ b/src/lib/onboard/diagnostics/jetson-openrm-process-proof.ts @@ -0,0 +1,326 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { DockerGpuPatchDeps } from "../docker-gpu-patch-types"; + +const PROOF_TIMEOUT_MS = 30_000; +const CUDA_RESULT_PATTERN = /cuInit\(0\)=(-?\d+)/u; + +// OpenShell v0.0.85 blocks these syscalls unconditionally in its inherited +// supervisor prelude and runtime seccomp filters on aarch64. clone3 uses ENOSYS +// so libc can fall back to clone; the other denials use EPERM. +const OPEN_SHELL_BLOCKED_SYSCALLS = [ + ["umount2", 39, 1], + ["mount", 40, 1], + ["pivot_root", 41, 1], + ["kexec_load", 104, 1], + ["init_module", 105, 1], + ["delete_module", 106, 1], + ["ptrace", 117, 1], + ["perf_event_open", 241, 1], + ["setns", 268, 1], + ["process_vm_readv", 270, 1], + ["process_vm_writev", 271, 1], + ["finit_module", 273, 1], + ["memfd_create", 279, 1], + ["bpf", 280, 1], + ["userfaultfd", 282, 1], + ["kexec_file_load", 294, 1], + ["open_tree", 428, 1], + ["move_mount", 429, 1], + ["fsopen", 430, 1], + ["fsconfig", 431, 1], + ["fsmount", 432, 1], + ["fspick", 433, 1], + ["pidfd_open", 434, 1], + ["clone3", 435, 38], + ["pidfd_getfd", 438, 1], + ["pidfd_send_signal", 424, 1], + ["io_uring_setup", 425, 1], +] as const; + +const OPEN_SHELL_CONDITIONAL_RULES = [ + "socket-af-packet", + "socket-af-bluetooth", + "socket-af-vsock", + "socket-netlink-non-route", + "execveat-empty-path", + "unshare-newuser", + "clone-newuser", + "seccomp-set-filter", +] as const; + +const PROCESS_BOUNDARY_PROBE = String.raw` +import ctypes +import errno +import glob +import os +import pwd +import resource +import stat +import sys + +mode = sys.argv[1] +blocked = { +${OPEN_SHELL_BLOCKED_SYSCALLS.map(([name, number, error]) => ` ${JSON.stringify(name)}: (${String(number)}, ${String(error)}),`).join("\n")} +} +conditional = { + "socket-af-packet": ("eq", 198, 0, 17), + "socket-af-bluetooth": ("eq", 198, 0, 31), + "socket-af-vsock": ("eq", 198, 0, 40), + "socket-netlink-non-route": ("netlink", 198, 0, 16), + "execveat-empty-path": ("masked", 281, 4, 0x1000), + "unshare-newuser": ("masked", 97, 0, 0x10000000), + "clone-newuser": ("masked", 220, 0, 0x10000000), + "seccomp-set-filter": ("eq", 277, 0, 1), +} + +libc = ctypes.CDLL(None, use_errno=True) + +def prctl(option, arg2=0, arg3=0, arg4=0, arg5=0, allow_einval=False): + ctypes.set_errno(0) + rc = libc.prctl( + ctypes.c_int(option), + ctypes.c_ulong(arg2), + ctypes.c_ulong(arg3), + ctypes.c_ulong(arg4), + ctypes.c_ulong(arg5), + ) + error = ctypes.get_errno() + if rc != 0 and not (allow_einval and error == errno.EINVAL): + raise OSError(error, os.strerror(error)) + return rc + +def prctl_get(option): + ctypes.set_errno(0) + rc = libc.prctl(ctypes.c_int(option), 0, 0, 0, 0) + if rc < 0: + error = ctypes.get_errno() + raise OSError(error, os.strerror(error)) + return rc + +def drop_bounding_set(): + for capability in range(64): + prctl(24, capability, allow_einval=True) + +def drop_to_sandbox(): + account = pwd.getpwnam("sandbox") + groups = {account.pw_gid} + for pattern in ("/dev/nvmap", "/dev/nvhost-*", "/dev/dri/renderD*", "/dev/dri/card*"): + for device in glob.glob(pattern): + try: + device_stat = os.stat(device) + except OSError: + continue + if stat.S_ISCHR(device_stat.st_mode) and device_stat.st_gid > 0: + groups.add(device_stat.st_gid) + os.setgroups(sorted(groups)) + os.setgid(account.pw_gid) + os.setuid(account.pw_uid) + +class SockFilter(ctypes.Structure): + _fields_ = [ + ("code", ctypes.c_ushort), + ("jt", ctypes.c_ubyte), + ("jf", ctypes.c_ubyte), + ("k", ctypes.c_uint), + ] + +class SockFprog(ctypes.Structure): + _fields_ = [ + ("length", ctypes.c_ushort), + ("filters", ctypes.POINTER(SockFilter)), + ] + +def install_filter(denials, conditional_denials): + instructions = [(0x20, 0, 0, 0)] + for syscall_number, syscall_errno in denials: + instructions.append((0x15, 0, 1, syscall_number)) + instructions.append((0x06, 0, 0, 0x00050000 | syscall_errno)) + for kind, syscall_number, arg_index, value in conditional_denials: + instructions.append((0x20, 0, 0, 0)) + if kind == "masked": + instructions.append((0x15, 0, 4, syscall_number)) + instructions.append((0x20, 0, 0, 16 + (arg_index * 8))) + instructions.append((0x54, 0, 0, value)) + instructions.append((0x15, 0, 1, value)) + elif kind == "netlink": + instructions.append((0x15, 0, 5, syscall_number)) + instructions.append((0x20, 0, 0, 16)) + instructions.append((0x15, 0, 3, value)) + instructions.append((0x20, 0, 0, 32)) + instructions.append((0x15, 1, 0, 0)) + else: + instructions.append((0x15, 0, 3, syscall_number)) + instructions.append((0x20, 0, 0, 16 + (arg_index * 8))) + instructions.append((0x15, 0, 1, value)) + instructions.append((0x06, 0, 0, 0x00050001)) + instructions.append((0x06, 0, 0, 0x7fff0000)) + filters = (SockFilter * len(instructions))( + *(SockFilter(*instruction) for instruction in instructions) + ) + program = SockFprog(len(instructions), filters) + prctl(38, 1) + prctl(22, 2, ctypes.addressof(program)) + +drop_caps = mode in ( + "empty-capability-bounding", + "openshell-hardening", + "hardening-plus-openshell-seccomp", +) or mode == "empty-capability-bounding-plus-openshell-seccomp" +if drop_caps: + drop_bounding_set() +drop_to_sandbox() +prctl(4, 1) + +if mode in ("core-zero", "openshell-hardening", "hardening-plus-openshell-seccomp", "core-zero-plus-openshell-seccomp"): + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) +if mode in ("nondumpable", "openshell-hardening", "hardening-plus-openshell-seccomp", "nondumpable-plus-openshell-seccomp"): + prctl(4, 0) +if mode in ("no-new-privs", "openshell-hardening", "hardening-plus-openshell-seccomp", "no-new-privs-plus-openshell-seccomp"): + prctl(38, 1) + +denials = [] +conditional_denials = [] +if mode.startswith("deny-"): + rule = mode.removeprefix("deny-") + if rule in blocked: + denials = [blocked[rule]] + else: + conditional_denials = [conditional[rule]] +elif mode == "openshell-seccomp" or mode.endswith("-plus-openshell-seccomp"): + denials = list(blocked.values()) + conditional_denials = list(conditional.values()) +if mode == "allow-all-seccomp" or denials or conditional_denials: + install_filter(denials, conditional_denials) + +status = {} +with open("/proc/self/status", encoding="utf8") as status_file: + for line in status_file: + key, _, value = line.partition(":") + if key in ("Uid", "Gid", "Groups", "CapBnd", "NoNewPrivs", "Seccomp", "Seccomp_filters"): + status[key] = value.strip() +print("process_status=" + "; ".join(f"{key}={value}" for key, value in status.items())) +print(f"dumpable={prctl_get(3)} core_limit={resource.getrlimit(resource.RLIMIT_CORE)[0]}") + +cuda = ctypes.CDLL("libcuda.so.1") +cuda.cuInit.argtypes = [ctypes.c_uint] +cuda.cuInit.restype = ctypes.c_int +result = cuda.cuInit(0) +print(f"cuInit(0)={result}") +raise SystemExit(0 if result == 0 else 1) +`.trim(); + +type DockerRun = NonNullable; + +function cudaResult(value: string): string { + return value.match(CUDA_RESULT_PATTERN)?.[1] ?? "missing"; +} + +function runCase(containerId: string, mode: string, dockerRun: DockerRun): string { + const result = dockerRun( + ["exec", "--user", "0", containerId, "python3", "-c", PROCESS_BOUNDARY_PROBE, mode], + { ignoreError: true, suppressOutput: true, timeout: PROOF_TIMEOUT_MS }, + ); + const output = `${result.stderr ?? ""}\n${result.stdout ?? ""}`; + const cuda = cudaResult(output); + if (cuda === "missing") { + console.error( + ` process_case_error[${mode}]=${output.trim().replaceAll(/\s+/gu, " ").slice(0, 500) || "no output"}`, + ); + } + return cuda; +} + +/** Isolate the process controls that differ between direct Docker and OpenShell. */ +export function runJetsonOpenRmProcessProof(containerId: string, dockerRun: DockerRun): boolean { + const fixedCases = [ + "baseline", + "no-new-privs", + "nondumpable", + "core-zero", + "empty-capability-bounding", + "allow-all-seccomp", + "openshell-hardening", + ]; + const results = new Map(fixedCases.map((mode) => [mode, runCase(containerId, mode, dockerRun)])); + + console.log(""); + console.log(" === Jetson OpenRM process boundary matrix ==="); + console.log( + fixedCases.map((mode) => `${mode.replaceAll("-", "_")}_cuInit=${results.get(mode)}`).join(" "), + ); + if (results.get("baseline") !== "0") { + console.error(" INCONCLUSIVE: the direct-Docker process probe baseline did not pass."); + return false; + } + const isolatedHardening = fixedCases.slice(1, 5).filter((mode) => results.get(mode) === "801"); + if (isolatedHardening.length > 0) { + console.log(` ISOLATED: cuInit fails after ${isolatedHardening.join(", ")}.`); + return false; + } + if (results.get("allow-all-seccomp") === "801") { + console.log(" ISOLATED: cuInit fails when any additional seccomp filter is installed."); + return false; + } + if (results.get("openshell-hardening") === "801") { + console.log( + " ISOLATED: cuInit fails only when the non-seccomp process controls are combined.", + ); + return false; + } + + const exactSeccomp = runCase(containerId, "openshell-seccomp", dockerRun); + console.log(` openshell_seccomp_cuInit=${exactSeccomp}`); + if (exactSeccomp === "801") { + const rules = [ + ...OPEN_SHELL_BLOCKED_SYSCALLS.map(([name]) => name), + ...OPEN_SHELL_CONDITIONAL_RULES, + ]; + const ruleResults = new Map( + rules.map((name) => [name, runCase(containerId, `deny-${name}`, dockerRun)]), + ); + console.log( + ` seccomp_rule_cuInit=${[...ruleResults].map(([name, result]) => `${name}:${result}`).join(",")}`, + ); + const isolatedRules = [...ruleResults] + .filter(([, result]) => result === "801") + .map(([name]) => name); + if (isolatedRules.length > 0) { + console.log( + ` ISOLATED: OpenShell blocks CUDA-required rule(s): ${isolatedRules.join(", ")}.`, + ); + } else { + console.log(" ISOLATED: CUDA requires a combination of OpenShell seccomp rules."); + } + return false; + } + + const full = runCase(containerId, "hardening-plus-openshell-seccomp", dockerRun); + console.log(` hardening_plus_openshell_seccomp_cuInit=${full}`); + if (full === "801") { + const hardening = ["no-new-privs", "nondumpable", "core-zero", "empty-capability-bounding"]; + const interactionResults = new Map( + hardening.map((name) => [ + name, + runCase(containerId, `${name}-plus-openshell-seccomp`, dockerRun), + ]), + ); + console.log( + ` hardening_seccomp_interaction_cuInit=${[...interactionResults].map(([name, result]) => `${name}:${result}`).join(",")}`, + ); + const isolatedInteractions = [...interactionResults] + .filter(([, result]) => result === "801") + .map(([name]) => name); + console.log( + isolatedInteractions.length > 0 + ? ` ISOLATED: CUDA fails when OpenShell seccomp is combined with ${isolatedInteractions.join(", ")}.` + : " ISOLATED: CUDA requires the combined OpenShell hardening and seccomp state.", + ); + return false; + } + console.error( + " INCONCLUSIVE: the complete OpenShell process-control model passes; Landlock or an unmodeled launch difference remains.", + ); + return true; +} diff --git a/src/lib/onboard/diagnostics/jetson-openrm-proof.test.ts b/src/lib/onboard/diagnostics/jetson-openrm-proof.test.ts new file mode 100644 index 00000000000..5c4373903aa --- /dev/null +++ b/src/lib/onboard/diagnostics/jetson-openrm-proof.test.ts @@ -0,0 +1,303 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { DockerGpuPatchResult } from "../docker-gpu-patch-types"; +import { + JetsonOpenRmPolicyRestorationError, + maybeRunJetsonOpenRmPolicyProof, +} from "./jetson-openrm-proof"; + +const BASE_POLICY = `Version: 2 +Hash: fixture +--- +version: 1 +filesystem_policy: + read_only: + - /opt/nvidia + read_write: + - /dev/nvmap +network_policies: {} +`; + +function result(): DockerGpuPatchResult { + return { + applied: true, + oldContainerId: "a".repeat(64), + newContainerId: "b".repeat(64), + originalName: "openshell-alpha-fixture", + backupContainerName: "openshell-alpha-fixture-backup", + mode: { + kind: "nvidia-runtime", + label: "--runtime nvidia", + device: "all", + args: ["--runtime", "nvidia"], + }, + backupRemoved: false, + }; +} + +function dockerRunForBoundaryProof() { + return vi.fn((args: readonly string[]) => + args.includes("0") + ? { + status: 0, + stdout: ["/dev/nvidia-caps/nvidia-cap2", "/dev/nvhost-ctrl-pva0", "/dev/nvmap"].join( + "\n", + ), + stderr: "", + } + : { status: 0, stdout: "cuInit(0)=0", stderr: "" }, + ); +} + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe("Jetson OpenRM policy proof", () => { + it("isolates missing injected devices from sysfs and restores the baseline policy", () => { + const appliedPolicies: string[] = []; + const runOpenshell = vi.fn((args: string[]) => { + appliedPolicies.push(fs.readFileSync(args[3] ?? "", "utf8")); + return { status: 0 }; + }); + const log = vi.spyOn(console, "log").mockImplementation(() => undefined); + const verifyDirectSandboxGpu = vi + .fn() + .mockReturnValueOnce({ + status: "verified" as const, + cudaVerified: true, + at: "2026-08-06T00:00:00.000Z", + }) + .mockReturnValueOnce({ + status: "failed" as const, + cudaVerified: false, + detail: "cuInit(0)=801", + at: "2026-08-06T00:00:00.000Z", + }) + .mockReturnValueOnce({ + status: "verified" as const, + cudaVerified: true, + at: "2026-08-06T00:00:00.000Z", + }) + .mockReturnValueOnce({ + status: "verified" as const, + cudaVerified: true, + at: "2026-08-06T00:00:00.000Z", + }) + .mockReturnValueOnce({ + status: "failed" as const, + cudaVerified: false, + detail: "cuInit(0)=801", + at: "2026-08-06T00:00:00.000Z", + }); + + maybeRunJetsonOpenRmPolicyProof({ + backend: "jetson", + enabled: true, + failure: new Error("cuInit(0)=801"), + preserveJetsonDeviceGroupMembership: true, + result: result(), + sandboxName: "alpha", + verifyDirectSandboxGpu, + deps: { + dockerRun: dockerRunForBoundaryProof(), + runCaptureOpenshell: vi.fn(() => BASE_POLICY), + runOpenshell, + }, + }); + + expect(appliedPolicies).toHaveLength(6); + expect(appliedPolicies[0]).toContain("/dev/nvidia-caps/nvidia-cap2"); + expect(appliedPolicies[0]).toContain("/dev/nvhost-ctrl-pva0"); + expect(appliedPolicies[0]).not.toContain("- /sys"); + expect(appliedPolicies[1]).toContain("- /sys"); + expect(appliedPolicies[1]).not.toContain("/dev/nvidia-caps/nvidia-cap2"); + expect(appliedPolicies[2]).toContain("/dev/nvidia-caps/nvidia-cap2"); + expect(appliedPolicies[2]).toContain("- /sys"); + expect(appliedPolicies[3]).toContain("/dev/nvhost-ctrl-pva0"); + expect(appliedPolicies[3]).not.toContain("/dev/nvidia-caps/nvidia-cap2"); + expect(appliedPolicies[4]).toContain("/dev/nvidia-caps/nvidia-cap2"); + expect(appliedPolicies[4]).not.toContain("/dev/nvhost-ctrl-pva0"); + expect(appliedPolicies[5]).not.toContain("/dev/nvidia-caps/nvidia-cap2"); + expect(appliedPolicies[5]).not.toContain("- /sys"); + expect(log).toHaveBeenCalledWith(expect.stringContaining("devices_cuInit=0 sysfs_cuInit=801")); + expect(log).toHaveBeenCalledWith(expect.stringContaining("ISOLATED:")); + expect(log).toHaveBeenCalledWith(expect.stringContaining("/dev/nvhost-ctrl-pva0")); + }); + + it("isolates a CUDA-required character device outside the known GPU name families (#7610)", () => { + const appliedPolicies: string[] = []; + const runOpenshell = vi.fn((args: string[]) => { + appliedPolicies.push(fs.readFileSync(args[3] ?? "", "utf8")); + return { status: 0 }; + }); + const verifyDirectSandboxGpu = vi + .fn() + .mockReturnValueOnce({ + status: "failed" as const, + cudaVerified: false, + detail: "cuInit(0)=801", + at: "2026-08-06T00:00:00.000Z", + }) + .mockReturnValueOnce({ + status: "verified" as const, + cudaVerified: true, + at: "2026-08-06T00:00:00.000Z", + }) + .mockReturnValueOnce({ + status: "verified" as const, + cudaVerified: true, + at: "2026-08-06T00:00:00.000Z", + }); + const log = vi.spyOn(console, "log").mockImplementation(() => undefined); + const dockerRun = vi.fn((args: readonly string[]) => + args.includes("0") + ? { status: 0, stdout: "/dev/nvmap\n/dev/special-gpu\n", stderr: "" } + : { status: 0, stdout: "cuInit(0)=0", stderr: "" }, + ); + + maybeRunJetsonOpenRmPolicyProof({ + backend: "jetson", + enabled: true, + failure: new Error("cuInit(0)=801"), + preserveJetsonDeviceGroupMembership: true, + result: result(), + sandboxName: "alpha", + verifyDirectSandboxGpu, + deps: { + dockerRun, + runCaptureOpenshell: vi.fn(() => BASE_POLICY), + runOpenshell, + }, + }); + + expect(appliedPolicies).toHaveLength(4); + expect(appliedPolicies[0]).toContain("- /sys"); + expect(appliedPolicies[1]).toContain("/dev/special-gpu"); + expect(appliedPolicies[2]).toContain("/dev/special-gpu"); + expect(appliedPolicies[3]).not.toContain("/dev/special-gpu"); + expect(log).toHaveBeenCalledWith(expect.stringContaining("/dev/special-gpu")); + }); + + it("restores the baseline when the candidate CUDA proof throws", () => { + const appliedPolicies: string[] = []; + const runOpenshell = vi.fn((args: string[]) => { + appliedPolicies.push(fs.readFileSync(args[3] ?? "", "utf8")); + return { status: 0 }; + }); + + expect(() => + maybeRunJetsonOpenRmPolicyProof({ + backend: "jetson", + enabled: true, + failure: new Error("cuInit(0)=801"), + preserveJetsonDeviceGroupMembership: true, + result: result(), + sandboxName: "alpha", + verifyDirectSandboxGpu: vi.fn(() => { + throw new Error("candidate probe failed"); + }), + deps: { + dockerRun: dockerRunForBoundaryProof(), + runCaptureOpenshell: vi.fn(() => BASE_POLICY), + runOpenshell, + }, + }), + ).toThrow("candidate probe failed"); + + expect(appliedPolicies).toHaveLength(2); + expect(appliedPolicies[0]).toContain("/dev/nvidia-caps/nvidia-cap2"); + expect(appliedPolicies[1]).not.toContain("/dev/nvidia-caps/nvidia-cap2"); + }); + + it("attempts baseline restoration when candidate policy application reports failure", () => { + const appliedPolicies: string[] = []; + const runOpenshell = vi.fn((args: string[]) => { + appliedPolicies.push(fs.readFileSync(args[3] ?? "", "utf8")); + return { status: appliedPolicies.length === 1 ? 1 : 0 }; + }); + + expect(() => + maybeRunJetsonOpenRmPolicyProof({ + backend: "jetson", + enabled: true, + failure: new Error("cuInit(0)=801"), + preserveJetsonDeviceGroupMembership: true, + result: result(), + sandboxName: "alpha", + verifyDirectSandboxGpu: vi.fn(), + deps: { + dockerRun: dockerRunForBoundaryProof(), + runCaptureOpenshell: vi.fn(() => BASE_POLICY), + runOpenshell, + }, + }), + ).toThrow("devices.yaml"); + + expect(appliedPolicies).toHaveLength(2); + expect(appliedPolicies[0]).toContain("/dev/nvidia-caps/nvidia-cap2"); + expect(appliedPolicies[1]).not.toContain("/dev/nvidia-caps/nvidia-cap2"); + }); + + it("preserves a candidate failure and cleans temporary files when baseline restoration fails", () => { + const candidateError = new Error("candidate probe failed"); + let temporaryDirectory = ""; + let policySetCount = 0; + const runOpenshell = vi.fn((args: string[]) => { + const policyPath = args[3] ?? ""; + temporaryDirectory = path.dirname(policyPath); + policySetCount += 1; + return { status: policySetCount === 1 ? 0 : 1 }; + }); + + let failure: unknown; + try { + maybeRunJetsonOpenRmPolicyProof({ + backend: "jetson", + enabled: true, + failure: new Error("cuInit(0)=801"), + preserveJetsonDeviceGroupMembership: true, + result: result(), + sandboxName: "alpha", + verifyDirectSandboxGpu: vi.fn(() => { + throw candidateError; + }), + deps: { + dockerRun: dockerRunForBoundaryProof(), + runCaptureOpenshell: vi.fn(() => BASE_POLICY), + runOpenshell, + }, + }); + } catch (error) { + failure = error; + } + + expect(failure).toBeInstanceOf(JetsonOpenRmPolicyRestorationError); + expect((failure as JetsonOpenRmPolicyRestorationError).candidateError).toBe(candidateError); + expect((failure as JetsonOpenRmPolicyRestorationError).restorationError).toEqual( + expect.objectContaining({ message: expect.stringContaining("baseline.yaml") }), + ); + expect((failure as JetsonOpenRmPolicyRestorationError).cleanupError).toBeNull(); + expect(temporaryDirectory).not.toBe(""); + expect(fs.existsSync(temporaryDirectory)).toBe(false); + }); + + it("does nothing outside the exact Jetson cuInit 801 failure", () => { + const dockerRun = vi.fn(); + maybeRunJetsonOpenRmPolicyProof({ + backend: "jetson", + enabled: true, + failure: new Error("cuInit(0)=100"), + preserveJetsonDeviceGroupMembership: true, + result: result(), + sandboxName: "alpha", + verifyDirectSandboxGpu: vi.fn(), + deps: { dockerRun }, + }); + expect(dockerRun).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/onboard/diagnostics/jetson-openrm-proof.ts b/src/lib/onboard/diagnostics/jetson-openrm-proof.ts new file mode 100644 index 00000000000..06b393172fb --- /dev/null +++ b/src/lib/onboard/diagnostics/jetson-openrm-proof.ts @@ -0,0 +1,463 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import YAML from "yaml"; +import { parseOpenShellPolicy } from "../../policy/merge"; +import type { SandboxGpuProofResult } from "../../state/registry"; +import { dockerRun as defaultDockerRun } from "../docker-gpu-patch"; +import type { + DockerGpuPatchBackend, + DockerGpuPatchDeps, + DockerGpuPatchResult, +} from "../docker-gpu-patch-types"; +import { runJetsonOpenRmNamespaceProof } from "./jetson-openrm-namespace-proof"; +import { runJetsonOpenRmProcessProof } from "./jetson-openrm-process-proof"; + +const CUDA_RESULT_PATTERN = /cuInit\(0\)=(-?\d+)/u; +const PROOF_TIMEOUT_MS = 30_000; +const SYSFS_ROOT = "/sys"; +const MAX_CHARACTER_DEVICE_PATHS = 256; +const CUDA_PROBE = [ + "import ctypes", + 'lib = ctypes.CDLL("libcuda.so.1")', + "lib.cuInit.argtypes = [ctypes.c_uint]", + "lib.cuInit.restype = ctypes.c_int", + "rc = lib.cuInit(0)", + 'print(f"cuInit(0)={rc}")', + "raise SystemExit(0 if rc == 0 else 1)", +].join("; "); +const CHARACTER_DEVICE_DISCOVERY_PROBE = [ + "import os, stat", + "for root, dirs, files in os.walk('/dev'):", + " for name in files:", + " device = os.path.join(root, name)", + " try:", + " if stat.S_ISCHR(os.lstat(device).st_mode): print(device)", + " except OSError:", + " pass", +].join("\n"); +const PROCESS_STATUS_PROBE = + "grep -E '^(Uid|Gid|Groups|CapInh|CapPrm|CapEff|CapBnd|CapAmb|NoNewPrivs|Seccomp|Seccomp_filters):' /proc/self/status"; +const CHARACTER_DEVICE_PATH_PATTERN = /^\/dev(?:\/[A-Za-z0-9._-]+)+$/u; +const GPU_DEVICE_PATH_PATTERN = + /^\/dev\/(?:nvidia[A-Za-z0-9._/-]*|nvhost[A-Za-z0-9._/-]*|nvgpu(?:\/[A-Za-z0-9._-]+)*|nvsci[A-Za-z0-9._/-]*|tegra[A-Za-z0-9._/-]*|nvmap|dri\/(?:renderD|card)\d+)$/u; +const FILESYSTEM_PATH_CANDIDATES: readonly PolicyCandidate[] = [ + { name: "dev-shm-read", readOnly: ["/dev/shm"], readWrite: [] }, + { name: "dev-shm-read-write", readOnly: [], readWrite: ["/dev/shm"] }, + { name: "run-read", readOnly: ["/run"], readWrite: [] }, + { name: "run-read-write", readOnly: [], readWrite: ["/run"] }, + { name: "var-read", readOnly: ["/var"], readWrite: [] }, + { name: "var-read-write", readOnly: [], readWrite: ["/var"] }, + { name: "home-read", readOnly: ["/home"], readWrite: [] }, + { name: "opt-read", readOnly: ["/opt"], readWrite: [] }, + { name: "mnt-read", readOnly: ["/mnt"], readWrite: [] }, + { name: "media-read", readOnly: ["/media"], readWrite: [] }, + { name: "srv-read", readOnly: ["/srv"], readWrite: [] }, +]; + +type OpenRmProofDeps = Pick< + DockerGpuPatchDeps, + "dockerRun" | "runCaptureOpenshell" | "runOpenshell" +>; + +type OpenRmProofOptions = { + backend?: DockerGpuPatchBackend; + enabled?: boolean; + failure: Error; + preserveJetsonDeviceGroupMembership?: boolean; + result: DockerGpuPatchResult | null; + sandboxName: string; + verifyDirectSandboxGpu: (sandboxName: string) => SandboxGpuProofResult; + deps: OpenRmProofDeps; +}; + +type PolicyDocument = { + filesystem_policy?: { + read_only?: unknown; + read_write?: unknown; + }; +}; + +type PolicyCandidate = { + name: string; + readOnly: string[]; + readWrite: string[]; +}; + +export class JetsonOpenRmPolicyRestorationError extends Error { + readonly candidateError: unknown | null; + readonly restorationError: unknown; + readonly cleanupError: unknown | null; + + constructor(options: { + candidateError: unknown | null; + restorationError: unknown; + cleanupError: unknown | null; + }) { + const detail = + options.restorationError instanceof Error + ? options.restorationError.message + : String(options.restorationError); + super(`NemoClaw could not confirm that OpenShell restored the baseline policy: ${detail}`); + this.name = "JetsonOpenRmPolicyRestorationError"; + this.candidateError = options.candidateError; + this.restorationError = options.restorationError; + this.cleanupError = options.cleanupError; + } +} + +function cudaResult(value: string): string { + return value.match(CUDA_RESULT_PATTERN)?.[1] ?? "missing"; +} + +function proofCudaResult(proof: SandboxGpuProofResult): string { + if (proof.status === "verified" && proof.cudaVerified) return "0"; + return cudaResult(proof.detail ?? ""); +} + +function setPolicy( + sandboxName: string, + policyPath: string, + runOpenshell: NonNullable, +): void { + const result = runOpenshell(["policy", "set", "--policy", policyPath, "--wait", sandboxName], { + ignoreError: true, + suppressOutput: true, + timeout: PROOF_TIMEOUT_MS, + }); + if (result.status !== 0) { + throw new Error(`OpenShell rejected diagnostic policy file ${policyPath}.`); + } +} + +function parseFilesystemPolicy(policyYaml: string): { + policy: PolicyDocument; + readOnly: string[]; + readWrite: string[]; +} { + const policy = YAML.parse(policyYaml) as PolicyDocument | null; + const filesystemPolicy = policy?.filesystem_policy; + if (!filesystemPolicy || typeof filesystemPolicy !== "object") { + throw new Error("OpenShell base policy has no filesystem_policy mapping."); + } + if (!Array.isArray(filesystemPolicy.read_only) || !Array.isArray(filesystemPolicy.read_write)) { + throw new Error("OpenShell base policy filesystem policy paths are not lists."); + } + return { + policy, + readOnly: filesystemPolicy.read_only.map(String), + readWrite: filesystemPolicy.read_write.map(String), + }; +} + +function candidatePolicy(policyYaml: string, candidate: PolicyCandidate): string { + const { policy, readOnly, readWrite } = parseFilesystemPolicy(policyYaml); + const filesystemPolicy = policy.filesystem_policy; + if (!filesystemPolicy) throw new Error("OpenShell base policy has no filesystem_policy mapping."); + const readWriteSet = new Set(readWrite); + for (const devicePath of candidate.readWrite) readWriteSet.add(devicePath); + const readOnlySet = new Set(readOnly.filter((policyPath) => !readWriteSet.has(policyPath))); + for (const policyPath of candidate.readOnly) { + if (!readWriteSet.has(policyPath)) readOnlySet.add(policyPath); + } + filesystemPolicy.read_only = [...readOnlySet]; + filesystemPolicy.read_write = [...readWriteSet]; + return YAML.stringify(policy); +} + +function discoverCharacterDevicePaths( + containerId: string, + dockerRun: NonNullable, +): string[] { + const result = dockerRun( + ["exec", "--user", "0", containerId, "python3", "-c", CHARACTER_DEVICE_DISCOVERY_PROBE], + { ignoreError: true, suppressOutput: true, timeout: PROOF_TIMEOUT_MS }, + ); + if (result.status !== 0) throw new Error("Could not enumerate container character devices."); + const devicePaths = [ + ...new Set( + String(result.stdout ?? "") + .split(/\r?\n/u) + .map((devicePath) => devicePath.trim()) + .filter((devicePath) => CHARACTER_DEVICE_PATH_PATTERN.test(devicePath)), + ), + ].sort(); + if (devicePaths.length === 0 || devicePaths.length > MAX_CHARACTER_DEVICE_PATHS) { + throw new Error("Container character-device enumeration is empty or excessive."); + } + return devicePaths; +} + +function policyPathCovers(policyPath: string, targetPath: string): boolean { + return policyPath === "/" || targetPath === policyPath || targetPath.startsWith(`${policyPath}/`); +} + +function compactProcessStatus(value: string | Buffer | null | undefined): string { + return String(value ?? "") + .trim() + .split(/\r?\n/u) + .join("; "); +} + +/** + * Maintainer-only hardware A/B for issue #7610. The caller invokes this after + * the live OpenShell CUDA proof returns 801 and before it rolls the exact + * replacement container back. The baseline policy is restored in `finally`. + */ +export function maybeRunJetsonOpenRmPolicyProof(options: OpenRmProofOptions): void { + const enabled = options.enabled ?? process.env.NEMOCLAW_DIAGNOSE_JETSON_OPENRM_POLICY === "1"; + if ( + !enabled || + options.backend !== "jetson" || + options.preserveJetsonDeviceGroupMembership !== true || + options.result?.mode.kind !== "nvidia-runtime" || + !/cuInit\(0\)=801/u.test(options.failure.message) + ) { + return; + } + + const dockerRun = options.deps.dockerRun ?? defaultDockerRun; + const { runCaptureOpenshell: captureOpenshell, runOpenshell } = options.deps; + if (!captureOpenshell || !runOpenshell) { + console.error(" OpenRM A/B inconclusive: required OpenShell adapters are unavailable."); + return; + } + const direct = dockerRun( + ["exec", "--user", "sandbox", options.result.newContainerId, "python3", "-c", CUDA_PROBE], + { ignoreError: true, suppressOutput: true, timeout: PROOF_TIMEOUT_MS }, + ); + const directOutput = `${direct.stderr ?? ""}\n${direct.stdout ?? ""}`; + const directResult = cudaResult(directOutput); + const characterDevicePaths = discoverCharacterDevicePaths( + options.result.newContainerId, + dockerRun, + ); + const injectedDevicePaths = characterDevicePaths.filter((devicePath) => + GPU_DEVICE_PATH_PATTERN.test(devicePath), + ); + const rawPolicy = captureOpenshell(["policy", "get", "--base", options.sandboxName], { + ignoreError: false, + timeout: PROOF_TIMEOUT_MS, + }); + const baselinePolicy = parseOpenShellPolicy(rawPolicy).yamlBody; + if (!baselinePolicy) throw new Error("OpenShell returned no round-trippable base policy."); + const baselineFilesystemPolicy = parseFilesystemPolicy(baselinePolicy); + const missingDevicePaths = injectedDevicePaths.filter( + (devicePath) => + !baselineFilesystemPolicy.readWrite.some((policyPath) => + policyPathCovers(policyPath, devicePath), + ), + ); + const missingCharacterDevicePaths = characterDevicePaths.filter( + (devicePath) => + !baselineFilesystemPolicy.readWrite.some((policyPath) => + policyPathCovers(policyPath, devicePath), + ), + ); + const sysfsMissing = + !baselineFilesystemPolicy.readOnly.includes(SYSFS_ROOT) && + !baselineFilesystemPolicy.readWrite.includes(SYSFS_ROOT); + const candidates: PolicyCandidate[] = []; + if (missingDevicePaths.length > 0) { + candidates.push({ name: "devices", readOnly: [], readWrite: missingDevicePaths }); + } + if (sysfsMissing) { + candidates.push({ name: "sysfs", readOnly: [SYSFS_ROOT], readWrite: [] }); + } + if (missingDevicePaths.length > 0 && sysfsMissing) { + candidates.push({ + name: "devices-plus-sysfs", + readOnly: [SYSFS_ROOT], + readWrite: missingDevicePaths, + }); + } + const temporaryDirectory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-openrm-proof-")); + const baselinePath = path.join(temporaryDirectory, "baseline.yaml"); + fs.writeFileSync(baselinePath, baselinePolicy, { encoding: "utf8", mode: 0o600 }); + + const candidateResults = new Map(); + const individualCharacterDeviceResults = new Map(); + let candidateFailure: { readonly error: unknown } | null = null; + try { + const applyCandidate = (candidate: PolicyCandidate): string => { + const candidatePath = path.join(temporaryDirectory, `${candidate.name}.yaml`); + fs.writeFileSync(candidatePath, candidatePolicy(baselinePolicy, candidate), { + encoding: "utf8", + mode: 0o600, + }); + setPolicy(options.sandboxName, candidatePath, runOpenshell); + const result = proofCudaResult(options.verifyDirectSandboxGpu(options.sandboxName)); + candidateResults.set(candidate.name, result); + return result; + }; + for (const candidate of candidates) applyCandidate(candidate); + + if (candidateResults.get("devices") === "0") { + const onlyPath = missingDevicePaths.length === 1 ? missingDevicePaths[0] : null; + if (onlyPath) { + individualCharacterDeviceResults.set(onlyPath, "0"); + } else { + for (const [index, devicePath] of missingDevicePaths.entries()) { + const result = applyCandidate({ + name: `gpu-device-${String(index)}`, + readOnly: [], + readWrite: [devicePath], + }); + individualCharacterDeviceResults.set(devicePath, result); + } + } + } + + if (![...candidateResults.values()].includes("0") && missingCharacterDevicePaths.length > 0) { + const allCharacterDevicesResult = applyCandidate({ + name: "all-character-devices", + readOnly: [], + readWrite: missingCharacterDevicePaths, + }); + if (allCharacterDevicesResult === "0") { + for (const [index, devicePath] of missingCharacterDevicePaths.entries()) { + const result = applyCandidate({ + name: `character-device-${String(index)}`, + readOnly: [], + readWrite: [devicePath], + }); + individualCharacterDeviceResults.set(devicePath, result); + } + } + } + + if (![...candidateResults.values()].includes("0")) { + for (const candidate of FILESYSTEM_PATH_CANDIDATES) applyCandidate(candidate); + } + + if (![...candidateResults.values()].includes("0")) { + applyCandidate({ name: "read-root", readOnly: ["/"], readWrite: [] }); + if (candidateResults.get("read-root") !== "0") { + applyCandidate({ name: "read-write-root", readOnly: [], readWrite: ["/"] }); + } + } + } catch (error) { + candidateFailure = { error }; + } + let restorationFailure: { readonly error: unknown } | null = null; + try { + setPolicy(options.sandboxName, baselinePath, runOpenshell); + } catch (error) { + restorationFailure = { error }; + } + let cleanupFailure: { readonly error: unknown } | null = null; + try { + fs.rmSync(temporaryDirectory, { recursive: true, force: true }); + } catch (error) { + cleanupFailure = { error }; + } + if (restorationFailure !== null) { + throw new JetsonOpenRmPolicyRestorationError({ + candidateError: candidateFailure?.error ?? null, + restorationError: restorationFailure.error, + cleanupError: cleanupFailure?.error ?? null, + }); + } + if (candidateFailure !== null) throw candidateFailure.error; + if (cleanupFailure !== null) throw cleanupFailure.error; + + const deviceResult = candidateResults.get("devices") ?? "not-tested"; + const sysfsResult = candidateResults.get("sysfs") ?? "not-tested"; + const combinedResult = candidateResults.get("devices-plus-sysfs") ?? "not-tested"; + const allCharacterDevicesResult = candidateResults.get("all-character-devices") ?? "not-tested"; + const readRootResult = candidateResults.get("read-root") ?? "not-tested"; + const readWriteRootResult = candidateResults.get("read-write-root") ?? "not-tested"; + const isolatedCharacterDevices = [...individualCharacterDeviceResults] + .filter(([, result]) => result === "0") + .map(([devicePath]) => devicePath); + const passingFilesystemCandidates = FILESYSTEM_PATH_CANDIDATES.filter( + (candidate) => candidateResults.get(candidate.name) === "0", + ); + const isolatedFilesystemCandidates = passingFilesystemCandidates.filter((candidate) => { + const writePath = candidate.readWrite[0]; + return ( + !writePath || !passingFilesystemCandidates.some((other) => other.readOnly.includes(writePath)) + ); + }); + + console.log(""); + console.log(" === Jetson OpenRM policy boundary matrix ==="); + console.log(` injected_gpu_devices=${injectedDevicePaths.join(",")}`); + console.log(` policy_missing_gpu_devices=${missingDevicePaths.join(",") || "none"}`); + console.log( + ` policy_missing_character_devices=${missingCharacterDevicePaths.join(",") || "none"}`, + ); + console.log( + ` direct_docker_cuInit=${directResult} baseline_openshell_cuInit=801 devices_cuInit=${deviceResult} sysfs_cuInit=${sysfsResult} devices_plus_sysfs_cuInit=${combinedResult} all_character_devices_cuInit=${allCharacterDevicesResult} read_root_cuInit=${readRootResult} read_write_root_cuInit=${readWriteRootResult}`, + ); + if (FILESYSTEM_PATH_CANDIDATES.some((candidate) => candidateResults.has(candidate.name))) { + console.log( + ` filesystem_candidate_cuInit=${FILESYSTEM_PATH_CANDIDATES.map((candidate) => `${candidate.name}:${candidateResults.get(candidate.name) ?? "not-tested"}`).join(",")}`, + ); + } + if (directResult === "0" && isolatedCharacterDevices.length > 0) { + console.log( + ` ISOLATED: OpenShell policy is missing CUDA-required character device(s): ${isolatedCharacterDevices.join(",")}.`, + ); + } else if (directResult === "0" && allCharacterDevicesResult === "0") { + console.log( + " ISOLATED: CUDA requires a combination of character devices currently missing from OpenShell policy.", + ); + } else if (directResult === "0" && deviceResult === "0" && sysfsResult !== "0") { + console.log( + " ISOLATED: OpenShell policy is missing one or more NVIDIA/Tegra character devices; no sysfs grant is required.", + ); + } else if (directResult === "0" && sysfsResult === "0" && deviceResult !== "0") { + console.log( + " ISOLATED: OpenShell policy is missing CUDA-required sysfs visibility; exact sysfs paths still need narrowing.", + ); + } else if (directResult === "0" && combinedResult === "0") { + console.log( + " ISOLATED: CUDA requires both the missing GPU devices and sysfs visibility through OpenShell.", + ); + } else if (directResult === "0" && isolatedFilesystemCandidates.length > 0) { + console.log( + ` ISOLATED: CUDA requires additional Landlock path access: ${isolatedFilesystemCandidates.map((candidate) => `${candidate.readWrite.length > 0 ? "read-write" : "read-only"}:${candidate.readWrite[0] ?? candidate.readOnly[0]}`).join(",")}.`, + ); + } else if (directResult === "0" && readRootResult === "0") { + console.log( + " ISOLATED: Landlock is missing CUDA-required read access outside the tested GPU devices and /sys.", + ); + } else if (directResult === "0" && readWriteRootResult === "0") { + console.log( + " ISOLATED: Landlock is missing CUDA-required write access outside the tested character devices.", + ); + } else { + console.error( + " INCONCLUSIVE: the filesystem-policy matrix did not restore CUDA; no production policy change is justified.", + ); + const directStatus = dockerRun( + [ + "exec", + "--user", + "sandbox", + options.result.newContainerId, + "sh", + "-lc", + PROCESS_STATUS_PROBE, + ], + { ignoreError: true, suppressOutput: true, timeout: PROOF_TIMEOUT_MS }, + ); + const openshellStatus = runOpenshell( + ["sandbox", "exec", "-n", options.sandboxName, "--", "sh", "-lc", PROCESS_STATUS_PROBE], + { ignoreError: true, suppressOutput: true, timeout: PROOF_TIMEOUT_MS }, + ); + console.log(` direct_process_status=${compactProcessStatus(directStatus.stdout)}`); + console.log(` openshell_process_status=${compactProcessStatus(openshellStatus.stdout)}`); + const processModelPasses = runJetsonOpenRmProcessProof( + options.result.newContainerId, + dockerRun, + ); + if (processModelPasses && readWriteRootResult !== "0") { + runJetsonOpenRmNamespaceProof(options.result.newContainerId, dockerRun); + } + } +} diff --git a/src/lib/onboard/diagnostics/jetson-openrm-standalone.test.ts b/src/lib/onboard/diagnostics/jetson-openrm-standalone.test.ts new file mode 100644 index 00000000000..c2fa5950f46 --- /dev/null +++ b/src/lib/onboard/diagnostics/jetson-openrm-standalone.test.ts @@ -0,0 +1,100 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; +import { runStandaloneJetsonOpenRmPolicyProof } from "./jetson-openrm-standalone"; + +function verifierFactory() { + return vi.fn(() => vi.fn()); +} + +describe("standalone Jetson OpenRM policy proof", () => { + it("runs the production recreation boundary and restores it after cuInit 801 (#7610)", async () => { + const policyProofEnv = "NEMOCLAW_DIAGNOSE_JETSON_OPENRM_POLICY"; + const previousPolicyProofSetting = process.env[policyProofEnv]; + let observedPolicyProofSetting: string | undefined; + const rollback = vi.fn(async () => undefined); + const createPatch = vi.fn(() => ({ + ensureApplied: vi.fn(async () => undefined), + waitForSupervisorReconnectIfNeeded: vi.fn(), + verifyGpuOrExit: vi.fn(async () => { + observedPolicyProofSetting = process.env[policyProofEnv]; + throw new Error("Sandbox GPU proof returned failed status (cuInit(0)=801)"); + }), + rollbackManagedStartupAfterCreateFailure: rollback, + })); + + await runStandaloneJetsonOpenRmPolicyProof("alpha", { + createPatch, + createVerifier: verifierFactory(), + runCaptureOpenshell: vi.fn(), + runOpenshell: vi.fn(), + }); + + expect(createPatch).toHaveBeenCalledWith( + expect.objectContaining({ + route: "compatibility", + sandboxName: "alpha", + backend: "jetson", + preserveJetsonDeviceGroupMembership: true, + }), + ); + expect(observedPolicyProofSetting).toBe("1"); + expect(process.env[policyProofEnv]).toBe(previousPolicyProofSetting); + expect(rollback).toHaveBeenCalledOnce(); + }); + + it("restores the original container after an unexpected proof failure (#7610)", async () => { + const rollback = vi.fn(async () => undefined); + const createPatch = vi.fn(() => ({ + ensureApplied: vi.fn(async () => undefined), + waitForSupervisorReconnectIfNeeded: vi.fn(), + verifyGpuOrExit: vi.fn(async () => { + throw new Error("cuInit(0)=100"); + }), + rollbackManagedStartupAfterCreateFailure: rollback, + })); + + await expect( + runStandaloneJetsonOpenRmPolicyProof("alpha", { + createPatch, + createVerifier: verifierFactory(), + runCaptureOpenshell: vi.fn(), + runOpenshell: vi.fn(), + }), + ).rejects.toThrow("cuInit(0)=100"); + expect(rollback).toHaveBeenCalledOnce(); + }); + + it("does not accept cuInit 801 when the production rollback failed (#7610)", async () => { + const rollbackError = new Error("pre-patch container was not restored"); + const proofError = Object.assign(new Error("cuInit(0)=801"), { + managedBootstrapRollbackError: rollbackError, + }); + const createPatch = vi.fn(() => ({ + ensureApplied: vi.fn(async () => undefined), + waitForSupervisorReconnectIfNeeded: vi.fn(), + verifyGpuOrExit: vi.fn(async () => { + throw proofError; + }), + rollbackManagedStartupAfterCreateFailure: vi.fn(async () => undefined), + })); + + await expect( + runStandaloneJetsonOpenRmPolicyProof("alpha", { + createPatch, + createVerifier: verifierFactory(), + runCaptureOpenshell: vi.fn(), + runOpenshell: vi.fn(), + }), + ).rejects.toThrow("cuInit(0)=801"); + }); + + it("rejects an invalid sandbox name before creating a patch (#7610)", async () => { + const createPatch = vi.fn(); + await expect( + runStandaloneJetsonOpenRmPolicyProof("alpha;docker ps", { createPatch }), + ).rejects.toThrow("Invalid sandbox name"); + expect(createPatch).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/onboard/diagnostics/jetson-openrm-standalone.ts b/src/lib/onboard/diagnostics/jetson-openrm-standalone.ts new file mode 100644 index 00000000000..07d96b6f607 --- /dev/null +++ b/src/lib/onboard/diagnostics/jetson-openrm-standalone.ts @@ -0,0 +1,137 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { SandboxGpuProofResult } from "../../state/registry"; +import { createDockerGpuDiagnosticRedactor } from "../docker-gpu-diagnostic-redaction"; +import type { DockerGpuPatchDeps } from "../docker-gpu-patch-types"; +import { createDockerGpuSandboxCreatePatch } from "../docker-gpu-sandbox-create"; +import { createOpenshellCliHelpers } from "../openshell-cli"; +import { createDirectSandboxGpuVerifier } from "../sandbox-gpu-preflight"; + +const SANDBOX_NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_.-]*$/u; +const CUINIT_801_PATTERN = /cuInit\(0\)=801/u; +const RECREATE_TIMEOUT_SECS = 180; +const OPENRM_POLICY_PROOF_ENV = "NEMOCLAW_DIAGNOSE_JETSON_OPENRM_POLICY"; + +type SandboxPatch = Pick< + ReturnType, + | "ensureApplied" + | "rollbackManagedStartupAfterCreateFailure" + | "verifyGpuOrExit" + | "waitForSupervisorReconnectIfNeeded" +>; + +type StandaloneProofDeps = { + createPatch?: (options: Parameters[0]) => SandboxPatch; + createVerifier?: typeof createDirectSandboxGpuVerifier; + runCaptureOpenshell?: NonNullable; + runOpenshell?: NonNullable; +}; + +function compactText(value: string): string { + return String(value).replace(/\s+/gu, " ").trim(); +} + +function liveOpenShellRunners(): Pick< + ReturnType, + "runCaptureOpenshell" | "runOpenshell" +> { + let cachedBinary: string | null = null; + const helpers = createOpenshellCliHelpers({ + getCachedBinary: () => cachedBinary, + setCachedBinary: (binary) => { + cachedBinary = binary; + }, + getGatewayPort: () => 0, + getDockerDriverGatewayEndpoint: () => "", + }); + return { + runCaptureOpenshell: helpers.runCaptureOpenshell, + runOpenshell: helpers.runOpenshell, + }; +} + +/** + * Run the production Jetson Docker recreation, OpenShell CUDA proof, and + * OpenRM policy matrix without entering the onboarding state machine. The + * production rollback path restores the original container on every exit. + */ +export async function runStandaloneJetsonOpenRmPolicyProof( + sandboxName: string, + deps: StandaloneProofDeps = {}, +): Promise { + if (!SANDBOX_NAME_PATTERN.test(sandboxName)) { + throw new Error(`Invalid sandbox name: ${sandboxName}`); + } + + const liveRunners = deps.runOpenshell && deps.runCaptureOpenshell ? null : liveOpenShellRunners(); + const run = deps.runOpenshell ?? liveRunners?.runOpenshell; + const capture = deps.runCaptureOpenshell ?? liveRunners?.runCaptureOpenshell; + if (!run || !capture) throw new Error("OpenShell command runners are unavailable."); + const createVerifier = deps.createVerifier ?? createDirectSandboxGpuVerifier; + const redactor = createDockerGpuDiagnosticRedactor(); + const verifyGpu = createVerifier({ + runOpenshell: run, + compactText, + redact: (value) => redactor.redactText(String(value ?? "")), + detectNvidiaPlatform: () => "jetson", + }); + const createPatch = deps.createPatch ?? createDockerGpuSandboxCreatePatch; + const patch = createPatch({ + route: "compatibility", + sandboxName, + timeoutSecs: RECREATE_TIMEOUT_SECS, + backend: "jetson", + preserveJetsonDeviceGroupMembership: true, + deps: { + runCaptureOpenshell: capture, + runOpenshell: run, + }, + }); + + let proof: SandboxGpuProofResult | null = null; + let expectedBoundaryFailure = false; + let rollbackFailure: unknown = null; + const previousPolicyProofSetting = process.env[OPENRM_POLICY_PROOF_ENV]; + try { + await patch.ensureApplied(); + patch.waitForSupervisorReconnectIfNeeded(); + process.env[OPENRM_POLICY_PROOF_ENV] = "1"; + try { + proof = await patch.verifyGpuOrExit(verifyGpu); + } catch (error) { + rollbackFailure = + error && typeof error === "object" + ? (error as { managedBootstrapRollbackError?: unknown }).managedBootstrapRollbackError + : null; + if (rollbackFailure) throw error; + const message = error instanceof Error ? error.message : String(error); + if (!CUINIT_801_PATTERN.test(message)) throw error; + expectedBoundaryFailure = true; + } + } finally { + if (previousPolicyProofSetting === undefined) { + delete process.env[OPENRM_POLICY_PROOF_ENV]; + } else { + process.env[OPENRM_POLICY_PROOF_ENV] = previousPolicyProofSetting; + } + await patch.rollbackManagedStartupAfterCreateFailure(); + if (!rollbackFailure) { + console.log(" ✓ Original sandbox container restored after the standalone proof."); + } + } + + if (expectedBoundaryFailure) { + console.log( + " ✓ Reproduced the cuInit(0)=801 OpenShell boundary; use the policy matrix above as the result.", + ); + return; + } + if (proof?.status === "verified" && proof.cudaVerified) { + console.log( + " ✓ CUDA already passes through OpenShell; the cuInit(0)=801 boundary did not reproduce.", + ); + return; + } + throw new Error("The standalone run did not execute a conclusive CUDA proof."); +} diff --git a/src/lib/onboard/docker-gpu-jetson-groups.test.ts b/src/lib/onboard/docker-gpu-jetson-groups.test.ts index 90b6dfb5446..c6cacdd6fee 100644 --- a/src/lib/onboard/docker-gpu-jetson-groups.test.ts +++ b/src/lib/onboard/docker-gpu-jetson-groups.test.ts @@ -5,7 +5,37 @@ import fs from "node:fs"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { detectTegraDeviceGroupGids } from "./docker-gpu-jetson-groups"; +import { detectTegraDeviceGroupGids, detectTegraGpuDevicePaths } from "./docker-gpu-jetson-groups"; + +describe("detectTegraGpuDevicePaths", () => { + it("returns only existing character devices without following symlinks (#7610)", () => { + const paths = ["/dev/nvmap", "/dev/nvhost-gpu", "/dev/nvgpu/igpu0/link"]; + + expect( + detectTegraGpuDevicePaths({ + listDevicePaths: () => paths, + statDevicePath: (devicePath) => + devicePath === "/dev/nvmap" + ? { isCharacterDevice: true, isSymbolicLink: false } + : devicePath === "/dev/nvgpu/igpu0/link" + ? { isCharacterDevice: true, isSymbolicLink: true } + : null, + }), + ).toEqual(["/dev/nvmap"]); + }); + + it("requires a character device at /dev/nvmap before returning DRI render devices (#7610)", () => { + expect( + detectTegraGpuDevicePaths({ + listDevicePaths: () => ["/dev/nvmap", "/dev/dri/renderD128"], + statDevicePath: (devicePath) => ({ + isCharacterDevice: devicePath === "/dev/dri/renderD128", + isSymbolicLink: false, + }), + }), + ).toEqual([]); + }); +}); describe("detectTegraDeviceGroupGids", () => { afterEach(() => { diff --git a/src/lib/onboard/docker-gpu-jetson-groups.ts b/src/lib/onboard/docker-gpu-jetson-groups.ts index 4cfcd098f47..c8e6decf2d6 100644 --- a/src/lib/onboard/docker-gpu-jetson-groups.ts +++ b/src/lib/onboard/docker-gpu-jetson-groups.ts @@ -16,6 +16,7 @@ const TEGRA_GPU_DEVICE_NODES = [ "/dev/nvgpu/igpu0/as", "/dev/nvgpu/igpu0/prof", ] as const; +const NVMAP_DEVICE = "/dev/nvmap"; const READ_WRITE_PERMISSION_BITS = 0o6; const MAX_DOCKER_SUPPLEMENTARY_GID = 2_147_483_647; @@ -24,6 +25,11 @@ type DeviceGroupAccess = { mode: number; }; +type DevicePathAccess = { + isCharacterDevice: boolean; + isSymbolicLink: boolean; +}; + /** * Find real DRI render character devices without following symlinks or * scanning other DRI device families. @@ -51,16 +57,50 @@ function listTegraGpuDevicePaths(): string[] { return [...TEGRA_GPU_DEVICE_NODES, ...discoverTegraRenderDevicePaths()]; } +/** + * Require a non-symlink /dev/nvmap character device before returning the + * curated paths for the OpenShell filesystem policy. + */ +export function detectTegraGpuDevicePaths( + deps: { + statDevicePath?: (path: string) => DevicePathAccess | null; + listDevicePaths?: () => string[]; + } = {}, +): string[] { + const devicePaths = deps.listDevicePaths?.() ?? listTegraGpuDevicePaths(); + const statDevicePath = + deps.statDevicePath ?? + ((devicePath: string): DevicePathAccess | null => { + try { + const stat = fs.lstatSync(devicePath); + return { + isCharacterDevice: stat.isCharacterDevice(), + isSymbolicLink: stat.isSymbolicLink(), + }; + } catch { + return null; + } + }); + + const detectedPaths = devicePaths.filter((devicePath) => { + const access = statDevicePath(devicePath); + return access?.isCharacterDevice === true && access.isSymbolicLink === false; + }); + return detectedPaths.includes(NVMAP_DEVICE) ? detectedPaths : []; +} + /** * Source-of-truth boundary for Jetson/Tegra supplementary device groups: * - * - Invalid state: the non-root sandbox user can see `/dev/nvmap` and `/dev/nvhost-*` but cannot - * open them because Docker did not copy their host-owned supplementary GIDs into the container. + * - Invalid state: the non-root sandbox user can see `/dev/nvmap` and `/dev/nvhost-*` but loses + * access when OpenShell rebuilds supplementary groups from the container group database. * - Source boundary: host device-node ownership is authoritative; NemoClaw only carries each * bounded, non-root numeric GID with effective group read/write permission into the Jetson - * compatibility recreation via `--group-add`. - * - Source-fix constraint: changing host udev ownership or image-local groups cannot reliably fix - * device nodes whose ownership is assigned by the Jetson host at runtime. + * compatibility recreation via `--group-add`. OpenClaw also records those same GIDs in the + * replacement container's sandbox account before OpenShell rebuilds its supplementary group + * list. + * - Source-fix constraint: the replacement container membership must be derived from the current + * host device nodes; a static group name or GID cannot represent Jetson hosts reliably. * - Regression coverage: docker-gpu-jetson-groups.test.ts covers discovery and hostile numeric * values; docker-gpu-patch-jetson.test.ts covers clone-envelope propagation and generic-host * exclusion. diff --git a/src/lib/onboard/docker-gpu-patch-clone.ts b/src/lib/onboard/docker-gpu-patch-clone.ts index 9433c06a88e..7a3f03c612c 100644 --- a/src/lib/onboard/docker-gpu-patch-clone.ts +++ b/src/lib/onboard/docker-gpu-patch-clone.ts @@ -11,6 +11,9 @@ import { openshellSandboxCommandEnvValue } from "./docker-startup-command-env"; const OPENSHELL_SANDBOX_COMMAND_ENV = "OPENSHELL_SANDBOX_COMMAND"; const OPENSHELL_SANDBOX_ENTRYPOINT = "/opt/openshell/bin/openshell-sandbox"; +export const JETSON_DEVICE_GROUP_BOOTSTRAP = + "/usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh"; +const MAX_JETSON_DEVICE_GROUPS = 16; const OPENSHELL_V0_0_99_WORKDIR_COMMAND = ["--workdir", "/sandbox"] as const; const OPENSHELL_OCI_IMAGE_USER_ENV = "OPENSHELL_OCI_IMAGE_USER"; const OPENSHELL_SANDBOX_UID_ENV = "OPENSHELL_SANDBOX_UID"; @@ -455,6 +458,21 @@ export function buildDockerGpuCloneRunArgs( } const args: string[] = ["--name", containerName, ...mode.args]; const gpuAugment = mode.kind !== "startup-command"; + const extraGroupGids = [ + ...new Set((options.extraGroupGids ?? []).map((gid) => String(gid).trim())), + ]; + if ( + extraGroupGids.length > MAX_JETSON_DEVICE_GROUPS || + extraGroupGids.some((gid) => { + if (!/^[1-9][0-9]*$/u.test(gid)) return true; + const parsed = Number(gid); + return !Number.isSafeInteger(parsed) || parsed > 2_147_483_647; + }) + ) { + throw new Error("Docker clone received invalid or excessive supplementary group IDs."); + } + const preserveJetsonGroups = + options.preserveJetsonDeviceGroupMembership === true && extraGroupGids.length > 0; // Startup-command recreation must retain OpenShell's native CDI attachment. if (!gpuAugment) { @@ -536,11 +554,10 @@ export function buildDockerGpuCloneRunArgs( for (const hostEntry of stringArray(host.ExtraHosts)) args.push("--add-host", hostEntry); const groupAdds = new Set(stringArray(host.GroupAdd)); for (const group of groupAdds) args.push("--group-add", group); - for (const gid of options.extraGroupGids ?? []) { - const normalized = String(gid).trim(); - if (normalized && !groupAdds.has(normalized)) { - groupAdds.add(normalized); - args.push("--group-add", normalized); + for (const gid of extraGroupGids) { + if (!groupAdds.has(gid)) { + groupAdds.add(gid); + args.push("--group-add", gid); } } for (const ulimit of dockerUlimits(inspect, options.requiredUlimits)) { @@ -574,17 +591,32 @@ export function buildDockerGpuCloneRunArgs( const entrypoint = stringArray(config.Entrypoint); const replacementEntrypoint = String(options.containerEntrypoint ?? "").trim(); - if (replacementEntrypoint) { + const groupBootstrapTarget = replacementEntrypoint || entrypoint[0] || ""; + if (preserveJetsonGroups && groupBootstrapTarget !== OPENSHELL_SANDBOX_ENTRYPOINT) { + throw new Error("Jetson device-group bootstrap requires the OpenShell supervisor entrypoint."); + } + if (preserveJetsonGroups) { + args.push("--entrypoint", JETSON_DEVICE_GROUP_BOOTSTRAP); + } else if (replacementEntrypoint) { args.push("--entrypoint", replacementEntrypoint); } else if (entrypoint.length > 0) { args.push("--entrypoint", entrypoint[0]); } - const commandArgs = dockerContainerCommandArgs( + const targetCommandArgs = dockerContainerCommandArgs( entrypoint, stringArray(config.Cmd), sandboxCommand, options.containerCommand, ); + const commandArgs = preserveJetsonGroups + ? [ + "--device-group-gids", + extraGroupGids.join(","), + "--", + groupBootstrapTarget, + ...targetCommandArgs, + ] + : targetCommandArgs; args.push(image, ...commandArgs); return args; } diff --git a/src/lib/onboard/docker-gpu-patch-jetson.test.ts b/src/lib/onboard/docker-gpu-patch-jetson.test.ts index e091c45a137..55d516dbee7 100644 --- a/src/lib/onboard/docker-gpu-patch-jetson.test.ts +++ b/src/lib/onboard/docker-gpu-patch-jetson.test.ts @@ -32,6 +32,7 @@ describe("Jetson device-node group propagation (#4231, #7610)", () => { args.filter((arg, index) => args[index - 1] === "--group-add" && arg === "44").length, ).toBe(1); expect(args).toEqual(expect.arrayContaining(["--group-add", "110"])); + expect(args).not.toContain("/usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh"); }); it("does not add --group-add when extraGroupGids is absent", () => { @@ -41,8 +42,61 @@ describe("Jetson device-node group propagation (#4231, #7610)", () => { expect(args).not.toEqual(expect.arrayContaining(["--group-add"])); }); + it("runs the OpenShell supervisor through the Jetson group bootstrap (#7610)", () => { + const args = buildDockerGpuCloneRunArgs( + inspectFixture(), + buildDockerGpuMode("nvidia-runtime", null, { backend: "jetson" }), + { + extraGroupGids: ["44"], + preserveJetsonDeviceGroupMembership: true, + }, + ); + + expect(args.slice(args.indexOf("openshell/sandbox:abc"))).toEqual([ + "openshell/sandbox:abc", + "--device-group-gids", + "44", + "--", + "/opt/openshell/bin/openshell-sandbox", + ]); + }); + + it("rejects Jetson group preservation outside the OpenShell supervisor boundary (#7610)", () => { + const inspect = inspectFixture(); + inspect.Config!.Entrypoint = ["/custom/entrypoint"]; + expect(() => + buildDockerGpuCloneRunArgs( + inspect, + buildDockerGpuMode("nvidia-runtime", null, { backend: "jetson" }), + { + extraGroupGids: ["44"], + preserveJetsonDeviceGroupMembership: true, + }, + ), + ).toThrow("Jetson device-group bootstrap requires the OpenShell supervisor entrypoint."); + }); + + it("rejects invalid or excessive supplementary group IDs before clone creation (#7610)", () => { + const options = { + preserveJetsonDeviceGroupMembership: true, + } as const; + const build = (extraGroupGids: readonly string[]) => + buildDockerGpuCloneRunArgs( + inspectFixture(), + buildDockerGpuMode("nvidia-runtime", null, { backend: "jetson" }), + { ...options, extraGroupGids }, + ); + + expect(() => build(["0"])).toThrow( + "Docker clone received invalid or excessive supplementary group IDs.", + ); + expect(() => build(Array.from({ length: 17 }, (_, index) => String(index + 1)))).toThrow( + "Docker clone received invalid or excessive supplementary group IDs.", + ); + }); + it("passes all detected Tegra device GIDs into the Jetson recreate as --group-add", () => { - const dockerRunDetached = vi.fn(() => ({ + const dockerRunDetached = vi.fn((_args: readonly string[], _options?: unknown) => ({ status: 0, stdout: "new-container-id\n", })); @@ -59,7 +113,12 @@ describe("Jetson device-node group propagation (#4231, #7610)", () => { ); recreateOpenShellDockerSandboxWithGpu( - { sandboxName: "alpha", timeoutSecs: 1, backend: "jetson" }, + { + sandboxName: "alpha", + timeoutSecs: 1, + backend: "jetson", + preserveJetsonDeviceGroupMembership: true, + }, { dockerCapture: dockerCaptureFixture(), dockerRun: vi.fn(() => ({ status: 0, stdout: "probe-id\n" })), @@ -80,6 +139,20 @@ describe("Jetson device-node group propagation (#4231, #7610)", () => { expect.arrayContaining(["--group-add", "44", "--group-add", "104", "--group-add", "995"]), expect.objectContaining({ ignoreError: true }), ); + const createArgs = dockerRunDetached.mock.calls[0]?.[0] ?? []; + expect(createArgs).toEqual( + expect.arrayContaining([ + "--entrypoint", + "/usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh", + ]), + ); + expect(createArgs.slice(createArgs.indexOf("openshell/sandbox:abc"))).toEqual([ + "openshell/sandbox:abc", + "--device-group-gids", + "44,104,995", + "--", + "/opt/openshell/bin/openshell-sandbox", + ]); }); it("does not add Tegra device GIDs for the generic (non-Jetson) backend", () => { @@ -112,4 +185,36 @@ describe("Jetson device-node group propagation (#4231, #7610)", () => { expect.anything(), ); }); + + it("refuses to stop the original container when the OpenClaw wrapper is missing (#7610)", () => { + const dockerStop = vi.fn(() => ({ status: 0 })); + + expect(() => + recreateOpenShellDockerSandboxWithGpu( + { + sandboxName: "alpha", + timeoutSecs: 1, + backend: "jetson", + preserveJetsonDeviceGroupMembership: true, + }, + { + dockerCapture: dockerCaptureFixture(), + dockerRun: vi.fn((args: readonly string[]) => ({ + status: args[0] === "exec" ? 1 : 0, + stdout: "probe-id\n", + })), + dockerRunDetached: vi.fn(() => ({ status: 0, stdout: "new-container-id\n" })), + dockerRename: vi.fn(() => ({ status: 0 })), + dockerStop, + dockerRm: vi.fn(() => ({ status: 0 })), + runOpenshell: vi.fn(() => ({ status: 0 })), + sleep: vi.fn(), + now: () => new Date("2026-05-15T00:00:00Z"), + detectSandboxFallbackDns: () => null, + detectTegraDeviceGroupGids: () => ["44"], + }, + ), + ).toThrow("OpenClaw sandbox image is missing executable"); + expect(dockerStop).not.toHaveBeenCalled(); + }); }); diff --git a/src/lib/onboard/docker-gpu-patch-recreate.ts b/src/lib/onboard/docker-gpu-patch-recreate.ts index 4eb1c0ad8d4..03657c740e3 100644 --- a/src/lib/onboard/docker-gpu-patch-recreate.ts +++ b/src/lib/onboard/docker-gpu-patch-recreate.ts @@ -18,6 +18,7 @@ import { buildDockerGpuCloneRunOptions, dockerContainerName, getDockerGpuCloneFallbackDns, + JETSON_DEVICE_GROUP_BOOTSTRAP, parseDockerInspectJson, sameContainerId, validateRequiredDockerUlimits, @@ -159,6 +160,7 @@ export function recreateOpenShellDockerSandboxContainer( requiredUlimits?: readonly import("./docker-gpu-patch-types").DockerUlimit[] | null; expectedOldContainerId?: string | null; backend?: "generic" | "jetson"; + preserveJetsonDeviceGroupMembership?: boolean; dockerDesktopWsl?: boolean; modeOverride?: DockerGpuPatchMode; }, @@ -266,11 +268,35 @@ export function recreateOpenShellDockerSandboxContainer( const tegraGroupGids = d.detectTegraDeviceGroupGids(); if (tegraGroupGids.length > 0) { cloneOptions.extraGroupGids = tegraGroupGids; - console.log( - ` ✓ Granting sandbox user the detected Jetson GPU device groups via --group-add ${tegraGroupGids.join( - ", ", - )} (so CUDA can initialize as a non-root user)`, - ); + if (options.preserveJetsonDeviceGroupMembership === true) { + const wrapperProbe = d.dockerRun( + [ + "exec", + "--user", + "0", + oldContainerId, + "/usr/bin/test", + "-x", + JETSON_DEVICE_GROUP_BOOTSTRAP, + ], + { ignoreError: true, suppressOutput: true, timeout: DOCKER_GPU_PATCH_TIMEOUT_MS }, + ); + if (!hasZeroDockerExitStatus(wrapperProbe)) { + throw new Error( + `OpenClaw sandbox image is missing executable ${JETSON_DEVICE_GROUP_BOOTSTRAP}.`, + ); + } + cloneOptions.preserveJetsonDeviceGroupMembership = true; + console.log( + ` ✓ Preserving the detected Jetson GPU device groups through OpenShell startup: ${tegraGroupGids.join(", ")}`, + ); + } else { + console.log( + ` ✓ Granting sandbox user the detected Jetson GPU device groups via --group-add ${tegraGroupGids.join( + ", ", + )} (so CUDA can initialize as a non-root user)`, + ); + } } else { console.warn( " ⚠ Could not resolve the group owning Jetson Tegra GPU device nodes (/dev/nvmap); CUDA may fail with NvRmMemInitNvmap permission denied. Confirm /dev/nvmap exists and is group-readable on the host.", diff --git a/src/lib/onboard/docker-gpu-patch-types.ts b/src/lib/onboard/docker-gpu-patch-types.ts index fb1940fca7f..054e0fa7a83 100644 --- a/src/lib/onboard/docker-gpu-patch-types.ts +++ b/src/lib/onboard/docker-gpu-patch-types.ts @@ -125,12 +125,17 @@ export type DockerGpuCloneRunOptions = { containerName?: string | null; /** * Extra supplementary group IDs to add to the recreated container via - * `--group-add`. On Jetson these are the host group(s) owning the Tegra GPU - * device nodes; granting the sandbox user membership lets CUDA's nvmap init - * open them instead of failing with `NvRmMemInitNvmap ... Permission - * denied` (#4231, #7610). + * `--group-add`. The OpenClaw Jetson path also records these validated GIDs + * in the replacement container's group database before OpenShell calls + * initgroups() (#7610). */ extraGroupGids?: readonly string[] | null; + /** + * Add the detected Jetson device GIDs to the replacement container's sandbox + * account before OpenShell rebuilds supplementary groups with initgroups() + * (#7610). + */ + preserveJetsonDeviceGroupMembership?: boolean; }; export type DockerGpuPatchDiagnostics = { diff --git a/src/lib/onboard/docker-gpu-patch.ts b/src/lib/onboard/docker-gpu-patch.ts index 4331d2f56f0..74d64efd086 100644 --- a/src/lib/onboard/docker-gpu-patch.ts +++ b/src/lib/onboard/docker-gpu-patch.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { dockerCapture } from "../adapters/docker"; +import { dockerCapture, dockerRun } from "../adapters/docker"; import { parseLiveSandboxEntries } from "../runtime-recovery"; import { createDockerGpuDiagnosticRedactor } from "./docker-gpu-diagnostic-redaction"; import { DOCKER_GPU_PATCH_TIMEOUT_MS } from "./docker-gpu-patch-constants"; @@ -27,6 +27,7 @@ export { getDockerGpuPatchNetworkMode, parseDockerInspectJson, } from "./docker-gpu-patch-clone"; +export { dockerRun }; import { collectDockerGpuPatchDiagnostics } from "./docker-gpu-patch-diagnostics"; import { diff --git a/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts b/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts index 0206ce9a80d..4b042397882 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts @@ -1,11 +1,27 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import fs from "node:fs"; +import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { JetsonOpenRmPolicyRestorationError } from "./diagnostics/jetson-openrm-proof"; import type { DockerGpuPatchFailureContext, DockerGpuPatchResult } from "./docker-gpu-patch"; import { createDockerGpuSandboxCreatePatch } from "./docker-gpu-sandbox-create"; +const OPENRM_BASE_POLICY = `Version: 2 +Hash: fixture +--- +version: 1 +filesystem_policy: + read_only: + - /opt/nvidia + - /sys + read_write: + - /dev/nvmap +network_policies: {} +`; + function deferredCreateResult(): DockerGpuPatchResult { return { applied: true, @@ -59,6 +75,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { route: "compatibility", sandboxName: "alpha", timeoutSecs: 60, + preserveJetsonDeviceGroupMembership: true, deps, overrides: { findContainerIds, @@ -72,7 +89,10 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { patch.maybeApplyDuringCreate(); expect(recreatePatch).toHaveBeenCalledWith( - expect.objectContaining({ waitForSupervisor: false }), + expect.objectContaining({ + waitForSupervisor: false, + preserveJetsonDeviceGroupMembership: true, + }), expect.objectContaining({ runCaptureOpenshell: deps.runCaptureOpenshell, }), @@ -411,4 +431,79 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { expect.stringContaining("pre-patch container was not restored"), ); }); + + it("blocks lifecycle rollback when OpenShell cannot confirm baseline policy restoration", async () => { + vi.stubEnv("NEMOCLAW_DIAGNOSE_JETSON_OPENRM_POLICY", "1"); + let policySetCount = 0; + let temporaryDirectory = ""; + const deps = { + ...makeDeps(), + runCaptureOpenshell: vi.fn((args: string[]) => + args[0] === "policy" ? OPENRM_BASE_POLICY : "", + ), + runOpenshell: vi.fn((args: string[]) => { + const policyPath = args[3] ?? ""; + temporaryDirectory = path.dirname(policyPath); + policySetCount += 1; + return { status: policySetCount === 1 ? 0 : 1 }; + }), + dockerRun: vi.fn((args: readonly string[]) => + args.includes("0") + ? { + status: 0, + stdout: "/dev/nvidia-caps/nvidia-cap2\n/dev/nvmap", + stderr: "", + } + : { status: 0, stdout: "cuInit(0)=0", stderr: "" }, + ), + }; + const result = { + ...deferredCreateResult(), + mode: { + kind: "nvidia-runtime" as const, + label: "--runtime nvidia", + device: "all", + args: ["--runtime", "nvidia"], + }, + }; + const finalizeBackup = vi.fn(() => ({ backupRemoved: false, rolledBack: true })); + const patch = createDockerGpuSandboxCreatePatch({ + route: "compatibility", + sandboxName: "alpha", + timeoutSecs: 60, + backend: "jetson", + preserveJetsonDeviceGroupMembership: true, + deps, + overrides: { + findContainerIds: vi.fn(() => ["existing-container"]), + recreatePatch: vi.fn(() => result), + waitForSupervisor: vi.fn(() => true), + finalizeBackup, + }, + }); + const verifyDirectSandboxGpu = vi + .fn() + .mockImplementationOnce(() => { + throw new Error("cuInit(0)=801"); + }) + .mockReturnValue({ + status: "verified" as const, + cudaVerified: true, + at: "2026-08-06T00:00:00.000Z", + }); + + patch.maybeApplyDuringCreate(); + patch.waitForSupervisorReconnectIfNeeded(); + + await expect(patch.verifyGpuOrExit(verifyDirectSandboxGpu)).rejects.toBeInstanceOf( + JetsonOpenRmPolicyRestorationError, + ); + await expect(patch.rollbackManagedStartupAfterCreateFailure()).rejects.toBeInstanceOf( + JetsonOpenRmPolicyRestorationError, + ); + + expect(finalizeBackup).not.toHaveBeenCalled(); + expect(temporaryDirectory).not.toBe(""); + expect(fs.existsSync(temporaryDirectory)).toBe(false); + }); }); diff --git a/src/lib/onboard/docker-gpu-sandbox-create.ts b/src/lib/onboard/docker-gpu-sandbox-create.ts index d4356c79d0a..fcc8a0c771e 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create.ts @@ -3,6 +3,10 @@ import { getSandboxFailurePhase } from "../state/gateway"; import type { SandboxGpuProofResult } from "../state/registry"; +import { + JetsonOpenRmPolicyRestorationError, + maybeRunJetsonOpenRmPolicyProof, +} from "./diagnostics/jetson-openrm-proof"; import { getDockerGpuSupervisorReconnectTimeoutSecs, printDockerGpuPatchFailureAndExit, @@ -84,6 +88,7 @@ type DockerGpuSandboxCreatePatchOptions = { requiredUlimits?: Parameters[0]["requiredUlimits"]; timeoutSecs: number; backend?: DockerGpuPatchBackend; + preserveJetsonDeviceGroupMembership?: boolean; /** * Whether the host is Docker Desktop WSL. Defaults to the cached * `isDockerDesktopWslRuntime()` probe. When true, the GPU patch skips the CDI @@ -160,6 +165,7 @@ export function createDockerGpuSandboxCreatePatch( let cutoverFinalization: Promise | null = null; let cutoverFinalizationOutcome: "commit" | "rollback" | null = null; let cutoverFinalizationFailure: Error | null = null; + let policyRestorationFailure: JetsonOpenRmPolicyRestorationError | null = null; const findContainerIds = options.overrides?.findContainerIds ?? findOpenShellDockerSandboxContainerIds; @@ -187,6 +193,7 @@ export function createDockerGpuSandboxCreatePatch( requiredUlimits: options.requiredUlimits ?? null, timeoutSecs: options.timeoutSecs, backend: options.backend, + preserveJetsonDeviceGroupMembership: options.preserveJetsonDeviceGroupMembership, dockerDesktopWsl: options.dockerDesktopWsl ?? isDockerDesktopWslRuntime(), }; const recreationEnabled = @@ -313,6 +320,7 @@ export function createDockerGpuSandboxCreatePatch( }, async rollbackManagedStartupAfterCreateFailure() { + if (policyRestorationFailure) throw policyRestorationFailure; const rollbackError = await rollbackAfterFailure(); if (!rollbackError) return; onPatchFailureExit(options.sandboxName, rollbackError, { @@ -555,6 +563,25 @@ export function createDockerGpuSandboxCreatePatch( return proof; } catch (error) { const failure = error instanceof Error ? error : new Error(String(error)); + try { + maybeRunJetsonOpenRmPolicyProof({ + backend: options.backend, + failure, + preserveJetsonDeviceGroupMembership: options.preserveJetsonDeviceGroupMembership, + result, + sandboxName, + verifyDirectSandboxGpu, + deps: options.deps, + }); + } catch (diagnosticError) { + console.error( + ` OpenRM A/B inconclusive: ${diagnosticError instanceof Error ? diagnosticError.message : String(diagnosticError)}`, + ); + if (diagnosticError instanceof JetsonOpenRmPolicyRestorationError) { + policyRestorationFailure = diagnosticError; + throw diagnosticError; + } + } printDockerGpuProofFailure(sandboxName, failure, selectedMode(), { runCaptureOpenshell: options.deps.runCaptureOpenshell, dockerCapture: options.deps.dockerCapture, diff --git a/src/lib/onboard/docker-startup-command-agent.ts b/src/lib/onboard/docker-startup-command-agent.ts index de483bda8db..fdb60a63181 100644 --- a/src/lib/onboard/docker-startup-command-agent.ts +++ b/src/lib/onboard/docker-startup-command-agent.ts @@ -18,14 +18,16 @@ export function resolveDockerStartupCommandPatch( agent: AgentDefinition | null | undefined, dockerDriverGateway: boolean | null | undefined, ): { + agentName: string; persistStartupCommand: boolean; requiredUlimits: readonly DockerUlimit[] | null; } { + const agentName = agent?.name ?? "openclaw"; if (dockerDriverGateway !== true) { - return { persistStartupCommand: false, requiredUlimits: null }; + return { agentName, persistStartupCommand: false, requiredUlimits: null }; } - const agentName = agent?.name ?? "openclaw"; return { + agentName, persistStartupCommand: agentName === "openclaw" || agentName === "hermes" || agentName === DCODE_AGENT_NAME, requiredUlimits: agentName === DCODE_AGENT_NAME ? DCODE_DOCKER_ULIMITS : null, diff --git a/src/lib/onboard/dockerfile-remote-dashboard-bind-contract.ts b/src/lib/onboard/dockerfile-remote-dashboard-bind-contract.ts index 842a69a6e16..9fd3ad0b0e3 100644 --- a/src/lib/onboard/dockerfile-remote-dashboard-bind-contract.ts +++ b/src/lib/onboard/dockerfile-remote-dashboard-bind-contract.ts @@ -59,6 +59,7 @@ const CANONICAL_POST_GENERATOR_RUN_SHA256 = new Set([ "a619aead6cdf253dc7bf4504267e6b1d724fed672597072394b7400c08f81fd0", "c0b409e1bf4d33a9e44f407c6bd9b0445b2ffd0b796823fe3cfa5989314d6603", "9fcc674a44a152707380cdb09a67f8594f568288406c96f5354f1c87f5b939a6", + "a91a2cb7531542b50169301d172079cc63132bd6d5f9e986fe0e125722a055fe", "83567d1fa0e73bef6a3333383c13ace05e26704964ae6a7a76ee24a2f2be3d7e", "ca1f7b1cb9dd5d467f806792c4072a84ef1e6402c3e8650b6325b95cc186ccdf", "4165899eb1f0f948f8883eddf4136136caac21cee1df39b12afea7672b23a378", diff --git a/src/lib/onboard/initial-policy.test.ts b/src/lib/onboard/initial-policy.test.ts index 2cde41037a6..b12909739b1 100644 --- a/src/lib/onboard/initial-policy.test.ts +++ b/src/lib/onboard/initial-policy.test.ts @@ -369,6 +369,68 @@ network_policies: {} } }); + it("moves detected Jetson device paths to read-write and adds /opt/nvidia as read-only (#7610)", () => { + const gpuPolicy = buildDirectGpuPolicyYaml( + ` +version: 1 +filesystem_policy: + read_only: + - /usr + - /dev/nvmap + read_write: + - /tmp + - /dev/nvhost-gpu +network_policies: {} +`, + { + jetsonGpuDevicePaths: ["/dev/nvmap", "/dev/nvhost-gpu", "/dev/nvmap"], + }, + ); + const gpuDoc = YAML.parse(gpuPolicy); + + expect(gpuDoc.filesystem_policy.read_only).toContain("/opt/nvidia"); + expect(gpuDoc.filesystem_policy.read_only).not.toContain("/dev/nvmap"); + expectSingleOccurrence(gpuDoc.filesystem_policy.read_write, "/dev/nvmap"); + expectSingleOccurrence(gpuDoc.filesystem_policy.read_write, "/dev/nvhost-gpu"); + }); + + it("keeps Jetson filesystem grants scoped to OpenClaw direct GPU policy (#7610)", () => { + const basePolicyPath = tmpPolicy(BASE_POLICY_FIXTURE); + const devicePaths = ["/dev/nvmap", "/dev/nvhost-gpu"]; + const defaultOpenclaw = prepareInitialSandboxCreatePolicy(basePolicyPath, [], { + directGpu: true, + jetsonGpuDevicePaths: devicePaths, + stationGb300SysfsReadOnlyPaths: [], + }); + const openclaw = prepareInitialSandboxCreatePolicy(basePolicyPath, [], { + directGpu: true, + agentName: "openclaw", + jetsonGpuDevicePaths: devicePaths, + stationGb300SysfsReadOnlyPaths: [], + }); + const hermes = prepareInitialSandboxCreatePolicy(basePolicyPath, [], { + directGpu: true, + agentName: "hermes", + jetsonGpuDevicePaths: devicePaths, + stationGb300SysfsReadOnlyPaths: [], + }); + const defaultOpenclawDoc = YAML.parse(fs.readFileSync(defaultOpenclaw.policyPath, "utf-8")); + const openclawDoc = YAML.parse(fs.readFileSync(openclaw.policyPath, "utf-8")); + const hermesDoc = YAML.parse(fs.readFileSync(hermes.policyPath, "utf-8")); + + expect(defaultOpenclawDoc.filesystem_policy.read_only).toContain("/opt/nvidia"); + expect(defaultOpenclawDoc.filesystem_policy.read_write).toEqual( + expect.arrayContaining(devicePaths), + ); + expect(openclawDoc.filesystem_policy.read_only).toContain("/opt/nvidia"); + expect(openclawDoc.filesystem_policy.read_write).toEqual(expect.arrayContaining(devicePaths)); + expect(hermesDoc.filesystem_policy.read_only).not.toContain("/opt/nvidia"); + expect(hermesDoc.filesystem_policy.read_write).not.toEqual(expect.arrayContaining(devicePaths)); + expect(defaultOpenclaw.cleanup?.()).toBe(true); + expect(openclaw.cleanup?.()).toBe(true); + expect(hermes.cleanup?.()).toBe(true); + }); + it("preserves best-effort Landlock for missing Station sysfs paths (#7103)", () => { const sysfsRoot = tmpSysfsRoot(); addPciDevice(sysfsRoot, "0009:06:00.0", "0x10de\n", "0x030200\n"); diff --git a/src/lib/onboard/initial-policy.ts b/src/lib/onboard/initial-policy.ts index 4d56cecb127..17680d8e880 100644 --- a/src/lib/onboard/initial-policy.ts +++ b/src/lib/onboard/initial-policy.ts @@ -23,6 +23,7 @@ import { isStationGb300ProductName, type StationProfile, } from "../readiness/station-qualification"; +import { detectTegraGpuDevicePaths } from "./docker-gpu-jetson-groups"; import { allMessagingChannelPolicyPresets, requiredMessagingChannelPolicyPresets, @@ -46,6 +47,7 @@ export function discloseInitialSandboxPolicy(policy: InitialSandboxPolicy): void const HERMES_MESSAGING_POLICY_KEYS = getMessagingPolicyKeysByChannel({ agent: "hermes" }); const PROC_PATH = "/proc"; +const JETSON_GPU_LIBRARY_ROOT = "/opt/nvidia"; const PROC_COMM_READ_WRITE_PATHS = ["/proc/self/comm", "/proc/self/task/*/comm"]; const SYSFS_PATH = "/sys"; const PCI_BDF_PATTERN = /^[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]$/iu; @@ -77,6 +79,7 @@ function deduplicateDirectGpuSysfsEntries( type DirectGpuPolicyOptions = { procReadWrite?: boolean; sysfsReadOnlyPaths?: readonly string[]; + jetsonGpuDevicePaths?: readonly string[]; }; export { isStationGb300ProductName }; @@ -223,6 +226,23 @@ export function buildDirectGpuPolicyYaml( } } } + const jetsonGpuDevicePaths = [...new Set(options.jetsonGpuDevicePaths ?? [])]; + if (jetsonGpuDevicePaths.length > 0) { + if ( + !fsPolicy.read_only.includes(JETSON_GPU_LIBRARY_ROOT) && + !fsPolicy.read_write.includes(JETSON_GPU_LIBRARY_ROOT) + ) { + fsPolicy.read_only.push(JETSON_GPU_LIBRARY_ROOT); + } + + const jetsonGpuDevicePathSet = new Set(jetsonGpuDevicePaths); + fsPolicy.read_only = fsPolicy.read_only.filter( + (entry: string) => !jetsonGpuDevicePathSet.has(entry), + ); + for (const devicePath of jetsonGpuDevicePaths) { + if (!fsPolicy.read_write.includes(devicePath)) fsPolicy.read_write.push(devicePath); + } + } if (options.procReadWrite && !fsPolicy.read_write.includes(PROC_PATH)) { // This exists only for the legacy post-create Docker GPU compatibility // path, which recreates the container after `openshell sandbox create` and @@ -383,6 +403,7 @@ export function prepareInitialSandboxCreatePolicy( dockerGpuPatch?: boolean; hostGpuAvailable?: boolean; stationGb300SysfsReadOnlyPaths?: readonly string[]; + jetsonGpuDevicePaths?: readonly string[]; additionalPresets?: string[]; agentName?: string | null; policyTier?: string | null; @@ -397,6 +418,10 @@ export function prepareInitialSandboxCreatePolicy( discoverHostStationGb300SysfsReadOnlyPaths({ hasNvidiaGpu: options.hostGpuAvailable, }), + jetsonGpuDevicePaths: + (options.agentName ?? "openclaw") === "openclaw" + ? (options.jetsonGpuDevicePaths ?? detectTegraGpuDevicePaths()) + : [], }) : null; let effectiveBasePolicyPath = directGpuPolicy?.policyPath || basePolicyPath; diff --git a/src/lib/onboard/machine/handlers/sandbox-checkpoint-crash-recovery.test.ts b/src/lib/onboard/machine/handlers/sandbox-checkpoint-crash-recovery.test.ts index 9b54893fb2f..520af17538c 100644 --- a/src/lib/onboard/machine/handlers/sandbox-checkpoint-crash-recovery.test.ts +++ b/src/lib/onboard/machine/handlers/sandbox-checkpoint-crash-recovery.test.ts @@ -883,6 +883,28 @@ describe("sandbox crash-recovery replay (#5961, #6228)", () => { expect(calls.error.mock.calls.flat().join("\n")).toContain("--recreate-sandbox"); }); + it("recreates after build or policy drift when explicitly requested (#7022)", async () => { + const session = sessionWithCheckpoint( + crashedCheckpoint({ + effectGroups: { + sandbox_create: { completedAt: "2026-01-01T00:00:00.000Z", fingerprint: "stale-build" }, + }, + }), + ); + session.machine.state = "openclaw"; + const { deps, calls } = createDeps({ getSandboxReuseState: () => "ready" }, session); + + await handleSandboxState({ + ...baseOptions(deps, session), + resume: true, + sandboxName: "my-assistant", + recreateSandbox: () => true, + }); + + expect(calls.createSandbox).toHaveBeenCalledOnce(); + expect(calls.error).not.toHaveBeenCalled(); + }); + it("rejects reuse when a resolved policy or package input drifted despite an unchanged build version and policy tier (#7022)", async () => { const { deps, calls } = createDeps({ getSandboxReuseState: () => "ready" }); const session = sessionWithCheckpoint(crashedCheckpoint()); diff --git a/src/lib/onboard/machine/handlers/sandbox.ts b/src/lib/onboard/machine/handlers/sandbox.ts index 9f56af7edcd..24f235f3bb4 100644 --- a/src/lib/onboard/machine/handlers/sandbox.ts +++ b/src/lib/onboard/machine/handlers/sandbox.ts @@ -795,6 +795,7 @@ class SandboxStateFlow< sandboxName: string, createIntent: ResolvedSandboxCreateIntent, ): void { + if (this.options.recreateSandbox(false)) return; const recordedFingerprint = state.session?.checkpoint?.effectGroups.sandbox_create?.fingerprint; if (!recordedFingerprint) return; if (recordedFingerprint !== this.currentSandboxCreateFingerprint(sandboxName, createIntent)) { diff --git a/src/lib/onboard/sandbox-gpu-create-flow.test.ts b/src/lib/onboard/sandbox-gpu-create-flow.test.ts index 922afe3e17b..6a13d177e76 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.test.ts @@ -574,7 +574,24 @@ describe("runSandboxGpuCreateFlow native failure and readiness", () => { }); expect(mocks.createDockerGpuSandboxCreatePatch).toHaveBeenCalledWith( - expect.objectContaining({ route: "native", persistStartupCommand: false }), + expect.objectContaining({ + route: "native", + persistStartupCommand: false, + preserveJetsonDeviceGroupMembership: true, + }), + ); + }); + + it("does not apply the OpenClaw Jetson group bootstrap to Hermes (#7610)", async () => { + const input = createInput(); + input.agentName = "hermes"; + + await expect(runSandboxGpuCreateFlow(input, createDeps())).resolves.toMatchObject({ + route: "native", + }); + + expect(mocks.createDockerGpuSandboxCreatePatch).toHaveBeenCalledWith( + expect.objectContaining({ preserveJetsonDeviceGroupMembership: false }), ); }); diff --git a/src/lib/onboard/sandbox-gpu-create-flow.ts b/src/lib/onboard/sandbox-gpu-create-flow.ts index 6f52aefdbdf..66209d85460 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.ts @@ -70,6 +70,7 @@ type LifecycleRegistrationFields = Pick; export interface SandboxGpuCreateFlowInput { sandboxName: string; + agentName: string; provider: string; sandboxGpuConfig: SandboxGpuConfig; gpuRoutePlan: import("./docker-gpu-route").DockerGpuRoutePlan; diff --git a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts index 5b91caa7368..e565ee949dd 100644 --- a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts +++ b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts @@ -147,6 +147,7 @@ export function createSandboxGpuCreateAttemptRunner( requiredUlimits: input.requiredUlimits, timeoutSecs: input.sandboxReadyTimeoutSecs, backend: input.sandboxGpuConfig.hostGpuPlatform === "jetson" ? "jetson" : "generic", + preserveJetsonDeviceGroupMembership: input.agentName === "openclaw", deps, }); const recovery = await managedLifecycle?.recoverUnfinished(); diff --git a/src/lib/onboard/sandbox-gpu-preflight-routing.test.ts b/src/lib/onboard/sandbox-gpu-preflight-routing.test.ts index 9a1e70c3f2b..8407fa39eb5 100644 --- a/src/lib/onboard/sandbox-gpu-preflight-routing.test.ts +++ b/src/lib/onboard/sandbox-gpu-preflight-routing.test.ts @@ -11,6 +11,7 @@ import type { SandboxGpuConfig } from "./sandbox-gpu-mode"; import { dockerNvidiaRuntimeAvailable, formatSandboxGpuPassthroughNote, + jetsonGpuProofRemediationLines, parseDockerRuntimeNames, sandboxGpuRemediationLines, validateSandboxGpuPreflight, @@ -28,6 +29,13 @@ function sandboxGpuConfig(overrides: Partial = {}): SandboxGpu }; } describe("sandbox GPU preflight routing", () => { + it("checks retained Jetson group membership instead of repeating --group-add advice (#7610)", () => { + const remediation = jetsonGpuProofRemediationLines().join("\n"); + + expect(remediation).toContain("id (must include the groups that own those device nodes)"); + expect(remediation).not.toContain("via --group-add"); + }); + it("formats Jetson sandbox GPU notes around the NVIDIA runtime backend", () => { expect(formatSandboxGpuPassthroughNote({ hostGpuPlatform: "jetson" })).toContain( "Docker NVIDIA runtime", diff --git a/src/lib/onboard/sandbox-gpu-preflight.ts b/src/lib/onboard/sandbox-gpu-preflight.ts index 65cc1f8479f..0416b60d444 100644 --- a/src/lib/onboard/sandbox-gpu-preflight.ts +++ b/src/lib/onboard/sandbox-gpu-preflight.ts @@ -66,9 +66,9 @@ export function resolveSandboxGpuFlagFromOptions(opts: SandboxGpuFlagOptions): S export function jetsonGpuProofRemediationLines(): string[] { return [ "Jetson/Tegra CUDA proof did not pass. CUDA needs access to the Tegra device", - "nodes; confirm the sandbox propagates them and the agent user's groups:", + "nodes; confirm the sandbox receives them and keeps the device group memberships:", " ls -l /dev/nvmap /dev/nvhost-* (must be readable by the sandbox)", - " add the host video/render groups via --group-add when recreating", + " id (must include the groups that own those device nodes)", "Then recreate the sandbox, or force CPU behavior with NEMOCLAW_SANDBOX_GPU=0.", ]; } diff --git a/src/lib/sandbox/build-context.ts b/src/lib/sandbox/build-context.ts index 4a080151565..7f466a89038 100644 --- a/src/lib/sandbox/build-context.ts +++ b/src/lib/sandbox/build-context.ts @@ -296,6 +296,10 @@ function stageOptimizedSandboxBuildContext( path.join(rootDir, "scripts", "managed-bootstrap-trampoline.sh"), path.join(stagedScriptsDir, "managed-bootstrap-trampoline.sh"), ); + fs.copyFileSync( + path.join(rootDir, "scripts", "jetson-device-group-bootstrap.sh"), + path.join(stagedScriptsDir, "jetson-device-group-bootstrap.sh"), + ); fs.copyFileSync( path.join(rootDir, "scripts", "gateway-control.sh"), path.join(stagedScriptsDir, "gateway-control.sh"), diff --git a/test/jetson-device-group-bootstrap.test.ts b/test/jetson-device-group-bootstrap.test.ts new file mode 100644 index 00000000000..11accf58c66 --- /dev/null +++ b/test/jetson-device-group-bootstrap.test.ts @@ -0,0 +1,415 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterAll, beforeAll, describe, expect, it } from "vitest"; + +import { dockerSpawnSync } from "../src/lib/adapters/docker/exec"; + +const REPO_ROOT = path.join(import.meta.dirname, ".."); +const BOOTSTRAP_SCRIPT = path.join(REPO_ROOT, "scripts", "jetson-device-group-bootstrap.sh"); +const FIXTURE_BASE_IMAGE = + "ghcr.io/nvidia/nemoclaw/sandbox-base@sha256:16b01f6d7e0b843a72331538f3bf690b6112840064e7d8b81e361a61277bebf7"; +const SUPERVISOR = "/opt/openshell/bin/openshell-sandbox"; +const CONTAINER_TIMEOUT_MS = 20_000; + +type BootstrapRunOptions = { + environment?: Record; + groupDatabase?: "regular" | "symlink"; +}; + +type BootstrapRun = { + after: FixtureState; + before: FixtureState; + status: number | null; + stderr: string; + stdout: string; +}; + +type FixtureState = { + groupAddLog: string; + groupMap: string; + memberships: string; + supervisorArgv: Buffer | null; + usermodLog: string; +}; + +const fixtureParent = process.platform === "darwin" ? "/private/tmp" : os.tmpdir(); +const fixtureId = `${String(process.pid)}-${String(Date.now())}`; +const fixtureImage = `nemoclaw-jetson-bootstrap-test:${fixtureId}`; +let containerFixtureRoot = path.join( + fixtureParent, + `nemoclaw-jetson-bootstrap-not-created-${fixtureId}`, +); + +const GROUP_DATABASE_DOCKER_ARGS = { + regular: [], + symlink: ["--tmpfs", "/etc:rw,nosuid,nodev,noexec,size=1m"], +} as const satisfies Record, readonly string[]>; + +function writeExecutable(filePath: string, source: string): void { + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, source, { mode: 0o755 }); +} + +const INITIAL_STATE: FixtureState = { + groupAddLog: "", + groupMap: "44:video\n", + memberships: "1000 44\n", + supervisorArgv: null, + usermodLog: "", +}; + +function createContainerFixture(): string { + const root = fs.mkdtempSync(path.join(fixtureParent, "nemoclaw-jetson-bootstrap-")); + const usrBin = path.join(root, "usr-bin"); + const usrSbin = path.join(root, "usr-sbin"); + const supervisorDir = path.join(root, "supervisor"); + + writeExecutable( + path.join(usrBin, "id"), + `#!/bin/sh +set -eu +if [ "\${1:-}" = "-u" ]; then + printf '%s\n' "\${TEST_ID_UID:-0}" + exit 0 +fi +if [ "\${1:-}" = "sandbox" ]; then + [ "\${TEST_SANDBOX_MISSING:-0}" != "1" ] || exit 1 + printf 'uid=1000(sandbox) gid=1000(sandbox) groups=%s\n' "$(cat /test-state/memberships)" + exit 0 +fi +if [ "\${1:-}" = "-G" ] && [ "\${2:-}" = "sandbox" ]; then + cat /test-state/memberships + exit 0 +fi +exit 2 +`, + ); + writeExecutable( + path.join(usrBin, "getent"), + `#!/bin/sh +set -eu +[ "\${1:-}" = "group" ] || exit 2 +gid="\${2:-}" +if [ "\${TEST_GETENT_MALFORMED_GID:-}" = "$gid" ]; then + printf ':x:999:\n' + exit 0 +fi +record="$(awk -F: -v gid="$gid" '$1 == gid { print; exit }' /test-state/group-map)" +[ -n "$record" ] || exit 2 +name="\${record#*:}" +printf '%s:x:%s:\n' "$name" "$gid" +`, + ); + writeExecutable( + path.join(usrSbin, "groupadd"), + `#!/bin/sh +set -eu +[ "$#" -eq 3 ] && [ "$1" = "--gid" ] +printf '%s\n' "$*" >>/test-state/groupadd.log +printf '%s:%s\n' "$2" "$3" >>/test-state/group-map +`, + ); + writeExecutable( + path.join(usrSbin, "usermod"), + `#!/bin/sh +set -eu +[ "$#" -eq 4 ] && [ "$1" = "--append" ] && [ "$2" = "--groups" ] && [ "$4" = "sandbox" ] +printf '%s\n' "$*" >>/test-state/usermod.log +[ "\${TEST_USERMOD_NOOP:-0}" != "1" ] || exit 0 +gid="$(awk -F: -v name="$3" '$2 == name { print $1; exit }' /test-state/group-map)" +[ -n "$gid" ] +memberships="$(cat /test-state/memberships)" +case " $memberships " in + *" $gid "*) ;; + *) printf '%s %s\n' "$memberships" "$gid" >/test-state/memberships ;; +esac +`, + ); + writeExecutable( + path.join(supervisorDir, "openshell-sandbox"), + `#!/bin/sh +set -eu +printf '%s\\0' "$@" >/test-state/supervisor.argv +printf 'SUPERVISOR_EXECUTED\n' +`, + ); + writeExecutable( + path.join(root, "fixture-runner"), + `#!/bin/bash +set -uo pipefail +printf '1000 44\n' >/test-state/memberships +printf '44:video\n' >/test-state/group-map +case "\${TEST_GROUP_DATABASE:-regular}" in + regular) ;; + symlink) /bin/ln -s /tmp/nemoclaw-missing-group /etc/group ;; + *) exit 2 ;; +esac +set +e +/usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh "$@" +status=$? +set -e +printf 'NEMOCLAW_TEST_STATE_BEGIN\n' +for file in groupadd.log group-map memberships supervisor.argv usermod.log; do + printf '%s=' "$file" + if [ -f "/test-state/$file" ]; then + base64 "/test-state/$file" | tr -d '\n' + fi + printf '\n' +done +printf 'NEMOCLAW_TEST_STATE_END\n' +exit "$status" +`, + ); + fs.copyFileSync(BOOTSTRAP_SCRIPT, path.join(root, "jetson-device-group-bootstrap.sh")); + fs.writeFileSync( + path.join(root, "Dockerfile"), + `FROM ${FIXTURE_BASE_IMAGE} +COPY jetson-device-group-bootstrap.sh /usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh +COPY fixture-runner /test-fixture/run +COPY supervisor/ /opt/openshell/bin/ +COPY usr-bin/id /usr/bin/id +COPY usr-bin/getent /usr/bin/getent +COPY usr-sbin/ /usr/sbin/ +`, + ); + return root; +} + +function parseFixtureState(stdout: string): { state: FixtureState; stdout: string } { + const startMarker = "NEMOCLAW_TEST_STATE_BEGIN\n"; + const endMarker = "NEMOCLAW_TEST_STATE_END\n"; + const start = stdout.indexOf(startMarker); + const end = stdout.indexOf(endMarker, start + startMarker.length); + expect(start, "fixture state start marker is missing").toBeGreaterThanOrEqual(0); + expect(end, "fixture state end marker is missing").toBeGreaterThan(start); + const encoded = new Map( + stdout + .slice(start + startMarker.length, end) + .trimEnd() + .split("\n") + .map((line) => { + const separator = line.indexOf("="); + return [line.slice(0, separator), line.slice(separator + 1)] as const; + }), + ); + const decode = (name: string): Buffer | null => { + const value = encoded.get(name); + return value ? Buffer.from(value, "base64") : null; + }; + return { + state: { + groupAddLog: decode("groupadd.log")?.toString("utf8") ?? "", + groupMap: decode("group-map")?.toString("utf8") ?? "", + memberships: decode("memberships")?.toString("utf8") ?? "", + supervisorArgv: decode("supervisor.argv"), + usermodLog: decode("usermod.log")?.toString("utf8") ?? "", + }, + stdout: stdout.slice(0, start), + }; +} + +function runBootstrap(args: readonly string[], options: BootstrapRunOptions = {}): BootstrapRun { + const groupDatabase = options.groupDatabase ?? "regular"; + const dockerArgs = [ + "run", + "--rm", + "--network", + "none", + "--read-only", + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges", + "--tmpfs", + "/tmp:rw,nosuid,nodev,noexec,size=1m", + "--tmpfs", + "/test-state:rw,nosuid,nodev,noexec,size=1m", + ...GROUP_DATABASE_DOCKER_ARGS[groupDatabase], + ...Object.entries({ + ...options.environment, + TEST_GROUP_DATABASE: groupDatabase, + }).flatMap(([key, value]) => ["--env", `${key}=${value}`]), + "--entrypoint", + "/test-fixture/run", + fixtureImage, + ...args, + ]; + const result = dockerSpawnSync(dockerArgs, { + encoding: "utf8", + killSignal: "SIGKILL", + timeout: CONTAINER_TIMEOUT_MS, + }); + expect(result.error, result.error?.message).toBeUndefined(); + const parsed = parseFixtureState(String(result.stdout)); + return { + after: parsed.state, + before: INITIAL_STATE, + status: result.status, + stderr: String(result.stderr), + stdout: parsed.stdout, + }; +} + +function expectNoMutation(run: BootstrapRun): void { + expect(run.after).toEqual(run.before); +} + +const dockerProbe = dockerSpawnSync(["info", "--format", "{{.ServerVersion}}"], { + encoding: "utf8", + killSignal: "SIGKILL", + timeout: 5_000, +}); +const suite = dockerProbe.status === 0 || process.platform === "linux" ? describe : describe.skip; + +suite("Jetson device-group bootstrap", () => { + beforeAll(() => { + expect( + dockerProbe.status, + `Docker is required for the Linux bootstrap security boundary: ${String(dockerProbe.stderr)}`, + ).toBe(0); + containerFixtureRoot = createContainerFixture(); + const build = dockerSpawnSync( + ["build", "--network", "none", "--tag", fixtureImage, containerFixtureRoot], + { + encoding: "utf8", + killSignal: "SIGKILL", + timeout: 60_000, + }, + ); + expect(build.error, build.error?.message).toBeUndefined(); + expect(build.status, `${String(build.stderr)}\n${String(build.stdout)}`).toBe(0); + }, 65_000); + + afterAll(() => { + dockerSpawnSync(["image", "rm", "--force", fixtureImage], { + encoding: "utf8", + killSignal: "SIGKILL", + timeout: 10_000, + }); + fs.rmSync(containerFixtureRoot, { force: true, recursive: true }); + }); + + it("adds existing and new device groups before the fixed supervisor handoff (#8099)", () => { + const run = runBootstrap([ + "--device-group-gids", + "44,110", + "--", + SUPERVISOR, + "--ready", + "value with space", + ]); + + expect(run.status, run.stderr).toBe(0); + expect(run.stdout).toBe("SUPERVISOR_EXECUTED\n"); + expect(run.after.groupAddLog).toBe("--gid 110 nemoclaw_gpu_110\n"); + expect(run.after.usermodLog).toBe( + "--append --groups video sandbox\n--append --groups nemoclaw_gpu_110 sandbox\n", + ); + expect(run.after.memberships.trim().split(/\s+/)).toEqual(["1000", "44", "110"]); + expect(run.after.supervisorArgv).toEqual(Buffer.from("--ready\0value with space\0", "utf8")); + }); + + it.each([ + { + args: ["--device-group-gids", "44,invalid", "--", SUPERVISOR], + error: "device group ID is invalid", + title: "a later invalid group ID", + }, + { + args: ["--device-group-gids", "44,44", "--", SUPERVISOR], + error: "device group ID is duplicated", + title: "a later duplicate group ID", + }, + { + args: ["--device-group-gids", "2147483648", "--", SUPERVISOR], + error: "device group ID is out of range", + title: "an out-of-range group ID", + }, + { + args: [ + "--device-group-gids", + Array.from({ length: 17 }, (_, index) => String(index + 1)).join(","), + "--", + SUPERVISOR, + ], + error: "device group count is invalid", + title: "more than 16 group IDs", + }, + { + args: ["--device-group-gids", "44", "not-a-delimiter", SUPERVISOR], + error: "supervisor delimiter is missing", + title: "an invalid supervisor delimiter", + }, + { + args: ["--device-group-gids", "44", "--", "/tmp/openshell-sandbox"], + error: "OpenShell supervisor entrypoint is invalid", + title: "a different supervisor entrypoint", + }, + ])("rejects $title before account mutation (#8099)", ({ args, error }) => { + const run = runBootstrap(args); + + expect(run.status).toBe(1); + expect(run.stderr).toContain(`Jetson device-group bootstrap: ${error}`); + expectNoMutation(run); + }); + + it("rejects a missing sandbox account before group mutation (#8099)", () => { + const run = runBootstrap(["--device-group-gids", "44", "--", SUPERVISOR], { + environment: { TEST_SANDBOX_MISSING: "1" }, + }); + + expect(run.status).toBe(1); + expect(run.stderr).toContain("Jetson device-group bootstrap: sandbox user is missing"); + expectNoMutation(run); + }); + + it("rejects a non-root caller before group mutation (#8099)", () => { + const run = runBootstrap(["--device-group-gids", "44", "--", SUPERVISOR], { + environment: { TEST_ID_UID: "1000" }, + }); + + expect(run.status).toBe(1); + expect(run.stderr).toContain("Jetson device-group bootstrap: must run as root"); + expectNoMutation(run); + }); + + it("rejects a malformed existing group before account mutation (#8099)", () => { + const run = runBootstrap(["--device-group-gids", "44", "--", SUPERVISOR], { + environment: { TEST_GETENT_MALFORMED_GID: "44" }, + }); + + expect(run.status).toBe(1); + expect(run.stderr).toContain("Jetson device-group bootstrap: device group record is invalid"); + expectNoMutation(run); + }); + + it("rejects a symlinked group database before account mutation (#8099)", () => { + const run = runBootstrap(["--device-group-gids", "44", "--", SUPERVISOR], { + groupDatabase: "symlink", + }); + + expect(run.status).toBe(1); + expect(run.stderr).toContain( + "Jetson device-group bootstrap: container group database is invalid", + ); + expectNoMutation(run); + }); + + it("stops before supervisor handoff when membership verification fails (#8099)", () => { + const run = runBootstrap(["--device-group-gids", "110", "--", SUPERVISOR], { + environment: { TEST_USERMOD_NOOP: "1" }, + }); + + expect(run.status).toBe(1); + expect(run.stderr).toContain( + "Jetson device-group bootstrap: sandbox membership verification failed", + ); + expect(run.after.supervisorArgv).toBeNull(); + expect(run.after.memberships).toBe(run.before.memberships); + }); +}); diff --git a/test/openclaw-final-image-layout.test.ts b/test/openclaw-final-image-layout.test.ts index 16230300267..7a708f9ccb7 100644 --- a/test/openclaw-final-image-layout.test.ts +++ b/test/openclaw-final-image-layout.test.ts @@ -92,6 +92,7 @@ describe("OpenClaw final image layout", () => { "COPY scripts/lib/entrypoint-env-wrapper.sh /usr/local/lib/nemoclaw/entrypoint-env-wrapper.sh", "COPY scripts/lib/gateway-supervisor.sh /usr/local/lib/nemoclaw/gateway-supervisor.sh", "COPY scripts/lib/sandbox-rlimits.sh /usr/local/lib/nemoclaw/sandbox-rlimits.sh", + "COPY scripts/jetson-device-group-bootstrap.sh /usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh", "COPY scripts/lib/openclaw_device_approval_policy.py /usr/local/lib/nemoclaw/openclaw_device_approval_policy.py", "COPY scripts/lib/clean_runtime_shell_env_shim.py /usr/local/lib/nemoclaw/clean_runtime_shell_env_shim.py", "COPY scripts/lib/normalize_mutable_config_perms.py /usr/local/lib/nemoclaw/normalize_mutable_config_perms.py", @@ -164,6 +165,7 @@ describe("OpenClaw final image layout", () => { "/usr/local/lib/nemoclaw/patch-openclaw-gateway-daemon-dialback.mts 'root:root:755'", "/usr/local/bin/nemoclaw-managed-bootstrap 'root:root:755'", "/usr/local/lib/nemoclaw/managed-bootstrap-trampoline.sh 'root:root:444'", + "/usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh 'root:root:500'", "/usr/local/bin/nemoclaw-gateway-control 'root:root:700'", "/usr/local/lib/nemoclaw/state-dir-guard.py 'root:root:500'", "/usr/local/share/nemoclaw/state-lock-plan.json 'root:root:444'", diff --git a/test/sandbox-build-context.test.ts b/test/sandbox-build-context.test.ts index f75c936d424..65724b6c373 100644 --- a/test/sandbox-build-context.test.ts +++ b/test/sandbox-build-context.test.ts @@ -215,6 +215,7 @@ describe("sandbox build context staging", () => { writeFixture(path.join("scripts", "managed-startup-hold.sh")); writeFixture(path.join("scripts", "managed-bootstrap-entrypoint.c")); writeFixture(path.join("scripts", "managed-bootstrap-trampoline.sh")); + writeFixture(path.join("scripts", "jetson-device-group-bootstrap.sh")); writeFixture(path.join("scripts", "gateway-control.sh")); writeFixture(path.join("scripts", "managed-gateway-control.py")); writeFixture(path.join("scripts", "state-dir-guard.py")); @@ -851,6 +852,9 @@ describe("sandbox build context staging", () => { expect(fs.existsSync(path.join(buildCtx, "scripts", "managed-bootstrap-trampoline.sh"))).toBe( true, ); + expect( + fs.existsSync(path.join(buildCtx, "scripts", "jetson-device-group-bootstrap.sh")), + ).toBe(true); expect(fs.existsSync(path.join(buildCtx, "scripts", "gateway-control.sh"))).toBe(true); expect(fs.existsSync(path.join(buildCtx, "scripts", "managed-gateway-control.py"))).toBe( true, diff --git a/test/sandbox-provisioning-helper-permissions.test.ts b/test/sandbox-provisioning-helper-permissions.test.ts index 0d411300f1a..ac9febb9c57 100644 --- a/test/sandbox-provisioning-helper-permissions.test.ts +++ b/test/sandbox-provisioning-helper-permissions.test.ts @@ -85,6 +85,7 @@ describe("sandbox provisioning: copied OpenClaw helper permissions (#2861)", () const nestedPluginFile = path.join(nestedPluginDir, "helper.js"); const gatewayControlPath = path.join(localBin, "nemoclaw-gateway-control"); const gatewaySupervisorPath = path.join(localLib, "gateway-supervisor.sh"); + const jetsonDeviceGroupBootstrapPath = path.join(localLib, "jetson-device-group-bootstrap.sh"); const stateDirGuardPath = path.join(localLib, "state-dir-guard.py"); const stateLockPlanPath = path.join(localShare, "state-lock-plan.json"); const configGuardPath = path.join(localLib, "openclaw-config-guard.py"); @@ -99,6 +100,7 @@ describe("sandbox provisioning: copied OpenClaw helper permissions (#2861)", () path.join(localLib, "sandbox-init.sh"), path.join(localLib, "sandbox-rlimits.sh"), gatewaySupervisorPath, + jetsonDeviceGroupBootstrapPath, stateDirGuardPath, stateLockPlanPath, configGuardPath, @@ -161,6 +163,7 @@ describe("sandbox provisioning: copied OpenClaw helper permissions (#2861)", () expect((fs.statSync(nestedPluginFile).mode & 0o777).toString(8)).toBe("644"); expect((fs.statSync(gatewayControlPath).mode & 0o777).toString(8)).toBe("700"); expect((fs.statSync(gatewaySupervisorPath).mode & 0o777).toString(8)).toBe("444"); + expect((fs.statSync(jetsonDeviceGroupBootstrapPath).mode & 0o777).toString(8)).toBe("500"); expect((fs.statSync(stateDirGuardPath).mode & 0o777).toString(8)).toBe("500"); expect((fs.statSync(stateLockPlanPath).mode & 0o777).toString(8)).toBe("444"); expect((fs.statSync(configGuardPath).mode & 0o777).toString(8)).toBe("500");