diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 91441b91000..56fb6175928 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1695,14 +1695,19 @@ jobs: hermes-gpu-startup: needs: generate-matrix - if: ${{ contains(format(',{0},', inputs.jobs), ',hermes-gpu-startup,') || contains(format(',{0},', inputs.targets), ',hermes-gpu-startup,') }} + if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && (contains(format(',{0},', inputs.jobs), ',hermes-gpu-startup,') || contains(format(',{0},', inputs.targets), ',hermes-gpu-startup,')) }} runs-on: linux-amd64-gpu-rtxpro6000-latest-1 - timeout-minutes: 75 + timeout-minutes: 90 + strategy: + fail-fast: false + max-parallel: 1 + matrix: + scenario: [native, fallback, compatibility-only] env: E2E_JOB: "1" - E2E_DEFAULT_ENABLED: "0" E2E_TARGET_ID: "hermes-gpu-startup" - E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/hermes-gpu-startup + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/hermes-gpu-startup/${{ matrix.scenario }} + E2E_HERMES_GPU_STARTUP_SCENARIO: ${{ matrix.scenario }} NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_LIVE_E2E: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" @@ -1710,7 +1715,7 @@ jobs: NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_RECREATE_SANDBOX: "1" NEMOCLAW_SANDBOX_GPU: "1" - NEMOCLAW_SANDBOX_NAME: e2e-hermes-gpu-startup + NEMOCLAW_SANDBOX_NAME: e2e-hermes-gpu-startup-${{ matrix.scenario }} NEMOCLAW_ONBOARD_VALIDATION_TIMEOUT_SECONDS: "60" steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 @@ -1718,21 +1723,230 @@ jobs: ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false + - name: Checkout trusted Hermes GPU runtime fixture + if: ${{ matrix.scenario == 'fallback' }} + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + repository: NVIDIA/NemoClaw + ref: ${{ github.workflow_sha }} + path: .trusted-hermes-gpu-fixture-${{ github.run_id }}-${{ github.run_attempt }} + sparse-checkout: tools/e2e/hermes-gpu-docker-runtime-fixture.sh + sparse-checkout-cone-mode: false + persist-credentials: false + + - name: Install trusted Hermes GPU runtime fixture + if: ${{ matrix.scenario == 'fallback' }} + shell: /bin/bash --noprofile --norc -e -o pipefail {0} + env: + BASH_ENV: /dev/null + E2E_HERMES_GPU_STARTUP_SCENARIO: ${{ matrix.scenario }} + ENV: /dev/null + TRUSTED_DISPATCH_SHA: ${{ github.sha }} + TRUSTED_FIXTURE_SHA256: e273c4baa7fe89546d64517cf56eafec30aeda7b355971263605ab1327fade02 + TRUSTED_WORKFLOW_SHA: ${{ github.workflow_sha }} + run: | + set -euo pipefail + trusted_checkout="$GITHUB_WORKSPACE/.trusted-hermes-gpu-fixture-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + trusted_source="$trusted_checkout/tools/e2e/hermes-gpu-docker-runtime-fixture.sh" + trusted_fixture="/usr/local/libexec/nemoclaw/hermes-gpu-docker-runtime-fixture.${GITHUB_RUN_ID}.${GITHUB_RUN_ATTEMPT}.${E2E_HERMES_GPU_STARTUP_SCENARIO}" + trusted_state_root=/var/lib/nemoclaw-e2e + + run_trusted_fixture() { + /usr/bin/sudo -n /usr/bin/env -i \ + PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ + /bin/bash "$trusted_fixture" "$@" + } + + [[ "$TRUSTED_WORKFLOW_SHA" =~ ^[a-f0-9]{40}$ ]] \ + || { echo "Trusted workflow SHA must be an immutable commit" >&2; exit 1; } + [[ "$TRUSTED_DISPATCH_SHA" = "$TRUSTED_WORKFLOW_SHA" ]] \ + || { echo "Trusted fixture must match the dispatched main workflow" >&2; exit 1; } + [[ "$TRUSTED_FIXTURE_SHA256" =~ ^[a-f0-9]{64}$ ]] \ + || { echo "Trusted fixture SHA-256 must be pinned" >&2; exit 1; } + [ "$(/usr/bin/git -C "$trusted_checkout" rev-parse HEAD)" = "$TRUSTED_WORKFLOW_SHA" ] \ + || { echo "Trusted fixture checkout does not match the workflow SHA" >&2; exit 1; } + [ -f "$trusted_source" ] && [ ! -L "$trusted_source" ] \ + || { echo "Trusted Docker fixture must be a regular non-symlink file" >&2; exit 1; } + + /usr/bin/sudo /usr/bin/install -d -o root -g root -m 0755 /usr/local/libexec/nemoclaw + /usr/bin/sudo /usr/bin/install -o root -g root -m 0500 \ + "$trusted_source" "$trusted_fixture" + [ "$(/usr/bin/sudo /usr/bin/stat -c '%a %u %g' "$trusted_fixture")" = "500 0 0" ] \ + || { echo "Trusted Docker fixture ownership or mode is invalid" >&2; exit 1; } + printf '%s %s\n' "$TRUSTED_FIXTURE_SHA256" "$trusted_fixture" \ + | /usr/bin/sudo /usr/bin/sha256sum -c - + /usr/bin/sudo /usr/bin/cmp -s "$trusted_source" "$trusted_fixture" \ + || { echo "Installed Docker fixture does not match trusted workflow code" >&2; exit 1; } + /usr/bin/sudo /usr/bin/install -d -o root -g root -m 0700 "$trusted_state_root" + + # Recover any root-owned snapshot left by a hard-cancelled earlier run before + # PR-controlled build or test code executes on this persistent GPU runner. + if ! /usr/bin/sudo /usr/bin/find "$trusted_state_root" -mindepth 1 -maxdepth 1 \ + -type d -name 'hermes-gpu-fallback-docker-runtime.*' -print0 \ + | while IFS= read -r -d '' stale_state_dir; do + if ! run_trusted_fixture restore \ + "$stale_state_dir" /etc/docker/daemon.json >/dev/null; then + exit 1 + fi + done; then + echo "Could not recover stale Docker fallback state" >&2 + exit 1 + fi + - *dockerhub-auth - name: Prepare E2E workspace uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28 + - name: Reassert trusted Node runtime + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + env: + BASH_ENV: /dev/null + E2E_HERMES_GPU_STARTUP_SCENARIO: ${{ matrix.scenario }} + ENV: /dev/null + NODE_OPTIONS: "" + with: + node-version: "22" + - name: Run Hermes GPU startup live Vitest test + shell: /bin/bash --noprofile --norc -e -o pipefail {0} + env: + BASH_ENV: /dev/null + E2E_HERMES_GPU_STARTUP_SCENARIO: ${{ matrix.scenario }} + ENV: /dev/null run: | set -euo pipefail + + if [ "$E2E_HERMES_GPU_STARTUP_SCENARIO" = fallback ]; then + umask 077 + daemon_json=/etc/docker/daemon.json + trusted_fixture="/usr/local/libexec/nemoclaw/hermes-gpu-docker-runtime-fixture.${GITHUB_RUN_ID}.${GITHUB_RUN_ATTEMPT}.${E2E_HERMES_GPU_STARTUP_SCENARIO}" + trusted_state_root=/var/lib/nemoclaw-e2e + + run_trusted_fixture() { + /usr/bin/sudo -n /usr/bin/env -i \ + PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ + /bin/bash "$trusted_fixture" "$@" + } + + mkdir -p "$E2E_ARTIFACT_DIR" + [ "$(/usr/bin/sudo /usr/bin/stat -c '%a %u %g' "$trusted_fixture")" = "500 0 0" ] \ + || { echo "Trusted Docker fixture ownership or mode changed" >&2; exit 1; } + /usr/bin/sudo /usr/bin/install -d -o root -g root -m 0700 "$trusted_state_root" + state_dir="$(/usr/bin/sudo /usr/bin/mktemp -d \ + "$trusted_state_root/hermes-gpu-fallback-docker-runtime.${GITHUB_RUN_ID}.${GITHUB_RUN_ATTEMPT}.fallback.XXXXXX")" + /usr/bin/sudo /usr/bin/chown root:root "$state_dir" + /usr/bin/sudo /usr/bin/chmod 0700 "$state_dir" + + # SOURCE_OF_TRUTH_REVIEW (Hermes GPU fallback fixture; #6110): + # invalidState: a cancelled fallback test leaves the runner-global Docker default + # runtime changed, contaminating later OpenShell jobs on this self-hosted runner. + # sourceBoundary: daemon mutation uses only immutable root-owned workflow_sha + # code; the PR test cannot replace the helper or its root-owned snapshot state. + # whyNotSourceFix: production must fail closed when GPU attachment is ambiguous; + # this fixture needs a real no-GPU partial container on an nvidia-default runner. + # regressionTest: hermes-gpu-startup-workflow-boundary requires this same-step trap. + # removalCondition: remove the daemon fixture when OpenShell can create a provably + # GPU-unattached partial sandbox without consulting Docker's default runtime. + restore_docker_default_runtime() { + local command_status=$? + local restore_status=0 + local restored_runtime="" + trap - EXIT INT TERM + set +e + restored_runtime="$(run_trusted_fixture restore "$state_dir" "$daemon_json")" + restore_status=$? + if [ "$restore_status" -eq 0 ] && [ -n "$restored_runtime" ]; then + printf '%s\n' "$restored_runtime" \ + >"$E2E_ARTIFACT_DIR/docker-default-runtime-restored.txt" \ + || restore_status=1 + fi + if [ "$restore_status" -ne 0 ]; then + exit 1 + fi + exit "$command_status" + } + + # OpenShell leaves Runtime unset after the wrapper strips --gpu, so Docker + # otherwise substitutes its daemon default. Force runc only within this + # trap-guarded process so production can prove the partial container has no GPU. + trap restore_docker_default_runtime EXIT + trap 'exit 130' INT + trap 'exit 143' TERM + original_runtime="$(run_trusted_fixture capture "$state_dir" "$daemon_json")" + printf '%s\n' "$original_runtime" \ + >"$E2E_ARTIFACT_DIR/docker-default-runtime-before.txt" + selected_runtime="$(run_trusted_fixture select-runc "$state_dir" "$daemon_json")" + printf '%s\n' "$selected_runtime" \ + >"$E2E_ARTIFACT_DIR/docker-default-runtime-during.txt" + fi + npx vitest run --project e2e-live \ test/e2e/live/hermes-gpu-startup.test.ts \ --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts + - name: Recover Docker daemon after Hermes GPU fallback fixture + if: always() + shell: /bin/bash --noprofile --norc -e -o pipefail {0} + env: + BASH_ENV: /dev/null + E2E_HERMES_GPU_STARTUP_SCENARIO: ${{ matrix.scenario }} + ENV: /dev/null + run: | + set -euo pipefail + if [ "$E2E_HERMES_GPU_STARTUP_SCENARIO" != fallback ]; then + exit 0 + fi + + recovery_failed=0 + restored_runtime="" + trusted_fixture="/usr/local/libexec/nemoclaw/hermes-gpu-docker-runtime-fixture.${GITHUB_RUN_ID}.${GITHUB_RUN_ATTEMPT}.${E2E_HERMES_GPU_STARTUP_SCENARIO}" + trusted_state_root=/var/lib/nemoclaw-e2e + state_prefix="hermes-gpu-fallback-docker-runtime.${GITHUB_RUN_ID}.${GITHUB_RUN_ATTEMPT}.fallback." + + run_trusted_fixture() { + /usr/bin/sudo -n /usr/bin/env -i \ + PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ + /bin/bash "$trusted_fixture" "$@" + } + + if ! /usr/bin/sudo /usr/bin/find "$trusted_state_root" -mindepth 1 -maxdepth 1 \ + -type d -name "${state_prefix}*" -print0 \ + | while IFS= read -r -d '' state_dir; do + if ! restored_runtime="$(run_trusted_fixture restore \ + "$state_dir" /etc/docker/daemon.json)"; then + exit 1 + fi + if [ -n "$restored_runtime" ]; then + printf '%s\n' "$restored_runtime" \ + >"$E2E_ARTIFACT_DIR/docker-default-runtime-restored.txt" + fi + done; then + recovery_failed=1 + fi + if [ "$recovery_failed" -ne 0 ]; then + echo "Independent Docker daemon recovery could not prove restoration" >&2 + exit 1 + fi + + - name: Remove trusted Hermes GPU runtime fixture + if: ${{ always() && matrix.scenario == 'fallback' }} + shell: /bin/bash --noprofile --norc -e -o pipefail {0} + env: + BASH_ENV: /dev/null + E2E_HERMES_GPU_STARTUP_SCENARIO: ${{ matrix.scenario }} + ENV: /dev/null + run: | + set -euo pipefail + trusted_fixture="/usr/local/libexec/nemoclaw/hermes-gpu-docker-runtime-fixture.${GITHUB_RUN_ID}.${GITHUB_RUN_ATTEMPT}.${E2E_HERMES_GPU_STARTUP_SCENARIO}" + /usr/bin/sudo /usr/bin/rm -f -- "$trusted_fixture" + - name: Upload Hermes GPU startup artifacts if: always() uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 + with: + name: e2e-hermes-gpu-startup-${{ matrix.scenario }} + path: e2e-artifacts/live/hermes-gpu-startup/${{ matrix.scenario }}/ - name: Clean up Docker auth if: always() diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index cc217d27a50..ab912bc7a07 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -719,12 +719,18 @@ Use `--no-gpu` to opt out when you want host-side inference providers only and d Use `--gpu` to require GPU passthrough and fail fast if an NVIDIA GPU is not detected. Use `--sandbox-gpu` or `--no-sandbox-gpu` to control only direct NVIDIA GPU access inside the sandbox. Use `--sandbox-gpu --sandbox-gpu-device ` to pass a specific OpenShell GPU device selector to `openshell sandbox create`; device selectors require explicit sandbox GPU enablement. -On ordinary native Linux Docker-driver hosts with usable CDI, NemoClaw uses OpenShell native GPU injection by default. -On Docker Desktop WSL and Jetson/Tegra, NemoClaw creates the sandbox first and then recreates the OpenShell-managed Docker container with NVIDIA GPU access by default. -When you force this compatibility path on ordinary native Linux, NemoClaw uses an available NVIDIA CDI spec before falling back to Docker `--gpus all` or the NVIDIA runtime. +On ordinary native Linux Docker-driver hosts, NemoClaw uses native OpenShell GPU injection by default and never broadens confinement automatically. +Set `NEMOCLAW_DOCKER_GPU_PATCH=fallback` to explicitly authorize one native attempt followed by one compatibility retry. +NemoClaw permits the retry only after it confirms either a trusted host-side GPU routing failure or an explicit driver proof plus exact-container host configuration showing that no GPU was attached. +It then saves redacted diagnostics and removes the incomplete sandbox before retrying. +Sandbox-reported CUDA output alone never authorizes the broader compatibility envelope, even when the operator enabled fallback. +That case fails closed and points to the explicit `NEMOCLAW_DOCKER_GPU_PATCH=1` compatibility-only control. +NemoClaw retries only after it verifies that no OpenShell-managed Docker container labeled for that sandbox remains; if cleanup cannot be proven safe, onboarding stops and prints cleanup guidance instead. +On Docker Desktop WSL and Jetson/Tegra, automatic GPU onboarding uses the compatibility path directly. +On ordinary native Linux, the compatibility path uses an available NVIDIA CDI spec before falling back to Docker `--gpus all` or the NVIDIA runtime. On Docker Desktop WSL, the compatibility path skips CDI and tries Docker `--gpus all` before the NVIDIA runtime. On Jetson/Tegra hosts, the compatibility path uses the NVIDIA runtime and adds the host group IDs that own `/dev/nvmap` and `/dev/nvhost-*` so the sandbox user can initialize CUDA. -If the patch fails, onboarding keeps diagnostics and prints a manual cleanup command rather than deleting the failed sandbox automatically. +If the compatibility attempt fails, onboarding keeps its diagnostics and the failed sandbox in place and prints a manual cleanup command. Prerequisites: @@ -737,9 +743,11 @@ When GPU passthrough is enabled and a gateway already exists without it, onboard If no other registered sandbox depends on that gateway, or if `--recreate-sandbox` is recreating the only registered sandbox with the same name, onboarding cleans up the stale gateway and continues. If other sandboxes depend on the gateway or Docker state is unclear, onboarding exits without cleanup and prints targeted destroy or gateway-removal guidance. To add GPU to an existing sandbox, rerun with `--recreate-sandbox`. -Leave `NEMOCLAW_DOCKER_GPU_PATCH` unset or set it to `auto` to use the platform default. -Set `NEMOCLAW_DOCKER_GPU_PATCH=1` to force the legacy Docker container-swap path on ordinary native Linux. -Set `NEMOCLAW_DOCKER_GPU_PATCH=0` to select native OpenShell GPU injection on ordinary native Linux or Jetson/Tegra. +Leave `NEMOCLAW_DOCKER_GPU_PATCH` unset or set it to `auto` for native-only GPU onboarding on ordinary native Linux. +Set `NEMOCLAW_DOCKER_GPU_PATCH=fallback` to explicitly opt into one bounded native-to-compatibility retry on ordinary native Linux. +Set `NEMOCLAW_DOCKER_GPU_PATCH=0` to require native OpenShell GPU injection on ordinary native Linux or Jetson/Tegra. +Set `NEMOCLAW_DOCKER_GPU_PATCH=1` to use only the compatibility path on ordinary native Linux. +Other legacy nonzero values keep that behavior through the `v0.0.x` release line and will be removed in `v0.1.0`. Use `NEMOCLAW_DOCKER_GPU_PATCH=0` on Jetson/Tegra only for troubleshooting because it bypasses Tegra device-group propagation and CUDA may not initialize. Docker Desktop WSL ignores `NEMOCLAW_DOCKER_GPU_PATCH=0` because GPU passthrough on that runtime requires the compatibility patch. Use `--no-sandbox-gpu`, `--no-gpu`, or `NEMOCLAW_SANDBOX_GPU=0` when you want to disable sandbox GPU passthrough on Docker Desktop WSL. @@ -2383,7 +2391,8 @@ The command backs up workspace state, destroys the old sandbox (including its ho Credentials are stripped from backups before storage. Policy presets applied to the old sandbox are reapplied to the new one so your egress rules survive the rebuild. The replacement uses the recorded compatible-endpoint reasoning mode and web search selection instead of ambient shell values. -The recorded sandbox GPU mode is preserved across rebuild. +Rebuild preserves the recorded sandbox GPU enablement mode and, for an explicitly enabled sandbox, its recorded device selector. +It re-resolves the Docker-driver GPU route from the current host and current `NEMOCLAW_DOCKER_GPU_PATCH` value, so native-only, explicitly authorized native-with-fallback, and compatibility-only routing may differ from the original onboarding run. A rebuild preserves the recorded tool-disclosure mode unless `--tool-disclosure` explicitly changes it; it ignores an ambient `NEMOCLAW_TOOL_DISCLOSURE` value while recreating the sandbox. A rebuild preserves the recorded Deep Agents Code observability choice and matching local OTLP policy state unless `--observability` or `--no-observability` explicitly changes them. A rebuild preserves the recorded Deep Agents Code auto-approval capability unless `--dcode-auto-approval` explicitly changes it. @@ -3380,7 +3389,7 @@ Set them before running `$$nemoclaw onboard`. | `NEMOCLAW_SANDBOX_GPU_DEVICE` | OpenShell GPU device selector | Selects the GPU device passed with `openshell sandbox create --gpu-device`. Requires explicit sandbox GPU enablement with `NEMOCLAW_SANDBOX_GPU=1` (or `--sandbox-gpu` for CLI-driven onboarding); otherwise onboarding rejects the selector instead of treating it as an implicit opt-in. | | `NEMOCLAW_SANDBOX_BASE_IMAGE_REFRESH` | `1`, `true`, `yes`, or `on` to enable | Bypasses recorded sandbox base-image resolution metadata during onboarding, recreation, and rebuild. NemoClaw reruns candidate resolution but can still use a compatible image from Docker's local image store. Versioned release candidates that exist locally but fail validation are refreshed from the registry once during normal resolution. This setting does not discard onboarding session state. | | `NEMOCLAW_SANDBOX_BASE_LOCAL_BUILD` | unset or `auto` (default); `1`, `true`, `yes`, or `on` to enable; `0`, `false`, `no`, or `off` to disable | Controls whether base-image resolution may build a compatible image locally. The default allows builds during normal CLI runs and disables them when `NODE_ENV=test` or `VITEST=true`. When source inputs or a missing/incompatible release-version base require a fresh build, disabling local builds makes resolution fail instead of using an unproven image. | -| `NEMOCLAW_DOCKER_GPU_PATCH` | unset, `auto`, `1`, or `0` | Selects Linux Docker-driver GPU routing. Unset or `auto` uses native OpenShell GPU injection on ordinary native Linux and the compatibility patch on Docker Desktop WSL and Jetson/Tegra. `1` forces the compatibility patch. `0` selects native injection on ordinary native Linux and Jetson/Tegra, but Docker Desktop WSL ignores it. On Jetson/Tegra, use `0` only for troubleshooting because it bypasses the device-group propagation needed for CUDA. | +| `NEMOCLAW_DOCKER_GPU_PATCH` | unset, `auto`, `fallback`, `1`, or `0`; other legacy nonzero values remain accepted through `v0.0.x` and will be removed in `v0.1.0` | Selects Linux Docker-driver GPU routing. Unset, `auto`, or `0` uses native OpenShell GPU injection on ordinary native Linux. `fallback` explicitly opts into one native attempt followed by one bounded compatibility retry when trusted host evidence identifies a GPU-routing failure. `1` and legacy nonzero values select the compatibility patch from the outset. Docker Desktop WSL and Jetson/Tegra use the compatibility path by default; Docker Desktop WSL ignores `0`, while Jetson/Tegra accepts `0` only as a troubleshooting override that bypasses device-group propagation. | | `NEMOCLAW_OPENSHELL_GATEWAY_CONTAINER_PATCH` | `1` to enable; disabled by default | This setting explicitly opts into the Linux gateway compatibility container for an older host ABI or a diagnostic run; use it only on a trusted local host because it uses host networking and mounts the Docker socket read-only even though the socket still exposes the privileged Docker API; prefer OpenShell 0.0.72's directly supported glibc 2.28+ path; see the [OpenShell 0.0.72 compatibility review](/user-guide/openclaw/security/openshell-0.0.72-compatibility-review#source-of-truth-boundaries) for details. | | `NEMOCLAW_OPENSHELL_GATEWAY_BIN` | path | Advanced override for the `openshell-gateway` binary used by the Linux Docker-driver standalone fallback. Defaults to the binary next to `openshell`, then common install paths. | | `NEMOCLAW_OPENSHELL_SANDBOX_BIN` | path | Advanced override for the `openshell-sandbox` binary used by the Linux Docker-driver standalone fallback. Defaults to the binary next to `openshell`, then common install paths. | diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 17953d27bb3..ec155d86c74 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -669,8 +669,8 @@ On Windows or WSL hosts, some systems report a placeholder display adapter name NVIDIA NIM and GPU-backed sandbox setup require a real NVIDIA GPU. If NemoClaw rejects the detected GPU name during preflight, select a CPU or remote inference provider, or move the setup to a host with a supported NVIDIA GPU and current drivers. -Jetson hosts can still run NemoClaw, but sandbox GPU passthrough is not supported there. -If onboarding reports that sandbox GPU passthrough is unavailable on Jetson, rerun onboarding without `--sandbox-gpu`. +Jetson/Tegra hosts support sandbox GPU passthrough through the compatibility route. +Onboarding detects those hosts separately and propagates the supplementary groups required for `/dev/nvmap` and `/dev/nvhost-*`; if that path fails, follow the Jetson/Tegra compatibility guidance below instead of treating a missing `nvidia-smi` result as a placeholder adapter. ### Colima socket not detected (macOS) @@ -2246,12 +2246,57 @@ docker run --rm --gpus all nvcr.io/nvidia/k8s/cuda-sample:nbody nbody -gpu -benc If GPU passthrough is not required on this host, rerun onboarding with `--no-gpu` instead. -### Docker GPU patch failed during sandbox create - -On ordinary native Linux Docker-driver hosts with usable CDI, NemoClaw uses OpenShell native GPU injection by default. -On Docker Desktop WSL and Jetson/Tegra, NemoClaw uses the compatibility patch by default, which creates the sandbox first and then recreates the OpenShell-managed Docker container with NVIDIA GPU flags. -Set `NEMOCLAW_DOCKER_GPU_PATCH=1` to force this compatibility path on ordinary native Linux for diagnostics or older host compatibility. -If that compatibility patch fails, onboarding leaves the failed sandbox and diagnostic bundle in place so you can inspect the OpenShell and Docker state. +### GPU routing or compatibility patch failed + +The route depends on the host environment and the operator control. +Identify the matching path before applying the recovery guidance. + +| Symptom | Route or stage | Recovery | +| --- | --- | --- | +| Native `--gpu` is rejected, host runtime evidence identifies GPU injection failure, or an explicit driver proof fails and host configuration confirms no GPU attachment | Ordinary Linux native attempt | The default native-only route stops. Retry with `NEMOCLAW_DOCKER_GPU_PATCH=fallback` only if you explicitly accept one bounded compatibility retry, or use `=1` to select compatibility before creation. | +| `Cleanup could not be proven safe` | Native-to-compatibility handoff | Run the printed sandbox deletion command, verify both the gateway row and OpenShell-managed Docker containers labeled for that sandbox are absent, then rerun onboarding. | +| The patched container exits or the compatibility attempt fails | Compatibility recreation | Inspect the saved diagnostics, repair the NVIDIA Container Toolkit/CDI configuration, clean up the failed sandbox, and rerun onboarding. | +| A recreated container inherits only a loopback DNS stub and no usable upstream | Compatibility DNS fallback | Repair the host's `systemd-resolved` upstream configuration, then rerun onboarding. | + +#### Ordinary native Linux bounded fallback + +Ordinary Linux GPU onboarding uses native OpenShell GPU injection and stops on failure by default. +Unset, `auto`, and `0` all preserve this native-only confinement boundary. +`NEMOCLAW_DOCKER_GPU_PATCH=fallback` is the explicit operator authorization for one bounded retry. +With that control set, if sandbox creation rejects the native GPU flag before progress, the exact OpenShell-managed container labeled for that sandbox records a host runtime GPU-injection error, or an explicit `nvidia-smi` driver proof fails while that container's immutable host configuration confirms that no GPU was attached, NemoClaw captures redacted diagnostics, deletes the incomplete sandbox, verifies that no OpenShell-managed Docker container labeled for that sandbox remains, and retries exactly once through the compatibility path. +Free-form build/list text and sandbox-reported CUDA output never independently authorize the broader retry. +Without corroborating host evidence, onboarding fails closed even when `fallback` is set and directs the operator to clean up and explicitly select compatibility with `NEMOCLAW_DOCKER_GPU_PATCH=1` if desired. +Before the authorized retry, NemoClaw warns that the legacy GPU compatibility envelope recreates the OpenShell-managed Docker container and may relax container confinement compared with native injection. +Specifically, compatibility recreation adds `SYS_PTRACE`, adds `apparmor=unconfined` when the original container has no AppArmor option, and uses a compatibility policy that makes `/proc` writable for the NVIDIA runtime's process-name initialization. +These broader settings are why onboarding warns before the swap and retains a native-only opt-out. +NemoClaw verifies cleanup with two stable checks (that sandbox is absent from the gateway list and no OpenShell-managed Docker containers labeled for that sandbox remain) before retrying through the compatibility path. +Cleanup is polled at most five times, one second apart, and both conditions must pass twice consecutively; otherwise onboarding stops before the retry. +These fail-closed safety limits are the internal constants `STABLE_ABSENCE_CHECKS` (2), `MAX_CLEANUP_ATTEMPTS` (5), and `CLEANUP_POLL_INTERVAL_MS` (1,000 ms); they are not configurable through environment variables. +The first observation is immediate, so the default bound performs at most four one-second sleeps plus the five gateway/container queries. +The bounds are intentionally fixed. +Allowing environment input to weaken or extend the cleanup proof would make a security gate deployment-dependent. +On a host that cannot prove absence within the bound, onboarding fails closed; select compatibility from the outset with `NEMOCLAW_DOCKER_GPU_PATCH=1` instead of weakening the handoff proof. +If deletion or container cleanup cannot be proven safe, onboarding stops before the retry and prints manual cleanup guidance. +Image build, upload, TLS, provider, policy, dashboard, and inference failures stay on their existing error paths and do not trigger the GPU compatibility fallback. +Set `NEMOCLAW_DOCKER_GPU_PATCH=1` to use only the compatibility path for diagnostics or older host compatibility. +Other legacy nonzero values keep that behavior through the `v0.0.x` release line and will be removed in `v0.1.0`. + +#### Docker Desktop WSL compatibility route + +Automatic GPU onboarding uses the compatibility path directly; it does not make a native attempt first. +The path creates the sandbox and then recreates the OpenShell-managed Docker container with NVIDIA GPU flags. +`NEMOCLAW_DOCKER_GPU_PATCH=0` is ignored because this runtime requires the compatibility patch for GPU passthrough, and onboarding logs a warning when it is set. +To skip GPU passthrough entirely, rerun with `--no-gpu` or set `NEMOCLAW_SANDBOX_GPU=0`. + +#### Jetson and Tegra compatibility default + +Automatic GPU onboarding uses the compatibility path directly; it does not make a native attempt first. +The path recreates the OpenShell-managed Docker container with NVIDIA GPU flags and propagates the supplementary groups required for `/dev/nvmap` and `/dev/nvhost-*` access. +Use `NEMOCLAW_DOCKER_GPU_PATCH=0` only for troubleshooting because it bypasses that group propagation and CUDA may not initialize. + +#### Common compatibility-path recovery + +If the compatibility attempt fails on any host, onboarding leaves the failed sandbox and diagnostic bundle in place so you can inspect the OpenShell and Docker state. Starting with NemoClaw v0.0.43, the standard installer handles the `/proc//task//comm` permission case during this patch path. @@ -2266,11 +2311,6 @@ openshell sandbox delete Fix the NVIDIA Container Toolkit or CDI configuration reported in the diagnostics, clean up the failed sandbox, then rerun onboarding. If you do not need GPU access inside the sandbox, rerun with `--no-sandbox-gpu`. -Set `NEMOCLAW_DOCKER_GPU_PATCH=0` to select native OpenShell GPU injection on ordinary native Linux or Jetson/Tegra. -Use this override on Jetson/Tegra only for troubleshooting because it bypasses the compatibility path's `/dev/nvmap` and `/dev/nvhost-*` group propagation, so CUDA may not initialize. -On Docker Desktop WSL, the patch is required for GPU passthrough. -`NEMOCLAW_DOCKER_GPU_PATCH=0` is ignored on that runtime, and onboarding logs a warning when it is set there. -To skip GPU passthrough entirely on Docker Desktop WSL, rerun with `--no-gpu` or set `NEMOCLAW_SANDBOX_GPU=0`. If sandbox creation fails with `CDI device injection failed: unresolvable CDI devices nvidia.com/gpu=all`, the OpenShell gateway tried `docker create --device nvidia.com/gpu=all` and Docker could not resolve the CDI spec. This injection happens inside the gateway, so `NEMOCLAW_DOCKER_GPU_PATCH=0` does not bypass it. diff --git a/src/lib/actions/sandbox/rebuild-target-runtime.test.ts b/src/lib/actions/sandbox/rebuild-target-runtime.test.ts new file mode 100644 index 00000000000..bd0939c6b44 --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-target-runtime.test.ts @@ -0,0 +1,126 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + detectGpu: vi.fn(), + enforceDockerGpuPatchPreserveNetwork: vi.fn(), + isDockerDesktopWslRuntime: vi.fn(), + isLinuxDockerDriverGatewayEnabled: vi.fn(), + preflightRebuildCredentials: vi.fn(), +})); + +vi.mock("../../inference/nim", () => ({ + detectGpu: mocks.detectGpu, +})); + +vi.mock("../../onboard/docker-driver-platform", () => ({ + isLinuxDockerDriverGatewayEnabled: mocks.isLinuxDockerDriverGatewayEnabled, +})); + +vi.mock("../../onboard/docker-gpu-local-inference", () => ({ + enforceDockerGpuPatchPreserveNetwork: mocks.enforceDockerGpuPatchPreserveNetwork, +})); + +vi.mock("../../onboard/docker-gpu-sandbox-create", () => ({ + isDockerDesktopWslRuntime: mocks.isDockerDesktopWslRuntime, +})); + +vi.mock("./rebuild-credential-preflight", () => ({ + preflightRebuildCredentials: mocks.preflightRebuildCredentials, +})); + +import type { RebuildSandboxEntry } from "./rebuild-flow-helpers"; +import type { RebuildRecreateOnboardOpts } from "./rebuild-gpu-opt-out"; +import type { RebuildTargetConfig } from "./rebuild-target-config"; +import { preflightRebuildTargetRuntime } from "./rebuild-target-runtime"; + +const platformDescriptor = Object.getOwnPropertyDescriptor(process, "platform")!; +const TARGET = { + resumeConfig: { + provider: "ollama-local", + model: "test-model", + }, + durableConfig: { + webSearchConfig: null, + }, + hermesToolGateways: [], + credentialEnv: null, + fromDockerfile: null, + agentDefinition: null, +} as unknown as RebuildTargetConfig; +const ENTRY = { mcp: null } as unknown as RebuildSandboxEntry; +const RECREATE_OPTIONS = { + sandboxGpu: "enable", + sandboxGpuDevice: null, + controlUiPort: 18789, + targetGatewayPort: 8080, +} as RebuildRecreateOnboardOpts; + +describe("preflightRebuildTargetRuntime GPU route", () => { + beforeEach(() => { + vi.clearAllMocks(); + Object.defineProperty(process, "platform", { ...platformDescriptor, value: "linux" }); + mocks.detectGpu.mockReturnValue({ + type: "nvidia", + name: "NVIDIA test GPU", + count: 1, + totalMemoryMB: 24_576, + perGpuMB: 24_576, + nimCapable: true, + platform: "linux", + }); + mocks.isLinuxDockerDriverGatewayEnabled.mockReturnValue(true); + mocks.isDockerDesktopWslRuntime.mockReturnValue(false); + mocks.enforceDockerGpuPatchPreserveNetwork.mockResolvedValue(false); + mocks.preflightRebuildCredentials.mockReturnValue(true); + }); + + afterEach(() => { + Object.defineProperty(process, "platform", platformDescriptor); + vi.unstubAllEnvs(); + }); + + it.each([ + { control: "auto", selectedRoute: "native" }, + { control: "fallback", selectedRoute: "native" }, + { control: "1", selectedRoute: "compatibility" }, + ] as const)("passes the $selectedRoute rebuild GPU route into network preflight (#6110)", async ({ + control, + selectedRoute, + }) => { + vi.stubEnv("NEMOCLAW_DOCKER_GPU_PATCH", control); + const log = vi.fn(); + const bail = vi.fn((message: string): never => { + throw new Error(message); + }); + + await expect( + preflightRebuildTargetRuntime(TARGET, ENTRY, RECREATE_OPTIONS, log, bail, { + skipImagePreflight: true, + }), + ).resolves.toEqual({ + ok: true, + preparedImage: null, + requiresGatewayProviderReconfigure: false, + }); + + expect(mocks.enforceDockerGpuPatchPreserveNetwork).toHaveBeenCalledOnce(); + expect(mocks.enforceDockerGpuPatchPreserveNetwork).toHaveBeenCalledWith( + "ollama-local", + expect.objectContaining({ + sandboxGpuEnabled: true, + hostGpuPlatform: "linux", + sandboxGpuDevice: null, + }), + { + dockerDriverGateway: true, + selectedRoute, + gatewayPort: 8080, + log, + }, + ); + expect(bail).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/actions/sandbox/rebuild-target-runtime.ts b/src/lib/actions/sandbox/rebuild-target-runtime.ts index 446fd5a42e0..3e7f7b95a30 100644 --- a/src/lib/actions/sandbox/rebuild-target-runtime.ts +++ b/src/lib/actions/sandbox/rebuild-target-runtime.ts @@ -10,6 +10,8 @@ import { import { shouldManageDashboardForAgent } from "../../onboard/dashboard-runtime"; import { isLinuxDockerDriverGatewayEnabled } from "../../onboard/docker-driver-platform"; import { enforceDockerGpuPatchPreserveNetwork } from "../../onboard/docker-gpu-local-inference"; +import { initialDockerGpuRoute, resolveDockerGpuRoutePlan } from "../../onboard/docker-gpu-route"; +import { isDockerDesktopWslRuntime } from "../../onboard/docker-gpu-sandbox-create"; import { resolveSandboxGpuConfig } from "../../onboard/sandbox-gpu-mode"; import { agentSupportsWebSearchProvider } from "../../onboard/web-search-support"; import { redact } from "../../security/redact"; @@ -130,8 +132,16 @@ export async function preflightRebuildTargetRuntime( return { ok: false }; } try { + const dockerDriverGateway = isLinuxDockerDriverGatewayEnabled(); + const selectedRoute = initialDockerGpuRoute( + resolveDockerGpuRoutePlan(sandboxGpuConfig, { + dockerDriverGateway, + dockerDesktopWsl: isDockerDesktopWslRuntime(), + }), + ); await enforceDockerGpuPatchPreserveNetwork(target.resumeConfig.provider, sandboxGpuConfig, { - dockerDriverGateway: isLinuxDockerDriverGatewayEnabled(), + dockerDriverGateway, + selectedRoute, gatewayPort: recreateOptions.targetGatewayPort, log, }); diff --git a/src/lib/build-context.test.ts b/src/lib/build-context.test.ts index f290f019d70..86d906f0984 100644 --- a/src/lib/build-context.test.ts +++ b/src/lib/build-context.test.ts @@ -296,4 +296,8 @@ describe("extractBuiltImageRef", () => { expect(extractBuiltImageRef("nothing relevant here")).toBeNull(); expect(extractBuiltImageRef("")).toBeNull(); }); + + it("does not treat an immutable Docker image ID as a registry tag", () => { + expect(extractBuiltImageRef(`Built image sha256:${"a".repeat(64)}`)).toBeNull(); + }); }); diff --git a/src/lib/build-context.ts b/src/lib/build-context.ts index f2cbc9d0230..2af7c78adcc 100644 --- a/src/lib/build-context.ts +++ b/src/lib/build-context.ts @@ -56,7 +56,10 @@ export function extractBuiltImageRef(output = ""): string | null { const patterns = [/^Successfully tagged\s+(\S+)/im, /^\s*Built image\s+(\S+)/im]; for (const pattern of patterns) { const match = text.match(pattern); - if (match?.[1]) return match[1]; + const candidate = match?.[1]; + // Docker image IDs are valid create inputs but are not repository tags. + // Persisting one as registry `imageTag` breaks tag-based destroy/GC logic. + if (candidate && !/^sha256:[0-9a-f]{64}$/i.test(candidate)) return candidate; } return null; } diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 0955f92d36e..18522211bfc 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -68,9 +68,10 @@ const { runSandboxConfigSync, writeSandboxConfigSyncFile, }: typeof import("./onboard/config-sync") = require("./onboard/config-sync"); -const dockerGpuPatch: typeof import("./onboard/docker-gpu-patch") = require("./onboard/docker-gpu-patch"); const dockerGpuLocalInference: typeof import("./onboard/docker-gpu-local-inference") = require("./onboard/docker-gpu-local-inference"); const dockerGpuSandboxCreate: typeof import("./onboard/docker-gpu-sandbox-create") = require("./onboard/docker-gpu-sandbox-create"); +const dockerGpuRoute: typeof import("./onboard/docker-gpu-route") = require("./onboard/docker-gpu-route"); +const sandboxGpuCreateFlow: typeof import("./onboard/sandbox-gpu-create-flow") = require("./onboard/sandbox-gpu-create-flow"); const dockerDriverGatewayLaunch: typeof import("./onboard/docker-driver-gateway-launch") = require("./onboard/docker-driver-gateway-launch"); const dockerDriverGatewayRuntime: typeof import("./onboard/docker-driver-gateway-runtime") = require("./onboard/docker-driver-gateway-runtime"); const dockerDriverGatewayCutover: typeof import("./onboard/docker-driver-gateway-cutover") = require("./onboard/docker-driver-gateway-cutover"); @@ -117,13 +118,6 @@ const { const { finalizeCreatedSandbox, }: typeof import("./onboard/created-sandbox-finalization") = require("./onboard/created-sandbox-finalization"); -const { - reportSandboxCreateFailure, - reportSandboxReadinessFailure, -}: typeof import("./onboard/created-sandbox-failure") = require("./onboard/created-sandbox-failure"); -const { - runSandboxCreateStep, -}: typeof import("./onboard/sandbox-create-step") = require("./onboard/sandbox-create-step"); const providerKeyBridge: typeof import("./onboard/provider-key-bridge") = require("./onboard/provider-key-bridge"); const { isLinuxDockerDriverGatewayEnabled, @@ -556,7 +550,6 @@ const buildContext = require("./build-context"); const httpProbe: typeof import("./adapters/http/probe") = require("./adapters/http/probe"); const modelPrompts: typeof import("./inference/model-prompts") = require("./inference/model-prompts"); const providerModels: typeof import("./inference/provider-models") = require("./inference/provider-models"); -const sandboxCreateStream: typeof import("./sandbox/create-stream") = require("./sandbox/create-stream"); const validationRecovery: typeof import("./validation-recovery") = require("./validation-recovery"); const webSearch: typeof import("./inference/web-search") = require("./inference/web-search"); const openshellInstallFlow: typeof import("./onboard/openshell-install") = @@ -788,8 +781,6 @@ const { getSandboxReuseState, repairRecordedSandbox } = sandboxReuse.createSandb note, }); -const { streamSandboxCreate } = sandboxCreateStream; - const { executeSandboxCommandForVerification, }: typeof import("./onboard/sandbox-verification-exec") = @@ -1777,11 +1768,21 @@ async function startGatewayWithOptions( step(2, 8, "Starting OpenShell gateway"); if (isLinuxDockerDriverGatewayEnabled()) { + const selectedGpuRoute = dockerGpuRoute.initialDockerGpuRoute( + dockerGpuRoute.resolveDockerGpuRoutePlan( + { sandboxGpuEnabled: gpuPassthrough, hostGpuPlatform: _gpu?.platform }, + { + dockerDriverGateway: true, + dockerDesktopWsl: dockerGpuSandboxCreate.isDockerDesktopWslRuntime(), + }, + ), + ); return startDockerDriverGateway({ exitOnFailure, skipSandboxBridgeReachability: dockerGpuLocalInference.shouldSkipGpuBridgeProbe( gpuPassthrough, _gpu?.platform, + selectedGpuRoute, ), }); } @@ -2715,16 +2716,22 @@ async function createSandboxWithBaseImageResolution( "policies", "openclaw-sandbox.yaml", ); + // TODO: Keep GPU route resolution, policy materialization, and create execution in their focused + // modules; extract this remaining handoff if it acquires another responsibility. This boundary + // deliberately coordinates those modules without re-implementing their trust-boundary logic. const basePolicyPath = (agent && agentOnboard.getAgentPolicyPath(agent)) || defaultPolicyPath; const dockerDriverGateway = isLinuxDockerDriverGatewayEnabled(); + const { gpuRoutePlan, logMessage: sandboxGpuLogMessage } = + dockerGpuSandboxCreate.resolveDockerGpuSandboxCreatePlan(effectiveSandboxGpuConfig, { + dockerDriverGateway, + }); const { activeMessagingChannels, initialSandboxPolicy, policyTier: resolvedCreatePolicyTier, createArgs, messagingProviders, - useDockerGpuPatch, - sandboxGpuLogMessage, + compatibilityPolicyPath, } = sandboxCreatePlan.prepareSandboxCreatePlan({ basePolicyPath, buildCtx, @@ -2739,7 +2746,8 @@ async function createSandboxWithBaseImageResolution( extraProviders: createIntent?.extraProviders ?? reconcileRegisteredExtraProviders(GATEWAY_NAME, { runOpenshell }), hermesToolGateways, sandboxGpuConfig: effectiveSandboxGpuConfig, - dockerDriverGateway, + gpuRoutePlan, + sandboxGpuLogMessage, appendResourceFlags: (args) => appendResourceFlagsForProfile(args, resourceProfile, getOpenshellBinary(), { isNonInteractive, @@ -2776,6 +2784,7 @@ async function createSandboxWithBaseImageResolution( const configuredMessagingChannels = getChannelsFromPlan(plannedMessagingState?.plan) ?? activeMessagingChannels; sandboxBuildPatchConfig.prepareSandboxBuildPatchConfig({ configuredMessagingChannels }); + const initialGpuRoute = dockerGpuRoute.initialDockerGpuRoute(gpuRoutePlan); const { buildId, dashboardRemoteBindPrepared } = await preparedDcodeRebuild.resolveSandboxBuildPatch({ preparedBuildContext, @@ -2792,43 +2801,65 @@ async function createSandboxWithBaseImageResolution( ...(isManagedDcodeAgent ? { dcodeAutoApprovalMode: dcodeAutoApprovalPlan.mode } : {}), hermesToolGateways, sandboxGpuConfig: effectiveSandboxGpuConfig, + selectedGpuRoute: initialGpuRoute, ...baseImageResolutionFlow.getBaseImageResolutionPatchOptions(baseImageResolutionContext), gatewayPort: GATEWAY_PORT, }); const sandboxReadyTimeoutSecs = getSandboxReadyTimeoutSecs(effectiveSandboxGpuConfig); - const { createResult, prebuild, effectiveDashboardPort, dockerGpuCreatePatch } = - await runSandboxCreateStep( - { - agent, - observabilityEnabled: createIntent?.observabilityEnabled === true, - chatUiUrl, - createArgs, - sandboxName, - env: process.env, - extraPlaceholderKeys, - getDashboardForwardPort, - hermesDashboardState, - manageDashboard, - openshellShellCommand, - openshellArgv, - prebuild: { buildCtx, buildId, dockerDriverGateway, origin }, - useDockerGpuPatch, - gpuDevice: effectiveSandboxGpuConfig.sandboxGpuDevice, - gpuBackend: effectiveSandboxGpuConfig.hostGpuPlatform === "jetson" ? "jetson" : "generic", - timeoutSecs: sandboxReadyTimeoutSecs, - }, - { - prepareCreateLaunch: sandboxCreateLaunch.prepareSandboxCreateLaunchWithPrebuild, - createDockerGpuPatch: dockerGpuSandboxCreate.createDockerGpuSandboxCreatePatch, - streamCreate: streamSandboxCreate, - isSandboxReady, - isTerminalAgent: agentDefs.isTerminalAgent, - addTraceEvent: onboardTracing.addTraceEvent, - runOpenshell, - runCaptureOpenshell, - sleepSeconds, - }, - ); + const { createArgv, effectiveDashboardPort, prebuild, sandboxEnv, sandboxStartupCommand } = + await sandboxCreateLaunch.prepareSandboxCreateLaunchWithPrebuild({ + agent, + observabilityEnabled: createIntent?.observabilityEnabled === true, + chatUiUrl, + createArgs: dockerGpuRoute.renderSandboxCreateArgsForGpuRoute(createArgs, initialGpuRoute, { + compatibilityPolicyPath, + }), + sandboxName, + env: process.env, + extraPlaceholderKeys, + getDashboardForwardPort, + hermesDashboardState, + manageDashboard, + openshellShellCommand, + openshellArgv, + prebuild: { buildCtx, buildId, dockerDriverGateway, origin }, + }); + const restoreBackupPath = + pendingStateRestore?.manifest?.backupPath ?? pendingStateRestoreBackupPath; + const { + createResult, + dockerGpuCreatePatch, + route: selectedGpuRoute, + firstCreateOutput, + registryImageRef, + } = await sandboxGpuCreateFlow.runSandboxGpuCreateFlow( + { + sandboxName, + provider, + sandboxGpuConfig: effectiveSandboxGpuConfig, + gpuRoutePlan, + initialGpuRoute, + compatibilityPolicyPath, + dockerDriverGateway, + gatewayPort: GATEWAY_PORT, + sandboxReadyTimeoutSecs, + createArgv, + sandboxEnv, + sandboxStartupCommand, + prebuild, + restoreBackupPath, + terminalAgent: agentDefs.isTerminalAgent(agent), + persistStartupCommand: dockerDriverGateway === true && agent?.name === "hermes", + }, + { + runOpenshell, + runCaptureOpenshell, + sleep: sleepSeconds, + openshellArgv, + verifyDirectSandboxGpu, + }, + ); + if (initialSandboxPolicy.cleanup && initialSandboxPolicy.cleanup()) { process.removeListener("exit", initialSandboxPolicy.cleanup); } @@ -2842,73 +2873,6 @@ async function createSandboxWithBaseImageResolution( process.removeListener("exit", cleanupBuildCtx); } - dockerGpuCreatePatch.exitOnPatchError(); - - const restoreBackupPath = - pendingStateRestore?.manifest?.backupPath ?? pendingStateRestoreBackupPath; - - if (createResult.status !== 0) { - reportSandboxCreateFailure( - { - sandboxName, - createStatus: createResult.status, - createOutput: createResult.output, - restoreBackupPath, - createArgs: prebuild.createArgs, - }, - { - classifyCreateFailure: classifySandboxCreateFailure, - printCreateFailureDiagnostics: (name, options) => - sandboxCreateFailureDiagnostics.printSandboxCreateFailureDiagnostics(name, options), - printRecoveryHints: printSandboxCreateRecoveryHints, - warn: (message) => console.warn(message), - error: (message) => console.error(message), - exitProcess: (code) => process.exit(code), - }, - ); - } - - dockerGpuCreatePatch.ensureApplied(); - dockerGpuCreatePatch.waitForSupervisorReconnectIfNeeded(); - - // Wait for OpenShell to report the sandbox Ready before registering. - // On first run the sandbox can take longer to initialize; - // without this gate, NemoClaw registers a phantom sandbox that - // causes "sandbox not found" on every subsequent connect/status call. - console.log(" Waiting for sandbox to become ready..."); - const readiness = sandboxReadinessTracing.waitForCreatedSandboxReadyWithTrace({ - sandboxName, - timeoutSecs: sandboxReadyTimeoutSecs, - runCaptureOpenshell, - isSandboxReady, - getSandboxFailurePhase: gatewayState.getSandboxFailurePhase, - sleep: sleepSeconds, - }); - - if (!readiness.ready) { - reportSandboxReadinessFailure( - { - sandboxName, - readiness, - createStatus: createResult.status, - timeoutSecs: sandboxReadyTimeoutSecs, - restoreBackupPath, - useDockerGpuPatch, - }, - { - printReadinessFailure: (result, name, timeoutSecs) => - sandboxReadinessTracing.printReadinessFailure(result, name, timeoutSecs), - printCreateFailureDiagnostics: (name, options) => - sandboxCreateFailureDiagnostics.printSandboxCreateFailureDiagnostics(name, options), - printDockerGpuReadinessFailure: () => dockerGpuCreatePatch.printReadinessFailureIfEnabled(), - deleteSandbox: (name) => runOpenshell(["sandbox", "delete", name], { ignoreError: true }), - cliName, - error: (message) => console.error(message), - exitProcess: (code) => process.exit(code), - }, - ); - } - if (manageDashboard) { console.log(" Waiting for NemoClaw dashboard to become ready..."); sandboxReadinessTracing.waitForDashboardReadyWithTrace({ @@ -2920,18 +2884,20 @@ async function createSandboxWithBaseImageResolution( } if (effectiveSandboxGpuConfig.sandboxGpuEnabled) { - // Runs the GPU proof, preserving Docker-GPU patch Error-phase diagnostics - // when applicable, then gates host-network local inference reachability (#4509). - dockerGpuLocalInference.verifyGpuSandboxAfterReady(effectiveSandboxGpuConfig, provider, { - sandboxName, - dockerDriverGateway, - useDockerGpuPatch, - verifyDirectSandboxGpu, - verifyGpuOrExit: dockerGpuCreatePatch.verifyGpuOrExit, - selectedMode: dockerGpuCreatePatch.selectedMode, - runCaptureOpenshell, - log: console.log, - }); + dockerGpuLocalInference.verifyGpuSandboxLocalInferenceAfterReady( + effectiveSandboxGpuConfig, + provider, + { + sandboxName, + dockerDriverGateway, + selectedRoute: selectedGpuRoute, + verifyDirectSandboxGpu, + verifyGpuOrExit: dockerGpuCreatePatch.verifyGpuOrExit, + selectedMode: dockerGpuCreatePatch.selectedMode, + runCaptureOpenshell, + log: console.log, + }, + ); } let actualDashboardPort = 0; @@ -2950,7 +2916,10 @@ async function createSandboxWithBaseImageResolution( // openshell tags images with seconds; buildId is ms. Parse actual tag from output. Fixes #2672. const resolvedImageTag = - prebuild.imageRef ?? resolveSandboxImageTagFromCreateOutput(createResult.output, buildId); + registryImageRef ?? + prebuild.imageRef ?? + buildContext.extractBuiltImageRef(`${firstCreateOutput}\n${createResult.output}`) ?? + resolveSandboxImageTagFromCreateOutput(`${firstCreateOutput}\n${createResult.output}`, buildId); const sandboxRuntimeFields = getSandboxRuntimeRegistryFields(effectiveSandboxGpuConfig); finalizeCreatedSandbox( { diff --git a/src/lib/onboard/__test-helpers__/docker-gpu-patch-fixtures.ts b/src/lib/onboard/__test-helpers__/docker-gpu-patch-fixtures.ts new file mode 100644 index 00000000000..8c6566a5600 --- /dev/null +++ b/src/lib/onboard/__test-helpers__/docker-gpu-patch-fixtures.ts @@ -0,0 +1,117 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { DockerContainerInspect } from "../docker-gpu-patch-types"; + +function openshellNetworkSettings(): NonNullable { + return { + Networks: { + "openshell-docker": { + IPAddress: "172.18.0.2", + Gateway: "172.18.0.1", + Aliases: ["openshell-alpha"], + }, + }, + }; +} + +export function createDockerGpuInspectFixture(): DockerContainerInspect { + return { + Id: "old-container-id", + Image: `sha256:${"c".repeat(64)}`, + Name: "/openshell-alpha", + Config: { + Image: "openshell/sandbox:abc", + Env: [ + "A=1", + "OPENSHELL_ENDPOINT=http://host.openshell.internal:8080/", + "OPENSHELL_TEST=1", + "OPENSHELL_SANDBOX_COMMAND=sleep infinity", + "NVIDIA_VISIBLE_DEVICES=void", + ], + Labels: { + "openshell.ai/managed-by": "openshell", + "openshell.ai/sandbox-name": "alpha", + "openshell.ai/sandbox-id": "sandbox-id", + }, + Entrypoint: ["/opt/openshell/bin/openshell-sandbox"], + Cmd: [], + User: "0", + WorkingDir: "/workspace", + Hostname: "alpha-host", + Tty: true, + }, + HostConfig: { + Binds: ["/host:/container:rw"], + NetworkMode: "openshell-docker", + RestartPolicy: { Name: "unless-stopped" }, + CapAdd: ["SYS_ADMIN", "NET_ADMIN"], + SecurityOpt: ["apparmor=unconfined"], + ExtraHosts: ["host.openshell.internal:172.17.0.1"], + Memory: 8 * 1024 * 1024 * 1024, + NanoCpus: 2_500_000_000, + }, + NetworkSettings: openshellNetworkSettings(), + }; +} + +export function createDockerGpuDnsInspectFixture(): DockerContainerInspect { + return { + Id: "old-container-id", + Name: "/openshell-alpha", + Config: { + Image: "openshell/sandbox:abc", + Env: ["OPENSHELL_SANDBOX_COMMAND=sleep infinity"], + Labels: { + "openshell.ai/managed-by": "openshell", + "openshell.ai/sandbox-name": "alpha", + }, + Entrypoint: ["/opt/openshell/bin/openshell-sandbox"], + Cmd: [], + }, + HostConfig: { + NetworkMode: "openshell-docker", + RestartPolicy: { Name: "unless-stopped" }, + ExtraHosts: ["host.openshell.internal:172.17.0.1"], + }, + NetworkSettings: openshellNetworkSettings(), + }; +} + +export function createDockerGpuJetsonInspectFixture(): DockerContainerInspect { + return { + Id: "old-container-id", + Name: "/openshell-alpha", + Config: { + Image: "openshell/sandbox:abc", + Env: ["OPENSHELL_ENDPOINT=http://host.openshell.internal:8080/"], + Labels: { + "openshell.ai/managed-by": "openshell", + "openshell.ai/sandbox-name": "alpha", + }, + Entrypoint: ["/opt/openshell/bin/openshell-sandbox"], + Cmd: [], + }, + HostConfig: { NetworkMode: "openshell-docker" }, + }; +} + +export function createDockerGpuDiagnosticsInspectFixture(): DockerContainerInspect { + return { + Id: "old-container-id", + Name: "/openshell-alpha", + Config: { + Image: "openshell/sandbox:abc", + Env: ["A=1", "OPENSHELL_ENDPOINT=http://host.openshell.internal:8080/", "OPENSHELL_TEST=1"], + Labels: { + "openshell.ai/managed-by": "openshell", + "openshell.ai/sandbox-name": "alpha", + }, + }, + HostConfig: { + NetworkMode: "openshell-docker", + ExtraHosts: ["host.openshell.internal:172.17.0.1"], + }, + NetworkSettings: openshellNetworkSettings(), + }; +} diff --git a/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts b/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts new file mode 100644 index 00000000000..f0224759b04 --- /dev/null +++ b/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts @@ -0,0 +1,111 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { vi } from "vitest"; + +import type { SandboxGpuProofResult } from "../../state/registry"; +import type { + SandboxGpuCreateFlowDeps, + SandboxGpuCreateFlowInput, +} from "../sandbox-gpu-create-flow"; + +export const VERIFIED_GPU_PROOF: SandboxGpuProofResult = { + status: "verified", + cudaVerified: true, + label: "CUDA initialization", + detail: null, + at: "2026-07-06T00:00:00.000Z", +}; +export const GPU_IMAGE_ID = `sha256:${"a".repeat(64)}`; + +export function createGpuFlowInput(): SandboxGpuCreateFlowInput { + return { + sandboxName: "alpha", + provider: "nim", + sandboxGpuConfig: { + mode: "1", + hostGpuDetected: true, + hostGpuPlatform: null, + sandboxGpuEnabled: true, + sandboxGpuDevice: null, + errors: [], + }, + gpuRoutePlan: "native-with-fallback", + initialGpuRoute: "native", + compatibilityPolicyPath: "/tmp/compatibility-policy.yaml", + dockerDriverGateway: true, + gatewayPort: 8080, + sandboxReadyTimeoutSecs: 60, + createArgv: ["openshell", "sandbox", "create", "--gpu"], + sandboxEnv: {}, + sandboxStartupCommand: ["nemoclaw-start"], + prebuild: { + createArgs: ["--from", "openshell/sandbox-from:test", "--name", "alpha", "--gpu"], + imageRef: "openshell/sandbox-from:test", + imageId: GPU_IMAGE_ID, + }, + restoreBackupPath: null, + terminalAgent: false, + }; +} + +export function createGpuFlowDeps(): SandboxGpuCreateFlowDeps { + return { + runOpenshell: vi.fn(() => ({ status: 0 })), + runCaptureOpenshell: vi.fn(() => "alpha Ready"), + sleep: vi.fn(), + openshellArgv: vi.fn((args: string[]) => ["openshell", ...args]), + verifyDirectSandboxGpu: vi.fn(() => VERIFIED_GPU_PROOF), + }; +} + +export function createGpuPatchFixture() { + return { + maybeApplyDuringCreate: vi.fn(), + createFailureMessage: vi.fn(() => null), + exitOnPatchError: vi.fn(), + ensureApplied: vi.fn(), + waitForSupervisorReconnectIfNeeded: vi.fn(), + selectedMode: vi.fn(() => null), + printReadinessFailureIfEnabled: vi.fn(), + verifyGpuOrExit: vi.fn(() => VERIFIED_GPU_PROOF), + }; +} + +export function setupGpuFlowMocks(mocks: Record>): void { + mocks.streamSandboxCreate.mockResolvedValue({ + status: 0, + output: "Created sandbox: alpha", + sawProgress: true, + }); + mocks.createDockerGpuSandboxCreatePatch.mockImplementation(createGpuPatchFixture); + mocks.waitForCreatedSandboxReadyWithTrace.mockReturnValue({ + ready: true, + reason: "ready", + failurePhase: null, + }); + mocks.verifyGpuSandboxAccessAfterReady.mockImplementation((_config, options) => + options.verifyGpuOrExit + ? options.verifyGpuOrExit(options.verifyDirectSandboxGpu) + : options.verifyDirectSandboxGpu(options.sandboxName), + ); + mocks.enforceDockerGpuPatchPreserveNetwork.mockResolvedValue(false); + mocks.collectDockerGpuPatchDiagnostics.mockReturnValue(null); + mocks.queryOpenShellDockerSandboxContainers.mockReturnValue({ ok: true, ids: [] }); + mocks.queryOpenShellDockerSandboxRuntimeSnapshot.mockReturnValue({ + ok: true, + imageId: GPU_IMAGE_ID, + bookkeepingImageRef: "openshell/sandbox-from:test", + stateError: "", + nativeGpuAttachmentState: "present", + containerId: "container-a", + }); + for (const method of ["log", "warn", "error"] as const) { + vi.spyOn(console, method).mockImplementation(() => {}); + } +} + +export function resetGpuFlowMocks(): void { + vi.restoreAllMocks(); + vi.clearAllMocks(); +} diff --git a/src/lib/onboard/docker-command-result.ts b/src/lib/onboard/docker-command-result.ts new file mode 100644 index 00000000000..7f888720b92 --- /dev/null +++ b/src/lib/onboard/docker-command-result.ts @@ -0,0 +1,12 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Docker adapters use `status: null` when a process cannot start or times out. + * Mutation and cleanup gates therefore accept only an explicit zero status. + */ +export function hasZeroDockerExitStatus( + result: { status?: number | null } | null | undefined, +): boolean { + return result?.status === 0; +} diff --git a/src/lib/onboard/docker-gpu-diagnostic-redaction.ts b/src/lib/onboard/docker-gpu-diagnostic-redaction.ts index b14232a63f6..cf3a1af5ae0 100644 --- a/src/lib/onboard/docker-gpu-diagnostic-redaction.ts +++ b/src/lib/onboard/docker-gpu-diagnostic-redaction.ts @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { redact, redactFull } from "../security/redact"; -import type { DockerContainerInspect } from "./docker-gpu-patch"; +import type { DockerContainerInspect } from "./docker-gpu-patch-types"; const SENSITIVE_ENV_KEY = /(?:api_?key|private_?key|(?:^|_)key$|token|secret|password|credential|authorization|cookie|proxy)/i; @@ -69,9 +69,12 @@ export function discoverDockerGpuDiagnosticSensitiveValues( } /** - * Owns the redaction state for one diagnostic bundle. Full Docker inspect - * records are observed before any artifact is written so conventionally - * sensitive and custom-placeholder values are removed from every sink. + * SOURCE_OF_TRUTH_REVIEW (shared Docker GPU diagnostic redaction; #6110): + * invalidState: inspect, network, log, or startup-command credentials reach an artifact sink. + * sourceBoundary: this shared collector redacts Docker/OpenShell output before every write. + * whyNotSourceFix: supported Docker and OpenShell versions expose unredacted runtime metadata. + * regressionTest: docker-gpu-diagnostic-redaction.test.ts proves sink-wide canary removal. + * removalCondition: supported upstream inspect and log APIs provide equivalent secret redaction. */ export function createDockerGpuDiagnosticRedactor( initialSensitiveValues: Iterable = [], diff --git a/src/lib/onboard/docker-gpu-dns-fallback.test.ts b/src/lib/onboard/docker-gpu-dns-fallback.test.ts new file mode 100644 index 00000000000..c05e542e847 --- /dev/null +++ b/src/lib/onboard/docker-gpu-dns-fallback.test.ts @@ -0,0 +1,114 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { detectSandboxFallbackDns, parseResolvConfNameservers } from "./docker-gpu-dns-fallback"; + +const RESOLV_CONF = "/etc/resolv.conf"; +const SYSTEMD_RESOLV_CONF = "/run/systemd/resolve/resolv.conf"; +const LOOPBACK_STUB = "nameserver 127.0.0.53\n"; + +function readSequence(...contents: Array) { + const readFile = vi.fn<(path: string) => string | null>(); + for (const content of contents) readFile.mockReturnValueOnce(content); + return readFile; +} + +describe("parseResolvConfNameservers", () => { + it.each([ + [ + "parses loopback and direct nameservers across resolv.conf whitespace", + "# generated\n nameserver 127.0.0.53\n\tnameserver\t192.168.1.1\nsearch lan\n", + ["127.0.0.53", "192.168.1.1"], + ], + [ + "ignores empty, malformed, and non-IP resolver lines", + "\nsearch lan\nnameserver\nnameserver not-an-ip\nnameserver-alias 8.8.8.8\nnot-a-resolver 1.1.1.1\n", + [], + ], + ])("%s (resolver parsing)", (_title, source, expected) => { + expect(parseResolvConfNameservers(source)).toEqual(expected); + }); +}); + +describe("detectSandboxFallbackDns", () => { + it("returns the systemd-resolved upstream when /etc/resolv.conf is loopback-only", () => { + const files: Record = { + [RESOLV_CONF]: `${LOOPBACK_STUB}search lan\n`, + [SYSTEMD_RESOLV_CONF]: + "# Generated by systemd-resolved\nnameserver 8.8.8.8\nnameserver 1.1.1.1\n", + }; + + const readFile = vi.fn((path: string) => files[path] ?? null); + expect(detectSandboxFallbackDns({ readFile })).toBe("8.8.8.8"); + expect(readFile.mock.calls.map(([path]) => path)).toEqual([RESOLV_CONF, SYSTEMD_RESOLV_CONF]); + }); + + it.each([ + ["direct resolver", ["nameserver 192.168.1.1\n"]], + ["missing resolver", [null]], + ["missing systemd upstream", [LOOPBACK_STUB, null]], + ])("returns null without a loopback-only resolver and usable systemd upstream: %s", (_case, responses) => { + expect(detectSandboxFallbackDns({ readFile: readSequence(...responses) })).toBeNull(); + }); + + it.each([ + ["direct", "nameserver 192.168.1.1\n"], + ["mixed", `${LOOPBACK_STUB}nameserver 10.0.0.2\n`], + ])("does not read the upstream file for a %s resolver configuration", (_case, root) => { + const readFile = readSequence(root); + expect(detectSandboxFallbackDns({ readFile })).toBeNull(); + expect(readFile).toHaveBeenCalledTimes(1); + }); + + it.each([ + ["empty", [""]], + ["missing", [null]], + ["malformed", ["search lan\nnameserver\n"]], + ["unreadable upstream", [LOOPBACK_STUB, null]], + ])("returns null for %s resolver files", (_case, responses) => { + const readFile = readSequence(...responses); + expect(detectSandboxFallbackDns({ readFile })).toBeNull(); + expect(readFile).toHaveBeenCalledTimes(responses.length); + }); + + it.each([ + ["loopback-only", "nameserver 127.0.0.54\n"], + ["malformed", "search lan\nnameserver\n"], + ])("ignores %s upstream entries", (_case, upstream) => { + expect( + detectSandboxFallbackDns({ readFile: readSequence(LOOPBACK_STUB, upstream) }), + ).toBeNull(); + }); + + it.each([ + [ + "unspecified and multicast", + "nameserver 0.0.0.0\nnameserver 224.0.0.1\nnameserver ::\nnameserver ff02::1\n", + null, + ], + ["valid link-local", "nameserver 169.254.169.253\n", "169.254.169.253"], + ])("rejects or preserves %s upstreams", (_case, upstream, expected) => { + expect(detectSandboxFallbackDns({ readFile: readSequence(LOOPBACK_STUB, upstream) })).toBe( + expected, + ); + }); + + it.each([ + [ + "accepts a private upstream because host resolver integrity is the trust boundary", + LOOPBACK_STUB, + "nameserver 10.20.30.53\n", + "10.20.30.53", + ], + [ + "supports an IPv6 loopback stub with a unicast IPv6 upstream", + "nameserver ::1\n", + "nameserver 2001:4860:4860::8888\n", + "2001:4860:4860::8888", + ], + ])("%s (resolver selection)", (_title, root, upstream, expected) => { + expect(detectSandboxFallbackDns({ readFile: readSequence(root, upstream) })).toBe(expected); + }); +}); diff --git a/src/lib/onboard/docker-gpu-dns-fallback.ts b/src/lib/onboard/docker-gpu-dns-fallback.ts new file mode 100644 index 00000000000..5f196de2334 --- /dev/null +++ b/src/lib/onboard/docker-gpu-dns-fallback.ts @@ -0,0 +1,62 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import { isIP } from "node:net"; + +export function parseResolvConfNameservers(content: string): string[] { + return content + .split("\n") + .map((line) => line.trim()) + .filter((line) => /^nameserver(?:\s|$)/.test(line)) + .map((line) => line.split(/\s+/)[1]) + .filter((ip): ip is string => Boolean(ip) && isIP(ip) !== 0); +} + +function isLoopbackResolver(ip: string): boolean { + return /^127\./.test(ip) || ip === "::1"; +} + +function isUsableUpstreamResolver(ip: string): boolean { + if (isLoopbackResolver(ip) || ip === "0.0.0.0" || ip === "::") return false; + if (isIP(ip) === 4) { + const firstOctet = Number(ip.split(".")[0]); + // Keep unicast link-local resolvers: cloud hosts legitimately publish + // addresses such as the Route 53 Resolver at 169.254.169.253. + return firstOctet > 0 && firstOctet < 224; + } + return !/^ff/i.test(ip); +} + +/** + * SOURCE_OF_TRUTH_REVIEW (compatibility DNS fallback) + * invalidState: a recreated container inherits a host-only loopback resolver. + * sourceBoundary: host resolver files supply data only to Docker's `--dns`; the sandbox gets no + * file access, and syntactically valid unicast includes legitimate private/link-local resolvers. + * whyNotSourceFix: atomic recreation cannot reconfigure the host or upgrade OpenShell/Docker. + * regressionTest: DNS parser/host-file tests and the recreate-command envelope test. + * removalCondition: compatibility is retired or supported stacks always supply non-loopback DNS. + */ +export function detectSandboxFallbackDns( + deps: { readFile?: (path: string) => string | null } = {}, +): string | null { + // The test seam is invoked only with these hardcoded host-trusted paths; its result is data, + // never a command. + const readFile = + deps.readFile ?? + ((path: string): string | null => { + try { + return fs.readFileSync(path, "utf-8"); + } catch { + return null; + } + }); + const resolvConf = readFile("/etc/resolv.conf"); + if (!resolvConf) return null; + const nameservers = parseResolvConfNameservers(resolvConf); + if (nameservers.length === 0 || !nameservers.every(isLoopbackResolver)) return null; + const upstreamFile = readFile("/run/systemd/resolve/resolv.conf"); + return upstreamFile + ? (parseResolvConfNameservers(upstreamFile).find(isUsableUpstreamResolver) ?? null) + : null; +} diff --git a/src/lib/onboard/docker-gpu-jetson-groups.test.ts b/src/lib/onboard/docker-gpu-jetson-groups.test.ts new file mode 100644 index 00000000000..2caf66eb0a1 --- /dev/null +++ b/src/lib/onboard/docker-gpu-jetson-groups.test.ts @@ -0,0 +1,50 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { detectTegraDeviceGroupGids } from "./docker-gpu-jetson-groups"; + +describe("detectTegraDeviceGroupGids", () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("returns unique host-owned GIDs while skipping missing, root-owned, and oversized nodes", () => { + const deviceGids: Record = { + "/dev/nvmap": 44, + "/dev/nvhost-ctrl": 44, + "/dev/nvhost-gpu": 0, + "/dev/nvgpu/igpu0/ctrl": 110, + "/dev/nvgpu/igpu0/as": 2_147_483_648, + }; + + expect( + detectTegraDeviceGroupGids({ + statDeviceGid: (path: string) => (path in deviceGids ? deviceGids[path] : null), + }), + ).toEqual(["44", "110"]); + }); + + it("rejects non-integer, non-numeric, negative, zero, and oversized supplementary GIDs", () => { + const gids = [Number.NaN, 1.5, -1, 0, 2_147_483_648]; + let index = 0; + + expect( + detectTegraDeviceGroupGids({ + statDeviceGid: () => gids[index++] ?? null, + }), + ).toEqual([]); + }); + + it("returns no GIDs when Tegra nodes are missing or unreadable", () => { + expect(detectTegraDeviceGroupGids({ statDeviceGid: () => null })).toEqual([]); + + vi.spyOn(fs, "statSync").mockImplementation(() => { + throw new Error("EACCES"); + }); + expect(detectTegraDeviceGroupGids()).toEqual([]); + }); +}); diff --git a/src/lib/onboard/docker-gpu-jetson-groups.ts b/src/lib/onboard/docker-gpu-jetson-groups.ts new file mode 100644 index 00000000000..0754fbb1889 --- /dev/null +++ b/src/lib/onboard/docker-gpu-jetson-groups.ts @@ -0,0 +1,61 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; + +const TEGRA_GPU_DEVICE_NODES = [ + "/dev/nvmap", + "/dev/nvhost-ctrl", + "/dev/nvhost-ctrl-gpu", + "/dev/nvhost-gpu", + "/dev/nvhost-as-gpu", + "/dev/nvhost-prof-gpu", + "/dev/nvhost-dbg-gpu", + "/dev/nvhost-tsg-gpu", + "/dev/nvgpu/igpu0/ctrl", + "/dev/nvgpu/igpu0/as", + "/dev/nvgpu/igpu0/prof", +] as const; +const MAX_DOCKER_SUPPLEMENTARY_GID = 2_147_483_647; + +/** + * Source-of-truth boundary for Jetson/Tegra supplementary device groups: + * + * - Invalid state: the non-root sandbox user can see `/dev/nvmap` and `/dev/nvhost-*` but cannot + * open them because Docker did not copy their host-owned supplementary GIDs into the container. + * - Source boundary: host device-node ownership is authoritative; NemoClaw only carries each + * bounded, non-root numeric GID into the Jetson compatibility recreation via `--group-add`. + * - Source-fix constraint: changing host udev ownership or image-local groups cannot reliably fix + * device nodes whose ownership is assigned by the Jetson host at runtime. + * - Regression coverage: docker-gpu-jetson-groups.test.ts covers discovery and hostile numeric + * values; docker-gpu-patch-jetson.test.ts covers clone-envelope propagation and generic-host + * exclusion. + * - Removal condition: remove this probe when the minimum supported native OpenShell Jetson path + * propagates the host device groups without compatibility container recreation. + */ +export function detectTegraDeviceGroupGids( + deps: { statDeviceGid?: (path: string) => number | null } = {}, +): string[] { + const statGid = + deps.statDeviceGid ?? + ((path: string): number | null => { + try { + return fs.statSync(path).gid; + } catch { + return null; + } + }); + const gids = new Set(); + for (const node of TEGRA_GPU_DEVICE_NODES) { + const gid = statGid(node); + if ( + gid !== null && + Number.isSafeInteger(gid) && + gid > 0 && + gid <= MAX_DOCKER_SUPPLEMENTARY_GID + ) { + gids.add(String(gid)); + } + } + return [...gids].sort((left, right) => Number(left) - Number(right)); +} diff --git a/src/lib/onboard/docker-gpu-local-inference.test.ts b/src/lib/onboard/docker-gpu-local-inference.test.ts index f3959d5bf5e..60dfe28c639 100644 --- a/src/lib/onboard/docker-gpu-local-inference.test.ts +++ b/src/lib/onboard/docker-gpu-local-inference.test.ts @@ -24,6 +24,7 @@ function gpuPatchOptions(extra: Record = {}) { return { sandboxName: "alpha", dockerDriverGateway: true, + selectedRoute: "compatibility" as const, platform: "linux" as NodeJS.Platform, env: { ...LEGACY_PATCH_ENV }, ...extra, @@ -42,6 +43,7 @@ describe("shouldUseDockerGpuPatchHostNetwork", () => { expect( shouldUseDockerGpuPatchHostNetwork(GPU_CONFIG, { dockerDriverGateway: true, + selectedRoute: "compatibility", platform: "linux", env: HOST_NETWORK_ENV, }), @@ -50,6 +52,7 @@ describe("shouldUseDockerGpuPatchHostNetwork", () => { expect( shouldUseDockerGpuPatchHostNetwork(GPU_CONFIG, { dockerDriverGateway: true, + selectedRoute: "compatibility", platform: "linux", env: {}, }), @@ -58,6 +61,7 @@ describe("shouldUseDockerGpuPatchHostNetwork", () => { expect( shouldUseDockerGpuPatchHostNetwork(GPU_CONFIG, { dockerDriverGateway: true, + selectedRoute: "compatibility", platform: "darwin", env: HOST_NETWORK_ENV, }), @@ -68,23 +72,14 @@ describe("shouldUseDockerGpuPatchHostNetwork", () => { describe("shouldSkipGpuBridgeProbe", () => { it("forces the gateway context and skips only for an active legacy host-network patch", () => { expect( - shouldSkipGpuBridgeProbe(true, "linux", { + shouldSkipGpuBridgeProbe(true, "linux", "compatibility", { dockerDriverGateway: false, - dockerDesktopWsl: false, env: HOST_NETWORK_ENV, platform: "linux", }), ).toBe(true); expect( - shouldSkipGpuBridgeProbe(true, "linux", { - dockerDesktopWsl: false, - env: { NEMOCLAW_DOCKER_GPU_PATCH_NETWORK: "host" }, - platform: "linux", - }), - ).toBe(false); - expect( - shouldSkipGpuBridgeProbe(false, "linux", { - dockerDesktopWsl: false, + shouldSkipGpuBridgeProbe(false, "linux", "compatibility", { env: HOST_NETWORK_ENV, platform: "linux", }), @@ -99,6 +94,7 @@ describe("enforceDockerGpuPatchPreserveNetwork", () => { const reverifyBridgeReachability = vi.fn(); const downgraded = await enforceDockerGpuPatchPreserveNetwork("ollama-local", GPU_CONFIG, { dockerDriverGateway: true, + selectedRoute: "compatibility", platform: "linux", env, log, @@ -117,6 +113,7 @@ describe("enforceDockerGpuPatchPreserveNetwork", () => { expect( await enforceDockerGpuPatchPreserveNetwork("nvidia", GPU_CONFIG, { dockerDriverGateway: true, + selectedRoute: "compatibility", platform: "linux", env, reverifyBridgeReachability, @@ -131,6 +128,7 @@ describe("enforceDockerGpuPatchPreserveNetwork", () => { expect( await enforceDockerGpuPatchPreserveNetwork("ollama-local", GPU_CONFIG, { dockerDriverGateway: true, + selectedRoute: "compatibility", platform: "linux", env, reverifyBridgeReachability: vi.fn(), @@ -147,6 +145,7 @@ describe("enforceDockerGpuPatchPreserveNetwork", () => { { sandboxGpuEnabled: false }, { dockerDriverGateway: true, + selectedRoute: "compatibility", platform: "linux", env, reverifyBridgeReachability: vi.fn(), @@ -170,18 +169,21 @@ describe("getSandboxRuntimeInferenceEndpoint", () => { }); describe("verifyDockerGpuSandboxLocalInference", () => { - it("skips when the Docker GPU patch is not active", () => { + it("skips for non-local providers", () => { + const result = verifyDockerGpuSandboxLocalInference(GPU_CONFIG, "build", gpuPatchOptions()); + expect(result).toEqual({ status: "skipped", reason: "not-local-provider" }); + }); + + it("skips the compatibility-only inference gate on the native route", () => { + const execInSandbox = vi.fn(); const result = verifyDockerGpuSandboxLocalInference( GPU_CONFIG, "ollama-local", - gpuPatchOptions({ env: { NEMOCLAW_DOCKER_GPU_PATCH: "0" } }), + gpuPatchOptions({ selectedRoute: "native", deps: { execInSandbox } }), ); - expect(result).toEqual({ status: "skipped", reason: "not-docker-gpu-patch" }); - }); - it("skips for non-local providers", () => { - const result = verifyDockerGpuSandboxLocalInference(GPU_CONFIG, "build", gpuPatchOptions()); - expect(result).toEqual({ status: "skipped", reason: "not-local-provider" }); + expect(result).toEqual({ status: "skipped", reason: "not-docker-gpu-patch" }); + expect(execInSandbox).not.toHaveBeenCalled(); }); it("probes inference.local from the runtime context, never a loopback or docker exec", () => { @@ -298,9 +300,9 @@ describe("verifyGpuSandboxAfterReady", () => { return { sandboxName: "alpha", dockerDriverGateway: true, + selectedRoute: "compatibility" as const, platform: "linux" as NodeJS.Platform, env: { ...LEGACY_PATCH_ENV }, - useDockerGpuPatch: true, verifyDirectSandboxGpu: vi.fn(), selectedMode: () => null, runCaptureOpenshell: vi.fn(() => ""), @@ -381,22 +383,6 @@ describe("verifyGpuSandboxAfterReady", () => { exitSpy.mockRestore(); } }); - - it("skips the inference gate when the Docker GPU patch did not run", () => { - const execInSandbox = vi.fn(); - const verifyDirectSandboxGpu = vi.fn(); - verifyGpuSandboxAfterReady( - GPU_CONFIG, - "ollama-local", - baseOptions({ - useDockerGpuPatch: false, - verifyDirectSandboxGpu, - deps: { execInSandbox, sleep: vi.fn() }, - }), - ); - expect(verifyDirectSandboxGpu).toHaveBeenCalledWith("alpha"); - expect(execInSandbox).not.toHaveBeenCalled(); - }); }); describe("printDockerGpuSandboxInferenceVerificationFailure", () => { diff --git a/src/lib/onboard/docker-gpu-local-inference.ts b/src/lib/onboard/docker-gpu-local-inference.ts index 5f4600a856d..da58f8c44d8 100644 --- a/src/lib/onboard/docker-gpu-local-inference.ts +++ b/src/lib/onboard/docker-gpu-local-inference.ts @@ -5,12 +5,12 @@ import { getLocalProviderLabel } from "../inference/local"; import type { SandboxGpuProofResult } from "../state/registry"; import { DOCKER_GPU_PATCH_NETWORK_ENV, - type DockerGpuPatchMode, getDockerGpuPatchNetworkMode, printDockerGpuProofFailure, - shouldApplyDockerGpuPatch, } from "./docker-gpu-patch"; -import { isDockerDesktopWslRuntime } from "./docker-gpu-sandbox-create"; +import type { DockerGpuPatchMode } from "./docker-gpu-patch-types"; +import type { SelectedDockerGpuRoute } from "./docker-gpu-route"; +import { adaptDockerGpuRouteForPatch } from "./docker-gpu-route-patch-adapter"; import { executeSandboxCommandForVerification } from "./sandbox-verification-exec"; const { @@ -41,17 +41,13 @@ type DockerGpuLocalInferenceConfig = { type DockerGpuLocalInferenceOptions = { dockerDriverGateway: boolean; + selectedRoute: SelectedDockerGpuRoute; gatewayPort?: number; - dockerDesktopWsl?: boolean; env?: NodeJS.ProcessEnv; platform?: NodeJS.Platform; log?: (message: string) => void; }; -function resolveDockerDesktopWsl(options: DockerGpuLocalInferenceOptions): boolean { - return options.dockerDesktopWsl ?? isDockerDesktopWslRuntime(); -} - function isLocalInferenceProvider(provider: string | null | undefined): provider is string { return Boolean(provider && LOCAL_INFERENCE_PROVIDERS.includes(provider)); } @@ -59,13 +55,14 @@ function isLocalInferenceProvider(provider: string | null | undefined): provider export function shouldSkipGpuBridgeProbe( gpuPassthrough: boolean, hostGpuPlatform?: string | null, - options: Partial = {}, + selectedRoute: SelectedDockerGpuRoute = "none", + options: Omit, "selectedRoute"> = {}, ): boolean { return ( gpuPassthrough && shouldUseDockerGpuPatchHostNetwork( { sandboxGpuEnabled: true, hostGpuPlatform }, - { ...options, dockerDriverGateway: true }, + { ...options, dockerDriverGateway: true, selectedRoute }, ) ); } @@ -80,12 +77,11 @@ export function shouldUseDockerGpuPatchHostNetwork( options: DockerGpuLocalInferenceOptions, ): boolean { return ( - shouldApplyDockerGpuPatch(config, { - dockerDriverGateway: options.dockerDriverGateway, - dockerDesktopWsl: resolveDockerDesktopWsl(options), - env: options.env, - platform: options.platform, - }) && getDockerGpuPatchNetworkMode(options.env ?? process.env) === "host" + config.sandboxGpuEnabled && + options.selectedRoute === "compatibility" && + options.dockerDriverGateway && + (options.platform ?? process.platform) === "linux" && + getDockerGpuPatchNetworkMode(options.env ?? process.env) === "host" ); } @@ -111,12 +107,12 @@ export function shouldUseDockerGpuPatchHostNetwork( * Scoped to LOCAL inference providers — that is the only case the host-network * opt-in was meant to serve, and the only one this breaks. Non-local (cloud / * routed / custom) GPU sandboxes keep their requested network mode untouched. - * Runs at sandbox build time, after the provider is resolved and before the GPU + * Runs after the provider and actual route are resolved, before the GPU * container recreate reads the network mode. * * When a downgrade is applied, also re-runs the sandbox bridge reachability - * probe (with UFW auto-fix): gateway startup skipped it on the assumption that - * the sandbox would be on host networking, but the sandbox is now committed to + * probe (with UFW auto-fix): gateway startup may have skipped it on the assumption + * that the sandbox would be on host networking, but the sandbox is now committed to * the OpenShell bridge, so a default-deny firewall must fail fast / self-heal * before the build rather than surface as a late, opaque failure. * @@ -155,7 +151,11 @@ function defaultReverifyBridgeReachability(gatewayPort?: number): Promise }); } -export type SandboxExecResult = { status: number; stdout: string; stderr: string } | null; +export type SandboxExecResult = { + status: number; + stdout: string; + stderr: string; +} | null; export type DockerGpuSandboxInferenceVerifyDeps = { execInSandbox?: (sandboxName: string, script: string) => SandboxExecResult; @@ -257,7 +257,10 @@ function probeSandboxRuntimeInference( // an exec failure, NOT a missing-curl soft-skip, so we never declare // success without actually exercising the runtime (#4509 review). const noise = (out || result.stderr || "").slice(0, 160); - last = { kind: "exec-failed", detail: `unexpected sandbox exec output: ${noise}` }; + last = { + kind: "exec-failed", + detail: `unexpected sandbox exec output: ${noise}`, + }; } } if (attempt < DOCKER_GPU_INFERENCE_PROBE_MAX_ATTEMPTS) { @@ -289,14 +292,7 @@ export function verifyDockerGpuSandboxLocalInference( deps?: DockerGpuSandboxInferenceVerifyDeps; }, ): DockerGpuSandboxInferenceVerification { - if ( - !shouldApplyDockerGpuPatch(config, { - dockerDriverGateway: options.dockerDriverGateway, - dockerDesktopWsl: resolveDockerDesktopWsl(options), - env: options.env, - platform: options.platform, - }) - ) { + if (options.selectedRoute !== "compatibility") { return { status: "skipped", reason: "not-docker-gpu-patch" }; } if (!isLocalInferenceProvider(provider)) { @@ -385,11 +381,12 @@ export function printDockerGpuSandboxInferenceVerificationFailure( export type GpuSandboxAfterReadyOptions = { sandboxName: string; dockerDriverGateway: boolean; - useDockerGpuPatch: boolean; + selectedRoute: SelectedDockerGpuRoute; verifyDirectSandboxGpu: (sandboxName: string) => SandboxGpuProofResult; verifyGpuOrExit?: ( verifyDirectSandboxGpu: (sandboxName: string) => SandboxGpuProofResult, ) => SandboxGpuProofResult; + reportGpuProofFailure?: boolean; selectedMode: () => DockerGpuPatchMode | null; runCaptureOpenshell: (args: string[], opts?: Record) => string; env?: NodeJS.ProcessEnv; @@ -412,33 +409,48 @@ export function verifyGpuSandboxAfterReady( provider: string | null | undefined, options: GpuSandboxAfterReadyOptions, ): void { + verifyGpuSandboxAccessAfterReady(config, options); + verifyGpuSandboxLocalInferenceAfterReady(config, provider, options); +} + +export function verifyGpuSandboxAccessAfterReady( + config: DockerGpuLocalInferenceConfig, + options: GpuSandboxAfterReadyOptions, +): SandboxGpuProofResult { try { // Capture the CUDA-usability proof result and write it back onto the shared // config so onboarding can persist it to the registry and `status` can // report proven usability rather than mere configuration (#4231). - config.sandboxGpuProof = options.verifyGpuOrExit + const proof = options.verifyGpuOrExit ? options.verifyGpuOrExit(options.verifyDirectSandboxGpu) : options.verifyDirectSandboxGpu(options.sandboxName); + config.sandboxGpuProof = proof; + return proof; } catch (error) { // `verifyGpuOrExit` is supplied by the Docker GPU create patch and already // prints the richer Error-phase / patched-container diagnostics before // rethrowing. Avoid a second generic proof-failure block in that path. - if (!options.verifyGpuOrExit) { + if (!options.verifyGpuOrExit && options.reportGpuProofFailure !== false) { printDockerGpuProofFailure(options.sandboxName, error, options.selectedMode(), { runCaptureOpenshell: options.runCaptureOpenshell, + additionalSummaryLines: adaptDockerGpuRouteForPatch(options.selectedRoute) + .additionalSummaryLines, }); } throw error; } +} - // When the resolved create plan disabled the Docker GPU patch (e.g. - // NEMOCLAW_DOCKER_GPU_PATCH=0 honoured outside Docker Desktop WSL), there is - // no GPU-patched sandbox to gate, so skip the local inference reachability - // gate. - if (!options.useDockerGpuPatch) return; +export function verifyGpuSandboxLocalInferenceAfterReady( + config: DockerGpuLocalInferenceConfig, + provider: string | null | undefined, + options: GpuSandboxAfterReadyOptions, +): void { + if (options.selectedRoute !== "compatibility") return; const verification = verifyDockerGpuSandboxLocalInference(config, provider, { sandboxName: options.sandboxName, dockerDriverGateway: options.dockerDriverGateway, + selectedRoute: options.selectedRoute, env: options.env, platform: options.platform, log: options.log, diff --git a/src/lib/onboard/docker-gpu-patch-clone.test.ts b/src/lib/onboard/docker-gpu-patch-clone.test.ts new file mode 100644 index 00000000000..e21d31f7ba9 --- /dev/null +++ b/src/lib/onboard/docker-gpu-patch-clone.test.ts @@ -0,0 +1,172 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; +import { createDockerGpuInspectFixture as inspectFixture } from "./__test-helpers__/docker-gpu-patch-fixtures"; +import { + buildDockerGpuCloneRunArgs, + buildDockerGpuCloneRunOptions, + buildDockerGpuMode, + getDockerGpuPatchNetworkMode, +} from "./docker-gpu-patch"; + +describe("Docker GPU clone envelope", () => { + it("builds clone args that preserve OpenShell labels and runtime settings", () => { + const args = buildDockerGpuCloneRunArgs(inspectFixture(), buildDockerGpuMode("gpus")); + + expect(args).toEqual( + expect.arrayContaining([ + "--name", + "openshell-alpha", + "--gpus", + "all", + "--env", + "A=1", + "--env", + "OPENSHELL_ENDPOINT=http://host.openshell.internal:8080/", + "--env", + "OPENSHELL_TEST=1", + "--label", + "openshell.ai/managed-by=openshell", + "--label", + "openshell.ai/sandbox-name=alpha", + "--volume", + "/host:/container:rw", + "--network", + "openshell-docker", + "--network-alias", + "openshell-alpha", + "--restart", + "unless-stopped", + "--cap-add", + "SYS_ADMIN", + "--security-opt", + "apparmor=unconfined", + "--add-host", + "host.openshell.internal:172.17.0.1", + "--memory", + String(8 * 1024 * 1024 * 1024), + "--cpus", + "2.5", + "--entrypoint", + "/opt/openshell/bin/openshell-sandbox", + "openshell/sandbox:abc", + ]), + ); + expect(args).not.toEqual(expect.arrayContaining(["--env", "NVIDIA_VISIBLE_DEVICES=void"])); + }); + + it("adds OpenShell's sandbox command env when the inspected container lacks one", () => { + const inspect = inspectFixture(); + inspect.Config!.Env = inspect.Config!.Env!.filter( + (entry) => !entry.startsWith("OPENSHELL_SANDBOX_COMMAND="), + ); + const args = buildDockerGpuCloneRunArgs(inspect, buildDockerGpuMode("gpus"), { + openshellSandboxCommand: ["env", "CHAT_UI_URL=http://127.0.0.1:8642", "nemoclaw-start"], + }); + + expect(args).toEqual( + expect.arrayContaining([ + "--env", + "OPENSHELL_SANDBOX_COMMAND=env CHAT_UI_URL=http://127.0.0.1:8642 nemoclaw-start", + ]), + ); + }); + + it("adds SYS_PTRACE to the GPU clone when the baseline container lacks it", () => { + const inspect = inspectFixture(); + inspect.HostConfig!.CapAdd = ["SYS_ADMIN", "NET_ADMIN"]; + const args = buildDockerGpuCloneRunArgs(inspect, buildDockerGpuMode("gpus")); + + expect(args).toEqual(expect.arrayContaining(["--cap-add", "SYS_PTRACE"])); + expect(args).toEqual(expect.arrayContaining(["--cap-add", "SYS_ADMIN"])); + expect(args).toEqual(expect.arrayContaining(["--cap-add", "NET_ADMIN"])); + }); + + it("does not duplicate SYS_PTRACE when the baseline container already has it", () => { + const inspect = inspectFixture(); + inspect.HostConfig!.CapAdd = ["SYS_ADMIN", "SYS_PTRACE"]; + const args = buildDockerGpuCloneRunArgs(inspect, buildDockerGpuMode("gpus")); + + expect(args.filter((arg) => arg === "SYS_PTRACE").length).toBe(1); + }); + + it("injects apparmor=unconfined when the baseline container has no apparmor profile", () => { + const inspect = inspectFixture(); + inspect.HostConfig!.SecurityOpt = []; + const args = buildDockerGpuCloneRunArgs(inspect, buildDockerGpuMode("gpus")); + + expect(args).toEqual(expect.arrayContaining(["--security-opt", "apparmor=unconfined"])); + }); + + it("respects a baseline-pinned apparmor profile instead of overriding it", () => { + const inspect = inspectFixture(); + inspect.HostConfig!.SecurityOpt = ["apparmor=docker-default", "no-new-privileges"]; + const args = buildDockerGpuCloneRunArgs(inspect, buildDockerGpuMode("gpus")); + + expect(args).toEqual(expect.arrayContaining(["--security-opt", "apparmor=docker-default"])); + expect(args).toEqual(expect.arrayContaining(["--security-opt", "no-new-privileges"])); + expect(args).not.toEqual(expect.arrayContaining(["--security-opt", "apparmor=unconfined"])); + }); + + it("can switch the recreated sandbox to host networking for OpenShell callbacks", () => { + const inspect = inspectFixture(); + const options = buildDockerGpuCloneRunOptions(inspect, { + NEMOCLAW_DOCKER_GPU_PATCH_NETWORK: "host", + }); + const args = buildDockerGpuCloneRunArgs(inspect, buildDockerGpuMode("gpus"), options); + + expect(options).toEqual({ + networkMode: "host", + openshellEndpoint: "http://127.0.0.1:8080/", + }); + expect(args).toEqual(expect.arrayContaining(["--network", "host"])); + expect(args).toEqual( + expect.arrayContaining(["--env", "OPENSHELL_ENDPOINT=http://127.0.0.1:8080/"]), + ); + expect(args).toEqual( + expect.arrayContaining(["--add-host", "host.openshell.internal:172.17.0.1"]), + ); + expect(args).not.toEqual(expect.arrayContaining(["--network-alias", "openshell-alpha"])); + expect( + buildDockerGpuCloneRunOptions(inspect, { + NEMOCLAW_DOCKER_GPU_PATCH_NETWORK: "preserve", + }), + ).toEqual({}); + }); + + it.each([ + { name: "missing", endpoint: null }, + { name: "unrewritable", endpoint: "http://gateway.example.test:8080/" }, + ])("fails closed when host networking is requested with a $name OpenShell endpoint (#6110)", ({ + endpoint, + }) => { + const inspect = inspectFixture(); + inspect.Config!.Env = [ + ...inspect.Config!.Env!.filter((entry) => !entry.startsWith("OPENSHELL_ENDPOINT=")), + ...(endpoint === null ? [] : [`OPENSHELL_ENDPOINT=${endpoint}`]), + ]; + + expect(() => + buildDockerGpuCloneRunOptions(inspect, { + NEMOCLAW_DOCKER_GPU_PATCH_NETWORK: "host", + }), + ).toThrow(/NEMOCLAW_DOCKER_GPU_PATCH_NETWORK=host requires .*OPENSHELL_ENDPOINT/i); + }); + + it("reports the Docker GPU patch network mode", () => { + expect(getDockerGpuPatchNetworkMode({})).toBe("preserve"); + expect(getDockerGpuPatchNetworkMode({ NEMOCLAW_DOCKER_GPU_PATCH_NETWORK: "host" })).toBe( + "host", + ); + expect(getDockerGpuPatchNetworkMode({ NEMOCLAW_DOCKER_GPU_PATCH_NETWORK: "preserve" })).toBe( + "preserve", + ); + expect(getDockerGpuPatchNetworkMode({ NEMOCLAW_DOCKER_GPU_PATCH_NETWORK: "bridge" })).toBe( + "preserve", + ); + expect(getDockerGpuPatchNetworkMode({ NEMOCLAW_DOCKER_GPU_PATCH_NETWORK: "bogus" })).toBe( + "preserve", + ); + }); +}); diff --git a/src/lib/onboard/docker-gpu-patch-clone.ts b/src/lib/onboard/docker-gpu-patch-clone.ts new file mode 100644 index 00000000000..59087d10b4c --- /dev/null +++ b/src/lib/onboard/docker-gpu-patch-clone.ts @@ -0,0 +1,260 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { + DockerContainerInspect, + DockerGpuCloneRunOptions, + DockerGpuPatchMode, +} from "./docker-gpu-patch-types"; +import { openshellSandboxCommandEnvValue } from "./docker-startup-command-env"; + +const OPENSHELL_SANDBOX_COMMAND_ENV = "OPENSHELL_SANDBOX_COMMAND"; +const GPU_ENV_KEYS = new Set([ + "NVIDIA_VISIBLE_DEVICES", + "NVIDIA_DRIVER_CAPABILITIES", + "NVIDIA_REQUIRE_CUDA", + "NVIDIA_DISABLE_REQUIRE", +]); + +export const DOCKER_GPU_PATCH_NETWORK_ENV = "NEMOCLAW_DOCKER_GPU_PATCH_NETWORK"; + +export function dockerContainerName(inspect: DockerContainerInspect): string { + const raw = String(inspect.Name || "") + .replace(/^\/+/, "") + .trim(); + if (!raw) throw new Error("Docker inspect output did not include a container name."); + return raw; +} + +function stringArray(value: string[] | string | null | undefined): string[] { + if (Array.isArray(value)) return value.map((entry) => String(entry)); + if (typeof value === "string" && value.length > 0) return [value]; + return []; +} + +function envKey(env: string): string { + const index = env.indexOf("="); + return index === -1 ? env : env.slice(0, index); +} + +function envValue(env: string[] | null | undefined, key: string): string | null { + const prefix = `${key}=`; + const entry = stringArray(env).find((value) => value.startsWith(prefix)); + return entry ? entry.slice(prefix.length) : null; +} + +function replaceEnvValue(entry: string, key: string, value: string | null | undefined): string { + if (!value || envKey(entry) !== key) return entry; + return `${key}=${value}`; +} + +function dockerGpuHostEndpointFromOpenShellEndpoint(endpoint: string): string | null { + try { + const url = new URL(endpoint); + if (url.hostname !== "host.openshell.internal") return null; + url.hostname = "127.0.0.1"; + return url.toString(); + } catch { + return null; + } +} + +function pushStringFlag(args: string[], flag: string, value: unknown): void { + const normalized = String(value ?? "").trim(); + if (normalized) args.push(flag, normalized); +} + +function pushNumberFlag(args: string[], flag: string, value: unknown): void { + if (typeof value === "number" && Number.isFinite(value) && value > 0) { + args.push(flag, String(value)); + } +} + +function dockerCpusFromNanoCpus(nanoCpus: number): string { + return (nanoCpus / 1_000_000_000).toFixed(3).replace(/\.?0+$/, ""); +} + +export function buildDockerGpuCloneRunOptions( + inspect: DockerContainerInspect, + env: Record = process.env, +): DockerGpuCloneRunOptions { + if (getDockerGpuPatchNetworkMode(env) !== "host") return {}; + const endpoint = envValue(inspect.Config?.Env, "OPENSHELL_ENDPOINT"); + if (!endpoint) { + throw new Error( + `${DOCKER_GPU_PATCH_NETWORK_ENV}=host requires the inspected sandbox to include OPENSHELL_ENDPOINT.`, + ); + } + const hostEndpoint = dockerGpuHostEndpointFromOpenShellEndpoint(endpoint); + if (!hostEndpoint) { + throw new Error( + `${DOCKER_GPU_PATCH_NETWORK_ENV}=host requires OPENSHELL_ENDPOINT to use host.openshell.internal so NemoClaw can rewrite it to host loopback.`, + ); + } + return { networkMode: "host", openshellEndpoint: hostEndpoint }; +} + +export function getDockerGpuPatchNetworkMode( + env: Record = process.env, +): "host" | "preserve" { + const networkOverride = String(env[DOCKER_GPU_PATCH_NETWORK_ENV] || "") + .trim() + .toLowerCase(); + return networkOverride === "host" ? "host" : "preserve"; +} + +export function sameContainerId( + left: string | null | undefined, + right: string | null | undefined, +): boolean { + if (!left || !right) return false; + return left.startsWith(right) || right.startsWith(left); +} + +function dockerNetworkAliases( + inspect: DockerContainerInspect, + networkMode: string | null | undefined, +): string[] { + const network = String(networkMode || "").trim(); + if ( + !network || + ["bridge", "default", "host", "none"].includes(network) || + network.includes(":") + ) { + return []; + } + const networkInfo = inspect.NetworkSettings?.Networks?.[network]; + const containerId = String(inspect.Id || "").trim(); + return Array.from(new Set(stringArray(networkInfo?.Aliases))) + .map((alias) => alias.trim()) + .filter(Boolean) + .filter((alias) => !sameContainerId(alias, containerId)); +} + +export function buildDockerGpuCloneRunArgs( + inspect: DockerContainerInspect, + mode: DockerGpuPatchMode, + options: DockerGpuCloneRunOptions = {}, +): string[] { + const config = inspect.Config || {}; + const host = inspect.HostConfig || {}; + const image = String(options.image || config.Image || "").trim(); + if (!image) throw new Error("Docker inspect output did not include Config.Image."); + + const args: string[] = ["--name", dockerContainerName(inspect), ...mode.args]; + const gpuAugment = mode.kind !== "startup-command"; + + // Startup-command recreation must retain OpenShell's native CDI attachment. + if (!gpuAugment) { + const cdiDeviceIds = new Set( + (host.DeviceRequests ?? []) + .filter((request) => request.Driver === "cdi") + .flatMap((request) => stringArray(request.DeviceIDs)) + .map((deviceId) => deviceId.trim()) + .filter(Boolean), + ); + for (const deviceId of cdiDeviceIds) args.push("--device", deviceId); + } + pushStringFlag(args, "--hostname", config.Hostname); + pushStringFlag(args, "--user", config.User); + pushStringFlag(args, "--workdir", config.WorkingDir); + if (config.Tty) args.push("--tty"); + if (config.OpenStdin) args.push("--interactive"); + + const sandboxCommand = openshellSandboxCommandEnvValue(options.openshellSandboxCommand); + let sawSandboxCommand = false; + for (const env of stringArray(config.Env).filter( + (entry) => !gpuAugment || !GPU_ENV_KEYS.has(envKey(entry)), + )) { + const key = envKey(env); + if (key === OPENSHELL_SANDBOX_COMMAND_ENV && sandboxCommand) { + sawSandboxCommand = true; + args.push("--env", `${OPENSHELL_SANDBOX_COMMAND_ENV}=${sandboxCommand}`); + continue; + } + args.push("--env", replaceEnvValue(env, "OPENSHELL_ENDPOINT", options.openshellEndpoint)); + } + if (sandboxCommand && !sawSandboxCommand) { + args.push("--env", `${OPENSHELL_SANDBOX_COMMAND_ENV}=${sandboxCommand}`); + } + + const labels = config.Labels || {}; + for (const key of Object.keys(labels).sort()) { + const value = labels[key]; + if (value !== undefined && value !== null) args.push("--label", `${key}=${value}`); + } + for (const bind of stringArray(host.Binds)) args.push("--volume", bind); + const networkMode = options.networkMode ?? host.NetworkMode; + pushStringFlag(args, "--network", networkMode); + for (const alias of dockerNetworkAliases(inspect, networkMode)) + args.push("--network-alias", alias); + + const restart = host.RestartPolicy; + if (restart?.Name && restart.Name !== "no") { + const value = + restart.Name === "on-failure" && restart.MaximumRetryCount + ? `${restart.Name}:${restart.MaximumRetryCount}` + : restart.Name; + args.push("--restart", value); + } + + const capAdd = new Set(stringArray(host.CapAdd)); + if (gpuAugment) capAdd.add("SYS_PTRACE"); + for (const cap of capAdd) args.push("--cap-add", cap); + for (const cap of stringArray(host.CapDrop)) args.push("--cap-drop", cap); + const securityOpt = new Set(stringArray(host.SecurityOpt)); + if (gpuAugment && ![...securityOpt].some((entry) => entry.startsWith("apparmor"))) { + securityOpt.add("apparmor=unconfined"); + } + for (const option of securityOpt) args.push("--security-opt", option); + for (const hostEntry of stringArray(host.ExtraHosts)) args.push("--add-host", hostEntry); + const groupAdds = new Set(stringArray(host.GroupAdd)); + for (const group of groupAdds) args.push("--group-add", group); + for (const gid of options.extraGroupGids ?? []) { + const normalized = String(gid).trim(); + if (normalized && !groupAdds.has(normalized)) { + groupAdds.add(normalized); + args.push("--group-add", normalized); + } + } + if (networkMode !== "host") { + const dnsServers = stringArray(host.Dns); + for (const dns of dnsServers) args.push("--dns", dns); + for (const dnsSearch of stringArray(host.DnsSearch)) args.push("--dns-search", dnsSearch); + if (dnsServers.length === 0 && options.sandboxFallbackDns) { + args.push("--dns", options.sandboxFallbackDns); + } + } + + pushNumberFlag(args, "--memory", host.Memory); + pushNumberFlag(args, "--memory-reservation", host.MemoryReservation); + pushNumberFlag(args, "--memory-swap", host.MemorySwap); + pushNumberFlag(args, "--cpu-shares", host.CpuShares); + pushNumberFlag(args, "--cpu-quota", host.CpuQuota); + pushNumberFlag(args, "--cpu-period", host.CpuPeriod); + pushNumberFlag(args, "--shm-size", host.ShmSize); + if (typeof host.NanoCpus === "number" && host.NanoCpus > 0) { + args.push("--cpus", dockerCpusFromNanoCpus(host.NanoCpus)); + } + pushStringFlag(args, "--cpuset-cpus", host.CpusetCpus); + pushStringFlag(args, "--cpuset-mems", host.CpusetMems); + pushStringFlag(args, "--ipc", host.IpcMode); + pushStringFlag(args, "--pid", host.PidMode); + if (host.Privileged) args.push("--privileged"); + if (host.Init) args.push("--init"); + + const entrypoint = stringArray(config.Entrypoint); + if (entrypoint.length > 0) args.push("--entrypoint", entrypoint[0]); + const commandArgs = sandboxCommand ? [] : [...entrypoint.slice(1), ...stringArray(config.Cmd)]; + args.push(image, ...commandArgs); + return args; +} + +export function parseDockerInspectJson(output: string): DockerContainerInspect { + const parsed = JSON.parse(output); + const inspect = Array.isArray(parsed) ? parsed[0] : parsed; + if (!inspect || typeof inspect !== "object") { + throw new Error("Docker inspect did not return a container object."); + } + return inspect as DockerContainerInspect; +} diff --git a/src/lib/onboard/docker-gpu-patch-constants.ts b/src/lib/onboard/docker-gpu-patch-constants.ts new file mode 100644 index 00000000000..c01febdaee1 --- /dev/null +++ b/src/lib/onboard/docker-gpu-patch-constants.ts @@ -0,0 +1,5 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** Default timeout for one Docker CLI operation in the compatibility GPU patch path. */ +export const DOCKER_GPU_PATCH_TIMEOUT_MS = 30_000; diff --git a/src/lib/onboard/docker-gpu-patch-diagnostics-classification.test.ts b/src/lib/onboard/docker-gpu-patch-diagnostics-classification.test.ts new file mode 100644 index 00000000000..faba1386fdb --- /dev/null +++ b/src/lib/onboard/docker-gpu-patch-diagnostics-classification.test.ts @@ -0,0 +1,204 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { getSandboxFailurePhase } from "../state/gateway"; +import { + buildDockerGpuMode, + captureDockerGpuPatchSandboxSnapshot, + classifyDockerGpuPatchFailure, +} from "./docker-gpu-patch"; + +function sandboxCapture(getOutput: string, listOutput: string) { + const responses: Record = { + "sandbox:get": getOutput, + "sandbox:list": listOutput, + }; + return vi.fn((args: readonly string[]) => responses[args.slice(0, 2).join(":")] ?? ""); +} + +const GPU_MODE = buildDockerGpuMode("gpus"); +type PatchSnapshot = Parameters[0]; +const RUNNING = { Status: "running", Running: true, ExitCode: 0 }; + +function captureSnapshot( + getOutput: string, + listOutput: string, + patchedContainerState?: Record, +) { + return captureDockerGpuPatchSandboxSnapshot( + "alpha", + { patchedContainerId: patchedContainerState ? "new-container-id" : null }, + { + runCaptureOpenshell: sandboxCapture(getOutput, listOutput), + ...(patchedContainerState + ? { dockerCapture: vi.fn(() => JSON.stringify(patchedContainerState)) } + : {}), + }, + ); +} + +function classify( + snapshot: PatchSnapshot, + proofError?: Error, + mode: Parameters[1] = GPU_MODE, +) { + return classifyDockerGpuPatchFailure(snapshot, mode, proofError ? { proofError } : {}); +} + +function failureSnapshot( + sandboxPhase: string, + patchedContainerState: PatchSnapshot["patchedContainerState"] = null, + sandboxListLine: string | null = `alpha ${sandboxPhase} 30s ago`, +): PatchSnapshot { + return { sandboxPhase, sandboxListLine, patchedContainerState }; +} + +describe("Docker GPU patch diagnostics", () => { + it("detects terminal failure phases in `openshell sandbox list` output", () => { + const phase = (output: string) => getSandboxFailurePhase(output, "my-sandbox"); + expect(phase("my-sandbox Error 2s ago")).toBe("Error"); + expect(phase("my-sandbox CrashLoopBackOff 3s ago")).toBe("CrashLoopBackOff"); + expect(phase("my-sandbox Failed 3s ago")).toBe("Failed"); + expect(phase("my-sandbox Ready 3s ago")).toBeNull(); + expect(phase("other Error 3s ago")).toBeNull(); + expect(phase("")).toBeNull(); + }); + + it.each([ + [ + "prefers `sandbox list` phase over `sandbox get` when both are present (stale get)", + "Name: alpha\nPhase: Provisioning\n", + "alpha Error 2s ago\n", + "Error", + "alpha Error 2s ago", + ], + [ + "uses the list-derived phase whenever the sandbox row is present", + "Name: alpha\nPhase: Error\nReason: ContainerCannotRun\n", + "alpha Ready 1m ago\n", + "Ready", + "alpha Ready 1m ago", + ], + [ + "keeps the get-derived phase when the sandbox row is absent from list output", + "Name: alpha\nPhase: Terminated\n", + "other-box Ready 2s ago\n", + "Terminated", + null, + ], + ])("%s (phase precedence)", (_title, getOutput, listOutput, expectedPhase, expectedListLine) => { + const snapshot = captureSnapshot(getOutput, listOutput); + + expect(snapshot.sandboxPhase).toBe(expectedPhase); + expect(snapshot.sandboxListLine).toBe(expectedListLine); + }); + + it("captures sandbox phase and patched container State via the snapshot helper", () => { + const state = { + Status: "exited", + Running: false, + ExitCode: 125, + Error: 'could not select device driver "nvidia" with capabilities: [[gpu]]', + OOMKilled: false, + StartedAt: "2026-05-12T00:00:00Z", + FinishedAt: "2026-05-12T00:00:01Z", + }; + const snapshot = captureSnapshot( + "Name: alpha\nPhase: Error\nReason: ContainerExit\n", + "alpha Error 1m ago\n", + state, + ); + + expect(snapshot.sandboxPhase).toBe("Error"); + expect(snapshot.sandboxListLine).toBe("alpha Error 1m ago"); + expect(snapshot.patchedContainerState?.ExitCode).toBe(125); + expect(snapshot.patchedContainerState?.Error).toContain("could not select device driver"); + }); + + it("classifies a dead patched container as patched_container_failed with the failed mode", () => { + const result = classify( + failureSnapshot( + "Error", + { + Status: "exited", + ExitCode: 125, + Error: 'could not select device driver "nvidia" with capabilities: [[gpu]]', + }, + "alpha Error 1m ago", + ), + ); + + expect(result.kind).toBe("patched_container_failed"); + expect(result.headline).toContain("Patched GPU container exited with code 125"); + expect(result.headline).toContain("--gpus all"); + const flat = result.summaryLines.join("\n"); + expect(flat).toContain("sandbox_phase=Error"); + expect(flat).toContain("patched_container_exit_code=125"); + expect(flat).toContain("could not select device driver"); + expect(flat).toContain("patched_create_option=--gpus all"); + }); + + it("classifies an Error-phase sandbox with unknown container state as sandbox_error_phase", () => { + const result = classify(failureSnapshot("Error", null, null)); + + expect(result.kind).toBe("sandbox_error_phase"); + expect(result.headline).toContain("OpenShell sandbox entered Error phase"); + }); + + it("classifies a live container but timed-out supervisor as supervisor_unreachable", () => { + const result = classify(failureSnapshot("Provisioning", RUNNING)); + + expect(result.kind).toBe("supervisor_unreachable"); + expect(result.headline).toContain("Provisioning"); + }); + + it("prefers supervisor_unreachable over proof_failure when the sandbox is non-live but non-terminal", () => { + const result = classify( + failureSnapshot("Provisioning"), + new Error("openshell sandbox exec refused: sandbox not ready"), + ); + + expect(result.kind).toBe("supervisor_unreachable"); + expect(result.headline).toContain("Provisioning"); + expect(result.summaryLines.join("\n")).toContain("proof_error="); + }); + + it("does not blame the supervisor when the patch failed before a container existed", () => { + const result = classify( + failureSnapshot("Provisioning", null, "alpha Provisioning 3s ago"), + undefined, + null, + ); + + expect(result.kind).toBe("unknown"); + expect(result.headline).not.toMatch(/supervisor/i); + }); + + it("treats proof failures inside a Ready sandbox as proof_failure, not patched_container_failed", () => { + const result = classify( + failureSnapshot("Ready", RUNNING), + new Error("nvidia-smi exited with status 9"), + ); + + expect(result.kind).toBe("proof_failure"); + expect(result.summaryLines.join("\n")).toContain("proof_error=nvidia-smi exited with status 9"); + }); + + it("does not inspect the original/backup container when newContainerId is missing", () => { + const dockerCapture = vi.fn((_args: readonly string[]) => + JSON.stringify({ Status: "exited", ExitCode: 1 }), + ); + const snapshot = captureDockerGpuPatchSandboxSnapshot( + "alpha", + { patchedContainerId: null }, + { dockerCapture }, + ); + + expect(snapshot.patchedContainerState).toBeNull(); + expect( + dockerCapture.mock.calls.some(([args]) => args[0] === "inspect" && args[1] === "--format"), + ).toBe(false); + }); +}); diff --git a/src/lib/onboard/docker-gpu-patch-diagnostics-collection.test.ts b/src/lib/onboard/docker-gpu-patch-diagnostics-collection.test.ts new file mode 100644 index 00000000000..3ba96e6ede0 --- /dev/null +++ b/src/lib/onboard/docker-gpu-patch-diagnostics-collection.test.ts @@ -0,0 +1,116 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it, vi } from "vitest"; + +const dockerAdapterMocks = vi.hoisted(() => ({ + dockerCapture: vi.fn((args: readonly string[]) => + args[0] === "ps" ? "default-container-id\n" : "", + ), +})); + +vi.mock("../adapters/docker", async (importOriginal) => ({ + ...(await importOriginal()), + dockerCapture: dockerAdapterMocks.dockerCapture, +})); + +import { + buildDockerGpuMode, + classifyDockerGpuPatchFailure, + collectDockerGpuPatchDiagnostics, +} from "./docker-gpu-patch"; + +describe("Docker GPU patch diagnostics", () => { + it.each(["", "relative-home"])("rejects non-absolute diagnostic home %j", (home) => { + const dockerCapture = vi.fn(); + expect( + collectDockerGpuPatchDiagnostics("alpha", {}, { dockerCapture, homedir: () => home }), + ).toBeNull(); + expect(dockerCapture).not.toHaveBeenCalled(); + }); + + it("preserves the default Docker capture when callers omit dockerCapture from deps", () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-gpu-default-")); + try { + dockerAdapterMocks.dockerCapture.mockClear(); + const diagnostics = collectDockerGpuPatchDiagnostics( + "alpha", + { + context: { + sandboxName: "alpha", + newContainerId: "new-container-id", + selectedMode: buildDockerGpuMode("gpus"), + }, + }, + { + dockerLogs: vi.fn(() => ""), + homedir: () => tmpDir, + now: () => new Date("2026-05-12T00:00:00Z"), + }, + ); + + expect(diagnostics?.dir).toBeTruthy(); + expect( + fs.readFileSync(path.join(diagnostics?.dir || "", "docker-ps.txt"), "utf-8"), + ).toContain("default-container-id"); + expect(dockerAdapterMocks.dockerCapture).toHaveBeenCalledWith( + expect.arrayContaining(["ps"]), + expect.objectContaining({ ignoreError: true }), + ); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); + + it("writes patched-container-state.json and surfaces failure_kind/sandbox_phase in the summary", () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-gpu-4316-")); + try { + const snapshot = { + sandboxPhase: "Error", + sandboxListLine: "alpha Error 1m ago", + patchedContainerState: { + Status: "exited", + ExitCode: 125, + Error: 'could not select device driver "nvidia"', + }, + }; + const classification = classifyDockerGpuPatchFailure(snapshot, buildDockerGpuMode("gpus")); + const diagnostics = collectDockerGpuPatchDiagnostics( + "alpha", + { + context: { + sandboxName: "alpha", + newContainerId: "new-container-id", + selectedMode: buildDockerGpuMode("gpus"), + }, + selectedMode: buildDockerGpuMode("gpus"), + snapshot, + classification, + }, + { + dockerCapture: vi.fn(() => ""), + dockerLogs: vi.fn(() => ""), + homedir: () => tmpDir, + now: () => new Date("2026-05-12T00:00:00Z"), + }, + ); + + expect(diagnostics?.dir).toBeTruthy(); + const summary = fs.readFileSync(path.join(diagnostics?.dir || "", "summary.txt"), "utf-8"); + expect(summary).toContain("failure_kind=patched_container_failed"); + expect(summary).toContain("sandbox_phase=Error"); + expect(summary).toContain("patched_container_exit_code=125"); + const state = fs.readFileSync( + path.join(diagnostics?.dir || "", "patched-container-state.json"), + "utf-8", + ); + expect(state).toContain("could not select device driver"); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); +}); diff --git a/src/lib/onboard/docker-gpu-patch-diagnostics-network.test.ts b/src/lib/onboard/docker-gpu-patch-diagnostics-network.test.ts new file mode 100644 index 00000000000..1971eb0d57d --- /dev/null +++ b/src/lib/onboard/docker-gpu-patch-diagnostics-network.test.ts @@ -0,0 +1,78 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it, vi } from "vitest"; + +import { createDockerGpuDiagnosticsInspectFixture as inspectFixture } from "./__test-helpers__/docker-gpu-patch-fixtures"; +import { + collectDockerGpuPatchDiagnostics, + formatDockerInspectNetworkSummary, +} from "./docker-gpu-patch"; + +describe("Docker GPU patch diagnostics", () => { + it("formats sanitized network diagnostics without dumping provider secrets", () => { + const inspect = inspectFixture(); + inspect.Config?.Env?.push("NVIDIA_INFERENCE_API_KEY=secret"); + const summary = formatDockerInspectNetworkSummary("old-container-id", inspect); + + expect(summary).toContain("target=old-container-id"); + expect(summary).toContain("network_mode=openshell-docker"); + expect(summary).toContain("host.openshell.internal:172.17.0.1"); + expect(summary).toContain("env.OPENSHELL_ENDPOINT=http://host.openshell.internal:8080/"); + expect(summary).toContain("openshell-docker: ip=172.18.0.2 gateway=172.18.0.1"); + expect(summary).not.toContain("NVIDIA_INFERENCE_API_KEY"); + expect(summary).not.toContain("secret"); + }); + + it("keeps Docker network diagnostics when old patch containers are gone", () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-gpu-diag-")); + try { + const liveInspect = inspectFixture(); + liveInspect.Id = "new-container-id"; + const responses: Record = { + "ps:": "new-container-id\n", + "inspect:new-container-id": JSON.stringify([liveInspect]), + }; + const dockerCapture = vi.fn((args: readonly string[]) => { + const key = `${args[0]}:${String(args[1] ?? "")}`; + return ( + responses[key] ?? + (() => { + throw new Error(`missing target ${String(args[1])}`); + })() + ); + }); + const diagnostics = collectDockerGpuPatchDiagnostics( + "alpha", + { + context: { + sandboxName: "alpha", + oldContainerId: "old-container-id", + newContainerId: "new-container-id", + backupContainerName: "backup-container", + }, + }, + { + dockerCapture, + dockerLogs: vi.fn(() => ""), + homedir: () => tmpDir, + now: () => new Date("2026-05-12T00:00:00Z"), + }, + ); + + expect(diagnostics?.dir).toBeTruthy(); + const summary = fs.readFileSync( + path.join(diagnostics?.dir || "", "docker-network-summary.txt"), + "utf-8", + ); + expect(summary).toContain("target=new-container-id"); + expect(summary).toContain("network_mode=openshell-docker"); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); +}); diff --git a/src/lib/onboard/docker-gpu-patch-diagnostics.ts b/src/lib/onboard/docker-gpu-patch-diagnostics.ts new file mode 100644 index 00000000000..f92c6f62941 --- /dev/null +++ b/src/lib/onboard/docker-gpu-patch-diagnostics.ts @@ -0,0 +1,327 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { dockerCapture, dockerLogs } from "../adapters/docker"; +import { createDockerGpuDiagnosticRedactor } from "./docker-gpu-diagnostic-redaction"; +import { DOCKER_GPU_PATCH_TIMEOUT_MS } from "./docker-gpu-patch-constants"; +import { getDockerGpuPatchFailureContext } from "./docker-gpu-patch-recreate"; +import type { + DockerContainerInspect, + DockerContainerState, + DockerGpuPatchDeps, + DockerGpuPatchDiagnostics, + DockerGpuPatchFailureClassification, + DockerGpuPatchFailureContext, + DockerGpuPatchMode, + DockerGpuPatchSandboxSnapshot, +} from "./docker-gpu-patch-types"; +import { + findOpenShellDockerSandboxContainerIds, + OPENSHELL_MANAGED_BY_LABEL, + OPENSHELL_MANAGED_BY_VALUE, + OPENSHELL_SANDBOX_NAME_LABEL, +} from "./openshell-docker-sandbox-containers"; + +function stringArray(value: string[] | string | null | undefined): string[] { + if (Array.isArray(value)) return value.map((entry) => String(entry)); + if (typeof value === "string" && value.length > 0) return [value]; + return []; +} + +function envKey(env: string): string { + const index = env.indexOf("="); + return index === -1 ? env : env.slice(0, index); +} + +function writeTextFile(dir: string, name: string, content: string): void { + fs.writeFileSync(path.join(dir, name), content.endsWith("\n") ? content : `${content}\n`, { + mode: 0o600, + }); +} + +function uniqueStrings(values: Array): string[] { + return [...new Set(values.map((value) => String(value || "").trim()).filter(Boolean))]; +} + +function sanitizePathPart(value: string): string { + return value.replace(/[^A-Za-z0-9._-]/g, "_").slice(0, 80) || "sandbox"; +} + +function timestampForPath(now: Date): string { + return now.toISOString().replace(/[:.]/g, "-"); +} + +const DIAGNOSTIC_ENV_KEYS = new Set([ + "OPENSHELL_ENDPOINT", + "OPENSHELL_SANDBOX_ID", + "OPENSHELL_SANDBOX", + "OPENSHELL_LOG_LEVEL", + "OPENSHELL_TLS_CA", + "OPENSHELL_TLS_CERT", + "OPENSHELL_TLS_KEY", +]); + +function diagnosticEnvLines(env: string[] | null | undefined): string[] { + return stringArray(env) + .filter((entry) => DIAGNOSTIC_ENV_KEYS.has(envKey(entry))) + .sort() + .map((entry) => ` env.${envKey(entry)}=${entry.slice(envKey(entry).length + 1)}`); +} + +export function formatDockerInspectNetworkSummary( + target: string, + inspect: DockerContainerInspect, +): string { + const lines = [ + `target=${target}`, + `id=${inspect.Id ?? "unknown"}`, + `name=${String(inspect.Name || "").replace(/^\/+/, "") || "unknown"}`, + `image=${inspect.Config?.Image ?? "unknown"}`, + `network_mode=${inspect.HostConfig?.NetworkMode ?? "unknown"}`, + ]; + const extraHosts = stringArray(inspect.HostConfig?.ExtraHosts); + if (extraHosts.length > 0) { + lines.push("extra_hosts:"); + for (const entry of extraHosts) lines.push(` ${entry}`); + } + const envLines = diagnosticEnvLines(inspect.Config?.Env); + if (envLines.length > 0) lines.push("openshell_env:", ...envLines); + const networks = inspect.NetworkSettings?.Networks || {}; + const names = Object.keys(networks).sort(); + if (names.length > 0) { + lines.push("networks:"); + for (const name of names) { + const network = networks[name] || {}; + lines.push( + ` ${name}: ip=${network.IPAddress || "unknown"} gateway=${network.Gateway || "unknown"}`, + ); + const aliases = stringArray(network.Aliases); + if (aliases.length > 0) lines.push(` aliases=${aliases.join(",")}`); + } + } + return lines.join("\n"); +} + +function describePatchedContainerState(state: DockerContainerState | null): string[] { + if (!state) return []; + const lines: string[] = []; + if (state.Status) lines.push(`patched_container_status=${state.Status}`); + if (typeof state.ExitCode === "number") { + lines.push(`patched_container_exit_code=${state.ExitCode}`); + } + if (state.OOMKilled) lines.push("patched_container_oom_killed=true"); + if (state.Error) lines.push(`patched_container_error=${state.Error}`); + if (state.Health?.Status) lines.push(`patched_container_health=${state.Health.Status}`); + if (state.FinishedAt && state.FinishedAt !== "0001-01-01T00:00:00Z") { + lines.push(`patched_container_finished_at=${state.FinishedAt}`); + } + return lines; +} + +export function dockerGpuPatchCleanupCommands(sandboxName: string): string[] { + return [`openshell sandbox delete ${JSON.stringify(sandboxName)}`]; +} + +export function collectDockerGpuPatchDiagnostics( + sandboxName: string, + options: { + error?: unknown; + context?: DockerGpuPatchFailureContext | null; + selectedMode?: DockerGpuPatchMode | null; + snapshot?: DockerGpuPatchSandboxSnapshot | null; + classification?: DockerGpuPatchFailureClassification | null; + additionalSummaryLines?: readonly string[]; + additionalSensitiveValues?: readonly string[]; + dockerTopOutput?: string | null; + } = {}, + deps: DockerGpuPatchDeps = {}, +): DockerGpuPatchDiagnostics | null { + const home = (deps.homedir ?? os.homedir)(); + if (!path.isAbsolute(home)) return null; + const capture = deps.dockerCapture ?? dockerCapture; + const logs = deps.dockerLogs ?? dockerLogs; + const now = (deps.now ?? (() => new Date()))(); + const dir = path.join( + home, + ".nemoclaw", + "onboard-failures", + `${timestampForPath(now)}-${sanitizePathPart(sandboxName)}-docker-gpu-patch`, + ); + try { + fs.mkdirSync(dir, { recursive: true, mode: 0o700 }); + } catch { + return null; + } + + const context = options.context || getDockerGpuPatchFailureContext(options.error) || null; + const redactor = createDockerGpuDiagnosticRedactor(options.additionalSensitiveValues); + let discoveredContainerIds: string[] = []; + try { + discoveredContainerIds = findOpenShellDockerSandboxContainerIds(sandboxName, deps); + } catch { + discoveredContainerIds = []; + } + const containerTargets = uniqueStrings([ + ...(context + ? [context.oldContainerId, context.newContainerId, context.backupContainerName] + : []), + ...discoveredContainerIds, + ]); + const inspectedTargets: Array<{ target: string; entries: DockerContainerInspect[] }> = []; + for (const target of containerTargets) { + try { + const inspect = capture(["inspect", target], { + ignoreError: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }); + if (!inspect.trim()) continue; + const parsed = JSON.parse(inspect); + const entries = (Array.isArray(parsed) ? parsed : [parsed]) as DockerContainerInspect[]; + for (const entry of entries) redactor.rememberInspect(entry); + inspectedTargets.push({ target, entries }); + } catch { + // Best-effort diagnostics must not hide the original failure. + } + } + const writeDiagnosticText = (name: string, content: string): void => { + writeTextFile(dir, name, redactor.redactText(content)); + }; + const writeDiagnosticJson = (name: string, value: unknown): void => { + writeTextFile(dir, name, JSON.stringify(redactor.redactValue(value), null, 2)); + }; + + const cleanupCommands = dockerGpuPatchCleanupCommands(sandboxName).map(redactor.redactText); + const errorText = redactor.redactText( + options.error instanceof Error + ? options.error.message + : options.error + ? String(options.error) + : "none", + ); + const selectedMode = options.selectedMode || context?.selectedMode || null; + const snapshot = options.snapshot ?? null; + const classification = options.classification ?? null; + const summaryLines = [ + `created_at=${now.toISOString()}`, + `sandbox_name=${redactor.redactText(sandboxName)}`, + `error=${errorText}`, + ...(options.additionalSummaryLines ?? []).map(redactor.redactText), + `selected_gpu_mode=${redactor.redactText(selectedMode?.label ?? "none")}`, + `old_container_id=${redactor.redactText(context?.oldContainerId ?? "unknown")}`, + `new_container_id=${redactor.redactText(context?.newContainerId ?? "unknown")}`, + `backup_container_name=${redactor.redactText(context?.backupContainerName ?? "none")}`, + `rolled_back=${context?.rolledBack === true ? "yes" : context?.rolledBack === false ? "failed" : "no"}`, + "cleanup_commands:", + ...cleanupCommands.map((command) => ` ${command}`), + ]; + if (context?.modeAttempts?.length) { + summaryLines.push("gpu_mode_attempts:"); + for (const attempt of context.modeAttempts) { + summaryLines.push( + redactor.redactText( + ` ${attempt.mode.label}: ${attempt.ok ? "ok" : "failed"}${attempt.error ? `: ${attempt.error}` : ""}`, + ), + ); + } + } + if (classification) { + summaryLines.push(`failure_kind=${redactor.redactText(classification.kind)}`); + if (classification.headline) { + summaryLines.push(`failure_headline=${redactor.redactText(classification.headline)}`); + } + } + if (snapshot) { + if (snapshot.sandboxPhase) { + summaryLines.push(`sandbox_phase=${redactor.redactText(snapshot.sandboxPhase)}`); + } + if (snapshot.sandboxListLine) { + summaryLines.push(`sandbox_list_row=${redactor.redactText(snapshot.sandboxListLine)}`); + } + summaryLines.push( + ...describePatchedContainerState(snapshot.patchedContainerState).map(redactor.redactText), + ); + } + writeDiagnosticText("summary.txt", summaryLines.join("\n")); + if (snapshot?.patchedContainerState) { + writeDiagnosticJson("patched-container-state.json", snapshot.patchedContainerState); + } + if (options.dockerTopOutput?.trim()) + writeDiagnosticText("docker-top.txt", options.dockerTopOutput); + + try { + const ps = capture( + [ + "ps", + "-a", + "--filter", + `label=${OPENSHELL_MANAGED_BY_LABEL}=${OPENSHELL_MANAGED_BY_VALUE}`, + "--filter", + `label=${OPENSHELL_SANDBOX_NAME_LABEL}=${sandboxName}`, + ], + { ignoreError: true, timeout: DOCKER_GPU_PATCH_TIMEOUT_MS }, + ); + if (ps.trim()) writeDiagnosticText("docker-ps.txt", ps); + } catch { + // Best effort. + } + + if (containerTargets.length > 0) { + const inspectEntries: DockerContainerInspect[] = []; + const networkSummaries: string[] = []; + for (const { target, entries } of inspectedTargets) { + const sanitizedEntries = entries.map(redactor.sanitizeInspect); + inspectEntries.push(...sanitizedEntries); + for (const [index, entry] of sanitizedEntries.entries()) { + networkSummaries.push( + redactor.redactText( + formatDockerInspectNetworkSummary( + entries.length === 1 ? target : `${target}[${index}]`, + entry, + ), + ), + ); + } + } + if (inspectEntries.length > 0) writeDiagnosticJson("docker-inspect.json", inspectEntries); + if (networkSummaries.length > 0) { + writeDiagnosticText("docker-network-summary.txt", networkSummaries.join("\n\n")); + } + const containerLogs = containerTargets + .map((target) => { + try { + return redactor.redactText( + [`===== ${target} =====`, logs(target, { tail: 120 })].join("\n"), + ); + } catch { + return redactor.redactText(`===== ${target} =====\n(unavailable)`); + } + }) + .join("\n"); + if (containerLogs.trim()) writeDiagnosticText("docker-logs.txt", containerLogs); + } + + if (deps.runCaptureOpenshell) { + const captures: Array<[string, string[]]> = [ + ["openshell-sandbox-get.txt", ["sandbox", "get", sandboxName]], + ["openshell-sandbox-list.txt", ["sandbox", "list"]], + ["openshell-logs.txt", ["doctor", "logs", "--name", "nemoclaw"]], + ]; + for (const [fileName, args] of captures) { + try { + const output = deps.runCaptureOpenshell(args, { + ignoreError: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }); + if (output.trim()) writeDiagnosticText(fileName, output); + } catch { + // Best effort. + } + } + } + + return { dir, cleanupCommands, summaryLines }; +} diff --git a/src/lib/onboard/docker-gpu-patch-finalize.test.ts b/src/lib/onboard/docker-gpu-patch-finalize.test.ts index 9c87316f2ed..5a691435ce4 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.test.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.test.ts @@ -119,12 +119,40 @@ describe("finalizeDockerGpuPatchBackup", () => { ); }); - it("does not report backup removal when Docker returns no exit status", () => { + it("fails closed when backup removal has no exit status", () => { + const dockerRm = vi.fn((_name: string) => ({ status: null, stderr: "timed out" })); const outcome = finalizeDockerGpuPatchBackup( { result: deferredCreateResult(), supervisorReady: true }, - { dockerRm: vi.fn(() => ({ status: null, error: new Error("spawn timed out") })) }, + { dockerRm }, ); + expect(outcome).toEqual({ backupRemoved: false, rolledBack: false }); + }); + it("stops rollback before start when rename has no exit status", () => { + const dockerStart = vi.fn(() => ({ status: 0 })); + const outcome = finalizeDockerGpuPatchBackup( + { result: deferredCreateResult(), supervisorReady: false }, + { + dockerStop: vi.fn(() => ({ status: 0 })), + dockerRm: vi.fn(() => ({ status: 0 })), + dockerRename: vi.fn(() => ({ status: null })), + dockerStart, + }, + ); + expect(outcome).toEqual({ backupRemoved: false, rolledBack: false }); + expect(dockerStart).not.toHaveBeenCalled(); + }); + + it("fails closed when rollback start has no exit status", () => { + const outcome = finalizeDockerGpuPatchBackup( + { result: deferredCreateResult(), supervisorReady: false }, + { + dockerStop: vi.fn(() => ({ status: 0 })), + dockerRm: vi.fn(() => ({ status: 0 })), + dockerRename: vi.fn(() => ({ status: 0 })), + dockerStart: vi.fn(() => ({ status: null })), + }, + ); expect(outcome).toEqual({ backupRemoved: false, rolledBack: false }); }); }); diff --git a/src/lib/onboard/docker-gpu-patch-finalize.ts b/src/lib/onboard/docker-gpu-patch-finalize.ts index 542e27148cb..4eeeedf399b 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.ts @@ -23,83 +23,18 @@ // and delete this module along with its callers in docker-gpu-patch.ts and // docker-gpu-sandbox-create.ts. +import { hasZeroDockerExitStatus } from "./docker-command-result"; +import { DOCKER_GPU_PATCH_TIMEOUT_MS } from "./docker-gpu-patch-constants"; import { - dockerRename as defaultDockerRename, - dockerRm as defaultDockerRm, - dockerStart as defaultDockerStart, - dockerStop as defaultDockerStop, -} from "../adapters/docker"; -import type { DockerGpuPatchDeps, DockerGpuPatchResult } from "./docker-gpu-patch"; + resolveDockerGpuPatchRollbackDeps, + rollbackToBackupContainer, +} from "./docker-gpu-patch-rollback"; +import type { DockerGpuPatchDeps, DockerGpuPatchResult } from "./docker-gpu-patch-types"; -const DOCKER_GPU_PATCH_TIMEOUT_MS = 30_000; - -type DockerRunResult = { - status?: number | null; - stdout?: string | Buffer | null; - stderr?: string | Buffer | null; -}; - -type DockerRunOptions = Record; - -type DockerContainerFn = (containerName: string, opts?: DockerRunOptions) => DockerRunResult; -type DockerRenameFn = ( - oldContainerName: string, - newContainerName: string, - opts?: DockerRunOptions, -) => DockerRunResult; - -type ResolvedRollbackDeps = { - dockerStop: DockerContainerFn; - dockerRm: DockerContainerFn; - dockerRename: DockerRenameFn; - dockerStart: DockerContainerFn; -}; - -function isZeroStatus(result: DockerRunResult | null | undefined): boolean { - return result?.status === 0; -} - -function resolveRollbackDeps(deps: DockerGpuPatchDeps): ResolvedRollbackDeps { - return { - dockerStop: deps.dockerStop ?? defaultDockerStop, - dockerRm: deps.dockerRm ?? defaultDockerRm, - dockerRename: deps.dockerRename ?? defaultDockerRename, - dockerStart: deps.dockerStart ?? defaultDockerStart, - }; -} - -export function rollbackToBackupContainer( - refs: { newContainerId: string; backupContainerName: string; originalName: string }, - deps: ResolvedRollbackDeps, -): boolean { - const containerOpts = { - ignoreError: true, - suppressOutput: true, - timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, - }; - deps.dockerStop(refs.newContainerId, containerOpts); - deps.dockerRm(refs.newContainerId, containerOpts); - const restored = deps.dockerRename(refs.backupContainerName, refs.originalName, containerOpts); - if (!isZeroStatus(restored)) return false; - const started = deps.dockerStart(refs.originalName, containerOpts); - return isZeroStatus(started); -} - -/** - * Roll back a Docker GPU patch that failed *before* the supervisor wait — e.g. - * the recreate `docker run` itself failed after the original sandbox was already - * renamed to the backup. Restores the pre-patch sandbox so onboarding never - * leaves an orphaned `*-nemoclaw-gpu-backup-*` container (which otherwise - * collides on the next retry) (#5512). Accepts raw deps so the real - * `docker start`/`docker rename` defaults are resolved even when the caller's - * deps only carry the recreate subset. - */ -export function rollbackDockerGpuPatchOnRecreateFailure( - refs: { newContainerId: string; backupContainerName: string; originalName: string }, - deps: DockerGpuPatchDeps = {}, -): boolean { - return rollbackToBackupContainer(refs, resolveRollbackDeps(deps)); -} +export { + restoreDockerGpuPatchBackupAfterRecreateFailure as rollbackDockerGpuPatchOnRecreateFailure, + rollbackToBackupContainer, +} from "./docker-gpu-patch-rollback"; export type DockerGpuPatchFinalizeOptions = { result: DockerGpuPatchResult; @@ -115,7 +50,7 @@ export function finalizeDockerGpuPatchBackup( options: DockerGpuPatchFinalizeOptions, deps: DockerGpuPatchDeps = {}, ): DockerGpuPatchFinalizeOutcome { - const resolved = resolveRollbackDeps(deps); + const resolved = resolveDockerGpuPatchRollbackDeps(deps); const containerOpts = { ignoreError: true, suppressOutput: true, @@ -131,7 +66,7 @@ export function finalizeDockerGpuPatchBackup( // daemon timeout). Reflect the actual rm status in the outcome so // diagnostics can flag a leaked backup container. const rmResult = resolved.dockerRm(options.result.backupContainerName, containerOpts); - return { backupRemoved: isZeroStatus(rmResult), rolledBack: false }; + return { backupRemoved: hasZeroDockerExitStatus(rmResult), rolledBack: false }; } const rolledBack = rollbackToBackupContainer( { @@ -153,7 +88,7 @@ export function reconcileSupervisorReconnect( refs: { newContainerId: string; backupContainerName: string; originalName: string }, deps: DockerGpuPatchDeps, ): SupervisorReconnectOutcome { - const resolved = resolveRollbackDeps(deps); + const resolved = resolveDockerGpuPatchRollbackDeps(deps); const containerOpts = { ignoreError: true, suppressOutput: true, @@ -166,7 +101,7 @@ export function reconcileSupervisorReconnect( // Surface the actual rm status so callers can fold it into diagnostics // alongside the deferred-finalize path in `finalizeDockerGpuPatchBackup`. const rmResult = resolved.dockerRm(refs.backupContainerName, containerOpts); - return { execReady: true, backupRemoved: isZeroStatus(rmResult) }; + return { execReady: true, backupRemoved: hasZeroDockerExitStatus(rmResult) }; } const rolledBack = rollbackToBackupContainer(refs, resolved); return { diff --git a/src/lib/onboard/docker-gpu-patch-jetson.test.ts b/src/lib/onboard/docker-gpu-patch-jetson.test.ts new file mode 100644 index 00000000000..1a298fc7bd9 --- /dev/null +++ b/src/lib/onboard/docker-gpu-patch-jetson.test.ts @@ -0,0 +1,103 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; +import { createDockerGpuJetsonInspectFixture as inspectFixture } from "./__test-helpers__/docker-gpu-patch-fixtures"; +import { detectTegraDeviceGroupGids } from "./docker-gpu-jetson-groups"; +import { + buildDockerGpuCloneRunArgs, + buildDockerGpuMode, + recreateOpenShellDockerSandboxWithGpu, +} from "./docker-gpu-patch"; + +function dockerCaptureFixture() { + const responses: Record = { + ps: "old-container-id\n", + inspect: JSON.stringify([inspectFixture()]), + info: "", + }; + return vi.fn((args: readonly string[]) => responses[args[0]] ?? ""); +} + +describe("Jetson /dev/nvmap group propagation (#4231)", () => { + it("emits --group-add for extraGroupGids and dedupes against existing GroupAdd", () => { + const inspect = inspectFixture(); + inspect.HostConfig!.GroupAdd = ["44"]; + const args = buildDockerGpuCloneRunArgs( + inspect, + buildDockerGpuMode("nvidia-runtime", null, { backend: "jetson" }), + { extraGroupGids: ["44", "110"] }, + ); + expect( + args.filter((arg, index) => args[index - 1] === "--group-add" && arg === "44").length, + ).toBe(1); + expect(args).toEqual(expect.arrayContaining(["--group-add", "110"])); + }); + + it("does not add --group-add when extraGroupGids is absent", () => { + const inspect = inspectFixture(); + inspect.HostConfig!.GroupAdd = []; + const args = buildDockerGpuCloneRunArgs(inspect, buildDockerGpuMode("gpus")); + expect(args).not.toEqual(expect.arrayContaining(["--group-add"])); + }); + + it("plumbs detected Tegra device GIDs into the Jetson recreate as --group-add", () => { + const dockerRunDetached = vi.fn(() => ({ status: 0, stdout: "new-container-id\n" })); + const detectTegraDeviceGroupGidsStub = vi.fn(() => + detectTegraDeviceGroupGids({ + statDeviceGid: (path) => (path === "/dev/nvmap" ? 44 : null), + }), + ); + + recreateOpenShellDockerSandboxWithGpu( + { sandboxName: "alpha", timeoutSecs: 1, backend: "jetson" }, + { + dockerCapture: dockerCaptureFixture(), + dockerRun: vi.fn(() => ({ status: 0, stdout: "probe-id\n" })), + dockerRunDetached, + dockerRename: vi.fn(() => ({ status: 0 })), + dockerStop: vi.fn(() => ({ status: 0 })), + dockerRm: vi.fn(() => ({ status: 0 })), + runOpenshell: vi.fn(() => ({ status: 0 })), + sleep: vi.fn(), + now: () => new Date("2026-05-15T00:00:00Z"), + detectSandboxFallbackDns: () => null, + detectTegraDeviceGroupGids: detectTegraDeviceGroupGidsStub, + }, + ); + + expect(detectTegraDeviceGroupGidsStub).toHaveBeenCalled(); + expect(dockerRunDetached).toHaveBeenCalledWith( + expect.arrayContaining(["--group-add", "44"]), + expect.objectContaining({ ignoreError: true }), + ); + }); + + it("does not add Tegra device GIDs for the generic (non-Jetson) backend", () => { + const dockerRunDetached = vi.fn(() => ({ status: 0, stdout: "new-container-id\n" })); + const detectTegraDeviceGroupGidsStub = vi.fn(() => ["44"]); + + recreateOpenShellDockerSandboxWithGpu( + { sandboxName: "alpha", timeoutSecs: 1, backend: "generic" }, + { + dockerCapture: dockerCaptureFixture(), + dockerRun: vi.fn(() => ({ status: 0, stdout: "probe-id\n" })), + dockerRunDetached, + dockerRename: vi.fn(() => ({ status: 0 })), + dockerStop: vi.fn(() => ({ status: 0 })), + dockerRm: vi.fn(() => ({ status: 0 })), + runOpenshell: vi.fn(() => ({ status: 0 })), + sleep: vi.fn(), + now: () => new Date("2026-05-15T00:00:00Z"), + detectSandboxFallbackDns: () => null, + detectTegraDeviceGroupGids: detectTegraDeviceGroupGidsStub, + }, + ); + + expect(detectTegraDeviceGroupGidsStub).not.toHaveBeenCalled(); + expect(dockerRunDetached).not.toHaveBeenCalledWith( + expect.arrayContaining(["--group-add", "44"]), + expect.anything(), + ); + }); +}); diff --git a/src/lib/onboard/docker-gpu-patch-mode-selection.test.ts b/src/lib/onboard/docker-gpu-patch-mode-selection.test.ts index 07577cc4707..d75d25097b0 100644 --- a/src/lib/onboard/docker-gpu-patch-mode-selection.test.ts +++ b/src/lib/onboard/docker-gpu-patch-mode-selection.test.ts @@ -1,12 +1,18 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + import { describe, expect, it, vi } from "vitest"; import { + buildDockerGpuMode, buildDockerGpuModeCandidates, type DockerContainerInspect, type DockerGpuPatchDeps, + dockerReportsNvidiaCdiDevices, recreateOpenShellDockerSandboxWithGpu, selectDockerGpuPatchMode, } from "./docker-gpu-patch"; @@ -42,6 +48,28 @@ function inspectFixture(): DockerContainerInspect { } describe("docker-gpu-patch CDI-first mode selection (#4948)", () => { + it("maps default and explicit GPU devices to Docker --gpus values", () => { + expect(buildDockerGpuMode("gpus").args).toEqual(["--gpus", "all"]); + expect(buildDockerGpuMode("gpus", "nvidia.com/gpu=0").args).toEqual(["--gpus", "device=0"]); + expect(buildDockerGpuMode("gpus", "1,2").args).toEqual(["--gpus", "device=1,2"]); + }); + + it("uses Jetson NVIDIA runtime args without selecting generic --gpus or CDI candidates", () => { + expect(buildDockerGpuMode("nvidia-runtime", null, { backend: "jetson" }).args).toEqual([ + "--runtime", + "nvidia", + "--env", + "NVIDIA_VISIBLE_DEVICES=all", + "--env", + "NVIDIA_DRIVER_CAPABILITIES=compute,utility", + ]); + expect( + buildDockerGpuModeCandidates("all", { backend: "jetson", cdiAvailable: true }).map( + (mode) => mode.kind, + ), + ).toEqual(["nvidia-runtime"]); + }); + it("prefers CDI over --gpus when the host advertises an NVIDIA CDI spec", () => { // Repro for #4948: on a Docker-CDI GPU host (e.g. Ubuntu 24.04 with // /etc/cdi/nvidia.yaml), `docker create --gpus all` is *accepted* so the @@ -105,6 +133,151 @@ describe("docker-gpu-patch CDI-first mode selection (#4948)", () => { ]); }); + it("does not accept a GPU mode probe with no exit status", () => { + const selected = selectDockerGpuPatchMode( + { image: "openshell/sandbox:abc" }, + { + dockerCapture: vi.fn(() => ""), + dockerRun: vi.fn(() => ({ status: null, error: new Error("spawn timed out") })), + dockerRm: vi.fn(() => ({ status: 0 })), + readDir: vi.fn(() => null), + readFile: vi.fn(() => null), + }, + ); + + expect(selected.mode).toBeNull(); + expect(selected.attempts).toHaveLength(2); + expect(selected.attempts.map((attempt) => attempt.error)).toEqual([ + "spawn timed out", + "spawn timed out", + ]); + }); + + it("falls back to NVIDIA runtime when Docker rejects --gpus", () => { + const dockerRun = vi + .fn() + .mockReturnValueOnce({ status: 1, stderr: "could not select device driver" }) + .mockReturnValueOnce({ status: 0, stdout: "probe-id" }); + const selected = selectDockerGpuPatchMode( + { image: "openshell/sandbox:abc" }, + { + dockerCapture: vi.fn(() => ""), + dockerRun, + dockerRm: vi.fn(() => ({ status: 0 })), + readDir: vi.fn(() => null), + readFile: vi.fn(() => null), + }, + ); + + expect(selected.mode?.kind).toBe("nvidia-runtime"); + expect(selected.attempts.map((attempt) => attempt.mode.kind)).toEqual([ + "gpus", + "nvidia-runtime", + ]); + }); + + it("probes only NVIDIA runtime for Jetson Docker GPU mode", () => { + const dockerCapture = vi.fn(() => ""); + const dockerRun = vi.fn(() => ({ status: 0, stdout: "probe-id" })); + const selected = selectDockerGpuPatchMode( + { image: "openshell/sandbox:abc", backend: "jetson" }, + { dockerCapture, dockerRun, dockerRm: vi.fn(() => ({ status: 0 })) }, + ); + + expect(selected.mode?.kind).toBe("nvidia-runtime"); + expect(selected.attempts.map((attempt) => attempt.mode.kind)).toEqual(["nvidia-runtime"]); + expect(dockerRun).toHaveBeenCalledWith( + expect.arrayContaining([ + "create", + "--runtime", + "nvidia", + "--env", + "NVIDIA_DRIVER_CAPABILITIES=compute,utility", + ]), + expect.objectContaining({ ignoreError: true }), + ); + expect(dockerCapture).not.toHaveBeenCalled(); + }); + + it("prefers CDI only when Docker reports readable NVIDIA CDI specs", () => { + expect(buildDockerGpuModeCandidates("all", { cdiAvailable: false }).map((m) => m.kind)).toEqual( + ["gpus", "nvidia-runtime"], + ); + expect(buildDockerGpuModeCandidates("all", { cdiAvailable: true }).map((m) => m.kind)).toEqual([ + "cdi", + "gpus", + "nvidia-runtime", + ]); + + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-cdi-")); + try { + fs.writeFileSync( + path.join(tmpDir, "nvidia.yaml"), + "cdiVersion: 0.6.0\nkind: nvidia.com/gpu\ndevices:\n - name: all\n", + ); + expect( + dockerReportsNvidiaCdiDevices({ + dockerCapture: vi.fn(() => JSON.stringify([tmpDir])), + }), + ).toBe(true); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); + + it("detects NVIDIA CDI specs in /etc/cdi when docker info reports no dirs (#3575)", () => { + const readDir = vi.fn((dirPath: string) => (dirPath === "/etc/cdi" ? ["nvidia.yaml"] : null)); + const readFile = vi.fn((filePath: string) => + filePath === "/etc/cdi/nvidia.yaml" + ? "cdiVersion: 0.6.0\nkind: nvidia.com/gpu\ndevices:\n - name: all\n" + : null, + ); + expect( + dockerReportsNvidiaCdiDevices({ + dockerCapture: vi.fn(() => ""), + readDir, + readFile, + }), + ).toBe(true); + expect(readDir).toHaveBeenCalledWith("/etc/cdi"); + }); + + it("returns false when default CDI dirs hold no NVIDIA specs", () => { + expect( + dockerReportsNvidiaCdiDevices({ + dockerCapture: vi.fn(() => ""), + readDir: vi.fn(() => null), + readFile: vi.fn(() => null), + }), + ).toBe(false); + }); + + it("falls back to default CDI dirs even when docker info errors", () => { + const dockerCapture = vi.fn(() => { + throw new Error("docker daemon unreachable"); + }); + const readDir = vi.fn((dirPath: string) => + dirPath === "/var/run/cdi" ? ["nvidia.json"] : null, + ); + const readFile = vi.fn((filePath: string) => + filePath === "/var/run/cdi/nvidia.json" + ? JSON.stringify({ cdiVersion: "0.6.0", kind: "nvidia.com/gpu" }) + : null, + ); + expect(dockerReportsNvidiaCdiDevices({ dockerCapture, readDir, readFile })).toBe(true); + }); + + it("does not re-scan a directory that docker info already reported", () => { + const readDir = vi.fn((dirPath: string) => (dirPath === "/etc/cdi" ? ["nvidia.yaml"] : null)); + const readFile = vi.fn(() => "cdiVersion: 0.6.0\nkind: nvidia.com/gpu\n"); + dockerReportsNvidiaCdiDevices({ + dockerCapture: vi.fn(() => JSON.stringify(["/etc/cdi"])), + readDir, + readFile, + }); + expect(readDir.mock.calls.filter(([dir]) => dir === "/etc/cdi").length).toBe(1); + }); + it("passes the CDI --device flag to docker run when recreating on a CDI host", () => { // Proves the selected CDI mode propagates into the actual recreate command // (`dockerRunDetached`), not just the selection result. This is the create diff --git a/src/lib/onboard/docker-gpu-patch-mode.ts b/src/lib/onboard/docker-gpu-patch-mode.ts new file mode 100644 index 00000000000..03f746386d7 --- /dev/null +++ b/src/lib/onboard/docker-gpu-patch-mode.ts @@ -0,0 +1,249 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; + +import { dockerCapture, dockerRm, dockerRun } from "../adapters/docker"; +import { hasZeroDockerExitStatus } from "./docker-command-result"; +import { DOCKER_GPU_PATCH_TIMEOUT_MS } from "./docker-gpu-patch-constants"; +import type { + DockerGpuPatchBackend, + DockerGpuPatchDeps, + DockerGpuPatchMode, + DockerGpuPatchModeAttempt, + DockerGpuPatchModeKind, +} from "./docker-gpu-patch-types"; + +function resultText(result: { + stdout?: string | Buffer | null; + stderr?: string | Buffer | null; + error?: Error | null; +}): string { + return `${String(result.stderr || "")} ${String(result.stdout || "")} ${String( + result.error?.message || "", + )}`.trim(); +} + +function normalizeGpuDeviceForDocker(device: string | null | undefined): string { + const raw = String(device || "").trim(); + if (!raw || raw === "nvidia.com/gpu=all") return "all"; + if (raw.startsWith("nvidia.com/gpu=")) return raw.slice("nvidia.com/gpu=".length) || "all"; + return raw; +} + +function normalizeGpuDeviceForCdi(device: string | null | undefined): string { + const dockerDevice = normalizeGpuDeviceForDocker(device); + if ( + String(device || "") + .trim() + .startsWith("nvidia.com/gpu=") + ) { + return String(device).trim(); + } + return `nvidia.com/gpu=${dockerDevice || "all"}`; +} + +export function buildDockerGpuMode( + kind: DockerGpuPatchModeKind, + device?: string | null, + options: { backend?: DockerGpuPatchBackend } = {}, +): DockerGpuPatchMode { + if (kind === "startup-command") { + return { + kind, + label: "persistent sandbox startup command", + device: "", + args: [], + }; + } + const dockerDevice = normalizeGpuDeviceForDocker(device); + if (kind === "gpus") { + const gpuValue = dockerDevice === "all" ? "all" : `device=${dockerDevice}`; + return { + kind, + label: `--gpus ${gpuValue}`, + device: dockerDevice, + args: ["--gpus", gpuValue], + }; + } + if (kind === "nvidia-runtime") { + const args = ["--runtime", "nvidia", "--env", `NVIDIA_VISIBLE_DEVICES=${dockerDevice}`]; + if (options.backend === "jetson") { + args.push("--env", "NVIDIA_DRIVER_CAPABILITIES=compute,utility"); + } + return { + kind, + label: `--runtime nvidia (NVIDIA_VISIBLE_DEVICES=${dockerDevice})`, + device: dockerDevice, + args, + }; + } + const cdiDevice = normalizeGpuDeviceForCdi(device); + return { + kind, + label: `--device ${cdiDevice}`, + device: cdiDevice, + args: ["--device", cdiDevice], + }; +} + +export function buildDockerGpuModeCandidates( + device?: string | null, + options: { + cdiAvailable?: boolean; + backend?: DockerGpuPatchBackend; + dockerDesktopWsl?: boolean; + } = {}, +): DockerGpuPatchMode[] { + if (options.backend === "jetson") { + return [buildDockerGpuMode("nvidia-runtime", device, { backend: "jetson" })]; + } + // Match OpenShell's CDI preference when a usable NVIDIA spec is present, + // while retaining --gpus and the NVIDIA runtime as compatibility fallbacks. + // Docker Desktop WSL may advertise CDI directories without a resolvable + // nvidia.com/gpu device, so its compatibility route deliberately skips CDI. + const candidates: DockerGpuPatchMode[] = []; + if (options.cdiAvailable && !options.dockerDesktopWsl) { + candidates.push(buildDockerGpuMode("cdi", device)); + } + candidates.push(buildDockerGpuMode("gpus", device), buildDockerGpuMode("nvidia-runtime", device)); + return candidates; +} + +function parseDockerCdiSpecDirs(value: string | null | undefined): string[] { + const raw = String(value || "").trim(); + if (!raw || raw === "") return []; + try { + const parsed = JSON.parse(raw); + return Array.isArray(parsed) + ? parsed.map((entry) => String(entry || "").trim()).filter(Boolean) + : []; + } catch { + return raw + .split(/[\s,]+/) + .map((entry) => entry.trim()) + .filter(Boolean); + } +} + +export const DEFAULT_DOCKER_CDI_SPEC_DIRS = ["/etc/cdi", "/var/run/cdi"] as const; + +function readCdiSpecContent( + filePath: string, + readFile?: (path: string) => string | null, +): string | null { + if (readFile) return readFile(filePath); + try { + return fs.readFileSync(filePath, "utf-8"); + } catch { + return null; + } +} + +function isLikelyNvidiaCdiSpecFile( + filePath: string, + readFile?: (path: string) => string | null, +): boolean { + if (!/\.(json|ya?ml)$/i.test(filePath)) return false; + const content = readCdiSpecContent(filePath, readFile); + return content !== null && /nvidia\.com\/gpu|nvidia-container|libcuda|cuda/i.test(content); +} + +function listDirEntries( + dirPath: string, + readDir?: (path: string) => string[] | null, +): string[] | null { + if (readDir) return readDir(dirPath); + try { + return fs.readdirSync(dirPath); + } catch { + return null; + } +} + +function resolveCdiScanDirs(reportedDirs: readonly string[]): string[] { + const seen = new Set(); + const ordered: string[] = []; + for (const dir of [...reportedDirs, ...DEFAULT_DOCKER_CDI_SPEC_DIRS]) { + const trimmed = dir.trim(); + if (!trimmed || seen.has(trimmed)) continue; + seen.add(trimmed); + ordered.push(trimmed); + } + return ordered; +} + +export function dockerReportsNvidiaCdiDevices(deps: DockerGpuPatchDeps = {}): boolean { + const capture = deps.dockerCapture ?? dockerCapture; + let raw = ""; + try { + raw = capture(["info", "--format", "{{json .CDISpecDirs}}"], { + ignoreError: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }); + } catch { + // The default CDI directories may still contain a valid NVIDIA spec. + } + for (const dir of resolveCdiScanDirs(parseDockerCdiSpecDirs(raw))) { + const entries = listDirEntries(dir, deps.readDir); + if (!entries) continue; + if (entries.some((entry) => isLikelyNvidiaCdiSpecFile(path.join(dir, entry), deps.readFile))) { + return true; + } + } + return false; +} + +function probeDockerGpuMode( + mode: DockerGpuPatchMode, + image: string, + deps: DockerGpuPatchDeps, +): { ok: boolean; error: string | null } { + const run = deps.dockerRun ?? dockerRun; + const remove = deps.dockerRm ?? dockerRm; + const probeName = `nemoclaw-gpu-probe-${process.pid}-${Date.now()}-${Math.random() + .toString(16) + .slice(2, 8)}`; + try { + const result = run(["create", "--name", probeName, ...mode.args, image, "true"], { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }); + const ok = hasZeroDockerExitStatus(result); + return { ok, error: ok ? null : resultText(result) || "docker create failed" }; + } catch (error) { + return { ok: false, error: error instanceof Error ? error.message : String(error) }; + } finally { + remove(probeName, { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }); + } +} + +export function selectDockerGpuPatchMode( + options: { + image: string; + device?: string | null; + backend?: DockerGpuPatchBackend; + dockerDesktopWsl?: boolean; + }, + deps: DockerGpuPatchDeps = {}, +): { mode: DockerGpuPatchMode | null; attempts: DockerGpuPatchModeAttempt[] } { + const cdiAvailable = options.backend === "jetson" ? false : dockerReportsNvidiaCdiDevices(deps); + const attempts: DockerGpuPatchModeAttempt[] = []; + for (const mode of buildDockerGpuModeCandidates(options.device, { + cdiAvailable, + backend: options.backend, + dockerDesktopWsl: options.dockerDesktopWsl, + })) { + const result = probeDockerGpuMode(mode, options.image, deps); + const attempt = { mode, ok: result.ok, error: result.error }; + attempts.push(attempt); + if (attempt.ok) return { mode, attempts }; + } + return { mode: null, attempts }; +} diff --git a/src/lib/onboard/docker-gpu-patch-recreate-dns.test.ts b/src/lib/onboard/docker-gpu-patch-recreate-dns.test.ts new file mode 100644 index 00000000000..f72a39753de --- /dev/null +++ b/src/lib/onboard/docker-gpu-patch-recreate-dns.test.ts @@ -0,0 +1,86 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { createDockerGpuDnsInspectFixture as inspectFixture } from "./__test-helpers__/docker-gpu-patch-fixtures"; +import { buildDockerGpuCloneRunArgs } from "./docker-gpu-patch-clone"; +import { buildDockerGpuMode } from "./docker-gpu-patch-mode"; +import { recreateOpenShellDockerSandboxWithGpu } from "./docker-gpu-patch-recreate"; + +function recreateDeps(dns: string | null) { + const dockerResponses: Record = { + ps: "old-container-id\n", + inspect: JSON.stringify([inspectFixture()]), + }; + const dockerCapture = vi.fn((args: readonly string[]) => { + return dockerResponses[args[0] ?? ""] ?? ""; + }); + return { + dockerCapture, + dockerRun: vi.fn(() => ({ status: 0, stdout: "probe-id\n" })), + dockerRunDetached: vi.fn(() => ({ status: 0, stdout: "new-container-id\n" })), + dockerRename: vi.fn(() => ({ status: 0 })), + dockerStop: vi.fn(() => ({ status: 0 })), + dockerRm: vi.fn(() => ({ status: 0 })), + runOpenshell: vi.fn(() => ({ status: 0 })), + sleep: vi.fn(), + now: () => new Date("2026-05-15T00:00:00Z"), + detectSandboxFallbackDns: vi.fn(() => dns), + }; +} + +describe("Docker GPU recreate DNS fallback (#3579)", () => { + it("injects a discovered upstream only when Docker DNS is otherwise unavailable", () => { + const inspect = inspectFixture(); + const args = buildDockerGpuCloneRunArgs(inspect, buildDockerGpuMode("gpus"), { + sandboxFallbackDns: "8.8.8.8", + }); + expect(args).toEqual(expect.arrayContaining(["--dns", "8.8.8.8"])); + + inspect.HostConfig = { ...inspect.HostConfig, Dns: ["10.43.0.10"] }; + const configured = buildDockerGpuCloneRunArgs(inspect, buildDockerGpuMode("gpus"), { + sandboxFallbackDns: "8.8.8.8", + }); + expect(configured).toEqual(expect.arrayContaining(["--dns", "10.43.0.10"])); + expect(configured).not.toEqual(expect.arrayContaining(["--dns", "8.8.8.8"])); + + const host = buildDockerGpuCloneRunArgs(inspectFixture(), buildDockerGpuMode("gpus"), { + networkMode: "host", + sandboxFallbackDns: "8.8.8.8", + }); + expect(host).not.toEqual(expect.arrayContaining(["--dns", "8.8.8.8"])); + }); + + it("plumbs the discovered upstream through recreate into clone args", () => { + const deps = recreateDeps("9.9.9.9"); + recreateOpenShellDockerSandboxWithGpu({ sandboxName: "alpha", timeoutSecs: 1 }, deps); + + expect(deps.detectSandboxFallbackDns).toHaveBeenCalled(); + expect(deps.dockerRunDetached).toHaveBeenCalledWith( + expect.arrayContaining(["--dns", "9.9.9.9"]), + expect.objectContaining({ ignoreError: true }), + ); + }); + + it("does not add DNS through recreate when no upstream is discovered", () => { + const deps = recreateDeps(null); + recreateOpenShellDockerSandboxWithGpu({ sandboxName: "alpha", timeoutSecs: 1 }, deps); + + expect(deps.dockerRunDetached).not.toHaveBeenCalledWith( + expect.arrayContaining(["--dns"]), + expect.anything(), + ); + }); + + it("preserves the complete hostname regression contract", () => { + const args = buildDockerGpuCloneRunArgs(inspectFixture(), buildDockerGpuMode("gpus"), { + sandboxFallbackDns: "8.8.8.8", + }); + expect(args).toEqual( + expect.arrayContaining(["--add-host", "host.openshell.internal:172.17.0.1"]), + ); + expect(args).not.toEqual(expect.arrayContaining(["--network", "host"])); + expect(args).toEqual(expect.arrayContaining(["--dns", "8.8.8.8"])); + }); +}); diff --git a/src/lib/onboard/docker-gpu-patch-recreate.test.ts b/src/lib/onboard/docker-gpu-patch-recreate.test.ts new file mode 100644 index 00000000000..b48080bca2c --- /dev/null +++ b/src/lib/onboard/docker-gpu-patch-recreate.test.ts @@ -0,0 +1,136 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; +import { createDockerGpuInspectFixture as inspectFixture } from "./__test-helpers__/docker-gpu-patch-fixtures"; +import { recreateOpenShellDockerSandboxWithGpu } from "./docker-gpu-patch"; + +function dockerCaptureFixture() { + const responses: Record = { + ps: "old-container-id\n", + inspect: JSON.stringify([inspectFixture()]), + info: "", + }; + return vi.fn((args: readonly string[]) => responses[args[0]] ?? ""); +} + +describe("Docker GPU recreate orchestration", () => { + it("recreates the OpenShell-managed container and waits for supervisor exec", () => { + const dockerCapture = dockerCaptureFixture(); + const dockerRun = vi.fn(() => ({ status: 0, stdout: "probe-id\n" })); + const dockerRunDetached = vi.fn(() => ({ status: 0, stdout: "new-container-id\n" })); + const dockerRename = vi.fn(() => ({ status: 0 })); + const dockerStop = vi.fn(() => ({ status: 0 })); + const dockerRm = vi.fn(() => ({ status: 0 })); + const runOpenshell = vi.fn(() => ({ status: 0 })); + + const result = recreateOpenShellDockerSandboxWithGpu( + { sandboxName: "alpha", timeoutSecs: 1 }, + { + dockerCapture, + dockerRun, + dockerRunDetached, + dockerRename, + dockerStop, + dockerRm, + runOpenshell, + sleep: vi.fn(), + now: () => new Date("2026-05-12T00:00:00Z"), + readDir: vi.fn(() => null), + readFile: vi.fn(() => null), + }, + ); + + expect(result.newContainerId).toBe("new-container-id"); + expect(result.mode.kind).toBe("gpus"); + expect(dockerRunDetached).toHaveBeenCalledWith( + expect.arrayContaining([ + "--name", + "openshell-alpha", + "--gpus", + "all", + "--cap-add", + "SYS_ADMIN", + "--cap-add", + "SYS_PTRACE", + "--security-opt", + "apparmor=unconfined", + "--network", + "openshell-docker", + "--add-host", + "host.openshell.internal:172.17.0.1", + "--env", + "OPENSHELL_ENDPOINT=http://host.openshell.internal:8080/", + ]), + expect.objectContaining({ ignoreError: true }), + ); + expect(runOpenshell).toHaveBeenCalledWith( + ["sandbox", "exec", "-n", "alpha", "--", "true"], + expect.objectContaining({ ignoreError: true, suppressOutput: true }), + ); + const dockerRmCalls = dockerRm.mock.calls as unknown[][]; + const backupRmCall = dockerRmCalls.findIndex((call) => + String(call[0]).includes("nemoclaw-gpu-backup"), + ); + expect(backupRmCall).toBeGreaterThanOrEqual(0); + expect(dockerRm.mock.invocationCallOrder[backupRmCall]).toBeGreaterThan( + runOpenshell.mock.invocationCallOrder[0], + ); + }); + + it("can recreate during sandbox create before supervisor exec is allowed", () => { + const dockerCapture = dockerCaptureFixture(); + const dockerRunDetached = vi.fn( + (_args: readonly string[], _opts?: Record) => ({ + status: 0, + stdout: "new-container-id\n", + }), + ); + const dockerRm = vi.fn((_name: string) => ({ status: 0 })); + const runOpenshell = vi.fn(() => ({ status: 1, stderr: "phase: Provisioning" })); + + const result = recreateOpenShellDockerSandboxWithGpu( + { + sandboxName: "alpha", + timeoutSecs: 1, + waitForSupervisor: false, + openshellSandboxCommand: ["env", "CHAT_UI_URL=http://127.0.0.1:8642", "nemoclaw-start"], + }, + { + dockerCapture, + dockerRun: vi.fn(() => ({ status: 0, stdout: "probe-id\n" })), + dockerRunDetached, + dockerRename: vi.fn(() => ({ status: 0 })), + dockerStop: vi.fn(() => ({ status: 0 })), + dockerRm, + runOpenshell, + sleep: vi.fn(), + now: () => new Date("2026-05-12T00:00:00Z"), + }, + ); + + expect(result.newContainerId).toBe("new-container-id"); + expect(result.backupRemoved).toBe(false); + expect(result.originalName).toBe("openshell-alpha"); + expect(result.backupContainerName).toContain("nemoclaw-gpu-backup"); + expect(runOpenshell).not.toHaveBeenCalled(); + expect( + dockerRm.mock.calls.some((call) => String(call[0]).includes("nemoclaw-gpu-backup")), + ).toBe(false); + const cloneArgs = dockerRunDetached.mock.calls[0]?.[0] ?? []; + expect(cloneArgs).toEqual( + expect.arrayContaining([ + "--env", + "OPENSHELL_SANDBOX_COMMAND=env CHAT_UI_URL=http://127.0.0.1:8642 nemoclaw-start", + `sha256:${"c".repeat(64)}`, + ]), + ); + expect(cloneArgs.slice(cloneArgs.indexOf(`sha256:${"c".repeat(64)}`))).toEqual([ + `sha256:${"c".repeat(64)}`, + ]); + expect(dockerRunDetached).toHaveBeenCalledWith( + cloneArgs, + expect.objectContaining({ ignoreError: true }), + ); + }); +}); diff --git a/src/lib/onboard/docker-gpu-patch-recreate.ts b/src/lib/onboard/docker-gpu-patch-recreate.ts new file mode 100644 index 00000000000..22513f45fba --- /dev/null +++ b/src/lib/onboard/docker-gpu-patch-recreate.ts @@ -0,0 +1,376 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + dockerCapture, + dockerRename, + dockerRm, + dockerRun, + dockerRunDetached, + dockerStart, + dockerStop, +} from "../adapters/docker"; +import { hasZeroDockerExitStatus } from "./docker-command-result"; +import { detectSandboxFallbackDns } from "./docker-gpu-dns-fallback"; +import { detectTegraDeviceGroupGids } from "./docker-gpu-jetson-groups"; +import { + buildDockerGpuCloneRunArgs, + buildDockerGpuCloneRunOptions, + dockerContainerName, + parseDockerInspectJson, + sameContainerId, +} from "./docker-gpu-patch-clone"; +import { DOCKER_GPU_PATCH_TIMEOUT_MS } from "./docker-gpu-patch-constants"; +import { reconcileSupervisorReconnect } from "./docker-gpu-patch-finalize"; +import { selectDockerGpuPatchMode } from "./docker-gpu-patch-mode"; +import { restoreDockerGpuPatchBackupAfterRecreateFailure } from "./docker-gpu-patch-rollback"; +import type { + DockerContainerInspect, + DockerGpuPatchDeps, + DockerGpuPatchFailureContext, + DockerGpuPatchMode, + DockerGpuPatchResult, +} from "./docker-gpu-patch-types"; +import { waitForOpenShellSupervisorReconnect } from "./docker-gpu-supervisor-reconnect"; +import { openshellSandboxCommandEnvValue } from "./docker-startup-command-env"; +import { findOpenShellDockerSandboxContainerIds } from "./openshell-docker-sandbox-containers"; + +const DOCKER_GPU_PATCH_WAIT_SECS = 180; +const MAX_DOCKER_CONTAINER_NAME_LENGTH = 253; + +type RecreateDeps = Required< + Pick< + DockerGpuPatchDeps, + | "dockerCapture" + | "dockerRun" + | "dockerRunDetached" + | "dockerRename" + | "dockerRm" + | "dockerStart" + | "dockerStop" + | "sleep" + | "now" + | "detectSandboxFallbackDns" + | "detectTegraDeviceGroupGids" + > +> & + DockerGpuPatchDeps; + +function recreateDeps(deps: DockerGpuPatchDeps): RecreateDeps { + return { + dockerCapture, + dockerRun, + dockerRunDetached, + dockerRename, + dockerRm, + dockerStart, + dockerStop, + sleep: (seconds: number) => { + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, Math.max(0, seconds) * 1000); + }, + now: () => new Date(), + detectSandboxFallbackDns: () => detectSandboxFallbackDns(), + detectTegraDeviceGroupGids: () => detectTegraDeviceGroupGids(), + ...deps, + }; +} + +function resultText( + result: { + stdout?: string | Buffer | null; + stderr?: string | Buffer | null; + error?: Error | null; + } | null, +): string { + if (!result) return ""; + return `${String(result.stderr || "")} ${String(result.stdout || "")} ${String( + result.error?.message || "", + )}`.trim(); +} + +function inspectDockerContainer( + containerId: string, + deps: DockerGpuPatchDeps, +): DockerContainerInspect { + const capture = deps.dockerCapture ?? dockerCapture; + const output = capture(["inspect", "--type", "container", containerId], { + ignoreError: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }); + return parseDockerInspectJson(output); +} + +function buildBackupContainerName(originalName: string, now: Date): string { + const suffix = `-nemoclaw-gpu-backup-${String(now.getTime())}`; + const maxOriginalLength = MAX_DOCKER_CONTAINER_NAME_LENGTH - suffix.length; + return `${originalName.slice(0, Math.max(1, maxOriginalLength))}${suffix}`; +} + +function waitForNewContainerId( + sandboxName: string, + oldContainerId: string, + timeoutSecs: number, + deps: DockerGpuPatchDeps, +): string | null { + const d = recreateDeps(deps); + const deadline = Date.now() + Math.max(1, timeoutSecs) * 1000; + while (Date.now() <= deadline) { + const replacement = findOpenShellDockerSandboxContainerIds(sandboxName, deps).find( + (id) => !sameContainerId(id, oldContainerId), + ); + if (replacement) return replacement; + d.sleep(2); + } + return null; +} + +function decoratePatchError( + error: T, + context: DockerGpuPatchFailureContext, +): T & { dockerGpuPatch?: DockerGpuPatchFailureContext } { + (error as T & { dockerGpuPatch?: DockerGpuPatchFailureContext }).dockerGpuPatch = context; + return error; +} + +export function getDockerGpuPatchFailureContext( + error: unknown, +): DockerGpuPatchFailureContext | null { + if (error && typeof error === "object" && "dockerGpuPatch" in error) { + return (error as { dockerGpuPatch?: DockerGpuPatchFailureContext }).dockerGpuPatch || null; + } + return null; +} + +export function recreateOpenShellDockerSandboxContainer( + options: { + sandboxName: string; + gpuDevice?: string | null; + timeoutSecs?: number; + waitForSupervisor?: boolean; + openshellSandboxCommand?: readonly string[] | null; + expectedOldContainerId?: string | null; + backend?: "generic" | "jetson"; + dockerDesktopWsl?: boolean; + modeOverride?: DockerGpuPatchMode; + }, + deps: DockerGpuPatchDeps = {}, +): DockerGpuPatchResult { + const d = recreateDeps(deps); + const context: DockerGpuPatchFailureContext = { + sandboxName: options.sandboxName, + modeAttempts: [], + }; + try { + const containerIds = findOpenShellDockerSandboxContainerIds(options.sandboxName, deps); + const oldContainerId = containerIds[0]; + if (!oldContainerId) { + throw new Error( + `Could not find OpenShell Docker container for sandbox '${options.sandboxName}'.`, + ); + } + if ( + options.expectedOldContainerId != null && + (containerIds.length !== 1 || oldContainerId !== options.expectedOldContainerId) + ) { + throw new Error( + `OpenShell Docker container identity changed for sandbox '${options.sandboxName}'; ` + + "refusing startup-command recreation because the observed container differs from the pinned identity.", + ); + } + if (options.openshellSandboxCommand != null) { + // Validate the persisted command before image selection so malformed + // tokens fail before any container mutation can begin. + openshellSandboxCommandEnvValue(options.openshellSandboxCommand); + } + context.oldContainerId = oldContainerId; + const inspect = inspectDockerContainer(oldContainerId, deps); + const configuredImage = String(inspect.Config?.Image || "").trim(); + if (!configuredImage) { + throw new Error("OpenShell sandbox container inspect did not include an image."); + } + const immutableImage = String(inspect.Image || "").trim(); + const requiresImmutableImage = options.openshellSandboxCommand != null; + if (requiresImmutableImage && !/^sha256:[0-9a-f]{64}$/i.test(immutableImage)) { + throw new Error( + "OpenShell sandbox container inspect did not include a valid immutable image ID; " + + "refusing startup-command recreation from a mutable image tag.", + ); + } + const image = requiresImmutableImage ? immutableImage : configuredImage; + + const selection = options.modeOverride + ? { mode: options.modeOverride, attempts: [] } + : selectDockerGpuPatchMode( + { + image, + device: options.gpuDevice, + backend: options.backend, + dockerDesktopWsl: options.dockerDesktopWsl, + }, + deps, + ); + context.modeAttempts = selection.attempts; + context.selectedMode = selection.mode; + if (!selection.mode) { + throw new Error( + options.backend === "jetson" + ? "Docker did not accept the Jetson NVIDIA runtime GPU mode." + : "Docker did not accept --gpus, NVIDIA runtime, or CDI GPU modes.", + ); + } + + const originalName = dockerContainerName(inspect); + const backupContainerName = buildBackupContainerName(originalName, d.now()); + context.backupContainerName = backupContainerName; + const cloneOptions = buildDockerGpuCloneRunOptions(inspect); + cloneOptions.image = image; + cloneOptions.openshellSandboxCommand = options.openshellSandboxCommand ?? null; + const sandboxFallbackDns = d.detectSandboxFallbackDns(); + if (sandboxFallbackDns) cloneOptions.sandboxFallbackDns = sandboxFallbackDns; + if (selection.mode.kind !== "startup-command" && options.backend === "jetson") { + const tegraGroupGids = d.detectTegraDeviceGroupGids(); + if (tegraGroupGids.length > 0) { + cloneOptions.extraGroupGids = tegraGroupGids; + console.log( + ` ✓ Granting sandbox user access to Jetson Tegra GPU device nodes via --group-add ${tegraGroupGids.join( + ", ", + )} (so CUDA can open /dev/nvmap)`, + ); + } else { + console.warn( + " ⚠ Could not resolve the group owning Jetson Tegra GPU device nodes (/dev/nvmap); CUDA may fail with NvRmMemInitNvmap permission denied. Confirm /dev/nvmap exists and is group-readable on the host.", + ); + } + } + const cloneArgs = buildDockerGpuCloneRunArgs(inspect, selection.mode, cloneOptions); + + const containerMutationOptions = { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }; + const stopResult = d.dockerStop(oldContainerId, containerMutationOptions); + if (!hasZeroDockerExitStatus(stopResult)) { + context.rolledBack = hasZeroDockerExitStatus( + d.dockerStart(oldContainerId, containerMutationOptions), + ); + throw new Error( + `Could not stop original sandbox container: ${resultText(stopResult)}; ${ + context.rolledBack + ? "original sandbox container confirmed running" + : "restart failed; original sandbox container may be stopped" + }`, + ); + } + const renameResult = d.dockerRename( + oldContainerId, + backupContainerName, + containerMutationOptions, + ); + if (!hasZeroDockerExitStatus(renameResult)) { + d.dockerRename(backupContainerName, originalName, containerMutationOptions); + const restarted = hasZeroDockerExitStatus( + d.dockerStart(oldContainerId, containerMutationOptions), + ); + let originalNameRestored = false; + try { + originalNameRestored = + dockerContainerName(inspectDockerContainer(oldContainerId, deps)) === originalName; + } catch { + originalNameRestored = false; + } + context.rolledBack = restarted && originalNameRestored; + throw new Error( + `Could not move original sandbox container aside: ${resultText(renameResult)}; ${ + context.rolledBack + ? "original sandbox container restored" + : "restore failed; original sandbox container state is uncertain" + }`, + ); + } + + const runResult = d.dockerRunDetached(cloneArgs, { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }); + if (!hasZeroDockerExitStatus(runResult)) { + context.rolledBack = restoreDockerGpuPatchBackupAfterRecreateFailure( + { newContainerId: originalName, backupContainerName, originalName }, + deps, + ); + const containerDescription = + selection.mode.kind === "startup-command" + ? "recreated sandbox container" + : "GPU-enabled sandbox container"; + throw new Error( + `Could not start ${containerDescription}: ${resultText(runResult)}; ${ + context.rolledBack + ? "pre-patch sandbox restored" + : "rollback failed; pre-patch sandbox was NOT restored" + }`, + ); + } + + const newContainerId = + String(runResult.stdout || "").trim() || + waitForNewContainerId( + options.sandboxName, + oldContainerId, + options.timeoutSecs ?? DOCKER_GPU_PATCH_WAIT_SECS, + deps, + ); + if (!newContainerId) { + context.rolledBack = restoreDockerGpuPatchBackupAfterRecreateFailure( + { newContainerId: originalName, backupContainerName, originalName }, + deps, + ); + const containerDescription = + selection.mode.kind === "startup-command" + ? "Recreated sandbox container" + : "GPU-enabled sandbox container"; + throw new Error( + `${containerDescription} started, but Docker did not report its ID; ${ + context.rolledBack + ? "pre-patch sandbox restored" + : "rollback failed; pre-patch sandbox was NOT restored" + }`, + ); + } + context.newContainerId = newContainerId; + const selectedMode = selection.mode; + const result = (backupRemoved: boolean): DockerGpuPatchResult => ({ + applied: true, + oldContainerId, + newContainerId, + originalName, + backupContainerName, + mode: selectedMode, + backupRemoved, + }); + if (options.waitForSupervisor === false) return result(false); + + const execReady = waitForOpenShellSupervisorReconnect( + options.sandboxName, + options.timeoutSecs ?? DOCKER_GPU_PATCH_WAIT_SECS, + deps, + ); + const reconcile = reconcileSupervisorReconnect( + execReady, + { newContainerId, backupContainerName, originalName }, + deps, + ); + if (!reconcile.execReady) { + context.rolledBack = reconcile.rolledBack; + throw reconcile.error; + } + return result(reconcile.backupRemoved); + } catch (error) { + throw decoratePatchError(error instanceof Error ? error : new Error(String(error)), context); + } +} + +export const recreateOpenShellDockerSandboxWithGpu: ( + options: Omit[0], "modeOverride">, + deps?: DockerGpuPatchDeps, +) => DockerGpuPatchResult = recreateOpenShellDockerSandboxContainer; diff --git a/src/lib/onboard/docker-gpu-patch-rollback.test.ts b/src/lib/onboard/docker-gpu-patch-rollback.test.ts index 7a9341df38b..0d06a52dfe9 100644 --- a/src/lib/onboard/docker-gpu-patch-rollback.test.ts +++ b/src/lib/onboard/docker-gpu-patch-rollback.test.ts @@ -106,7 +106,10 @@ describe("recreateOpenShellDockerSandboxWithGpu rollback path", () => { ).toBe(false); }); - it("restores the pre-patch sandbox when the recreate run fails before the supervisor wait (#5512)", () => { + it.each([ + 1, + null, + ])("restores the pre-patch sandbox when the recreate run returns status %s before the supervisor wait (#5512)", (runStatus) => { const captureResponses: Record = { ps: "old-container-id\n", inspect: JSON.stringify([inspectFixture()]), @@ -117,7 +120,7 @@ describe("recreateOpenShellDockerSandboxWithGpu rollback path", () => { ); const dockerRun = vi.fn(() => ({ status: 0, stdout: "probe-id\n" })); // The recreate `docker run` fails after the original was renamed aside. - const dockerRunDetached = vi.fn(() => ({ status: 1, stderr: "docker: boom" })); + const dockerRunDetached = vi.fn(() => ({ status: runStatus, stderr: "docker: boom" })); const dockerRename = vi.fn((_old: string, _next: string) => ({ status: 0 })); const dockerStop = vi.fn(() => ({ status: 0 })); const dockerStart = vi.fn(() => ({ status: 0 })); @@ -162,6 +165,34 @@ describe("recreateOpenShellDockerSandboxWithGpu rollback path", () => { ).toBe(false); }); + it("does not start a replacement when the original-container rename has no exit status", () => { + const captureResponses: Record = { + ps: "old-container-id\n", + inspect: JSON.stringify([inspectFixture()]), + }; + const dockerCapture = vi.fn( + (args: readonly string[]) => captureResponses[String(args[0])] ?? "", + ); + const dockerRunDetached = vi.fn(() => ({ status: 0, stdout: "new-container-id\n" })); + + expect(() => + recreateOpenShellDockerSandboxWithGpu( + { sandboxName: "alpha", timeoutSecs: 1 }, + { + dockerCapture, + dockerRun: vi.fn(() => ({ status: 0, stdout: "probe-id\n" })), + dockerRunDetached, + dockerRename: vi.fn(() => ({ status: null, stderr: "timed out" })), + dockerStop: vi.fn(() => ({ status: 0 })), + dockerRm: vi.fn(() => ({ status: 0 })), + sleep: vi.fn(), + now: () => new Date("2026-05-12T00:00:00Z"), + }, + ), + ).toThrow(/Could not move original sandbox container aside/); + expect(dockerRunDetached).not.toHaveBeenCalled(); + }); + it("reports early recreate rollback failure when backup rename back fails (#5512)", () => { const captureResponses: Record = { ps: "old-container-id\n", diff --git a/src/lib/onboard/docker-gpu-patch-rollback.ts b/src/lib/onboard/docker-gpu-patch-rollback.ts new file mode 100644 index 00000000000..81532529503 --- /dev/null +++ b/src/lib/onboard/docker-gpu-patch-rollback.ts @@ -0,0 +1,69 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + dockerRename as defaultDockerRename, + dockerRm as defaultDockerRm, + dockerStart as defaultDockerStart, + dockerStop as defaultDockerStop, +} from "../adapters/docker"; +import { hasZeroDockerExitStatus } from "./docker-command-result"; +import { DOCKER_GPU_PATCH_TIMEOUT_MS } from "./docker-gpu-patch-constants"; +import type { DockerGpuPatchDeps } from "./docker-gpu-patch-types"; + +type DockerRunResult = { + status?: number | null; + stdout?: string | Buffer | null; + stderr?: string | Buffer | null; +}; + +type DockerRunOptions = Record; +type DockerContainerFn = (containerName: string, opts?: DockerRunOptions) => DockerRunResult; +type DockerRenameFn = ( + oldContainerName: string, + newContainerName: string, + opts?: DockerRunOptions, +) => DockerRunResult; + +export type ResolvedDockerGpuPatchRollbackDeps = { + dockerStop: DockerContainerFn; + dockerRm: DockerContainerFn; + dockerRename: DockerRenameFn; + dockerStart: DockerContainerFn; +}; + +export function resolveDockerGpuPatchRollbackDeps( + deps: DockerGpuPatchDeps, +): ResolvedDockerGpuPatchRollbackDeps { + return { + dockerStop: deps.dockerStop ?? defaultDockerStop, + dockerRm: deps.dockerRm ?? defaultDockerRm, + dockerRename: deps.dockerRename ?? defaultDockerRename, + dockerStart: deps.dockerStart ?? defaultDockerStart, + }; +} + +export function rollbackToBackupContainer( + refs: { newContainerId: string; backupContainerName: string; originalName: string }, + deps: ResolvedDockerGpuPatchRollbackDeps, +): boolean { + const containerOpts = { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }; + deps.dockerStop(refs.newContainerId, containerOpts); + deps.dockerRm(refs.newContainerId, containerOpts); + const restored = deps.dockerRename(refs.backupContainerName, refs.originalName, containerOpts); + if (!hasZeroDockerExitStatus(restored)) return false; + const started = deps.dockerStart(refs.originalName, containerOpts); + return hasZeroDockerExitStatus(started); +} + +/** Restore the original sandbox after `docker run` fails during GPU recreation. */ +export function restoreDockerGpuPatchBackupAfterRecreateFailure( + refs: { newContainerId: string; backupContainerName: string; originalName: string }, + deps: DockerGpuPatchDeps = {}, +): boolean { + return rollbackToBackupContainer(refs, resolveDockerGpuPatchRollbackDeps(deps)); +} diff --git a/src/lib/onboard/docker-gpu-patch-types.ts b/src/lib/onboard/docker-gpu-patch-types.ts new file mode 100644 index 00000000000..b0ac01836a7 --- /dev/null +++ b/src/lib/onboard/docker-gpu-patch-types.ts @@ -0,0 +1,224 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +type DockerRunResult = { + status?: number | null; + stdout?: string | Buffer | null; + stderr?: string | Buffer | null; + error?: Error | null; +}; + +type DockerRunOptions = Record; +type DockerCaptureFn = (args: readonly string[], opts?: DockerRunOptions) => string; +type DockerRunFn = (args: readonly string[], opts?: DockerRunOptions) => DockerRunResult; +type DockerContainerFn = (containerName: string, opts?: DockerRunOptions) => DockerRunResult; +type DockerRenameFn = ( + oldContainerName: string, + newContainerName: string, + opts?: DockerRunOptions, +) => DockerRunResult; +type DockerLogsFn = (containerName: string, opts?: { tail?: number; timeout?: number }) => string; + +export type DockerGpuPatchDeps = { + dockerCapture?: DockerCaptureFn; + dockerRun?: DockerRunFn; + dockerRunDetached?: DockerRunFn; + dockerRename?: DockerRenameFn; + dockerRm?: DockerContainerFn; + dockerStart?: DockerContainerFn; + dockerStop?: DockerContainerFn; + dockerLogs?: DockerLogsFn; + runOpenshell?: (args: string[], opts?: Record) => DockerRunResult; + runCaptureOpenshell?: (args: string[], opts?: Record) => string; + sleep?: (seconds: number) => void; + homedir?: () => string; + now?: () => Date; + detectSandboxFallbackDns?: () => string | null; + /** + * Resolve the host group ID(s) that own the Jetson/Tegra GPU device nodes + * (`/dev/nvmap`, `/dev/nvhost-*`). Used by the Jetson recreate to grant the + * sandbox user matching `--group-add` membership so CUDA can open them + * (#4231). Injectable so the Jetson permission path is testable without + * Tegra hardware. + */ + detectTegraDeviceGroupGids?: () => string[]; + /** Injectable directory lister for unit testing CDI spec discovery. */ + readDir?: (dirPath: string) => string[] | null; + /** Injectable file reader for unit testing CDI spec content checks. */ + readFile?: (filePath: string) => string | null; + /** + * Forwarded to the supervisor-reconnect wait. See + * `DockerGpuSupervisorReconnectDeps.errorPhaseDebouncePolls`. + */ + errorPhaseDebouncePolls?: number; +}; + +export type DockerGpuPatchModeKind = "gpus" | "nvidia-runtime" | "cdi" | "startup-command"; +export type DockerGpuPatchBackend = "generic" | "jetson"; + +export type DockerGpuPatchMode = { + kind: DockerGpuPatchModeKind; + label: string; + device: string; + args: string[]; +}; + +export type DockerGpuPatchModeAttempt = { + mode: DockerGpuPatchMode; + ok: boolean; + error: string | null; +}; + +export type DockerGpuPatchFailureContext = { + sandboxName: string; + oldContainerId?: string | null; + newContainerId?: string | null; + backupContainerName?: string | null; + selectedMode?: DockerGpuPatchMode | null; + modeAttempts?: DockerGpuPatchModeAttempt[]; + rolledBack?: boolean; +}; + +export type DockerGpuPatchResult = { + applied: true; + oldContainerId: string; + newContainerId: string; + originalName: string; + backupContainerName: string; + mode: DockerGpuPatchMode; + // True when the patch path also confirmed supervisor reconnect AND removed + // the backup container. False when the caller deferred the reconnect wait + // (via `waitForSupervisor: false`); the backup is still in place and the + // caller is responsible for calling `finalizeDockerGpuPatchBackup` after + // its own supervisor wait completes. + backupRemoved: boolean; +}; + +export type DockerGpuCloneRunOptions = { + image?: string | null; + networkMode?: string | null; + openshellEndpoint?: string | null; + sandboxFallbackDns?: string | null; + openshellSandboxCommand?: readonly string[] | null; + /** + * Extra supplementary group IDs to add to the recreated container via + * `--group-add`. On Jetson these are the host group(s) owning the Tegra GPU + * device nodes (`/dev/nvmap`, `/dev/nvhost-*`); granting the sandbox user + * membership lets CUDA's nvmap init open them instead of failing with + * `NvRmMemInitNvmap ... Permission denied` (#4231). + */ + extraGroupGids?: readonly string[] | null; +}; + +export type DockerGpuPatchDiagnostics = { + dir: string; + cleanupCommands: string[]; + summaryLines: string[]; +}; + +/** + * Subset of `docker inspect --format '{{json .State}}'` fields surfaced when + * the patched GPU sandbox container fails to become executable. We capture + * just the runtime/exit/health state — not the full inspect — because that + * is what tells the user *why* the patched create option broke (e.g. a + * non-zero ExitCode with `Error: "could not select device driver"`). + */ +export type DockerContainerState = { + Status?: string; + Running?: boolean; + Paused?: boolean; + Restarting?: boolean; + OOMKilled?: boolean; + Dead?: boolean; + ExitCode?: number; + Error?: string; + StartedAt?: string; + FinishedAt?: string; + Health?: { Status?: string; FailingStreak?: number } | null; +}; + +/** + * Snapshot of "is the patched sandbox even runnable?" — sandbox phase from + * OpenShell plus the patched Docker container's State. This is the data the + * caller needs to tell the user whether the failure is at the OpenShell + * sandbox layer (Error phase) vs. the Docker container layer (non-zero exit + * with a driver/runtime error) — see #4316. + */ +export type DockerGpuPatchSandboxSnapshot = { + sandboxPhase: string | null; + sandboxListLine: string | null; + patchedContainerState: DockerContainerState | null; +}; + +export type DockerGpuPatchFailureKind = + | "patched_container_failed" + | "sandbox_error_phase" + | "supervisor_unreachable" + | "proof_failure" + | "unknown"; + +export type DockerGpuPatchFailureClassification = { + kind: DockerGpuPatchFailureKind; + headline: string; + summaryLines: string[]; +}; + +export type DockerContainerInspect = { + Id?: string; + Image?: string; + Name?: string; + Config?: { + Image?: string; + Env?: string[] | null; + Labels?: Record | null; + Entrypoint?: string[] | string | null; + Cmd?: string[] | string | null; + User?: string; + WorkingDir?: string; + Hostname?: string; + Tty?: boolean; + OpenStdin?: boolean; + } | null; + HostConfig?: { + Binds?: string[] | null; + NetworkMode?: string; + RestartPolicy?: { Name?: string; MaximumRetryCount?: number } | null; + CapAdd?: string[] | null; + CapDrop?: string[] | null; + SecurityOpt?: string[] | null; + ExtraHosts?: string[] | null; + Memory?: number; + MemoryReservation?: number; + MemorySwap?: number; + NanoCpus?: number; + CpuShares?: number; + CpuQuota?: number; + CpuPeriod?: number; + CpusetCpus?: string; + CpusetMems?: string; + Privileged?: boolean; + Init?: boolean; + IpcMode?: string; + PidMode?: string; + GroupAdd?: string[] | null; + Dns?: string[] | null; + DnsSearch?: string[] | null; + DeviceRequests?: Array<{ + Driver?: string; + DeviceIDs?: string[] | null; + }> | null; + ShmSize?: number; + ReadonlyPaths?: string[] | null; + MaskedPaths?: string[] | null; + } | null; + NetworkSettings?: { + Networks?: Record< + string, + { + IPAddress?: string; + Gateway?: string; + Aliases?: string[] | null; + } + > | null; + } | null; +}; diff --git a/src/lib/onboard/docker-gpu-patch-wsl.test.ts b/src/lib/onboard/docker-gpu-patch-wsl.test.ts index 8bcc16bbcf4..0d239525edd 100644 --- a/src/lib/onboard/docker-gpu-patch-wsl.test.ts +++ b/src/lib/onboard/docker-gpu-patch-wsl.test.ts @@ -3,7 +3,8 @@ import { describe, expect, it } from "vitest"; -import { buildDockerGpuModeCandidates, shouldApplyDockerGpuPatch } from "./docker-gpu-patch"; +import { buildDockerGpuModeCandidates } from "./docker-gpu-patch"; +import { shouldApplyDockerGpuPatch } from "./docker-gpu-route-patch-adapter"; describe("shouldApplyDockerGpuPatch on Docker Desktop WSL", () => { it("ignores NEMOCLAW_DOCKER_GPU_PATCH=0 on Docker Desktop WSL where the patch is required", () => { diff --git a/src/lib/onboard/docker-gpu-patch.test.ts b/src/lib/onboard/docker-gpu-patch.test.ts deleted file mode 100644 index a51a707b5ad..00000000000 --- a/src/lib/onboard/docker-gpu-patch.test.ts +++ /dev/null @@ -1,1311 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; - -import { describe, expect, it, vi } from "vitest"; -import { getSandboxFailurePhase } from "../state/gateway"; -import { - buildDockerGpuCloneRunArgs, - buildDockerGpuCloneRunOptions, - buildDockerGpuMode, - buildDockerGpuModeCandidates, - captureDockerGpuPatchSandboxSnapshot, - classifyDockerGpuPatchFailure, - collectDockerGpuPatchDiagnostics, - type DockerContainerInspect, - detectSandboxFallbackDns, - detectTegraDeviceGroupGids, - dockerReportsNvidiaCdiDevices, - formatDockerInspectNetworkSummary, - getDockerGpuPatchNetworkMode, - getDockerGpuSupervisorReconnectTimeoutSecs, - recreateOpenShellDockerSandboxWithGpu, - selectDockerGpuPatchMode, - shouldApplyDockerGpuPatch, - waitForOpenShellSupervisorReconnect, -} from "./docker-gpu-patch"; - -function inspectFixture(): DockerContainerInspect { - return { - Id: "old-container-id", - Image: `sha256:${"c".repeat(64)}`, - Name: "/openshell-alpha", - Config: { - Image: "openshell/sandbox:abc", - Env: [ - "A=1", - "OPENSHELL_ENDPOINT=http://host.openshell.internal:8080/", - "OPENSHELL_TEST=1", - "OPENSHELL_SANDBOX_COMMAND=sleep infinity", - "NVIDIA_VISIBLE_DEVICES=void", - ], - Labels: { - "openshell.ai/managed-by": "openshell", - "openshell.ai/sandbox-name": "alpha", - "openshell.ai/sandbox-id": "sandbox-id", - }, - Entrypoint: ["/opt/openshell/bin/openshell-sandbox"], - Cmd: [], - User: "0", - WorkingDir: "/workspace", - Hostname: "alpha-host", - Tty: true, - }, - HostConfig: { - Binds: ["/host:/container:rw"], - NetworkMode: "openshell-docker", - RestartPolicy: { Name: "unless-stopped" }, - CapAdd: ["SYS_ADMIN", "NET_ADMIN"], - SecurityOpt: ["apparmor=unconfined"], - ExtraHosts: ["host.openshell.internal:172.17.0.1"], - Memory: 8 * 1024 * 1024 * 1024, - NanoCpus: 2_500_000_000, - }, - NetworkSettings: { - Networks: { - "openshell-docker": { - IPAddress: "172.18.0.2", - Gateway: "172.18.0.1", - Aliases: ["openshell-alpha"], - }, - }, - }, - }; -} - -describe("docker-gpu-patch", () => { - it("routes native CDI Linux directly unless the legacy patch is forced", () => { - expect( - shouldApplyDockerGpuPatch( - { sandboxGpuEnabled: true }, - { env: {}, platform: "linux", dockerDriverGateway: true }, - ), - ).toBe(false); - expect( - shouldApplyDockerGpuPatch( - { sandboxGpuEnabled: true }, - { - env: { NEMOCLAW_DOCKER_GPU_PATCH: "auto" }, - platform: "linux", - dockerDriverGateway: true, - }, - ), - ).toBe(false); - expect( - shouldApplyDockerGpuPatch( - { sandboxGpuEnabled: true }, - { env: { NEMOCLAW_DOCKER_GPU_PATCH: "1" }, platform: "linux", dockerDriverGateway: true }, - ), - ).toBe(true); - expect( - shouldApplyDockerGpuPatch( - { sandboxGpuEnabled: true }, - { env: { NEMOCLAW_DOCKER_GPU_PATCH: "0" }, platform: "linux", dockerDriverGateway: true }, - ), - ).toBe(false); - expect( - shouldApplyDockerGpuPatch( - { sandboxGpuEnabled: true }, - { env: {}, platform: "darwin", dockerDriverGateway: true }, - ), - ).toBe(false); - expect( - shouldApplyDockerGpuPatch( - { sandboxGpuEnabled: false }, - { env: {}, platform: "linux", dockerDriverGateway: true }, - ), - ).toBe(false); - }); - - it("keeps Jetson on the compatibility patch by default while honoring its opt-out", () => { - expect( - shouldApplyDockerGpuPatch( - { sandboxGpuEnabled: true, hostGpuPlatform: "jetson" }, - { env: {}, platform: "linux", dockerDriverGateway: true }, - ), - ).toBe(true); - expect( - shouldApplyDockerGpuPatch( - { sandboxGpuEnabled: true, hostGpuPlatform: "jetson" }, - { env: { NEMOCLAW_DOCKER_GPU_PATCH: "0" }, platform: "linux", dockerDriverGateway: true }, - ), - ).toBe(false); - }); - - it("builds clone args that preserve OpenShell labels and runtime settings", () => { - const args = buildDockerGpuCloneRunArgs(inspectFixture(), buildDockerGpuMode("gpus")); - - expect(args).toEqual( - expect.arrayContaining([ - "--name", - "openshell-alpha", - "--gpus", - "all", - "--env", - "A=1", - "--env", - "OPENSHELL_ENDPOINT=http://host.openshell.internal:8080/", - "--env", - "OPENSHELL_TEST=1", - "--label", - "openshell.ai/managed-by=openshell", - "--label", - "openshell.ai/sandbox-name=alpha", - "--volume", - "/host:/container:rw", - "--network", - "openshell-docker", - "--network-alias", - "openshell-alpha", - "--restart", - "unless-stopped", - "--cap-add", - "SYS_ADMIN", - "--security-opt", - "apparmor=unconfined", - "--add-host", - "host.openshell.internal:172.17.0.1", - "--memory", - String(8 * 1024 * 1024 * 1024), - "--cpus", - "2.5", - "--entrypoint", - "/opt/openshell/bin/openshell-sandbox", - "openshell/sandbox:abc", - ]), - ); - expect(args).not.toEqual(expect.arrayContaining(["--env", "NVIDIA_VISIBLE_DEVICES=void"])); - }); - - it("adds OpenShell's sandbox command env when the inspected container lacks one", () => { - const inspect = inspectFixture(); - inspect.Config!.Env = inspect.Config!.Env!.filter( - (entry) => !entry.startsWith("OPENSHELL_SANDBOX_COMMAND="), - ); - - const args = buildDockerGpuCloneRunArgs(inspect, buildDockerGpuMode("gpus"), { - openshellSandboxCommand: ["env", "CHAT_UI_URL=http://127.0.0.1:8642", "nemoclaw-start"], - }); - - expect(args).toEqual( - expect.arrayContaining([ - "--env", - "OPENSHELL_SANDBOX_COMMAND=env CHAT_UI_URL=http://127.0.0.1:8642 nemoclaw-start", - ]), - ); - }); - - it("adds SYS_PTRACE to the GPU clone when the baseline container lacks it", () => { - const inspect = inspectFixture(); - inspect.HostConfig!.CapAdd = ["SYS_ADMIN", "NET_ADMIN"]; - - const args = buildDockerGpuCloneRunArgs(inspect, buildDockerGpuMode("gpus")); - - expect(args).toEqual(expect.arrayContaining(["--cap-add", "SYS_PTRACE"])); - // The baseline caps are preserved alongside SYS_PTRACE. - expect(args).toEqual(expect.arrayContaining(["--cap-add", "SYS_ADMIN"])); - expect(args).toEqual(expect.arrayContaining(["--cap-add", "NET_ADMIN"])); - }); - - it("does not duplicate SYS_PTRACE when the baseline container already has it", () => { - const inspect = inspectFixture(); - inspect.HostConfig!.CapAdd = ["SYS_ADMIN", "SYS_PTRACE"]; - - const args = buildDockerGpuCloneRunArgs(inspect, buildDockerGpuMode("gpus")); - - const sysPtraceCount = args.filter((arg) => arg === "SYS_PTRACE").length; - expect(sysPtraceCount).toBe(1); - }); - - it("injects apparmor=unconfined when the baseline container has no apparmor profile", () => { - const inspect = inspectFixture(); - inspect.HostConfig!.SecurityOpt = []; - - const args = buildDockerGpuCloneRunArgs(inspect, buildDockerGpuMode("gpus")); - - expect(args).toEqual(expect.arrayContaining(["--security-opt", "apparmor=unconfined"])); - }); - - it("respects a baseline-pinned apparmor profile instead of overriding it", () => { - const inspect = inspectFixture(); - inspect.HostConfig!.SecurityOpt = ["apparmor=docker-default", "no-new-privileges"]; - - const args = buildDockerGpuCloneRunArgs(inspect, buildDockerGpuMode("gpus")); - - expect(args).toEqual(expect.arrayContaining(["--security-opt", "apparmor=docker-default"])); - expect(args).toEqual(expect.arrayContaining(["--security-opt", "no-new-privileges"])); - expect(args).not.toEqual(expect.arrayContaining(["--security-opt", "apparmor=unconfined"])); - }); - - it("formats sanitized network diagnostics without dumping provider secrets", () => { - const inspect = inspectFixture(); - inspect.Config?.Env?.push("NVIDIA_INFERENCE_API_KEY=secret"); - - const summary = formatDockerInspectNetworkSummary("old-container-id", inspect); - - expect(summary).toContain("target=old-container-id"); - expect(summary).toContain("network_mode=openshell-docker"); - expect(summary).toContain("host.openshell.internal:172.17.0.1"); - expect(summary).toContain("env.OPENSHELL_ENDPOINT=http://host.openshell.internal:8080/"); - expect(summary).toContain("openshell-docker: ip=172.18.0.2 gateway=172.18.0.1"); - expect(summary).not.toContain("NVIDIA_INFERENCE_API_KEY"); - expect(summary).not.toContain("secret"); - }); - - it("can switch the recreated sandbox to host networking for OpenShell callbacks", () => { - const inspect = inspectFixture(); - const options = buildDockerGpuCloneRunOptions(inspect, { - NEMOCLAW_DOCKER_GPU_PATCH_NETWORK: "host", - }); - const args = buildDockerGpuCloneRunArgs(inspect, buildDockerGpuMode("gpus"), options); - - expect(options).toEqual({ - networkMode: "host", - openshellEndpoint: "http://127.0.0.1:8080/", - }); - expect(args).toEqual(expect.arrayContaining(["--network", "host"])); - expect(args).toEqual( - expect.arrayContaining(["--env", "OPENSHELL_ENDPOINT=http://127.0.0.1:8080/"]), - ); - // --add-host writes to /etc/hosts (mount namespace), not the network - // stack, so it must survive even when --network=host is explicitly - // requested (#3562, #3568). - expect(args).toEqual( - expect.arrayContaining(["--add-host", "host.openshell.internal:172.17.0.1"]), - ); - expect(args).not.toEqual(expect.arrayContaining(["--network-alias", "openshell-alpha"])); - expect( - buildDockerGpuCloneRunOptions(inspect, { - NEMOCLAW_DOCKER_GPU_PATCH_NETWORK: "preserve", - }), - ).toEqual({}); - }); - - it("reports the Docker GPU patch network mode", () => { - expect(getDockerGpuPatchNetworkMode({})).toBe("preserve"); - expect(getDockerGpuPatchNetworkMode({ NEMOCLAW_DOCKER_GPU_PATCH_NETWORK: "host" })).toBe( - "host", - ); - expect(getDockerGpuPatchNetworkMode({ NEMOCLAW_DOCKER_GPU_PATCH_NETWORK: "preserve" })).toBe( - "preserve", - ); - expect(getDockerGpuPatchNetworkMode({ NEMOCLAW_DOCKER_GPU_PATCH_NETWORK: "bridge" })).toBe( - "preserve", - ); - expect(getDockerGpuPatchNetworkMode({ NEMOCLAW_DOCKER_GPU_PATCH_NETWORK: "bogus" })).toBe( - "preserve", - ); - }); - - it("maps default and explicit GPU devices to Docker --gpus values", () => { - expect(buildDockerGpuMode("gpus").args).toEqual(["--gpus", "all"]); - expect(buildDockerGpuMode("gpus", "nvidia.com/gpu=0").args).toEqual(["--gpus", "device=0"]); - expect(buildDockerGpuMode("gpus", "1,2").args).toEqual(["--gpus", "device=1,2"]); - }); - - it("uses Jetson NVIDIA runtime args without selecting generic --gpus or CDI candidates", () => { - expect(buildDockerGpuMode("nvidia-runtime", null, { backend: "jetson" }).args).toEqual([ - "--runtime", - "nvidia", - "--env", - "NVIDIA_VISIBLE_DEVICES=all", - "--env", - "NVIDIA_DRIVER_CAPABILITIES=compute,utility", - ]); - expect( - buildDockerGpuModeCandidates("all", { backend: "jetson", cdiAvailable: true }).map( - (m) => m.kind, - ), - ).toEqual(["nvidia-runtime"]); - }); - - it("uses a Docker-GPU-specific supervisor reconnect wait with an override", () => { - expect(getDockerGpuSupervisorReconnectTimeoutSecs(180, {})).toBe(900); - expect(getDockerGpuSupervisorReconnectTimeoutSecs(600, {})).toBe(900); - expect(getDockerGpuSupervisorReconnectTimeoutSecs(1200, {})).toBe(1200); - expect( - getDockerGpuSupervisorReconnectTimeoutSecs(180, { - NEMOCLAW_DOCKER_GPU_SUPERVISOR_RECONNECT_TIMEOUT: "30", - }), - ).toBe(30); - }); - - it("keeps Docker network diagnostics when old patch containers are gone", () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-gpu-diag-")); - try { - const liveInspect = inspectFixture(); - liveInspect.Id = "new-container-id"; - const dockerCapture = vi.fn((args: readonly string[]) => { - if (args[0] === "ps") return "new-container-id\n"; - if (args[0] === "inspect" && args[1] === "new-container-id") { - return JSON.stringify([liveInspect]); - } - throw new Error(`missing target ${String(args[1])}`); - }); - - const diagnostics = collectDockerGpuPatchDiagnostics( - "alpha", - { - context: { - sandboxName: "alpha", - oldContainerId: "old-container-id", - newContainerId: "new-container-id", - backupContainerName: "backup-container", - }, - }, - { - dockerCapture, - dockerLogs: vi.fn(() => ""), - homedir: () => tmpDir, - now: () => new Date("2026-05-12T00:00:00Z"), - }, - ); - - expect(diagnostics?.dir).toBeTruthy(); - const summary = fs.readFileSync( - path.join(diagnostics?.dir || "", "docker-network-summary.txt"), - "utf-8", - ); - expect(summary).toContain("target=new-container-id"); - expect(summary).toContain("network_mode=openshell-docker"); - } finally { - fs.rmSync(tmpDir, { recursive: true, force: true }); - } - }); - - it("falls back to NVIDIA runtime when Docker rejects --gpus", () => { - const dockerRun = vi - .fn() - .mockReturnValueOnce({ status: 1, stderr: "could not select device driver" }) - .mockReturnValueOnce({ status: 0, stdout: "probe-id" }); - - const selected = selectDockerGpuPatchMode( - { image: "openshell/sandbox:abc" }, - { - dockerCapture: vi.fn(() => ""), - dockerRun, - dockerRm: vi.fn(() => ({ status: 0 })), - readDir: vi.fn(() => null), - readFile: vi.fn(() => null), - }, - ); - - expect(selected.mode?.kind).toBe("nvidia-runtime"); - expect(selected.attempts.map((attempt) => attempt.mode.kind)).toEqual([ - "gpus", - "nvidia-runtime", - ]); - }); - - it("probes only NVIDIA runtime for Jetson Docker GPU mode", () => { - const dockerCapture = vi.fn(() => ""); - const dockerRun = vi.fn(() => ({ status: 0, stdout: "probe-id" })); - - const selected = selectDockerGpuPatchMode( - { image: "openshell/sandbox:abc", backend: "jetson" }, - { - dockerCapture, - dockerRun, - dockerRm: vi.fn(() => ({ status: 0 })), - }, - ); - - expect(selected.mode?.kind).toBe("nvidia-runtime"); - expect(selected.attempts.map((attempt) => attempt.mode.kind)).toEqual(["nvidia-runtime"]); - expect(dockerRun).toHaveBeenCalledWith( - expect.arrayContaining([ - "create", - "--runtime", - "nvidia", - "--env", - "NVIDIA_DRIVER_CAPABILITIES=compute,utility", - ]), - expect.objectContaining({ ignoreError: true }), - ); - expect(dockerCapture).not.toHaveBeenCalled(); - }); - - it("prefers CDI only when Docker reports readable NVIDIA CDI specs", () => { - expect(buildDockerGpuModeCandidates("all", { cdiAvailable: false }).map((m) => m.kind)).toEqual( - ["gpus", "nvidia-runtime"], - ); - // When a CDI spec is present, CDI is preferred first (see #4948); --gpus - // and the NVIDIA runtime remain as fallbacks. - expect(buildDockerGpuModeCandidates("all", { cdiAvailable: true }).map((m) => m.kind)).toEqual([ - "cdi", - "gpus", - "nvidia-runtime", - ]); - - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-cdi-")); - try { - fs.writeFileSync( - path.join(tmpDir, "nvidia.yaml"), - "cdiVersion: 0.6.0\nkind: nvidia.com/gpu\ndevices:\n - name: all\n", - ); - expect( - dockerReportsNvidiaCdiDevices({ - dockerCapture: vi.fn(() => JSON.stringify([tmpDir])), - }), - ).toBe(true); - } finally { - fs.rmSync(tmpDir, { recursive: true, force: true }); - } - }); - - it("detects NVIDIA CDI specs in /etc/cdi when docker info reports no dirs (#3575)", () => { - // Reproduces the Docker 29 + nvidia-container-toolkit + no daemon.json - // case: `docker info` returns an empty CDISpecDirs list, but Docker is - // still reading specs from its well-known default /etc/cdi. The detector - // should mirror Docker's behavior and surface cdi as available so the - // candidate list prefers `cdi` ahead of `--gpus all` on CDI hosts (#4948). - const readDir = vi.fn((dirPath: string) => (dirPath === "/etc/cdi" ? ["nvidia.yaml"] : null)); - const readFile = vi.fn((filePath: string) => - filePath === "/etc/cdi/nvidia.yaml" - ? "cdiVersion: 0.6.0\nkind: nvidia.com/gpu\ndevices:\n - name: all\n" - : null, - ); - expect( - dockerReportsNvidiaCdiDevices({ - dockerCapture: vi.fn(() => ""), - readDir, - readFile, - }), - ).toBe(true); - expect(readDir).toHaveBeenCalledWith("/etc/cdi"); - }); - - it("returns false when default CDI dirs hold no NVIDIA specs", () => { - expect( - dockerReportsNvidiaCdiDevices({ - dockerCapture: vi.fn(() => ""), - readDir: vi.fn(() => null), - readFile: vi.fn(() => null), - }), - ).toBe(false); - }); - - it("falls back to default CDI dirs even when docker info errors", () => { - const dockerCapture = vi.fn(() => { - throw new Error("docker daemon unreachable"); - }); - const readDir = vi.fn((dirPath: string) => - dirPath === "/var/run/cdi" ? ["nvidia.json"] : null, - ); - const readFile = vi.fn((filePath: string) => - filePath === "/var/run/cdi/nvidia.json" - ? JSON.stringify({ cdiVersion: "0.6.0", kind: "nvidia.com/gpu" }) - : null, - ); - expect(dockerReportsNvidiaCdiDevices({ dockerCapture, readDir, readFile })).toBe(true); - }); - - it("does not re-scan a directory that docker info already reported", () => { - const readDir = vi.fn((dirPath: string) => (dirPath === "/etc/cdi" ? ["nvidia.yaml"] : null)); - const readFile = vi.fn(() => "cdiVersion: 0.6.0\nkind: nvidia.com/gpu\n"); - dockerReportsNvidiaCdiDevices({ - dockerCapture: vi.fn(() => JSON.stringify(["/etc/cdi"])), - readDir, - readFile, - }); - const etcCdiCalls = readDir.mock.calls.filter(([dir]) => dir === "/etc/cdi"); - expect(etcCdiCalls.length).toBe(1); - }); - - it("recreates the OpenShell-managed container and waits for supervisor exec", () => { - const dockerCapture = vi.fn((args: readonly string[]) => { - if (args[0] === "ps") return "old-container-id\n"; - if (args[0] === "inspect") return JSON.stringify([inspectFixture()]); - if (args[0] === "info") return ""; - return ""; - }); - const dockerRun = vi.fn(() => ({ status: 0, stdout: "probe-id\n" })); - const dockerRunDetached = vi.fn(() => ({ status: 0, stdout: "new-container-id\n" })); - const dockerRename = vi.fn(() => ({ status: 0 })); - const dockerStop = vi.fn(() => ({ status: 0 })); - const dockerRm = vi.fn(() => ({ status: 0 })); - const runOpenshell = vi.fn(() => ({ status: 0 })); - - const result = recreateOpenShellDockerSandboxWithGpu( - { sandboxName: "alpha", timeoutSecs: 1 }, - { - dockerCapture, - dockerRun, - dockerRunDetached, - dockerRename, - dockerStop, - dockerRm, - runOpenshell, - sleep: vi.fn(), - now: () => new Date("2026-05-12T00:00:00Z"), - readDir: vi.fn(() => null), - readFile: vi.fn(() => null), - }, - ); - - expect(result.newContainerId).toBe("new-container-id"); - expect(result.mode.kind).toBe("gpus"); - expect(dockerRunDetached).toHaveBeenCalledWith( - expect.arrayContaining([ - "--name", - "openshell-alpha", - "--gpus", - "all", - "--cap-add", - "SYS_ADMIN", - "--cap-add", - "SYS_PTRACE", - "--security-opt", - "apparmor=unconfined", - "--network", - "openshell-docker", - "--add-host", - "host.openshell.internal:172.17.0.1", - "--env", - "OPENSHELL_ENDPOINT=http://host.openshell.internal:8080/", - ]), - expect.objectContaining({ ignoreError: true }), - ); - expect(runOpenshell).toHaveBeenCalledWith( - ["sandbox", "exec", "-n", "alpha", "--", "true"], - expect.objectContaining({ ignoreError: true, suppressOutput: true }), - ); - const dockerRmCalls = dockerRm.mock.calls as unknown[][]; - const backupRmCall = dockerRmCalls.findIndex((call) => - String(call[0]).includes("nemoclaw-gpu-backup"), - ); - expect(backupRmCall).toBeGreaterThanOrEqual(0); - // Backup container is removed only AFTER supervisor reconnect confirms - // the GPU container is reachable. If reconnect fails the rollback path - // restores the backup under the original name (see the rollback test - // below), so the backup must outlive the supervisor probe. - expect(dockerRm.mock.invocationCallOrder[backupRmCall]).toBeGreaterThan( - runOpenshell.mock.invocationCallOrder[0], - ); - }); - - it("can recreate during sandbox create before supervisor exec is allowed", () => { - const dockerCapture = vi.fn((args: readonly string[]) => { - if (args[0] === "ps") return "old-container-id\n"; - if (args[0] === "inspect") return JSON.stringify([inspectFixture()]); - if (args[0] === "info") return ""; - return ""; - }); - const dockerRunDetached = vi.fn( - (_args: readonly string[], _opts?: Record) => ({ - status: 0, - stdout: "new-container-id\n", - }), - ); - const dockerRm = vi.fn((_name: string) => ({ status: 0 })); - const runOpenshell = vi.fn(() => ({ status: 1, stderr: "phase: Provisioning" })); - - const result = recreateOpenShellDockerSandboxWithGpu( - { - sandboxName: "alpha", - timeoutSecs: 1, - waitForSupervisor: false, - openshellSandboxCommand: ["env", "CHAT_UI_URL=http://127.0.0.1:8642", "nemoclaw-start"], - }, - { - dockerCapture, - dockerRun: vi.fn(() => ({ status: 0, stdout: "probe-id\n" })), - dockerRunDetached, - dockerRename: vi.fn(() => ({ status: 0 })), - dockerStop: vi.fn(() => ({ status: 0 })), - dockerRm, - runOpenshell, - sleep: vi.fn(), - now: () => new Date("2026-05-12T00:00:00Z"), - }, - ); - - expect(result.newContainerId).toBe("new-container-id"); - expect(result.backupRemoved).toBe(false); - expect(result.originalName).toBe("openshell-alpha"); - expect(result.backupContainerName).toContain("nemoclaw-gpu-backup"); - expect(runOpenshell).not.toHaveBeenCalled(); - // The create path takes the supervisor wait into its own hands later in - // the flow. The patch helper must NOT remove the backup yet — that would - // re-introduce the deleted-backup / failed-new state #4664 fixes. - expect( - dockerRm.mock.calls.some((call) => String(call[0]).includes("nemoclaw-gpu-backup")), - ).toBe(false); - const cloneArgs = dockerRunDetached.mock.calls[0]?.[0] ?? []; - expect(cloneArgs).toEqual( - expect.arrayContaining([ - "--env", - "OPENSHELL_SANDBOX_COMMAND=env CHAT_UI_URL=http://127.0.0.1:8642 nemoclaw-start", - `sha256:${"c".repeat(64)}`, - ]), - ); - expect(cloneArgs.slice(cloneArgs.indexOf(`sha256:${"c".repeat(64)}`))).toEqual([ - `sha256:${"c".repeat(64)}`, - ]); - expect(dockerRunDetached).toHaveBeenCalledWith( - cloneArgs, - expect.objectContaining({ ignoreError: true }), - ); - }); -}); - -describe("docker-gpu-patch sandbox DNS fallback (#3579)", () => { - it("returns the systemd-resolved upstream when /etc/resolv.conf is loopback-only", () => { - const readFile = (p: string): string | null => { - if (p === "/etc/resolv.conf") return "nameserver 127.0.0.53\nsearch lan\n"; - if (p === "/run/systemd/resolve/resolv.conf") { - return "# Generated by systemd-resolved\nnameserver 8.8.8.8\nnameserver 1.1.1.1\n"; - } - return null; - }; - expect(detectSandboxFallbackDns({ readFile })).toBe("8.8.8.8"); - }); - - it("returns null when /etc/resolv.conf has a non-loopback resolver", () => { - const readFile = (_p: string): string | null => "nameserver 192.168.1.1\n"; - expect(detectSandboxFallbackDns({ readFile })).toBeNull(); - }); - - it("returns null when /etc/resolv.conf is missing", () => { - expect(detectSandboxFallbackDns({ readFile: () => null })).toBeNull(); - }); - - it("returns null when /etc/resolv.conf is loopback-only but systemd upstream is missing", () => { - const readFile = (p: string): string | null => { - if (p === "/etc/resolv.conf") return "nameserver 127.0.0.53\n"; - return null; - }; - expect(detectSandboxFallbackDns({ readFile })).toBeNull(); - }); - - it("injects sandboxFallbackDns via --dns on non-host networks", () => { - const inspect = inspectFixture(); - const args = buildDockerGpuCloneRunArgs(inspect, buildDockerGpuMode("gpus"), { - sandboxFallbackDns: "8.8.8.8", - }); - expect(args).toEqual(expect.arrayContaining(["--dns", "8.8.8.8"])); - }); - - it("does not inject sandboxFallbackDns when OpenShell already configured --dns", () => { - const inspect = inspectFixture(); - inspect.HostConfig = { ...inspect.HostConfig, Dns: ["10.43.0.10"] }; - const args = buildDockerGpuCloneRunArgs(inspect, buildDockerGpuMode("gpus"), { - sandboxFallbackDns: "8.8.8.8", - }); - expect(args).toEqual(expect.arrayContaining(["--dns", "10.43.0.10"])); - expect(args).not.toEqual(expect.arrayContaining(["--dns", "8.8.8.8"])); - }); - - it("does not inject --dns when network mode is host (Docker ignores --dns on host)", () => { - const inspect = inspectFixture(); - const args = buildDockerGpuCloneRunArgs(inspect, buildDockerGpuMode("gpus"), { - networkMode: "host", - sandboxFallbackDns: "8.8.8.8", - }); - expect(args).not.toEqual(expect.arrayContaining(["--dns", "8.8.8.8"])); - }); - - it("plumbs detectSandboxFallbackDns through recreateOpenShellDockerSandboxWithGpu into clone args", () => { - // Wire-through test: the production callsite at docker-gpu-patch.ts - // calls d.detectSandboxFallbackDns() and merges the result into - // cloneOptions.sandboxFallbackDns before building the run args. Stub - // the deps hook and verify --dns lands in the final dockerRunDetached call. - const dockerCapture = vi.fn((args: readonly string[]) => { - if (args[0] === "ps") return "old-container-id\n"; - if (args[0] === "inspect") return JSON.stringify([inspectFixture()]); - if (args[0] === "info") return ""; - return ""; - }); - const dockerRunDetached = vi.fn(() => ({ status: 0, stdout: "new-container-id\n" })); - const detectSandboxFallbackDnsStub = vi.fn(() => "9.9.9.9"); - - recreateOpenShellDockerSandboxWithGpu( - { sandboxName: "alpha", timeoutSecs: 1 }, - { - dockerCapture, - dockerRun: vi.fn(() => ({ status: 0, stdout: "probe-id\n" })), - dockerRunDetached, - dockerRename: vi.fn(() => ({ status: 0 })), - dockerStop: vi.fn(() => ({ status: 0 })), - dockerRm: vi.fn(() => ({ status: 0 })), - runOpenshell: vi.fn(() => ({ status: 0 })), - sleep: vi.fn(), - now: () => new Date("2026-05-15T00:00:00Z"), - detectSandboxFallbackDns: detectSandboxFallbackDnsStub, - }, - ); - - expect(detectSandboxFallbackDnsStub).toHaveBeenCalled(); - expect(dockerRunDetached).toHaveBeenCalledWith( - expect.arrayContaining(["--dns", "9.9.9.9"]), - expect.objectContaining({ ignoreError: true }), - ); - }); - - it("does not inject --dns through recreate when fallback detection returns null", () => { - const dockerCapture = vi.fn((args: readonly string[]) => { - if (args[0] === "ps") return "old-container-id\n"; - if (args[0] === "inspect") return JSON.stringify([inspectFixture()]); - if (args[0] === "info") return ""; - return ""; - }); - const dockerRunDetached = vi.fn(() => ({ status: 0, stdout: "new-container-id\n" })); - - recreateOpenShellDockerSandboxWithGpu( - { sandboxName: "alpha", timeoutSecs: 1 }, - { - dockerCapture, - dockerRun: vi.fn(() => ({ status: 0, stdout: "probe-id\n" })), - dockerRunDetached, - dockerRename: vi.fn(() => ({ status: 0 })), - dockerStop: vi.fn(() => ({ status: 0 })), - dockerRm: vi.fn(() => ({ status: 0 })), - runOpenshell: vi.fn(() => ({ status: 0 })), - sleep: vi.fn(), - now: () => new Date("2026-05-15T00:00:00Z"), - detectSandboxFallbackDns: () => null, - }, - ); - - // No --dns from the fallback path (and inspectFixture() does not preset host.Dns). - expect(dockerRunDetached).not.toHaveBeenCalledWith( - expect.arrayContaining(["--dns"]), - expect.anything(), - ); - }); - - it("includes every hostname from the manager-provided regression manifest (#3579)", () => { - // The four hostnames called out in #3579's manager-provided spec: - // host.openshell.internal → resolved via --add-host (mount namespace) - // google.com → public DNS via embedded Docker resolver - // gateway.discord.gg → public DNS via embedded Docker resolver - // integrate.api.nvidia.com → public DNS via embedded Docker resolver - // - // Unit-testable invariants that together cover all four: - // 1. --add-host preserves the host.openshell.internal mapping - // 2. Network mode is NOT "host" by default (so Docker's embedded DNS - // at 127.0.0.11 kicks in for the three public hostnames) - // 3. When the host has a loopback-only resolver, the real upstream - // is injected via --dns so DNS works even if the daemon's - // embedded resolver can't reach the upstream by itself. - const inspect = inspectFixture(); - const args = buildDockerGpuCloneRunArgs(inspect, buildDockerGpuMode("gpus"), { - sandboxFallbackDns: "8.8.8.8", - }); - - // host.openshell.internal - expect(args).toEqual( - expect.arrayContaining(["--add-host", "host.openshell.internal:172.17.0.1"]), - ); - // google.com / gateway.discord.gg / integrate.api.nvidia.com — covered by - // (a) not pinning --network=host and (b) injecting --dns when the host - // has a loopback-only resolver. - expect(args).not.toEqual(expect.arrayContaining(["--network", "host"])); - expect(args).toEqual(expect.arrayContaining(["--dns", "8.8.8.8"])); - }); -}); - -// Jetson `/dev/nvmap` group-permission propagation (#4231). The reporter's -// Jetson Orin sandbox saw the GPU devices mounted but CUDA failed with -// `NvRmMemInitNvmap ... Permission denied` / `cuInit(0)=999` because the -// sandbox user (uid/gid 998) was not in the `video` group that owns -// `/dev/nvmap` (`crw-rw---- root video`). The Jetson recreate must grant that -// group via `--group-add` so CUDA can initialize. -describe("Jetson /dev/nvmap group propagation (#4231)", () => { - it("returns the owning GID(s) of present Tegra device nodes, skipping missing and root-owned", () => { - const deviceGids: Record = { - "/dev/nvmap": 44, // root video - "/dev/nvhost-ctrl": 44, - "/dev/nvhost-gpu": 0, // root root — skipped (root already has access) - "/dev/nvgpu/igpu0/ctrl": 110, // render - // every other Tegra node is absent on this host - }; - const gids = detectTegraDeviceGroupGids({ - statDeviceGid: (p: string) => (p in deviceGids ? deviceGids[p] : null), - }); - // Deduped, sorted numerically, root (0) and missing nodes excluded. - expect(gids).toEqual(["44", "110"]); - }); - - it("returns no GIDs when no Tegra device nodes are present (non-Jetson host)", () => { - expect(detectTegraDeviceGroupGids({ statDeviceGid: () => null })).toEqual([]); - }); - - it("emits --group-add for extraGroupGids and dedupes against existing GroupAdd", () => { - const inspect = inspectFixture(); - inspect.HostConfig!.GroupAdd = ["44"]; // baseline already carries video - const args = buildDockerGpuCloneRunArgs( - inspect, - buildDockerGpuMode("nvidia-runtime", null, { backend: "jetson" }), - { extraGroupGids: ["44", "110"] }, - ); - // `44` is added exactly once (baseline + extra deduped); `110` added. - expect(args.filter((arg, i) => args[i - 1] === "--group-add" && arg === "44").length).toBe(1); - expect(args).toEqual(expect.arrayContaining(["--group-add", "110"])); - }); - - it("does not add --group-add when extraGroupGids is absent", () => { - const inspect = inspectFixture(); - inspect.HostConfig!.GroupAdd = []; - const args = buildDockerGpuCloneRunArgs(inspect, buildDockerGpuMode("gpus")); - expect(args).not.toEqual(expect.arrayContaining(["--group-add"])); - }); - - it("plumbs detected Tegra device GIDs into the Jetson recreate as --group-add", () => { - const dockerCapture = vi.fn((args: readonly string[]) => { - if (args[0] === "ps") return "old-container-id\n"; - if (args[0] === "inspect") return JSON.stringify([inspectFixture()]); - if (args[0] === "info") return ""; - return ""; - }); - const dockerRunDetached = vi.fn(() => ({ status: 0, stdout: "new-container-id\n" })); - const detectTegraDeviceGroupGidsStub = vi.fn(() => ["44"]); - - recreateOpenShellDockerSandboxWithGpu( - { sandboxName: "alpha", timeoutSecs: 1, backend: "jetson" }, - { - dockerCapture, - dockerRun: vi.fn(() => ({ status: 0, stdout: "probe-id\n" })), - dockerRunDetached, - dockerRename: vi.fn(() => ({ status: 0 })), - dockerStop: vi.fn(() => ({ status: 0 })), - dockerRm: vi.fn(() => ({ status: 0 })), - runOpenshell: vi.fn(() => ({ status: 0 })), - sleep: vi.fn(), - now: () => new Date("2026-05-15T00:00:00Z"), - detectSandboxFallbackDns: () => null, - detectTegraDeviceGroupGids: detectTegraDeviceGroupGidsStub, - }, - ); - - expect(detectTegraDeviceGroupGidsStub).toHaveBeenCalled(); - expect(dockerRunDetached).toHaveBeenCalledWith( - expect.arrayContaining(["--group-add", "44"]), - expect.objectContaining({ ignoreError: true }), - ); - }); - - it("does not add Tegra device GIDs for the generic (non-Jetson) backend", () => { - const dockerCapture = vi.fn((args: readonly string[]) => { - if (args[0] === "ps") return "old-container-id\n"; - if (args[0] === "inspect") return JSON.stringify([inspectFixture()]); - if (args[0] === "info") return ""; - return ""; - }); - const dockerRunDetached = vi.fn(() => ({ status: 0, stdout: "new-container-id\n" })); - const detectTegraDeviceGroupGidsStub = vi.fn(() => ["44"]); - - recreateOpenShellDockerSandboxWithGpu( - { sandboxName: "alpha", timeoutSecs: 1, backend: "generic" }, - { - dockerCapture, - dockerRun: vi.fn(() => ({ status: 0, stdout: "probe-id\n" })), - dockerRunDetached, - dockerRename: vi.fn(() => ({ status: 0 })), - dockerStop: vi.fn(() => ({ status: 0 })), - dockerRm: vi.fn(() => ({ status: 0 })), - runOpenshell: vi.fn(() => ({ status: 0 })), - sleep: vi.fn(), - now: () => new Date("2026-05-15T00:00:00Z"), - detectSandboxFallbackDns: () => null, - detectTegraDeviceGroupGids: detectTegraDeviceGroupGidsStub, - }, - ); - - // Generic backend never queries Tegra device groups and never emits the - // extra --group-add (inspectFixture has no baseline GroupAdd). - expect(detectTegraDeviceGroupGidsStub).not.toHaveBeenCalled(); - expect(dockerRunDetached).not.toHaveBeenCalledWith( - expect.arrayContaining(["--group-add", "44"]), - expect.anything(), - ); - }); -}); - -// Regression coverage for NemoClaw issue #4316: the Docker GPU patch path -// must distinguish "sandbox never became executable" (Error phase / dead -// container) from "GPU proof failed inside an executable sandbox", and the -// readiness wait must short-circuit on a terminal failure phase instead of -// burning the full timeout window. -describe("docker-gpu-patch Error-phase diagnostics (#4316)", () => { - it("detects terminal failure phases in `openshell sandbox list` output", () => { - const errorList = "my-sandbox Error 2s ago"; - expect(getSandboxFailurePhase(errorList, "my-sandbox")).toBe("Error"); - expect(getSandboxFailurePhase("my-sandbox CrashLoopBackOff 3s ago", "my-sandbox")).toBe( - "CrashLoopBackOff", - ); - expect(getSandboxFailurePhase("my-sandbox Failed 3s ago", "my-sandbox")).toBe("Failed"); - - expect(getSandboxFailurePhase("my-sandbox Ready 3s ago", "my-sandbox")).toBeNull(); - expect(getSandboxFailurePhase("other Error 3s ago", "my-sandbox")).toBeNull(); - expect(getSandboxFailurePhase("", "my-sandbox")).toBeNull(); - }); - - // Create/readiness-wait Error-phase behavior (including the #6043 transient - // debounce and its env contract) lives in sandbox-readiness-tracing.test.ts. - - it("short-circuits the supervisor-reconnect wait when the sandbox enters Error phase", () => { - // Without the short-circuit, a patched container that crashes on startup - // leaves users waiting the full 900s+ supervisor-reconnect timeout before - // any Error-phase diagnostics run. With the debounce now in place, this - // test asserts the K=1 (no-debounce) behavior explicitly so the original - // fast-fail intent is preserved when the operator opts out of the - // debounce. - const runOpenshell = vi.fn(() => ({ status: 1, stderr: "sandbox not ready" })); - const listOutputs = ["alpha Provisioning 1s ago", "alpha Error 3s ago"]; - let i = 0; - const runCaptureOpenshell = vi.fn(() => listOutputs[Math.min(i++, listOutputs.length - 1)]); - const sleep = vi.fn(); - - const ok = waitForOpenShellSupervisorReconnect("alpha", 600, { - runOpenshell, - runCaptureOpenshell, - sleep, - errorPhaseDebouncePolls: 1, - }); - - expect(ok).toBe(false); - // Without short-circuit we'd loop ~300 iterations. With K=1 the second - // iteration's list output shows Error and the wait bails out. - expect(runOpenshell).toHaveBeenCalledTimes(2); - expect(sleep).toHaveBeenCalledTimes(1); - }); - - it("prefers `sandbox list` phase over `sandbox get` when both are present (stale get)", () => { - // Regression guard for #4316 CodeRabbit feedback: when `sandbox get` - // returns a stale Phase (e.g. Provisioning while the gateway has already - // transitioned the row to Error), the list-derived phase must take - // precedence so the classifier doesn't act on stale data. - const runCaptureOpenshell = vi.fn((args: readonly string[]) => { - if (args[0] === "sandbox" && args[1] === "get") { - return "Name: alpha\nPhase: Provisioning\n"; - } - if (args[0] === "sandbox" && args[1] === "list") { - return "alpha Error 2s ago\n"; - } - return ""; - }); - - const snapshot = captureDockerGpuPatchSandboxSnapshot( - "alpha", - { patchedContainerId: null }, - { runCaptureOpenshell }, - ); - - expect(snapshot.sandboxPhase).toBe("Error"); - expect(snapshot.sandboxListLine).toContain("Error"); - }); - - it("uses the list-derived phase whenever the sandbox row is present", () => { - // Regression guard for CodeRabbit feedback: `sandbox list` reflects the - // gateway's table row and should be the phase used by the failure - // classifier whenever that row is available, even if `sandbox get` reports - // a different phase. - const runCaptureOpenshell = vi.fn((args: readonly string[]) => { - if (args[0] === "sandbox" && args[1] === "get") { - return "Name: alpha\nPhase: Error\nReason: ContainerCannotRun\n"; - } - if (args[0] === "sandbox" && args[1] === "list") { - return "alpha Ready 1m ago\n"; - } - return ""; - }); - - const snapshot = captureDockerGpuPatchSandboxSnapshot( - "alpha", - { patchedContainerId: null }, - { runCaptureOpenshell }, - ); - - expect(snapshot.sandboxPhase).toBe("Ready"); - expect(snapshot.sandboxListLine).toContain("Ready"); - }); - - it("keeps the get-derived phase when the sandbox row is absent from list output", () => { - // Complement to the precedence test: if `sandbox list` has no row for - // the named sandbox (e.g. the gateway lost track of it), the get-derived - // phase is the only signal we have — don't drop it. - const runCaptureOpenshell = vi.fn((args: readonly string[]) => { - if (args[0] === "sandbox" && args[1] === "get") { - return "Name: alpha\nPhase: Terminated\n"; - } - if (args[0] === "sandbox" && args[1] === "list") { - return "other-box Ready 2s ago\n"; - } - return ""; - }); - - const snapshot = captureDockerGpuPatchSandboxSnapshot( - "alpha", - { patchedContainerId: null }, - { runCaptureOpenshell }, - ); - - expect(snapshot.sandboxPhase).toBe("Terminated"); - expect(snapshot.sandboxListLine).toBeNull(); - }); - - it("captures sandbox phase and patched container State via the snapshot helper", () => { - const runCaptureOpenshell = vi.fn((args: readonly string[]) => { - if (args[0] === "sandbox" && args[1] === "get") { - return "Name: alpha\nPhase: Error\nReason: ContainerExit\n"; - } - if (args[0] === "sandbox" && args[1] === "list") { - return "alpha Error 1m ago\n"; - } - return ""; - }); - const dockerCapture = vi.fn((args: readonly string[]) => { - if (args[0] === "inspect" && args[1] === "--format" && args[2] === "{{json .State}}") { - return JSON.stringify({ - Status: "exited", - Running: false, - ExitCode: 125, - Error: 'could not select device driver "nvidia" with capabilities: [[gpu]]', - OOMKilled: false, - StartedAt: "2026-05-12T00:00:00Z", - FinishedAt: "2026-05-12T00:00:01Z", - }); - } - return ""; - }); - - const snapshot = captureDockerGpuPatchSandboxSnapshot( - "alpha", - { patchedContainerId: "new-container-id" }, - { runCaptureOpenshell, dockerCapture }, - ); - - expect(snapshot.sandboxPhase).toBe("Error"); - expect(snapshot.sandboxListLine).toBe("alpha Error 1m ago"); - expect(snapshot.patchedContainerState?.ExitCode).toBe(125); - expect(snapshot.patchedContainerState?.Error).toContain("could not select device driver"); - }); - - it("classifies a dead patched container as patched_container_failed with the failed mode", () => { - const result = classifyDockerGpuPatchFailure( - { - sandboxPhase: "Error", - sandboxListLine: "alpha Error 1m ago", - patchedContainerState: { - Status: "exited", - ExitCode: 125, - Error: 'could not select device driver "nvidia" with capabilities: [[gpu]]', - }, - }, - buildDockerGpuMode("gpus"), - ); - - expect(result.kind).toBe("patched_container_failed"); - expect(result.headline).toContain("Patched GPU container exited with code 125"); - expect(result.headline).toContain("--gpus all"); - const flat = result.summaryLines.join("\n"); - expect(flat).toContain("sandbox_phase=Error"); - expect(flat).toContain("patched_container_exit_code=125"); - expect(flat).toContain("could not select device driver"); - expect(flat).toContain("patched_create_option=--gpus all"); - }); - - it("classifies an Error-phase sandbox with unknown container state as sandbox_error_phase", () => { - const result = classifyDockerGpuPatchFailure( - { - sandboxPhase: "Error", - sandboxListLine: null, - patchedContainerState: null, - }, - buildDockerGpuMode("gpus"), - ); - - expect(result.kind).toBe("sandbox_error_phase"); - expect(result.headline).toContain("OpenShell sandbox entered Error phase"); - }); - - it("classifies a live container but timed-out supervisor as supervisor_unreachable", () => { - const result = classifyDockerGpuPatchFailure( - { - sandboxPhase: "Provisioning", - sandboxListLine: "alpha Provisioning 30s ago", - patchedContainerState: { Status: "running", Running: true, ExitCode: 0 }, - }, - buildDockerGpuMode("gpus"), - ); - - expect(result.kind).toBe("supervisor_unreachable"); - expect(result.headline).toContain("Provisioning"); - }); - - it("prefers supervisor_unreachable over proof_failure when the sandbox is non-live but non-terminal", () => { - // Regression guard for #4316 review: a proof failing while the sandbox is - // still in a transient/non-live phase (Provisioning, NotReady) is really - // a lifecycle failure — classifying it as proof_failure would tell users - // `nvidia-smi` failed inside an executable sandbox, which masks the real - // cause. - const result = classifyDockerGpuPatchFailure( - { - sandboxPhase: "Provisioning", - sandboxListLine: "alpha Provisioning 30s ago", - patchedContainerState: null, - }, - buildDockerGpuMode("gpus"), - { proofError: new Error("openshell sandbox exec refused: sandbox not ready") }, - ); - - expect(result.kind).toBe("supervisor_unreachable"); - expect(result.headline).toContain("Provisioning"); - expect(result.summaryLines.join("\n")).toContain("proof_error="); - }); - - it("does not blame the supervisor when the patch failed before a container existed", () => { - // Regression guard for #4316 review: an early patch failure (e.g. all GPU - // mode probes were rejected, or detached `docker run` failed) leaves no - // patched container. If the original sandbox happens to still be in a - // transient phase like Provisioning, the classifier must not point at - // an OpenShell supervisor reconnect issue. - const result = classifyDockerGpuPatchFailure( - { - sandboxPhase: "Provisioning", - sandboxListLine: "alpha Provisioning 3s ago", - patchedContainerState: null, - }, - null, - ); - - expect(result.kind).toBe("unknown"); - expect(result.headline).not.toMatch(/supervisor/i); - }); - - it("treats proof failures inside a Ready sandbox as proof_failure, not patched_container_failed", () => { - const result = classifyDockerGpuPatchFailure( - { - sandboxPhase: "Ready", - sandboxListLine: "alpha Ready 30s ago", - patchedContainerState: { Status: "running", Running: true, ExitCode: 0 }, - }, - buildDockerGpuMode("gpus"), - { proofError: new Error("nvidia-smi exited with status 9") }, - ); - - expect(result.kind).toBe("proof_failure"); - expect(result.summaryLines.join("\n")).toContain("proof_error=nvidia-smi exited with status 9"); - }); - - it("preserves the default Docker capture when callers omit dockerCapture from deps", () => { - // Regression guard for #4316 review: passing `dockerCapture: undefined` - // through to `depsWithDefaults` would shadow the module's real Docker - // adapter. The print/diagnostic helpers must NOT forward an explicit - // `undefined` — they should let the default flow through so `docker ps` - // and `docker inspect ` still run. - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-gpu-default-")); - try { - const dockerCapture = vi.fn((_args: readonly string[]) => ""); - const dockerLogs = vi.fn(() => ""); - collectDockerGpuPatchDiagnostics( - "alpha", - { - context: { - sandboxName: "alpha", - newContainerId: "new-container-id", - selectedMode: buildDockerGpuMode("gpus"), - }, - }, - { - // `runCaptureOpenshell` intentionally omitted — exercises the - // "caller has no openshell capture either" path. - dockerCapture, - dockerLogs, - homedir: () => tmpDir, - now: () => new Date("2026-05-12T00:00:00Z"), - }, - ); - - // Without the fix, `depsWithDefaults` would still see `dockerCapture` as - // a function here (the explicit one), so this is more of a structural - // sanity check. The substantive regression is exercised at the print- - // helper level (printDockerGpuPatchFailureAndExit must not pass - // `dockerCapture: undefined`). Here we just confirm collect() invokes - // the supplied dockerCapture for ps/inspect. - expect(dockerCapture.mock.calls.some(([args]) => args?.[0] === "ps")).toBe(true); - } finally { - fs.rmSync(tmpDir, { recursive: true, force: true }); - } - }); - - it("does not inspect the original/backup container when newContainerId is missing", () => { - // Regression guard for #4316 review: when `recreateOpenShellDockerSandboxWithGpu` - // throws before the patched container exists, only `oldContainerId` is set - // in the failure context. The snapshot must NOT inspect the old/backup - // container as if it were the patched one — that would mis-attribute the - // patched container's State. - const dockerCapture = vi.fn((args: readonly string[]) => { - if (args[0] === "inspect" && args[1] === "--format" && args[2] === "{{json .State}}") { - // If this is called for old-container-id, return State that *looks* - // like a failed patch; the test would then incorrectly classify it. - return JSON.stringify({ Status: "exited", ExitCode: 1 }); - } - return ""; - }); - - const snapshot = captureDockerGpuPatchSandboxSnapshot( - "alpha", - { patchedContainerId: null }, - { dockerCapture }, - ); - - expect(snapshot.patchedContainerState).toBeNull(); - // The `--format '{{json .State}}'` invocation should not have happened. - expect( - dockerCapture.mock.calls.some(([args]) => args[0] === "inspect" && args[1] === "--format"), - ).toBe(false); - }); - - it("writes patched-container-state.json and surfaces failure_kind/sandbox_phase in the summary", () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-docker-gpu-4316-")); - try { - const snapshot = { - sandboxPhase: "Error", - sandboxListLine: "alpha Error 1m ago", - patchedContainerState: { - Status: "exited", - ExitCode: 125, - Error: 'could not select device driver "nvidia"', - }, - }; - const classification = classifyDockerGpuPatchFailure(snapshot, buildDockerGpuMode("gpus")); - const diagnostics = collectDockerGpuPatchDiagnostics( - "alpha", - { - context: { - sandboxName: "alpha", - newContainerId: "new-container-id", - selectedMode: buildDockerGpuMode("gpus"), - }, - selectedMode: buildDockerGpuMode("gpus"), - snapshot, - classification, - }, - { - dockerCapture: vi.fn(() => ""), - dockerLogs: vi.fn(() => ""), - homedir: () => tmpDir, - now: () => new Date("2026-05-12T00:00:00Z"), - }, - ); - - expect(diagnostics?.dir).toBeTruthy(); - const summary = fs.readFileSync(path.join(diagnostics?.dir || "", "summary.txt"), "utf-8"); - expect(summary).toContain("failure_kind=patched_container_failed"); - expect(summary).toContain("sandbox_phase=Error"); - expect(summary).toContain("patched_container_exit_code=125"); - const state = fs.readFileSync( - path.join(diagnostics?.dir || "", "patched-container-state.json"), - "utf-8", - ); - expect(state).toContain("could not select device driver"); - } finally { - fs.rmSync(tmpDir, { recursive: true, force: true }); - } - }); -}); diff --git a/src/lib/onboard/docker-gpu-patch.ts b/src/lib/onboard/docker-gpu-patch.ts index 30cf0ae4a3a..ba3b154fd04 100644 --- a/src/lib/onboard/docker-gpu-patch.ts +++ b/src/lib/onboard/docker-gpu-patch.ts @@ -1,25 +1,40 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; +import { dockerCapture } from "../adapters/docker"; +import { createDockerGpuDiagnosticRedactor } from "./docker-gpu-diagnostic-redaction"; +import { DOCKER_GPU_PATCH_TIMEOUT_MS } from "./docker-gpu-patch-constants"; +import type { + DockerContainerState, + DockerGpuPatchBackend, + DockerGpuPatchDeps, + DockerGpuPatchFailureClassification, + DockerGpuPatchFailureContext, + DockerGpuPatchFailureKind, + DockerGpuPatchMode, + DockerGpuPatchResult, + DockerGpuPatchSandboxSnapshot, +} from "./docker-gpu-patch-types"; + +export { detectSandboxFallbackDns } from "./docker-gpu-dns-fallback"; +export { detectTegraDeviceGroupGids } from "./docker-gpu-jetson-groups"; +export { + buildDockerGpuCloneRunArgs, + buildDockerGpuCloneRunOptions, + DOCKER_GPU_PATCH_NETWORK_ENV, + getDockerGpuPatchNetworkMode, + parseDockerInspectJson, +} from "./docker-gpu-patch-clone"; import { - dockerCapture, - dockerLogs, - dockerRename, - dockerRm, - dockerRun, - dockerRunDetached, - dockerStart, - dockerStop, -} from "../adapters/docker"; -import { createDockerGpuDiagnosticRedactor } from "./docker-gpu-diagnostic-redaction"; + collectDockerGpuPatchDiagnostics, + dockerGpuPatchCleanupCommands, +} from "./docker-gpu-patch-diagnostics"; import { - reconcileSupervisorReconnect, - rollbackDockerGpuPatchOnRecreateFailure, -} from "./docker-gpu-patch-finalize"; + getDockerGpuPatchFailureContext, + recreateOpenShellDockerSandboxContainer, + recreateOpenShellDockerSandboxWithGpu, +} from "./docker-gpu-patch-recreate"; import { DOCKER_GPU_SUPERVISOR_RECONNECT_ERROR_DEBOUNCE_ENV, DOCKER_GPU_SUPERVISOR_RECONNECT_TIMEOUT_ENV, @@ -28,7 +43,51 @@ import { getDockerGpuSupervisorReconnectTimeoutSecs, waitForOpenShellSupervisorReconnect, } from "./docker-gpu-supervisor-reconnect"; -import { openshellSandboxCommandEnvValue } from "./docker-startup-command-env"; + +export { + collectDockerGpuPatchDiagnostics, + dockerGpuPatchCleanupCommands, + formatDockerInspectNetworkSummary, +} from "./docker-gpu-patch-diagnostics"; +export { + buildDockerGpuMode, + buildDockerGpuModeCandidates, + DEFAULT_DOCKER_CDI_SPEC_DIRS, + dockerReportsNvidiaCdiDevices, + selectDockerGpuPatchMode, +} from "./docker-gpu-patch-mode"; +export { + getDockerGpuPatchFailureContext, + recreateOpenShellDockerSandboxContainer, + recreateOpenShellDockerSandboxWithGpu, +} from "./docker-gpu-patch-recreate"; +export type { + DockerContainerInspect, + DockerContainerState, + DockerGpuCloneRunOptions, + DockerGpuPatchBackend, + DockerGpuPatchDeps, + DockerGpuPatchDiagnostics, + DockerGpuPatchFailureClassification, + DockerGpuPatchFailureContext, + DockerGpuPatchFailureKind, + DockerGpuPatchMode, + DockerGpuPatchModeAttempt, + DockerGpuPatchModeKind, + DockerGpuPatchResult, + DockerGpuPatchSandboxSnapshot, +} from "./docker-gpu-patch-types"; +export { + findOpenShellDockerSandboxContainerIds, + isImmutableDockerImageId, + OPENSHELL_MANAGED_BY_LABEL, + OPENSHELL_MANAGED_BY_VALUE, + OPENSHELL_SANDBOX_NAME_LABEL, + type OpenShellDockerSandboxContainerQuery, + type OpenShellDockerSandboxRuntimeSnapshotQuery, + queryOpenShellDockerSandboxContainers, + queryOpenShellDockerSandboxRuntimeSnapshot, +} from "./openshell-docker-sandbox-containers"; export type { DockerGpuSupervisorReconnectDeps }; export { @@ -39,1309 +98,6 @@ export { waitForOpenShellSupervisorReconnect, }; -export const OPENSHELL_MANAGED_BY_LABEL = "openshell.ai/managed-by"; -export const OPENSHELL_MANAGED_BY_VALUE = "openshell"; -export const OPENSHELL_SANDBOX_NAME_LABEL = "openshell.ai/sandbox-name"; -const OPENSHELL_SANDBOX_COMMAND_ENV = "OPENSHELL_SANDBOX_COMMAND"; - -const DOCKER_GPU_PATCH_TIMEOUT_MS = 30_000; -const DOCKER_GPU_PATCH_WAIT_SECS = 180; -export const DOCKER_GPU_PATCH_NETWORK_ENV = "NEMOCLAW_DOCKER_GPU_PATCH_NETWORK"; -const MAX_DOCKER_CONTAINER_NAME_LENGTH = 253; -const GPU_ENV_KEYS = new Set([ - "NVIDIA_VISIBLE_DEVICES", - "NVIDIA_DRIVER_CAPABILITIES", - "NVIDIA_REQUIRE_CUDA", - "NVIDIA_DISABLE_REQUIRE", -]); - -type DockerRunResult = { - status?: number | null; - stdout?: string | Buffer | null; - stderr?: string | Buffer | null; - error?: Error | null; -}; - -type DockerRunOptions = Record; -type DockerCaptureFn = (args: readonly string[], opts?: DockerRunOptions) => string; -type DockerRunFn = (args: readonly string[], opts?: DockerRunOptions) => DockerRunResult; -type DockerContainerFn = (containerName: string, opts?: DockerRunOptions) => DockerRunResult; -type DockerRenameFn = ( - oldContainerName: string, - newContainerName: string, - opts?: DockerRunOptions, -) => DockerRunResult; -type DockerLogsFn = (containerName: string, opts?: { tail?: number; timeout?: number }) => string; - -export type DockerGpuPatchDeps = { - dockerCapture?: DockerCaptureFn; - dockerRun?: DockerRunFn; - dockerRunDetached?: DockerRunFn; - dockerRename?: DockerRenameFn; - dockerRm?: DockerContainerFn; - dockerStart?: DockerContainerFn; - dockerStop?: DockerContainerFn; - dockerLogs?: DockerLogsFn; - runOpenshell?: (args: string[], opts?: Record) => DockerRunResult; - runCaptureOpenshell?: (args: string[], opts?: Record) => string; - sleep?: (seconds: number) => void; - homedir?: () => string; - now?: () => Date; - detectSandboxFallbackDns?: () => string | null; - /** - * Resolve the host group ID(s) that own the Jetson/Tegra GPU device nodes - * (`/dev/nvmap`, `/dev/nvhost-*`). Used by the Jetson recreate to grant the - * sandbox user matching `--group-add` membership so CUDA can open them - * (#4231). Injectable so the Jetson permission path is testable without - * Tegra hardware. - */ - detectTegraDeviceGroupGids?: () => string[]; - /** Injectable directory lister for unit testing CDI spec discovery. */ - readDir?: (dirPath: string) => string[] | null; - /** Injectable file reader for unit testing CDI spec content checks. */ - readFile?: (filePath: string) => string | null; - /** - * Forwarded to the supervisor-reconnect wait. See - * `DockerGpuSupervisorReconnectDeps.errorPhaseDebouncePolls`. - */ - errorPhaseDebouncePolls?: number; -}; - -export type DockerGpuPatchModeKind = "gpus" | "nvidia-runtime" | "cdi" | "startup-command"; -export type DockerGpuPatchBackend = "generic" | "jetson"; - -export type DockerGpuPatchMode = { - kind: DockerGpuPatchModeKind; - label: string; - device: string; - args: string[]; -}; - -export type DockerGpuPatchModeAttempt = { - mode: DockerGpuPatchMode; - ok: boolean; - error: string | null; -}; - -export type DockerGpuPatchFailureContext = { - sandboxName: string; - oldContainerId?: string | null; - newContainerId?: string | null; - backupContainerName?: string | null; - selectedMode?: DockerGpuPatchMode | null; - modeAttempts?: DockerGpuPatchModeAttempt[]; - rolledBack?: boolean; -}; - -export type DockerGpuPatchResult = { - applied: true; - oldContainerId: string; - newContainerId: string; - originalName: string; - backupContainerName: string; - mode: DockerGpuPatchMode; - // True when the patch path also confirmed supervisor reconnect AND removed - // the backup container. False when the caller deferred the reconnect wait - // (via `waitForSupervisor: false`); the backup is still in place and the - // caller is responsible for calling `finalizeDockerGpuPatchBackup` after - // its own supervisor wait completes. - backupRemoved: boolean; -}; - -export type DockerGpuCloneRunOptions = { - image?: string | null; - networkMode?: string | null; - openshellEndpoint?: string | null; - sandboxFallbackDns?: string | null; - openshellSandboxCommand?: readonly string[] | null; - /** - * Extra supplementary group IDs to add to the recreated container via - * `--group-add`. On Jetson these are the host group(s) owning the Tegra GPU - * device nodes (`/dev/nvmap`, `/dev/nvhost-*`); granting the sandbox user - * membership lets CUDA's nvmap init open them instead of failing with - * `NvRmMemInitNvmap ... Permission denied` (#4231). - */ - extraGroupGids?: readonly string[] | null; -}; - -export type DockerGpuPatchDiagnostics = { - dir: string; - cleanupCommands: string[]; - summaryLines: string[]; -}; - -/** - * Subset of `docker inspect --format '{{json .State}}'` fields surfaced when - * the patched GPU sandbox container fails to become executable. We capture - * just the runtime/exit/health state — not the full inspect — because that - * is what tells the user *why* the patched create option broke (e.g. a - * non-zero ExitCode with `Error: "could not select device driver"`). - */ -export type DockerContainerState = { - Status?: string; - Running?: boolean; - Paused?: boolean; - Restarting?: boolean; - OOMKilled?: boolean; - Dead?: boolean; - ExitCode?: number; - Error?: string; - StartedAt?: string; - FinishedAt?: string; - Health?: { Status?: string; FailingStreak?: number } | null; -}; - -/** - * Snapshot of "is the patched sandbox even runnable?" — sandbox phase from - * OpenShell plus the patched Docker container's State. This is the data the - * caller needs to tell the user whether the failure is at the OpenShell - * sandbox layer (Error phase) vs. the Docker container layer (non-zero exit - * with a driver/runtime error) — see #4316. - */ -export type DockerGpuPatchSandboxSnapshot = { - sandboxPhase: string | null; - sandboxListLine: string | null; - patchedContainerState: DockerContainerState | null; -}; - -export type DockerGpuPatchFailureKind = - | "patched_container_failed" - | "sandbox_error_phase" - | "supervisor_unreachable" - | "proof_failure" - | "unknown"; - -export type DockerGpuPatchFailureClassification = { - kind: DockerGpuPatchFailureKind; - headline: string; - summaryLines: string[]; -}; - -export type DockerContainerInspect = { - Id?: string; - Image?: string; - Name?: string; - Config?: { - Image?: string; - Env?: string[] | null; - Labels?: Record | null; - Entrypoint?: string[] | string | null; - Cmd?: string[] | string | null; - User?: string; - WorkingDir?: string; - Hostname?: string; - Tty?: boolean; - OpenStdin?: boolean; - } | null; - HostConfig?: { - Binds?: string[] | null; - NetworkMode?: string; - RestartPolicy?: { Name?: string; MaximumRetryCount?: number } | null; - CapAdd?: string[] | null; - CapDrop?: string[] | null; - SecurityOpt?: string[] | null; - ExtraHosts?: string[] | null; - Memory?: number; - MemoryReservation?: number; - MemorySwap?: number; - NanoCpus?: number; - CpuShares?: number; - CpuQuota?: number; - CpuPeriod?: number; - CpusetCpus?: string; - CpusetMems?: string; - Privileged?: boolean; - Init?: boolean; - IpcMode?: string; - PidMode?: string; - GroupAdd?: string[] | null; - Dns?: string[] | null; - DnsSearch?: string[] | null; - DeviceRequests?: Array<{ - Driver?: string; - DeviceIDs?: string[] | null; - }> | null; - ShmSize?: number; - ReadonlyPaths?: string[] | null; - MaskedPaths?: string[] | null; - } | null; - NetworkSettings?: { - Networks?: Record< - string, - { - IPAddress?: string; - Gateway?: string; - Aliases?: string[] | null; - } - > | null; - } | null; -}; - -function depsWithDefaults( - deps: DockerGpuPatchDeps, -): Required< - Pick< - DockerGpuPatchDeps, - | "dockerCapture" - | "dockerRun" - | "dockerRunDetached" - | "dockerRename" - | "dockerRm" - | "dockerStart" - | "dockerStop" - | "dockerLogs" - | "sleep" - | "homedir" - | "now" - | "detectSandboxFallbackDns" - | "detectTegraDeviceGroupGids" - > -> & - DockerGpuPatchDeps { - return { - dockerCapture, - dockerRun, - dockerRunDetached, - dockerRename, - dockerRm, - dockerStart, - dockerStop, - dockerLogs, - sleep: (seconds: number) => { - Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, Math.max(0, seconds) * 1000); - }, - homedir: os.homedir, - now: () => new Date(), - detectSandboxFallbackDns: () => detectSandboxFallbackDns(), - detectTegraDeviceGroupGids: () => detectTegraDeviceGroupGids(), - ...deps, - }; -} - -// Jetson/Tegra device nodes that CUDA opens during driver initialization. -// `/dev/nvmap` is the memory manager whose `NvRmMemInitNvmap` failure the -// reporter hit (#4231); the `nvhost-*`/`nvgpu` nodes are the compute/control -// channels. On L4T these are owned by a non-root group (typically `video`, -// mode `crw-rw----`). -const TEGRA_GPU_DEVICE_NODES = [ - "/dev/nvmap", - "/dev/nvhost-ctrl", - "/dev/nvhost-ctrl-gpu", - "/dev/nvhost-gpu", - "/dev/nvhost-as-gpu", - "/dev/nvhost-prof-gpu", - "/dev/nvhost-dbg-gpu", - "/dev/nvhost-tsg-gpu", - "/dev/nvgpu/igpu0/ctrl", - "/dev/nvgpu/igpu0/as", - "/dev/nvgpu/igpu0/prof", -] as const; - -/** - * Resolve the host group ID(s) that own the Jetson/Tegra GPU device nodes. - * - * The NVIDIA Container Runtime bind-mounts these nodes into the sandbox - * preserving the host's numeric owner/group, but the OpenShell sandbox runs - * the agent as an unprivileged user that is not a member of that group — so - * CUDA's nvmap init fails with `Permission denied` and `cuInit(0)` returns 999 - * even though the devices are present (#4231). Returning the owning GID(s) - * lets the recreate grant the sandbox user matching `--group-add` membership. - * - * Numeric GIDs (not group names) are returned on purpose: the sandbox image's - * group database need not define a `video`/`render` group at the host's GID, - * and `docker run --group-add ` adds the supplementary group by ID - * regardless of whether a matching name exists inside the container. - */ -export function detectTegraDeviceGroupGids( - deps: { statDeviceGid?: (path: string) => number | null } = {}, -): string[] { - const statGid = - deps.statDeviceGid ?? - ((p: string): number | null => { - try { - return fs.statSync(p).gid; - } catch { - return null; - } - }); - const gids = new Set(); - for (const node of TEGRA_GPU_DEVICE_NODES) { - const gid = statGid(node); - // Skip missing nodes and root-owned (gid 0) nodes: `--group-add 0` would - // not help an unprivileged user, and root already has access regardless. - if (gid !== null && gid > 0) gids.add(String(gid)); - } - return [...gids].sort((a, b) => Number(a) - Number(b)); -} - -function resultText(result: DockerRunResult | null | undefined): string { - if (!result) return ""; - return `${String(result.stderr || "")} ${String(result.stdout || "")} ${String( - result.error?.message || "", - )}`.trim(); -} - -function isZeroStatus(result: DockerRunResult | null | undefined): boolean { - return result?.status === 0; -} - -function sanitizePathPart(value: string): string { - return value.replace(/[^A-Za-z0-9._-]/g, "_").slice(0, 80) || "sandbox"; -} - -function timestampForPath(now: Date): string { - return now.toISOString().replace(/[:.]/g, "-"); -} - -function dockerContainerName(inspect: DockerContainerInspect): string { - const raw = String(inspect.Name || "") - .replace(/^\/+/, "") - .trim(); - if (!raw) throw new Error("Docker inspect output did not include a container name."); - return raw; -} - -function stringArray(value: string[] | string | null | undefined): string[] { - if (Array.isArray(value)) return value.map((entry) => String(entry)); - if (typeof value === "string" && value.length > 0) return [value]; - return []; -} - -function envKey(env: string): string { - const idx = env.indexOf("="); - return idx === -1 ? env : env.slice(0, idx); -} - -function envValue(env: string[] | null | undefined, key: string): string | null { - const prefix = `${key}=`; - const entry = stringArray(env).find((value) => value.startsWith(prefix)); - return entry ? entry.slice(prefix.length) : null; -} - -function replaceEnvValue(entry: string, key: string, value: string | null | undefined): string { - if (!value || envKey(entry) !== key) return entry; - return `${key}=${value}`; -} - -function dockerGpuHostEndpointFromOpenShellEndpoint(endpoint: string): string | null { - try { - const url = new URL(endpoint); - if (url.hostname !== "host.openshell.internal") return null; - url.hostname = "127.0.0.1"; - return url.toString(); - } catch { - return null; - } -} - -function pushStringFlag(args: string[], flag: string, value: unknown): void { - const normalized = String(value ?? "").trim(); - if (normalized) args.push(flag, normalized); -} - -function pushNumberFlag(args: string[], flag: string, value: unknown): void { - if (typeof value === "number" && Number.isFinite(value) && value > 0) { - args.push(flag, String(value)); - } -} - -function dockerCpusFromNanoCpus(nanoCpus: number): string { - return (nanoCpus / 1_000_000_000).toFixed(3).replace(/\.?0+$/, ""); -} - -function normalizeGpuDeviceForDocker(device: string | null | undefined): string { - const raw = String(device || "").trim(); - if (!raw || raw === "nvidia.com/gpu=all") return "all"; - if (raw.startsWith("nvidia.com/gpu=")) return raw.slice("nvidia.com/gpu=".length) || "all"; - return raw; -} - -function normalizeGpuDeviceForCdi(device: string | null | undefined): string { - const dockerDevice = normalizeGpuDeviceForDocker(device); - if ( - String(device || "") - .trim() - .startsWith("nvidia.com/gpu=") - ) { - return String(device).trim(); - } - return `nvidia.com/gpu=${dockerDevice || "all"}`; -} - -export function buildDockerGpuMode( - kind: DockerGpuPatchModeKind, - device?: string | null, - options: { backend?: DockerGpuPatchBackend } = {}, -): DockerGpuPatchMode { - if (kind === "startup-command") { - return { - kind, - label: "persistent sandbox startup command", - device: "", - args: [], - }; - } - const dockerDevice = normalizeGpuDeviceForDocker(device); - if (kind === "gpus") { - const gpuValue = dockerDevice === "all" ? "all" : `device=${dockerDevice}`; - return { - kind, - label: `--gpus ${gpuValue}`, - device: dockerDevice, - args: ["--gpus", gpuValue], - }; - } - if (kind === "nvidia-runtime") { - const args = ["--runtime", "nvidia", "--env", `NVIDIA_VISIBLE_DEVICES=${dockerDevice}`]; - if (options.backend === "jetson") { - args.push("--env", "NVIDIA_DRIVER_CAPABILITIES=compute,utility"); - } - return { - kind, - label: `--runtime nvidia (NVIDIA_VISIBLE_DEVICES=${dockerDevice})`, - device: dockerDevice, - args, - }; - } - const cdiDevice = normalizeGpuDeviceForCdi(device); - return { - kind, - label: `--device ${cdiDevice}`, - device: cdiDevice, - args: ["--device", cdiDevice], - }; -} - -export function buildDockerGpuModeCandidates( - device?: string | null, - options: { - cdiAvailable?: boolean; - backend?: DockerGpuPatchBackend; - dockerDesktopWsl?: boolean; - } = {}, -): DockerGpuPatchMode[] { - if (options.backend === "jetson") { - return [buildDockerGpuMode("nvidia-runtime", device, { backend: "jetson" })]; - } - // When the host advertises an NVIDIA CDI spec, prefer the CDI mode - // (`--device nvidia.com/gpu=all`) ahead of --gpus. OpenShell's gateway owns - // supervisor GPU injection and wires Docker-CDI hosts from that spec; this - // NemoClaw patch only chooses the recreate mode while matching that source - // boundary. On Docker-CDI hosts `docker create --gpus all` is accepted (the - // create-only probe passes), but the legacy --gpus injection diverges from - // gateway wiring and the supervisor never reconnects (#4948). Keep --gpus - // and the NVIDIA runtime as fallbacks until OpenShell exposes an - // authoritative GPU mode contract that can replace CDI-spec probing. - // - // #5512: Docker Desktop WSL advertises CDI spec directories (so cdiAvailable - // is true) while the WSL distro has no usable nvidia.com/gpu spec. There the - // CDI mode passes the create-only probe but fails the real recreate with - // "unresolvable CDI devices nvidia.com/gpu=all", so skip CDI and use the - // --gpus compatibility path that preflight already commits to on this runtime. - const candidates: DockerGpuPatchMode[] = []; - if (options.cdiAvailable && !options.dockerDesktopWsl) { - candidates.push(buildDockerGpuMode("cdi", device)); - } - candidates.push(buildDockerGpuMode("gpus", device), buildDockerGpuMode("nvidia-runtime", device)); - return candidates; -} - -export function shouldApplyDockerGpuPatch( - config: { sandboxGpuEnabled: boolean; hostGpuPlatform?: string | null }, - options: { - env?: NodeJS.ProcessEnv; - platform?: NodeJS.Platform; - dockerDriverGateway?: boolean; - dockerDesktopWsl?: boolean; - log?: (message: string) => void; - } = {}, -): boolean { - const env = options.env ?? process.env; - const platform = options.platform ?? process.platform; - const dockerDesktopWsl = options.dockerDesktopWsl === true; - const dockerDriverGateway = - options.dockerDriverGateway ?? (platform === "linux" || dockerDesktopWsl); - if ( - !(config.sandboxGpuEnabled && (platform === "linux" || dockerDesktopWsl) && dockerDriverGateway) - ) { - return false; - } - const control = String(env.NEMOCLAW_DOCKER_GPU_PATCH || "") - .trim() - .toLowerCase(); - const optedOut = control === "0"; - if (optedOut && dockerDesktopWsl) { - const log = options.log ?? ((message: string) => console.warn(message)); - log( - " NEMOCLAW_DOCKER_GPU_PATCH=0 ignored on Docker Desktop WSL: GPU passthrough on this runtime requires the patch.", - ); - log(" Skip GPU passthrough entirely with --no-gpu or NEMOCLAW_SANDBOX_GPU=0."); - return true; - } - if (dockerDesktopWsl) return true; - if (config.hostGpuPlatform === "jetson") return !optedOut; - // OpenShell 0.0.71 natively injects CDI devices for ordinary native Linux. - // Keep the container-swap path as an explicit compatibility control while - // WSL and Jetson retain the legacy defaults they still require. - return control === "1"; -} - -export function buildDockerGpuCloneRunOptions( - inspect: DockerContainerInspect, - env: Record = process.env, -): DockerGpuCloneRunOptions { - if (getDockerGpuPatchNetworkMode(env) !== "host") return {}; - - const endpoint = envValue(inspect.Config?.Env, "OPENSHELL_ENDPOINT"); - const hostEndpoint = endpoint ? dockerGpuHostEndpointFromOpenShellEndpoint(endpoint) : null; - if (!hostEndpoint) return {}; - return { networkMode: "host", openshellEndpoint: hostEndpoint }; -} - -function parseResolvConfNameservers(content: string): string[] { - return content - .split("\n") - .map((line) => line.trim()) - .filter((line) => line.startsWith("nameserver")) - .map((line) => line.split(/\s+/)[1]) - .filter((ip): ip is string => Boolean(ip)); -} - -// #3579: when the host's /etc/resolv.conf points only at 127.0.0.x (e.g. -// 127.0.0.53 from systemd-resolved), a sandbox in its own network namespace -// can't reach that resolver — systemd-resolved listens in the host namespace -// only. Return the first non-loopback nameserver from -// /run/systemd/resolve/resolv.conf so the caller can inject it via --dns -// rather than relying on inherited /etc/resolv.conf. -export function detectSandboxFallbackDns( - deps: { readFile?: (path: string) => string | null } = {}, -): string | null { - const readFile = - deps.readFile ?? - ((p: string): string | null => { - try { - return fs.readFileSync(p, "utf-8"); - } catch { - return null; - } - }); - const resolvConf = readFile("/etc/resolv.conf"); - if (!resolvConf) return null; - const nameservers = parseResolvConfNameservers(resolvConf); - if (nameservers.length === 0) return null; - if (!nameservers.every((ip) => /^127\./.test(ip))) return null; - const upstreamFile = readFile("/run/systemd/resolve/resolv.conf"); - if (!upstreamFile) return null; - return parseResolvConfNameservers(upstreamFile).find((ip) => !/^127\./.test(ip)) ?? null; -} - -export function getDockerGpuPatchNetworkMode( - env: Record = process.env, -): "host" | "preserve" { - const networkOverride = String(env[DOCKER_GPU_PATCH_NETWORK_ENV] || "") - .trim() - .toLowerCase(); - if (networkOverride === "host") return "host"; - if (networkOverride === "preserve" || networkOverride === "bridge") return "preserve"; - return "preserve"; -} - -function dockerNetworkAliases( - inspect: DockerContainerInspect, - networkMode: string | null | undefined, -): string[] { - const network = String(networkMode || "").trim(); - if ( - !network || - ["bridge", "default", "host", "none"].includes(network) || - network.includes(":") - ) { - return []; - } - - const networkInfo = inspect.NetworkSettings?.Networks?.[network]; - const containerId = String(inspect.Id || "").trim(); - return Array.from(new Set(stringArray(networkInfo?.Aliases))) - .map((alias) => alias.trim()) - .filter(Boolean) - .filter((alias) => !sameContainerId(alias, containerId)); -} - -export function buildDockerGpuCloneRunArgs( - inspect: DockerContainerInspect, - mode: DockerGpuPatchMode, - options: DockerGpuCloneRunOptions = {}, -): string[] { - const config = inspect.Config || {}; - const host = inspect.HostConfig || {}; - const image = String(options.image || config.Image || "").trim(); - if (!image) throw new Error("Docker inspect output did not include Config.Image."); - - const args: string[] = ["--name", dockerContainerName(inspect), ...mode.args]; - const gpuAugment = mode.kind !== "startup-command"; - - // Hermes restart persistence recreates the OpenShell-managed container - // without selecting NemoClaw's compatibility GPU mode. Preserve Docker's - // native CDI requests from the inspected container so that recreation does - // not silently drop OpenShell's GPU attachment (and the injected libcuda). - if (!gpuAugment) { - const cdiDeviceIds = new Set( - (host.DeviceRequests ?? []) - .filter((request) => request.Driver === "cdi") - .flatMap((request) => stringArray(request.DeviceIDs)) - .map((deviceId) => deviceId.trim()) - .filter(Boolean), - ); - for (const deviceId of cdiDeviceIds) args.push("--device", deviceId); - } - - pushStringFlag(args, "--hostname", config.Hostname); - pushStringFlag(args, "--user", config.User); - pushStringFlag(args, "--workdir", config.WorkingDir); - if (config.Tty) args.push("--tty"); - if (config.OpenStdin) args.push("--interactive"); - - const openshellSandboxCommandEnv = openshellSandboxCommandEnvValue( - options.openshellSandboxCommand, - ); - let sawOpenShellSandboxCommandEnv = false; - for (const env of stringArray(config.Env).filter( - (entry) => !gpuAugment || !GPU_ENV_KEYS.has(envKey(entry)), - )) { - const key = envKey(env); - if (key === OPENSHELL_SANDBOX_COMMAND_ENV && openshellSandboxCommandEnv) { - sawOpenShellSandboxCommandEnv = true; - args.push("--env", `${OPENSHELL_SANDBOX_COMMAND_ENV}=${openshellSandboxCommandEnv}`); - continue; - } - args.push("--env", replaceEnvValue(env, "OPENSHELL_ENDPOINT", options.openshellEndpoint)); - } - if (openshellSandboxCommandEnv && !sawOpenShellSandboxCommandEnv) { - args.push("--env", `${OPENSHELL_SANDBOX_COMMAND_ENV}=${openshellSandboxCommandEnv}`); - } - - const labels = config.Labels || {}; - for (const key of Object.keys(labels).sort()) { - const value = labels[key]; - if (value !== undefined && value !== null) args.push("--label", `${key}=${value}`); - } - - for (const bind of stringArray(host.Binds)) args.push("--volume", bind); - const networkMode = options.networkMode ?? host.NetworkMode; - pushStringFlag(args, "--network", networkMode); - for (const alias of dockerNetworkAliases(inspect, networkMode)) { - args.push("--network-alias", alias); - } - - const restart = host.RestartPolicy; - if (restart?.Name && restart.Name !== "no") { - const value = - restart.Name === "on-failure" && restart.MaximumRetryCount - ? `${restart.Name}:${restart.MaximumRetryCount}` - : restart.Name; - args.push("--restart", value); - } - - // GPU bring-up requires writing to /proc//task//comm (see - // PROC_COMM_WRITE_PROBE in initial-policy.ts). On some Docker/distro - // baselines, the OpenShell-created container that we inspect here lacks - // SYS_PTRACE and/or apparmor=unconfined, which the kernel/LSM combination - // requires for that write. Augment the recreate flags to make the - // GPU-capable container self-sufficient for the operations the GPU proof - // checks, regardless of what the non-GPU baseline happened to set (#3511). - const capAdd = new Set(stringArray(host.CapAdd)); - if (gpuAugment) capAdd.add("SYS_PTRACE"); - for (const cap of capAdd) args.push("--cap-add", cap); - for (const cap of stringArray(host.CapDrop)) args.push("--cap-drop", cap); - const securityOpt = new Set(stringArray(host.SecurityOpt)); - // Only inject apparmor=unconfined when the baseline did not pin a specific - // apparmor profile. Docker rejects multiple `--security-opt apparmor=...` - // entries, and a baseline that explicitly chose `apparmor=docker-default` - // (or similar) should be respected — we are scoped to the GPU recreate - // path, not to overriding deliberate operator choices. - if (gpuAugment && ![...securityOpt].some((entry) => entry.startsWith("apparmor"))) { - securityOpt.add("apparmor=unconfined"); - } - for (const opt of securityOpt) args.push("--security-opt", opt); - // --add-host writes to the container's /etc/hosts (mount namespace), not - // the network stack, so OpenShell's host.openshell.internal mapping must - // survive even when the caller explicitly opts into --network=host via - // NEMOCLAW_DOCKER_GPU_PATCH_NETWORK=host (#3562, #3568). - for (const hostEntry of stringArray(host.ExtraHosts)) args.push("--add-host", hostEntry); - const groupAdds = new Set(stringArray(host.GroupAdd)); - for (const group of groupAdds) args.push("--group-add", group); - // Jetson/Tegra: grant the sandbox user membership in the host group(s) that - // own /dev/nvmap and the nvhost device nodes so CUDA's nvmap init can open - // them. Without this the unprivileged agent user hits EACCES on /dev/nvmap - // and cuInit(0) returns 999 even though the GPU devices are mounted (#4231). - // Dedupe against any GroupAdd the baseline container already carried. - for (const gid of options.extraGroupGids ?? []) { - const normalized = String(gid).trim(); - if (normalized && !groupAdds.has(normalized)) { - groupAdds.add(normalized); - args.push("--group-add", normalized); - } - } - if (networkMode !== "host") { - const dnsServers = stringArray(host.Dns); - for (const dns of dnsServers) args.push("--dns", dns); - for (const dnsSearch of stringArray(host.DnsSearch)) args.push("--dns-search", dnsSearch); - // #3579: when the host has only a loopback resolver (systemd-resolved), - // inject the real upstream so the sandbox doesn't inherit an unreachable - // 127.0.0.53. Only kicks in if OpenShell didn't already set --dns. - if (dnsServers.length === 0 && options.sandboxFallbackDns) { - args.push("--dns", options.sandboxFallbackDns); - } - } - - pushNumberFlag(args, "--memory", host.Memory); - pushNumberFlag(args, "--memory-reservation", host.MemoryReservation); - pushNumberFlag(args, "--memory-swap", host.MemorySwap); - pushNumberFlag(args, "--cpu-shares", host.CpuShares); - pushNumberFlag(args, "--cpu-quota", host.CpuQuota); - pushNumberFlag(args, "--cpu-period", host.CpuPeriod); - pushNumberFlag(args, "--shm-size", host.ShmSize); - if (typeof host.NanoCpus === "number" && host.NanoCpus > 0) { - args.push("--cpus", dockerCpusFromNanoCpus(host.NanoCpus)); - } - pushStringFlag(args, "--cpuset-cpus", host.CpusetCpus); - pushStringFlag(args, "--cpuset-mems", host.CpusetMems); - pushStringFlag(args, "--ipc", host.IpcMode); - pushStringFlag(args, "--pid", host.PidMode); - if (host.Privileged) args.push("--privileged"); - if (host.Init) args.push("--init"); - - const entrypoint = stringArray(config.Entrypoint); - if (entrypoint.length > 0) args.push("--entrypoint", entrypoint[0]); - // OpenShell 0.0.71's Docker driver deliberately clears Config.Cmd and passes - // the workload only through OPENSHELL_SANDBOX_COMMAND. Keep that contract - // when recreating the container: appending the same workload after the image - // puts `nemoclaw-start` in the supervisor's own argv. The serializer above - // rejects whitespace-bearing tokens because OpenShell uses split_whitespace() - // when reading this environment value (#6110). Remove this compatibility - // rewrite only after the Docker container-swap GPU patch itself is retired. - const commandArgs = openshellSandboxCommandEnv - ? [] - : [...entrypoint.slice(1), ...stringArray(config.Cmd)]; - args.push(image, ...commandArgs); - return args; -} - -export function parseDockerInspectJson(output: string): DockerContainerInspect { - const parsed = JSON.parse(output); - const inspect = Array.isArray(parsed) ? parsed[0] : parsed; - if (!inspect || typeof inspect !== "object") { - throw new Error("Docker inspect did not return a container object."); - } - return inspect as DockerContainerInspect; -} - -export function findOpenShellDockerSandboxContainerIds( - sandboxName: string, - deps: DockerGpuPatchDeps = {}, -): string[] { - const d = depsWithDefaults(deps); - const output = d.dockerCapture( - [ - "ps", - "-a", - "--no-trunc", - "--filter", - `label=${OPENSHELL_MANAGED_BY_LABEL}=${OPENSHELL_MANAGED_BY_VALUE}`, - "--filter", - `label=${OPENSHELL_SANDBOX_NAME_LABEL}=${sandboxName}`, - "--format", - "{{.ID}}", - ], - { ignoreError: true, timeout: DOCKER_GPU_PATCH_TIMEOUT_MS }, - ); - return output - .split(/\r?\n/) - .map((line) => line.trim()) - .filter(Boolean); -} - -function inspectDockerContainer( - containerId: string, - deps: DockerGpuPatchDeps, -): DockerContainerInspect { - const d = depsWithDefaults(deps); - const output = d.dockerCapture(["inspect", "--type", "container", containerId], { - ignoreError: true, - timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, - }); - return parseDockerInspectJson(output); -} - -function sameContainerId( - left: string | null | undefined, - right: string | null | undefined, -): boolean { - if (!left || !right) return false; - return left.startsWith(right) || right.startsWith(left); -} - -function parseDockerCdiSpecDirs(value: string | null | undefined): string[] { - const raw = String(value || "").trim(); - if (!raw || raw === "") return []; - try { - const parsed = JSON.parse(raw); - return Array.isArray(parsed) - ? parsed.map((entry) => String(entry || "").trim()).filter(Boolean) - : []; - } catch { - return raw - .split(/[\s,]+/) - .map((entry) => entry.trim()) - .filter(Boolean); - } -} - -/** - * Docker's well-known default CDI spec directories. Docker reads CDI specs - * from these paths even when `docker info` reports an empty `CDISpecDirs` - * (for example, on Docker 29 hosts with `nvidia-container-toolkit` installed - * but no `/etc/docker/daemon.json`). Scanning them lets us detect that the - * `cdi` GPU mode is viable when the docker-info detection alone would miss - * it (NemoClaw issue #3575). - */ -export const DEFAULT_DOCKER_CDI_SPEC_DIRS = ["/etc/cdi", "/var/run/cdi"] as const; - -function readCdiSpecContent( - filePath: string, - readFile?: (p: string) => string | null, -): string | null { - if (readFile) return readFile(filePath); - try { - return fs.readFileSync(filePath, "utf-8"); - } catch { - return null; - } -} - -function isLikelyNvidiaCdiSpecFile( - filePath: string, - readFile?: (p: string) => string | null, -): boolean { - if (!/\.(json|ya?ml)$/i.test(filePath)) return false; - const content = readCdiSpecContent(filePath, readFile); - if (content === null) return false; - return /nvidia\.com\/gpu|nvidia-container|libcuda|cuda/i.test(content); -} - -function listDirEntries( - dirPath: string, - readDir?: (p: string) => string[] | null, -): string[] | null { - if (readDir) return readDir(dirPath); - try { - return fs.readdirSync(dirPath); - } catch { - return null; - } -} - -/** - * Returns the set of directories to scan for CDI specs: those reported by - * `docker info` (if any), plus Docker's well-known defaults. Deduplicated - * so a host that surfaces `/etc/cdi` explicitly is not scanned twice. - */ -function resolveCdiScanDirs(reportedDirs: readonly string[]): string[] { - const seen = new Set(); - const ordered: string[] = []; - for (const dir of [...reportedDirs, ...DEFAULT_DOCKER_CDI_SPEC_DIRS]) { - const trimmed = dir.trim(); - if (!trimmed || seen.has(trimmed)) continue; - seen.add(trimmed); - ordered.push(trimmed); - } - return ordered; -} - -export function dockerReportsNvidiaCdiDevices(deps: DockerGpuPatchDeps = {}): boolean { - const d = depsWithDefaults(deps); - let raw = ""; - try { - raw = d.dockerCapture(["info", "--format", "{{json .CDISpecDirs}}"], { - ignoreError: true, - timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, - }); - } catch { - // `docker info` failed, but the default CDI dirs may still hold a valid - // spec (e.g. issue #3575). Continue with the defaults below. - } - const reported = parseDockerCdiSpecDirs(raw); - for (const dir of resolveCdiScanDirs(reported)) { - const entries = listDirEntries(dir, deps.readDir); - if (!entries) continue; - if (entries.some((entry) => isLikelyNvidiaCdiSpecFile(path.join(dir, entry), deps.readFile))) { - return true; - } - } - return false; -} - -function probeDockerGpuMode( - mode: DockerGpuPatchMode, - image: string, - deps: DockerGpuPatchDeps, -): { ok: boolean; error: string | null } { - const d = depsWithDefaults(deps); - const probeName = `nemoclaw-gpu-probe-${process.pid}-${Date.now()}-${Math.random() - .toString(16) - .slice(2, 8)}`; - try { - const result = d.dockerRun(["create", "--name", probeName, ...mode.args, image, "true"], { - ignoreError: true, - suppressOutput: true, - timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, - }); - return { - ok: isZeroStatus(result), - error: isZeroStatus(result) ? null : resultText(result) || `docker create failed`, - }; - } catch (error) { - return { ok: false, error: error instanceof Error ? error.message : String(error) }; - } finally { - d.dockerRm(probeName, { - ignoreError: true, - suppressOutput: true, - timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, - }); - } -} - -export function selectDockerGpuPatchMode( - options: { - image: string; - device?: string | null; - backend?: DockerGpuPatchBackend; - dockerDesktopWsl?: boolean; - }, - deps: DockerGpuPatchDeps = {}, -): { mode: DockerGpuPatchMode | null; attempts: DockerGpuPatchModeAttempt[] } { - const cdiAvailable = options.backend === "jetson" ? false : dockerReportsNvidiaCdiDevices(deps); - const attempts: DockerGpuPatchModeAttempt[] = []; - for (const mode of buildDockerGpuModeCandidates(options.device, { - cdiAvailable, - backend: options.backend, - dockerDesktopWsl: options.dockerDesktopWsl, - })) { - const result = probeDockerGpuMode(mode, options.image, deps); - const attempt = { mode, ok: result.ok, error: result.error }; - attempts.push(attempt); - if (attempt.ok) return { mode, attempts }; - } - return { mode: null, attempts }; -} - -function buildBackupContainerName(originalName: string, now: Date): string { - const suffix = `-nemoclaw-gpu-backup-${String(now.getTime())}`; - const maxOriginalLength = MAX_DOCKER_CONTAINER_NAME_LENGTH - suffix.length; - return `${originalName.slice(0, Math.max(1, maxOriginalLength))}${suffix}`; -} - -function waitForNewContainerId( - sandboxName: string, - oldContainerId: string, - timeoutSecs: number, - deps: DockerGpuPatchDeps, -): string | null { - const d = depsWithDefaults(deps); - const deadline = Date.now() + Math.max(1, timeoutSecs) * 1000; - while (Date.now() <= deadline) { - const ids = findOpenShellDockerSandboxContainerIds(sandboxName, deps); - const replacement = ids.find((id) => !sameContainerId(id, oldContainerId)); - if (replacement) return replacement; - d.sleep(2); - } - return null; -} - -function decoratePatchError( - error: T, - context: DockerGpuPatchFailureContext, -): T & { dockerGpuPatch?: DockerGpuPatchFailureContext } { - (error as T & { dockerGpuPatch?: DockerGpuPatchFailureContext }).dockerGpuPatch = context; - return error as T & { dockerGpuPatch?: DockerGpuPatchFailureContext }; -} - -export function getDockerGpuPatchFailureContext( - error: unknown, -): DockerGpuPatchFailureContext | null { - if (error && typeof error === "object" && "dockerGpuPatch" in error) { - return (error as { dockerGpuPatch?: DockerGpuPatchFailureContext }).dockerGpuPatch || null; - } - return null; -} - -export function recreateOpenShellDockerSandboxContainer( - options: { - sandboxName: string; - gpuDevice?: string | null; - timeoutSecs?: number; - waitForSupervisor?: boolean; - openshellSandboxCommand?: readonly string[] | null; - expectedOldContainerId?: string | null; - backend?: DockerGpuPatchBackend; - dockerDesktopWsl?: boolean; - modeOverride?: DockerGpuPatchMode; - }, - deps: DockerGpuPatchDeps = {}, -): DockerGpuPatchResult { - const d = depsWithDefaults(deps); - const context: DockerGpuPatchFailureContext = { - sandboxName: options.sandboxName, - modeAttempts: [], - }; - try { - const containerIds = findOpenShellDockerSandboxContainerIds(options.sandboxName, deps); - const oldContainerId = containerIds[0]; - if (!oldContainerId) { - throw new Error( - `Could not find OpenShell Docker container for sandbox '${options.sandboxName}'.`, - ); - } - if ( - options.expectedOldContainerId != null && - (containerIds.length !== 1 || oldContainerId !== options.expectedOldContainerId) - ) { - throw new Error( - `OpenShell Docker container identity changed for sandbox '${options.sandboxName}'; ` + - "refusing startup-command recreation because the observed container differs from the pinned identity.", - ); - } - if (options.openshellSandboxCommand != null) { - // Validate the persisted command before image selection so malformed - // tokens remain the first fail-closed result and no container mutation - // can begin regardless of inspect metadata quality. - openshellSandboxCommandEnvValue(options.openshellSandboxCommand); - } - context.oldContainerId = oldContainerId; - - const inspect = inspectDockerContainer(oldContainerId, deps); - const configuredImage = String(inspect.Config?.Image || "").trim(); - if (!configuredImage) { - throw new Error("OpenShell sandbox container inspect did not include an image."); - } - const immutableImage = String(inspect.Image || "").trim(); - const requiresImmutableImage = options.openshellSandboxCommand != null; - if (requiresImmutableImage && !/^sha256:[0-9a-f]{64}$/i.test(immutableImage)) { - throw new Error( - "OpenShell sandbox container inspect did not include a valid immutable image ID; " + - "refusing startup-command recreation from a mutable image tag.", - ); - } - const image = requiresImmutableImage ? immutableImage : configuredImage; - - const selection = options.modeOverride - ? { mode: options.modeOverride, attempts: [] } - : selectDockerGpuPatchMode( - { - image, - device: options.gpuDevice, - backend: options.backend, - dockerDesktopWsl: options.dockerDesktopWsl, - }, - deps, - ); - context.modeAttempts = selection.attempts; - context.selectedMode = selection.mode; - if (!selection.mode) { - const modeMessage = - options.backend === "jetson" - ? "Docker did not accept the Jetson NVIDIA runtime GPU mode." - : "Docker did not accept --gpus, NVIDIA runtime, or CDI GPU modes."; - throw new Error(modeMessage); - } - - const originalName = dockerContainerName(inspect); - const backupContainerName = buildBackupContainerName(originalName, d.now()); - context.backupContainerName = backupContainerName; - - const cloneOptions = buildDockerGpuCloneRunOptions(inspect); - cloneOptions.image = image; - cloneOptions.openshellSandboxCommand = options.openshellSandboxCommand ?? null; - const sandboxFallbackDns = d.detectSandboxFallbackDns(); - if (sandboxFallbackDns) cloneOptions.sandboxFallbackDns = sandboxFallbackDns; - // On Jetson the Tegra GPU device nodes (`/dev/nvmap`, `/dev/nvhost-*`) are - // owned by a non-root group, but the sandbox user is not a member — so - // CUDA fails with `NvRmMemInitNvmap ... Permission denied` and `cuInit(0)` - // returns 999 even though the devices are mounted (#4231). Grant the - // sandbox user the owning group(s) so CUDA can initialize. - if (selection.mode.kind !== "startup-command" && options.backend === "jetson") { - const tegraGroupGids = d.detectTegraDeviceGroupGids(); - if (tegraGroupGids.length > 0) { - cloneOptions.extraGroupGids = tegraGroupGids; - console.log( - ` ✓ Granting sandbox user access to Jetson Tegra GPU device nodes via --group-add ${tegraGroupGids.join( - ", ", - )} (so CUDA can open /dev/nvmap)`, - ); - } else { - console.warn( - " ⚠ Could not resolve the group owning Jetson Tegra GPU device nodes (/dev/nvmap); CUDA may fail with NvRmMemInitNvmap permission denied. Confirm /dev/nvmap exists and is group-readable on the host.", - ); - } - } - // Validate and build the replacement command before touching the original - // container. A malformed startup envelope must fail without stopping or - // renaming the user's working sandbox. - const cloneArgs = buildDockerGpuCloneRunArgs(inspect, selection.mode, cloneOptions); - - const containerMutationOptions = { - ignoreError: true, - suppressOutput: true, - timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, - }; - const stopResult = d.dockerStop(oldContainerId, containerMutationOptions); - if (!isZeroStatus(stopResult)) { - context.rolledBack = isZeroStatus(d.dockerStart(oldContainerId, containerMutationOptions)); - throw new Error( - `Could not stop original sandbox container: ${resultText(stopResult)}; ${ - context.rolledBack - ? "original sandbox container confirmed running" - : "restart failed; original sandbox container may be stopped" - }`, - ); - } - const renameResult = d.dockerRename( - oldContainerId, - backupContainerName, - containerMutationOptions, - ); - if (!isZeroStatus(renameResult)) { - // A timed-out rename can still have reached the daemon. Normalize both - // possible outcomes toward the original name, then prove by container ID - // that the original is named correctly and running before calling the - // recovery successful. - d.dockerRename(backupContainerName, originalName, containerMutationOptions); - const restarted = isZeroStatus(d.dockerStart(oldContainerId, containerMutationOptions)); - let originalNameRestored = false; - try { - originalNameRestored = - dockerContainerName(inspectDockerContainer(oldContainerId, deps)) === originalName; - } catch { - originalNameRestored = false; - } - context.rolledBack = restarted && originalNameRestored; - throw new Error( - `Could not move original sandbox container aside: ${resultText(renameResult)}; ${ - context.rolledBack - ? "original sandbox container restored" - : "restore failed; original sandbox container state is uncertain" - }`, - ); - } - - const runResult = d.dockerRunDetached(cloneArgs, { - ignoreError: true, - suppressOutput: true, - timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, - }); - if (!isZeroStatus(runResult)) { - // #5512: the recreate failed after the original was renamed aside. Restore - // the pre-patch sandbox (remove the failed new container, rename the backup - // back, and start it) instead of leaving an orphaned - // `*-nemoclaw-gpu-backup-*` container and a sandbox with no live original - // behind — which otherwise collides on the next retry. - context.rolledBack = rollbackDockerGpuPatchOnRecreateFailure( - // newContainerId is originalName: the recreate `docker run` used `--name originalName`. - { newContainerId: originalName, backupContainerName, originalName }, - deps, - ); - const containerDescription = - selection.mode.kind === "startup-command" - ? "recreated sandbox container" - : "GPU-enabled sandbox container"; - throw new Error( - `Could not start ${containerDescription}: ${resultText(runResult)}; ${ - context.rolledBack - ? "pre-patch sandbox restored" - : "rollback failed; pre-patch sandbox was NOT restored" - }`, - ); - } - - const stdoutId = String(runResult.stdout || "").trim(); - const newContainerId = - stdoutId || - waitForNewContainerId( - options.sandboxName, - oldContainerId, - options.timeoutSecs ?? DOCKER_GPU_PATCH_WAIT_SECS, - deps, - ); - if (!newContainerId) { - context.rolledBack = rollbackDockerGpuPatchOnRecreateFailure( - // Docker accepted `run --name originalName`, but neither stdout nor - // labeled discovery identified the replacement. Use the deterministic - // requested name to remove any partial replacement before restoring - // the pinned backup. - { newContainerId: originalName, backupContainerName, originalName }, - deps, - ); - const containerDescription = - selection.mode.kind === "startup-command" - ? "Recreated sandbox container" - : "GPU-enabled sandbox container"; - throw new Error( - `${containerDescription} started, but Docker did not report its ID; ${ - context.rolledBack - ? "pre-patch sandbox restored" - : "rollback failed; pre-patch sandbox was NOT restored" - }`, - ); - } - context.newContainerId = newContainerId; - - const selectedMode = selection.mode; - const buildPatchResult = (backupRemoved: boolean): DockerGpuPatchResult => ({ - applied: true, - oldContainerId, - newContainerId, - originalName, - backupContainerName, - mode: selectedMode, - backupRemoved, - }); - - // Deferred: caller will run the supervisor wait and call - // `finalizeDockerGpuPatchBackup` (success → remove the backup, failure → - // roll back to it). Removing the backup here would strand the user with - // a deleted-backup / failed-new sandbox if the deferred reconnect fails. - if (options.waitForSupervisor === false) return buildPatchResult(false); - - const execReady = waitForOpenShellSupervisorReconnect( - options.sandboxName, - options.timeoutSecs ?? DOCKER_GPU_PATCH_WAIT_SECS, - deps, - ); - const reconcile = reconcileSupervisorReconnect( - execReady, - { newContainerId, backupContainerName, originalName }, - deps, - ); - if (!reconcile.execReady) { - context.rolledBack = reconcile.rolledBack; - throw reconcile.error; - } - return buildPatchResult(reconcile.backupRemoved); - } catch (error) { - const err = error instanceof Error ? error : new Error(String(error)); - throw decoratePatchError(err, context); - } -} - -export const recreateOpenShellDockerSandboxWithGpu: ( - options: Omit[0], "modeOverride">, - deps?: DockerGpuPatchDeps, -) => DockerGpuPatchResult = recreateOpenShellDockerSandboxContainer; - -export function dockerGpuPatchCleanupCommands(sandboxName: string): string[] { - return [`openshell sandbox delete ${JSON.stringify(sandboxName)}`]; -} - function printDockerGpuPatchCleanup(sandboxName: string): void { console.error(" The failed sandbox/container has been left in place for inspection."); console.error(" Manual cleanup:"); @@ -1417,6 +173,7 @@ export function printDockerGpuPatchFailureAndExit( deps: Pick & { context?: DockerGpuPatchFailureContext | null; selectedMode?: DockerGpuPatchMode | null; + additionalSummaryLines?: readonly string[]; }, ): never { const context = deps.context || getDockerGpuPatchFailureContext(error) || null; @@ -1430,7 +187,14 @@ export function printDockerGpuPatchFailureAndExit( const classification = classifyDockerGpuPatchFailure(snapshot, selectedMode); const diagnostics = collectDockerGpuPatchDiagnostics( sandboxName, - { error, context, selectedMode, snapshot, classification }, + { + error, + context, + selectedMode, + snapshot, + classification, + additionalSummaryLines: deps.additionalSummaryLines, + }, inspectDeps, ); const errorMessage = @@ -1447,9 +211,7 @@ export function printDockerGpuPatchFailureAndExit( console.error(` Diagnostics saved: ${diagnostics.dir}`); } console.error(" Escape hatches:"); - console.error( - " NEMOCLAW_DOCKER_GPU_PATCH=1 force the legacy Docker GPU container-swap path.", - ); + console.error(" NEMOCLAW_DOCKER_GPU_PATCH=1 use only the Docker GPU compatibility path."); console.error( " NEMOCLAW_DOCKER_GPU_PATCH=0 use native OpenShell GPU injection (ignored on Docker Desktop WSL; Jetson also defaults to the compatibility path).", ); @@ -1465,6 +227,7 @@ export function printDockerGpuReadinessFailure( selectedMode: DockerGpuPatchMode | null, deps: Pick & { context?: DockerGpuPatchFailureContext | null; + additionalSummaryLines?: readonly string[]; }, ): void { const context = deps.context ?? null; @@ -1477,7 +240,13 @@ export function printDockerGpuReadinessFailure( const classification = classifyDockerGpuPatchFailure(snapshot, selectedMode); const diagnostics = collectDockerGpuPatchDiagnostics( sandboxName, - { selectedMode, context, snapshot, classification }, + { + selectedMode, + context, + snapshot, + classification, + additionalSummaryLines: deps.additionalSummaryLines, + }, inspectDeps, ); printDockerGpuPatchClassificationLines(classification); @@ -1493,6 +262,7 @@ export function printDockerGpuProofFailure( selectedMode: DockerGpuPatchMode | null, deps: Pick & { context?: DockerGpuPatchFailureContext | null; + additionalSummaryLines?: readonly string[]; }, ): void { const context = deps.context ?? null; @@ -1507,7 +277,14 @@ export function printDockerGpuProofFailure( }); const diagnostics = collectDockerGpuPatchDiagnostics( sandboxName, - { error, selectedMode, context, snapshot, classification }, + { + error, + selectedMode, + context, + snapshot, + classification, + additionalSummaryLines: deps.additionalSummaryLines, + }, inspectDeps, ); printDockerGpuPatchClassificationLines(classification); @@ -1517,70 +294,6 @@ export function printDockerGpuProofFailure( printDockerGpuPatchCleanup(sandboxName); } -function writeTextFile(dir: string, name: string, content: string): void { - fs.writeFileSync(path.join(dir, name), content.endsWith("\n") ? content : `${content}\n`, { - mode: 0o600, - }); -} - -function uniqueStrings(values: Array): string[] { - return [...new Set(values.map((value) => String(value || "").trim()).filter(Boolean))]; -} - -const DIAGNOSTIC_ENV_KEYS = new Set([ - "OPENSHELL_ENDPOINT", - "OPENSHELL_SANDBOX_ID", - "OPENSHELL_SANDBOX", - "OPENSHELL_LOG_LEVEL", - "OPENSHELL_TLS_CA", - "OPENSHELL_TLS_CERT", - "OPENSHELL_TLS_KEY", -]); - -function diagnosticEnvLines(env: string[] | null | undefined): string[] { - return stringArray(env) - .filter((entry) => DIAGNOSTIC_ENV_KEYS.has(envKey(entry))) - .sort() - .map((entry) => ` env.${envKey(entry)}=${entry.slice(envKey(entry).length + 1)}`); -} - -export function formatDockerInspectNetworkSummary( - target: string, - inspect: DockerContainerInspect, -): string { - const lines = [ - `target=${target}`, - `id=${inspect.Id ?? "unknown"}`, - `name=${String(inspect.Name || "").replace(/^\/+/, "") || "unknown"}`, - `image=${inspect.Config?.Image ?? "unknown"}`, - `network_mode=${inspect.HostConfig?.NetworkMode ?? "unknown"}`, - ]; - const extraHosts = stringArray(inspect.HostConfig?.ExtraHosts); - if (extraHosts.length > 0) { - lines.push("extra_hosts:"); - for (const entry of extraHosts) lines.push(` ${entry}`); - } - const envLines = diagnosticEnvLines(inspect.Config?.Env); - if (envLines.length > 0) { - lines.push("openshell_env:"); - lines.push(...envLines); - } - const networks = inspect.NetworkSettings?.Networks || {}; - const names = Object.keys(networks).sort(); - if (names.length > 0) { - lines.push("networks:"); - for (const name of names) { - const network = networks[name] || {}; - lines.push( - ` ${name}: ip=${network.IPAddress || "unknown"} gateway=${network.Gateway || "unknown"}`, - ); - const aliases = stringArray(network.Aliases); - if (aliases.length > 0) lines.push(` aliases=${aliases.join(",")}`); - } - } - return lines.join("\n"); -} - const SANDBOX_FAILURE_PHASE_TOKENS = new Set(["Error", "Failed", "CrashLoopBackOff"]); const SANDBOX_LIVE_PHASE_TOKENS = new Set(["Ready", "Running"]); @@ -1816,201 +529,3 @@ export function classifyDockerGpuPatchFailure( } return { kind, headline, summaryLines: lines }; } - -export function collectDockerGpuPatchDiagnostics( - sandboxName: string, - options: { - error?: unknown; - context?: DockerGpuPatchFailureContext | null; - selectedMode?: DockerGpuPatchMode | null; - snapshot?: DockerGpuPatchSandboxSnapshot | null; - classification?: DockerGpuPatchFailureClassification | null; - additionalSensitiveValues?: readonly string[]; - dockerTopOutput?: string | null; - } = {}, - deps: DockerGpuPatchDeps = {}, -): DockerGpuPatchDiagnostics | null { - const d = depsWithDefaults(deps); - const now = d.now(); - const dir = path.join( - d.homedir(), - ".nemoclaw", - "onboard-failures", - `${timestampForPath(now)}-${sanitizePathPart(sandboxName)}-docker-gpu-patch`, - ); - try { - fs.mkdirSync(dir, { recursive: true, mode: 0o700 }); - } catch { - return null; - } - - const context = options.context || getDockerGpuPatchFailureContext(options.error) || null; - const redactor = createDockerGpuDiagnosticRedactor(options.additionalSensitiveValues); - let discoveredContainerIds: string[] = []; - try { - discoveredContainerIds = findOpenShellDockerSandboxContainerIds(sandboxName, deps); - } catch { - discoveredContainerIds = []; - } - const containerTargets = uniqueStrings([ - ...(context - ? [context.oldContainerId, context.newContainerId, context.backupContainerName] - : []), - ...discoveredContainerIds, - ]); - const inspectedTargets: Array<{ target: string; entries: DockerContainerInspect[] }> = []; - for (const target of containerTargets) { - try { - const inspect = d.dockerCapture(["inspect", target], { - ignoreError: true, - timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, - }); - if (!inspect.trim()) continue; - const parsed = JSON.parse(inspect); - const entries = (Array.isArray(parsed) ? parsed : [parsed]) as DockerContainerInspect[]; - for (const entry of entries) redactor.rememberInspect(entry); - inspectedTargets.push({ target, entries }); - } catch { - /* best effort */ - } - } - const writeDiagnosticText = (name: string, content: string): void => { - writeTextFile(dir, name, redactor.redactText(content)); - }; - const writeDiagnosticJson = (name: string, value: unknown): void => { - writeTextFile(dir, name, JSON.stringify(redactor.redactValue(value), null, 2)); - }; - - const cleanupCommands = dockerGpuPatchCleanupCommands(sandboxName).map(redactor.redactText); - const errorText = redactor.redactText( - options.error instanceof Error - ? options.error.message - : options.error - ? String(options.error) - : "none", - ); - const selectedMode = options.selectedMode || context?.selectedMode || null; - const snapshot = options.snapshot ?? null; - const classification = options.classification ?? null; - const summaryLines = [ - `created_at=${now.toISOString()}`, - `sandbox_name=${redactor.redactText(sandboxName)}`, - `error=${errorText}`, - `selected_gpu_mode=${redactor.redactText(selectedMode?.label ?? "none")}`, - `old_container_id=${redactor.redactText(context?.oldContainerId ?? "unknown")}`, - `new_container_id=${redactor.redactText(context?.newContainerId ?? "unknown")}`, - `backup_container_name=${redactor.redactText(context?.backupContainerName ?? "none")}`, - `rolled_back=${context?.rolledBack === true ? "yes" : context?.rolledBack === false ? "failed" : "no"}`, - "cleanup_commands:", - ...cleanupCommands.map((command) => ` ${command}`), - ]; - if (context?.modeAttempts?.length) { - summaryLines.push("gpu_mode_attempts:"); - for (const attempt of context.modeAttempts) { - summaryLines.push( - redactor.redactText( - ` ${attempt.mode.label}: ${attempt.ok ? "ok" : "failed"}${attempt.error ? `: ${attempt.error}` : ""}`, - ), - ); - } - } - if (classification) { - summaryLines.push(`failure_kind=${redactor.redactText(classification.kind)}`); - if (classification.headline) { - summaryLines.push(`failure_headline=${redactor.redactText(classification.headline)}`); - } - } - if (snapshot) { - if (snapshot.sandboxPhase) { - summaryLines.push(`sandbox_phase=${redactor.redactText(snapshot.sandboxPhase)}`); - } - if (snapshot.sandboxListLine) { - summaryLines.push(`sandbox_list_row=${redactor.redactText(snapshot.sandboxListLine)}`); - } - summaryLines.push( - ...describePatchedContainerState(snapshot.patchedContainerState).map(redactor.redactText), - ); - } - writeDiagnosticText("summary.txt", summaryLines.join("\n")); - if (snapshot?.patchedContainerState) { - writeDiagnosticJson("patched-container-state.json", snapshot.patchedContainerState); - } - if (options.dockerTopOutput?.trim()) { - writeDiagnosticText("docker-top.txt", options.dockerTopOutput); - } - - try { - const ps = d.dockerCapture( - [ - "ps", - "-a", - "--filter", - `label=${OPENSHELL_MANAGED_BY_LABEL}=${OPENSHELL_MANAGED_BY_VALUE}`, - "--filter", - `label=${OPENSHELL_SANDBOX_NAME_LABEL}=${sandboxName}`, - ], - { ignoreError: true, timeout: DOCKER_GPU_PATCH_TIMEOUT_MS }, - ); - if (ps.trim()) writeDiagnosticText("docker-ps.txt", ps); - } catch { - /* best effort */ - } - - if (containerTargets.length > 0) { - const inspectEntries: DockerContainerInspect[] = []; - const networkSummaries: string[] = []; - for (const { target, entries } of inspectedTargets) { - const sanitizedEntries = entries.map(redactor.sanitizeInspect); - inspectEntries.push(...sanitizedEntries); - for (const [index, entry] of sanitizedEntries.entries()) { - networkSummaries.push( - redactor.redactText( - formatDockerInspectNetworkSummary( - entries.length === 1 ? target : `${target}[${index}]`, - entry, - ), - ), - ); - } - } - if (inspectEntries.length > 0) { - writeDiagnosticJson("docker-inspect.json", inspectEntries); - } - if (networkSummaries.length > 0) { - writeDiagnosticText("docker-network-summary.txt", networkSummaries.join("\n\n")); - } - const logs = containerTargets - .map((target) => { - try { - return redactor.redactText( - [`===== ${target} =====`, d.dockerLogs(target, { tail: 120 })].join("\n"), - ); - } catch { - return redactor.redactText(`===== ${target} =====\n(unavailable)`); - } - }) - .join("\n"); - if (logs.trim()) writeDiagnosticText("docker-logs.txt", logs); - } - - if (deps.runCaptureOpenshell) { - const captures: Array<[string, string[]]> = [ - ["openshell-sandbox-get.txt", ["sandbox", "get", sandboxName]], - ["openshell-sandbox-list.txt", ["sandbox", "list"]], - ["openshell-logs.txt", ["doctor", "logs", "--name", "nemoclaw"]], - ]; - for (const [fileName, args] of captures) { - try { - const output = deps.runCaptureOpenshell(args, { - ignoreError: true, - timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, - }); - if (output.trim()) writeDiagnosticText(fileName, output); - } catch { - /* best effort */ - } - } - } - - return { dir, cleanupCommands, summaryLines }; -} diff --git a/src/lib/onboard/docker-gpu-pre-rollback-diagnostics.ts b/src/lib/onboard/docker-gpu-pre-rollback-diagnostics.ts index 4e7f53cbe21..0d1ab1dbb61 100644 --- a/src/lib/onboard/docker-gpu-pre-rollback-diagnostics.ts +++ b/src/lib/onboard/docker-gpu-pre-rollback-diagnostics.ts @@ -6,21 +6,21 @@ import { dockerLogs as defaultDockerLogs, } from "../adapters/docker"; import { discoverDockerGpuDiagnosticSensitiveValues } from "./docker-gpu-diagnostic-redaction"; -import type { - DockerContainerInspect, - DockerGpuPatchDeps, - DockerGpuPatchDiagnostics, - DockerGpuPatchFailureContext, - DockerGpuPatchResult, -} from "./docker-gpu-patch"; import { captureDockerGpuPatchSandboxSnapshot, classifyDockerGpuPatchFailure, collectDockerGpuPatchDiagnostics, findOpenShellDockerSandboxContainerIds, } from "./docker-gpu-patch"; +import { DOCKER_GPU_PATCH_TIMEOUT_MS } from "./docker-gpu-patch-constants"; +import type { + DockerContainerInspect, + DockerGpuPatchDeps, + DockerGpuPatchDiagnostics, + DockerGpuPatchFailureContext, + DockerGpuPatchResult, +} from "./docker-gpu-patch-types"; -const DOCKER_GPU_PATCH_TIMEOUT_MS = 30_000; const PRE_ROLLBACK_DIAGNOSTICS_TOTAL_BUDGET_MS = 10_000; const PRE_ROLLBACK_DIAGNOSTICS_CALL_TIMEOUT_MS = 2_000; @@ -29,8 +29,21 @@ type PreRollbackDiagnosticsDeps = Pick< "runCaptureOpenshell" | "dockerCapture" | "dockerLogs" | "homedir" | "now" >; -// This wrapper owns only the pre-rollback time budget. The shared collector -// is the sole redaction and artifact-publication boundary for every caller. +/** + * SOURCE_OF_TRUTH_REVIEW + * invalidState: Docker created a replacement but OpenShell did not reconnect; rollback would + * erase its transient process, network, state, and log evidence. + * sourceBoundary: Docker/OpenShell own that ephemeral state; this wrapper snapshots it + * immediately before rollback, and the shared collector remains the sole redaction and + * artifact-publication boundary for every caller. + * whyNotSourceFix: this layer cannot reconnect the external supervisor or retain the failed + * replacement without delaying restoration, so capture is best effort and strictly bounded. + * regressionTest: docker-gpu-pre-rollback-diagnostics.test.ts covers the allowlisted bundle, + * redaction, and budget; docker-gpu-sandbox-create-diagnostics.test.ts proves capture precedes + * rollback and capture failure cannot block rollback. + * removalCondition: remove only when the replacement path emits equivalent bounded, redacted + * evidence before rollback, or no longer replaces a container. + */ function boundedDiagnosticsDeps(deps: PreRollbackDiagnosticsDeps): PreRollbackDiagnosticsDeps { const capture = deps.dockerCapture ?? defaultDockerCapture; const logs = deps.dockerLogs ?? defaultDockerLogs; diff --git a/src/lib/onboard/docker-gpu-route-consumers.test.ts b/src/lib/onboard/docker-gpu-route-consumers.test.ts new file mode 100644 index 00000000000..f2454e5d45d --- /dev/null +++ b/src/lib/onboard/docker-gpu-route-consumers.test.ts @@ -0,0 +1,187 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { + enforceDockerGpuPatchPreserveNetwork, + shouldSkipGpuBridgeProbe, + shouldUseDockerGpuPatchHostNetwork, + verifyDockerGpuSandboxLocalInference, + verifyGpuSandboxAfterReady, +} from "./docker-gpu-local-inference"; +import { resolveDockerGpuRoutePlan } from "./docker-gpu-route"; +import { prepareSandboxGpuRoutePolicies } from "./sandbox-gpu-route-policy"; + +const GPU_CONFIG = { sandboxGpuEnabled: true }; +const HOST_NETWORK_ENV = { + NEMOCLAW_DOCKER_GPU_PATCH: "1", + NEMOCLAW_DOCKER_GPU_PATCH_NETWORK: "host", +} as NodeJS.ProcessEnv; + +describe("route-specific policy materialization", () => { + it("keeps the native attempt narrow and prepares one broad fallback policy", () => { + const nativeCleanup = vi.fn(() => true); + const compatibilityCleanup = vi.fn(() => true); + const preparePolicy = vi.fn((_base, _channels, options) => ({ + policyPath: options?.dockerGpuPatch ? "/tmp/compatibility.yaml" : "/tmp/native.yaml", + appliedPresets: ["github"], + cleanup: options?.dockerGpuPatch ? compatibilityCleanup : nativeCleanup, + })); + const explicitFallbackPlan = resolveDockerGpuRoutePlan(GPU_CONFIG, { + dockerDriverGateway: true, + env: { NEMOCLAW_DOCKER_GPU_PATCH: "fallback" }, + platform: "linux", + }); + const policies = prepareSandboxGpuRoutePolicies( + "/repo/policy.yaml", + ["telegram"], + { directGpu: true, additionalPresets: ["github"] }, + explicitFallbackPlan, + preparePolicy, + ); + + expect(preparePolicy).toHaveBeenNthCalledWith( + 1, + "/repo/policy.yaml", + ["telegram"], + expect.objectContaining({ directGpu: true, dockerGpuPatch: false }), + ); + expect(preparePolicy).toHaveBeenNthCalledWith( + 2, + "/repo/policy.yaml", + ["telegram"], + expect.objectContaining({ directGpu: true, dockerGpuPatch: true }), + ); + expect(policies.initialSandboxPolicy.policyPath).toBe("/tmp/native.yaml"); + expect(policies.compatibilityPolicyPath).toBe("/tmp/compatibility.yaml"); + expect(policies.initialSandboxPolicy.cleanup?.()).toBe(true); + expect(nativeCleanup).toHaveBeenCalledOnce(); + expect(compatibilityCleanup).toHaveBeenCalledOnce(); + }); + + it("does not materialize a broader compatibility policy for ordinary Linux defaults (#6110)", () => { + const preparePolicy = vi.fn((_base, _channels, options) => ({ + policyPath: options?.dockerGpuPatch ? "/tmp/compatibility.yaml" : "/tmp/native.yaml", + appliedPresets: [], + })); + const defaultPlan = resolveDockerGpuRoutePlan(GPU_CONFIG, { + dockerDriverGateway: true, + env: { NEMOCLAW_DOCKER_GPU_PATCH: "auto" }, + platform: "linux", + }); + + const policies = prepareSandboxGpuRoutePolicies( + "/repo/policy.yaml", + [], + { directGpu: true }, + defaultPlan, + preparePolicy, + ); + + expect(defaultPlan).toBe("native-only"); + expect(preparePolicy).toHaveBeenCalledOnce(); + expect(preparePolicy).toHaveBeenCalledWith( + "/repo/policy.yaml", + [], + expect.objectContaining({ dockerGpuPatch: false }), + ); + expect(policies.compatibilityPolicyPath).toBeNull(); + }); + + it("cleans the initial temporary policy when fallback policy materialization fails", () => { + const nativeCleanup = vi.fn(() => true); + const preparePolicy = vi + .fn() + .mockReturnValueOnce({ + policyPath: "/tmp/native.yaml", + appliedPresets: [], + cleanup: nativeCleanup, + }) + .mockImplementationOnce(() => { + throw new Error("compatibility policy failed"); + }); + + expect(() => + prepareSandboxGpuRoutePolicies( + "/repo/policy.yaml", + [], + { directGpu: true }, + "native-with-fallback", + preparePolicy, + ), + ).toThrow("compatibility policy failed"); + expect(nativeCleanup).toHaveBeenCalledOnce(); + }); +}); + +describe("selected route consumers", () => { + it("keeps native selection out of compatibility networking", async () => { + const env = { ...HOST_NETWORK_ENV }; + const reverifyBridgeReachability = vi.fn(); + const options = { + dockerDriverGateway: true, + selectedRoute: "native" as const, + platform: "linux" as NodeJS.Platform, + env, + }; + expect(shouldUseDockerGpuPatchHostNetwork(GPU_CONFIG, options)).toBe(false); + expect(shouldSkipGpuBridgeProbe(true, "linux", "native", options)).toBe(false); + expect( + await enforceDockerGpuPatchPreserveNetwork("ollama-local", GPU_CONFIG, { + ...options, + reverifyBridgeReachability, + }), + ).toBe(false); + expect(env.NEMOCLAW_DOCKER_GPU_PATCH_NETWORK).toBe("host"); + expect(reverifyBridgeReachability).not.toHaveBeenCalled(); + }); + + it("skips compatibility-only inference gates after native wins", () => { + const execInSandbox = vi.fn(); + expect( + verifyDockerGpuSandboxLocalInference(GPU_CONFIG, "ollama-local", { + sandboxName: "alpha", + dockerDriverGateway: true, + selectedRoute: "native", + env: HOST_NETWORK_ENV, + }), + ).toEqual({ status: "skipped", reason: "not-docker-gpu-patch" }); + + const verifyDirectSandboxGpu = vi.fn(); + verifyGpuSandboxAfterReady(GPU_CONFIG, "ollama-local", { + sandboxName: "alpha", + dockerDriverGateway: true, + selectedRoute: "native", + verifyDirectSandboxGpu, + selectedMode: () => null, + runCaptureOpenshell: vi.fn(() => ""), + deps: { execInSandbox, sleep: vi.fn() }, + }); + expect(verifyDirectSandboxGpu).toHaveBeenCalledWith("alpha"); + expect(execInSandbox).not.toHaveBeenCalled(); + }); + + it("defers native proof diagnostics while automatic fallback owns recovery", () => { + const proofError = new Error("native CUDA proof failed"); + const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}); + try { + expect(() => + verifyGpuSandboxAfterReady(GPU_CONFIG, "ollama-local", { + sandboxName: "alpha", + dockerDriverGateway: true, + selectedRoute: "native", + verifyDirectSandboxGpu: vi.fn(() => { + throw proofError; + }), + reportGpuProofFailure: false, + selectedMode: () => null, + runCaptureOpenshell: vi.fn(() => ""), + }), + ).toThrow(proofError); + expect(consoleError).not.toHaveBeenCalled(); + } finally { + consoleError.mockRestore(); + } + }); +}); diff --git a/src/lib/onboard/docker-gpu-route-patch-adapter.ts b/src/lib/onboard/docker-gpu-route-patch-adapter.ts new file mode 100644 index 00000000000..efbabcd72b7 --- /dev/null +++ b/src/lib/onboard/docker-gpu-route-patch-adapter.ts @@ -0,0 +1,50 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + initialDockerGpuRoute, + resolveDockerGpuRoutePlan, + type SelectedDockerGpuRoute, +} from "./docker-gpu-route"; + +export type DockerGpuPatchRouteAdapter = { + enabled: boolean; + additionalSummaryLines: readonly string[]; +}; + +/** Translate orchestration policy into the route-agnostic patch interface. */ +export function adaptDockerGpuRouteForPatch( + route: SelectedDockerGpuRoute, +): DockerGpuPatchRouteAdapter { + return { + enabled: route === "compatibility", + additionalSummaryLines: [`selected_gpu_route=${route}`], + }; +} + +/** Compatibility facade for callers that only need the initial patch decision. */ +export function shouldApplyDockerGpuPatch( + config: { sandboxGpuEnabled: boolean; hostGpuPlatform?: string | null }, + options: { + env?: NodeJS.ProcessEnv; + platform?: NodeJS.Platform; + dockerDriverGateway?: boolean; + dockerDesktopWsl?: boolean; + log?: (message: string) => void; + } = {}, +): boolean { + const platform = options.platform ?? process.platform; + const dockerDesktopWsl = options.dockerDesktopWsl === true; + const dockerDriverGateway = + options.dockerDriverGateway ?? (platform === "linux" || dockerDesktopWsl); + const route = initialDockerGpuRoute( + resolveDockerGpuRoutePlan(config, { + dockerDriverGateway, + dockerDesktopWsl, + env: options.env, + platform, + log: options.log, + }), + ); + return adaptDockerGpuRouteForPatch(route).enabled; +} diff --git a/src/lib/onboard/docker-gpu-route-render.test.ts b/src/lib/onboard/docker-gpu-route-render.test.ts new file mode 100644 index 00000000000..68595c61b33 --- /dev/null +++ b/src/lib/onboard/docker-gpu-route-render.test.ts @@ -0,0 +1,114 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + canFallbackToDockerGpuCompatibility, + initialDockerGpuRoute, + isDockerGpuCompatibilityRoute, + renderCompatibilityFallbackCreateArgs, + renderSandboxCreateArgsForGpuRoute, + supportsDockerGpuCompatibility, +} from "./docker-gpu-route"; +import { shouldApplyDockerGpuPatch } from "./docker-gpu-route-patch-adapter"; + +const GPU_CONFIG = { sandboxGpuEnabled: true }; +const IMAGE_ID = `sha256:${"a".repeat(64)}`; + +describe("Docker GPU route rendering", () => { + it.each([ + [GPU_CONFIG, {}, false], + [GPU_CONFIG, { NEMOCLAW_DOCKER_GPU_PATCH: "auto" }, false], + [GPU_CONFIG, { NEMOCLAW_DOCKER_GPU_PATCH: "fallback" }, false], + [GPU_CONFIG, { NEMOCLAW_DOCKER_GPU_PATCH: "0" }, false], + [GPU_CONFIG, { NEMOCLAW_DOCKER_GPU_PATCH: "1" }, true], + [{ sandboxGpuEnabled: false }, {}, false], + [{ sandboxGpuEnabled: true, hostGpuPlatform: "jetson" }, {}, true], + [ + { sandboxGpuEnabled: true, hostGpuPlatform: "jetson" }, + { NEMOCLAW_DOCKER_GPU_PATCH: "0" }, + false, + ], + ] as const)("adapts plan %j and control %j to patch enabled=%s", (config, env, expected) => { + expect( + shouldApplyDockerGpuPatch(config, { + env, + platform: "linux", + dockerDriverGateway: true, + }), + ).toBe(expected); + }); + + it.each([ + ["none", "none", false, false], + ["native-only", "native", false, false], + ["compatibility-only", "compatibility", true, false], + ["native-with-fallback", "native", true, true], + ] as const)("describes %s", (plan, initialRoute, compatibilitySupported, fallbackSupported) => { + expect(initialDockerGpuRoute(plan)).toBe(initialRoute); + expect(supportsDockerGpuCompatibility(plan)).toBe(compatibilitySupported); + expect(canFallbackToDockerGpuCompatibility(plan)).toBe(fallbackSupported); + }); + + it("identifies only the selected compatibility route", () => { + expect(isDockerGpuCompatibilityRoute("compatibility")).toBe(true); + expect(isDockerGpuCompatibilityRoute("native")).toBe(false); + expect(isDockerGpuCompatibilityRoute("none")).toBe(false); + }); + + it("renders native and compatibility argv from one materialized plan", () => { + const args = [ + "--from", + "/tmp/build/Dockerfile", + "--name", + "alpha", + "--policy", + "/tmp/native-policy.yaml", + "--gpu", + "--gpu-device", + "nvidia.com/gpu=0", + "--provider", + "provider-a", + ]; + expect(renderSandboxCreateArgsForGpuRoute(args, "native")).toEqual(args); + expect( + renderSandboxCreateArgsForGpuRoute(args, "compatibility", { + compatibilityPolicyPath: "/tmp/compatibility-policy.yaml", + }), + ).toEqual([ + "--from", + "/tmp/build/Dockerfile", + "--name", + "alpha", + "--policy", + "/tmp/compatibility-policy.yaml", + "--provider", + "provider-a", + ]); + }); + + it("reuses a proven image without rebuilding the fallback source", () => { + const args = ["--from", "/tmp/build/Dockerfile", "--gpu", "--policy", "/tmp/native.yaml"]; + expect( + renderCompatibilityFallbackCreateArgs(args, { + imageRef: IMAGE_ID, + compatibilityPolicyPath: "/tmp/compatibility.yaml", + }), + ).toEqual(["--from", IMAGE_ID, "--policy", "/tmp/compatibility.yaml"]); + expect( + renderCompatibilityFallbackCreateArgs(args, { + allowUnbuiltSource: true, + compatibilityPolicyPath: "/tmp/compatibility.yaml", + }), + ).toEqual(["--from", "/tmp/build/Dockerfile", "--policy", "/tmp/compatibility.yaml"]); + expect(() => + renderCompatibilityFallbackCreateArgs(args, { + compatibilityPolicyPath: "/tmp/compatibility.yaml", + }), + ).toThrow(/refusing to rebuild/i); + expect(() => renderSandboxCreateArgsForGpuRoute(args, "compatibility")).toThrow( + /route-specific sandbox policy/i, + ); + }); +}); diff --git a/src/lib/onboard/docker-gpu-route.test.ts b/src/lib/onboard/docker-gpu-route.test.ts new file mode 100644 index 00000000000..14364c4ba72 --- /dev/null +++ b/src/lib/onboard/docker-gpu-route.test.ts @@ -0,0 +1,182 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { + type DockerGpuRouteConfig, + type DockerGpuRouteOptions, + type DockerGpuRoutePlan, + initialDockerGpuRoute, + renderSandboxCreateArgsForGpuRoute, + resolveDockerGpuRoutePlan, +} from "./docker-gpu-route"; + +const GPU_CONFIG = { sandboxGpuEnabled: true }; +const LINUX_DOCKER: DockerGpuRouteOptions = { + dockerDriverGateway: true, + platform: "linux", + dockerDesktopWsl: false, + env: {}, +}; + +describe("resolveDockerGpuRoutePlan", () => { + const controls = [undefined, "auto", "fallback", "0", "1", "true"] as const; + const environments = [ + { + name: "GPU disabled", + config: { sandboxGpuEnabled: false }, + options: LINUX_DOCKER, + expected: ["none", "none", "none", "none", "none", "none"], + }, + { + name: "non-Docker driver", + config: GPU_CONFIG, + options: { ...LINUX_DOCKER, dockerDriverGateway: false }, + expected: [ + "native-only", + "native-only", + "native-only", + "native-only", + "native-only", + "native-only", + ], + }, + { + name: "non-Linux host", + config: GPU_CONFIG, + options: { ...LINUX_DOCKER, platform: "darwin" as const }, + expected: [ + "native-only", + "native-only", + "native-only", + "native-only", + "native-only", + "native-only", + ], + }, + { + name: "ordinary Linux Docker", + config: GPU_CONFIG, + options: LINUX_DOCKER, + expected: [ + "native-only", + "native-only", + "native-with-fallback", + "native-only", + "compatibility-only", + "compatibility-only", + ], + }, + { + name: "Docker Desktop WSL", + config: GPU_CONFIG, + options: { ...LINUX_DOCKER, dockerDesktopWsl: true, platform: "win32" as const }, + expected: [ + "compatibility-only", + "compatibility-only", + "compatibility-only", + "compatibility-only", + "compatibility-only", + "compatibility-only", + ], + }, + { + name: "Jetson/Tegra", + config: { sandboxGpuEnabled: true, hostGpuPlatform: "jetson" }, + options: LINUX_DOCKER, + expected: [ + "compatibility-only", + "compatibility-only", + "compatibility-only", + "native-only", + "compatibility-only", + "compatibility-only", + ], + }, + ] as const; + + const routingMatrix: Array<{ + name: string; + control: string; + expected: DockerGpuRoutePlan; + config: DockerGpuRouteConfig; + options: DockerGpuRouteOptions; + }> = environments.flatMap((environment) => + controls.map((control, index) => ({ + name: environment.name, + control: control ?? "unset", + expected: environment.expected[index], + config: environment.config, + options: { + ...environment.options, + env: control === undefined ? {} : { NEMOCLAW_DOCKER_GPU_PATCH: control }, + log: vi.fn(), + }, + })), + ); + + it.each(routingMatrix)("maps $name with control $control to $expected", ({ + expected, + config, + options, + }) => { + expect(resolveDockerGpuRoutePlan(config, options)).toBe(expected); + }); + + it("covers every environment/control pair and every route-plan outcome (#6110)", () => { + const matrixByKey = new Map( + routingMatrix.map((row) => [`${row.name}:${row.control}`, row.expected]), + ); + expect(routingMatrix).toHaveLength(environments.length * controls.length); + expect(new Set(routingMatrix.map(({ expected }) => expected))).toEqual( + new Set(["none", "native-only", "compatibility-only", "native-with-fallback"]), + ); + expect(matrixByKey.get("Docker Desktop WSL:0")).toBe("compatibility-only"); + expect(matrixByKey.get("Jetson/Tegra:0")).toBe("native-only"); + expect(matrixByKey.get("ordinary Linux Docker:unset")).toBe("native-only"); + expect(matrixByKey.get("ordinary Linux Docker:auto")).toBe("native-only"); + expect(matrixByKey.get("ordinary Linux Docker:fallback")).toBe("native-with-fallback"); + expect(matrixByKey.get("ordinary Linux Docker:true")).toBe("compatibility-only"); + expect(matrixByKey.get("non-Linux host:true")).toBe("native-only"); + }); + + it.each([ + "2", + "yes", + "on", + ])("preserves legacy nonzero compatibility routing for $control with a removal warning (#6110)", (control) => { + const log = vi.fn(); + const plan = resolveDockerGpuRoutePlan(GPU_CONFIG, { + ...LINUX_DOCKER, + env: { NEMOCLAW_DOCKER_GPU_PATCH: control }, + log, + }); + + expect(plan).toBe("compatibility-only"); + expect(log).toHaveBeenCalledWith(expect.stringMatching(/unrecognized.*compatibility-only/i)); + expect(log).toHaveBeenCalledWith(expect.stringContaining("removed in v0.1.0")); + expect( + renderSandboxCreateArgsForGpuRoute( + ["--from", "sandbox:built", "--policy", "/tmp/native.yaml", "--gpu"], + initialDockerGpuRoute(plan), + { compatibilityPolicyPath: "/tmp/compatibility.yaml" }, + ), + ).toEqual(["--from", "sandbox:built", "--policy", "/tmp/compatibility.yaml"]); + }); + + it("keeps Docker Desktop WSL on compatibility and explains why zero is ignored", () => { + const log = vi.fn(); + expect( + resolveDockerGpuRoutePlan(GPU_CONFIG, { + ...LINUX_DOCKER, + dockerDesktopWsl: true, + env: { NEMOCLAW_DOCKER_GPU_PATCH: "0" }, + log, + }), + ).toBe("compatibility-only"); + expect(log.mock.calls.map(([message]) => message).join("\n")).toMatch( + /0 ignored on Docker Desktop WSL.*--no-gpu/s, + ); + }); +}); diff --git a/src/lib/onboard/docker-gpu-route.ts b/src/lib/onboard/docker-gpu-route.ts new file mode 100644 index 00000000000..38b2e083546 --- /dev/null +++ b/src/lib/onboard/docker-gpu-route.ts @@ -0,0 +1,172 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export type DockerGpuRoutePlan = + | "none" + | "native-only" + | "compatibility-only" + | "native-with-fallback"; + +export type SelectedDockerGpuRoute = "none" | "native" | "compatibility"; + +export type DockerGpuRouteConfig = { + sandboxGpuEnabled: boolean; + hostGpuPlatform?: string | null; +}; + +export type DockerGpuRouteOptions = { + dockerDriverGateway: boolean; + dockerDesktopWsl?: boolean; + env?: NodeJS.ProcessEnv; + platform?: NodeJS.Platform; + log?: (message: string) => void; +}; + +const LEGACY_NONZERO_CONTROL_REMOVAL_VERSION = "v0.1.0"; + +/** + * Legacy control boundary: + * - invalidState: an undocumented nonzero value requests the old compatibility patch. + * - sourceBoundary: operator and deployment automation set NEMOCLAW_DOCKER_GPU_PATCH. + * - whyNotSourceFix: existing automation cannot be migrated atomically with this release. + * - regressionTest: docker-gpu-route.test.ts covers legacy nonzero routing and its warning. + * - removalCondition: remove legacy nonzero values in v0.1.0 as documented for operators. + */ +function warnForLegacyNonzeroControl(control: string, log: (message: string) => void): void { + if ( + control === "" || + control === "0" || + control === "1" || + control === "auto" || + control === "fallback" + ) + return; + log( + ` Warning: unrecognized NEMOCLAW_DOCKER_GPU_PATCH value '${control}'; preserving legacy compatibility-only behavior through v0.0.x. Other nonzero values will be removed in ${LEGACY_NONZERO_CONTROL_REMOVAL_VERSION}; use 0, 1, auto, or fallback.`, + ); +} + +/** + * SOURCE_OF_TRUTH_REVIEW (explicit ordinary-Linux GPU fallback; #6110): + * invalidState: explicit fallback sees native rejection or trusted proof of no GPU attachment. + * sourceBoundary: OpenShell `--gpu` plus structured Docker/NVIDIA evidence and proven cleanup. + * whyNotSourceFix: supported stacks cannot upgrade atomically; WSL/Jetson still need compatibility. + * regressionTest: sandbox-gpu-create-failure-classification.test.ts and + * sandbox-gpu-fallback-orchestration.test.ts prove authorization, cleanup, and one retry. + * removalCondition: native injection works on every supported host and compatibility is retired. + * + * Resolves the internal Docker-driver GPU strategy without exposing a new user contract. + */ +export function resolveDockerGpuRoutePlan( + config: DockerGpuRouteConfig, + options: DockerGpuRouteOptions, +): DockerGpuRoutePlan { + const env = options.env ?? process.env; + const platform = options.platform ?? process.platform; + const dockerDesktopWsl = options.dockerDesktopWsl === true; + if (!config.sandboxGpuEnabled) return "none"; + // The compatibility swap is specific to the Linux Docker driver. Other + // OpenShell drivers keep their existing direct `--gpu` behavior. + if (!options.dockerDriverGateway || (platform !== "linux" && !dockerDesktopWsl)) { + return "native-only"; + } + + const control = String(env.NEMOCLAW_DOCKER_GPU_PATCH ?? "") + .trim() + .toLowerCase(); + const log = options.log ?? ((message: string) => console.warn(message)); + warnForLegacyNonzeroControl(control, log); + + if (dockerDesktopWsl) { + if (control === "0") { + log( + " NEMOCLAW_DOCKER_GPU_PATCH=0 ignored on Docker Desktop WSL: GPU passthrough on this runtime requires the compatibility path.", + ); + log(" Skip GPU passthrough entirely with --no-gpu or NEMOCLAW_SANDBOX_GPU=0."); + } + return "compatibility-only"; + } + + if (config.hostGpuPlatform === "jetson") { + return control === "0" ? "native-only" : "compatibility-only"; + } + if (control === "fallback") return "native-with-fallback"; + if (control === "" || control === "auto" || control === "0") return "native-only"; + + // Before native routing was introduced, every nonzero value enabled the + // compatibility patch. Preserve that automation contract, including values + // other than the documented "1". + return "compatibility-only"; +} + +export function initialDockerGpuRoute(plan: DockerGpuRoutePlan): SelectedDockerGpuRoute { + if (plan === "none") return "none"; + return plan === "compatibility-only" ? "compatibility" : "native"; +} + +export function supportsDockerGpuCompatibility(plan: DockerGpuRoutePlan): boolean { + return plan === "compatibility-only" || plan === "native-with-fallback"; +} + +export function canFallbackToDockerGpuCompatibility(plan: DockerGpuRoutePlan): boolean { + return plan === "native-with-fallback"; +} + +export function isDockerGpuCompatibilityRoute(route: SelectedDockerGpuRoute): boolean { + return route === "compatibility"; +} + +/** Render one already-materialized create plan for the selected GPU route. */ +export function renderSandboxCreateArgsForGpuRoute( + createArgs: readonly string[], + route: SelectedDockerGpuRoute, + options: { compatibilityPolicyPath?: string | null } = {}, +): string[] { + if (route !== "compatibility") return [...createArgs]; + const rendered: string[] = []; + for (let index = 0; index < createArgs.length; index += 1) { + const arg = createArgs[index]; + if (arg === "--gpu") continue; + if (arg === "--gpu-device") { + index += 1; + continue; + } + rendered.push(arg); + } + const policyIndex = rendered.indexOf("--policy"); + if (policyIndex >= 0 && rendered[policyIndex + 1]) { + if (!options.compatibilityPolicyPath) { + throw new Error("Compatibility GPU route requires its route-specific sandbox policy."); + } + rendered[policyIndex + 1] = options.compatibilityPolicyPath; + } + return rendered; +} + +function replaceSandboxCreateImage(createArgs: readonly string[], imageRef: string): string[] { + const rendered = [...createArgs]; + const fromIndex = rendered.indexOf("--from"); + if (fromIndex < 0 || !rendered[fromIndex + 1]) { + throw new Error("Cannot reuse sandbox image; create arguments do not contain --from."); + } + rendered[fromIndex + 1] = imageRef; + return rendered; +} + +export function renderCompatibilityFallbackCreateArgs( + createArgs: readonly string[], + options: { + imageRef?: string | null; + allowUnbuiltSource?: boolean; + compatibilityPolicyPath: string; + }, +): string[] { + const compatibilityArgs = renderSandboxCreateArgsForGpuRoute(createArgs, "compatibility", { + compatibilityPolicyPath: options.compatibilityPolicyPath, + }); + if (options.imageRef) return replaceSandboxCreateImage(compatibilityArgs, options.imageRef); + if (options.allowUnbuiltSource) return compatibilityArgs; + throw new Error( + "Native GPU fallback cannot reuse the completed sandbox image; refusing to rebuild it.", + ); +} diff --git a/src/lib/onboard/docker-gpu-sandbox-create-diagnostics.test.ts b/src/lib/onboard/docker-gpu-sandbox-create-diagnostics.test.ts index ee77ddaccdf..e527ddbad7e 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create-diagnostics.test.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create-diagnostics.test.ts @@ -33,10 +33,13 @@ describe("Docker GPU create diagnostics fail-safety (#6110)", () => { sleep: vi.fn(), dockerCapture: vi.fn(() => ""), }; - const finalizeBackup = vi.fn(() => ({ backupRemoved: false, rolledBack: true })); + const finalizeBackup = vi.fn(() => ({ + backupRemoved: false, + rolledBack: true, + })); const onPatchFailureExit = vi.fn(); const patch = createDockerGpuSandboxCreatePatch({ - enabled: true, + route: "compatibility", sandboxName: "alpha", timeoutSecs: 60, deps, @@ -75,10 +78,13 @@ describe("Docker GPU create diagnostics fail-safety (#6110)", () => { const recreatePatch = vi.fn(() => RESULT); const waitForSupervisor = vi.fn(() => false); const capturePreRollbackDiagnostics = vi.fn(() => null); - const finalizeBackup = vi.fn(() => ({ backupRemoved: false, rolledBack: true })); + const finalizeBackup = vi.fn(() => ({ + backupRemoved: false, + rolledBack: true, + })); const onPatchFailureExit = vi.fn(); const patch = createDockerGpuSandboxCreatePatch({ - enabled: true, + route: "compatibility", sandboxName: "alpha", timeoutSecs: 60, deps, diff --git a/src/lib/onboard/docker-gpu-sandbox-create.test.ts b/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts similarity index 63% rename from src/lib/onboard/docker-gpu-sandbox-create.test.ts rename to src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts index e5a28f7eca5..fb94eba175e 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create.test.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts @@ -4,11 +4,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { DockerGpuPatchFailureContext, DockerGpuPatchResult } from "./docker-gpu-patch"; -import { - createDockerGpuSandboxCreatePatch, - resolveDockerGpuSandboxCreatePlan, -} from "./docker-gpu-sandbox-create"; -import { buildSandboxGpuCreateArgs } from "./sandbox-gpu-create"; +import { createDockerGpuSandboxCreatePatch } from "./docker-gpu-sandbox-create"; function deferredCreateResult(): DockerGpuPatchResult { return { @@ -51,13 +47,16 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { const result = deferredCreateResult(); const recreatePatch = vi.fn(() => result); const waitForSupervisor = vi.fn(() => true); - const finalizeBackup = vi.fn(() => ({ backupRemoved: true, rolledBack: false })); + const finalizeBackup = vi.fn(() => ({ + backupRemoved: true, + rolledBack: false, + })); const capturePreRollbackDiagnostics = vi.fn(() => null); const onPatchFailureExit = vi.fn(); const findContainerIds = vi.fn(() => ["existing-container"]); const patch = createDockerGpuSandboxCreatePatch({ - enabled: true, + route: "compatibility", sandboxName: "alpha", timeoutSecs: 60, deps, @@ -74,7 +73,9 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { patch.maybeApplyDuringCreate(); expect(recreatePatch).toHaveBeenCalledWith( expect.objectContaining({ waitForSupervisor: false }), - expect.objectContaining({ runCaptureOpenshell: deps.runCaptureOpenshell }), + expect.objectContaining({ + runCaptureOpenshell: deps.runCaptureOpenshell, + }), ); // Critical invariant: the patch helper must NOT remove the backup during // create (recreatePatch was called with waitForSupervisor: false; the @@ -89,18 +90,61 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { expect(onPatchFailureExit).not.toHaveBeenCalled(); }); + it("refuses compatibility success when the backup container cannot be removed", () => { + const deps = makeDeps(); + const result = deferredCreateResult(); + const onPatchFailureExit = vi.fn(); + const patch = createDockerGpuSandboxCreatePatch({ + route: "compatibility", + sandboxName: "alpha", + timeoutSecs: 60, + deps, + overrides: { + findContainerIds: vi.fn(() => ["existing-container"]), + recreatePatch: vi.fn(() => result), + waitForSupervisor: vi.fn(() => true), + finalizeBackup: vi.fn(() => ({ + backupRemoved: false, + rolledBack: false, + })), + onPatchFailureExit, + }, + }); + + patch.maybeApplyDuringCreate(); + patch.waitForSupervisorReconnectIfNeeded(); + + expect(onPatchFailureExit).toHaveBeenCalledOnce(); + expect(onPatchFailureExit.mock.calls[0]?.[1]).toEqual( + expect.objectContaining({ + message: expect.stringContaining("backup container"), + }), + ); + expect(onPatchFailureExit.mock.calls[0]?.[2]).toEqual( + expect.objectContaining({ + additionalSummaryLines: ["selected_gpu_route=compatibility"], + context: expect.objectContaining({ + backupContainerName: result.backupContainerName, + }), + }), + ); + }); + it("rolls back to the backup container and surfaces rolledBack=true diagnostics when supervisorReady=false", () => { const deps = makeDeps(); const result = deferredCreateResult(); const recreatePatch = vi.fn(() => result); const waitForSupervisor = vi.fn(() => false); const capturePreRollbackDiagnostics = vi.fn(() => null); - const finalizeBackup = vi.fn(() => ({ backupRemoved: false, rolledBack: true })); + const finalizeBackup = vi.fn(() => ({ + backupRemoved: false, + rolledBack: true, + })); const onPatchFailureExit = vi.fn(); const findContainerIds = vi.fn(() => ["existing-container"]); const patch = createDockerGpuSandboxCreatePatch({ - enabled: true, + route: "compatibility", sandboxName: "alpha", timeoutSecs: 60, deps, @@ -137,13 +181,16 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { const result = deferredCreateResult(); const recreatePatch = vi.fn(() => result); const waitForSupervisor = vi.fn(() => false); - const finalizeBackup = vi.fn(() => ({ backupRemoved: false, rolledBack: false })); + const finalizeBackup = vi.fn(() => ({ + backupRemoved: false, + rolledBack: false, + })); const capturePreRollbackDiagnostics = vi.fn(() => null); const onPatchFailureExit = vi.fn(); const findContainerIds = vi.fn(() => ["existing-container"]); const patch = createDockerGpuSandboxCreatePatch({ - enabled: true, + route: "compatibility", sandboxName: "alpha", timeoutSecs: 60, deps, @@ -176,7 +223,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { const findContainerIds = vi.fn(() => []); const patch = createDockerGpuSandboxCreatePatch({ - enabled: true, + route: "compatibility", sandboxName: "alpha", timeoutSecs: 60, deps, @@ -209,7 +256,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { const findContainerIds = vi.fn(() => ["existing-container"]); const patch = createDockerGpuSandboxCreatePatch({ - enabled: true, + route: "compatibility", sandboxName: "alpha", timeoutSecs: 60, deps, @@ -231,112 +278,27 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { expect(waitForSupervisor).not.toHaveBeenCalled(); expect(finalizeBackup).not.toHaveBeenCalled(); }); -}); - -describe("resolveDockerGpuSandboxCreatePlan Docker Desktop WSL handling", () => { - it("keeps useDockerGpuPatch=true on Docker Desktop WSL even when NEMOCLAW_DOCKER_GPU_PATCH=0", () => { - const originalEnv = process.env.NEMOCLAW_DOCKER_GPU_PATCH; - process.env.NEMOCLAW_DOCKER_GPU_PATCH = "0"; - try { - const plan = resolveDockerGpuSandboxCreatePlan( - { sandboxGpuEnabled: true }, - { - dockerDriverGateway: true, - detectDockerDesktopWsl: () => true, - }, - ); - expect(plan.useDockerGpuPatch).toBe(true); - } finally { - if (originalEnv === undefined) delete process.env.NEMOCLAW_DOCKER_GPU_PATCH; - else process.env.NEMOCLAW_DOCKER_GPU_PATCH = originalEnv; - } - }); - - it("honors NEMOCLAW_DOCKER_GPU_PATCH=0 when not on Docker Desktop WSL", () => { - const originalEnv = process.env.NEMOCLAW_DOCKER_GPU_PATCH; - process.env.NEMOCLAW_DOCKER_GPU_PATCH = "0"; - try { - const plan = resolveDockerGpuSandboxCreatePlan( - { sandboxGpuEnabled: true }, - { - dockerDriverGateway: true, - detectDockerDesktopWsl: () => false, - }, - ); - expect(plan.useDockerGpuPatch).toBe(false); - } finally { - if (originalEnv === undefined) delete process.env.NEMOCLAW_DOCKER_GPU_PATCH; - else process.env.NEMOCLAW_DOCKER_GPU_PATCH = originalEnv; - } - }); - - it("uses native OpenShell GPU by default and preserves the explicit legacy force", () => { - vi.stubEnv("NEMOCLAW_DOCKER_GPU_PATCH", ""); - try { - expect( - resolveDockerGpuSandboxCreatePlan( - { sandboxGpuEnabled: true }, - { - dockerDriverGateway: true, - detectDockerDesktopWsl: () => false, - platform: "linux", - }, - ).useDockerGpuPatch, - ).toBe(false); - - vi.stubEnv("NEMOCLAW_DOCKER_GPU_PATCH", "1"); - expect( - resolveDockerGpuSandboxCreatePlan( - { sandboxGpuEnabled: true }, - { - dockerDriverGateway: true, - detectDockerDesktopWsl: () => false, - platform: "linux", - }, - ).useDockerGpuPatch, - ).toBe(true); - } finally { - vi.unstubAllEnvs(); - } - }); - it("suppresses the openshell sandbox create --gpu flag on Docker Desktop WSL when the opt-out is ignored", () => { - const originalEnv = process.env.NEMOCLAW_DOCKER_GPU_PATCH; - process.env.NEMOCLAW_DOCKER_GPU_PATCH = "0"; - try { - const sandboxGpuConfig = { sandboxGpuEnabled: true }; - const plan = resolveDockerGpuSandboxCreatePlan(sandboxGpuConfig, { - dockerDriverGateway: true, - detectDockerDesktopWsl: () => true, - }); - expect(plan.useDockerGpuPatch).toBe(true); - const createArgs = buildSandboxGpuCreateArgs(sandboxGpuConfig, { - suppressGpuFlag: plan.useDockerGpuPatch, - }); - expect(createArgs).toEqual([]); - } finally { - if (originalEnv === undefined) delete process.env.NEMOCLAW_DOCKER_GPU_PATCH; - else process.env.NEMOCLAW_DOCKER_GPU_PATCH = originalEnv; - } - }); + it("hard-stops a structured failed GPU proof on the compatibility route", () => { + const deps = makeDeps(); + const patch = createDockerGpuSandboxCreatePatch({ + route: "compatibility", + sandboxName: "alpha", + timeoutSecs: 60, + deps, + overrides: { + findContainerIds: vi.fn(() => []), + }, + }); - it("emits --gpu when the patch is disabled outside Docker Desktop WSL", () => { - const originalEnv = process.env.NEMOCLAW_DOCKER_GPU_PATCH; - process.env.NEMOCLAW_DOCKER_GPU_PATCH = "0"; - try { - const sandboxGpuConfig = { sandboxGpuEnabled: true }; - const plan = resolveDockerGpuSandboxCreatePlan(sandboxGpuConfig, { - dockerDriverGateway: true, - detectDockerDesktopWsl: () => false, - }); - expect(plan.useDockerGpuPatch).toBe(false); - const createArgs = buildSandboxGpuCreateArgs(sandboxGpuConfig, { - suppressGpuFlag: plan.useDockerGpuPatch, - }); - expect(createArgs).toEqual(["--gpu"]); - } finally { - if (originalEnv === undefined) delete process.env.NEMOCLAW_DOCKER_GPU_PATCH; - else process.env.NEMOCLAW_DOCKER_GPU_PATCH = originalEnv; - } + expect(() => + patch.verifyGpuOrExit(() => ({ + status: "failed", + cudaVerified: false, + label: "nvidia-smi when available", + detail: "No devices were found", + at: "2026-07-07T00:00:00.000Z", + })), + ).toThrow("Sandbox GPU proof returned failed status: nvidia-smi when available"); }); }); diff --git a/src/lib/onboard/docker-gpu-sandbox-create-plan.ts b/src/lib/onboard/docker-gpu-sandbox-create-plan.ts new file mode 100644 index 00000000000..85ff02238f9 --- /dev/null +++ b/src/lib/onboard/docker-gpu-sandbox-create-plan.ts @@ -0,0 +1,89 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { type DockerGpuRoutePlan, resolveDockerGpuRoutePlan } from "./docker-gpu-route"; +import { detectWslDockerDesktopStatus } from "./wsl-docker-desktop-gpu"; + +type DockerGpuSandboxConfig = { + sandboxGpuEnabled: boolean; + sandboxGpuDevice?: string | null; + hostGpuPlatform?: string | null; +}; + +type DockerGpuSandboxCreatePlan = { + gpuRoutePlan: DockerGpuRoutePlan; + logMessage: string | null; +}; + +// NemoClaw onboarding is a short-lived process, and the active Docker daemon cannot switch +// between native Linux and Docker Desktop WSL during one run. Cache that stable host fact for the +// process lifetime; tests that substitute the probe explicitly reset it between scenarios. +let cachedDockerDesktopWslRuntime: boolean | null = null; + +export function isDockerDesktopWslRuntime(): boolean { + if (cachedDockerDesktopWslRuntime === null) { + cachedDockerDesktopWslRuntime = detectWslDockerDesktopStatus({}) === "docker-desktop"; + } + return cachedDockerDesktopWslRuntime; +} + +export function resetIsDockerDesktopWslRuntimeCache(): void { + cachedDockerDesktopWslRuntime = null; +} + +/** + * SOURCE_OF_TRUTH_REVIEW (GPU create route selection; #6110) + * invalidState: one attempt combines native `--gpu` with compatibility recreation. + * sourceBoundary: this host-control step selects the route; renderers may only implement it. + * whyNotSourceFix: the shipped suppressGpuFlag seam cannot be removed atomically with consumers. + * regressionTest: Docker GPU route matrix plus the legacy suppression case. + * removalCondition: migrate that seam separately, and retire compatibility after WSL, Jetson, and + * legacy nonzero NEMOCLAW_DOCKER_GPU_PATCH no longer require recreation. + */ +export function resolveDockerGpuSandboxCreatePlan( + config: DockerGpuSandboxConfig, + options: { + dockerDriverGateway: boolean; + dockerDesktopWsl?: boolean; + detectDockerDesktopWsl?: () => boolean; + env?: NodeJS.ProcessEnv; + platform?: NodeJS.Platform; + log?: (message: string) => void; + }, +): DockerGpuSandboxCreatePlan { + const dockerDesktopWsl = + options.dockerDesktopWsl ?? (options.detectDockerDesktopWsl ?? isDockerDesktopWslRuntime)(); + const gpuRoutePlan = resolveDockerGpuRoutePlan(config, { + dockerDriverGateway: options.dockerDriverGateway, + dockerDesktopWsl, + env: options.env, + platform: options.platform, + log: options.log, + }); + const logMessage = config.sandboxGpuEnabled + ? gpuRouteLogMessage(gpuRoutePlan, config.hostGpuPlatform) + : null; + return { gpuRoutePlan, logMessage }; +} + +function gpuRouteLogMessage( + route: DockerGpuRoutePlan, + hostGpuPlatform: string | null | undefined, +): string | null { + switch (route) { + case "none": + return null; + case "compatibility-only": + return hostGpuPlatform === "jetson" + ? " Jetson sandbox GPU enabled; using NVIDIA Container Runtime instead of CDI/--gpus." + : " Docker-driver GPU patch active; allowing /proc writes required by Docker GPU initialization."; + case "native-with-fallback": + return " Operator-authorized GPU fallback enabled; trying native OpenShell injection with one compatibility retry."; + case "native-only": + return " Direct sandbox GPU enabled; allowing OpenShell GPU policy enrichment."; + default: { + const exhaustiveRoute: never = route; + throw new Error(`Unhandled Docker GPU route: ${exhaustiveRoute}`); + } + } +} diff --git a/src/lib/onboard/docker-gpu-sandbox-create-route-plan.test.ts b/src/lib/onboard/docker-gpu-sandbox-create-route-plan.test.ts new file mode 100644 index 00000000000..2ab0b6af31f --- /dev/null +++ b/src/lib/onboard/docker-gpu-sandbox-create-route-plan.test.ts @@ -0,0 +1,133 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { + type DockerGpuRoutePlan, + resolveDockerGpuSandboxCreatePlan, +} from "./docker-gpu-sandbox-create"; + +describe("resolveDockerGpuSandboxCreatePlan", () => { + type RouteCase = { + label: string; + gpuEnabled?: boolean; + hostGpuPlatform?: string; + control?: string; + dockerDriverGateway?: boolean; + dockerDesktopWsl?: boolean; + platform?: NodeJS.Platform; + expected: DockerGpuRoutePlan; + }; + + it.each([ + { label: "GPU disabled", gpuEnabled: false, expected: "none" }, + { label: "ordinary Linux default", expected: "native-only" }, + { + label: "ordinary Linux auto", + control: "auto", + expected: "native-only", + }, + { + label: "ordinary Linux explicit fallback", + control: "fallback", + expected: "native-with-fallback", + }, + { label: "ordinary Linux opt-out", control: "0", expected: "native-only" }, + { + label: "ordinary Linux forced compatibility", + control: "1", + expected: "compatibility-only", + }, + { + label: "ordinary Linux legacy nonzero", + control: "2", + expected: "compatibility-only", + }, + { + label: "non-Docker driver", + dockerDriverGateway: false, + expected: "native-only", + }, + { + label: "non-Linux Docker driver", + platform: "darwin", + expected: "native-only", + }, + { + label: "Docker Desktop WSL default", + dockerDesktopWsl: true, + expected: "compatibility-only", + }, + { + label: "Jetson default", + hostGpuPlatform: "jetson", + expected: "compatibility-only", + }, + { + label: "Jetson auto", + hostGpuPlatform: "jetson", + control: "auto", + expected: "compatibility-only", + }, + { + label: "Jetson opt-out", + hostGpuPlatform: "jetson", + control: "0", + expected: "native-only", + }, + ])("resolves $label to $expected", (testCase) => { + const log = vi.fn(); + const result = resolveDockerGpuSandboxCreatePlan( + { + sandboxGpuEnabled: testCase.gpuEnabled ?? true, + hostGpuPlatform: testCase.hostGpuPlatform, + }, + { + dockerDriverGateway: testCase.dockerDriverGateway ?? true, + dockerDesktopWsl: testCase.dockerDesktopWsl ?? false, + env: { NEMOCLAW_DOCKER_GPU_PATCH: testCase.control }, + platform: testCase.platform ?? "linux", + log, + }, + ); + + expect(result.gpuRoutePlan).toBe(testCase.expected); + }); + + it("ignores opt-out on Docker Desktop WSL", () => { + const log = vi.fn(); + + const result = resolveDockerGpuSandboxCreatePlan( + { sandboxGpuEnabled: true }, + { + dockerDriverGateway: true, + dockerDesktopWsl: true, + env: { NEMOCLAW_DOCKER_GPU_PATCH: "0" }, + platform: "linux", + log, + }, + ); + + expect(result.gpuRoutePlan).toBe("compatibility-only"); + expect(log).toHaveBeenCalledWith(expect.stringContaining("ignored on Docker Desktop WSL")); + }); + + it("forwards the legacy nonzero warning through the create-plan boundary", () => { + const log = vi.fn(); + + const result = resolveDockerGpuSandboxCreatePlan( + { sandboxGpuEnabled: true }, + { + dockerDriverGateway: true, + dockerDesktopWsl: false, + env: { NEMOCLAW_DOCKER_GPU_PATCH: "true" }, + platform: "linux", + log, + }, + ); + + expect(result.gpuRoutePlan).toBe("compatibility-only"); + expect(log).toHaveBeenCalledWith(expect.stringMatching(/unrecognized.*compatibility-only/i)); + }); +}); diff --git a/src/lib/onboard/docker-gpu-sandbox-create.ts b/src/lib/onboard/docker-gpu-sandbox-create.ts index bf58c7d6b7f..dd3c80408af 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create.ts @@ -3,44 +3,42 @@ import { getSandboxFailurePhase } from "../state/gateway"; import type { SandboxGpuProofResult } from "../state/registry"; -import type { - DockerGpuPatchBackend, - DockerGpuPatchDeps, - DockerGpuPatchFailureContext, - DockerGpuPatchMode, - DockerGpuPatchResult, -} from "./docker-gpu-patch"; import { - findOpenShellDockerSandboxContainerIds, getDockerGpuSupervisorReconnectTimeoutSecs, printDockerGpuPatchFailureAndExit, printDockerGpuProofFailure, printDockerGpuReadinessFailure, recreateOpenShellDockerSandboxWithGpu, - shouldApplyDockerGpuPatch, waitForOpenShellSupervisorReconnect, } from "./docker-gpu-patch"; import { finalizeDockerGpuPatchBackup } from "./docker-gpu-patch-finalize"; +import type { + DockerGpuPatchBackend, + DockerGpuPatchDeps, + DockerGpuPatchFailureContext, + DockerGpuPatchMode, + DockerGpuPatchResult, +} from "./docker-gpu-patch-types"; import { captureDockerGpuPreRollbackDiagnostics } from "./docker-gpu-pre-rollback-diagnostics"; +import type { SelectedDockerGpuRoute } from "./docker-gpu-route"; +import { adaptDockerGpuRouteForPatch } from "./docker-gpu-route-patch-adapter"; +import { isDockerDesktopWslRuntime } from "./docker-gpu-sandbox-create-plan"; import { createDockerSandboxRecreator, type RecreateGpuPatchFn, type RecreateStartupPatchFn, } from "./docker-startup-command-sandbox-create"; -import { detectWslDockerDesktopStatus } from "./wsl-docker-desktop-gpu"; +import { findOpenShellDockerSandboxContainerIds } from "./openshell-docker-sandbox-containers"; -let cachedDockerDesktopWslRuntime: boolean | null = null; - -export function isDockerDesktopWslRuntime(): boolean { - if (cachedDockerDesktopWslRuntime === null) { - cachedDockerDesktopWslRuntime = detectWslDockerDesktopStatus({}) === "docker-desktop"; - } - return cachedDockerDesktopWslRuntime; -} - -export function resetIsDockerDesktopWslRuntimeCache(): void { - cachedDockerDesktopWslRuntime = null; -} +export type { + DockerGpuRoutePlan, + SelectedDockerGpuRoute, +} from "./docker-gpu-route"; +export { + isDockerDesktopWslRuntime, + resetIsDockerDesktopWslRuntimeCache, + resolveDockerGpuSandboxCreatePlan, +} from "./docker-gpu-sandbox-create-plan"; type DockerGpuSandboxCreateDeps = Pick< DockerGpuPatchDeps, @@ -61,7 +59,7 @@ type PatchFailureExitFn = ( ) => void; type DockerGpuSandboxCreatePatchOptions = { - enabled: boolean; + route: SelectedDockerGpuRoute; persistStartupCommand?: boolean; sandboxName: string; gpuDevice?: string | null; @@ -92,17 +90,6 @@ type DockerGpuSandboxCreatePatchOptions = { }; }; -type DockerGpuSandboxConfig = { - sandboxGpuEnabled: boolean; - sandboxGpuDevice?: string | null; - hostGpuPlatform?: string | null; -}; - -type DockerGpuSandboxCreatePlan = { - useDockerGpuPatch: boolean; - logMessage: string | null; -}; - export type DockerGpuSandboxCreatePatch = { maybeApplyDuringCreate: () => void; createFailureMessage: () => string | null; @@ -131,6 +118,7 @@ export type DockerGpuSandboxCreatePatch = { export function createDockerGpuSandboxCreatePatch( options: DockerGpuSandboxCreatePatchOptions, ): DockerGpuSandboxCreatePatch { + const routeAdapter = adaptDockerGpuRouteForPatch(options.route); let result: DockerGpuPatchResult | null = null; let patchError: unknown = null; let needsSupervisorWait = false; @@ -155,10 +143,10 @@ export function createDockerGpuSandboxCreatePatch( backend: options.backend, dockerDesktopWsl: options.dockerDesktopWsl ?? isDockerDesktopWslRuntime(), }; - const patchEnabled = options.enabled || options.persistStartupCommand === true; - const patchTarget = options.enabled ? "NVIDIA GPU access" : "restart-safe startup"; + const patchEnabled = routeAdapter.enabled || options.persistStartupCommand === true; + const patchTarget = routeAdapter.enabled ? "NVIDIA GPU access" : "restart-safe startup"; const recreateSelectedPatch = createDockerSandboxRecreator({ - gpuEnabled: options.enabled, + gpuEnabled: routeAdapter.enabled, gpuOptions: applyOptions, startupCommand: options.openshellSandboxCommand, recreateGpu: recreatePatch, @@ -187,7 +175,7 @@ export function createDockerGpuSandboxCreatePatch( createFailureMessage() { if (!patchError) return null; - return options.enabled + return routeAdapter.enabled ? "Docker GPU patch failed while OpenShell sandbox create was still waiting." : "Docker startup-command patch failed while OpenShell sandbox create was still waiting."; }, @@ -197,6 +185,7 @@ export function createDockerGpuSandboxCreatePatch( onPatchFailureExit(options.sandboxName, patchError, { runCaptureOpenshell: options.deps.runCaptureOpenshell, dockerCapture: options.deps.dockerCapture, + additionalSummaryLines: routeAdapter.additionalSummaryLines, }); }, @@ -211,6 +200,7 @@ export function createDockerGpuSandboxCreatePatch( onPatchFailureExit(options.sandboxName, error, { runCaptureOpenshell: options.deps.runCaptureOpenshell, dockerCapture: options.deps.dockerCapture, + additionalSummaryLines: routeAdapter.additionalSummaryLines, }); } }, @@ -248,7 +238,30 @@ export function createDockerGpuSandboxCreatePatch( const finalizeOutcome = result ? finalizeBackup({ result, supervisorReady }, options.deps) : null; - if (supervisorReady) return; + if (supervisorReady) { + if (finalizeOutcome && !finalizeOutcome.backupRemoved) { + onPatchFailureExit( + options.sandboxName, + new Error( + "OpenShell supervisor reconnected, but the recreated backup container could not be removed.", + ), + { + runCaptureOpenshell: options.deps.runCaptureOpenshell, + dockerCapture: options.deps.dockerCapture, + additionalSummaryLines: routeAdapter.additionalSummaryLines, + context: { + sandboxName: options.sandboxName, + oldContainerId: result?.oldContainerId, + newContainerId: result?.newContainerId, + backupContainerName: result?.backupContainerName, + selectedMode: result?.mode ?? null, + rolledBack: false, + }, + }, + ); + } + return; + } const failureMessage = (() => { if (!finalizeOutcome) { return "OpenShell supervisor did not reconnect to the recreated container."; @@ -260,6 +273,7 @@ export function createDockerGpuSandboxCreatePatch( onPatchFailureExit(options.sandboxName, new Error(failureMessage), { runCaptureOpenshell: options.deps.runCaptureOpenshell, dockerCapture: options.deps.dockerCapture, + additionalSummaryLines: routeAdapter.additionalSummaryLines, context: { sandboxName: options.sandboxName, oldContainerId: result?.oldContainerId, @@ -276,11 +290,12 @@ export function createDockerGpuSandboxCreatePatch( }, printReadinessFailureIfEnabled() { - if (!options.enabled) return; + if (!routeAdapter.enabled) return; printDockerGpuReadinessFailure(options.sandboxName, result?.mode ?? null, { runCaptureOpenshell: options.deps.runCaptureOpenshell, dockerCapture: options.deps.dockerCapture, context: buildFailureContext(options.sandboxName, result), + additionalSummaryLines: routeAdapter.additionalSummaryLines, }); }, @@ -295,7 +310,7 @@ export function createDockerGpuSandboxCreatePatch( // (#4316). const sandboxName = options.sandboxName; const failureContext = buildFailureContext(sandboxName, result); - if (options.enabled && options.deps.runCaptureOpenshell) { + if (routeAdapter.enabled && options.deps.runCaptureOpenshell) { const list = options.deps.runCaptureOpenshell(["sandbox", "list"], { ignoreError: true, }); @@ -313,18 +328,26 @@ export function createDockerGpuSandboxCreatePatch( runCaptureOpenshell: options.deps.runCaptureOpenshell, dockerCapture: options.deps.dockerCapture, context: failureContext, + additionalSummaryLines: routeAdapter.additionalSummaryLines, }, ); process.exit(1); } } try { - return verifyDirectSandboxGpu(sandboxName); + const proof = verifyDirectSandboxGpu(sandboxName); + if (proof.status === "failed") { + const label = proof.label ? `: ${proof.label}` : ""; + const detail = proof.detail ? ` (${proof.detail})` : ""; + throw new Error(`Sandbox GPU proof returned failed status${label}${detail}`); + } + return proof; } catch (error) { printDockerGpuProofFailure(sandboxName, error, result?.mode ?? null, { runCaptureOpenshell: options.deps.runCaptureOpenshell, dockerCapture: options.deps.dockerCapture, - context: options.enabled ? failureContext : null, + context: routeAdapter.enabled ? failureContext : null, + additionalSummaryLines: routeAdapter.additionalSummaryLines, }); throw error; } @@ -347,54 +370,3 @@ function buildFailureContext( selectedMode: result?.mode ?? null, }; } - -export function shouldUseDockerGpuPatchForCreate( - config: DockerGpuSandboxConfig, - options: { - dockerDriverGateway: boolean; - dockerDesktopWsl?: boolean; - platform?: NodeJS.Platform; - log?: (message: string) => void; - }, -): boolean { - const enabled = shouldApplyDockerGpuPatch(config, { - dockerDriverGateway: options.dockerDriverGateway, - dockerDesktopWsl: options.dockerDesktopWsl, - platform: options.platform, - log: options.log, - }); - if (enabled) { - options.log?.( - config.hostGpuPlatform === "jetson" - ? " Jetson Docker GPU patch active; creating sandbox first, then recreating the Docker container with NVIDIA runtime GPU access." - : " Docker-driver GPU patch active; creating sandbox first, then recreating the Docker container with GPU access.", - ); - } - return enabled; -} - -export function resolveDockerGpuSandboxCreatePlan( - config: DockerGpuSandboxConfig, - options: { - dockerDriverGateway: boolean; - dockerDesktopWsl?: boolean; - detectDockerDesktopWsl?: () => boolean; - platform?: NodeJS.Platform; - }, -): DockerGpuSandboxCreatePlan { - const dockerDesktopWsl = - options.dockerDesktopWsl ?? (options.detectDockerDesktopWsl ?? isDockerDesktopWslRuntime)(); - const useDockerGpuPatch = shouldUseDockerGpuPatchForCreate(config, { - dockerDriverGateway: options.dockerDriverGateway, - dockerDesktopWsl, - platform: options.platform, - }); - const logMessage = config.sandboxGpuEnabled - ? useDockerGpuPatch - ? config.hostGpuPlatform === "jetson" - ? " Jetson sandbox GPU enabled; using NVIDIA Container Runtime instead of CDI/--gpus." - : " Docker-driver GPU patch active; allowing /proc writes required by Docker GPU initialization." - : " Direct sandbox GPU enabled; allowing OpenShell GPU policy enrichment." - : null; - return { useDockerGpuPatch, logMessage }; -} diff --git a/src/lib/onboard/docker-gpu-supervisor-reconnect.test.ts b/src/lib/onboard/docker-gpu-supervisor-reconnect.test.ts index c0fae2d0ce1..ed7a1b82e1e 100644 --- a/src/lib/onboard/docker-gpu-supervisor-reconnect.test.ts +++ b/src/lib/onboard/docker-gpu-supervisor-reconnect.test.ts @@ -5,6 +5,7 @@ import { describe, expect, it, vi } from "vitest"; import { getDockerGpuSupervisorReconnectErrorDebouncePolls, + getDockerGpuSupervisorReconnectTimeoutSecs, waitForOpenShellSupervisorReconnect, } from "./docker-gpu-supervisor-reconnect"; @@ -16,6 +17,36 @@ import { // GPU sandbox whose container is running and whose supervisor has already // logged `LIFECYCLE:INSTALL OpenShell Sandbox Supervisor success`. describe("docker-gpu-supervisor-reconnect Error-phase debounce", () => { + it("uses a Docker-GPU-specific supervisor reconnect wait with an override", () => { + expect(getDockerGpuSupervisorReconnectTimeoutSecs(180, {})).toBe(900); + expect(getDockerGpuSupervisorReconnectTimeoutSecs(600, {})).toBe(900); + expect(getDockerGpuSupervisorReconnectTimeoutSecs(1200, {})).toBe(1200); + expect( + getDockerGpuSupervisorReconnectTimeoutSecs(180, { + NEMOCLAW_DOCKER_GPU_SUPERVISOR_RECONNECT_TIMEOUT: "30", + }), + ).toBe(30); + }); + + it("short-circuits the supervisor-reconnect wait when the sandbox enters Error phase", () => { + const runOpenshell = vi.fn(() => ({ status: 1, stderr: "sandbox not ready" })); + const listOutputs = ["alpha Provisioning 1s ago", "alpha Error 3s ago"]; + let index = 0; + const runCaptureOpenshell = vi.fn(() => listOutputs[Math.min(index++, listOutputs.length - 1)]); + const sleep = vi.fn(); + + const ok = waitForOpenShellSupervisorReconnect("alpha", 600, { + runOpenshell, + runCaptureOpenshell, + sleep, + errorPhaseDebouncePolls: 1, + }); + + expect(ok).toBe(false); + expect(runOpenshell).toHaveBeenCalledTimes(2); + expect(sleep).toHaveBeenCalledTimes(1); + }); + it("absorbs a transient Error phase shorter than the debounce window", () => { const execOutputs = [ { status: 1, stderr: "sandbox not ready" }, @@ -67,6 +98,21 @@ describe("docker-gpu-supervisor-reconnect Error-phase debounce", () => { expect(sleep).toHaveBeenCalledTimes(2); }); + it("does not accept a supervisor exec with no exit status", () => { + const runOpenshell = vi.fn(() => ({ status: null, stderr: "timed out" })); + const runCaptureOpenshell = vi.fn(() => "alpha Error 1s ago"); + + const ok = waitForOpenShellSupervisorReconnect("alpha", 600, { + runOpenshell, + runCaptureOpenshell, + sleep: vi.fn(), + errorPhaseDebouncePolls: 1, + }); + + expect(ok).toBe(false); + expect(runOpenshell).toHaveBeenCalledOnce(); + }); + it("resets the consecutive-Error counter when the phase recovers", () => { // Error, Error, Provisioning (counter resets), Error, Error, Error // -> bails out on the 3rd post-recovery Error, not earlier. diff --git a/src/lib/onboard/docker-gpu-supervisor-reconnect.ts b/src/lib/onboard/docker-gpu-supervisor-reconnect.ts index ad93417c5fe..5ba17fa1987 100644 --- a/src/lib/onboard/docker-gpu-supervisor-reconnect.ts +++ b/src/lib/onboard/docker-gpu-supervisor-reconnect.ts @@ -24,9 +24,10 @@ * recovers to Ready is the runtime evidence required. */ +import { hasZeroDockerExitStatus } from "./docker-command-result"; +import { DOCKER_GPU_PATCH_TIMEOUT_MS } from "./docker-gpu-patch-constants"; import { envInt } from "./env"; -const DOCKER_GPU_PATCH_TIMEOUT_MS = 30_000; const DOCKER_GPU_SUPERVISOR_RECONNECT_MIN_SECS = 900; // Default consecutive Error-phase polls required before fast-fail. With a // 2-second poll interval this is ~2 minutes of sustained Error, leaving @@ -76,10 +77,6 @@ function defaultSleep(seconds: number): void { Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, Math.max(0, seconds) * 1000); } -function isZeroStatus(result: DockerRunResult | null | undefined): boolean { - return result?.status === 0; -} - const ANSI_RE = /\x1b\[[0-9;]*m/g; function parseSandboxListFailurePhase(output: string, sandboxName: string): string | null { @@ -130,7 +127,7 @@ export function waitForOpenShellSupervisorReconnect( suppressOutput: true, timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, }); - if (isZeroStatus(result)) return true; + if (hasZeroDockerExitStatus(result)) return true; if ( deps.runCaptureOpenshell && sandboxListShowsErrorPhase(sandboxName, deps.runCaptureOpenshell) diff --git a/src/lib/onboard/docker-startup-command-sandbox-create.test.ts b/src/lib/onboard/docker-startup-command-sandbox-create.test.ts index a84482ac09e..b9a166c9e5d 100644 --- a/src/lib/onboard/docker-startup-command-sandbox-create.test.ts +++ b/src/lib/onboard/docker-startup-command-sandbox-create.test.ts @@ -86,7 +86,7 @@ describe("Docker startup-command sandbox creation", () => { }; const recreatePatch = vi.fn(); const patch = createDockerGpuSandboxCreatePatch({ - enabled: false, + route: "native", persistStartupCommand: true, sandboxName: "alpha", openshellSandboxCommand: ["env", "nemoclaw-start"], @@ -113,7 +113,7 @@ describe("Docker startup-command sandbox creation", () => { const finalizeBackup = vi.fn(() => ({ backupRemoved: false, rolledBack: true })); const onPatchFailureExit = vi.fn(); const patch = createDockerGpuSandboxCreatePatch({ - enabled: false, + route: "native", persistStartupCommand: true, sandboxName: "alpha", openshellSandboxCommand: ["env", "nemoclaw-start"], @@ -148,7 +148,7 @@ describe("Docker startup-command sandbox creation", () => { const deps = makeDeps(); const onPatchFailureExit = vi.fn(); const patch = createDockerGpuSandboxCreatePatch({ - enabled: false, + route: "native", persistStartupCommand: true, sandboxName: "alpha", openshellSandboxCommand: ["env", "nemoclaw-start"], diff --git a/src/lib/onboard/initial-policy.test.ts b/src/lib/onboard/initial-policy.test.ts index 1567aa363b2..556f780f715 100644 --- a/src/lib/onboard/initial-policy.test.ts +++ b/src/lib/onboard/initial-policy.test.ts @@ -267,6 +267,35 @@ network_policies: expect(prepared.cleanup?.()).toBe(true); }); + it("removes all temporary policies when a later materialization write fails", () => { + const basePolicyPath = tmpPolicy("version: 1\nnetwork_policies:\n base: {}\n"); + const realWriteFileSync = fs.writeFileSync; + const mkdtempSpy = vi.spyOn(fs, "mkdtempSync"); + let createdDirs: string[] = []; + const writeSpy = vi + .spyOn(fs, "writeFileSync") + .mockImplementationOnce((...args) => realWriteFileSync(...args)) + .mockImplementationOnce(() => { + throw new Error("policy write failed"); + }); + + try { + expect(() => + prepareInitialSandboxCreatePolicy(basePolicyPath, [], { + directGpu: true, + additionalPresets: ["slack"], + }), + ).toThrow("policy write failed"); + } finally { + createdDirs = mkdtempSpy.mock.results.map(({ value }) => String(value)); + writeSpy.mockRestore(); + mkdtempSpy.mockRestore(); + } + + expect(createdDirs).toHaveLength(2); + for (const dir of createdDirs) expect(fs.existsSync(dir)).toBe(false); + }); + it("merges openclaw-diagnostics-otel-local at create time when OTEL is enabled and the tier is known non-restricted", () => { const basePolicyPath = tmpPolicy("version: 1\nnetwork_policies:\n base: {}\n"); process.env.NEMOCLAW_OPENCLAW_OTEL = "1"; diff --git a/src/lib/onboard/initial-policy.ts b/src/lib/onboard/initial-policy.ts index fe230e5113e..54690a3f690 100644 --- a/src/lib/onboard/initial-policy.ts +++ b/src/lib/onboard/initial-policy.ts @@ -127,27 +127,37 @@ export function buildDirectSandboxGpuProofCommands( ]; } +function createPolicyTempCleanup(policyPath: string, expectedPrefix: string): () => boolean { + return () => { + try { + cleanupTempDir(policyPath, expectedPrefix); + return true; + } catch { + return false; + } + }; +} + function prepareDirectGpuSandboxPolicy( basePolicyPath: string, options: DirectGpuPolicyOptions = {}, ): InitialSandboxPolicy { const basePolicy = fs.readFileSync(basePolicyPath, "utf-8"); const policyPath = secureTempFile("nemoclaw-gpu-policy", ".yaml"); - fs.writeFileSync(policyPath, buildDirectGpuPolicyYaml(basePolicy, options), { - encoding: "utf-8", - mode: 0o600, - }); + const cleanup = createPolicyTempCleanup(policyPath, "nemoclaw-gpu-policy"); + try { + fs.writeFileSync(policyPath, buildDirectGpuPolicyYaml(basePolicy, options), { + encoding: "utf-8", + mode: 0o600, + }); + } catch (error) { + cleanup(); + throw error; + } return { policyPath, appliedPresets: [], - cleanup: () => { - try { - cleanupTempDir(policyPath, "nemoclaw-gpu-policy"); - return true; - } catch { - return false; - } - }, + cleanup, }; } @@ -216,117 +226,117 @@ export function prepareInitialSandboxCreatePolicy( const cleanupFns = directGpuPolicy?.cleanup ? [directGpuPolicy.cleanup] : []; const buildCleanup = () => cleanupFns.length > 0 ? () => cleanupFns.map((cleanup) => cleanup()).every(Boolean) : undefined; - // Fail closed: the OpenClaw OTEL preset is added at create time only when the - // selected policy tier is known and is not Restricted. When the tier is null - // (interactive flow that selects later) the preset is deferred to the - // post-boot policy step, so a later Restricted selection cannot leave a - // transient host-local OTLP egress allowance during sandbox boot. The same - // suppression filter still runs so an explicit `policyTier: "restricted"` - // (non-interactive flow) drops openclaw-pricing from `additionalPresets`. - const tierKnown = typeof options.policyTier === "string" && options.policyTier.length > 0; - const otelCreateTimePresets = - tierKnown && options.policyTier !== "restricted" - ? requiredOpenclawOtelPolicyPresets(options.agentName ?? "openclaw") - : []; - const isHermesPolicyFromPath = isHermesPolicyPath(basePolicyPath); - const isHermesPolicy = options.agentName === "hermes" || isHermesPolicyFromPath; - const policyAgent = options.agentName ?? (isHermesPolicyFromPath ? "hermes" : null); - const messagingCreateTimePresets = isHermesPolicy - ? allMessagingChannelPolicyPresets(activeMessagingChannels) - : requiredMessagingChannelPolicyPresets(activeMessagingChannels); - const requestedCreateTimePresets = filterSuppressedAgentRequiredPresets( - [ - ...new Set([ - ...messagingCreateTimePresets, - ...otelCreateTimePresets, - ...(options.additionalPresets || []), - ]), - ], - options.policyTier ?? null, - options.agentName ?? null, - ); - const dedupe = (values: string[]) => [...new Set(values.filter(Boolean))]; - - let basePolicy = fs.readFileSync(effectiveBasePolicyPath, "utf-8"); - if (isHermesPolicy) { - const filtered = filterHermesInactiveMessagingPolicies(basePolicy, activeMessagingChannels); - if (filtered.changed) { - const policyPath = secureTempFile("nemoclaw-agent-policy", ".yaml"); - fs.writeFileSync(policyPath, filtered.content, { encoding: "utf-8", mode: 0o600 }); - cleanupFns.push(() => { - try { - cleanupTempDir(policyPath, "nemoclaw-agent-policy"); - return true; - } catch { - return false; - } - }); - effectiveBasePolicyPath = policyPath; - basePolicy = filtered.content; + const cleanupOnError = () => { + for (const cleanup of [...cleanupFns].reverse()) { + try { + cleanup(); + } catch { + // Preserve the policy preparation error; cleanup is best effort and fail-closed upstream. + } } - } + }; + try { + // Fail closed: the OpenClaw OTEL preset is added at create time only when the + // selected policy tier is known and is not Restricted. When the tier is null + // (interactive flow that selects later) the preset is deferred to the + // post-boot policy step, so a later Restricted selection cannot leave a + // transient host-local OTLP egress allowance during sandbox boot. The same + // suppression filter still runs so an explicit `policyTier: "restricted"` + // (non-interactive flow) drops openclaw-pricing from `additionalPresets`. + const tierKnown = typeof options.policyTier === "string" && options.policyTier.length > 0; + const otelCreateTimePresets = + tierKnown && options.policyTier !== "restricted" + ? requiredOpenclawOtelPolicyPresets(options.agentName ?? "openclaw") + : []; + const isHermesPolicyFromPath = isHermesPolicyPath(basePolicyPath); + const isHermesPolicy = options.agentName === "hermes" || isHermesPolicyFromPath; + const policyAgent = options.agentName ?? (isHermesPolicyFromPath ? "hermes" : null); + const messagingCreateTimePresets = isHermesPolicy + ? allMessagingChannelPolicyPresets(activeMessagingChannels) + : requiredMessagingChannelPolicyPresets(activeMessagingChannels); + const requestedCreateTimePresets = filterSuppressedAgentRequiredPresets( + [ + ...new Set([ + ...messagingCreateTimePresets, + ...otelCreateTimePresets, + ...(options.additionalPresets || []), + ]), + ], + options.policyTier ?? null, + options.agentName ?? null, + ); + const dedupe = (values: string[]) => [...new Set(values.filter(Boolean))]; - const basePolicyNames = getNetworkPolicyNames(basePolicy); - if (basePolicyNames === null) { - return { - policyPath: effectiveBasePolicyPath, - appliedPresets: [], - cleanup: buildCleanup(), - }; - } - const existingChannelPresets = activeMessagingChannels.filter((channel) => - basePolicyNames.has(channel), - ); + let basePolicy = fs.readFileSync(effectiveBasePolicyPath, "utf-8"); + if (isHermesPolicy) { + const filtered = filterHermesInactiveMessagingPolicies(basePolicy, activeMessagingChannels); + if (filtered.changed) { + const policyPath = secureTempFile("nemoclaw-agent-policy", ".yaml"); + cleanupFns.push(createPolicyTempCleanup(policyPath, "nemoclaw-agent-policy")); + fs.writeFileSync(policyPath, filtered.content, { encoding: "utf-8", mode: 0o600 }); + effectiveBasePolicyPath = policyPath; + basePolicy = filtered.content; + } + } - if (requestedCreateTimePresets.length === 0) { - return { - policyPath: effectiveBasePolicyPath, - appliedPresets: dedupe(existingChannelPresets), - cleanup: buildCleanup(), - }; - } + const basePolicyNames = getNetworkPolicyNames(basePolicy); + if (basePolicyNames === null) { + return { + policyPath: effectiveBasePolicyPath, + appliedPresets: [], + cleanup: buildCleanup(), + }; + } + const existingChannelPresets = activeMessagingChannels.filter((channel) => + basePolicyNames.has(channel), + ); - const existingCreateTimePresets = requestedCreateTimePresets.filter((preset) => - basePolicyNames.has(preset), - ); - const createTimePresets = requestedCreateTimePresets.filter( - (preset) => !basePolicyNames.has(preset), - ); - if (createTimePresets.length === 0) { - return { - policyPath: effectiveBasePolicyPath, - appliedPresets: dedupe([...existingChannelPresets, ...existingCreateTimePresets]), - cleanup: buildCleanup(), - }; - } + if (requestedCreateTimePresets.length === 0) { + return { + policyPath: effectiveBasePolicyPath, + appliedPresets: dedupe(existingChannelPresets), + cleanup: buildCleanup(), + }; + } - const mergedPolicy = policies.mergePresetNamesIntoPolicy(basePolicy, createTimePresets, { - agent: policyAgent, - }); - if (mergedPolicy.missingPresets.length > 0) { - throw new Error( - `Cannot prepare sandbox create policy; missing policy preset(s): ${mergedPolicy.missingPresets.join(", ")}`, + const existingCreateTimePresets = requestedCreateTimePresets.filter((preset) => + basePolicyNames.has(preset), ); - } + const createTimePresets = requestedCreateTimePresets.filter( + (preset) => !basePolicyNames.has(preset), + ); + if (createTimePresets.length === 0) { + return { + policyPath: effectiveBasePolicyPath, + appliedPresets: dedupe([...existingChannelPresets, ...existingCreateTimePresets]), + cleanup: buildCleanup(), + }; + } - const policyPath = secureTempFile("nemoclaw-initial-policy", ".yaml"); - fs.writeFileSync(policyPath, mergedPolicy.policy, { encoding: "utf-8", mode: 0o600 }); - cleanupFns.push(() => { - try { - cleanupTempDir(policyPath, "nemoclaw-initial-policy"); - return true; - } catch { - return false; + const mergedPolicy = policies.mergePresetNamesIntoPolicy(basePolicy, createTimePresets, { + agent: policyAgent, + }); + if (mergedPolicy.missingPresets.length > 0) { + throw new Error( + `Cannot prepare sandbox create policy; missing policy preset(s): ${mergedPolicy.missingPresets.join(", ")}`, + ); } - }); - return { - policyPath, - appliedPresets: dedupe([ - ...existingChannelPresets, - ...existingCreateTimePresets, - ...mergedPolicy.appliedPresets, - ]), - cleanup: buildCleanup(), - }; + const policyPath = secureTempFile("nemoclaw-initial-policy", ".yaml"); + cleanupFns.push(createPolicyTempCleanup(policyPath, "nemoclaw-initial-policy")); + fs.writeFileSync(policyPath, mergedPolicy.policy, { encoding: "utf-8", mode: 0o600 }); + + return { + policyPath, + appliedPresets: dedupe([ + ...existingChannelPresets, + ...existingCreateTimePresets, + ...mergedPolicy.appliedPresets, + ]), + cleanup: buildCleanup(), + }; + } catch (error) { + cleanupOnError(); + throw error; + } } diff --git a/src/lib/onboard/openshell-docker-sandbox-containers.test.ts b/src/lib/onboard/openshell-docker-sandbox-containers.test.ts new file mode 100644 index 00000000000..e26eb9b8ec5 --- /dev/null +++ b/src/lib/onboard/openshell-docker-sandbox-containers.test.ts @@ -0,0 +1,239 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; +import { queryOpenShellDockerSandboxRuntimeSnapshot } from "./openshell-docker-sandbox-containers"; + +const IMAGE_ID = `sha256:${"a".repeat(64)}`; +const BOOKKEEPING_IMAGE_REF = "openshell/sandbox-from:alpha"; +const EMPTY_RUNTIME_FIELDS = [IMAGE_ID, BOOKKEEPING_IMAGE_REF, "", null, [], "runc"]; + +function querySnapshot(fields: unknown) { + const dockerRun = vi + .fn() + .mockReturnValueOnce({ status: 0, stdout: "container-a\n", stderr: "" }) + .mockReturnValueOnce({ status: 0, stdout: JSON.stringify(fields), stderr: "" }); + return { + dockerRun, + result: queryOpenShellDockerSandboxRuntimeSnapshot("alpha", { dockerRun }), + }; +} + +describe("queryOpenShellDockerSandboxRuntimeSnapshot", () => { + it("returns immutable identity, bookkeeping ref, and safe absence from one exact container", () => { + const { dockerRun, result } = querySnapshot(EMPTY_RUNTIME_FIELDS); + + expect(result).toEqual({ + ok: true, + imageId: IMAGE_ID, + bookkeepingImageRef: BOOKKEEPING_IMAGE_REF, + stateError: "", + deviceRequests: null, + devices: [], + runtime: "runc", + nativeGpuAttachmentState: "absent", + containerId: "container-a", + }); + expect(dockerRun).toHaveBeenLastCalledWith( + [ + "inspect", + "--type", + "container", + "--format", + "[{{json .Image}},{{json .Config.Image}},{{json .State.Error}},{{json .HostConfig.DeviceRequests}},{{json .HostConfig.Devices}},{{json .HostConfig.Runtime}}]", + "container-a", + ], + expect.objectContaining({ ignoreError: true }), + ); + }); + + it.each([ + [ + "Docker GPU capability request", + [ + { + Driver: "", + Count: -1, + DeviceIDs: null, + Capabilities: [["gpu"]], + Options: {}, + }, + ], + [], + "runc", + ], + [ + "NVIDIA CDI request", + [ + { + Driver: "cdi", + Count: 0, + DeviceIDs: ["nvidia.com/gpu=all"], + Capabilities: null, + Options: {}, + }, + ], + [], + "runc", + ], + [ + "direct NVIDIA device", + null, + [ + { + PathOnHost: "/dev/nvidia0", + PathInContainer: "/dev/nvidia0", + CgroupPermissions: "rwm", + }, + ], + "runc", + ], + [ + "DRI device", + null, + [ + { + PathOnHost: "/dev/dri/renderD128", + PathInContainer: "/dev/dri/renderD128", + CgroupPermissions: "rwm", + }, + ], + "runc", + ], + [ + "Jetson device", + null, + [ + { + PathOnHost: "/dev/nvhost-gpu", + PathInContainer: "/dev/nvhost-gpu", + CgroupPermissions: "rwm", + }, + ], + "runc", + ], + ["NVIDIA runtime", null, [], "nvidia"], + ])("detects a host-configured GPU attachment from %s", (_label, requests, devices, runtime) => { + const { result } = querySnapshot([ + IMAGE_ID, + BOOKKEEPING_IMAGE_REF, + "", + requests, + devices, + runtime, + ]); + + expect(result).toMatchObject({ + ok: true, + nativeGpuAttachmentState: "present", + }); + }); + + it.each([ + ["unknown runtime", null, [], "nvidia-container-runtime"], + [ + "non-NVIDIA CDI request", + [ + { + Driver: "cdi", + Count: 0, + DeviceIDs: ["example.com/widget=all"], + Capabilities: null, + Options: {}, + }, + ], + [], + "runc", + ], + [ + "unrecognized direct device", + null, + [ + { + PathOnHost: "/dev/custom-accelerator0", + PathInContainer: "/dev/custom-accelerator0", + CgroupPermissions: "rwm", + }, + ], + "runc", + ], + ])("keeps well-formed open-world GPU configuration %s unknown", (_label, requests, devices, runtime) => { + const { result } = querySnapshot([ + IMAGE_ID, + BOOKKEEPING_IMAGE_REF, + "", + requests, + devices, + runtime, + ]); + + expect(result).toMatchObject({ + ok: true, + nativeGpuAttachmentState: "unknown", + }); + }); + + it.each([ + ["zero", ""], + ["multiple", "container-a\ncontainer-b\n"], + ])("refuses %s labeled containers", (_label, ids) => { + const dockerRun = vi.fn(() => ({ status: 0, stdout: ids, stderr: "" })); + + expect(queryOpenShellDockerSandboxRuntimeSnapshot("alpha", { dockerRun })).toEqual({ + ok: false, + error: `expected one labeled sandbox container, found ${ids ? 2 : 0}`, + }); + expect(dockerRun).toHaveBeenCalledOnce(); + }); + + it.each([ + [ + "mutable retry identity", + ["registry.example/team/image:latest", BOOKKEEPING_IMAGE_REF, "", null, [], "runc"], + ], + ["short image ID", ["sha256:abc", BOOKKEEPING_IMAGE_REF, "", null, [], "runc"]], + ["unsafe bookkeeping ref", [IMAGE_ID, "image:tag with-space", "", null, [], "runc"]], + ["malformed device requests", [IMAGE_ID, BOOKKEEPING_IMAGE_REF, "", [{}], [], "runc"]], + [ + "malformed GPU capabilities", + [ + IMAGE_ID, + BOOKKEEPING_IMAGE_REF, + "", + [ + { + Driver: "", + Count: -1, + DeviceIDs: null, + Capabilities: ["gpu"], + Options: {}, + }, + ], + [], + "runc", + ], + ], + ["malformed device mappings", [IMAGE_ID, BOOKKEEPING_IMAGE_REF, "", null, [{}], "runc"]], + ["malformed runtime", [IMAGE_ID, BOOKKEEPING_IMAGE_REF, "", null, [], null]], + ["wrong field count", [IMAGE_ID, BOOKKEEPING_IMAGE_REF]], + ])("refuses %s instead of proving GPU attachment absence", (_label, fields) => { + const { result } = querySnapshot(fields); + + expect(result).toEqual({ + ok: false, + error: "docker inspect returned malformed runtime metadata", + }); + }); + + it("refuses malformed inspect JSON", () => { + const dockerRun = vi + .fn() + .mockReturnValueOnce({ status: 0, stdout: "container-a\n", stderr: "" }) + .mockReturnValueOnce({ status: 0, stdout: "not-json", stderr: "" }); + + expect(queryOpenShellDockerSandboxRuntimeSnapshot("alpha", { dockerRun })).toEqual({ + ok: false, + error: "docker inspect returned malformed runtime metadata", + }); + }); +}); diff --git a/src/lib/onboard/openshell-docker-sandbox-containers.ts b/src/lib/onboard/openshell-docker-sandbox-containers.ts new file mode 100644 index 00000000000..2c045cdb086 --- /dev/null +++ b/src/lib/onboard/openshell-docker-sandbox-containers.ts @@ -0,0 +1,309 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { dockerCapture, dockerRun } from "../adapters/docker"; +import type { DockerGpuPatchDeps } from "./docker-gpu-patch-types"; + +export const OPENSHELL_MANAGED_BY_LABEL = "openshell.ai/managed-by"; +export const OPENSHELL_MANAGED_BY_VALUE = "openshell"; +export const OPENSHELL_SANDBOX_NAME_LABEL = "openshell.ai/sandbox-name"; + +const DOCKER_SANDBOX_QUERY_TIMEOUT_MS = 30_000; + +type DockerSandboxContainerQueryDeps = Pick; + +function sandboxContainerFilterArgs(sandboxName: string): string[] { + return [ + "ps", + "-a", + "--no-trunc", + "--filter", + `label=${OPENSHELL_MANAGED_BY_LABEL}=${OPENSHELL_MANAGED_BY_VALUE}`, + "--filter", + `label=${OPENSHELL_SANDBOX_NAME_LABEL}=${sandboxName}`, + ]; +} + +function commandResultText(result: { + stdout?: string | Buffer | null; + stderr?: string | Buffer | null; +}): string { + return `${String(result.stderr || "")} ${String(result.stdout || "")}`.trim(); +} + +/** Best-effort labeled-container lookup used by patch discovery and diagnostics. */ +export function findOpenShellDockerSandboxContainerIds( + sandboxName: string, + deps: DockerSandboxContainerQueryDeps = {}, +): string[] { + const capture = deps.dockerCapture ?? dockerCapture; + const output = capture([...sandboxContainerFilterArgs(sandboxName), "--format", "{{.ID}}"], { + ignoreError: true, + timeout: DOCKER_SANDBOX_QUERY_TIMEOUT_MS, + }); + return output + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean); +} + +export type OpenShellDockerSandboxContainerQuery = + | { ok: true; ids: string[] } + | { ok: false; ids: []; error: string }; + +/** + * Status-bearing lookup used when an empty container list is a safety proof. + * Unlike the best-effort discovery helper, this distinguishes Docker failure + * from a successful query with zero labeled matches. + */ +export function queryOpenShellDockerSandboxContainers( + sandboxName: string, + deps: DockerSandboxContainerQueryDeps = {}, +): OpenShellDockerSandboxContainerQuery { + const run = deps.dockerRun ?? dockerRun; + const result = run([...sandboxContainerFilterArgs(sandboxName), "--format", "{{.ID}}"], { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_SANDBOX_QUERY_TIMEOUT_MS, + }); + if (Number(result.status ?? 1) !== 0) { + return { + ok: false, + ids: [], + error: commandResultText(result) || "docker ps did not complete successfully", + }; + } + const ids = String(result.stdout ?? "") + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean); + return { ok: true, ids }; +} + +export type OpenShellDockerDeviceRequest = { + Driver: string; + Count: number; + DeviceIDs: string[] | null; + Capabilities: string[][] | null; + Options: Record | null; +}; + +export type OpenShellDockerDeviceMapping = { + PathOnHost: string; + PathInContainer: string; + CgroupPermissions: string; +}; + +export type OpenShellDockerGpuAttachmentState = "absent" | "present" | "unknown"; + +export type OpenShellDockerSandboxRuntimeSnapshotQuery = + | { + ok: true; + /** Immutable identity used when rendering a compatibility retry. */ + imageId: string; + /** Original Docker source reference, retained only for registry bookkeeping. */ + bookkeepingImageRef: string; + stateError: string; + deviceRequests: OpenShellDockerDeviceRequest[] | null; + devices: OpenShellDockerDeviceMapping[] | null; + runtime: string; + /** Closed-world classification of host-owned Docker GPU configuration. */ + nativeGpuAttachmentState: OpenShellDockerGpuAttachmentState; + containerId: string; + } + | { ok: false; error: string }; + +export function isImmutableDockerImageId(value: string): boolean { + return /^sha256:[0-9a-f]{64}$/i.test(value); +} + +function isSafeBookkeepingImageRef(value: string): boolean { + return value.length > 0 && value.length <= 4096 && !/[\s\u0000-\u001f\u007f]/.test(value); +} + +function isStringArray(value: unknown): value is string[] { + return Array.isArray(value) && value.every((item) => typeof item === "string"); +} + +function isStringMatrix(value: unknown): value is string[][] { + return Array.isArray(value) && value.every(isStringArray); +} + +function isStringRecord(value: unknown): value is Record { + return ( + typeof value === "object" && + value !== null && + !Array.isArray(value) && + Object.values(value).every((item) => typeof item === "string") + ); +} + +function hasExactKeys(value: Record, expected: readonly string[]): boolean { + const keys = Object.keys(value); + return keys.length === expected.length && expected.every((key) => Object.hasOwn(value, key)); +} + +function isDockerDeviceRequest(value: unknown): value is OpenShellDockerDeviceRequest { + if (typeof value !== "object" || value === null || Array.isArray(value)) return false; + const request = value as Record; + return ( + hasExactKeys(request, ["Driver", "Count", "DeviceIDs", "Capabilities", "Options"]) && + typeof request.Driver === "string" && + typeof request.Count === "number" && + Number.isInteger(request.Count) && + (request.DeviceIDs === null || isStringArray(request.DeviceIDs)) && + (request.Capabilities === null || isStringMatrix(request.Capabilities)) && + (request.Options === null || isStringRecord(request.Options)) + ); +} + +function isDockerDeviceMapping(value: unknown): value is OpenShellDockerDeviceMapping { + if (typeof value !== "object" || value === null || Array.isArray(value)) return false; + const mapping = value as Record; + return ( + hasExactKeys(mapping, ["PathOnHost", "PathInContainer", "CgroupPermissions"]) && + typeof mapping.PathOnHost === "string" && + typeof mapping.PathInContainer === "string" && + typeof mapping.CgroupPermissions === "string" + ); +} + +function isDockerDeviceRequestList(value: unknown): value is OpenShellDockerDeviceRequest[] | null { + return value === null || (Array.isArray(value) && value.every(isDockerDeviceRequest)); +} + +function isDockerDeviceMappingList(value: unknown): value is OpenShellDockerDeviceMapping[] | null { + return value === null || (Array.isArray(value) && value.every(isDockerDeviceMapping)); +} + +function isNvidiaCdiDevice(value: string): boolean { + return /^nvidia\.com\/gpu(?:=|$)/i.test(value.trim()); +} + +function isKnownGpuDevicePath(value: string): boolean { + const path = value.trim(); + return ( + isNvidiaCdiDevice(path) || + /^\/dev\/nvidia(?:[a-z0-9._/-]*)$/i.test(path) || + /^\/dev\/dri(?:\/[a-z0-9._/-]+)?$/i.test(path) || + /^\/dev\/nvhost[-a-z0-9._/]*$/i.test(path) || + /^\/dev\/nvmap$/i.test(path) || + /^\/dev\/tegra[-a-z0-9._/]*$/i.test(path) + ); +} + +function classifyGpuAttachment( + deviceRequests: OpenShellDockerDeviceRequest[] | null, + devices: OpenShellDockerDeviceMapping[] | null, + runtime: string, +): OpenShellDockerGpuAttachmentState { + const normalizedRuntime = runtime.trim().toLowerCase(); + if (normalizedRuntime === "nvidia") return "present"; + if ( + deviceRequests?.some( + (request) => + request.Driver.trim().toLowerCase() === "nvidia" || + request.DeviceIDs?.some(isNvidiaCdiDevice) === true || + request.Capabilities?.some((group) => + group.some((capability) => capability.trim().toLowerCase() === "gpu"), + ) === true, + ) + ) { + return "present"; + } + if ( + devices?.some( + (mapping) => + isKnownGpuDevicePath(mapping.PathOnHost) || isKnownGpuDevicePath(mapping.PathInContainer), + ) === true + ) { + return "present"; + } + + const noDeviceRequests = deviceRequests === null || deviceRequests.length === 0; + const noDeviceMappings = devices === null || devices.length === 0; + const knownNonGpuRuntime = ["", "crun", "io.containerd.runc.v2", "runc"].includes( + normalizedRuntime, + ); + return noDeviceRequests && noDeviceMappings && knownNonGpuRuntime ? "absent" : "unknown"; +} + +/** + * Inspect the one exactly labeled native container before deletion. + * + * Docker owns the fields returned here: `.Image` is the immutable retry + * identity, `.Config.Image` is bookkeeping-only, and HostConfig supplies the + * structured GPU-attachment evidence. Malformed HostConfig shapes fail the + * whole snapshot. Unknown but well-formed configurations remain `unknown`, so + * only the closed-world `absent` state can authorize a broader retry. + */ +export function queryOpenShellDockerSandboxRuntimeSnapshot( + sandboxName: string, + deps: DockerSandboxContainerQueryDeps = {}, +): OpenShellDockerSandboxRuntimeSnapshotQuery { + const containers = queryOpenShellDockerSandboxContainers(sandboxName, deps); + if (!containers.ok) return { ok: false, error: containers.error }; + if (containers.ids.length !== 1) { + return { + ok: false, + error: `expected one labeled sandbox container, found ${containers.ids.length}`, + }; + } + const run = deps.dockerRun ?? dockerRun; + const containerId = containers.ids[0]; + const inspect = run( + [ + "inspect", + "--type", + "container", + "--format", + "[{{json .Image}},{{json .Config.Image}},{{json .State.Error}},{{json .HostConfig.DeviceRequests}},{{json .HostConfig.Devices}},{{json .HostConfig.Runtime}}]", + containerId, + ], + { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_SANDBOX_QUERY_TIMEOUT_MS, + }, + ); + if (Number(inspect.status ?? 1) !== 0) { + return { + ok: false, + error: commandResultText(inspect) || "docker inspect did not complete successfully", + }; + } + let fields: unknown; + try { + fields = JSON.parse(String(inspect.stdout ?? "").trim()); + } catch { + return { ok: false, error: "docker inspect returned malformed runtime metadata" }; + } + if ( + !Array.isArray(fields) || + fields.length !== 6 || + typeof fields[0] !== "string" || + typeof fields[1] !== "string" || + typeof fields[2] !== "string" || + !isImmutableDockerImageId(fields[0]) || + !isSafeBookkeepingImageRef(fields[1]) || + !isDockerDeviceRequestList(fields[3]) || + !isDockerDeviceMappingList(fields[4]) || + typeof fields[5] !== "string" + ) { + return { ok: false, error: "docker inspect returned malformed runtime metadata" }; + } + const deviceRequests = fields[3]; + const devices = fields[4]; + const runtime = fields[5]; + return { + ok: true, + imageId: fields[0].toLowerCase(), + bookkeepingImageRef: fields[1], + stateError: fields[2], + deviceRequests, + devices, + runtime, + nativeGpuAttachmentState: classifyGpuAttachment(deviceRequests, devices, runtime), + containerId, + }; +} diff --git a/src/lib/onboard/sandbox-create-intent-types.ts b/src/lib/onboard/sandbox-create-intent-types.ts index d8e9b733ce0..7c99d335bc4 100644 --- a/src/lib/onboard/sandbox-create-intent-types.ts +++ b/src/lib/onboard/sandbox-create-intent-types.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import type { DockerGpuRoutePlan } from "./docker-gpu-route"; import type { MessagingTokenDef } from "./messaging-prep"; import type { MessagingChannel } from "./messaging-state"; import type { SandboxGpuCreateConfig } from "./sandbox-gpu-create"; @@ -21,7 +22,6 @@ export type SandboxCreatePolicyRequest = { readonly activeMessagingChannels: readonly string[]; readonly options: { readonly directGpu: boolean; - readonly dockerGpuPatch: boolean; readonly additionalPresets: readonly string[]; readonly agentName?: string | null; readonly policyTier: string | null; @@ -47,7 +47,7 @@ export type SandboxCreateIntent = { readonly hermesToolGateways: readonly string[]; readonly policy: SandboxCreatePolicyRequest; readonly gpuCreateArgs: readonly string[]; - readonly useDockerGpuPatch: boolean; + readonly gpuRoutePlan: DockerGpuRoutePlan; readonly sandboxGpuLogMessage: string | null; readonly disabledChannelNames: readonly string[]; }; @@ -66,7 +66,7 @@ export type ResolveSandboxCreateIntentInput = { hermesToolGateways: readonly string[]; sandboxGpuConfig: SandboxGpuCreateConfig; gpuCreateArgs: readonly string[]; - useDockerGpuPatch: boolean; + gpuRoutePlan: DockerGpuRoutePlan; sandboxGpuLogMessage: string | null; agentName?: string | null; policyTier: string | null; diff --git a/src/lib/onboard/sandbox-create-intent.ts b/src/lib/onboard/sandbox-create-intent.ts new file mode 100644 index 00000000000..dbd2f625b80 --- /dev/null +++ b/src/lib/onboard/sandbox-create-intent.ts @@ -0,0 +1,190 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + listMessagingCredentialMetadata, + type MessagingCredentialMetadata, +} from "../messaging/channels"; +import type { MessagingTokenDef } from "./messaging-prep"; +import { resolveQrSelectedChannels } from "./messaging-state"; +import type { + ResolveSandboxCreateIntentInput, + SandboxCreateIntent, + SandboxCreateMessagingProviderRequest, +} from "./sandbox-create-intent-types"; + +function filterEnabledChannelNames( + channelNames: readonly string[], + disabledChannelNames: ReadonlySet, +): string[] { + return channelNames.filter((channelName) => !disabledChannelNames.has(channelName)); +} + +function filterMessagingProviderRequestsByEnabledChannel( + requests: readonly SandboxCreateMessagingProviderRequest[], + disabledChannelNames: ReadonlySet, +): SandboxCreateMessagingProviderRequest[] { + return requests.filter(({ channel }) => !channel || !disabledChannelNames.has(channel)); +} + +function resolveTokenProviderChannelMap( + requests: readonly SandboxCreateMessagingProviderRequest[], +): Map { + const providerChannels = new Map(); + for (const { channel, name } of requests) { + if (channel) providerChannels.set(name, channel); + } + return providerChannels; +} + +function filterMessagingProvidersByEnabledChannel( + providerNames: string[], + providerChannels: ReadonlyMap, + disabledChannelNames: ReadonlySet, +): string[] { + return providerNames.filter((providerName) => { + const channel = providerChannels.get(providerName); + return !channel || !disabledChannelNames.has(channel); + }); +} + +function resolveActiveMessagingChannels({ + channels, + disabledChannelNames, + enabledChannels, + messagingProviderRequests, + primaryMessagingCredentialEnvKeys, + reusableMessagingChannels, +}: Pick< + ResolveSandboxCreateIntentInput, + | "channels" + | "disabledChannelNames" + | "enabledChannels" + | "messagingProviderRequests" + | "primaryMessagingCredentialEnvKeys" + | "reusableMessagingChannels" +>): string[] { + const primaryCredentialEnvKeys = new Set(primaryMessagingCredentialEnvKeys); + const qrSelectedChannels = resolveQrSelectedChannels( + channels, + enabledChannels, + disabledChannelNames, + ); + return filterEnabledChannelNames( + [ + ...new Set([ + ...messagingProviderRequests + .filter(({ credentialConfigured }) => credentialConfigured) + .flatMap(({ channel, envKey }) => { + return channel && primaryCredentialEnvKeys.has(envKey) ? [channel] : []; + }), + ...reusableMessagingChannels, + ...qrSelectedChannels, + ]), + ], + disabledChannelNames, + ); +} + +function compareCredentialsForPrimarySelection( + left: MessagingCredentialMetadata, + right: MessagingCredentialMetadata, +): number { + return ( + left.credentialId.localeCompare(right.credentialId) || + left.providerEnvKey.localeCompare(right.providerEnvKey) + ); +} + +export function resolvePrimaryMessagingCredentialEnvKeys(): string[] { + const credentialsByChannel = new Map(); + for (const credential of listMessagingCredentialMetadata()) { + const credentials = credentialsByChannel.get(credential.channelId) ?? []; + credentials.push(credential); + credentialsByChannel.set(credential.channelId, credentials); + } + + const envKeys = new Set(); + for (const credentials of credentialsByChannel.values()) { + const primary = + credentials.find((credential) => credential.primary) ?? + [...credentials].sort(compareCredentialsForPrimarySelection)[0]; + if (primary) envKeys.add(primary.providerEnvKey); + } + return [...envKeys]; +} + +export function resolveSandboxCreateMessagingProviderRequests( + messagingTokenDefs: readonly MessagingTokenDef[], + getMessagingChannelForEnvKey: (envKey: string) => string | null, +): SandboxCreateMessagingProviderRequest[] { + return messagingTokenDefs.map(({ name, envKey, providerType, token }) => ({ + name, + envKey, + ...(providerType ? { providerType } : {}), + credentialConfigured: Boolean(token), + channel: getMessagingChannelForEnvKey(envKey), + })); +} + +export function resolveSandboxCreateIntent({ + basePolicyPath, + sandboxName, + channels, + enabledChannels, + disabledChannelNames, + messagingProviderRequests, + primaryMessagingCredentialEnvKeys, + reusableMessagingChannels, + reusableMessagingProviders, + extraProviders, + hermesToolGateways, + sandboxGpuConfig, + gpuCreateArgs, + gpuRoutePlan, + sandboxGpuLogMessage, + agentName, + policyTier, +}: ResolveSandboxCreateIntentInput): SandboxCreateIntent { + const enabledMessagingProviderRequests = filterMessagingProviderRequestsByEnabledChannel( + messagingProviderRequests, + disabledChannelNames, + ); + const providerChannels = resolveTokenProviderChannelMap(messagingProviderRequests); + const activeMessagingChannels = resolveActiveMessagingChannels({ + channels, + disabledChannelNames, + enabledChannels, + messagingProviderRequests: enabledMessagingProviderRequests, + primaryMessagingCredentialEnvKeys, + reusableMessagingChannels, + }); + const enabledReusableMessagingProviders = filterMessagingProvidersByEnabledChannel( + [...new Set(reusableMessagingProviders)], + providerChannels, + disabledChannelNames, + ); + + return { + sandboxName, + activeMessagingChannels, + messagingProviderRequests: messagingProviderRequests.map((request) => ({ ...request })), + reusableMessagingProviders: enabledReusableMessagingProviders, + extraProviders: [...new Set(extraProviders ?? [])].filter(Boolean), + hermesToolGateways: [...hermesToolGateways], + policy: { + basePolicyPath, + activeMessagingChannels: [...activeMessagingChannels], + options: { + directGpu: sandboxGpuConfig.sandboxGpuEnabled, + additionalPresets: [...hermesToolGateways], + ...(agentName !== undefined ? { agentName } : {}), + policyTier, + }, + }, + gpuCreateArgs: [...gpuCreateArgs], + gpuRoutePlan, + sandboxGpuLogMessage, + disabledChannelNames: [...disabledChannelNames], + }; +} diff --git a/src/lib/onboard/sandbox-create-launch.test.ts b/src/lib/onboard/sandbox-create-launch.test.ts index 3db3e6047e6..fef755927fc 100644 --- a/src/lib/onboard/sandbox-create-launch.test.ts +++ b/src/lib/onboard/sandbox-create-launch.test.ts @@ -17,6 +17,7 @@ import { } from "./sandbox-create-launch"; const disabledHermesDashboardState = { config: null, enabled: false }; +const IMAGE_ID = `sha256:${"a".repeat(64)}`; const temporaryBuildContexts: string[] = []; function createTrustedBuildContext(): string { @@ -376,6 +377,7 @@ describe("prepareSandboxCreateLaunchWithPrebuild", () => { dockerDriverGateway: true, env: { NEMOCLAW_SANDBOX_PREBUILD: "1" }, buildImage, + inspectImageId: () => IMAGE_ID, log: vi.fn(), origin: "generated", }, @@ -384,6 +386,7 @@ describe("prepareSandboxCreateLaunchWithPrebuild", () => { expect(result.prebuild).toEqual({ createArgs: ["--from", "nemoclaw-sandbox-local:demo-build-123", "--name", "demo"], imageRef: "nemoclaw-sandbox-local:demo-build-123", + imageId: IMAGE_ID, }); expect(result.createCommand).toContain( "sandbox create --from nemoclaw-sandbox-local:demo-build-123 --name demo", @@ -420,6 +423,7 @@ describe("prepareSandboxCreateLaunchWithPrebuild", () => { expect(result.prebuild).toEqual({ createArgs: ["--from", dockerfile, "--name", "demo"], imageRef: null, + imageId: null, }); expect(result.createCommand).toContain(`sandbox create --from ${dockerfile} --name demo`); expect(result.createCommand).not.toContain("nemoclaw-sandbox-local"); diff --git a/src/lib/onboard/sandbox-create-launch.ts b/src/lib/onboard/sandbox-create-launch.ts index 9446ba5ec94..1e5431bb479 100644 --- a/src/lib/onboard/sandbox-create-launch.ts +++ b/src/lib/onboard/sandbox-create-launch.ts @@ -75,6 +75,20 @@ export interface SandboxCreateLaunchWithPrebuild extends SandboxCreateLaunch { prebuild: SandboxPrebuildResult; } +export function renderSandboxCreateCommand( + createArgs: readonly string[], + sandboxStartupCommand: readonly string[], + openshellShellCommand: OpenshellShellCommand, +): string { + return `${openshellShellCommand([ + "sandbox", + "create", + ...createArgs, + "--", + ...sandboxStartupCommand, + ])} 2>&1`; +} + export interface SandboxRuntimeEnvArgsInput { agent: AgentDefinition | null; chatUiUrl: string; @@ -182,7 +196,11 @@ export function prepareSandboxCreateLaunch(input: SandboxCreateLaunchInput): San // lets the real exit code flow through to run(). const sandboxStartupCommand = ["env", ...envArgs, "nemoclaw-start"]; const openshellArgs = ["sandbox", "create", ...input.createArgs, "--", ...sandboxStartupCommand]; - const createCommand = `${input.openshellShellCommand(openshellArgs)} 2>&1`; + const createCommand = renderSandboxCreateCommand( + input.createArgs, + sandboxStartupCommand, + input.openshellShellCommand, + ); const createArgv = input.openshellArgv ? input.openshellArgv(openshellArgs) : ["bash", "-lc", createCommand]; diff --git a/src/lib/onboard/sandbox-create-plan-extra-providers.test.ts b/src/lib/onboard/sandbox-create-plan-extra-providers.test.ts index c68d370385b..35f96892d00 100644 --- a/src/lib/onboard/sandbox-create-plan-extra-providers.test.ts +++ b/src/lib/onboard/sandbox-create-plan-extra-providers.test.ts @@ -33,17 +33,14 @@ function buildPlan( extraProviders, hermesToolGateways: [], sandboxGpuConfig, - dockerDriverGateway: true, + gpuRoutePlan: "none", + sandboxGpuLogMessage: null, appendResourceFlags: vi.fn(), runProviderPreDeleteCleanup: vi.fn(), upsertMessagingProviders: vi.fn(() => []), getMessagingChannelForEnvKey: () => null, getHermesToolGatewayProviderName: vi.fn(), deps: { - resolveDockerGpuSandboxCreatePlan: vi.fn(() => ({ - useDockerGpuPatch: false, - logMessage: null, - })), prepareInitialSandboxCreatePolicy: vi.fn(() => ({ policyPath: "/tmp/policy.yaml", appliedPresets: [], diff --git a/src/lib/onboard/sandbox-create-plan-materialization.ts b/src/lib/onboard/sandbox-create-plan-materialization.ts new file mode 100644 index 00000000000..8488f9f330d --- /dev/null +++ b/src/lib/onboard/sandbox-create-plan-materialization.ts @@ -0,0 +1,168 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { InitialSandboxPolicy } from "./initial-policy"; +import type { MessagingTokenDef } from "./messaging-prep"; +import type { + MaterializeSandboxCreatePlanInput, + SandboxCreateIntent, + SandboxCreateMessagingProviderRequest, +} from "./sandbox-create-intent-types"; +import { prepareSandboxGpuRoutePolicies } from "./sandbox-gpu-route-policy"; + +type PrepareInitialSandboxCreatePolicy = + typeof import("./initial-policy").prepareInitialSandboxCreatePolicy; + +export type SandboxCreatePlan = { + activeMessagingChannels: string[]; + initialSandboxPolicy: InitialSandboxPolicy; + /** Tier resolved before create, persisted with the registry entry for safe resume. */ + policyTier: string | null; + createArgs: string[]; + messagingProviders: string[]; + gpuRoutePlan: SandboxCreateIntent["gpuRoutePlan"]; + compatibilityPolicyPath: string | null; + sandboxGpuLogMessage: string | null; +}; + +function getInitialSandboxCreatePolicy( + ...args: Parameters +): ReturnType { + const { prepareInitialSandboxCreatePolicy } = + require("./initial-policy") as typeof import("./initial-policy"); + return prepareInitialSandboxCreatePolicy(...args); +} + +function messagingProviderRequestKey( + request: Pick, +): string { + // Tuple encoding stays collision-free even if either value contains a separator. + return JSON.stringify([request.name, request.envKey]); +} + +function bindMessagingTokenDefs( + intent: SandboxCreateIntent, + messagingTokenDefs: readonly MessagingTokenDef[], +): MessagingTokenDef[] { + const disabledChannelNames = new Set(intent.disabledChannelNames); + const enabledRequests = intent.messagingProviderRequests.filter( + ({ channel }) => !channel || !disabledChannelNames.has(channel), + ); + const tokenDefsByRequest = new Map( + messagingTokenDefs.map((tokenDef) => [messagingProviderRequestKey(tokenDef), tokenDef]), + ); + + return enabledRequests.map((request) => { + const tokenDef = tokenDefsByRequest.get(messagingProviderRequestKey(request)); + if (!tokenDef) { + throw new Error( + `Cannot materialize sandbox create intent; missing credential binding '${request.envKey}' for provider '${request.name}'.`, + ); + } + if (Boolean(tokenDef.token) !== request.credentialConfigured) { + throw new Error( + `Cannot materialize sandbox create intent; credential availability changed for provider '${request.name}'.`, + ); + } + // Default providers omit this field; normalize an empty or missing binding + // to the intent's `undefined` representation before comparing. + const boundProviderType = tokenDef.providerType || undefined; + if (boundProviderType !== request.providerType) { + throw new Error( + `Cannot materialize sandbox create intent; provider type changed for '${request.name}'.`, + ); + } + return tokenDef; + }); +} + +function resolveProviderChannelMap( + requests: readonly SandboxCreateMessagingProviderRequest[], +): Map { + const providerChannels = new Map(); + for (const { channel, name } of requests) { + if (channel) providerChannels.set(name, channel); + } + return providerChannels; +} + +function filterDisabledMessagingProviders( + providerNames: string[], + providerChannels: ReadonlyMap, + disabledChannelNames: ReadonlySet, +): string[] { + return providerNames.filter((providerName) => { + const channel = providerChannels.get(providerName); + return !channel || !disabledChannelNames.has(channel); + }); +} + +/** Materialize policy, route metadata, resources, and providers from a secretless intent. */ +export function materializeSandboxCreatePlan({ + intent, + buildCtx, + messagingTokenDefs, + appendResourceFlags, + runProviderPreDeleteCleanup, + upsertMessagingProviders, + getHermesToolGatewayProviderName, + prepareInitialSandboxCreatePolicy = getInitialSandboxCreatePolicy, +}: MaterializeSandboxCreatePlanInput): SandboxCreatePlan { + const enabledMessagingTokenDefs = bindMessagingTokenDefs(intent, messagingTokenDefs); + const { initialSandboxPolicy, compatibilityPolicyPath } = prepareSandboxGpuRoutePolicies( + intent.policy.basePolicyPath, + [...intent.policy.activeMessagingChannels], + { + directGpu: intent.policy.options.directGpu, + additionalPresets: [...intent.policy.options.additionalPresets], + agentName: intent.policy.options.agentName, + policyTier: intent.policy.options.policyTier, + }, + intent.gpuRoutePlan, + prepareInitialSandboxCreatePolicy, + ); + const createArgs = [ + "--from", + `${buildCtx}/Dockerfile`, + "--name", + intent.sandboxName, + "--policy", + initialSandboxPolicy.policyPath, + ...intent.gpuCreateArgs, + ]; + + appendResourceFlags(createArgs); + runProviderPreDeleteCleanup(); + const providerChannels = resolveProviderChannelMap(intent.messagingProviderRequests); + const messagingProviders = filterDisabledMessagingProviders( + [ + ...new Set([ + ...upsertMessagingProviders(enabledMessagingTokenDefs, { replaceExisting: true }), + ...intent.reusableMessagingProviders, + ]), + ], + providerChannels, + new Set(intent.disabledChannelNames), + ); + for (const provider of messagingProviders) { + createArgs.push("--provider", provider); + } + if (intent.hermesToolGateways.length > 0) { + createArgs.push("--provider", getHermesToolGatewayProviderName(intent.sandboxName)); + } + for (const provider of intent.extraProviders) { + if (messagingProviders.includes(provider)) continue; + createArgs.push("--provider", provider); + } + + return { + activeMessagingChannels: [...intent.activeMessagingChannels], + initialSandboxPolicy, + policyTier: intent.policy.options.policyTier, + createArgs, + messagingProviders, + gpuRoutePlan: intent.gpuRoutePlan, + compatibilityPolicyPath, + sandboxGpuLogMessage: intent.sandboxGpuLogMessage, + }; +} diff --git a/src/lib/onboard/sandbox-create-plan.test.ts b/src/lib/onboard/sandbox-create-plan.test.ts index fd8f3b6c820..70d60351dd8 100644 --- a/src/lib/onboard/sandbox-create-plan.test.ts +++ b/src/lib/onboard/sandbox-create-plan.test.ts @@ -70,7 +70,7 @@ function expectCredentialBindingFailure({ hermesToolGateways: [], sandboxGpuConfig, gpuCreateArgs: [], - useDockerGpuPatch: false, + gpuRoutePlan: "native-only", sandboxGpuLogMessage: null, policyTier: null, }); @@ -165,7 +165,7 @@ describe("resolveSandboxCreateIntent", () => { hermesToolGateways: ["github"], sandboxGpuConfig, gpuCreateArgs: ["--gpu", "--gpu-device", "nvidia.com/gpu=0"], - useDockerGpuPatch: false, + gpuRoutePlan: "native-only" as const, sandboxGpuLogMessage: "gpu note", agentName: "hermes", policyTier: "balanced", @@ -187,7 +187,6 @@ describe("resolveSandboxCreateIntent", () => { activeMessagingChannels: ["telegram", "discord", "whatsapp"], options: { directGpu: true, - dockerGpuPatch: false, additionalPresets: ["github"], agentName: "hermes", policyTier: "balanced", @@ -222,7 +221,7 @@ describe("resolveSandboxCreateIntent", () => { hermesToolGateways: ["github"], sandboxGpuConfig, gpuCreateArgs: ["--gpu"], - useDockerGpuPatch: false, + gpuRoutePlan: "native-only", sandboxGpuLogMessage: null, agentName: "hermes", policyTier: "balanced", @@ -349,7 +348,7 @@ describe("prepareSandboxCreatePlan", () => { const prepareInitialSandboxCreatePolicy = vi.fn(() => ({ policyPath: "/tmp/policy.yaml", appliedPresets: ["telegram"], - cleanup: vi.fn(), + cleanup: vi.fn(() => true), })); const result = prepareSandboxCreatePlan({ @@ -380,7 +379,8 @@ describe("prepareSandboxCreatePlan", () => { reusableMessagingProviders: ["sandbox-existing-discord"], hermesToolGateways: ["github"], sandboxGpuConfig, - dockerDriverGateway: true, + gpuRoutePlan: "native-only", + sandboxGpuLogMessage: "gpu note", appendResourceFlags, runProviderPreDeleteCleanup, upsertMessagingProviders, @@ -393,10 +393,6 @@ describe("prepareSandboxCreatePlan", () => { getHermesToolGatewayProviderName: (sandboxName) => `${sandboxName}-hermes-tools`, agentName: "langchain-deepagents-code", deps: { - resolveDockerGpuSandboxCreatePlan: vi.fn(() => ({ - useDockerGpuPatch: false, - logMessage: "gpu note", - })), prepareInitialSandboxCreatePolicy, buildSandboxGpuCreateArgs: vi.fn(() => ["--gpu", "--gpu-device", "nvidia.com/gpu=0"]), }, @@ -442,6 +438,10 @@ describe("prepareSandboxCreatePlan", () => { "sandbox-existing-discord", ]); expect(result.sandboxGpuLogMessage).toBe("gpu note"); + expect(prepareInitialSandboxCreatePolicy).toHaveBeenCalledTimes(1); + expect(appendResourceFlags).toHaveBeenCalledTimes(1); + expect(runProviderPreDeleteCleanup).toHaveBeenCalledTimes(1); + expect(upsertMessagingProviders).toHaveBeenCalledTimes(1); expect(events).toEqual(["resources", "cleanup", "upsert"]); }); @@ -474,7 +474,8 @@ describe("prepareSandboxCreatePlan", () => { reusableMessagingProviders: ["sandbox-slack-bridge", "sandbox-existing-whatsapp"], hermesToolGateways: [], sandboxGpuConfig, - dockerDriverGateway: true, + gpuRoutePlan: "native-only", + sandboxGpuLogMessage: null, appendResourceFlags: vi.fn(), runProviderPreDeleteCleanup: vi.fn(), upsertMessagingProviders, @@ -486,10 +487,6 @@ describe("prepareSandboxCreatePlan", () => { : null, getHermesToolGatewayProviderName: vi.fn(), deps: { - resolveDockerGpuSandboxCreatePlan: vi.fn(() => ({ - useDockerGpuPatch: false, - logMessage: null, - })), prepareInitialSandboxCreatePolicy: vi.fn(() => ({ policyPath: "/tmp/policy.yaml", appliedPresets: [], @@ -517,7 +514,7 @@ describe("prepareSandboxCreatePlan", () => { expect(result.createArgs).not.toContain("sandbox-slack-bridge"); }); - it("does not activate slack from an app token alone and suppresses --gpu for Docker GPU patching", () => { + it("does not activate slack from an app token alone or a disabled QR channel", () => { const result = prepareSandboxCreatePlan({ basePolicyPath: "/repo/policy.yaml", buildCtx: "/tmp/nemoclaw-build-1", @@ -536,17 +533,14 @@ describe("prepareSandboxCreatePlan", () => { reusableMessagingProviders: [], hermesToolGateways: [], sandboxGpuConfig, - dockerDriverGateway: true, + gpuRoutePlan: "native-only", + sandboxGpuLogMessage: null, appendResourceFlags: vi.fn(), runProviderPreDeleteCleanup: vi.fn(), upsertMessagingProviders: vi.fn(() => []), getMessagingChannelForEnvKey: () => null, getHermesToolGatewayProviderName: vi.fn(), deps: { - resolveDockerGpuSandboxCreatePlan: vi.fn(() => ({ - useDockerGpuPatch: true, - logMessage: null, - })), prepareInitialSandboxCreatePolicy: vi.fn(() => ({ policyPath: "/tmp/policy.yaml", appliedPresets: [], @@ -556,7 +550,6 @@ describe("prepareSandboxCreatePlan", () => { }); expect(result.activeMessagingChannels).toEqual([]); - expect(result.useDockerGpuPatch).toBe(true); expect(result.createArgs).toEqual([ "--from", "/tmp/nemoclaw-build-1/Dockerfile", @@ -581,17 +574,14 @@ describe("prepareSandboxCreatePlan", () => { extraProviders: ["tavily-search", "tavily-search", "custom-provider"], hermesToolGateways: [], sandboxGpuConfig, - dockerDriverGateway: true, + gpuRoutePlan: "native-only", + sandboxGpuLogMessage: null, appendResourceFlags: vi.fn(), runProviderPreDeleteCleanup: vi.fn(), upsertMessagingProviders: vi.fn(() => []), getMessagingChannelForEnvKey: () => null, getHermesToolGatewayProviderName: vi.fn(), deps: { - resolveDockerGpuSandboxCreatePlan: vi.fn(() => ({ - useDockerGpuPatch: false, - logMessage: null, - })), prepareInitialSandboxCreatePolicy: vi.fn(() => ({ policyPath: "/tmp/policy.yaml", appliedPresets: [], @@ -626,7 +616,8 @@ describe("prepareSandboxCreatePlan", () => { extraProviders: ["sandbox-telegram-bridge", "tavily-search"], hermesToolGateways: [], sandboxGpuConfig, - dockerDriverGateway: true, + gpuRoutePlan: "native-only", + sandboxGpuLogMessage: null, appendResourceFlags: vi.fn(), runProviderPreDeleteCleanup: vi.fn(), upsertMessagingProviders: vi.fn(() => ["sandbox-telegram-bridge"]), @@ -634,10 +625,6 @@ describe("prepareSandboxCreatePlan", () => { envKey === "TELEGRAM_BOT_TOKEN" ? "telegram" : null, getHermesToolGatewayProviderName: vi.fn(), deps: { - resolveDockerGpuSandboxCreatePlan: vi.fn(() => ({ - useDockerGpuPatch: false, - logMessage: null, - })), prepareInitialSandboxCreatePolicy: vi.fn(() => ({ policyPath: "/tmp/policy.yaml", appliedPresets: [], diff --git a/src/lib/onboard/sandbox-create-plan.ts b/src/lib/onboard/sandbox-create-plan.ts index 84ec37405fc..ed8f78de1af 100644 --- a/src/lib/onboard/sandbox-create-plan.ts +++ b/src/lib/onboard/sandbox-create-plan.ts @@ -1,22 +1,24 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { - listMessagingCredentialMetadata, - type MessagingCredentialMetadata, -} from "../messaging/channels"; -import type { InitialSandboxPolicy } from "./initial-policy"; +import { type DockerGpuRoutePlan } from "./docker-gpu-route"; import type { MessagingTokenDef } from "./messaging-prep"; import type { MessagingChannel } from "./messaging-state"; -import { resolveQrSelectedChannels } from "./messaging-state"; -import type { - MaterializeSandboxCreatePlanInput, - ResolveSandboxCreateIntentInput, - SandboxCreateIntent, - SandboxCreateMessagingProviderRequest, -} from "./sandbox-create-intent-types"; +import { + resolvePrimaryMessagingCredentialEnvKeys, + resolveSandboxCreateIntent, + resolveSandboxCreateMessagingProviderRequests, +} from "./sandbox-create-intent"; +import { + materializeSandboxCreatePlan, + type SandboxCreatePlan, +} from "./sandbox-create-plan-materialization"; import { buildSandboxGpuCreateArgs, type SandboxGpuCreateConfig } from "./sandbox-gpu-create"; +export { + resolveSandboxCreateIntent, + resolveSandboxCreateMessagingProviderRequests, +} from "./sandbox-create-intent"; export type { MaterializeSandboxCreatePlanInput, ResolveSandboxCreateIntentInput, @@ -24,6 +26,8 @@ export type { SandboxCreateMessagingProviderRequest, SandboxCreatePolicyRequest, } from "./sandbox-create-intent-types"; +export type { SandboxCreatePlan } from "./sandbox-create-plan-materialization"; +export { materializeSandboxCreatePlan } from "./sandbox-create-plan-materialization"; // Known canonical policy tier names. Kept inline so the create-time path // validates the env value without pulling `../policy/tiers` (which transitively @@ -48,13 +52,10 @@ function readPolicyTierEnv(): string | null { return KNOWN_POLICY_TIER_NAMES.has(trimmed) ? trimmed : null; } -type ResolveDockerGpuSandboxCreatePlan = - typeof import("./docker-gpu-sandbox-create").resolveDockerGpuSandboxCreatePlan; type PrepareInitialSandboxCreatePolicy = typeof import("./initial-policy").prepareInitialSandboxCreatePolicy; export type SandboxCreatePlanDeps = { - resolveDockerGpuSandboxCreatePlan?: ResolveDockerGpuSandboxCreatePlan; prepareInitialSandboxCreatePolicy?: PrepareInitialSandboxCreatePolicy; buildSandboxGpuCreateArgs?: typeof buildSandboxGpuCreateArgs; }; @@ -72,7 +73,8 @@ export type PrepareSandboxCreatePlanInput = { extraProviders?: readonly string[]; hermesToolGateways: string[]; sandboxGpuConfig: SandboxGpuCreateConfig; - dockerDriverGateway: boolean; + gpuRoutePlan: DockerGpuRoutePlan; + sandboxGpuLogMessage: string | null; appendResourceFlags(createArgs: string[]): void; runProviderPreDeleteCleanup(): void; upsertMessagingProviders( @@ -86,320 +88,6 @@ export type PrepareSandboxCreatePlanInput = { deps?: SandboxCreatePlanDeps; }; -export type SandboxCreatePlan = { - activeMessagingChannels: string[]; - initialSandboxPolicy: InitialSandboxPolicy; - /** Tier resolved before create, persisted with the registry entry for safe resume. */ - policyTier: string | null; - createArgs: string[]; - messagingProviders: string[]; - useDockerGpuPatch: boolean; - sandboxGpuLogMessage: string | null; -}; - -function getDockerGpuSandboxCreatePlan( - ...args: Parameters -): ReturnType { - const { resolveDockerGpuSandboxCreatePlan } = - require("./docker-gpu-sandbox-create") as typeof import("./docker-gpu-sandbox-create"); - return resolveDockerGpuSandboxCreatePlan(...args); -} - -function getInitialSandboxCreatePolicy( - ...args: Parameters -): ReturnType { - const { prepareInitialSandboxCreatePolicy } = - require("./initial-policy") as typeof import("./initial-policy"); - return prepareInitialSandboxCreatePolicy(...args); -} - -function filterEnabledChannelNames( - channelNames: readonly string[], - disabledChannelNames: ReadonlySet, -): string[] { - return channelNames.filter((channelName) => !disabledChannelNames.has(channelName)); -} - -function filterMessagingProviderRequestsByEnabledChannel( - requests: readonly SandboxCreateMessagingProviderRequest[], - disabledChannelNames: ReadonlySet, -): SandboxCreateMessagingProviderRequest[] { - return requests.filter(({ channel }) => !channel || !disabledChannelNames.has(channel)); -} - -function resolveTokenProviderChannelMap( - requests: readonly SandboxCreateMessagingProviderRequest[], -): Map { - const providerChannels = new Map(); - for (const { channel, name } of requests) { - if (channel) providerChannels.set(name, channel); - } - return providerChannels; -} - -function filterMessagingProvidersByEnabledChannel( - providerNames: string[], - providerChannels: ReadonlyMap, - disabledChannelNames: ReadonlySet, -): string[] { - return providerNames.filter((providerName) => { - const channel = providerChannels.get(providerName); - return !channel || !disabledChannelNames.has(channel); - }); -} - -function resolveActiveMessagingChannels({ - channels, - disabledChannelNames, - enabledChannels, - messagingProviderRequests, - primaryMessagingCredentialEnvKeys, - reusableMessagingChannels, -}: Pick< - ResolveSandboxCreateIntentInput, - | "channels" - | "disabledChannelNames" - | "enabledChannels" - | "messagingProviderRequests" - | "primaryMessagingCredentialEnvKeys" - | "reusableMessagingChannels" ->): string[] { - const primaryCredentialEnvKeys = new Set(primaryMessagingCredentialEnvKeys); - const qrSelectedChannels = resolveQrSelectedChannels( - channels, - enabledChannels, - disabledChannelNames, - ); - return filterEnabledChannelNames( - [ - ...new Set([ - ...messagingProviderRequests - .filter(({ credentialConfigured }) => credentialConfigured) - .flatMap(({ channel, envKey }) => { - return channel && primaryCredentialEnvKeys.has(envKey) ? [channel] : []; - }), - ...reusableMessagingChannels, - ...qrSelectedChannels, - ]), - ], - disabledChannelNames, - ); -} - -function getPrimaryCredentialEnvKeys(): Set { - const credentialsByChannel = new Map(); - for (const credential of listMessagingCredentialMetadata()) { - const credentials = credentialsByChannel.get(credential.channelId) ?? []; - credentials.push(credential); - credentialsByChannel.set(credential.channelId, credentials); - } - - const envKeys = new Set(); - for (const credentials of credentialsByChannel.values()) { - const primary = - credentials.find((credential) => credential.primary) ?? - [...credentials].sort(compareCredentialsForPrimarySelection)[0]; - if (primary) envKeys.add(primary.providerEnvKey); - } - return envKeys; -} - -function compareCredentialsForPrimarySelection( - left: MessagingCredentialMetadata, - right: MessagingCredentialMetadata, -): number { - return ( - left.credentialId.localeCompare(right.credentialId) || - left.providerEnvKey.localeCompare(right.providerEnvKey) - ); -} - -export function resolveSandboxCreateMessagingProviderRequests( - messagingTokenDefs: readonly MessagingTokenDef[], - getMessagingChannelForEnvKey: (envKey: string) => string | null, -): SandboxCreateMessagingProviderRequest[] { - return messagingTokenDefs.map(({ name, envKey, providerType, token }) => ({ - name, - envKey, - ...(providerType ? { providerType } : {}), - credentialConfigured: Boolean(token), - channel: getMessagingChannelForEnvKey(envKey), - })); -} - -export function resolveSandboxCreateIntent({ - basePolicyPath, - sandboxName, - channels, - enabledChannels, - disabledChannelNames, - messagingProviderRequests, - primaryMessagingCredentialEnvKeys, - reusableMessagingChannels, - reusableMessagingProviders, - extraProviders, - hermesToolGateways, - sandboxGpuConfig, - gpuCreateArgs, - useDockerGpuPatch, - sandboxGpuLogMessage, - agentName, - policyTier, -}: ResolveSandboxCreateIntentInput): SandboxCreateIntent { - const enabledMessagingProviderRequests = filterMessagingProviderRequestsByEnabledChannel( - messagingProviderRequests, - disabledChannelNames, - ); - const providerChannels = resolveTokenProviderChannelMap(messagingProviderRequests); - const activeMessagingChannels = resolveActiveMessagingChannels({ - channels, - disabledChannelNames, - enabledChannels, - messagingProviderRequests: enabledMessagingProviderRequests, - primaryMessagingCredentialEnvKeys, - reusableMessagingChannels, - }); - const enabledReusableMessagingProviders = filterMessagingProvidersByEnabledChannel( - [...new Set(reusableMessagingProviders)], - providerChannels, - disabledChannelNames, - ); - - return { - sandboxName, - activeMessagingChannels, - messagingProviderRequests: messagingProviderRequests.map((request) => ({ ...request })), - reusableMessagingProviders: enabledReusableMessagingProviders, - extraProviders: [...new Set(extraProviders ?? [])].filter(Boolean), - hermesToolGateways: [...hermesToolGateways], - policy: { - basePolicyPath, - activeMessagingChannels: [...activeMessagingChannels], - options: { - directGpu: sandboxGpuConfig.sandboxGpuEnabled, - dockerGpuPatch: useDockerGpuPatch, - additionalPresets: [...hermesToolGateways], - ...(agentName !== undefined ? { agentName } : {}), - policyTier, - }, - }, - gpuCreateArgs: [...gpuCreateArgs], - useDockerGpuPatch, - sandboxGpuLogMessage, - disabledChannelNames: [...disabledChannelNames], - }; -} - -function messagingProviderRequestKey( - request: Pick, -): string { - // Tuple encoding stays collision-free even if either value contains a separator. - return JSON.stringify([request.name, request.envKey]); -} - -function bindMessagingTokenDefs( - intent: SandboxCreateIntent, - messagingTokenDefs: readonly MessagingTokenDef[], -): MessagingTokenDef[] { - const enabledRequests = filterMessagingProviderRequestsByEnabledChannel( - intent.messagingProviderRequests, - new Set(intent.disabledChannelNames), - ); - const tokenDefsByRequest = new Map( - messagingTokenDefs.map((tokenDef) => [messagingProviderRequestKey(tokenDef), tokenDef]), - ); - - return enabledRequests.map((request) => { - const tokenDef = tokenDefsByRequest.get(messagingProviderRequestKey(request)); - if (!tokenDef) { - throw new Error( - `Cannot materialize sandbox create intent; missing credential binding '${request.envKey}' for provider '${request.name}'.`, - ); - } - if (Boolean(tokenDef.token) !== request.credentialConfigured) { - throw new Error( - `Cannot materialize sandbox create intent; credential availability changed for provider '${request.name}'.`, - ); - } - // Default providers omit this field; normalize an empty or missing binding - // to the intent's `undefined` representation before comparing. - const boundProviderType = tokenDef.providerType || undefined; - if (boundProviderType !== request.providerType) { - throw new Error( - `Cannot materialize sandbox create intent; provider type changed for '${request.name}'.`, - ); - } - return tokenDef; - }); -} - -export function materializeSandboxCreatePlan({ - intent, - buildCtx, - messagingTokenDefs, - appendResourceFlags, - runProviderPreDeleteCleanup, - upsertMessagingProviders, - getHermesToolGatewayProviderName, - prepareInitialSandboxCreatePolicy = getInitialSandboxCreatePolicy, -}: MaterializeSandboxCreatePlanInput): SandboxCreatePlan { - const enabledMessagingTokenDefs = bindMessagingTokenDefs(intent, messagingTokenDefs); - const initialSandboxPolicy = prepareInitialSandboxCreatePolicy( - intent.policy.basePolicyPath, - [...intent.policy.activeMessagingChannels], - { - directGpu: intent.policy.options.directGpu, - dockerGpuPatch: intent.policy.options.dockerGpuPatch, - additionalPresets: [...intent.policy.options.additionalPresets], - agentName: intent.policy.options.agentName, - policyTier: intent.policy.options.policyTier, - }, - ); - const createArgs = [ - "--from", - `${buildCtx}/Dockerfile`, - "--name", - intent.sandboxName, - "--policy", - initialSandboxPolicy.policyPath, - ...intent.gpuCreateArgs, - ]; - - appendResourceFlags(createArgs); - runProviderPreDeleteCleanup(); - const providerChannels = resolveTokenProviderChannelMap(intent.messagingProviderRequests); - const messagingProviders = filterMessagingProvidersByEnabledChannel( - [ - ...new Set([ - ...upsertMessagingProviders(enabledMessagingTokenDefs, { replaceExisting: true }), - ...intent.reusableMessagingProviders, - ]), - ], - providerChannels, - new Set(intent.disabledChannelNames), - ); - for (const provider of messagingProviders) { - createArgs.push("--provider", provider); - } - if (intent.hermesToolGateways.length > 0) { - createArgs.push("--provider", getHermesToolGatewayProviderName(intent.sandboxName)); - } - for (const provider of intent.extraProviders) { - if (messagingProviders.includes(provider)) continue; - createArgs.push("--provider", provider); - } - - return { - activeMessagingChannels: [...intent.activeMessagingChannels], - initialSandboxPolicy, - policyTier: intent.policy.options.policyTier, - createArgs, - messagingProviders, - useDockerGpuPatch: intent.useDockerGpuPatch, - sandboxGpuLogMessage: intent.sandboxGpuLogMessage, - }; -} - export function prepareSandboxCreatePlan({ basePolicyPath, buildCtx, @@ -413,7 +101,8 @@ export function prepareSandboxCreatePlan({ extraProviders, hermesToolGateways, sandboxGpuConfig, - dockerDriverGateway, + gpuRoutePlan, + sandboxGpuLogMessage, appendResourceFlags, runProviderPreDeleteCleanup, upsertMessagingProviders, @@ -423,14 +112,8 @@ export function prepareSandboxCreatePlan({ policyTier = readPolicyTierEnv(), deps = {}, }: PrepareSandboxCreatePlanInput): SandboxCreatePlan { - const { useDockerGpuPatch, logMessage: sandboxGpuLogMessage } = ( - deps.resolveDockerGpuSandboxCreatePlan ?? getDockerGpuSandboxCreatePlan - )(sandboxGpuConfig, { dockerDriverGateway }); const gpuCreateArgs = (deps.buildSandboxGpuCreateArgs ?? buildSandboxGpuCreateArgs)( sandboxGpuConfig, - { - suppressGpuFlag: useDockerGpuPatch, - }, ); const messagingProviderRequests = resolveSandboxCreateMessagingProviderRequests( messagingTokenDefs, @@ -443,14 +126,14 @@ export function prepareSandboxCreatePlan({ enabledChannels, disabledChannelNames, messagingProviderRequests, - primaryMessagingCredentialEnvKeys: [...getPrimaryCredentialEnvKeys()], + primaryMessagingCredentialEnvKeys: resolvePrimaryMessagingCredentialEnvKeys(), reusableMessagingChannels, reusableMessagingProviders, extraProviders, hermesToolGateways, sandboxGpuConfig, gpuCreateArgs, - useDockerGpuPatch, + gpuRoutePlan, sandboxGpuLogMessage, agentName, policyTier, @@ -464,7 +147,8 @@ export function prepareSandboxCreatePlan({ runProviderPreDeleteCleanup, upsertMessagingProviders, getHermesToolGatewayProviderName, - prepareInitialSandboxCreatePolicy: - deps.prepareInitialSandboxCreatePolicy ?? getInitialSandboxCreatePolicy, + ...(deps.prepareInitialSandboxCreatePolicy + ? { prepareInitialSandboxCreatePolicy: deps.prepareInitialSandboxCreatePolicy } + : {}), }); } diff --git a/src/lib/onboard/sandbox-create-step.test.ts b/src/lib/onboard/sandbox-create-step.test.ts index 4cd3ea47b01..fe22a3f3003 100644 --- a/src/lib/onboard/sandbox-create-step.test.ts +++ b/src/lib/onboard/sandbox-create-step.test.ts @@ -112,7 +112,7 @@ describe("runSandboxCreateStep", () => { // GPU patch is created with the startup command from the launch result + backend/device. expect(deps.createDockerGpuPatch).toHaveBeenCalledWith( expect.objectContaining({ - enabled: true, + route: "compatibility", openshellSandboxCommand: ["run", "alpha"], gpuDevice: "nvidia.com/gpu=all", backend: "jetson", @@ -156,7 +156,7 @@ describe("runSandboxCreateStep", () => { expect(deps.createDockerGpuPatch).toHaveBeenCalledWith( expect.objectContaining({ - enabled: false, + route: "native", persistStartupCommand: true, openshellSandboxCommand: ["env", "CHAT_UI_URL=http://127.0.0.1:8642", "nemoclaw-start"], }), diff --git a/src/lib/onboard/sandbox-create-step.ts b/src/lib/onboard/sandbox-create-step.ts index 17b6568c3db..be97e726201 100644 --- a/src/lib/onboard/sandbox-create-step.ts +++ b/src/lib/onboard/sandbox-create-step.ts @@ -94,7 +94,7 @@ export async function runSandboxCreateStep( prebuild: context.prebuild, }); const dockerGpuCreatePatch = deps.createDockerGpuPatch({ - enabled: context.useDockerGpuPatch, + route: context.useDockerGpuPatch ? "compatibility" : "native", persistStartupCommand: context.prebuild.dockerDriverGateway === true && context.agent?.name === "hermes", sandboxName: context.sandboxName, diff --git a/src/lib/onboard/sandbox-dockerfile-patch-flow.test.ts b/src/lib/onboard/sandbox-dockerfile-patch-flow.test.ts index 549e22cb58b..fe8c52560e6 100644 --- a/src/lib/onboard/sandbox-dockerfile-patch-flow.test.ts +++ b/src/lib/onboard/sandbox-dockerfile-patch-flow.test.ts @@ -151,7 +151,9 @@ describe("prepareSandboxDockerfilePatch", () => { sandboxGpuConfig, { dockerDriverGateway: true, + gatewayPort: undefined, log, + selectedRoute: "none", }, ); expect(patchStagedDockerfile).toHaveBeenCalledWith( diff --git a/src/lib/onboard/sandbox-dockerfile-patch-flow.ts b/src/lib/onboard/sandbox-dockerfile-patch-flow.ts index 30589b67a2c..01067119366 100644 --- a/src/lib/onboard/sandbox-dockerfile-patch-flow.ts +++ b/src/lib/onboard/sandbox-dockerfile-patch-flow.ts @@ -9,6 +9,7 @@ import { } from "../sandbox-base-image"; import { DEFAULT_TOOL_DISCLOSURE, type ToolDisclosure } from "../tool-disclosure"; import type { DcodeAutoApprovalMode } from "./dcode-auto-approval"; +import type { SelectedDockerGpuRoute } from "./docker-gpu-route"; import type { SandboxGpuConfig } from "./sandbox-gpu-mode"; type DockerRunResult = { status: number | null }; @@ -46,6 +47,7 @@ export type PrepareSandboxDockerfilePatchInput = { dcodeAutoApprovalMode?: DcodeAutoApprovalMode; hermesToolGateways: string[]; sandboxGpuConfig: SandboxGpuConfig; + selectedGpuRoute?: SelectedDockerGpuRoute; resolutionHint?: SandboxBaseImageResolutionMetadata | null; preResolvedBaseImageMetadata?: SandboxBaseImageResolutionMetadata | null; forceBaseImageRefresh?: boolean; @@ -118,6 +120,7 @@ export async function prepareSandboxDockerfilePatch({ dcodeAutoApprovalMode, hermesToolGateways, sandboxGpuConfig, + selectedGpuRoute = "none", resolutionHint = null, preResolvedBaseImageMetadata = null, forceBaseImageRefresh = false, @@ -170,6 +173,7 @@ export async function prepareSandboxDockerfilePatch({ sandboxGpuConfig, { dockerDriverGateway, + selectedRoute: selectedGpuRoute, gatewayPort, log, }, diff --git a/src/lib/onboard/sandbox-gpu-cleanup-verification.test.ts b/src/lib/onboard/sandbox-gpu-cleanup-verification.test.ts new file mode 100644 index 00000000000..0902c178ce0 --- /dev/null +++ b/src/lib/onboard/sandbox-gpu-cleanup-verification.test.ts @@ -0,0 +1,197 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { + cleanupNativeGpuAttemptForFallback, + type NativeGpuFallbackCleanupResult, +} from "./sandbox-gpu-create-attempt"; +import { + CLEANUP_POLL_INTERVAL_MS, + MAX_CLEANUP_ATTEMPTS, + STABLE_ABSENCE_CHECKS, +} from "./sandbox-gpu-fallback-constants"; + +const SAFE_CLEANUP: NativeGpuFallbackCleanupResult = { + safe: true, + reason: null, + deleteStatus: 0, + sandboxPresent: false, + containerIds: [], +}; + +type CleanupDeps = Parameters[1]; +type CleanupOptions = Parameters[2]; +type CommandResult = ReturnType; +type ContainerResult = ReturnType>; + +const ABSENT = { ok: true as const, ids: [] }; + +function sequence(input: T | T[]) { + const values = Array.isArray(input) ? [...input] : [input]; + const last = values.at(-1) as T; + return () => values.shift() ?? last; +} + +function scenario({ + list, + containers = ABSENT, + deletion = { status: 0 }, + options, +}: { + list: CommandResult | CommandResult[]; + containers?: ContainerResult | ContainerResult[]; + deletion?: CommandResult; + options?: CleanupOptions; +}) { + const nextList = sequence(list); + const nextContainers = sequence(containers); + const runOpenshell = vi.fn((args: string[]) => (args[1] === "delete" ? deletion : nextList())); + const queryContainers = vi.fn(nextContainers); + const sleep = vi.fn(); + const result = cleanupNativeGpuAttemptForFallback( + "alpha", + { runOpenshell, queryContainers, sleep }, + options, + ); + return { queryContainers, result, runOpenshell, sleep }; +} + +describe("cleanupNativeGpuAttemptForFallback", () => { + it("uses the documented fail-closed cleanup limits by default", () => { + const { result, runOpenshell, sleep } = scenario({ + list: { status: 0, stdout: "alpha Ready" }, + }); + + expect(result.safe).toBe(false); + expect(MAX_CLEANUP_ATTEMPTS).toBe(5); + expect(STABLE_ABSENCE_CHECKS).toBe(2); + expect(CLEANUP_POLL_INTERVAL_MS).toBe(1_000); + expect(runOpenshell.mock.calls.filter(([args]) => args[1] === "list")).toHaveLength( + MAX_CLEANUP_ATTEMPTS, + ); + expect(sleep).toHaveBeenCalledTimes(MAX_CLEANUP_ATTEMPTS - 1); + expect(sleep).toHaveBeenCalledWith(CLEANUP_POLL_INTERVAL_MS / 1_000); + }); + + it("requires two stable sandbox and labeled-container absence checks", () => { + const { result, runOpenshell, queryContainers } = scenario({ + list: { status: 0, stdout: "" }, + options: { maxAttempts: 3, stableAbsenceChecks: 2 }, + }); + + expect(result).toEqual(SAFE_CLEANUP); + expect(runOpenshell).toHaveBeenNthCalledWith( + 1, + ["sandbox", "delete", "alpha"], + expect.objectContaining({ ignoreError: true }), + ); + expect(runOpenshell.mock.calls.filter(([args]) => args[1] === "list")).toHaveLength(2); + expect(queryContainers).toHaveBeenCalledTimes(2); + }); + + it("waits through propagated presence before proving two stable absence checks", () => { + const { result, runOpenshell, queryContainers, sleep } = scenario({ + list: [ + { status: 0, stdout: "alpha Ready" }, + { status: 0, stdout: "alpha Ready" }, + { status: 0, stdout: "" }, + { status: 0, stdout: "" }, + ], + containers: [{ ok: true, ids: ["container-a"] }, ABSENT, ABSENT, ABSENT], + options: { maxAttempts: 5, stableAbsenceChecks: 2 }, + }); + + expect(result).toEqual(SAFE_CLEANUP); + expect(runOpenshell).toHaveBeenCalledTimes(5); + expect(queryContainers).toHaveBeenCalledTimes(4); + expect(sleep).toHaveBeenCalledTimes(3); + expect(sleep).toHaveBeenCalledWith(1); + }); + + it("permits fallback after a nonzero delete only when two checks prove complete absence", () => { + const { result } = scenario({ + list: { status: 0, stdout: "" }, + deletion: { status: 1, stderr: "delete denied" }, + options: { maxAttempts: 2, stableAbsenceChecks: 2 }, + }); + + expect(result.safe).toBe(true); + expect(result.deleteStatus).toBe(1); + expect(result.reason).toBeNull(); + }); + + it("permits fallback after a transient gateway list failure recovers to stable absence", () => { + const { result, runOpenshell, queryContainers, sleep } = scenario({ + list: [ + { status: 1, stderr: "gateway unavailable" }, + { status: 0, stdout: "" }, + { status: 0, stdout: "" }, + ], + options: { maxAttempts: 3, stableAbsenceChecks: 2 }, + }); + + expect(result).toEqual(SAFE_CLEANUP); + expect(runOpenshell).toHaveBeenCalledTimes(4); + expect(queryContainers).toHaveBeenCalledTimes(3); + expect(sleep).toHaveBeenCalledTimes(2); + }); + + it("exhausts the fixed poll bound when gateway absence cannot be proven", () => { + const { result, runOpenshell, queryContainers, sleep } = scenario({ + list: { status: 1, stderr: "gateway unavailable" }, + }); + + expect(result).toMatchObject({ + safe: false, + sandboxPresent: null, + containerIds: [], + reason: "gateway unavailable", + }); + expect(runOpenshell.mock.calls.filter(([args]) => args[1] === "list")).toHaveLength( + MAX_CLEANUP_ATTEMPTS, + ); + expect(queryContainers).toHaveBeenCalledTimes(MAX_CLEANUP_ATTEMPTS); + expect(sleep).toHaveBeenCalledTimes(MAX_CLEANUP_ATTEMPTS - 1); + }); + + it.each([ + [ + "refuses fallback when the OpenShell sandbox query fails", + { list: { status: 1, stderr: "gateway unavailable" } }, + { sandboxPresent: null }, + "gateway unavailable", + ], + [ + "refuses fallback when the labeled-container query fails", + { + list: { status: 0, stdout: "" }, + containers: { ok: false as const, ids: [] as [], error: "docker daemon unavailable" }, + }, + { containerIds: null }, + "docker daemon unavailable", + ], + [ + "refuses fallback while any labeled container remains", + { + list: { status: 0, stdout: "" }, + deletion: { status: 1, stderr: "sandbox was never created" }, + containers: { ok: true as const, ids: ["container-a", "container-b"] as string[] }, + }, + { deleteStatus: 1, containerIds: ["container-a", "container-b"] }, + "container-a, container-b", + ], + [ + "treats an exact sandbox row with no parseable status as present", + { list: { status: 0, stdout: "alpha" } }, + { sandboxPresent: true }, + "still present", + ], + ] as const)("%s (fail-closed cleanup)", (_title, input, expected, reason) => { + const { result } = scenario({ ...input, options: { maxAttempts: 2 } }); + + expect(result).toMatchObject({ safe: false, ...expected }); + expect(result.reason).toContain(reason); + }); +}); diff --git a/src/lib/onboard/sandbox-gpu-create-attempt.ts b/src/lib/onboard/sandbox-gpu-create-attempt.ts new file mode 100644 index 00000000000..752166719b5 --- /dev/null +++ b/src/lib/onboard/sandbox-gpu-create-attempt.ts @@ -0,0 +1,291 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { hasSandboxListEntry } from "../state/gateway"; +import { + canFallbackToDockerGpuCompatibility, + type DockerGpuRoutePlan, + initialDockerGpuRoute, + type SelectedDockerGpuRoute, +} from "./docker-gpu-route"; +import { + type OpenShellDockerSandboxContainerQuery, + queryOpenShellDockerSandboxContainers, +} from "./openshell-docker-sandbox-containers"; +import { + CLEANUP_POLL_INTERVAL_MS, + MAX_CLEANUP_ATTEMPTS, + STABLE_ABSENCE_CHECKS, +} from "./sandbox-gpu-fallback-constants"; + +export type SandboxGpuCreateFailureStage = "create" | "readiness" | "gpu-proof"; + +export type SandboxGpuCreateAttemptSuccess = { + ok: true; + route: SelectedDockerGpuRoute; + value: T; +}; + +export type SandboxGpuCreateAttemptFailure = { + ok: false; + route: SelectedDockerGpuRoute; + stage: SandboxGpuCreateFailureStage; + error: unknown; + fallbackEligible: boolean; +}; + +export type SandboxGpuCreateAttemptResult = + | SandboxGpuCreateAttemptSuccess + | SandboxGpuCreateAttemptFailure; + +export type SandboxGpuCreatePlanFailure = SandboxGpuCreateAttemptFailure & { + cleanupRefused?: string; + preparationRefused?: string; +}; + +export type SandboxGpuCreatePlanResult = + | SandboxGpuCreateAttemptSuccess + | SandboxGpuCreatePlanFailure; + +export type NativeGpuFallbackCleanupResult = { + safe: boolean; + reason: string | null; + deleteStatus: number | null; + sandboxPresent: boolean | null; + containerIds: string[] | null; +}; + +type CommandResult = { + status?: number | null; + stdout?: string | Buffer | null; + stderr?: string | Buffer | null; +}; + +export type NativeGpuFallbackCleanupDeps = { + runOpenshell(args: string[], options?: Record): CommandResult; + queryContainers?: (sandboxName: string) => OpenShellDockerSandboxContainerQuery; + sleep?: (seconds: number) => void; +}; + +/** + * SOURCE_OF_TRUTH_REVIEW (native failure classification; #6110) + * invalidState: non-GPU failure or sandbox-controlled output authorizes a broader retry. + * sourceBoundary: accept only strict pre-progress `--gpu` rejection or exact-container Docker + * runtime evidence; proof output also requires host configuration proving attachment absent. + * whyNotSourceFix: supported OpenShell/Docker versions cannot be upgraded atomically. + * regressionTest: create failure classification, fallback orchestration, cleanup, and live Hermes. + * removalCondition: native injection replaces compatibility on every supported host. + * Ordinary Linux also requires explicit `NEMOCLAW_DOCKER_GPU_PATCH=fallback`; WSL/Jetson are + * separately gated, and unrelated create/readiness failures retain their existing paths. + */ +export function isNativeGpuCreatePreBuildRejection(output: string): boolean { + const text = String(output ?? ""); + if (text.length > 4096) return false; + const lines = text + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean); + if (lines.length === 0 || lines.length > 4) return false; + const [errorLine, ...envelope] = lines; + const exactError = + /^error:\s+(?:unexpected|unrecognized|unknown|unsupported)\s+(?:argument|option|flag)(?:\s+|:\s*)['"`]?--gpu['"`]?(?:\s+(?:found|provided|specified))?\.?$/i.test( + errorLine, + ) || + /^error:\s+(?:argument|option|flag)\s+['"`]?--gpu['"`]?\s+(?:is not supported|was rejected)\.?$/i.test( + errorLine, + ); + return ( + exactError && + envelope.every( + (line) => + /^tip:\s+to pass ['"`]--gpu['"`] as a value, use ['"`]-- --gpu['"`]\.?$/i.test(line) || + /^Usage:\s+openshell sandbox create(?:\s|$)/.test(line) || + /^For more information, try ['"`]--help['"`]\.?$/i.test(line), + ) + ); +} + +export function isNativeGpuCreateRoutingFailure( + output: string, + options: { sawProgress: boolean }, +): boolean { + // Create output can contain arbitrary image build logs. Only a strict CLI + // parser rejection before any build/create progress is trusted here. + return options.sawProgress !== true && isNativeGpuCreatePreBuildRejection(output); +} + +export function isTrustedNativeGpuRuntimeError(error: string): boolean { + const text = String(error ?? "").trim(); + if (!text) return false; + // Match complete Docker/NVIDIA runtime clauses, not a conjunction of tokens: + // `.State.Error` can quote image-controlled WORKDIR/CMD paths. + const selector = String.raw`nvidia\.com\/gpu=[A-Za-z0-9._-]+`; + const selectors = String.raw`${selector}(?:,\s*${selector})*`; + const directCdi = new RegExp( + String.raw`^(?:Error response from daemon:\s*)?(?:CDI device injection failed:\s*)?unresolvable CDI devices ${selectors}\.?$`, + "i", + ); + const customDeviceCdi = new RegExp( + String.raw`^error gathering device information while adding custom device ["']?${selector}["']?:\s*unresolvable CDI devices ${selectors}\.?$`, + "i", + ); + const ociCdi = new RegExp( + String.raw`^failed to create task for container:\s*failed to create shim task:\s*OCI runtime create failed:\s*(?:could not apply required modification to OCI specification:\s*)?error injecting CDI devices:\s*unresolvable CDI devices ${selectors}(?::\s*unknown)?$`, + "i", + ); + return ( + directCdi.test(text) || + customDeviceCdi.test(text) || + ociCdi.test(text) || + /^(?:error response from daemon:\s*)?could not select device driver[^\n]*with capabilities:\s*\[\[?['"]?gpu['"]?\]?\]\.?$/i.test( + text, + ) + ); +} + +export function isNativeGpuReadinessRoutingFailure(evidence: { + failurePhase: string | null; + runtimeError: string; +}): boolean { + return ( + evidence.failurePhase !== null && + ["Error", "Failed", "CrashLoopBackOff"].includes(evidence.failurePhase) && + isTrustedNativeGpuRuntimeError(evidence.runtimeError) + ); +} + +function commandText(result: CommandResult): string { + return `${String(result.stdout ?? "")}\n${String(result.stderr ?? "")}`.trim(); +} + +/** Delete a failed native attempt and prove two stable, status-bearing absences. */ +export function cleanupNativeGpuAttemptForFallback( + sandboxName: string, + deps: NativeGpuFallbackCleanupDeps, + options: { maxAttempts?: number; stableAbsenceChecks?: number } = {}, +): NativeGpuFallbackCleanupResult { + const maxAttempts = Math.max(1, options.maxAttempts ?? MAX_CLEANUP_ATTEMPTS); + const stableAbsenceChecks = Math.max(1, options.stableAbsenceChecks ?? STABLE_ABSENCE_CHECKS); + const deletion = deps.runOpenshell(["sandbox", "delete", sandboxName], { + ignoreError: true, + suppressOutput: true, + }); + const deleteStatus = deletion.status ?? null; + const queryContainers = + deps.queryContainers ?? ((name: string) => queryOpenShellDockerSandboxContainers(name)); + let stableChecks = 0; + let sandboxPresent: boolean | null = null; + let containerIds: string[] | null = null; + let lastReason = + deleteStatus === 0 ? "cleanup absence has not been verified" : commandText(deletion) || null; + + for (let attempt = 0; attempt < maxAttempts; attempt += 1) { + const list = deps.runOpenshell(["sandbox", "list"], { + ignoreError: true, + suppressOutput: true, + }); + const listOk = Number(list.status ?? 1) === 0; + sandboxPresent = listOk ? hasSandboxListEntry(String(list.stdout ?? ""), sandboxName) : null; + const containers = queryContainers(sandboxName); + containerIds = containers.ok ? containers.ids : null; + + if (listOk && sandboxPresent === false && containers.ok && containers.ids.length === 0) { + stableChecks += 1; + if (stableChecks >= stableAbsenceChecks) { + return { + safe: true, + reason: null, + deleteStatus, + sandboxPresent: false, + containerIds: [], + }; + } + } else { + stableChecks = 0; + lastReason = !listOk + ? commandText(list) || "openshell sandbox list failed" + : sandboxPresent + ? `sandbox '${sandboxName}' is still present` + : !containers.ok + ? containers.error + : `labeled Docker containers remain: ${containers.ids.join(", ")}`; + } + if (attempt < maxAttempts - 1) deps.sleep?.(CLEANUP_POLL_INTERVAL_MS / 1_000); + } + + return { + safe: false, + reason: lastReason || "cleanup absence could not be proven", + deleteStatus, + sandboxPresent, + containerIds, + }; +} + +export type SandboxGpuCreatePlanDeps = { + runAttempt(route: SelectedDockerGpuRoute): Promise>; + captureNativeFailure?(failure: SandboxGpuCreateAttemptFailure): void; + cleanupNativeFailure(): NativeGpuFallbackCleanupResult | Promise; + /** Validate and render the retry without mutating host or process state. */ + prepareCompatibilityAttempt(failure: SandboxGpuCreateAttemptFailure): void | Promise; + /** Apply compatibility side effects only after native cleanup is proven safe. */ + activateCompatibilityAttempt(failure: SandboxGpuCreateAttemptFailure): void | Promise; + traceEvent?(name: string, attributes?: Record): void; +}; + +/** Execute the internal GPU strategy with at most one compatibility retry. */ +export async function executeSandboxGpuCreatePlan( + plan: DockerGpuRoutePlan, + deps: SandboxGpuCreatePlanDeps, +): Promise> { + const initialRoute = initialDockerGpuRoute(plan); + const first = await deps.runAttempt(initialRoute); + if (first.ok) { + if (first.route === "native") { + deps.traceEvent?.("gpu_native_success", { route: first.route }); + } + return first; + } + if ( + first.route !== "native" || + !first.fallbackEligible || + !canFallbackToDockerGpuCompatibility(plan) + ) { + return first; + } + + try { + deps.captureNativeFailure?.(first); + } catch { + // Diagnostics are best effort; cleanup safety remains the retry gate. + } + try { + await deps.prepareCompatibilityAttempt(first); + } catch (error) { + return { + ...first, + preparationRefused: error instanceof Error ? error.message : String(error), + }; + } + const cleanup = await deps.cleanupNativeFailure(); + if (!cleanup.safe) { + return { + ...first, + cleanupRefused: cleanup.reason ?? "native GPU cleanup could not be proven safe", + }; + } + try { + await deps.activateCompatibilityAttempt(first); + } catch (error) { + return { + ...first, + preparationRefused: error instanceof Error ? error.message : String(error), + }; + } + deps.traceEvent?.("gpu_compatibility_fallback", { + from_route: "native", + to_route: "compatibility", + failure_stage: first.stage, + }); + return deps.runAttempt("compatibility"); +} diff --git a/src/lib/onboard/sandbox-gpu-create-failure-classification.test.ts b/src/lib/onboard/sandbox-gpu-create-failure-classification.test.ts new file mode 100644 index 00000000000..9b2e7959093 --- /dev/null +++ b/src/lib/onboard/sandbox-gpu-create-failure-classification.test.ts @@ -0,0 +1,68 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + isNativeGpuCreatePreBuildRejection, + isNativeGpuCreateRoutingFailure, + isNativeGpuReadinessRoutingFailure, + isTrustedNativeGpuRuntimeError, +} from "./sandbox-gpu-create-attempt"; + +describe("native GPU create failure classification", () => { + it("accepts an argument rejection without treating unrelated build failures as routing", () => { + const rejection = "error: unexpected argument '--gpu' found"; + expect(isNativeGpuCreatePreBuildRejection(rejection)).toBe(true); + for (const [message, sawProgress, expected] of [ + [rejection, false, true], + [rejection, true, false], + ["Docker build failed while compiling a GPU Python package for --gpu support", false, false], + ["x509: certificate signed by unknown authority", false, false], + ["notice: error: unexpected argument '--gpu' found while compiling docs", false, false], + ["error: unexpected argument '--gpu' found\nimage-controlled trailing output", false, false], + [ + "error: unexpected argument '--gpu' found\nUsage: openshell sandbox create [OPTIONS]\nFor more information, try '--help'.", + false, + true, + ], + ] as const) { + expect(isNativeGpuCreateRoutingFailure(message, { sawProgress })).toBe(expected); + } + }); + + it("requires exact-target terminal phase plus host runtime evidence for readiness fallback", () => { + for (const [failurePhase, runtimeError, expected] of [ + ["Failed", "policy denied startup exec for gpu-device-initialization-failed", false], + [null, "CDI device injection failed: unresolvable nvidia.com/gpu=all", false], + ["Error", "CDI device injection failed: unresolvable CDI devices nvidia.com/gpu=all", true], + ] as const) { + expect(isNativeGpuReadinessRoutingFailure({ failurePhase, runtimeError })).toBe(expected); + } + }); + + it("recognizes only narrow host-owned OCI/CDI GPU runtime errors", () => { + for (const [message, expected] of [ + ["unresolvable CDI devices nvidia.com/gpu=all", true], + [ + "failed to create task for container: failed to create shim task: OCI runtime create failed: error injecting CDI devices: unresolvable CDI devices nvidia.com/gpu=all: unknown", + true, + ], + ['could not select device driver "" with capabilities: [[gpu]]', true], + ["Docker build failed while compiling CUDA support", false], + ["CDI device injection failed: unresolvable CDI devices example.com/widget=all", false], + [ + 'failed to create task: exec: "CDI injection failed nvidia.com/gpu=all": executable file not found', + false, + ], + [ + 'chdir to cwd ("/CDI device injection failed/nvidia.com/gpu=all") set in config.json failed: no such file or directory', + false, + ], + ["nvidia-container-cli: requirement error: unsatisfied condition: cuda>=999", false], + ["nvidia-container-cli: mount error: failed to mount /image-controlled/path", false], + ] as const) { + expect(isTrustedNativeGpuRuntimeError(message)).toBe(expected); + } + }); +}); diff --git a/src/lib/onboard/sandbox-gpu-create-flow.test.ts b/src/lib/onboard/sandbox-gpu-create-flow.test.ts new file mode 100644 index 00000000000..dc0a63b01d7 --- /dev/null +++ b/src/lib/onboard/sandbox-gpu-create-flow.test.ts @@ -0,0 +1,689 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + streamSandboxCreate: vi.fn(), + waitForCreatedSandboxReadyWithTrace: vi.fn(), + printReadinessFailure: vi.fn(), + enforceDockerGpuPatchPreserveNetwork: vi.fn(), + verifyGpuSandboxAccessAfterReady: vi.fn(), + createDockerGpuSandboxCreatePatch: vi.fn(), + printSandboxCreateFailureDiagnostics: vi.fn(), + collectDockerGpuPatchDiagnostics: vi.fn(), + queryOpenShellDockerSandboxContainers: vi.fn(), + queryOpenShellDockerSandboxRuntimeSnapshot: vi.fn(), +})); + +vi.mock("../sandbox/create-stream", () => ({ + streamSandboxCreate: mocks.streamSandboxCreate, +})); + +vi.mock("./sandbox-readiness-tracing", () => ({ + waitForCreatedSandboxReadyWithTrace: mocks.waitForCreatedSandboxReadyWithTrace, + printReadinessFailure: mocks.printReadinessFailure, +})); + +vi.mock("./docker-gpu-local-inference", () => ({ + enforceDockerGpuPatchPreserveNetwork: mocks.enforceDockerGpuPatchPreserveNetwork, + verifyGpuSandboxAccessAfterReady: mocks.verifyGpuSandboxAccessAfterReady, +})); + +vi.mock("./docker-gpu-sandbox-create", () => ({ + createDockerGpuSandboxCreatePatch: mocks.createDockerGpuSandboxCreatePatch, +})); + +vi.mock("./sandbox-create-failure", () => ({ + printSandboxCreateFailureDiagnostics: mocks.printSandboxCreateFailureDiagnostics, +})); + +vi.mock("./docker-gpu-patch", async (importOriginal) => ({ + ...(await importOriginal()), + collectDockerGpuPatchDiagnostics: mocks.collectDockerGpuPatchDiagnostics, +})); + +vi.mock("./openshell-docker-sandbox-containers", async (importOriginal) => ({ + ...(await importOriginal()), + queryOpenShellDockerSandboxContainers: mocks.queryOpenShellDockerSandboxContainers, + queryOpenShellDockerSandboxRuntimeSnapshot: mocks.queryOpenShellDockerSandboxRuntimeSnapshot, +})); + +import type { SandboxGpuProofResult } from "../state/registry"; +import { + createGpuFlowDeps as createDeps, + createGpuFlowInput as createInput, + createGpuPatchFixture as createPatch, + GPU_IMAGE_ID as IMAGE_ID, + resetGpuFlowMocks, + setupGpuFlowMocks, + VERIFIED_GPU_PROOF as VERIFIED_PROOF, +} from "./__test-helpers__/sandbox-gpu-create-flow"; +import { + runSandboxGpuCreateFlow, + type SandboxGpuCreateFlowDeps, + type SandboxGpuCreateFlowInput, +} from "./sandbox-gpu-create-flow"; + +const FAILED_PROOF: SandboxGpuProofResult = { + status: "failed", + cudaVerified: false, + label: "cuInit(0) via libcuda.so.1", + detail: "cuInit(0)=999", + at: "2026-07-06T00:00:00.000Z", +}; +const NVIDIA_SMI_FAILED_PROOF: SandboxGpuProofResult = { + status: "failed", + cudaVerified: false, + label: "nvidia-smi when available", + detail: "Failed to initialize NVML: Driver/library version mismatch", + at: "2026-07-06T00:00:00.000Z", +}; +const DEFAULT_RUNTIME_SNAPSHOT = { + ok: true as const, + imageId: IMAGE_ID, + bookkeepingImageRef: "openshell/sandbox-from:test", + stateError: "", + nativeGpuAttachmentState: "absent" as const, + containerId: "container-a", +}; + +function failNativeCreate(output = "error: unexpected argument '--gpu' found"): void { + mocks.streamSandboxCreate.mockResolvedValueOnce({ status: 1, output, sawProgress: false }); +} + +async function expectFlowExit( + input: SandboxGpuCreateFlowInput, + deps: SandboxGpuCreateFlowDeps, +): Promise { + mockExit(); + await expect(runSandboxGpuCreateFlow(input, deps)).rejects.toThrow("process.exit:1"); +} + +function mockExit(status = 1) { + return vi.spyOn(process, "exit").mockImplementation(() => { + throw new Error(`process.exit:${status}`); + }); +} + +function mockRuntimeSnapshot(overrides: Record = {}): void { + mocks.queryOpenShellDockerSandboxRuntimeSnapshot.mockReturnValue({ + ...DEFAULT_RUNTIME_SNAPSHOT, + ...overrides, + }); +} + +function mockReadinessFailure(failurePhase = "Failed"): void { + mocks.waitForCreatedSandboxReadyWithTrace.mockReturnValue({ + ready: false, + reason: "terminal_failure_phase", + failurePhase, + }); +} + +function expectNativeStateKept(deps: ReturnType): void { + expect(mocks.streamSandboxCreate).toHaveBeenCalledOnce(); + expect(deps.runOpenshell).not.toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.anything(), + ); +} + +function errorOutput(): string { + return vi.mocked(console.error).mock.calls.flat().join("\n"); +} + +function createSourceInput(): SandboxGpuCreateFlowInput { + const input = createInput(); + input.prebuild = { + createArgs: ["--from", "/tmp/build/Dockerfile", "--name", "alpha", "--gpu"], + imageRef: null, + imageId: null, + }; + return input; +} + +beforeEach(() => setupGpuFlowMocks(mocks)); +afterEach(resetGpuFlowMocks); + +describe("runSandboxGpuCreateFlow proof authorization", () => { + it("does not retry compatibility when the native proof throws an exec/policy error (#6110)", async () => { + const deps = createDeps(); + vi.mocked(deps.verifyDirectSandboxGpu).mockImplementation(() => { + throw new Error("openshell sandbox exec denied by policy"); + }); + + await expect(runSandboxGpuCreateFlow(createInput(), deps)).rejects.toThrow( + "openshell sandbox exec denied by policy", + ); + expect(mocks.streamSandboxCreate).toHaveBeenCalledOnce(); + expect(deps.runOpenshell).not.toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.anything(), + ); + }); + + it("does not let sandbox-controlled CUDA output authorize compatibility fallback (#6110)", async () => { + const deps = createDeps(); + vi.mocked(deps.verifyDirectSandboxGpu).mockReturnValue(FAILED_PROOF); + vi.spyOn(process, "exit").mockImplementation(() => { + throw new Error("process.exit:1"); + }); + + await expect(runSandboxGpuCreateFlow(createInput(), deps)).rejects.toThrow("process.exit:1"); + + expect(mocks.streamSandboxCreate).toHaveBeenCalledOnce(); + expect(deps.runOpenshell).not.toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.anything(), + ); + expect(vi.mocked(console.error).mock.calls.flat().join("\n")).toContain( + "cannot authorize a less-confined compatibility retry", + ); + }); + + it("retries structured nvidia-smi failure only when host config proves no GPU attachment (#6110)", async () => { + const deps = createDeps(); + vi.mocked(deps.verifyDirectSandboxGpu) + .mockReturnValueOnce(NVIDIA_SMI_FAILED_PROOF) + .mockReturnValue(VERIFIED_PROOF); + mockRuntimeSnapshot(); + + await expect(runSandboxGpuCreateFlow(createInput(), deps)).resolves.toMatchObject({ + route: "compatibility", + registryImageRef: "openshell/sandbox-from:test", + }); + + expect(mocks.streamSandboxCreate).toHaveBeenCalledTimes(2); + expect(deps.runOpenshell).toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.objectContaining({ suppressOutput: true }), + ); + }); + + it.each([ + "present", + "unknown", + ] as const)("fails closed on sandbox nvidia-smi text when host GPU attachment is %s", async (nativeGpuAttachmentState) => { + const deps = createDeps(); + vi.mocked(deps.verifyDirectSandboxGpu).mockReturnValue(NVIDIA_SMI_FAILED_PROOF); + mockRuntimeSnapshot({ nativeGpuAttachmentState }); + mockExit(); + + await expect(runSandboxGpuCreateFlow(createInput(), deps)).rejects.toThrow("process.exit:1"); + + expect(mocks.streamSandboxCreate).toHaveBeenCalledOnce(); + expect(mocks.queryOpenShellDockerSandboxRuntimeSnapshot).toHaveBeenCalledOnce(); + expect(deps.runOpenshell).not.toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.anything(), + ); + expect(vi.mocked(console.error).mock.calls.flat().join("\n")).toContain( + "without corroborating host evidence cannot authorize", + ); + }); + + it("stops after one compatibility retry when its GPU proof also fails", async () => { + const deps = createDeps(); + vi.mocked(deps.verifyDirectSandboxGpu).mockReturnValue(NVIDIA_SMI_FAILED_PROOF); + mockRuntimeSnapshot(); + const nativePatch = createPatch(); + const compatibilityPatch = createPatch(); + compatibilityPatch.verifyGpuOrExit.mockReturnValue(NVIDIA_SMI_FAILED_PROOF); + mocks.createDockerGpuSandboxCreatePatch + .mockReturnValueOnce(nativePatch) + .mockReturnValueOnce(compatibilityPatch); + + await expect(runSandboxGpuCreateFlow(createInput(), deps)).rejects.toThrow( + "Sandbox GPU proof returned failed status", + ); + + expect(mocks.streamSandboxCreate).toHaveBeenCalledTimes(2); + expect( + vi + .mocked(deps.runOpenshell) + .mock.calls.filter(([args]) => (args as string[]).includes("delete")), + ).toHaveLength(1); + }); + + it("hard-stops a returned failed proof in compatibility-only mode", async () => { + const input = createInput(); + input.gpuRoutePlan = "compatibility-only"; + input.initialGpuRoute = "compatibility"; + mocks.createDockerGpuSandboxCreatePatch.mockImplementation(() => { + const patch = createPatch(); + patch.verifyGpuOrExit.mockReturnValue(NVIDIA_SMI_FAILED_PROOF); + return patch; + }); + + await expect(runSandboxGpuCreateFlow(input, createDeps())).rejects.toThrow( + "Sandbox GPU proof returned failed status", + ); + + expect(mocks.streamSandboxCreate).toHaveBeenCalledOnce(); + }); +}); + +describe("runSandboxGpuCreateFlow native failure and readiness", () => { + it("persists the Hermes startup command on the no-GPU Docker route", async () => { + const input = createInput(); + input.sandboxGpuConfig = { + ...input.sandboxGpuConfig, + mode: "0", + sandboxGpuEnabled: false, + }; + input.gpuRoutePlan = "none"; + input.initialGpuRoute = "none"; + input.createArgv = ["openshell", "sandbox", "create"]; + input.persistStartupCommand = true; + + await expect(runSandboxGpuCreateFlow(input, createDeps())).resolves.toMatchObject({ + route: "none", + }); + + expect(mocks.createDockerGpuSandboxCreatePatch).toHaveBeenCalledWith( + expect.objectContaining({ route: "none", persistStartupCommand: true }), + ); + }); + + it("does not replace a native GPU container solely to persist its startup command", async () => { + const input = createInput(); + input.persistStartupCommand = true; + + await expect(runSandboxGpuCreateFlow(input, createDeps())).resolves.toMatchObject({ + route: "native", + }); + + expect(mocks.createDockerGpuSandboxCreatePatch).toHaveBeenCalledWith( + expect.objectContaining({ route: "native", persistStartupCommand: false }), + ); + }); + + it.each([ + { + failure: "image build", + output: "Docker build failed while compiling a GPU Python package for --gpu support", + }, + { + failure: "image upload", + output: "[progress] Uploaded to gateway\nfailed to upload image tar into container", + }, + { + failure: "TLS handshake", + output: "x509: certificate signed by unknown authority", + }, + { + failure: "provider credential validation", + output: "Provider credential validation failed: required token is unavailable", + }, + { + failure: "policy application", + output: "Sandbox policy application failed: requested policy was denied", + }, + ])("does not retry compatibility for a $failure failure (#6110)", async ({ output }) => { + mocks.streamSandboxCreate.mockResolvedValueOnce({ + status: 1, + output, + sawProgress: true, + }); + const deps = createDeps(); + mockExit(); + + await expect(runSandboxGpuCreateFlow(createInput(), deps)).rejects.toThrow("process.exit:1"); + expect(mocks.streamSandboxCreate).toHaveBeenCalledOnce(); + expect(mocks.createDockerGpuSandboxCreatePatch).toHaveBeenCalledOnce(); + expect(mocks.createDockerGpuSandboxCreatePatch).toHaveBeenCalledWith( + expect.objectContaining({ route: "native" }), + ); + expect(deps.runOpenshell).not.toHaveBeenCalled(); + }); + + it("redacts create errors and preserves their exact nonzero status (#6110)", async () => { + mocks.streamSandboxCreate.mockResolvedValueOnce({ + status: 19, + output: "provider failed with NVIDIA_API_KEY=super-secret-create-value", + sawProgress: true, + }); + const exit = mockExit(19); + + await expect(runSandboxGpuCreateFlow(createInput(), createDeps())).rejects.toThrow( + "process.exit:19", + ); + + const output = vi.mocked(console.error).mock.calls.flat().join("\n"); + expect(exit).toHaveBeenCalledWith(19); + expect(output).toMatch(/NVIDIA_API_KEY=[^\n]*\*+/); + expect(output).not.toContain("super-secret-create-value"); + }); + + it("does not retry compatibility for a non-GPU native readiness failure (#6110)", async () => { + mockReadinessFailure(); + const deps = createDeps(); + vi.mocked(deps.runCaptureOpenshell).mockReturnValue( + "gpu-device-initialization-failed Failed\nother-sandbox Error NVIDIA GPU device unavailable", + ); + mockExit(); + + await expect(runSandboxGpuCreateFlow(createInput(), deps)).rejects.toThrow("process.exit:1"); + expect(mocks.streamSandboxCreate).toHaveBeenCalledOnce(); + expect(mocks.verifyGpuSandboxAccessAfterReady).not.toHaveBeenCalled(); + expect(deps.runOpenshell).toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.objectContaining({ ignoreError: true }), + ); + expect(mocks.streamSandboxCreate).toHaveBeenCalledOnce(); + }); + + it("preserves a nonzero create status when separate readiness polling fails (#6110)", async () => { + mocks.streamSandboxCreate.mockResolvedValueOnce({ + status: 23, + output: "Created sandbox: alpha", + sawProgress: true, + }); + mocks.waitForCreatedSandboxReadyWithTrace.mockReturnValue({ + ready: false, + reason: "timeout", + failurePhase: null, + }); + const exit = mockExit(23); + + await expect(runSandboxGpuCreateFlow(createInput(), createDeps())).rejects.toThrow( + "process.exit:23", + ); + + expect(exit).toHaveBeenCalledWith(23); + }); + + it("keeps native readiness on the single-Ready contract", async () => { + const deps = createDeps(); + + await expect(runSandboxGpuCreateFlow(createInput(), deps)).resolves.toMatchObject({ + route: "native", + }); + + expect(mocks.waitForCreatedSandboxReadyWithTrace).toHaveBeenCalledWith( + expect.objectContaining({ stableReadyPolls: 1 }), + ); + expect(mocks.enforceDockerGpuPatchPreserveNetwork).not.toHaveBeenCalled(); + }); +}); + +describe("runSandboxGpuCreateFlow fallback ordering", () => { + it("retries readiness only for exact-container host runtime evidence (#6110)", async () => { + mocks.waitForCreatedSandboxReadyWithTrace + .mockReturnValueOnce({ + ready: false, + reason: "terminal_failure_phase", + failurePhase: "Error", + }) + .mockReturnValue({ ready: true, reason: "ready", failurePhase: null }); + mockRuntimeSnapshot({ + stateError: "CDI device injection failed: unresolvable CDI devices nvidia.com/gpu=all", + }); + + await expect(runSandboxGpuCreateFlow(createInput(), createDeps())).resolves.toMatchObject({ + route: "compatibility", + registryImageRef: "openshell/sandbox-from:test", + }); + + expect(mocks.streamSandboxCreate).toHaveBeenCalledTimes(2); + }); + + it("streams native and compatibility attempts through direct argv without a shell (#6110)", async () => { + failNativeCreate(); + const input = createInput(); + + await expect(runSandboxGpuCreateFlow(input, createDeps())).resolves.toMatchObject({ + route: "compatibility", + }); + + expect(mocks.streamSandboxCreate).toHaveBeenNthCalledWith( + 1, + "openshell", + ["sandbox", "create", "--gpu"], + input.sandboxEnv, + expect.objectContaining({ + onPoll: expect.any(Function), + readyCheck: expect.any(Function), + }), + ); + expect(mocks.streamSandboxCreate).toHaveBeenNthCalledWith( + 2, + "openshell", + expect.arrayContaining(["sandbox", "create", "--from", IMAGE_ID]), + input.sandboxEnv, + expect.any(Object), + ); + expect(mocks.streamSandboxCreate.mock.calls.flat()).not.toContain("bash"); + expect(mocks.streamSandboxCreate.mock.calls.flat()).not.toContain("-lc"); + }); + + it("discloses the compatibility container-swap confinement tradeoff and native-only opt-out", async () => { + failNativeCreate(); + const deps = createDeps(); + + await expect(runSandboxGpuCreateFlow(createInput(), deps)).resolves.toMatchObject({ + route: "compatibility", + }); + + const warning = vi.mocked(console.warn).mock.calls.flat().join("\n"); + expect(warning).toContain("recreating the OpenShell-managed Docker container"); + expect(warning).toContain("legacy GPU compatibility envelope"); + expect(warning).toContain("may relax container confinement"); + expect(warning).toContain("NEMOCLAW_DOCKER_GPU_PATCH=fallback"); + expect(warning).toContain("explicitly authorized"); + expect(mocks.streamSandboxCreate).toHaveBeenCalledTimes(2); + expect(mocks.waitForCreatedSandboxReadyWithTrace).toHaveBeenCalledWith( + expect.objectContaining({ stableReadyPolls: 2 }), + ); + }); + + it("runs the local-provider bridge preflight only after selecting compatibility fallback", async () => { + const input = createInput(); + input.provider = "ollama-local"; + input.sandboxEnv = { + NEMOCLAW_DOCKER_GPU_PATCH_NETWORK: "host", + }; + input.sandboxGpuConfig.sandboxGpuProof = VERIFIED_PROOF; + failNativeCreate(); + const deps = createDeps(); + + await expect(runSandboxGpuCreateFlow(input, deps)).resolves.toMatchObject({ + route: "compatibility", + }); + + expect(mocks.enforceDockerGpuPatchPreserveNetwork).toHaveBeenCalledOnce(); + expect(mocks.enforceDockerGpuPatchPreserveNetwork).toHaveBeenCalledWith( + "ollama-local", + input.sandboxGpuConfig, + expect.objectContaining({ + dockerDriverGateway: true, + selectedRoute: "compatibility", + gatewayPort: 8080, + }), + ); + const cleanupComplete = + mocks.queryOpenShellDockerSandboxContainers.mock.invocationCallOrder.at(-1) ?? + Number.POSITIVE_INFINITY; + const networkPrepared = mocks.enforceDockerGpuPatchPreserveNetwork.mock.invocationCallOrder[0]; + const compatibilityCreate = mocks.streamSandboxCreate.mock.invocationCallOrder[1]; + expect(cleanupComplete).toBeLessThan(networkPrepared); + expect(networkPrepared).toBeLessThan(compatibilityCreate); + expect(input.sandboxGpuConfig.sandboxGpuProof).toBeNull(); + }); + + it("validates the full compatibility command before deleting native state (#6110)", async () => { + const input = createInput(); + input.compatibilityPolicyPath = null; + failNativeCreate(); + const deps = createDeps(); + await expectFlowExit(input, deps); + expectNativeStateKept(deps); + expect(errorOutput()).toContain("Compatibility retry policy was not materialized"); + }); + + it("keeps native state when compatibility command rendering fails (#6110)", async () => { + failNativeCreate(); + const deps = createDeps(); + vi.mocked(deps.openshellArgv).mockImplementation(() => { + throw new Error("compatibility command render rejected"); + }); + await expectFlowExit(createInput(), deps); + expectNativeStateKept(deps); + expect(errorOutput()).toContain("compatibility command render rejected"); + }); + + it("runs compatibility network preflight only after native cleanup succeeds (#6110)", async () => { + const input = createInput(); + input.provider = "ollama-local"; + failNativeCreate(); + mocks.enforceDockerGpuPatchPreserveNetwork.mockRejectedValueOnce( + new Error("compatibility bridge is unreachable"), + ); + const deps = createDeps(); + await expectFlowExit(input, deps); + expect(deps.openshellArgv).toHaveBeenCalledOnce(); + expect(deps.runOpenshell).toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.anything(), + ); + expect(mocks.streamSandboxCreate).toHaveBeenCalledOnce(); + expect(errorOutput()).toContain("compatibility bridge is unreachable"); + }); +}); + +describe("runSandboxGpuCreateFlow cleanup and provenance", () => { + it("does not let a stale same-label container authorize or receive fallback cleanup", async () => { + mocks.queryOpenShellDockerSandboxContainers.mockReturnValue({ + ok: true, + ids: ["stale-container"], + }); + failNativeCreate(); + const deps = createDeps(); + await expectFlowExit(createInput(), deps); + + expect(mocks.streamSandboxCreate).toHaveBeenCalledOnce(); + expect(mocks.queryOpenShellDockerSandboxRuntimeSnapshot).not.toHaveBeenCalled(); + expect(deps.runOpenshell).not.toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.anything(), + ); + }); + + it("reports manual cleanup when ordinary readiness deletion fails (#6110)", async () => { + mockReadinessFailure(); + const deps = createDeps(); + vi.mocked(deps.runOpenshell).mockReturnValue({ status: 7, stderr: "gateway unavailable" }); + await expectFlowExit(createInput(), deps); + + const output = vi.mocked(console.error).mock.calls.flat().join("\n"); + expect(output).toContain("could not be removed automatically"); + expect(output).toContain('Manual cleanup: openshell sandbox delete "alpha"'); + expect(output).not.toContain("Retry: nemoclaw onboard"); + }); + + it("treats an already-absent sandbox as successful ordinary readiness cleanup", async () => { + mockReadinessFailure(); + const deps = createDeps(); + vi.mocked(deps.runOpenshell).mockReturnValue({ + status: 1, + stderr: "sandbox alpha not found", + }); + await expectFlowExit(createInput(), deps); + + const output = vi.mocked(console.error).mock.calls.flat().join("\n"); + expect(output).toContain("Retry: nemoclaw onboard"); + expect(output).not.toContain("could not be removed automatically"); + }); + + it("fully redacts command diagnostics when cleanup cannot be proven safe", async () => { + failNativeCreate(); + const input = createInput(); + input.provider = "ollama-local"; + input.sandboxGpuConfig.sandboxGpuProof = VERIFIED_PROOF; + const deps = createDeps(); + vi.mocked(deps.runOpenshell).mockImplementation((args) => + args[1] === "delete" + ? { status: 0 } + : { status: 1, stderr: "NVIDIA_API_KEY=super-secret-cleanup-value" }, + ); + await expectFlowExit(input, deps); + + const diagnostic = vi.mocked(console.error).mock.calls.flat().join("\n"); + expect(diagnostic).toContain("Cleanup could not be proven safe"); + expect(diagnostic).toContain("NVIDIA_API_KEY="); + expect(diagnostic).not.toContain("super-secret-cleanup-value"); + expect(deps.openshellArgv).toHaveBeenCalledOnce(); + expect(mocks.enforceDockerGpuPatchPreserveNetwork).not.toHaveBeenCalled(); + expect(input.sandboxGpuConfig.sandboxGpuProof).toBe(VERIFIED_PROOF); + }); + + it("refuses nvidia-smi fallback when exact native container provenance is unavailable (#6110)", async () => { + const input = createSourceInput(); + mocks.queryOpenShellDockerSandboxRuntimeSnapshot.mockReturnValue({ + ok: false, + error: "expected one labeled sandbox container, found 2", + }); + const deps = createDeps(); + vi.mocked(deps.verifyDirectSandboxGpu).mockReturnValue({ + ...NVIDIA_SMI_FAILED_PROOF, + detail: "No devices were found", + }); + await expectFlowExit(input, deps); + + expect(mocks.streamSandboxCreate).toHaveBeenCalledOnce(); + expect(deps.runOpenshell).not.toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.anything(), + ); + expect(deps.openshellArgv).not.toHaveBeenCalled(); + }); + + it("ignores create-stream tags and reuses only the inspected immutable image", async () => { + const input = createSourceInput(); + mockRuntimeSnapshot({ + bookkeepingImageRef: "openshell/sandbox-from:built", + stateError: "CDI device injection failed: unresolvable CDI devices nvidia.com/gpu=all", + }); + mocks.streamSandboxCreate.mockResolvedValueOnce({ + status: 1, + output: + "Built image attacker.example/redirect:latest\nCDI device injection failed: unresolvable CDI devices nvidia.com/gpu=all", + sawProgress: true, + }); + const deps = createDeps(); + + await expect(runSandboxGpuCreateFlow(input, deps)).resolves.toMatchObject({ + route: "compatibility", + registryImageRef: "openshell/sandbox-from:built", + }); + + expect(deps.openshellArgv).toHaveBeenCalledWith(expect.arrayContaining(["--from", IMAGE_ID])); + expect(deps.openshellArgv).not.toHaveBeenCalledWith( + expect.arrayContaining(["--from", "attacker.example/redirect:latest"]), + ); + expect(mocks.streamSandboxCreate).toHaveBeenCalledTimes(2); + expect(mocks.queryOpenShellDockerSandboxRuntimeSnapshot).toHaveBeenCalledOnce(); + }); + + it("does not persist an immutable retry ID as the registry image tag", async () => { + const input = createSourceInput(); + mockRuntimeSnapshot({ + bookkeepingImageRef: IMAGE_ID, + stateError: "CDI device injection failed: unresolvable CDI devices nvidia.com/gpu=all", + }); + mocks.streamSandboxCreate.mockResolvedValueOnce({ + status: 1, + output: "CDI device injection failed: unresolvable CDI devices nvidia.com/gpu=all", + sawProgress: true, + }); + const deps = createDeps(); + + await expect(runSandboxGpuCreateFlow(input, deps)).resolves.toMatchObject({ + route: "compatibility", + registryImageRef: null, + }); + + expect(deps.openshellArgv).toHaveBeenCalledWith(expect.arrayContaining(["--from", IMAGE_ID])); + }); +}); diff --git a/src/lib/onboard/sandbox-gpu-create-flow.ts b/src/lib/onboard/sandbox-gpu-create-flow.ts new file mode 100644 index 00000000000..0bcd3c647d0 --- /dev/null +++ b/src/lib/onboard/sandbox-gpu-create-flow.ts @@ -0,0 +1,173 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { StreamSandboxCreateResult } from "../sandbox/create-stream"; +import { redactFull } from "../security/redact"; +import type { SandboxGpuProofResult } from "../state/registry"; +import * as dockerGpuLocalInference from "./docker-gpu-local-inference"; +import { collectDockerGpuPatchDiagnostics } from "./docker-gpu-patch"; +import type { DockerGpuPatchDeps } from "./docker-gpu-patch-types"; +import type { SelectedDockerGpuRoute } from "./docker-gpu-route"; +import { renderCompatibilityFallbackCreateArgs } from "./docker-gpu-route"; +import { adaptDockerGpuRouteForPatch } from "./docker-gpu-route-patch-adapter"; +import type { DockerGpuSandboxCreatePatch } from "./docker-gpu-sandbox-create"; +import { isImmutableDockerImageId } from "./openshell-docker-sandbox-containers"; +import * as sandboxGpuCreateAttempt from "./sandbox-gpu-create-attempt"; +import { createSandboxGpuCreateAttemptRunner } from "./sandbox-gpu-create-run-attempt"; +import type { SandboxGpuConfig } from "./sandbox-gpu-mode"; +import type { SandboxPrebuildResult } from "./sandbox-prebuild"; +import { addTraceEvent } from "./tracing"; + +type RunOpenshell = NonNullable; +type RunCaptureOpenshell = NonNullable; +type Sleep = NonNullable; + +export interface SandboxGpuCreateFlowInput { + sandboxName: string; + provider: string; + sandboxGpuConfig: SandboxGpuConfig; + gpuRoutePlan: import("./docker-gpu-route").DockerGpuRoutePlan; + initialGpuRoute: SelectedDockerGpuRoute; + compatibilityPolicyPath: string | null; + dockerDriverGateway: boolean; + gatewayPort: number; + sandboxReadyTimeoutSecs: number; + createArgv: string[]; + sandboxEnv: NodeJS.ProcessEnv; + sandboxStartupCommand: string[]; + prebuild: SandboxPrebuildResult; + restoreBackupPath: string | null; + terminalAgent: boolean; + persistStartupCommand?: boolean; +} + +export interface SandboxGpuCreateFlowDeps { + runOpenshell: RunOpenshell; + runCaptureOpenshell: RunCaptureOpenshell; + sleep: Sleep; + openshellArgv(args: string[]): string[]; + verifyDirectSandboxGpu(sandboxName: string): SandboxGpuProofResult; +} + +export interface SandboxGpuCreateFlowResult { + createResult: StreamSandboxCreateResult; + dockerGpuCreatePatch: DockerGpuSandboxCreatePatch; + route: SelectedDockerGpuRoute; + firstCreateOutput: string; + /** Mutable tag/reference retained only for registry and image-GC bookkeeping. */ + registryImageRef: string | null; +} + +/** + * SOURCE_OF_TRUTH_REVIEW (ordered native-GPU fallback; #6110) + * invalidState: native injection fails and a broader retry starts without exact evidence or cleanup. + * sourceBoundary: the operator authorizes fallback; Docker owns image, runtime, attachment, and + * cleanup evidence, while image-controlled proof output remains diagnostic only. + * whyNotSourceFix: supported OpenShell and Docker versions cannot be upgraded atomically. + * regressionTest: the create classification/orchestration/cleanup suites and live Hermes GPU flow. + * removalCondition: native injection works on all supported hosts and compatibility is retired. + * Build/upload/TLS/provider/policy/general-readiness failures retain their existing exit paths. + * The runner captures evidence; this module renders before cleanup, activates networking only + * after proven cleanup, and the attempt helper permits at most one retry. + */ +export async function runSandboxGpuCreateFlow( + input: SandboxGpuCreateFlowInput, + deps: SandboxGpuCreateFlowDeps, +): Promise { + let registryImageRef: string | null = input.prebuild.imageRef; + const attemptRunner = createSandboxGpuCreateAttemptRunner(input, deps); + const gpuCreateOutcome = await sandboxGpuCreateAttempt.executeSandboxGpuCreatePlan( + input.gpuRoutePlan, + { + runAttempt: attemptRunner.runAttempt, + captureNativeFailure: (failure) => { + const routeAdapter = adaptDockerGpuRouteForPatch(failure.route); + const diagnostics = collectDockerGpuPatchDiagnostics( + input.sandboxName, + { + error: failure.error, + additionalSummaryLines: routeAdapter.additionalSummaryLines, + }, + { runCaptureOpenshell: deps.runCaptureOpenshell }, + ); + if (diagnostics) console.error(` Native GPU diagnostics saved: ${diagnostics.dir}`); + }, + cleanupNativeFailure: () => + sandboxGpuCreateAttempt.cleanupNativeGpuAttemptForFallback(input.sandboxName, { + runOpenshell: deps.runOpenshell, + sleep: deps.sleep, + }), + prepareCompatibilityAttempt: async () => { + if (!input.compatibilityPolicyPath) { + throw new Error("Compatibility retry policy was not materialized."); + } + const nativeRuntimeSnapshot = attemptRunner.state.nativeRuntimeSnapshot; + const prebuildImageId = input.prebuild.imageId; + const imageId = + nativeRuntimeSnapshot?.imageId ?? + (prebuildImageId && isImmutableDockerImageId(prebuildImageId) + ? prebuildImageId.toLowerCase() + : null); + if ( + !registryImageRef && + nativeRuntimeSnapshot?.bookkeepingImageRef && + !isImmutableDockerImageId(nativeRuntimeSnapshot.bookkeepingImageRef) + ) { + registryImageRef = nativeRuntimeSnapshot.bookkeepingImageRef; + } + const compatibilityArgs = renderCompatibilityFallbackCreateArgs(input.prebuild.createArgs, { + imageRef: imageId, + allowUnbuiltSource: attemptRunner.state.allowUnbuiltCompatibilitySource, + compatibilityPolicyPath: input.compatibilityPolicyPath, + }); + attemptRunner.state.compatibilityArgv = deps.openshellArgv([ + "sandbox", + "create", + ...compatibilityArgs, + "--", + ...input.sandboxStartupCommand, + ]); + if (attemptRunner.state.compatibilityArgv.length === 0) { + throw new Error("Compatibility sandbox create executable is missing."); + } + }, + activateCompatibilityAttempt: async () => { + await dockerGpuLocalInference.enforceDockerGpuPatchPreserveNetwork( + input.provider, + input.sandboxGpuConfig, + { + dockerDriverGateway: input.dockerDriverGateway, + selectedRoute: "compatibility", + gatewayPort: input.gatewayPort, + log: console.log, + }, + ); + input.sandboxGpuConfig.sandboxGpuProof = null; + }, + traceEvent: addTraceEvent, + }, + ); + if (!gpuCreateOutcome.ok) { + console.error(""); + console.error(" Operator-authorized GPU fallback stopped before compatibility retry."); + if (gpuCreateOutcome.preparationRefused) { + console.error( + ` Compatibility retry could not be prepared: ${gpuCreateOutcome.preparationRefused}`, + ); + } + if (gpuCreateOutcome.cleanupRefused) { + console.error( + ` Cleanup could not be proven safe: ${redactFull(gpuCreateOutcome.cleanupRefused)}`, + ); + } + console.error(` Manual cleanup: openshell sandbox delete "${input.sandboxName}"`); + process.exit(1); + } + + return { + ...gpuCreateOutcome.value, + route: gpuCreateOutcome.route, + firstCreateOutput: attemptRunner.state.firstCreateOutput, + registryImageRef, + }; +} diff --git a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts new file mode 100644 index 00000000000..0cf98a659ad --- /dev/null +++ b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts @@ -0,0 +1,293 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { printSandboxCreateRecoveryHints } from "../build-context"; +import { getSandboxDeleteOutcome } from "../domain/sandbox/destroy"; +import { streamSandboxCreate } from "../sandbox/create-stream"; +import { getReadyCheckOutputPatternsForAgent } from "../sandbox/create-stream-ready-gate"; +import { getSandboxFailurePhase, isSandboxReady } from "../state/gateway"; +import type { SandboxGpuProofResult } from "../state/registry"; +import { classifySandboxCreateFailure } from "../validation"; +import { cliName } from "./branding"; +import { reportSandboxCreateFailure } from "./created-sandbox-failure"; +import * as dockerGpuLocalInference from "./docker-gpu-local-inference"; +import type { SelectedDockerGpuRoute } from "./docker-gpu-route"; +import { createDockerGpuSandboxCreatePatch } from "./docker-gpu-sandbox-create"; +import { + type OpenShellDockerSandboxRuntimeSnapshotQuery, + queryOpenShellDockerSandboxContainers, + queryOpenShellDockerSandboxRuntimeSnapshot, +} from "./openshell-docker-sandbox-containers"; +import { printSandboxCreateFailureDiagnostics } from "./sandbox-create-failure"; +import * as sandboxGpuCreateAttempt from "./sandbox-gpu-create-attempt"; +import type { + SandboxGpuCreateFlowDeps, + SandboxGpuCreateFlowInput, +} from "./sandbox-gpu-create-flow"; +import * as sandboxGpuPreflight from "./sandbox-gpu-preflight"; +import * as sandboxReadinessTracing from "./sandbox-readiness-tracing"; +import { addTraceEvent } from "./tracing"; + +type NativeRuntimeSnapshot = Extract; + +export type SandboxGpuCreateAttemptState = { + firstCreateOutput: string; + compatibilityArgv: string[] | null; + allowUnbuiltCompatibilitySource: boolean; + nativeRuntimeSnapshot: NativeRuntimeSnapshot | null; +}; + +// A compatibility recreate can briefly observe the original container's stale +// Ready row. Require one confirmation poll before advancing to the GPU proof. +const COMPATIBILITY_STABLE_READY_POLLS = 2; + +export function createSandboxGpuCreateAttemptRunner( + input: SandboxGpuCreateFlowInput, + deps: SandboxGpuCreateFlowDeps, +) { + const state: SandboxGpuCreateAttemptState = { + firstCreateOutput: "", + compatibilityArgv: null, + allowUnbuiltCompatibilitySource: false, + nativeRuntimeSnapshot: null, + }; + const nativeFallbackBaseline = + input.initialGpuRoute === "native" && input.gpuRoutePlan === "native-with-fallback" + ? queryOpenShellDockerSandboxContainers(input.sandboxName) + : null; + const nativeFallbackHasCleanBaseline = + nativeFallbackBaseline?.ok === true && nativeFallbackBaseline.ids.length === 0; + const inspectNativeRuntime = () => queryOpenShellDockerSandboxRuntimeSnapshot(input.sandboxName); + + const runAttempt = async (route: SelectedDockerGpuRoute) => { + const compatibility = route === "compatibility"; + if (compatibility && input.initialGpuRoute === "native") { + console.warn( + " Native OpenShell GPU onboarding did not complete; retrying once by recreating the OpenShell-managed Docker container with the legacy GPU compatibility envelope.", + ); + console.warn( + " This compatibility container swap may relax container confinement compared with native injection. The retry is running only because NEMOCLAW_DOCKER_GPU_PATCH=fallback explicitly authorized it.", + ); + } + const dockerGpuCreatePatch = createDockerGpuSandboxCreatePatch({ + route, + // Native attachment cannot be reproduced by a startup-only swap. The + // compatibility route owns its GPU envelope; no-GPU can persist startup. + persistStartupCommand: input.persistStartupCommand === true && route !== "native", + sandboxName: input.sandboxName, + gpuDevice: input.sandboxGpuConfig.sandboxGpuDevice, + openshellSandboxCommand: input.sandboxStartupCommand, + timeoutSecs: input.sandboxReadyTimeoutSecs, + backend: input.sandboxGpuConfig.hostGpuPlatform === "jetson" ? "jetson" : "generic", + deps, + }); + const attemptArgv = state.compatibilityArgv ?? input.createArgv; + const [createExecutable, ...createExecutableArgs] = attemptArgv; + if (!createExecutable) throw new Error("Sandbox create executable is missing."); + const createResult = await streamSandboxCreate( + createExecutable, + createExecutableArgs, + input.sandboxEnv, + { + readyCheck: () => { + const list = deps.runCaptureOpenshell(["sandbox", "list"], { ignoreError: true }); + return isSandboxReady(list, input.sandboxName); + }, + onPoll: () => dockerGpuCreatePatch.maybeApplyDuringCreate(), + readyCheckOutputPatterns: getReadyCheckOutputPatternsForAgent( + input.terminalAgent, + input.sandboxEnv, + ), + failureCheck: dockerGpuCreatePatch.createFailureMessage, + traceEvent: addTraceEvent, + initialPhase: + compatibility && (input.prebuild.imageRef || state.compatibilityArgv) + ? "create" + : undefined, + }, + ); + if (!state.firstCreateOutput) state.firstCreateOutput = createResult.output; + dockerGpuCreatePatch.exitOnPatchError(); + if (createResult.status !== 0) { + const failure = classifySandboxCreateFailure(createResult.output); + if (failure.kind === "sandbox_create_incomplete") { + console.warn(""); + console.warn( + ` Create stream exited with code ${createResult.status} after sandbox was created.`, + ); + console.warn(" Checking whether the sandbox reaches Ready state..."); + } else if ( + route === "native" && + input.gpuRoutePlan === "native-with-fallback" && + nativeFallbackHasCleanBaseline && + (() => { + if ( + sandboxGpuCreateAttempt.isNativeGpuCreateRoutingFailure(createResult.output, { + sawProgress: createResult.sawProgress, + }) + ) { + state.allowUnbuiltCompatibilitySource = input.prebuild.imageRef === null; + return true; + } + const snapshot = inspectNativeRuntime(); + if ( + snapshot.ok && + sandboxGpuCreateAttempt.isTrustedNativeGpuRuntimeError(snapshot.stateError) + ) { + state.nativeRuntimeSnapshot = snapshot; + return true; + } + return false; + })() + ) { + return { + ok: false, + route, + stage: "create", + error: new Error("Native OpenShell GPU sandbox creation was rejected."), + fallbackEligible: true, + } as const; + } else { + reportSandboxCreateFailure( + { + sandboxName: input.sandboxName, + createStatus: createResult.status, + createOutput: createResult.output, + restoreBackupPath: input.restoreBackupPath, + createArgs: input.prebuild.createArgs, + }, + { + classifyCreateFailure: classifySandboxCreateFailure, + printCreateFailureDiagnostics: printSandboxCreateFailureDiagnostics, + printRecoveryHints: printSandboxCreateRecoveryHints, + warn: (message) => console.warn(message), + error: (message) => console.error(message), + exitProcess: (code) => process.exit(code), + }, + ); + } + } + dockerGpuCreatePatch.ensureApplied(); + dockerGpuCreatePatch.waitForSupervisorReconnectIfNeeded(); + console.log(" Waiting for sandbox to become ready..."); + const readiness = sandboxReadinessTracing.waitForCreatedSandboxReadyWithTrace({ + sandboxName: input.sandboxName, + timeoutSecs: input.sandboxReadyTimeoutSecs, + runCaptureOpenshell: deps.runCaptureOpenshell, + isSandboxReady, + getSandboxFailurePhase, + stableReadyPolls: compatibility ? COMPATIBILITY_STABLE_READY_POLLS : 1, + sleep: deps.sleep, + }); + if (!readiness.ready) { + console.error(""); + sandboxReadinessTracing.printReadinessFailure( + readiness, + input.sandboxName, + input.sandboxReadyTimeoutSecs, + ); + const canClassifyNativeReadiness = + route === "native" && + input.gpuRoutePlan === "native-with-fallback" && + nativeFallbackHasCleanBaseline; + const runtimeSnapshot = canClassifyNativeReadiness ? inspectNativeRuntime() : null; + if ( + canClassifyNativeReadiness && + runtimeSnapshot?.ok && + sandboxGpuCreateAttempt.isNativeGpuReadinessRoutingFailure({ + failurePhase: readiness.failurePhase, + runtimeError: runtimeSnapshot.stateError, + }) + ) { + state.nativeRuntimeSnapshot = runtimeSnapshot; + return { + ok: false, + route, + stage: "readiness", + error: new Error( + `Native OpenShell GPU sandbox did not become ready${readiness.failurePhase ? ` (${readiness.failurePhase})` : ""}.`, + ), + fallbackEligible: true, + } as const; + } + printSandboxCreateFailureDiagnostics(input.sandboxName, { + backupPath: input.restoreBackupPath, + }); + if (compatibility) dockerGpuCreatePatch.printReadinessFailureIfEnabled(); + else { + const deletion = deps.runOpenshell(["sandbox", "delete", input.sandboxName], { + ignoreError: true, + suppressOutput: true, + }); + const { alreadyGone } = getSandboxDeleteOutcome({ + status: deletion.status ?? null, + stdout: String(deletion.stdout ?? ""), + stderr: String(deletion.stderr ?? ""), + }); + if (Number(deletion.status ?? 1) !== 0 && !alreadyGone) { + console.error(" The failed sandbox could not be removed automatically."); + console.error(` Manual cleanup: openshell sandbox delete "${input.sandboxName}"`); + } else console.error(` Retry: ${cliName()} onboard`); + } + process.exit(createResult.status === 0 ? 1 : createResult.status); + } + if (input.sandboxGpuConfig.sandboxGpuEnabled) { + const deferNativeProofFailure = + route === "native" && + input.gpuRoutePlan === "native-with-fallback" && + nativeFallbackHasCleanBaseline; + const proof: SandboxGpuProofResult = dockerGpuLocalInference.verifyGpuSandboxAccessAfterReady( + input.sandboxGpuConfig, + { + sandboxName: input.sandboxName, + dockerDriverGateway: input.dockerDriverGateway, + selectedRoute: route, + verifyDirectSandboxGpu: deps.verifyDirectSandboxGpu, + verifyGpuOrExit: deferNativeProofFailure + ? undefined + : dockerGpuCreatePatch.verifyGpuOrExit, + reportGpuProofFailure: !deferNativeProofFailure, + selectedMode: dockerGpuCreatePatch.selectedMode, + runCaptureOpenshell: deps.runCaptureOpenshell, + log: console.log, + }, + ); + if (deferNativeProofFailure && proof.status === "failed") { + if (sandboxGpuPreflight.isExplicitNvidiaSmiDriverProofFailure(proof)) { + const snapshot = inspectNativeRuntime(); + if (snapshot.ok && snapshot.nativeGpuAttachmentState === "absent") { + state.nativeRuntimeSnapshot = snapshot; + return { + ok: false, + route, + stage: "gpu-proof", + error: new Error( + "Native OpenShell GPU proof failed and the host confirms no GPU attachment.", + ), + fallbackEligible: true, + } as const; + } + } + console.error(""); + console.error(" Native sandbox GPU proof failed."); + console.error( + " Sandbox-reported GPU output without corroborating host evidence cannot authorize a less-confined compatibility retry.", + ); + console.error( + " To explicitly select the compatibility route, clean up the sandbox and retry with NEMOCLAW_DOCKER_GPU_PATCH=1.", + ); + process.exit(1); + } + if (proof.status === "failed") { + throw new Error("Sandbox GPU proof returned failed status."); + } + } + return { + ok: true, + route, + value: { createResult, dockerGpuCreatePatch }, + } as const; + }; + + return { state, runAttempt }; +} diff --git a/src/lib/onboard/sandbox-gpu-preflight.test.ts b/src/lib/onboard/sandbox-gpu-direct-proof.test.ts similarity index 52% rename from src/lib/onboard/sandbox-gpu-preflight.test.ts rename to src/lib/onboard/sandbox-gpu-direct-proof.test.ts index c78a044453f..a6d55681eec 100644 --- a/src/lib/onboard/sandbox-gpu-preflight.test.ts +++ b/src/lib/onboard/sandbox-gpu-direct-proof.test.ts @@ -7,157 +7,12 @@ vi.mock("../adapters/docker", () => ({ dockerInfoFormat: vi.fn(), })); -import type { SandboxGpuConfig } from "./sandbox-gpu-mode"; import { createDirectSandboxGpuVerifier, - dockerNvidiaRuntimeAvailable, - formatSandboxGpuPassthroughNote, - parseDockerRuntimeNames, - sandboxGpuRemediationLines, - validateSandboxGpuPreflight, + isExplicitNvidiaSmiDriverProofFailure, } from "./sandbox-gpu-preflight"; -function sandboxGpuConfig(overrides: Partial = {}): SandboxGpuConfig { - return { - mode: "auto", - hostGpuDetected: true, - hostGpuPlatform: "linux", - sandboxGpuEnabled: true, - sandboxGpuDevice: null, - errors: [], - ...overrides, - }; -} - -describe("sandbox GPU preflight", () => { - it("formats Jetson sandbox GPU notes around the NVIDIA runtime backend", () => { - expect(formatSandboxGpuPassthroughNote({ hostGpuPlatform: "jetson" })).toContain( - "Docker NVIDIA runtime", - ); - expect( - formatSandboxGpuPassthroughNote({ - resumeHasResolvedGpuIntent: true, - recordedGpuPassthroughBeforePreflight: true, - }), - ).toContain("Continuing GPU passthrough"); - expect(formatSandboxGpuPassthroughNote({ requestedGpuPassthrough: true })).toContain( - "GPU passthrough requested", - ); - }); - - it("parses Docker runtime names from JSON and plain-text output", () => { - expect(parseDockerRuntimeNames('{"io.containerd.runc.v2":{},"nvidia":{}}')).toContain("nvidia"); - expect(parseDockerRuntimeNames("runc nvidia io.containerd.runc.v2")).toContain("nvidia"); - expect(parseDockerRuntimeNames("")).toEqual([]); - }); - - it("checks Jetson sandbox GPU support through Docker NVIDIA runtime availability", () => { - const dockerInfo = vi.fn(() => '{"runc":{},"nvidia":{}}'); - expect(dockerNvidiaRuntimeAvailable({ dockerInfoFormat: dockerInfo })).toBe(true); - - expect(() => - validateSandboxGpuPreflight(sandboxGpuConfig({ hostGpuPlatform: "jetson" }), { - platform: "linux", - dockerInfoFormat: dockerInfo, - getDockerCdiSpecDirs: vi.fn(() => { - throw new Error("Jetson preflight must not require CDI"); - }), - findReadableNvidiaCdiSpecFiles: vi.fn(() => { - throw new Error("Jetson preflight must not inspect CDI specs"); - }), - }), - ).not.toThrow(); - expect(dockerInfo).toHaveBeenCalledWith( - "{{json .Runtimes}}", - expect.objectContaining({ ignoreError: true }), - ); - }); - - it("keeps generic Linux sandbox GPU preflight on the CDI path", () => { - const getDockerCdiSpecDirs = vi.fn(() => ["/etc/cdi"]); - const findReadableNvidiaCdiSpecFiles = vi.fn(() => ["/etc/cdi/nvidia.yaml"]); - const dockerInfo = vi.fn(() => '{"runc":{},"nvidia":{}}'); - - expect(() => - validateSandboxGpuPreflight(sandboxGpuConfig(), { - platform: "linux", - env: {}, - release: "6.8.0-generic", - procVersion: "Linux version 6.8.0-generic", - dockerInfoFormat: dockerInfo, - getDockerCdiSpecDirs, - findReadableNvidiaCdiSpecFiles, - }), - ).not.toThrow(); - expect(getDockerCdiSpecDirs).toHaveBeenCalled(); - expect(findReadableNvidiaCdiSpecFiles).toHaveBeenCalledWith(["/etc/cdi"]); - expect(dockerInfo).not.toHaveBeenCalled(); - }); - - it("skips CDI spec validation on Docker Desktop WSL so Docker --gpus can be used", () => { - const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined); - const getDockerCdiSpecDirs = vi.fn(() => ["/etc/cdi"]); - const findReadableNvidiaCdiSpecFiles = vi.fn(() => []); - - try { - expect(() => - validateSandboxGpuPreflight(sandboxGpuConfig(), { - platform: "linux", - env: { WSL_DISTRO_NAME: "Ubuntu" }, - dockerInfoFormat: vi.fn(() => '"Docker Desktop"'), - getDockerCdiSpecDirs, - findReadableNvidiaCdiSpecFiles, - }), - ).not.toThrow(); - expect(getDockerCdiSpecDirs).not.toHaveBeenCalled(); - expect(findReadableNvidiaCdiSpecFiles).not.toHaveBeenCalled(); - expect(logSpy.mock.calls.map((call) => call[0]).join("\n")).toContain( - "Docker --gpus compatibility path", - ); - } finally { - logSpy.mockRestore(); - } - }); - - it("prints neutral WSL remediation when Docker runtime cannot be determined", () => { - const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); - const exitSpy = vi.spyOn(process, "exit").mockImplementation((( - code?: number | string | null, - ) => { - throw new Error(`exit:${code}`); - }) as never); - - try { - expect(() => - validateSandboxGpuPreflight(sandboxGpuConfig(), { - platform: "linux", - env: { WSL_DISTRO_NAME: "Ubuntu" }, - dockerInfoFormat: vi.fn(() => ""), - getDockerCdiSpecDirs: vi.fn(() => ["/etc/cdi"]), - findReadableNvidiaCdiSpecFiles: vi.fn(() => []), - }), - ).toThrow("exit:1"); - const message = errorSpy.mock.calls.map((call) => call[0]).join("\n"); - expect(message).toContain("could not determine whether Docker is Docker Desktop"); - expect(message).toContain("If using Docker Desktop"); - expect(message).toContain("If using native Docker Engine inside WSL"); - expect(message).not.toContain("sudo systemctl restart docker"); - } finally { - errorSpy.mockRestore(); - exitSpy.mockRestore(); - } - }); - - it("keeps generic Linux CDI remediation outside Docker Desktop WSL", () => { - expect(sandboxGpuRemediationLines().join("\n")).toContain("sudo nvidia-ctk"); - expect(sandboxGpuRemediationLines({ wslDockerDesktop: true }).join("\n")).toContain( - "Docker Desktop WSL", - ); - expect(sandboxGpuRemediationLines({ wslDockerDesktopStatus: "unknown" }).join("\n")).toContain( - "could not determine", - ); - }); - +describe("direct sandbox GPU proof", () => { it("treats optional direct sandbox GPU proof failures as non-fatal and reports unverified", () => { const runOpenshell = vi.fn(() => ({ status: 1, stdout: "", stderr: "optional proof failed" })); const verifier = createDirectSandboxGpuVerifier({ @@ -192,6 +47,123 @@ describe("sandbox GPU preflight", () => { expect(result?.cudaVerified).toBe(false); }); + it.each([ + "Failed to initialize NVML: Driver/library version mismatch", + "NVIDIA-SMI has failed because it couldn't communicate with the NVIDIA driver.", + "No devices were found", + "Unable to determine the device handle for GPU 0000:01:00.0: Unknown Error", + ])("returns a structured required nvidia-smi failure for %s", (diagnostic) => { + const verifier = createDirectSandboxGpuVerifier({ + runOpenshell: vi.fn(() => ({ status: 1, stdout: "", stderr: diagnostic })), + detectNvidiaPlatform: () => "linux", + buildDirectSandboxGpuProofCommands: vi.fn(() => [ + { + id: "nvidia-smi", + args: ["sandbox", "exec", "demo", "--", "nvidia-smi"], + label: "nvidia-smi when available", + }, + ]), + compactText: (value) => value.trim(), + redact: (value) => String(value), + }); + + const result = verifier("demo"); + + expect(result).toMatchObject({ + status: "failed", + cudaVerified: false, + label: "nvidia-smi when available", + detail: expect.stringContaining(diagnostic), + }); + expect(isExplicitNvidiaSmiDriverProofFailure(result)).toBe(true); + }); + + it("keeps a required nvidia-smi exec or policy error on the hard-failure path", () => { + const verifier = createDirectSandboxGpuVerifier({ + runOpenshell: vi.fn(() => ({ + status: 1, + stdout: "", + stderr: "openshell sandbox exec denied by policy", + })), + detectNvidiaPlatform: () => "linux", + buildDirectSandboxGpuProofCommands: vi.fn(() => [ + { + id: "nvidia-smi", + args: ["sandbox", "exec", "demo", "--", "nvidia-smi"], + label: "nvidia-smi when available", + }, + ]), + compactText: (value) => value.trim(), + redact: (value) => String(value), + }); + + expect(() => verifier("demo")).toThrow( + "GPU proof failed: nvidia-smi when available (status 1): openshell sandbox exec denied by policy", + ); + }); + + it("keeps an explicit nvidia-smi failure authoritative over a later CUDA pass", () => { + const runOpenshell = vi + .fn() + .mockReturnValueOnce({ + status: 1, + stdout: "", + stderr: "Failed to initialize NVML: Driver/library version mismatch", + }) + .mockReturnValueOnce({ status: 0, stdout: "cuInit(0)=0", stderr: "" }); + const verifier = createDirectSandboxGpuVerifier({ + runOpenshell, + detectNvidiaPlatform: () => "linux", + buildDirectSandboxGpuProofCommands: vi.fn(() => [ + { + id: "nvidia-smi", + args: ["sandbox", "exec", "demo", "--", "nvidia-smi"], + label: "nvidia-smi when available", + }, + { + id: "cuda-init", + args: ["sandbox", "exec", "demo", "--", "cuda-init"], + label: "cuInit(0) via libcuda.so.1", + optional: true, + }, + ]), + compactText: (value) => value.trim(), + redact: (value) => String(value), + }); + + const result = verifier("demo"); + + expect(runOpenshell).toHaveBeenCalledTimes(2); + expect(result).toMatchObject({ + status: "failed", + cudaVerified: false, + label: "nvidia-smi when available", + detail: expect.stringContaining("Failed to initialize NVML"), + }); + expect(isExplicitNvidiaSmiDriverProofFailure(result)).toBe(true); + }); + + it("rejects lookalike structured nvidia-smi results", () => { + expect( + isExplicitNvidiaSmiDriverProofFailure({ + status: "failed", + cudaVerified: false, + label: "nvidia-smi when available", + detail: "openshell sandbox exec denied by policy", + at: "2026-07-07T00:00:00.000Z", + }), + ).toBe(false); + expect( + isExplicitNvidiaSmiDriverProofFailure({ + status: "failed", + cudaVerified: false, + label: "cuInit(0) via libcuda.so.1", + detail: "Failed to initialize NVML", + at: "2026-07-07T00:00:00.000Z", + }), + ).toBe(false); + }); + it("reports failed when the CUDA usability proof reaches the driver and fails (#4231)", () => { const verifier = createDirectSandboxGpuVerifier({ runOpenshell: vi.fn((args: string[]) => { @@ -339,29 +311,4 @@ describe("sandbox GPU preflight", () => { errorSpy.mockRestore(); } }); - - it("exits with an explicit Jetson NVIDIA runtime message when runtime support is missing", () => { - const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); - const exitSpy = vi.spyOn(process, "exit").mockImplementation((( - code?: number | string | null, - ) => { - throw new Error(`exit:${code}`); - }) as never); - - try { - expect(() => - validateSandboxGpuPreflight(sandboxGpuConfig({ hostGpuPlatform: "jetson" }), { - platform: "linux", - dockerInfoFormat: vi.fn(() => '{"runc":{}}'), - }), - ).toThrow("exit:1"); - const message = errorSpy.mock.calls.map((call) => call[0]).join("\n"); - expect(message).toContain("Docker NVIDIA runtime was not detected"); - expect(message).toContain("NVIDIA Container Runtime semantics, not CDI"); - expect(message).toContain("nvidia-ctk runtime configure --runtime=docker"); - } finally { - errorSpy.mockRestore(); - exitSpy.mockRestore(); - } - }); }); diff --git a/src/lib/onboard/sandbox-gpu-fallback-constants.ts b/src/lib/onboard/sandbox-gpu-fallback-constants.ts new file mode 100644 index 00000000000..9b73a05f469 --- /dev/null +++ b/src/lib/onboard/sandbox-gpu-fallback-constants.ts @@ -0,0 +1,13 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Fixed fail-closed cleanup limits documented in docs/reference/troubleshooting.mdx. + * + * Do not source these from the environment: operator-controlled tuning would make the proof that + * gates a broader compatibility envelope deployment-dependent. A slow host must fail closed or + * select compatibility before creation rather than weaken the native-to-compatibility handoff. + */ +export const MAX_CLEANUP_ATTEMPTS = 5; +export const STABLE_ABSENCE_CHECKS = 2; +export const CLEANUP_POLL_INTERVAL_MS = 1_000; diff --git a/src/lib/onboard/sandbox-gpu-fallback-orchestration.test.ts b/src/lib/onboard/sandbox-gpu-fallback-orchestration.test.ts new file mode 100644 index 00000000000..8c165ef4bd0 --- /dev/null +++ b/src/lib/onboard/sandbox-gpu-fallback-orchestration.test.ts @@ -0,0 +1,309 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { + renderCompatibilityFallbackCreateArgs, + type SelectedDockerGpuRoute, +} from "./docker-gpu-route"; +import { + executeSandboxGpuCreatePlan, + type NativeGpuFallbackCleanupResult, + type SandboxGpuCreateAttemptFailure, + type SandboxGpuCreateFailureStage, + type SandboxGpuCreatePlanDeps, +} from "./sandbox-gpu-create-attempt"; + +const SAFE_CLEANUP: NativeGpuFallbackCleanupResult = { + safe: true, + reason: null, + deleteStatus: 0, + sandboxPresent: false, + containerIds: [], +}; + +function nativeFailure(stage: SandboxGpuCreateFailureStage): SandboxGpuCreateAttemptFailure { + return { + ok: false, + route: "native", + stage, + error: new Error(`native ${stage} failed`), + fallbackEligible: true, + }; +} + +function planDeps( + runAttempt: SandboxGpuCreatePlanDeps["runAttempt"], + overrides: Partial, "runAttempt">> = {}, +): SandboxGpuCreatePlanDeps { + return { + runAttempt, + cleanupNativeFailure: vi.fn(async () => SAFE_CLEANUP), + prepareCompatibilityAttempt: vi.fn(), + activateCompatibilityAttempt: vi.fn(), + ...overrides, + }; +} + +function execute( + deps: SandboxGpuCreatePlanDeps, + plan: Parameters[0] = "native-with-fallback", +) { + return executeSandboxGpuCreatePlan(plan, deps); +} + +const attemptedRoutes = (runAttempt: ReturnType) => + runAttempt.mock.calls.map(([route]) => route); + +function record(order: string[], event: string, result?: T) { + order.push(event); + return result as T; +} + +describe("executeSandboxGpuCreatePlan", () => { + it("accepts native success without cleanup or compatibility work and emits a trace event", async () => { + const runAttempt = vi.fn(async (route: SelectedDockerGpuRoute) => ({ + ok: true as const, + route, + value: "native-ready", + })); + const captureNativeFailure = vi.fn(); + const cleanupNativeFailure = vi.fn(async () => SAFE_CLEANUP); + const prepareCompatibilityAttempt = vi.fn(); + const activateCompatibilityAttempt = vi.fn(); + const traceEvent = vi.fn(); + + await expect( + execute( + planDeps(runAttempt, { + captureNativeFailure, + cleanupNativeFailure, + prepareCompatibilityAttempt, + activateCompatibilityAttempt, + traceEvent, + }), + ), + ).resolves.toEqual({ ok: true, route: "native", value: "native-ready" }); + + expect(runAttempt).toHaveBeenCalledTimes(1); + expect(runAttempt).toHaveBeenCalledWith("native"); + expect(captureNativeFailure).not.toHaveBeenCalled(); + expect(cleanupNativeFailure).not.toHaveBeenCalled(); + expect(prepareCompatibilityAttempt).not.toHaveBeenCalled(); + expect(activateCompatibilityAttempt).not.toHaveBeenCalled(); + expect(traceEvent).toHaveBeenCalledWith("gpu_native_success", { route: "native" }); + }); + + it.each([ + "create", + "readiness", + "gpu-proof", + ] as const)("falls back once after a native %s failure and preserves diagnostics/cleanup ordering", async (stage) => { + const order: string[] = []; + const runAttempt = vi.fn(async (route: SelectedDockerGpuRoute) => { + order.push(`attempt:${route}`); + return route === "native" + ? nativeFailure(stage) + : { ok: true as const, route, value: "compatibility-ready" }; + }); + const traceEvent = vi.fn((name: string) => order.push(`trace:${name}`)); + + const result = await execute({ + runAttempt, + captureNativeFailure: () => record(order, "diagnostics"), + cleanupNativeFailure: async () => record(order, "cleanup", SAFE_CLEANUP), + prepareCompatibilityAttempt: async () => record(order, "prepare-compatibility"), + activateCompatibilityAttempt: async () => record(order, "activate-compatibility"), + traceEvent, + }); + + expect(result).toEqual({ + ok: true, + route: "compatibility", + value: "compatibility-ready", + }); + expect(attemptedRoutes(runAttempt)).toEqual(["native", "compatibility"]); + expect(order).toEqual([ + "attempt:native", + "diagnostics", + "prepare-compatibility", + "cleanup", + "activate-compatibility", + "trace:gpu_compatibility_fallback", + "attempt:compatibility", + ]); + expect(traceEvent).toHaveBeenCalledWith("gpu_compatibility_fallback", { + from_route: "native", + to_route: "compatibility", + failure_stage: stage, + }); + }); + + it("prepares and renders the built image before the single compatibility retry", async () => { + const imageRef = `sha256:${"a".repeat(64)}`; + let compatibilityArgs: string[] | null = null; + const runAttempt = vi.fn(async (route: SelectedDockerGpuRoute) => + route === "native" + ? nativeFailure("create") + : { ok: true as const, route, value: compatibilityArgs }, + ); + + await expect( + execute( + planDeps(runAttempt, { + prepareCompatibilityAttempt: () => { + compatibilityArgs = renderCompatibilityFallbackCreateArgs( + ["--from", "/tmp/build/Dockerfile", "--policy", "/tmp/native.yaml", "--gpu"], + { + imageRef, + compatibilityPolicyPath: "/tmp/compatibility.yaml", + }, + ); + }, + }), + ), + ).resolves.toEqual({ + ok: true, + route: "compatibility", + value: ["--from", imageRef, "--policy", "/tmp/compatibility.yaml"], + }); + expect(attemptedRoutes(runAttempt)).toEqual(["native", "compatibility"]); + }); + + it("refuses fallback when native cleanup cannot be proven safe", async () => { + const runAttempt = vi.fn(async () => nativeFailure("readiness")); + const prepareCompatibilityAttempt = vi.fn(); + const activateCompatibilityAttempt = vi.fn(); + const traceEvent = vi.fn(); + + const result = await execute( + planDeps(runAttempt, { + cleanupNativeFailure: async () => ({ + safe: false, + reason: "labeled Docker containers remain: deadbeef", + deleteStatus: 0, + sandboxPresent: false, + containerIds: ["deadbeef"], + }), + prepareCompatibilityAttempt, + activateCompatibilityAttempt, + traceEvent, + }), + ); + + expect(result.ok).toBe(false); + expect(result).toMatchObject({ + ok: false, + cleanupRefused: expect.stringContaining("labeled Docker containers remain"), + }); + expect(runAttempt).toHaveBeenCalledTimes(1); + expect(prepareCompatibilityAttempt).toHaveBeenCalledOnce(); + expect(activateCompatibilityAttempt).not.toHaveBeenCalled(); + expect(traceEvent).not.toHaveBeenCalledWith("gpu_compatibility_fallback", expect.anything()); + }); + + it("keeps the failed native sandbox when compatibility retry preparation fails", async () => { + const cleanupNativeFailure = vi.fn(async () => SAFE_CLEANUP); + const activateCompatibilityAttempt = vi.fn(); + const result = await execute( + planDeps( + vi.fn(async () => nativeFailure("readiness")), + { + prepareCompatibilityAttempt: vi.fn(() => { + throw new Error("no reusable image"); + }), + activateCompatibilityAttempt, + cleanupNativeFailure, + }, + ), + ); + + expect(result.ok).toBe(false); + expect(result).toMatchObject({ ok: false, preparationRefused: "no reusable image" }); + expect(cleanupNativeFailure).not.toHaveBeenCalled(); + expect(activateCompatibilityAttempt).not.toHaveBeenCalled(); + }); + + it("returns a compatibility failure without attempting a third route", async () => { + const compatibilityFailure: SandboxGpuCreateAttemptFailure = { + ok: false, + route: "compatibility", + stage: "readiness", + error: new Error("compatibility failed"), + fallbackEligible: false, + }; + const runAttempt = vi.fn(async (route: SelectedDockerGpuRoute) => + route === "native" ? nativeFailure("create") : compatibilityFailure, + ); + + const result = await execute(planDeps(runAttempt)); + + expect(result).toBe(compatibilityFailure); + expect(attemptedRoutes(runAttempt)).toEqual(["native", "compatibility"]); + expect(runAttempt).toHaveBeenCalledTimes(2); + }); + + it.each([ + ["does not fallback when the native failure is ineligible", "native-with-fallback", false], + ["does not fallback when the route plan is native-only", "native-only", true], + ] as const)("%s (fallback gating)", async (_title, plan, fallbackEligible) => { + const failure = { ...nativeFailure("create"), fallbackEligible }; + const runAttempt = vi.fn(async () => failure); + const cleanupNativeFailure = vi.fn(async () => SAFE_CLEANUP); + + await expect(execute(planDeps(runAttempt, { cleanupNativeFailure }), plan)).resolves.toBe( + failure, + ); + expect(runAttempt).toHaveBeenCalledTimes(1); + expect(cleanupNativeFailure).not.toHaveBeenCalled(); + }); + + it("isolates cleanup verification across concurrent native fallback plans (#6110)", async () => { + function deferred() { + let resolve!: () => void; + const promise = new Promise((resolvePromise) => { + resolve = resolvePromise; + }); + return { promise, resolve }; + } + const firstCleanupEntered = deferred(); + const secondCleanupEntered = deferred(); + const firstRoutes: SelectedDockerGpuRoute[] = []; + const secondRoutes: SelectedDockerGpuRoute[] = []; + + const runPlan = ( + routes: SelectedDockerGpuRoute[], + cleanupNativeFailure: () => Promise, + ) => + execute( + planDeps( + vi.fn(async (route: SelectedDockerGpuRoute) => { + routes.push(route); + return route === "native" + ? nativeFailure("create") + : { ok: true as const, route, value: "compatibility-ready" }; + }), + { cleanupNativeFailure }, + ), + ); + + const first = runPlan(firstRoutes, async () => { + firstCleanupEntered.resolve(); + await secondCleanupEntered.promise; + return SAFE_CLEANUP; + }); + const second = runPlan(secondRoutes, async () => { + secondCleanupEntered.resolve(); + await firstCleanupEntered.promise; + return SAFE_CLEANUP; + }); + + await expect(Promise.all([first, second])).resolves.toEqual([ + { ok: true, route: "compatibility", value: "compatibility-ready" }, + { ok: true, route: "compatibility", value: "compatibility-ready" }, + ]); + expect(firstRoutes).toEqual(["native", "compatibility"]); + expect(secondRoutes).toEqual(["native", "compatibility"]); + }); +}); diff --git a/src/lib/onboard/sandbox-gpu-preflight-routing.test.ts b/src/lib/onboard/sandbox-gpu-preflight-routing.test.ts new file mode 100644 index 00000000000..d7b19cf0a9d --- /dev/null +++ b/src/lib/onboard/sandbox-gpu-preflight-routing.test.ts @@ -0,0 +1,183 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +vi.mock("../adapters/docker", () => ({ + dockerInfoFormat: vi.fn(), +})); + +import type { SandboxGpuConfig } from "./sandbox-gpu-mode"; +import { + dockerNvidiaRuntimeAvailable, + formatSandboxGpuPassthroughNote, + parseDockerRuntimeNames, + sandboxGpuRemediationLines, + validateSandboxGpuPreflight, +} from "./sandbox-gpu-preflight"; + +function sandboxGpuConfig(overrides: Partial = {}): SandboxGpuConfig { + return { + mode: "auto", + hostGpuDetected: true, + hostGpuPlatform: "linux", + sandboxGpuEnabled: true, + sandboxGpuDevice: null, + errors: [], + ...overrides, + }; +} +describe("sandbox GPU preflight routing", () => { + it("formats Jetson sandbox GPU notes around the NVIDIA runtime backend", () => { + expect(formatSandboxGpuPassthroughNote({ hostGpuPlatform: "jetson" })).toContain( + "Docker NVIDIA runtime", + ); + expect( + formatSandboxGpuPassthroughNote({ + resumeHasResolvedGpuIntent: true, + recordedGpuPassthroughBeforePreflight: true, + }), + ).toContain("Continuing GPU passthrough"); + expect(formatSandboxGpuPassthroughNote({ requestedGpuPassthrough: true })).toContain( + "GPU passthrough requested", + ); + }); + + it("parses Docker runtime names from JSON and plain-text output", () => { + expect(parseDockerRuntimeNames('{"io.containerd.runc.v2":{},"nvidia":{}}')).toContain("nvidia"); + expect(parseDockerRuntimeNames("runc nvidia io.containerd.runc.v2")).toContain("nvidia"); + expect(parseDockerRuntimeNames("")).toEqual([]); + }); + + it("checks Jetson sandbox GPU support through Docker NVIDIA runtime availability", () => { + const dockerInfo = vi.fn(() => '{"runc":{},"nvidia":{}}'); + expect(dockerNvidiaRuntimeAvailable({ dockerInfoFormat: dockerInfo })).toBe(true); + + expect(() => + validateSandboxGpuPreflight(sandboxGpuConfig({ hostGpuPlatform: "jetson" }), { + platform: "linux", + dockerInfoFormat: dockerInfo, + getDockerCdiSpecDirs: vi.fn(() => { + throw new Error("Jetson preflight must not require CDI"); + }), + findReadableNvidiaCdiSpecFiles: vi.fn(() => { + throw new Error("Jetson preflight must not inspect CDI specs"); + }), + }), + ).not.toThrow(); + expect(dockerInfo).toHaveBeenCalledWith( + "{{json .Runtimes}}", + expect.objectContaining({ ignoreError: true }), + ); + }); + + it("keeps generic Linux sandbox GPU preflight on the CDI path", () => { + const getDockerCdiSpecDirs = vi.fn(() => ["/etc/cdi"]); + const findReadableNvidiaCdiSpecFiles = vi.fn(() => ["/etc/cdi/nvidia.yaml"]); + const dockerInfo = vi.fn(() => '{"runc":{},"nvidia":{}}'); + + expect(() => + validateSandboxGpuPreflight(sandboxGpuConfig(), { + platform: "linux", + env: {}, + release: "6.8.0-generic", + procVersion: "Linux version 6.8.0-generic", + dockerInfoFormat: dockerInfo, + getDockerCdiSpecDirs, + findReadableNvidiaCdiSpecFiles, + }), + ).not.toThrow(); + expect(getDockerCdiSpecDirs).toHaveBeenCalled(); + expect(findReadableNvidiaCdiSpecFiles).toHaveBeenCalledWith(["/etc/cdi"]); + expect(dockerInfo).not.toHaveBeenCalled(); + }); + + it("skips CDI spec validation on Docker Desktop WSL so Docker --gpus can be used", () => { + const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined); + const getDockerCdiSpecDirs = vi.fn(() => ["/etc/cdi"]); + const findReadableNvidiaCdiSpecFiles = vi.fn(() => []); + + try { + expect(() => + validateSandboxGpuPreflight(sandboxGpuConfig(), { + platform: "linux", + env: { WSL_DISTRO_NAME: "Ubuntu" }, + dockerInfoFormat: vi.fn(() => '"Docker Desktop"'), + getDockerCdiSpecDirs, + findReadableNvidiaCdiSpecFiles, + }), + ).not.toThrow(); + expect(getDockerCdiSpecDirs).not.toHaveBeenCalled(); + expect(findReadableNvidiaCdiSpecFiles).not.toHaveBeenCalled(); + expect(logSpy.mock.calls.map((call) => call[0]).join("\n")).toContain( + "Docker --gpus compatibility path", + ); + } finally { + logSpy.mockRestore(); + } + }); + + it("prints neutral WSL remediation when Docker runtime cannot be determined", () => { + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); + const exitSpy = vi.spyOn(process, "exit").mockImplementation((( + code?: number | string | null, + ) => { + throw new Error(`exit:${code}`); + }) as never); + + try { + expect(() => + validateSandboxGpuPreflight(sandboxGpuConfig(), { + platform: "linux", + env: { WSL_DISTRO_NAME: "Ubuntu" }, + dockerInfoFormat: vi.fn(() => ""), + getDockerCdiSpecDirs: vi.fn(() => ["/etc/cdi"]), + findReadableNvidiaCdiSpecFiles: vi.fn(() => []), + }), + ).toThrow("exit:1"); + const message = errorSpy.mock.calls.map((call) => call[0]).join("\n"); + expect(message).toContain("could not determine whether Docker is Docker Desktop"); + expect(message).toContain("If using Docker Desktop"); + expect(message).toContain("If using native Docker Engine inside WSL"); + expect(message).not.toContain("sudo systemctl restart docker"); + } finally { + errorSpy.mockRestore(); + exitSpy.mockRestore(); + } + }); + + it("keeps generic Linux CDI remediation outside Docker Desktop WSL", () => { + expect(sandboxGpuRemediationLines().join("\n")).toContain("sudo nvidia-ctk"); + expect(sandboxGpuRemediationLines({ wslDockerDesktop: true }).join("\n")).toContain( + "Docker Desktop WSL", + ); + expect(sandboxGpuRemediationLines({ wslDockerDesktopStatus: "unknown" }).join("\n")).toContain( + "could not determine", + ); + }); + + it("exits with an explicit Jetson NVIDIA runtime message when runtime support is missing", () => { + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); + const exitSpy = vi.spyOn(process, "exit").mockImplementation((( + code?: number | string | null, + ) => { + throw new Error(`exit:${code}`); + }) as never); + + try { + expect(() => + validateSandboxGpuPreflight(sandboxGpuConfig({ hostGpuPlatform: "jetson" }), { + platform: "linux", + dockerInfoFormat: vi.fn(() => '{"runc":{}}'), + }), + ).toThrow("exit:1"); + const message = errorSpy.mock.calls.map((call) => call[0]).join("\n"); + expect(message).toContain("Docker NVIDIA runtime was not detected"); + expect(message).toContain("NVIDIA Container Runtime semantics, not CDI"); + expect(message).toContain("nvidia-ctk runtime configure --runtime=docker"); + } finally { + errorSpy.mockRestore(); + exitSpy.mockRestore(); + } + }); +}); diff --git a/src/lib/onboard/sandbox-gpu-preflight.ts b/src/lib/onboard/sandbox-gpu-preflight.ts index eb7be13eb58..79c6745df05 100644 --- a/src/lib/onboard/sandbox-gpu-preflight.ts +++ b/src/lib/onboard/sandbox-gpu-preflight.ts @@ -166,17 +166,41 @@ export interface DirectSandboxGpuVerifierDeps extends WslDockerDesktopDetectionD detectNvidiaPlatform?: () => GpuDetection["platform"] | null; } -// The proof whose result decides CUDA usability. `cuInit(0)` via libcuda is the -// authoritative usability signal (it actually initializes the CUDA driver), so -// a clean pass means "verified" and a run that reaches the driver and fails -// means "failed" rather than merely "unverified". +// The proof whose result decides reported CUDA usability. `cuInit(0)` via +// libcuda actually initializes the CUDA driver, so a clean pass means +// "verified" and a run that reaches the driver and fails means "failed" rather +// than merely "unverified". The image controls this process and its output, so +// this result is status/diagnostic evidence only: it must never by itself +// authorize a retry with a broader container-confinement envelope. const CUDA_USABILITY_PROOF_ID = "cuda-init"; +const NVIDIA_SMI_PROOF_ID = "nvidia-smi"; +const NVIDIA_SMI_PROOF_LABEL = "nvidia-smi when available"; +const EXPLICIT_NVIDIA_SMI_DRIVER_FAILURE_PATTERN = + /(?:failed to initialize NVML|(?:couldn['’]t|could not|cannot|can['’]t) communicate with the NVIDIA driver|no devices were found|unable to determine the device handle)/i; // Capture the cuInit(0) return code so we can require it to be 0 for a verified // result. Matching only the marker text is not enough: a wrapper that swallows // the probe's non-zero exit but still prints `cuInit(0)=` would otherwise // read as verified for an unusable GPU (#4231). const CUDA_INIT_RESULT_PATTERN = /cuInit\(0\)=(-?\d+)/; +/** + * Identify the canonical structured result for a required `nvidia-smi` + * driver/device failure. Keep this discriminator on existing registry fields: + * the verifier owns the exact label, while the detail must retain one of the + * narrow diagnostics that caused it to classify the command as failed. + */ +export function isExplicitNvidiaSmiDriverProofFailure( + proof: SandboxGpuProofResult | null | undefined, +): boolean { + return ( + proof?.status === "failed" && + proof.cudaVerified === false && + proof.label === NVIDIA_SMI_PROOF_LABEL && + typeof proof.detail === "string" && + EXPLICIT_NVIDIA_SMI_DRIVER_FAILURE_PATTERN.test(proof.detail) + ); +} + export type VerifyDirectSandboxGpu = ( sandboxName: string, hostGpuPlatform?: GpuDetection["platform"] | null, @@ -201,6 +225,7 @@ export function createDirectSandboxGpuVerifier( // A CUDA-usability proof that reached the driver and failed (vs one that // could not run at all). Records the proof that determines "failed" status. let cudaFailure: { label: string; detail: string } | null = null; + let explicitNvidiaSmiFailure: { label: string; detail: string } | null = null; for (const proof of buildProofCommands(sandboxName)) { const result = deps.runOpenshell(proof.args, { ignoreError: true, @@ -211,6 +236,9 @@ export function createDirectSandboxGpuVerifier( // 300 chars is only for display/storage, so a verbose proof cannot push // the marker past the cutoff and silently downgrade the classification. const rawOutput = deps.redact(`${result.stderr || ""} ${result.stdout || ""}`); + const explicitNvidiaSmiDiagnostic = rawOutput.match( + EXPLICIT_NVIDIA_SMI_DRIVER_FAILURE_PATTERN, + )?.[0]; const cudaInitMatch = rawOutput.match(CUDA_INIT_RESULT_PATTERN); const cudaInitRan = cudaInitMatch !== null; // Only `cuInit(0)=0` proves usability; any other return code means the @@ -232,6 +260,27 @@ export function createDirectSandboxGpuVerifier( } continue; } + if ( + proof.id === NVIDIA_SMI_PROOF_ID && + proof.optional !== true && + typeof result.status === "number" && + result.status > 0 && + explicitNvidiaSmiDiagnostic + ) { + // Preserve a narrow driver/device failure as structured proof so the + // caller can combine it with host-owned runtime evidence. Continue + // through the CUDA probe for diagnostics, but never let a later marker + // override this required-proof failure. + explicitNvidiaSmiFailure = { + label: NVIDIA_SMI_PROOF_LABEL, + detail: EXPLICIT_NVIDIA_SMI_DRIVER_FAILURE_PATTERN.test(diagnostic) + ? diagnostic + : explicitNvidiaSmiDiagnostic, + }; + console.warn(warnLine(`GPU proof failed: ${NVIDIA_SMI_PROOF_LABEL}`)); + if (diagnostic) console.warn(` ${diagnostic}`); + continue; + } if (proof.optional !== true) { // Required proof (e.g. the sandbox-exec wrapper itself): keep the // historical hard-fail so onboarding aborts and rolls back. @@ -258,15 +307,22 @@ export function createDirectSandboxGpuVerifier( console.warn(warnLine(`GPU proof inconclusive: ${proof.label}`)); if (diagnostic) console.warn(` ${diagnostic}`); } - const status: SandboxGpuProofResult["status"] = cudaVerified - ? "verified" - : cudaFailure - ? "failed" - : "unverified"; + const status: SandboxGpuProofResult["status"] = explicitNvidiaSmiFailure + ? "failed" + : cudaVerified + ? "verified" + : cudaFailure + ? "failed" + : "unverified"; + const reportedCudaVerified = explicitNvidiaSmiFailure ? false : cudaVerified; + const reportedFailure = explicitNvidiaSmiFailure ?? cudaFailure; if (status === "verified") { console.log(" ✓ Sandbox CUDA usability proven (cuInit succeeded)."); } else if (status === "failed") { - console.warn(warnLine(`Sandbox CUDA proof failed: ${cudaFailure?.label}`)); + const failureKind = explicitNvidiaSmiFailure + ? "Sandbox NVIDIA driver/device proof failed" + : "Sandbox CUDA proof failed"; + console.warn(warnLine(`${failureKind}: ${reportedFailure?.label}`)); const lines = resolvedPlatform === "jetson" ? jetsonGpuProofRemediationLines() @@ -281,9 +337,9 @@ export function createDirectSandboxGpuVerifier( } return { status, - cudaVerified, - label: cudaFailure?.label ?? null, - detail: cudaFailure?.detail ?? null, + cudaVerified: reportedCudaVerified, + label: reportedFailure?.label ?? null, + detail: reportedFailure?.detail ?? null, at: new Date().toISOString(), }; }; diff --git a/src/lib/onboard/sandbox-gpu-route-policy.ts b/src/lib/onboard/sandbox-gpu-route-policy.ts new file mode 100644 index 00000000000..ffd0617ef31 --- /dev/null +++ b/src/lib/onboard/sandbox-gpu-route-policy.ts @@ -0,0 +1,66 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + canFallbackToDockerGpuCompatibility, + type DockerGpuRoutePlan, + initialDockerGpuRoute, +} from "./docker-gpu-route"; +import type { InitialSandboxPolicy } from "./initial-policy"; + +type PrepareInitialSandboxCreatePolicy = + typeof import("./initial-policy").prepareInitialSandboxCreatePolicy; +type InitialPolicyOptions = Parameters[2]; + +export type SandboxGpuRoutePolicies = { + initialSandboxPolicy: InitialSandboxPolicy; + compatibilityPolicyPath: string | null; +}; + +/** + * Materialize narrow native and compatibility fallback policies before sandbox-create side + * effects. `preparePolicy` may create secure temporary files; every successful result carries its + * cleanup, this function combines both cleanups, and a failed second materialization immediately + * cleans the first. No provider, registry, gateway, or sandbox mutation occurs here. + */ +export function prepareSandboxGpuRoutePolicies( + basePolicyPath: string, + activeMessagingChannels: string[], + options: InitialPolicyOptions, + gpuRoutePlan: DockerGpuRoutePlan, + preparePolicy: PrepareInitialSandboxCreatePolicy, +): SandboxGpuRoutePolicies { + const initialCompatibility = initialDockerGpuRoute(gpuRoutePlan) === "compatibility"; + const initialSandboxPolicy = preparePolicy(basePolicyPath, activeMessagingChannels, { + ...options, + dockerGpuPatch: initialCompatibility, + }); + let compatibilityPolicy: InitialSandboxPolicy | null = null; + try { + if (canFallbackToDockerGpuCompatibility(gpuRoutePlan)) { + compatibilityPolicy = preparePolicy(basePolicyPath, activeMessagingChannels, { + ...options, + dockerGpuPatch: true, + }); + } + } catch (error) { + initialSandboxPolicy.cleanup?.(); + throw error; + } + + const cleanupFns = [initialSandboxPolicy.cleanup, compatibilityPolicy?.cleanup].filter( + (cleanup): cleanup is () => boolean => Boolean(cleanup), + ); + return { + initialSandboxPolicy: { + ...initialSandboxPolicy, + cleanup: + cleanupFns.length > 0 + ? () => cleanupFns.map((cleanup) => cleanup()).every(Boolean) + : undefined, + }, + compatibilityPolicyPath: initialCompatibility + ? initialSandboxPolicy.policyPath + : (compatibilityPolicy?.policyPath ?? null), + }; +} diff --git a/src/lib/onboard/sandbox-prebuild.test.ts b/src/lib/onboard/sandbox-prebuild.test.ts index 96cf387f537..b16616fb1de 100644 --- a/src/lib/onboard/sandbox-prebuild.test.ts +++ b/src/lib/onboard/sandbox-prebuild.test.ts @@ -16,6 +16,7 @@ import { } from "./sandbox-prebuild"; const BUILD_ID = "1234567890"; +const IMAGE_ID = `sha256:${"a".repeat(64)}`; const temporaryDirectories: string[] = []; function createBuildContext( @@ -121,7 +122,11 @@ describe("sandbox BuildKit prebuild", () => { env: {}, buildImage, }), - ).resolves.toEqual({ createArgs: ["--from", "/other/Dockerfile"], imageRef: null }); + ).resolves.toEqual({ + createArgs: ["--from", "/other/Dockerfile"], + imageRef: null, + imageId: null, + }); expect(buildImage).not.toHaveBeenCalled(); }); @@ -142,7 +147,7 @@ describe("sandbox BuildKit prebuild", () => { buildImage, log, }), - ).resolves.toEqual({ createArgs, imageRef: null }); + ).resolves.toEqual({ createArgs, imageRef: null, imageId: null }); expect(buildImage).not.toHaveBeenCalled(); expect(log).toHaveBeenCalledWith(expect.stringContaining("custom Dockerfile")); }); @@ -166,7 +171,7 @@ describe("sandbox BuildKit prebuild", () => { buildImage, log: () => {}, }), - ).resolves.toEqual({ createArgs, imageRef: null }); + ).resolves.toEqual({ createArgs, imageRef: null, imageId: null }); expect(buildImage).not.toHaveBeenCalled(); }); @@ -186,7 +191,7 @@ describe("sandbox BuildKit prebuild", () => { buildImage, log: () => {}, }), - ).resolves.toEqual({ createArgs, imageRef: null }); + ).resolves.toEqual({ createArgs, imageRef: null, imageId: null }); expect(buildImage).not.toHaveBeenCalled(); }); @@ -208,7 +213,7 @@ describe("sandbox BuildKit prebuild", () => { buildImage, log, }), - ).resolves.toEqual({ createArgs, imageRef: null }); + ).resolves.toEqual({ createArgs, imageRef: null, imageId: null }); expect(buildImage).not.toHaveBeenCalled(); expect(log).toHaveBeenCalledWith(expect.stringContaining("failed trust validation")); }); @@ -232,7 +237,7 @@ describe("sandbox BuildKit prebuild", () => { buildImage, log: () => {}, }), - ).resolves.toEqual({ createArgs, imageRef: null }); + ).resolves.toEqual({ createArgs, imageRef: null, imageId: null }); expect(buildImage).not.toHaveBeenCalled(); }); @@ -254,7 +259,7 @@ describe("sandbox BuildKit prebuild", () => { buildImage, log: () => {}, }), - ).resolves.toEqual({ createArgs, imageRef: null }); + ).resolves.toEqual({ createArgs, imageRef: null, imageId: null }); expect(buildImage).not.toHaveBeenCalled(); }); @@ -280,7 +285,7 @@ describe("sandbox BuildKit prebuild", () => { buildImage, log: () => {}, }), - ).resolves.toEqual({ createArgs, imageRef: null }); + ).resolves.toEqual({ createArgs, imageRef: null, imageId: null }); expect(buildImage).not.toHaveBeenCalled(); }); @@ -304,7 +309,7 @@ describe("sandbox BuildKit prebuild", () => { buildImage, log, }), - ).resolves.toEqual({ createArgs, imageRef: null }); + ).resolves.toEqual({ createArgs, imageRef: null, imageId: null }); expect(buildImage).not.toHaveBeenCalled(); expect(log).toHaveBeenCalledWith(expect.stringContaining("too many open files")); expect(log).toHaveBeenCalledWith(expect.stringContaining("could not be inspected")); @@ -324,6 +329,7 @@ describe("sandbox BuildKit prebuild", () => { dockerDriverGateway: true, env: {}, buildImage, + inspectImageId: () => IMAGE_ID, log: () => {}, }); @@ -344,6 +350,7 @@ describe("sandbox BuildKit prebuild", () => { expect(result).toEqual({ createArgs: ["--from", "nemoclaw-sandbox-local:alpha-1234567890", "--name", "alpha"], imageRef: "nemoclaw-sandbox-local:alpha-1234567890", + imageId: IMAGE_ID, }); }); @@ -363,7 +370,7 @@ describe("sandbox BuildKit prebuild", () => { buildImage, log: () => {}, }); - expect(result).toEqual({ createArgs, imageRef: null }); + expect(result).toEqual({ createArgs, imageRef: null, imageId: null }); }); it("falls back to OpenShell when the Docker helper throws", async () => { @@ -381,6 +388,6 @@ describe("sandbox BuildKit prebuild", () => { }, log: () => {}, }); - expect(result).toEqual({ createArgs, imageRef: null }); + expect(result).toEqual({ createArgs, imageRef: null, imageId: null }); }); }); diff --git a/src/lib/onboard/sandbox-prebuild.ts b/src/lib/onboard/sandbox-prebuild.ts index 44dac578c63..5ed957227e0 100644 --- a/src/lib/onboard/sandbox-prebuild.ts +++ b/src/lib/onboard/sandbox-prebuild.ts @@ -5,6 +5,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { dockerImageInspectFormat } from "../adapters/docker"; import { dockerSpawn } from "../adapters/docker/exec"; import { LOCAL_SANDBOX_IMAGE_REPO } from "../domain/sandbox/image-tag"; import { @@ -12,6 +13,7 @@ import { type SandboxBuildContextOrigin, } from "../sandbox/build-context"; import { buildSubprocessEnv } from "../subprocess-env"; +import { isImmutableDockerImageId } from "./openshell-docker-sandbox-containers"; const TRUTHY_FLAG_VALUES = new Set(["1", "true", "yes", "on"]); const FALSY_FLAG_VALUES = new Set(["0", "false", "no", "off"]); @@ -36,12 +38,15 @@ export interface SandboxPrebuildInput { args: readonly string[], options: { env: NodeJS.ProcessEnv; stdio: "inherit" }, ) => Promise; + inspectImageId?: (imageRef: string) => string; log?: (message: string) => void; } export interface SandboxPrebuildResult { createArgs: string[]; imageRef: string | null; + /** Immutable local image identity; mutable tags never authorize fallback. */ + imageId: string | null; } interface TrustedStagedBuildContext { @@ -150,13 +155,13 @@ export async function prebuildSandboxImageIfEligible( const env = input.env ?? process.env; const log = input.log ?? console.log; if (!resolveSandboxPrebuildEnabled(env, input.dockerDriverGateway)) { - return { createArgs, imageRef: null }; + return { createArgs, imageRef: null, imageId: null }; } if (input.origin !== "generated") { log( " Local BuildKit build skipped for a custom Dockerfile; using the gateway builder instead.", ); - return { createArgs, imageRef: null }; + return { createArgs, imageRef: null, imageId: null }; } const fromIndex = createArgs.indexOf("--from"); const fromDockerfile = createArgs[fromIndex + 1]; @@ -165,7 +170,7 @@ export async function prebuildSandboxImageIfEligible( !fromDockerfile || path.resolve(fromDockerfile) !== path.resolve(input.buildCtx, "Dockerfile") ) { - return { createArgs, imageRef: null }; + return { createArgs, imageRef: null, imageId: null }; } let trustedContext: TrustedStagedBuildContext | null; try { @@ -175,13 +180,13 @@ export async function prebuildSandboxImageIfEligible( log( ` Local BuildKit build skipped: staged build context could not be inspected (${detail}); using the gateway builder instead.`, ); - return { createArgs, imageRef: null }; + return { createArgs, imageRef: null, imageId: null }; } if (!trustedContext) { log( " Local BuildKit build skipped: staged build context failed trust validation; using the gateway builder instead.", ); - return { createArgs, imageRef: null }; + return { createArgs, imageRef: null, imageId: null }; } const imageRef = sandboxLocalImageRef(input.sandboxName, input.buildId); @@ -207,18 +212,39 @@ export async function prebuildSandboxImageIfEligible( } catch (error) { const detail = error instanceof Error ? error.message : String(error); log(` Local BuildKit build could not start (${detail}); using the gateway builder instead.`); - return { createArgs, imageRef: null }; + return { createArgs, imageRef: null, imageId: null }; } if (status !== 0) { const detail = status === null ? " without an exit status" : ` (exit ${status})`; log(` Local BuildKit build failed${detail}; using the gateway builder instead.`); - return { createArgs, imageRef: null }; + return { createArgs, imageRef: null, imageId: null }; } createArgs[fromIndex + 1] = imageRef; + const inspectImageId = + input.inspectImageId ?? + ((ref: string) => + dockerImageInspectFormat("{{.Id}}", ref, { + ignoreError: true, + }).trim()); + let imageId: string | null = null; + try { + const inspected = inspectImageId(imageRef).trim(); + if (isImmutableDockerImageId(inspected)) imageId = inspected.toLowerCase(); + } catch { + // Native creation can still use the local tag. Automatic compatibility + // fallback will refuse it unless the exact container supplies an immutable + // image ID before cleanup. + } + if (!imageId) { + log( + " Local image identity could not be proven; an operator-authorized GPU compatibility fallback may fail closed if no exact native container identity becomes available.", + ); + } return { createArgs, imageRef, + imageId, }; } diff --git a/src/lib/onboard/sandbox-readiness-stability.test.ts b/src/lib/onboard/sandbox-readiness-stability.test.ts new file mode 100644 index 00000000000..13f5488334a --- /dev/null +++ b/src/lib/onboard/sandbox-readiness-stability.test.ts @@ -0,0 +1,60 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { getSandboxFailurePhase, isSandboxReady } from "../state/gateway"; +import { waitForCreatedSandboxReadyWithTrace } from "./sandbox-readiness-tracing"; + +const NAME = "my-sandbox"; + +function replay(outputs: readonly string[]) { + let index = 0; + const runCaptureOpenshell = vi.fn(() => outputs[Math.min(index++, outputs.length - 1)] ?? ""); + return { runCaptureOpenshell, sleep: vi.fn() }; +} + +describe("created sandbox Ready stability", () => { + it("preserves single-poll Ready acceptance by default", () => { + const { runCaptureOpenshell, sleep } = replay([`${NAME} Ready 1s ago`]); + + const ready = waitForCreatedSandboxReadyWithTrace({ + sandboxName: NAME, + timeoutSecs: 600, + runCaptureOpenshell, + isSandboxReady, + getSandboxFailurePhase, + sleep, + }); + + expect(ready).toEqual({ ready: true, reason: "ready", failurePhase: null }); + expect(runCaptureOpenshell).toHaveBeenCalledOnce(); + expect(sleep).not.toHaveBeenCalled(); + }); + + it("rejects a stale Ready row until compatibility recreation reaches stable Ready", () => { + // Exact fallback-run ordering from 28817562371: after a successful + // supervisor exec, sandbox-list first retained the old container's Ready + // row, then published the recreated supervisor's Error -> Ready sequence. + const { runCaptureOpenshell, sleep } = replay([ + `${NAME} Ready old-container`, + `${NAME} Error replacement-registering`, + `${NAME} Ready replacement-connected`, + `${NAME} Ready replacement-stable`, + ]); + + const ready = waitForCreatedSandboxReadyWithTrace({ + sandboxName: NAME, + timeoutSecs: 600, + runCaptureOpenshell, + isSandboxReady, + getSandboxFailurePhase, + stableReadyPolls: 2, + sleep, + }); + + expect(ready).toEqual({ ready: true, reason: "ready", failurePhase: null }); + expect(runCaptureOpenshell).toHaveBeenCalledTimes(4); + expect(sleep).toHaveBeenCalledTimes(3); + }); +}); diff --git a/src/lib/onboard/sandbox-readiness-tracing.ts b/src/lib/onboard/sandbox-readiness-tracing.ts index 32020161044..35ae0e075cc 100644 --- a/src/lib/onboard/sandbox-readiness-tracing.ts +++ b/src/lib/onboard/sandbox-readiness-tracing.ts @@ -176,6 +176,14 @@ export function waitForCreatedSandboxReadyWithTrace(options: { * timeout window before reporting "did not become ready" (#4316). */ getSandboxFailurePhase?: (output: string, sandboxName: string) => string | null; + /** + * Consecutive Ready polls required before returning success. Defaults to 1. + * The Docker GPU compatibility recreate passes 2 because the OpenShell + * gateway can briefly retain the pre-recreate Ready row before publishing + * the new supervisor's Error -> Ready registration transition. Requiring a + * confirmation poll keeps that stale row from reaching the GPU proof. + */ + stableReadyPolls?: number; /** * Consecutive Error-phase polls required before the wait treats the phase as * terminal. Defaults to {@link getSandboxReadyErrorDebouncePolls} (30 polls / @@ -213,16 +221,37 @@ export function waitForCreatedSandboxReadyWithTrace(options: { : // Round (not truncate) so a fractional override matches the env-var // path's envInt rounding — one consistent rule for both entry points. Math.max(1, Math.round(options.errorPhaseDebouncePolls)); + const stableReadyPolls = + options.stableReadyPolls == null || !Number.isFinite(options.stableReadyPolls) + ? 1 + : Math.max(1, Math.round(options.stableReadyPolls)); return withSandboxReadinessTrace(sandboxName, { timeout_seconds: timeoutSecs }, () => { const readyAttempts = Math.max(1, Math.ceil(timeoutSecs / 2)); + let consecutiveReadyPolls = 0; let consecutiveFailurePolls = 0; let lastFailurePhase: string | null = null; for (let i = 0; i < readyAttempts; i++) { const list = runCaptureOpenshell(["sandbox", "list"], { ignoreError: true }); if (isSandboxReady(list, sandboxName)) { - addTraceEvent("ready", { attempt: i + 1 }); - return { ready: true, reason: "ready", failurePhase: null }; + consecutiveReadyPolls += 1; + consecutiveFailurePolls = 0; + lastFailurePhase = null; + if (consecutiveReadyPolls >= stableReadyPolls) { + addTraceEvent("ready", { + attempt: i + 1, + consecutive_polls: consecutiveReadyPolls, + }); + return { ready: true, reason: "ready", failurePhase: null }; + } + addTraceEvent("ready_pending_stability", { + attempt: i + 1, + consecutive_polls: consecutiveReadyPolls, + required_polls: stableReadyPolls, + }); + if (i < readyAttempts - 1) sleep(2); + continue; } + consecutiveReadyPolls = 0; const failurePhase = getSandboxFailurePhase?.(list, sandboxName) ?? null; // Only the transient "Error" phase is debounced — it is the phase the // gateway briefly reports while re-registering the just-created sandbox diff --git a/src/lib/state/gateway.ts b/src/lib/state/gateway.ts index 1730487ca52..8c2f67975a8 100644 --- a/src/lib/state/gateway.ts +++ b/src/lib/state/gateway.ts @@ -35,6 +35,11 @@ function parseSandboxRow(output: string, sandboxName: string): string[] | null { return null; } +/** True when `sandbox list` contains an exact first-column sandbox name. */ +export function hasSandboxListEntry(output: string, sandboxName: string): boolean { + return parseSandboxRow(output, sandboxName) !== null; +} + export function parseSandboxStatus(output: string, sandboxName: string): string | null { const cols = parseSandboxRow(output, sandboxName); return cols && cols.length >= 2 ? cols[1] : null; diff --git a/test/e2e/live/hermes-gpu-startup-fallback.ts b/test/e2e/live/hermes-gpu-startup-fallback.ts new file mode 100644 index 00000000000..423d1fce1d0 --- /dev/null +++ b/test/e2e/live/hermes-gpu-startup-fallback.ts @@ -0,0 +1,193 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { REQUIRED_OPENSHELL_MCP_FEATURES } from "../../../src/lib/onboard/openshell-feature-gate"; + +export const HERMES_GPU_FALLBACK_EVENTS = { + delegateNativeCreateWithoutGpu: "delegate-native-create-without-gpu", + rejectNativeNvidiaSmiProof: "reject-native-nvidia-smi-proof", + delegateCompatibilityCreate: "delegate-compatibility-create", + delegateNvidiaSmiProofAfterRejection: "delegate-nvidia-smi-proof-after-rejection", +} as const; + +export const HERMES_GPU_NATIVE_NVIDIA_SMI_PROOF = [ + "set -eu;", + "if command -v nvidia-smi >/dev/null 2>&1; then", + "exec nvidia-smi;", + "fi;", + 'echo "nvidia-smi not installed; skipping optional visibility check"', +].join(" "); + +export interface HermesGpuFallbackWrapper { + componentEnv: NodeJS.ProcessEnv; + eventsPath: string; + rootDir: string; + wrapperPath: string; +} + +export type HermesGpuStartupScenario = "compatibility-only" | "fallback" | "native"; +export type HermesGpuStartupRoute = + | "compatibility-fallback" + | "compatibility-only" + | "native-success"; + +export function resolveHermesGpuStartupScenario( + rawScenario: string | undefined, + forceCompatibility: boolean, +): { route: HermesGpuStartupRoute; scenario: HermesGpuStartupScenario } { + const scenario = rawScenario ?? "native"; + if (scenario !== "native" && scenario !== "fallback" && scenario !== "compatibility-only") { + throw new Error( + `E2E_HERMES_GPU_STARTUP_SCENARIO must be native, fallback, or compatibility-only, got '${scenario}'`, + ); + } + if (scenario === "fallback" && forceCompatibility) { + throw new Error( + "fallback scenario requires automatic GPU routing, not compatibility-only mode", + ); + } + return { + scenario, + route: + forceCompatibility || scenario === "compatibility-only" + ? "compatibility-only" + : scenario === "fallback" + ? "compatibility-fallback" + : "native-success", + }; +} + +export function extractHermesGpuDiagnosticsDirectory(output: string): string { + return ( + output.match(/Pre-rollback diagnostics saved:\s*(\S+)/u)?.[1] ?? + output.match(/Native GPU diagnostics saved:\s*(\S+)/u)?.[1] ?? + "" + ); +} + +function requireAbsoluteExecutable(filePath: string, label: string): void { + if (!path.isAbsolute(filePath)) { + throw new Error(`${label} must be an absolute path`); + } + fs.accessSync(filePath, fs.constants.X_OK); +} + +function quoteShellLiteral(value: string): string { + return `'${value.replaceAll("'", `'"'"'`)}'`; +} + +/** + * Create an E2E-only OpenShell CLI wrapper that simulates a native injector + * silently dropping the exact `--gpu` request while still delegating creation, + * so a real partial sandbox exists with host-visible GPU attachment absent. + * It then atomically rejects the first exact post-create `nvidia-smi` proof + * with a narrow NVML driver error. Every other invocation transparently + * delegates its original argv to the real CLI. This + * test-only wrapper never logs argv: its sole artifact is an event log made of + * fixed labels, so sandbox-create environment arguments never enter artifacts. + * This interception pattern is specific to the #6110 fallback proof and must + * not be copied to another E2E path without security review. The caller owns + * the wrapper root and registers recursive removal with the test cleanup stack. + */ +export function createHermesGpuFallbackWrapper( + realOpenshellPath: string, + options: { rootDir?: string } = {}, +): HermesGpuFallbackWrapper { + requireAbsoluteExecutable(realOpenshellPath, "real OpenShell CLI"); + const componentDir = path.dirname(realOpenshellPath); + const gatewayPath = path.join(componentDir, "openshell-gateway"); + const sandboxPath = path.join(componentDir, "openshell-sandbox"); + requireAbsoluteExecutable(gatewayPath, "OpenShell gateway component"); + requireAbsoluteExecutable(sandboxPath, "OpenShell sandbox component"); + + const rootDir = + options.rootDir ?? + fs.mkdtempSync(path.join(process.env.RUNNER_TEMP ?? os.tmpdir(), "hermes-gpu-fallback-")); + fs.mkdirSync(rootDir, { recursive: true, mode: 0o700 }); + const stateDir = path.join(rootDir, "state"); + fs.mkdirSync(stateDir, { recursive: true, mode: 0o700 }); + const wrapperPath = path.join(rootDir, "openshell"); + const eventsPath = path.join(stateDir, "events.log"); + const wrapper = [ + "#!/usr/bin/env bash", + "set -euo pipefail", + "", + ...REQUIRED_OPENSHELL_MCP_FEATURES.map((marker) => `# capability: ${marker}`), + `REAL_OPENSHELL=${quoteShellLiteral(realOpenshellPath)}`, + `FALLBACK_STATE_DIR=${quoteShellLiteral(stateDir)}`, + `NATIVE_NVIDIA_SMI_PROOF=${quoteShellLiteral(HERMES_GPU_NATIVE_NVIDIA_SMI_PROOF)}`, + "", + "is_sandbox_create=0", + "has_gpu_flag=0", + 'if [[ "${1:-}" == "sandbox" && "${2:-}" == "create" ]]; then', + " is_sandbox_create=1", + ' for arg in "$@"; do', + ' if [[ "$arg" == "--gpu" ]]; then', + " has_gpu_flag=1", + " break", + " fi", + " done", + "fi", + "", + 'if [[ "$is_sandbox_create" == "1" ]]; then', + ' if [[ "$has_gpu_flag" == "1" ]]; then', + ` printf '%s\\n' '${HERMES_GPU_FALLBACK_EVENTS.delegateNativeCreateWithoutGpu}' >>"$FALLBACK_STATE_DIR/events.log"`, + " filtered_args=()", + " stripped_gpu=0", + ' for arg in "$@"; do', + ' if [[ "$stripped_gpu" == "0" && "$arg" == "--gpu" ]]; then', + " stripped_gpu=1", + " continue", + " fi", + ' filtered_args+=("$arg")', + " done", + ' exec "$REAL_OPENSHELL" "${filtered_args[@]}"', + " else", + ` printf '%s\\n' '${HERMES_GPU_FALLBACK_EVENTS.delegateCompatibilityCreate}' >>"$FALLBACK_STATE_DIR/events.log"`, + " fi", + "fi", + "", + "is_native_nvidia_smi_proof=0", + 'if [[ "$#" -eq 8 && "${1:-}" == "sandbox" && "${2:-}" == "exec" && "${3:-}" == "-n" && -n "${4:-}" && "${5:-}" == "--" && "${6:-}" == "sh" && "${7:-}" == "-lc" && "${8:-}" == "$NATIVE_NVIDIA_SMI_PROOF" ]]; then', + " is_native_nvidia_smi_proof=1", + "fi", + "", + 'if [[ "$is_native_nvidia_smi_proof" == "1" ]]; then', + ' if mkdir "$FALLBACK_STATE_DIR/native-nvidia-smi-proof-rejected" 2>/dev/null; then', + ` printf '%s\\n' '${HERMES_GPU_FALLBACK_EVENTS.rejectNativeNvidiaSmiProof}' >>"$FALLBACK_STATE_DIR/events.log"`, + ` printf '%s\\n' 'Failed to initialize NVML: Driver/library version mismatch' >&2`, + " exit 1", + " fi", + ` printf '%s\\n' '${HERMES_GPU_FALLBACK_EVENTS.delegateNvidiaSmiProofAfterRejection}' >>"$FALLBACK_STATE_DIR/events.log"`, + "fi", + "", + "# Transparent test-only delegation: argv is never written by this wrapper.", + 'exec "$REAL_OPENSHELL" "$@"', + "", + ].join("\n"); + fs.writeFileSync(wrapperPath, wrapper, { encoding: "utf8", mode: 0o700 }); + + return { + componentEnv: { + NEMOCLAW_OPENSHELL_BIN: wrapperPath, + NEMOCLAW_OPENSHELL_GATEWAY_BIN: gatewayPath, + NEMOCLAW_OPENSHELL_SANDBOX_BIN: sandboxPath, + }, + eventsPath, + rootDir, + wrapperPath, + }; +} + +export function readHermesGpuFallbackEvents(eventsPath: string): string[] { + if (!fs.existsSync(eventsPath)) return []; + return fs + .readFileSync(eventsPath, "utf8") + .split(/\r?\n/u) + .map((line) => line.trim()) + .filter(Boolean); +} diff --git a/test/e2e/live/hermes-gpu-startup-integrity.ts b/test/e2e/live/hermes-gpu-startup-integrity.ts index 16dbf661dca..36afd6a84b4 100644 --- a/test/e2e/live/hermes-gpu-startup-integrity.ts +++ b/test/e2e/live/hermes-gpu-startup-integrity.ts @@ -93,7 +93,7 @@ def parse_hash(data, label): ) if state_match is None: fail(f"{label} contains an unexpected MCP state record") - return tuple(digests) + return tuple(digests), state_match.groups() def digest(data): return hashlib.sha256(data).hexdigest() @@ -138,10 +138,16 @@ if api_key_lines != 1: fail("Hermes environment does not contain exactly one canonical generated API key") base_env_bytes = "".join(base_env_lines).encode("utf-8") -strict_config_digest, strict_env_digest = parse_hash(strict_hash_bytes, "Hermes strict hash") -compat_config_digest, compat_env_digest = parse_hash( +strict_digests, strict_mcp_state = parse_hash(strict_hash_bytes, "Hermes strict hash") +compat_digests, compat_mcp_state = parse_hash( compat_hash_bytes, "Hermes compatibility hash" ) +strict_config_digest, strict_env_digest = strict_digests +compat_config_digest, compat_env_digest = compat_digests +if strict_mcp_state[0] != strict_mcp_state[1]: + fail("Hermes strict hash contains pending MCP state") +if compat_mcp_state != strict_mcp_state: + fail("Hermes compatibility hash MCP state differs from the strict anchor") if not secrets.compare_digest(strict_config_digest, digest(config_bytes)): fail("Hermes config differs from the strict startup base") if not secrets.compare_digest(strict_env_digest, digest(base_env_bytes)): diff --git a/test/e2e/live/hermes-gpu-startup-proof.ts b/test/e2e/live/hermes-gpu-startup-proof.ts index 16e0f9c32d5..61d4e6b0cde 100644 --- a/test/e2e/live/hermes-gpu-startup-proof.ts +++ b/test/e2e/live/hermes-gpu-startup-proof.ts @@ -17,10 +17,17 @@ export const HERMES_GPU_EXTRA_PLACEHOLDER_KEYS = [ "TELEGRAM_BOT_TOKEN_AGENT_A", "SLACK_BOT_TOKEN_AGENT_B", ] as const; +export const HERMES_GPU_FALLBACK_DISCLOSURE_FRAGMENTS = [ + "recreating the OpenShell-managed Docker container", + "legacy GPU compatibility envelope", + "may relax container confinement", + "NEMOCLAW_DOCKER_GPU_PATCH=fallback", + "explicitly authorized", +] as const; interface HermesGpuStartupProofOptions { env: NodeJS.ProcessEnv; - gpuRoute: "legacy-patch" | "native-openshell"; + gpuRoute: "compatibility-fallback" | "compatibility-only" | "native-success"; host: HostCliClient; install: Pick; sandbox: SandboxClient; @@ -37,24 +44,42 @@ export async function assertHermesGpuStartupProof({ sandboxName, status, }: HermesGpuStartupProofOptions): Promise { - expect(resultText(install)).toContain("Starting OpenShell Docker-driver gateway..."); - expect(resultText(install)).toContain("Docker-driver gateway is healthy"); - expect(resultText(install)).not.toContain("Reusing healthy NemoClaw gateway."); - expect(resultText(install)).not.toContain("Reusing existing Docker-driver gateway"); - expect(resultText(install)).not.toContain("[reuse] Skipping gateway (running)"); - if (gpuRoute === "legacy-patch") { - expect(resultText(install)).toContain( + const installText = resultText(install); + expect(installText).toContain("Starting OpenShell Docker-driver gateway..."); + expect(installText).toContain("Docker-driver gateway is healthy"); + expect(installText).not.toContain("Reusing healthy NemoClaw gateway."); + expect(installText).not.toContain("Reusing existing Docker-driver gateway"); + expect(installText).not.toContain("[reuse] Skipping gateway (running)"); + if (gpuRoute === "compatibility-only") { + expect(installText).toContain( + "Recreating OpenShell Docker sandbox container with NVIDIA GPU access", + ); + expect(installText).toContain("Docker GPU mode selected:"); + for (const fragment of HERMES_GPU_FALLBACK_DISCLOSURE_FRAGMENTS) { + expect(installText).not.toContain(fragment); + } + } else if (gpuRoute === "compatibility-fallback") { + expect(installText).toContain( + "Operator-authorized GPU fallback enabled; trying native OpenShell injection with one compatibility retry.", + ); + for (const fragment of HERMES_GPU_FALLBACK_DISCLOSURE_FRAGMENTS) { + expect(installText).toContain(fragment); + } + expect(installText).toContain( "Recreating OpenShell Docker sandbox container with NVIDIA GPU access", ); - expect(resultText(install)).toContain("Docker GPU mode selected:"); + expect(installText).toContain("Docker GPU mode selected:"); } else { - expect(resultText(install)).toContain( + expect(installText).toContain( "Direct sandbox GPU enabled; allowing OpenShell GPU policy enrichment.", ); - expect(resultText(install)).not.toContain( + expect(installText).not.toContain( "Recreating OpenShell Docker sandbox container with NVIDIA GPU access", ); - expect(resultText(install)).not.toContain("Docker GPU mode selected:"); + expect(installText).not.toContain("Docker GPU mode selected:"); + for (const fragment of HERMES_GPU_FALLBACK_DISCLOSURE_FRAGMENTS) { + expect(installText).not.toContain(fragment); + } } const plainStatus = stripAnsi(resultText(status)); expect(plainStatus).toMatch(/Phase:\s*Ready/i); @@ -217,7 +242,7 @@ raise SystemExit(1)`, entrypoint: ["/opt/openshell/bin/openshell-sandbox"], has_openshell_sandbox_command: true, }); - if (gpuRoute === "legacy-patch") { + if (gpuRoute !== "native-success") { expect(commandBoundary.command_ends_with_nemoclaw_start).toBe(true); expect(commandBoundary.command_is_sleep_infinity).toBe(false); } else { @@ -254,10 +279,10 @@ raise SystemExit(1)`, }, ); expect(allContainers.exitCode, resultText(allContainers)).toBe(0); - expect( - allContainers.stdout - .split(/\r?\n/) - .map((line) => line.trim()) - .filter(Boolean), - ).toHaveLength(1); + const allContainerNames = allContainers.stdout + .split(/\r?\n/u) + .map((line) => line.trim()) + .filter(Boolean); + expect(allContainerNames).toHaveLength(1); + expect(allContainerNames.filter((name) => name.includes("-nemoclaw-gpu-backup-"))).toEqual([]); } diff --git a/test/e2e/live/hermes-gpu-startup.test.ts b/test/e2e/live/hermes-gpu-startup.test.ts index 7e1daede960..aa9d1a6e3ca 100644 --- a/test/e2e/live/hermes-gpu-startup.test.ts +++ b/test/e2e/live/hermes-gpu-startup.test.ts @@ -1,11 +1,13 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import fs from "node:fs"; import path from "node:path"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import { type HostCliClient, + outputContainsSandbox, resultText, type SandboxClient, trustedSandboxShellScript, @@ -14,9 +16,17 @@ import { import { expect, test } from "../fixtures/e2e-test.ts"; import { startFakeOpenAiCompatibleServer } from "../fixtures/fake-openai-compatible.ts"; import { REPO_ROOT } from "../fixtures/paths.ts"; +import { + createHermesGpuFallbackWrapper, + extractHermesGpuDiagnosticsDirectory, + HERMES_GPU_FALLBACK_EVENTS, + readHermesGpuFallbackEvents, + resolveHermesGpuStartupScenario, +} from "./hermes-gpu-startup-fallback.ts"; import { assertHermesGpuStartupProof, HERMES_GPU_EXTRA_PLACEHOLDER_KEYS, + HERMES_GPU_FALLBACK_DISCLOSURE_FRAGMENTS, } from "./hermes-gpu-startup-proof.ts"; const GATEWAY_CLEANUP_MODULE = path.join(REPO_ROOT, "dist/lib/actions/sandbox/destroy-gateway.js"); @@ -27,14 +37,24 @@ const GATEWAY_CLEANUP_SCRIPT = String.raw` command -v openshell >/dev/null 2>&1 || exit 0 exec node -e 'const { cleanupGatewayAfterLastSandbox } = require(process.argv[1]); cleanupGatewayAfterLastSandbox(process.argv[2]);' "$@" `; +const GATEWAY_ALREADY_ABSENT = + /gateway[^\n]*(?:does not exist|not found)|No (?:active )?gateway|No gateway metadata found/i; const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-hermes-gpu-startup"; const FAKE_API_KEY = "e2e-hermes-gpu-startup-key"; const FAKE_MODEL = "test-model"; const EXTRA_PLACEHOLDER_TOKEN_A = "e2e-hermes-gpu-extra-telegram-token"; const EXTRA_PLACEHOLDER_TOKEN_B = "e2e-hermes-gpu-extra-slack-token"; const LIVE_TIMEOUT_MS = 70 * 60_000; -const FORCE_LEGACY_GPU_PATCH = process.env.NEMOCLAW_DOCKER_GPU_PATCH === "1"; -const GPU_ROUTE = FORCE_LEGACY_GPU_PATCH ? "legacy-patch" : "native-openshell"; +const { route: GPU_ROUTE, scenario: GPU_STARTUP_SCENARIO } = resolveHermesGpuStartupScenario( + process.env.E2E_HERMES_GPU_STARTUP_SCENARIO, + process.env.NEMOCLAW_DOCKER_GPU_PATCH === "1", +); +const GPU_ROUTE_CONTROL = + GPU_ROUTE === "compatibility-only" + ? "1" + : GPU_ROUTE === "compatibility-fallback" + ? "fallback" + : undefined; validateSandboxName(SANDBOX_NAME); function commandEnv(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { @@ -48,7 +68,7 @@ function commandEnv(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { NEMOCLAW_SANDBOX_GPU: "1", NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, NEMOCLAW_ONBOARD_VALIDATION_TIMEOUT_SECONDS: "60", - ...(FORCE_LEGACY_GPU_PATCH ? { NEMOCLAW_DOCKER_GPU_PATCH: "1" } : {}), + ...(GPU_ROUTE_CONTROL ? { NEMOCLAW_DOCKER_GPU_PATCH: GPU_ROUTE_CONTROL } : {}), }; } @@ -66,7 +86,7 @@ async function cleanupHermes( label: string, ): Promise { await bestEffort(() => - host.nemoclaw([SANDBOX_NAME, "destroy", "--yes", "--cleanup-gateway"], { + host.nemoclaw([SANDBOX_NAME, "destroy", "--yes"], { artifactName: `${label}-nemoclaw-destroy`, env: commandEnv(), timeoutMs: 120_000, @@ -79,6 +99,27 @@ async function cleanupHermes( timeoutMs: 60_000, }), ); + // Check the OpenShell control-plane view before intentionally removing its + // gateway. A clean runner can have the CLI installed but no active gateway; + // that explicit state is also valid absence evidence. + const sandboxList = await host.command( + "bash", + [ + "-lc", + "if command -v openshell >/dev/null 2>&1; then openshell sandbox list; else printf '%s\\n' openshell-unavailable; fi", + ], + { + artifactName: `${label}-openshell-sandbox-absent`, + env: commandEnv(), + timeoutMs: 30_000, + }, + ); + const sandboxAbsenceProven = + sandboxList.exitCode === 0 + ? !outputContainsSandbox(sandboxList, SANDBOX_NAME) + : GATEWAY_ALREADY_ABSENT.test(resultText(sandboxList)); + expect(sandboxAbsenceProven, resultText(sandboxList)).toBe(true); + const runtimeCleanup = await host.command( "bash", ["-c", GATEWAY_CLEANUP_SCRIPT, "gateway-runtime-cleanup", GATEWAY_CLEANUP_MODULE, "nemoclaw"], @@ -119,6 +160,34 @@ async function cleanupHermes( portAvailable.exitCode, `gateway port ${gatewayPort} remains occupied after cleanup: ${resultText(portAvailable)}`, ).toBe(0); + + const labeledContainers = await host.command( + "docker", + ["ps", "-aq", "--filter", `label=openshell.ai/sandbox-name=${SANDBOX_NAME}`], + { + artifactName: `${label}-labeled-containers-absent`, + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }, + ); + expect(labeledContainers.exitCode, resultText(labeledContainers)).toBe(0); + expect(labeledContainers.stdout.trim()).toBe(""); + + const namedContainers = await host.command( + "docker", + ["ps", "-a", "--filter", `name=${SANDBOX_NAME}`, "--format", "{{.Names}}"], + { + artifactName: `${label}-backup-containers-absent`, + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }, + ); + expect(namedContainers.exitCode, resultText(namedContainers)).toBe(0); + expect( + namedContainers.stdout + .split(/\r?\n/u) + .filter((name) => name.includes(`${SANDBOX_NAME}-nemoclaw-gpu-backup-`)), + ).toEqual([]); } async function captureFailedGpuContainer( @@ -171,7 +240,7 @@ done`; ); } -test("hermes-gpu-startup: selected OpenShell GPU route reaches stable Ready state", { +test(`hermes-gpu-startup: ${GPU_STARTUP_SCENARIO} OpenShell GPU route reaches stable Ready state`, { timeout: LIVE_TIMEOUT_MS, }, async ({ artifacts, cleanup, host, sandbox }) => { await artifacts.target.declare({ @@ -180,6 +249,7 @@ test("hermes-gpu-startup: selected OpenShell GPU route reaches stable Ready stat sandboxName: SANDBOX_NAME, inference: "hermetic fake OpenAI-compatible endpoint", gpuRoute: GPU_ROUTE, + scenario: GPU_STARTUP_SCENARIO, }); await cleanupHermes(host, sandbox, "pre-cleanup"); @@ -205,15 +275,50 @@ test("hermes-gpu-startup: selected OpenShell GPU route reaches stable Ready stat await artifacts.writeJson("fake-openai-compatible-requests.json", fake.requests()); await fake.close(); }); - cleanup.add(`destroy Hermes sandbox ${SANDBOX_NAME}`, async () => { - await cleanupHermes(host, sandbox, "cleanup"); - }); + let cleanTeardownVerified = false; + cleanup.add(`destroy Hermes sandbox ${SANDBOX_NAME}`, () => + cleanTeardownVerified ? Promise.resolve() : cleanupHermes(host, sandbox, "cleanup"), + ); await artifacts.writeJson("fake-openai-compatible.json", { baseUrl: fake.baseUrl, model: FAKE_MODEL, publicHost: hostAddress, }); + const prepareFallbackWrapper = async () => { + const openshellInstall = await host.command( + "bash", + [path.join(REPO_ROOT, "scripts/install-openshell.sh")], + { + artifactName: "phase-2-install-openshell-for-gpu-fallback-wrapper", + cwd: REPO_ROOT, + env: commandEnv(), + timeoutMs: 5 * 60_000, + }, + ); + expect(openshellInstall.exitCode, resultText(openshellInstall)).toBe(0); + const realOpenshell = await host.command("bash", ["-lc", "command -v openshell"], { + artifactName: "phase-2-resolve-real-openshell-for-gpu-fallback-wrapper", + env: commandEnv(), + timeoutMs: 30_000, + }); + expect(realOpenshell.exitCode, resultText(realOpenshell)).toBe(0); + const wrapper = createHermesGpuFallbackWrapper(realOpenshell.stdout.trim()); + // Security-scoped #6110 fault injection: always remove the private wrapper root through the + // E2E cleanup stack. Do not generalize PATH interception to other tests without review. + cleanup.add("remove Hermes GPU fallback wrapper", () => + fs.rmSync(wrapper.rootDir, { recursive: true, force: true }), + ); + await artifacts.writeJson("gpu-fallback-wrapper.json", { + behavior: + "create real native state while dropping GPU attachment, reject exactly the first post-create nvidia-smi proof, then delegate compatibility retry", + eventVocabulary: HERMES_GPU_FALLBACK_EVENTS, + }); + return wrapper; + }; + const fallbackWrapper = + GPU_STARTUP_SCENARIO === "fallback" ? await prepareFallbackWrapper() : undefined; + const env = commandEnv({ COMPATIBLE_API_KEY: FAKE_API_KEY, NEMOCLAW_COMPAT_MODEL: FAKE_MODEL, @@ -223,6 +328,7 @@ test("hermes-gpu-startup: selected OpenShell GPU route reaches stable Ready stat NEMOCLAW_POLICY_MODE: "suggested", NEMOCLAW_PREFERRED_API: "openai-completions", NEMOCLAW_PROVIDER: "custom", + ...(fallbackWrapper?.componentEnv ?? {}), [HERMES_GPU_EXTRA_PLACEHOLDER_KEYS[0]]: EXTRA_PLACEHOLDER_TOKEN_A, [HERMES_GPU_EXTRA_PLACEHOLDER_KEYS[1]]: EXTRA_PLACEHOLDER_TOKEN_B, }); @@ -233,13 +339,28 @@ test("hermes-gpu-startup: selected OpenShell GPU route reaches stable Ready stat redactionValues: [FAKE_API_KEY, EXTRA_PLACEHOLDER_TOKEN_A, EXTRA_PLACEHOLDER_TOKEN_B], timeoutMs: 60 * 60_000, }); - const preRollbackDiagnosticsDir = - resultText(install).match(/Pre-rollback diagnostics saved:\s*(\S+)/)?.[1] ?? ""; + const gpuDiagnosticsDir = extractHermesGpuDiagnosticsDirectory(resultText(install)); await (install.exitCode !== 0 - ? captureFailedGpuContainer(host, preRollbackDiagnosticsDir) + ? captureFailedGpuContainer(host, gpuDiagnosticsDir) : Promise.resolve()); expect(install.exitCode, resultText(install)).toBe(0); + const verifyFallback = async (wrapper: ReturnType) => { + const fallbackEvents = readHermesGpuFallbackEvents(wrapper.eventsPath); + await artifacts.writeJson("gpu-fallback-events.json", fallbackEvents); + expect(fallbackEvents).toEqual([ + HERMES_GPU_FALLBACK_EVENTS.delegateNativeCreateWithoutGpu, + HERMES_GPU_FALLBACK_EVENTS.rejectNativeNvidiaSmiProof, + HERMES_GPU_FALLBACK_EVENTS.delegateCompatibilityCreate, + HERMES_GPU_FALLBACK_EVENTS.delegateNvidiaSmiProofAfterRejection, + ]); + expect(resultText(install)).toContain("Native GPU diagnostics saved:"); + for (const fragment of HERMES_GPU_FALLBACK_DISCLOSURE_FRAGMENTS) { + expect(resultText(install)).toContain(fragment); + } + }; + await (fallbackWrapper ? verifyFallback(fallbackWrapper) : Promise.resolve()); + const status = await host.command("nemoclaw", [SANDBOX_NAME, "status"], { artifactName: "phase-3-nemoclaw-status", env: commandEnv(), @@ -294,10 +415,20 @@ test("hermes-gpu-startup: selected OpenShell GPU route reaches stable Ready stat expect(JSON.stringify(fakeRequests)).not.toContain(EXTRA_PLACEHOLDER_TOKEN_A); expect(JSON.stringify(fakeRequests)).not.toContain(EXTRA_PLACEHOLDER_TOKEN_B); + await cleanupHermes(host, sandbox, "phase-5-clean-teardown"); + cleanTeardownVerified = true; + await artifacts.target.complete({ id: "hermes-gpu-startup", + gpuRoute: GPU_ROUTE, + scenario: GPU_STARTUP_SCENARIO, assertions: { selectedGpuRouteVerified: true, + ...(GPU_ROUTE === "compatibility-fallback" + ? { automaticCompatibilityFallbackVerified: true } + : GPU_ROUTE === "native-success" + ? { nativeGpuRouteVerified: true } + : { compatibilityOnlyRouteVerified: true }), openshellReady: true, sandboxCudaVerified: true, extraPlaceholderCommandRoundTripValid: true, @@ -306,6 +437,7 @@ test("hermes-gpu-startup: selected OpenShell GPU route reaches stable Ready stat supervisorTopologyValid: true, authenticatedInferenceRequestVerified: true, placeholderTokensAbsentFromInference: true, + cleanTeardownVerified: true, }, }); }); diff --git a/test/e2e/mock-parity.json b/test/e2e/mock-parity.json index 888a87c314d..2e24128df0b 100644 --- a/test/e2e/mock-parity.json +++ b/test/e2e/mock-parity.json @@ -2,6 +2,14 @@ "$comment": "SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\nSPDX-License-Identifier: Apache-2.0", "version": 1, "entries": [ + { + "live": "test/e2e/live/hermes-gpu-startup.test.ts", + "fast": [ + "test/e2e/support/hermes-gpu-startup-fallback.test.ts", + "test/e2e/support/hermes-gpu-startup-integrity.test.ts", + "test/e2e/support/hermes-workflow-boundary.test.ts" + ] + }, { "live": "test/e2e/live/inference-routing.test.ts", "fast": [ diff --git a/test/e2e/support/hermes-gpu-startup-fallback.test.ts b/test/e2e/support/hermes-gpu-startup-fallback.test.ts new file mode 100644 index 00000000000..e58b6afd5c2 --- /dev/null +++ b/test/e2e/support/hermes-gpu-startup-fallback.test.ts @@ -0,0 +1,282 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawn, spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { buildDirectSandboxGpuProofCommands } from "../../../src/lib/onboard/initial-policy"; +import { + hasRequiredOpenshellMessagingFeatures, + REQUIRED_OPENSHELL_SANDBOX_MCP_FEATURE, +} from "../../../src/lib/onboard/openshell-feature-gate"; +import { + createHermesGpuFallbackWrapper, + extractHermesGpuDiagnosticsDirectory, + HERMES_GPU_FALLBACK_EVENTS, + HERMES_GPU_NATIVE_NVIDIA_SMI_PROOF, + readHermesGpuFallbackEvents, + resolveHermesGpuStartupScenario, +} from "../live/hermes-gpu-startup-fallback.ts"; + +const roots: string[] = []; + +afterEach(() => { + for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); +}); + +function writeExecutable(filePath: string, body: string): void { + fs.writeFileSync(filePath, body, { encoding: "utf8", mode: 0o700 }); +} + +function createWrapperFixture( + prefix: string, + scripts: { openshell?: string; gateway?: string; sandbox?: string } = {}, +) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + roots.push(root); + const realDir = path.join(root, "real"); + fs.mkdirSync(realDir); + const fallback = "#!/usr/bin/env bash\nexit 0\n"; + const realOpenshell = path.join(realDir, "openshell"); + writeExecutable(realOpenshell, scripts.openshell ?? fallback); + writeExecutable(path.join(realDir, "openshell-gateway"), scripts.gateway ?? fallback); + writeExecutable(path.join(realDir, "openshell-sandbox"), scripts.sandbox ?? fallback); + return { + realDir, + realOpenshell, + root, + wrapper: createHermesGpuFallbackWrapper(realOpenshell, { rootDir: path.join(root, "wrapper") }), + }; +} + +function runWrapper(wrapperPath: string, args: string[], env: NodeJS.ProcessEnv) { + return spawnSync(wrapperPath, args, { encoding: "utf8", env }); +} + +function runWrapperConcurrently( + wrapperPath: string, + args: string[], + env: NodeJS.ProcessEnv, +): Promise { + return new Promise((resolve, reject) => { + const child = spawn(wrapperPath, args, { env, stdio: "ignore" }); + child.once("error", reject); + child.once("close", resolve); + }); +} + +describe("Hermes GPU startup scenario selection", () => { + it.each([ + [undefined, false, { route: "native-success", scenario: "native" }], + ["native", false, { route: "native-success", scenario: "native" }], + ["fallback", false, { route: "compatibility-fallback", scenario: "fallback" }], + ["compatibility-only", false, { route: "compatibility-only", scenario: "compatibility-only" }], + ["native", true, { route: "compatibility-only", scenario: "native" }], + ] as const)("maps scenario %s and compatibility=%s", (scenario, forced, expected) => { + expect(resolveHermesGpuStartupScenario(scenario, forced)).toEqual(expected); + }); + + it.each([ + ["unknown", false, /must be native, fallback, or compatibility-only/], + ["fallback", true, /requires automatic GPU routing/], + ] as const)("rejects invalid scenario/control combination %s", (scenario, forced, expected) => { + expect(() => resolveHermesGpuStartupScenario(scenario, forced)).toThrow(expected); + }); +}); + +describe("Hermes GPU startup failure diagnostics", () => { + it.each([ + [ + "recognizes native diagnostics", + "Native GPU diagnostics saved: /tmp/nemoclaw-native-gpu-diagnostics\n", + "/tmp/nemoclaw-native-gpu-diagnostics", + ], + [ + "prefers final compatibility diagnostics", + "Native GPU diagnostics saved: /tmp/native\nPre-rollback diagnostics saved: /tmp/compatibility", + "/tmp/compatibility", + ], + ["returns empty without a bundle", "GPU setup failed before diagnostics\n", ""], + ])("extracts %s", (_name, output, expected) => { + expect(extractHermesGpuDiagnosticsDirectory(output)).toBe(expected); + }); +}); + +describe("Hermes GPU startup fallback OpenShell wrapper", () => { + it("tracks the exact production nvidia-smi proof argv", () => { + const proof = buildDirectSandboxGpuProofCommands("alpha").find( + (candidate) => candidate.id === "nvidia-smi", + ); + expect(proof?.args).toEqual([ + "sandbox", + "exec", + "-n", + "alpha", + "--", + "sh", + "-lc", + HERMES_GPU_NATIVE_NVIDIA_SMI_PROOF, + ]); + }); + + it("creates native state without GPU, rejects one exact proof, and delegates compatibility", () => { + const { root, wrapper } = createWrapperFixture("hermes-gpu-fallback-test-", { + openshell: [ + "#!/usr/bin/env bash", + "marker=delegated", + 'if [[ "${1:-}" == "sandbox" && "${2:-}" == "create" ]]; then', + " marker=create-without-gpu", + ' for arg in "$@"; do', + ' if [[ "$arg" == "--gpu" ]]; then exit 97; fi', + " done", + "fi", + `printf '%s\\n' "$marker" >>"$E2E_FAKE_DELEGATE_LOG"`, + "", + ].join("\n"), + }); + const delegateMarkerLog = path.join(root, "delegate-markers.log"); + const env = { + ...process.env, + ...wrapper.componentEnv, + E2E_FAKE_DELEGATE_LOG: delegateMarkerLog, + }; + const secretMarkers = [ + "must-not-enter-wrapper-events", + "sk-wrapper-api-key", + "must-not-enter-wrapper-password", + ]; + + const nativeCreate = runWrapper( + wrapper.wrapperPath, + [ + "sandbox", + "create", + "--from", + "image", + "--gpu", + "--", + `TOKEN=${secretMarkers[0]}`, + `OPENAI_API_KEY=${secretMarkers[1]}`, + `PASSWORD=${secretMarkers[2]}`, + ], + env, + ); + expect(nativeCreate.status, nativeCreate.stderr).toBe(0); + + const nearMissProof = runWrapper( + wrapper.wrapperPath, + ["sandbox", "exec", "-n", "alpha", "--", "sh", "-lc", "nvidia-smi"], + env, + ); + expect(nearMissProof.status, nearMissProof.stderr).toBe(0); + + const rejectedProof = runWrapper( + wrapper.wrapperPath, + ["sandbox", "exec", "-n", "alpha", "--", "sh", "-lc", HERMES_GPU_NATIVE_NVIDIA_SMI_PROOF], + env, + ); + expect(rejectedProof.status).toBe(1); + expect(rejectedProof.stderr).toContain( + "Failed to initialize NVML: Driver/library version mismatch", + ); + + const compatibility = runWrapper( + wrapper.wrapperPath, + ["sandbox", "create", "--from", "image", "--gpu-device", "all"], + env, + ); + expect(compatibility.status, compatibility.stderr).toBe(0); + + const compatibilityProof = runWrapper( + wrapper.wrapperPath, + ["sandbox", "exec", "-n", "alpha", "--", "sh", "-lc", HERMES_GPU_NATIVE_NVIDIA_SMI_PROOF], + env, + ); + expect(compatibilityProof.status, compatibilityProof.stderr).toBe(0); + + const version = runWrapper(wrapper.wrapperPath, ["--version"], env); + expect(version.status, version.stderr).toBe(0); + expect(readHermesGpuFallbackEvents(wrapper.eventsPath)).toEqual([ + HERMES_GPU_FALLBACK_EVENTS.delegateNativeCreateWithoutGpu, + HERMES_GPU_FALLBACK_EVENTS.rejectNativeNvidiaSmiProof, + HERMES_GPU_FALLBACK_EVENTS.delegateCompatibilityCreate, + HERMES_GPU_FALLBACK_EVENTS.delegateNvidiaSmiProofAfterRejection, + ]); + const wrapperArtifacts = fs + .readdirSync(path.dirname(wrapper.eventsPath), { withFileTypes: true }) + .filter((entry) => entry.isFile()) + .map((entry) => + fs.readFileSync(path.join(path.dirname(wrapper.eventsPath), entry.name), "utf8"), + ) + .join("\n"); + for (const secretMarker of secretMarkers) { + expect(wrapperArtifacts).not.toContain(secretMarker); + } + expect(wrapperArtifacts).not.toMatch(/(?:TOKEN|API_KEY|PASSWORD)=/u); + // The fake delegate records a constant marker only; it never serializes argv. + expect(fs.readFileSync(delegateMarkerLog, "utf8").split(/\r?\n/u).filter(Boolean)).toEqual([ + "create-without-gpu", + "delegated", + "create-without-gpu", + "delegated", + "delegated", + ]); + }); + + it("rejects exactly one native nvidia-smi proof when wrapper calls race", async () => { + const { wrapper } = createWrapperFixture("hermes-gpu-fallback-race-test-"); + const env = { ...process.env, ...wrapper.componentEnv }; + const nativeCreate = runWrapper( + wrapper.wrapperPath, + ["sandbox", "create", "--from", "image", "--gpu"], + env, + ); + expect(nativeCreate.status, nativeCreate.stderr).toBe(0); + const statuses = await Promise.all( + Array.from({ length: 8 }, () => + runWrapperConcurrently( + wrapper.wrapperPath, + ["sandbox", "exec", "-n", "alpha", "--", "sh", "-lc", HERMES_GPU_NATIVE_NVIDIA_SMI_PROOF], + env, + ), + ), + ); + + expect(statuses.filter((status) => status === 1)).toHaveLength(1); + expect(statuses.filter((status) => status === 0)).toHaveLength(7); + const events = readHermesGpuFallbackEvents(wrapper.eventsPath); + expect( + events.filter((event) => event === HERMES_GPU_FALLBACK_EVENTS.delegateNativeCreateWithoutGpu), + ).toHaveLength(1); + expect( + events.filter((event) => event === HERMES_GPU_FALLBACK_EVENTS.rejectNativeNvidiaSmiProof), + ).toHaveLength(1); + expect( + events.filter( + (event) => event === HERMES_GPU_FALLBACK_EVENTS.delegateNvidiaSmiProofAfterRejection, + ), + ).toHaveLength(7); + }); + + it("preserves OpenShell version and capability detection without private wrapper env", () => { + const versionScript = "#!/usr/bin/env bash\nprintf '%s\\n' 'openshell 0.0.72'\n"; + const { realDir, wrapper } = createWrapperFixture("hermes-gpu-fallback-feature-test-", { + openshell: versionScript, + gateway: versionScript, + sandbox: `${versionScript}# ${REQUIRED_OPENSHELL_SANDBOX_MCP_FEATURE}\n`, + }); + expect( + hasRequiredOpenshellMessagingFeatures({ + openshellBin: wrapper.wrapperPath, + gatewayBin: path.join(realDir, "openshell-gateway"), + sandboxBin: path.join(realDir, "openshell-sandbox"), + allowExternalGatewayBin: true, + allowExternalSandboxBin: true, + }), + ).toBe(true); + }); +}); diff --git a/test/e2e/support/hermes-gpu-startup-integrity.test.ts b/test/e2e/support/hermes-gpu-startup-integrity.test.ts index 8a473179ba9..3141df45016 100644 --- a/test/e2e/support/hermes-gpu-startup-integrity.test.ts +++ b/test/e2e/support/hermes-gpu-startup-integrity.test.ts @@ -22,7 +22,8 @@ interface IntegrityFixture { } const roots: string[] = []; -const MCP_STATE_RECORD = `# nemoclaw-hermes-mcp-state-v1 intended=${"1".repeat(64)} applied=${"2".repeat(64)}`; +const MCP_STATE_DIGEST = "1".repeat(64); +const MCP_STATE_RECORD = `# nemoclaw-hermes-mcp-state-v1 intended=${MCP_STATE_DIGEST} applied=${MCP_STATE_DIGEST}`; afterEach(() => { for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); @@ -218,6 +219,21 @@ describe("Hermes managed startup integrity proof", () => { ); }); + it("rejects pending MCP state in the strict anchor (#6110)", () => { + const fixture = createFixture(); + fs.chmodSync(fixture.strictHashPath, 0o644); + const current = fs.readFileSync(fixture.strictHashPath, "utf-8"); + fs.writeFileSync( + fixture.strictHashPath, + current.replace(/applied=[0-9a-f]{64}/u, `applied=${"b".repeat(64)}`), + ); + fs.chmodSync(fixture.strictHashPath, 0o444); + + const proof = runProof(fixture); + expect(proof.status).not.toBe(0); + expect(proof.stderr).toContain("Hermes strict hash contains pending MCP state"); + }); + it("rejects a noncanonical API key assignment even when it belongs to the strict base", () => { const fixture = createFixture(); const config = fs.readFileSync(fixture.configPath, "utf-8"); diff --git a/test/e2e/support/hermes-workflow-boundary.test.ts b/test/e2e/support/hermes-workflow-boundary.test.ts index 2985a8f55e6..9fa433e5176 100644 --- a/test/e2e/support/hermes-workflow-boundary.test.ts +++ b/test/e2e/support/hermes-workflow-boundary.test.ts @@ -1,64 +1,241 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { spawnSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; import YAML from "yaml"; +import { validateHermesGpuStartupWorkflowBoundary } from "../../../tools/e2e/hermes-gpu-startup-workflow-boundary.mts"; import { validateE2eWorkflowBoundary } from "../../../tools/e2e/workflow-boundary.mts"; +import { readRepoText, readWorkflow } from "../../helpers/e2e-workflow-contract"; -describe("Hermes E2E workflow boundary", () => { - it("rejects hosted Hermes model and hermetic GPU-startup boundary drift", () => { - const workflow = YAML.parse(fs.readFileSync(".github/workflows/e2e.yaml", "utf8")); - workflow.jobs["hermes-e2e"].env.NEMOCLAW_MODEL = "minimaxai/minimax-m2.7"; - const gpuJob = workflow.jobs["hermes-gpu-startup"]; - gpuJob["runs-on"] = "ubuntu-latest"; - gpuJob.if = "${{ always() }}"; - gpuJob.env.NEMOCLAW_DOCKER_GPU_PATCH = "1"; - gpuJob.env.NEMOCLAW_E2E_USE_HOSTED_INFERENCE = "1"; - gpuJob.env.UNRELATED_SECRET = "${{ github.ref == 'refs/heads/main' && secrets.FOO || '' }}"; - const gpuRun = gpuJob.steps.find( - (step: { name?: string }) => step.name === "Run Hermes GPU startup live Vitest test", +const WF = ".github/workflows/e2e.yaml"; +const FX = "tools/e2e/hermes-gpu-docker-runtime-fixture.sh"; +const GPU = "hermes-gpu-startup"; +const KEY = "${{ secrets.NVIDIA_INFERENCE_API_KEY }}"; +type Doc = ReturnType; + +function tmp(use: (dir: string) => T): T { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-hermes-")); + try { + return use(dir); + } finally { + fs.rmSync(dir, { force: true, recursive: true }); + } +} + +function tempFile(content: string, use: (file: string) => T): T { + return tmp((dir) => { + const file = path.join(dir, "input"); + fs.writeFileSync(file, content); + return use(file); + }); +} + +function wfErrors( + mutate: (workflow: Doc) => void, + validate: (file: string) => string[] = validateHermesGpuStartupWorkflowBoundary, +): string[] { + const workflow = readWorkflow(); + mutate(workflow); + return tempFile(YAML.stringify(workflow), validate); +} + +function step(job: Doc, name: string): Doc { + return job.steps.find((candidate: { name?: string }) => candidate.name === name); +} + +function restoreHarness(tmp: string, content: string, fail: boolean) { + const state = path.join(tmp, "hermes-gpu-fallback-docker-runtime.123.1.fallback.ABC123"); + const daemon = path.join(tmp, "daemon.json"); + const bin = path.join(tmp, "bin"); + const sudoLog = path.join(tmp, "sudo.log"); + const { uid, gid } = os.userInfo(); + fs.mkdirSync(state, { mode: 0o700 }); + fs.mkdirSync(bin); + fs.writeFileSync(daemon, '{"default-runtime":"runc"}\n', { mode: 0o600 }); + const files = { + "capture.complete": "", + "daemon.json.metadata": `640 ${uid} ${gid}\n`, + "daemon.json.original": content, + "default-runtime.modified": "", + "default-runtime.original": "nvidia\n", + }; + for (const [name, value] of Object.entries(files)) { + fs.writeFileSync(path.join(state, name), value, { mode: 0o600 }); + } + const scripts = { + docker: `#!/bin/sh +if [ "$1" = info ] && [ "\${2:-}" = --format ]; then echo nvidia; fi +exit 0 +`, + stat: `#!/bin/bash +/usr/bin/stat -c "$2" "$3" 2>/dev/null && exit +exec /usr/bin/stat -f "$(printf %s "$2" | sed s/%a/%Lp/g)" "$3" +`, + sudo: `#!/bin/sh +printf '%s\n' "$*" >> "$FAKE_SUDO_LOG" +if [ "\${FAIL_INSTALL:-0}" = 1 ] && [ "\${1:-}" = install ]; then exit 42; fi +exec "$@" +`, + systemctl: "#!/bin/sh\n:\n", + }; + for (const [name, script] of Object.entries(scripts)) { + fs.writeFileSync(path.join(bin, name), script, { mode: 0o700 }); + } + return { bin, daemon, fail, gid, state, sudoLog, tmp, uid }; +} + +function restore(harness: ReturnType) { + return spawnSync("bash", [FX, "restore", harness.state, harness.daemon], { + encoding: "utf8", + env: { + ...process.env, + NEMOCLAW_E2E_FIXTURE_DAEMON_JSON: harness.daemon, + NEMOCLAW_E2E_FIXTURE_STATE_ROOT: harness.tmp, + FAIL_INSTALL: harness.fail ? "1" : "0", + FAKE_SUDO_LOG: harness.sudoLog, + PATH: `${harness.bin}:${process.env.PATH ?? ""}`, + }, + }); +} + +function snapshotFile(file: string) { + const fd = fs.openSync(file, "r"); + try { + return { stat: fs.fstatSync(fd), text: fs.readFileSync(fd, "utf8") }; + } finally { + fs.closeSync(fd); + } +} + +function withRestore( + fail: boolean, + check: (result: ReturnType, harness: ReturnType) => void, +): void { + tmp((tmp) => { + const harness = restoreHarness(tmp, '{"default-runtime":"nvidia"}\n', fail); + const result = restore(harness); + expect(fs.existsSync(harness.state)).toBe(false); + check(result, harness); + }); +} + +describe("Hermes GPU boundary", () => { + it("accepts baseline", () => { + expect(validateHermesGpuStartupWorkflowBoundary()).toEqual([]); + }); + + it("rejects broad drift", () => { + const errors = wfErrors((workflow) => { + workflow.jobs["hermes-e2e"].env.NEMOCLAW_MODEL = "minimaxai/minimax-m2.7"; + const job = workflow.jobs[GPU]; + job["runs-on"] = "ubuntu-latest"; + job.if = "${{ always() }}"; + job.strategy["max-parallel"] = 2; + job.strategy.matrix.scenario = ["native"]; + job.env.UNRELATED_SECRET = KEY; + const run = step(job, "Run Hermes GPU startup live Vitest test"); + run.env = { NVIDIA_INFERENCE_API_KEY: KEY }; + run.run = "npx vitest run --project e2e-live test/e2e/live/hermes-e2e.test.ts"; + step(job, "Upload Hermes GPU startup artifacts").with.path = "wrong"; + }, validateE2eWorkflowBoundary); + + expect(errors.join("\n")).toMatch( + /GPU runner.*generate-matrix.*serialize.*secrets.*hosted Hermes.*artifact path.*hosted-compatible/s, ); - gpuRun.env = { - COMPATIBLE_API_KEY: "${{ secrets.NVIDIA_INFERENCE_API_KEY }}", - NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1", - NVIDIA_API_KEY: "${{ secrets.NVIDIA_INFERENCE_API_KEY }}", - NVIDIA_INFERENCE_API_KEY: "${{ secrets.NVIDIA_INFERENCE_API_KEY }}", - }; - gpuRun.run = "npx vitest run --project e2e-live test/e2e/live/hermes-e2e.test.ts"; - gpuJob.steps.push({ - name: "Unexpected hosted test", - run: "npx vitest run --project e2e-live test/e2e/live/hermes-e2e.test.ts", - with: { token: "${{ github.ref == 'refs/heads/main' && secrets.FOO || '' }}" }, + }); + + it.each([ + ["hash", (fixture: string) => `${fixture}\n# drift`, "trusted SHA-256"], + ["mode", (fixture: string) => fixture.replace("install -m 0600", "install -m 0644"), "0644"], + [ + "path", + (fixture: string) => + fixture.replace( + "expected_daemon_json=/etc/docker/daemon.json", + "expected_daemon_json=/tmp/pr.json", + ), + "privileged state", + ], + [ + "metadata", + (fixture: string) => + fixture.replace('sudo chown "$original_uid:$original_gid"', "true # no chown"), + "mode, UID, GID", + ], + [ + "cleanup", + (fixture: string) => + fixture.replace('rm -rf -- "$state_dir" || restore_failed=1', "true # no cleanup"), + "before restore failure", + ], + ])("rejects fixture %s drift", (_name, mutate, expected) => { + const errors = tempFile(mutate(readRepoText(FX)), (file) => + validateHermesGpuStartupWorkflowBoundary(WF, file), + ); + expect(errors.join("\n")).toContain(expected); + }); + + it("preserves an invalid restore path", () => { + tmp((tmp) => { + const root = path.join(tmp, "root"); + const victim = path.join(tmp, "victim"); + fs.mkdirSync(root); + fs.mkdirSync(victim); + const result = spawnSync("bash", [FX, "restore", victim, path.join(tmp, "daemon.json")], { + encoding: "utf8", + env: { ...process.env, NEMOCLAW_E2E_FIXTURE_STATE_ROOT: root }, + }); + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("Refusing Docker restore"); + expect(fs.existsSync(victim)).toBe(true); + }); + }); + + it("rejects trusted-boundary drift", () => { + const errors = wfErrors((workflow) => { + const job = workflow.jobs[GPU]; + const checkout = step(job, "Checkout trusted Hermes GPU runtime fixture"); + const install = step(job, "Install trusted Hermes GPU runtime fixture"); + checkout.with.ref = "${{ inputs.checkout_sha }}"; + install.env.TRUSTED_FIXTURE_SHA256 = "0".repeat(64); + step(job, "Run Hermes GPU startup live Vitest test").run = `bash ${FX}`; + step(job, "Recover Docker daemon after Hermes GPU fallback fixture").run = + `done < <(bash ${FX})`; + step(job, "Remove trusted Hermes GPU runtime fixture").if = "${{ success() }}"; + job.steps.reverse(); }); - const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-hermes-workflow-")); - const workflowPath = path.join(tmp, "workflow.yaml"); - try { - fs.writeFileSync(workflowPath, YAML.stringify(workflow)); - expect(validateE2eWorkflowBoundary(workflowPath)).toEqual( - expect.arrayContaining([ - "hermes-e2e job must use the shared hosted-compatible model default", - "hermes-gpu-startup job must run on the native RTX PRO 6000 GPU runner", - "hermes-gpu-startup job must remain explicit-only behind generate-matrix", - "hermes-gpu-startup job must leave NEMOCLAW_DOCKER_GPU_PATCH unset to exercise auto routing", - "hermes-gpu-startup job env must not expose NEMOCLAW_E2E_USE_HOSTED_INFERENCE", - "hermes-gpu-startup job env must not consume repository secrets", - "hermes-gpu-startup step 'Run Hermes GPU startup live Vitest test' must not expose COMPATIBLE_API_KEY", - "hermes-gpu-startup step 'Run Hermes GPU startup live Vitest test' must not expose NEMOCLAW_E2E_USE_HOSTED_INFERENCE", - "hermes-gpu-startup step 'Run Hermes GPU startup live Vitest test' must not expose NVIDIA_API_KEY", - "hermes-gpu-startup step 'Run Hermes GPU startup live Vitest test' must not expose NVIDIA_INFERENCE_API_KEY", - "hermes-gpu-startup step 'Run Hermes GPU startup live Vitest test' must not consume repository secrets", - "hermes-gpu-startup step must run the dedicated Hermes GPU startup test", - "hermes-gpu-startup step 'Run Hermes GPU startup live Vitest test' must not run the hosted Hermes E2E test", - "hermes-gpu-startup step 'Unexpected hosted test' must not run the hosted Hermes E2E test", - "hermes-gpu-startup step 'Unexpected hosted test' must not consume repository secrets", - ]), + expect(errors.join("\n")).toMatch( + /root-owned.*trusted runtime.*trusted recovery.*always step/s, + ); + }); + + it("restores daemon content, metadata, ownership, and runtime", () => { + withRestore(false, (result, harness) => { + const snapshot = snapshotFile(harness.daemon); + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toBe("nvidia\n"); + expect(snapshot.text).toBe('{"default-runtime":"nvidia"}\n'); + expect([snapshot.stat.mode & 0o777, snapshot.stat.uid, snapshot.stat.gid]).toEqual([ + 0o640, + harness.uid, + harness.gid, + ]); + expect(fs.readFileSync(harness.sudoLog, "utf8")).toContain( + `chown ${harness.uid}:${harness.gid} ${harness.daemon}`, ); - } finally { - fs.rmSync(tmp, { recursive: true, force: true }); - } + }); + }); + + it("cleans private state after daemon restoration fails", () => { + withRestore(true, (result, harness) => { + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("Failed to prove restoration of the Docker daemon"); + expect(fs.readFileSync(harness.sudoLog, "utf8")).toContain("install -m 640"); + }); }); }); diff --git a/test/e2e/support/workflow-plan.test.ts b/test/e2e/support/workflow-plan.test.ts index caa83f96b1d..90f51e73d4f 100644 --- a/test/e2e/support/workflow-plan.test.ts +++ b/test/e2e/support/workflow-plan.test.ts @@ -30,6 +30,7 @@ describe("E2E workflow plan", () => { hermesSelected: true, explicitOnlyJobs: readFreeStandingJobsInventory().explicitOnlyJobs, }); + expect(plan.explicitOnlyJobs).toContain("hermes-gpu-startup"); }); it("validates jobs and selects only matching credential-free tests", () => { diff --git a/test/helpers/e2e-workflow-contract.ts b/test/helpers/e2e-workflow-contract.ts index 9207ba861ae..52be517a1f5 100644 --- a/test/helpers/e2e-workflow-contract.ts +++ b/test/helpers/e2e-workflow-contract.ts @@ -47,8 +47,12 @@ export type CompositeAction = { }; }; +export function readRepoText(path: string): string { + return readFileSync(join(REPO_ROOT, path), "utf-8"); +} + export function readYaml(path: string): T { - return YAML.parse(readFileSync(join(REPO_ROOT, path), "utf-8")) as T; + return YAML.parse(readRepoText(path)) as T; } export function readWorkflow(): Record { diff --git a/tools/e2e/hermes-gpu-docker-runtime-fixture.sh b/tools/e2e/hermes-gpu-docker-runtime-fixture.sh new file mode 100755 index 00000000000..e79a1f67600 --- /dev/null +++ b/tools/e2e/hermes-gpu-docker-runtime-fixture.sh @@ -0,0 +1,238 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# SOURCE_OF_TRUTH_REVIEW +# invalidState: a failed or cancelled fallback leaves privileged Docker daemon state modified. +# sourceBoundary: one root-owned 0500 entrypoint serves the live step and independent always recovery. +# whyNotSourceFix: inline PR shell cannot retain immutable provenance across step failure or cancellation. +# regressionTest: hermes-workflow-boundary.test.ts pins digest, modes, paths, metadata, and cleanup. +# removalCondition: remove this scenario-only helper when the test no longer mutates the host daemon. + +set -euo pipefail + +command_name="${1:-}" +state_dir="${2:-}" +daemon_json="${3:-}" +fixture_uid="$(id -u)" +if [ "$fixture_uid" -eq 0 ]; then + expected_state_root=/var/lib/nemoclaw-e2e + expected_daemon_json=/etc/docker/daemon.json +else + expected_state_root="${NEMOCLAW_E2E_FIXTURE_STATE_ROOT:-/var/lib/nemoclaw-e2e}" + expected_daemon_json="${NEMOCLAW_E2E_FIXTURE_DAEMON_JSON:-/etc/docker/daemon.json}" +fi + +fail() { + echo "$*" >&2 + return 1 +} + +wait_for_docker() { + local failure_message="$1" + local attempt + for attempt in $(seq 1 30); do + if docker info >/dev/null 2>&1; then + return 0 + fi + if [ "$attempt" -eq 30 ]; then + fail "$failure_message" + return 1 + fi + sleep 2 + done +} + +validate_state_dir() { + local expected_root_real="" + local state_name="" + local state_real="" + local state_mode="" + local state_uid="" + [ -n "$state_dir" ] && [ "$state_dir" != / ] && [ -d "$state_dir" ] \ + && [ ! -L "$state_dir" ] || return 1 + expected_root_real="$(cd -P -- "$expected_state_root" && pwd -P)" || return 1 + state_real="$(cd -P -- "$state_dir" && pwd -P)" || return 1 + [ "$(dirname -- "$state_real")" = "$expected_root_real" ] || return 1 + state_name="$(basename -- "$state_real")" + [[ "$state_name" =~ ^hermes-gpu-fallback-docker-runtime\.[0-9]+\.[0-9]+\.fallback\.[A-Za-z0-9]+$ ]] \ + || return 1 + read -r state_mode state_uid < <(stat -c '%a %u' "$state_dir") || return 1 + [ "$state_mode" = 700 ] && [ "$state_uid" = "$fixture_uid" ] +} + +validate_daemon_path() { + [ "$daemon_json" = "$expected_daemon_json" ] +} + +capture_original() { + local original_runtime="" + local original_mode="" + local original_uid="" + local original_gid="" + + umask 077 + validate_state_dir || fail "Docker fallback state directory must be private and fixture-owned" + validate_daemon_path || fail "Docker daemon path must use the fixed fixture target" + sudo -n true + + original_runtime="$(docker info --format '{{.DefaultRuntime}}')" + [ -n "$original_runtime" ] || fail "Docker did not report its original default runtime" + printf '%s\n' "$original_runtime" >"$state_dir/default-runtime.original" + chmod 0600 "$state_dir/default-runtime.original" + + if sudo test -f "$daemon_json"; then + read -r original_mode original_uid original_gid < <(sudo stat -c '%a %u %g' "$daemon_json") + [[ "$original_mode" =~ ^[0-7]{3,4}$ ]] || fail "Docker daemon mode could not be recorded" + [[ "$original_uid" =~ ^[0-9]+$ ]] || fail "Docker daemon UID could not be recorded" + [[ "$original_gid" =~ ^[0-9]+$ ]] || fail "Docker daemon GID could not be recorded" + printf '%s %s %s\n' "$original_mode" "$original_uid" "$original_gid" \ + >"$state_dir/daemon.json.metadata" + chmod 0600 "$state_dir/daemon.json.metadata" + install -m 0600 /dev/null "$state_dir/daemon.json.original" + # The fixture-owned redirection is intentional; sudo is needed only to read + # the root-owned source while the private backup remains fixture-owned. + # shellcheck disable=SC2024 + sudo cat "$daemon_json" >"$state_dir/daemon.json.original" + chmod 0600 "$state_dir/daemon.json.original" + elif sudo test -e "$daemon_json"; then + fail "$daemon_json exists but is not a regular file" + else + install -m 0600 /dev/null "$state_dir/daemon.json.absent" + printf '{}\n' >"$state_dir/daemon.json.original" + chmod 0600 "$state_dir/daemon.json.original" + fi + + install -m 0600 /dev/null "$state_dir/capture.complete" + printf '%s\n' "$original_runtime" +} + +select_runc() { + local original_runtime="" + local selected_runtime="" + + umask 077 + validate_state_dir || fail "Docker fallback state directory must be private and fixture-owned" + validate_daemon_path || fail "Docker daemon path must use the fixed fixture target" + [ -f "$state_dir/capture.complete" ] || fail "Docker fallback snapshot is incomplete" + original_runtime="$(cat "$state_dir/default-runtime.original")" + if [ "$original_runtime" != runc ]; then + /usr/bin/jq \ + 'if type == "object" then .["default-runtime"] = "runc" else error("Docker daemon.json must contain a top-level object") end' \ + "$state_dir/daemon.json.original" >"$state_dir/daemon.json.runc" + chmod 0600 "$state_dir/daemon.json.runc" + + # Mark the host mutation before it begins so either cleanup path knows that + # exact restoration and a daemon restart are mandatory after cancellation. + install -m 0600 /dev/null "$state_dir/default-runtime.modified" + sudo install -m 0600 "$state_dir/daemon.json.runc" "$daemon_json" + sudo systemctl restart docker + wait_for_docker "Docker did not recover after selecting the runc default runtime" + fi + + selected_runtime="$(docker info --format '{{.DefaultRuntime}}')" + [ "$selected_runtime" = runc ] || fail "Docker did not select the runc default runtime" + docker info --format '{{json .Runtimes}}' | grep -q 'nvidia' \ + || fail "Docker no longer reports the nvidia runtime" + printf '%s\n' "$selected_runtime" +} + +restore_original() { + local restore_failed=0 + local original_runtime="" + local restored_runtime="" + local original_mode="" + local original_uid="" + local original_gid="" + local restored_mode="" + local restored_uid="" + local restored_gid="" + + if ! validate_state_dir || ! validate_daemon_path; then + fail "Refusing Docker restore outside the fixed private fixture boundary" + return 1 + fi + + set +e + if [ -f "$state_dir/default-runtime.modified" ]; then + if [ ! -f "$state_dir/capture.complete" ]; then + restore_failed=1 + elif [ -f "$state_dir/daemon.json.absent" ]; then + sudo rm -f "$daemon_json" || restore_failed=1 + else + read -r original_mode original_uid original_gid \ + <"$state_dir/daemon.json.metadata" || restore_failed=1 + [[ "$original_mode" =~ ^[0-7]{3,4}$ ]] || restore_failed=1 + [[ "$original_uid" =~ ^[0-9]+$ ]] || restore_failed=1 + [[ "$original_gid" =~ ^[0-9]+$ ]] || restore_failed=1 + [ "$(stat -c '%a' "$state_dir/daemon.json.original" 2>/dev/null)" = 600 ] || restore_failed=1 + if [ "$restore_failed" -eq 0 ]; then + sudo install -m "$original_mode" "$state_dir/daemon.json.original" "$daemon_json" \ + || restore_failed=1 + sudo chown "$original_uid:$original_gid" "$daemon_json" || restore_failed=1 + sudo chmod "$original_mode" "$daemon_json" || restore_failed=1 + fi + fi + + # Restart even when a preceding restore operation failed. This is best-effort + # recovery; the verification below still refuses to report success. + sudo systemctl restart docker || restore_failed=1 + wait_for_docker "Docker did not recover while restoring its original default runtime" \ + || restore_failed=1 + fi + + if [ -f "$state_dir/capture.complete" ]; then + original_runtime="$(cat "$state_dir/default-runtime.original")" || restore_failed=1 + restored_runtime="$(docker info --format '{{.DefaultRuntime}}')" || restore_failed=1 + if [ -z "$original_runtime" ] || [ "$restored_runtime" != "$original_runtime" ]; then + echo "Docker default runtime was not restored: expected ${original_runtime:-}, got ${restored_runtime:-}" >&2 + restore_failed=1 + fi + + if [ -f "$state_dir/daemon.json.absent" ]; then + sudo test ! -e "$daemon_json" || restore_failed=1 + else + sudo cmp -s "$state_dir/daemon.json.original" "$daemon_json" || restore_failed=1 + read -r original_mode original_uid original_gid \ + <"$state_dir/daemon.json.metadata" || restore_failed=1 + read -r restored_mode restored_uid restored_gid \ + < <(sudo stat -c '%a %u %g' "$daemon_json") || restore_failed=1 + if [ "$restored_mode $restored_uid $restored_gid" != \ + "$original_mode $original_uid $original_gid" ]; then + echo "Docker daemon metadata was not restored" >&2 + restore_failed=1 + fi + fi + elif [ -f "$state_dir/default-runtime.modified" ]; then + restore_failed=1 + fi + + # The snapshot may contain registry/proxy credentials. Remove it regardless of + # whether restoration or verification succeeded, but preserve the failing exit. + rm -rf -- "$state_dir" || restore_failed=1 + if [ "$restore_failed" -ne 0 ]; then + fail "Failed to prove restoration of the Docker daemon after the fallback fixture" + return 1 + fi + printf '%s\n' "$restored_runtime" + return 0 +} + +case "$command_name" in + capture) + capture_original + ;; + select-runc) + select_runc + ;; + restore) + if [ ! -e "$state_dir" ]; then + exit 0 + fi + restore_original + ;; + *) + fail "usage: $0 {capture|select-runc|restore} STATE_DIR DAEMON_JSON" + exit 2 + ;; +esac diff --git a/tools/e2e/hermes-gpu-startup-workflow-boundary.mts b/tools/e2e/hermes-gpu-startup-workflow-boundary.mts index d19fbebd0ea..b4daf86eef7 100644 --- a/tools/e2e/hermes-gpu-startup-workflow-boundary.mts +++ b/tools/e2e/hermes-gpu-startup-workflow-boundary.mts @@ -1,15 +1,32 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { createHash } from "node:crypto"; import { readFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import YAML from "yaml"; +/** + * SOURCE_OF_TRUTH_REVIEW + * invalidState: untrusted workflow drift weakens privileged host mutation or daemon restoration. + * sourceBoundary: this shared workflow guard pins trust semantics that GitHub validates only as syntax. + * whyNotSourceFix: Actions cannot enforce fixture digest, mode, ownership, path, or recovery ordering. + * regressionTest: hermes-workflow-boundary.test.ts mutates each trust invariant independently. + * removalCondition: the Hermes GPU job no longer mutates privileged self-hosted runner state. + */ const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); const DEFAULT_WORKFLOW_PATH = join(REPO_ROOT, ".github", "workflows", "e2e.yaml"); +const FIXTURE = join(REPO_ROOT, "tools", "e2e", "hermes-gpu-docker-runtime-fixture.sh"); const JOB_NAME = "hermes-gpu-startup"; +const CHECKOUT = "actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10"; +const SOURCE = "tools/e2e/hermes-gpu-docker-runtime-fixture.sh"; +const SHA = "e273c4baa7fe89546d64517cf56eafec30aeda7b355971263605ab1327fade02"; +const F_PATH = + "/usr/local/libexec/nemoclaw/hermes-gpu-docker-runtime-fixture.${GITHUB_RUN_ID}.${GITHUB_RUN_ATTEMPT}.${E2E_HERMES_GPU_STARTUP_SCENARIO}"; +const FALLBACK = "${{ matrix.scenario == 'fallback' }}"; +const BASH = "/bin/bash --noprofile --norc -e -o pipefail {0}"; const RUN_STEP_NAME = "Run Hermes GPU startup live Vitest test"; const DOCKER_AUTH_STEP_NAME = "Authenticate to Docker Hub"; const HOSTED_PROVIDER_ENV_NAMES = [ @@ -20,7 +37,7 @@ const HOSTED_PROVIDER_ENV_NAMES = [ ] as const; const SECRET_REFERENCE_PATTERN = /\bsecrets\.[A-Za-z0-9_]+\b/u; const EXPECTED_SELECTOR = - "${{ contains(format(',{0},', inputs.jobs), ',hermes-gpu-startup,') || contains(format(',{0},', inputs.targets), ',hermes-gpu-startup,') }}"; + "${{ github.repository == 'NVIDIA/NemoClaw' && github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && (contains(format(',{0},', inputs.jobs), ',hermes-gpu-startup,') || contains(format(',{0},', inputs.targets), ',hermes-gpu-startup,')) }}"; type WorkflowRecord = Record; type WorkflowStep = WorkflowRecord & { @@ -42,8 +59,43 @@ function stringValue(value: unknown): string { return typeof value === "string" ? value : ""; } +// biome-ignore format: Compact declarative shell-proof vocabulary. +const TOKENS = { "@bash": '/bin/bash "$trusted_fixture" "$@"', "@bin": "/usr/bin", "@daemon": '"$daemon_json"', "@docker": "/etc/docker/daemon.json", "@env": "/usr/bin/sudo -n /usr/bin/env -i", "@fixture": '"$trusted_fixture"', "@gpu": "hermes-gpu-fallback-docker-runtime", "@install": "/usr/bin/sudo /usr/bin/install", "@root": '"$trusted_state_root"', "@run": "run_trusted_fixture", "@sha": '"$TRUSTED_FIXTURE_SHA256"', "@source": '"$trusted_source"', "@state": '"$state_dir"', "@sudo": "/usr/bin/sudo", "@workflow": '"$TRUSTED_WORKFLOW_SHA"' } as const; + +function proof(spec: string): string[] { + return spec + .trim() + .split("\n") + .map((line) => line.trim().replace(/@\w+/gu, (token) => TOKENS[token as keyof typeof TOKENS])); +} + +function hasProof(value: unknown, spec: string, ordered = false, raw = false): boolean { + const script = raw + ? stringValue(value) + : stringValue(value) + .replace(/\\\r?\n/gu, " ") + .replace(/\s+/gu, " ") + .trim(); + let offset = 0; + return proof(spec).every((fragment) => { + const index = script.indexOf(fragment, offset); + if (ordered && index >= 0) offset = index + fragment.length; + return index >= 0; + }); +} + +function trustedEnv(step: WorkflowStep | undefined): boolean { + const env = asRecord(step?.env); + return ( + env.BASH_ENV === "/dev/null" && + env.E2E_HERMES_GPU_STARTUP_SCENARIO === "${{ matrix.scenario }}" && + env.ENV === "/dev/null" + ); +} + export function validateHermesGpuStartupWorkflowBoundary( workflowPath = DEFAULT_WORKFLOW_PATH, + fixtureFile = FIXTURE, ): string[] { const workflow = asRecord(YAML.parse(readFileSync(workflowPath, "utf8"))); const job = asRecord(asRecord(workflow.jobs)[JOB_NAME]); @@ -58,29 +110,45 @@ export function validateHermesGpuStartupWorkflowBoundary( if (job.needs !== "generate-matrix" || job.if !== EXPECTED_SELECTOR) { errors.push(`${JOB_NAME} job must remain explicit-only behind generate-matrix`); } - if (job["timeout-minutes"] !== 75) { - errors.push(`${JOB_NAME} job must keep the 75 minute timeout`); + if (job["timeout-minutes"] !== 90) { + errors.push(`${JOB_NAME} requires a 90 minute timeout`); + } + const strategy = asRecord(job.strategy); + const matrix = asRecord(strategy.matrix); + if ( + strategy["fail-fast"] !== false || + strategy["max-parallel"] !== 1 || + !Array.isArray(matrix.scenario) || + matrix.scenario.length !== 3 || + matrix.scenario[0] !== "native" || + matrix.scenario[1] !== "fallback" || + matrix.scenario[2] !== "compatibility-only" + ) { + errors.push(`${JOB_NAME} must serialize GPU scenarios`); } const jobEnv = asRecord(job.env); const requiredEnv = { - E2E_DEFAULT_ENABLED: "0", + E2E_ARTIFACT_DIR: + "${{ github.workspace }}/e2e-artifacts/live/hermes-gpu-startup/${{ matrix.scenario }}", + E2E_HERMES_GPU_STARTUP_SCENARIO: "${{ matrix.scenario }}", E2E_JOB: "1", E2E_TARGET_ID: JOB_NAME, NEMOCLAW_AGENT: "hermes", NEMOCLAW_RUN_LIVE_E2E: "1", NEMOCLAW_SANDBOX_GPU: "1", - NEMOCLAW_SANDBOX_NAME: "e2e-hermes-gpu-startup", + NEMOCLAW_SANDBOX_NAME: "e2e-hermes-gpu-startup-${{ matrix.scenario }}", } as const; for (const [name, expected] of Object.entries(requiredEnv)) { if (jobEnv[name] !== expected) { errors.push(`${JOB_NAME} job must set ${name}=${expected}`); } } + if (Object.hasOwn(jobEnv, "E2E_DEFAULT_ENABLED")) { + errors.push(`${JOB_NAME} no E2E_DEFAULT_ENABLED`); + } if (Object.hasOwn(jobEnv, "NEMOCLAW_DOCKER_GPU_PATCH")) { - errors.push( - `${JOB_NAME} job must leave NEMOCLAW_DOCKER_GPU_PATCH unset to exercise auto routing`, - ); + errors.push(`${JOB_NAME} no NEMOCLAW_DOCKER_GPU_PATCH`); } for (const name of HOSTED_PROVIDER_ENV_NAMES) { if (Object.hasOwn(jobEnv, name)) { @@ -107,17 +175,237 @@ export function validateHermesGpuStartupWorkflowBoundary( errors.push(`${JOB_NAME} step '${stepName}' must not run the hosted Hermes E2E test`); } } + + const prI = steps.findIndex( + (step) => + step.uses === CHECKOUT && + asRecord(step.with).ref === "${{ inputs.checkout_sha || github.sha }}", + ); + const ci = steps.findIndex((step) => step.name === "Checkout trusted Hermes GPU runtime fixture"); + const checkout = steps[ci]; + const ii = steps.findIndex((step) => step.name === "Install trusted Hermes GPU runtime fixture"); + const install = steps[ii]; + const co = asRecord(checkout?.with); + const ie = asRecord(install?.env); + const spec = `trusted_checkout="$GITHUB_WORKSPACE/.trusted-hermes-gpu-fixture-\${GITHUB_RUN_ID}-\${GITHUB_RUN_ATTEMPT}" +trusted_source="$trusted_checkout/${SOURCE}" +trusted_fixture="${F_PATH}" +[[ @sha =~ ^[a-f0-9]{64}$ ]] +[[ @workflow =~ ^[a-f0-9]{40}$ ]] +[[ "$TRUSTED_DISPATCH_SHA" = @workflow ]] +@bin/git -C "$trusted_checkout" rev-parse HEAD +[ -f @source ] && [ ! -L @source ] +@install -d -o root -g root -m 0755 /usr/local/libexec/nemoclaw +@install -o root -g root -m 0500 @source @fixture +@sudo @bin/stat -c '%a %u %g' @fixture)" = "500 0 0" +printf '%s %s\\n' @sha @fixture | @sudo @bin/sha256sum -c - +@sudo @bin/cmp -s @source @fixture +trusted_state_root=/var/lib/nemoclaw-e2e +@install -d -o root -g root -m 0700 @root +@sudo @bin/find @root +-type d -name '@gpu.*' -print0 +@run restore "$stale_state_dir" @docker +@env +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +@bash +if ! @run restore`; + if ( + prI < 0 || + ci !== prI + 1 || + ii !== ci + 1 || + checkout?.if !== FALLBACK || + checkout?.uses !== CHECKOUT || + co.repository !== "NVIDIA/NemoClaw" || + co.ref !== "${{ github.workflow_sha }}" || + co.path !== ".trusted-hermes-gpu-fixture-${{ github.run_id }}-${{ github.run_attempt }}" || + co["sparse-checkout"] !== SOURCE || + co["sparse-checkout-cone-mode"] !== false || + co["persist-credentials"] !== false || + install?.if !== FALLBACK || + install?.shell !== BASH || + !trustedEnv(install) || + ie.TRUSTED_DISPATCH_SHA !== "${{ github.sha }}" || + ie.TRUSTED_FIXTURE_SHA256 !== SHA || + ie.TRUSTED_WORKFLOW_SHA !== "${{ github.workflow_sha }}" || + !hasProof(install?.run, spec) || + !hasProof(install?.run, proof(spec).slice(9, 13).join("\n"), true) + ) { + errors.push(`${JOB_NAME} root-owned fixture boundary failed`); + } + const runStep = steps.find((step) => step.name === RUN_STEP_NAME); if (!runStep) { errors.push(`${JOB_NAME} job missing step: ${RUN_STEP_NAME}`); return errors; } - const runScript = stringValue(runStep.run); - if (!runScript.includes("npx vitest run --project e2e-live")) { - errors.push(`${JOB_NAME} step must run the e2e-live Vitest project`); + const run = stringValue(runStep.run); + const pi = steps.findIndex((step) => step.name === "Prepare E2E workspace"); + const ni = steps.findIndex((step) => step.name === "Reassert trusted Node runtime"); + const node = steps[ni]; + if ( + runStep.shell !== BASH || + !trustedEnv(runStep) || + pi < 0 || + ni !== pi + 1 || + ni + 1 !== steps.indexOf(runStep) || + node?.uses !== "actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e" || + asRecord(node?.with)["node-version"] !== "22" || + !trustedEnv(node) || + asRecord(node?.env).NODE_OPTIONS !== "" || + run.includes(SOURCE) || + !hasProof( + runStep.run, + `trusted_fixture="${F_PATH}" +@env +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +@bash +@run capture @state @daemon +@run select-runc @state @daemon +@run restore @state @daemon +trusted_state_root=/var/lib/nemoclaw-e2e +@install -d -o root -g root -m 0700 @root +state_dir="$(@sudo @bin/mktemp -d "$trusted_state_root/@gpu. +@sudo @bin/chown root:root @state +@sudo @bin/chmod 0700 @state`, + ) || + steps.some( + (step) => + step.name === "Prepare no-GPU native fallback fixture" || + step.name === "Restore Docker default runtime after fallback fixture", + ) || + !hasProof( + run, + `umask 077 +mktemp -d +chmod 0700 @state +\${GITHUB_RUN_ID}.\${GITHUB_RUN_ATTEMPT}.fallback.XXXXXX +restore_docker_default_runtime() +trap restore_docker_default_runtime EXIT +trap 'exit 130' INT +trap 'exit 143' TERM +@run capture +@run select-runc +@run restore +SOURCE_OF_TRUTH_REVIEW +invalidState: +sourceBoundary: +whyNotSourceFix: +regressionTest: +removalCondition:`, + false, + true, + ) || + /\b(?:install\s+-m|chmod)\s+0?644\b/u.test(run) || + !run.includes("npx vitest run --project e2e-live") || + !run.includes("test/e2e/live/hermes-gpu-startup.test.ts") + ) { + errors.push(`${JOB_NAME} trusted runtime boundary failed`); } - if (!runScript.includes("test/e2e/live/hermes-gpu-startup.test.ts")) { - errors.push(`${JOB_NAME} step must run the dedicated Hermes GPU startup test`); + + const ri = steps.findIndex( + (step) => step.name === "Recover Docker daemon after Hermes GPU fallback fixture", + ); + const recovery = steps[ri]; + const rr = stringValue(recovery?.run); + if ( + recovery?.if !== "always()" || + recovery?.shell !== BASH || + !trustedEnv(recovery) || + ri <= steps.indexOf(runStep) || + rr.includes(SOURCE) || + rr.includes("done < <(") || + !hasProof( + recovery?.run, + `trusted_fixture="${F_PATH}" +trusted_state_root=/var/lib/nemoclaw-e2e +@sudo @bin/find @root +\${GITHUB_RUN_ID}.\${GITHUB_RUN_ATTEMPT}.fallback. +@env +@bash +@run restore @state @docker +recovery_failed=1`, + ) || + /\b(?:install\s+-m|chmod)\s+0?644\b/u.test(rr) + ) { + errors.push(`${JOB_NAME} trusted recovery boundary failed`); + } + + const ki = steps.findIndex((step) => step.name === "Remove trusted Hermes GPU runtime fixture"); + const cleanup = steps[ki]; + if ( + ki !== ri + 1 || + cleanup?.if !== "${{ always() && matrix.scenario == 'fallback' }}" || + cleanup?.shell !== BASH || + !trustedEnv(cleanup) || + !hasProof( + cleanup?.run, + `trusted_fixture="${F_PATH}" +@sudo @bin/rm -f -- @fixture`, + ) + ) { + errors.push(`${JOB_NAME} cleanup requires an always step`); + } + + let fixture = ""; + try { + fixture = readFileSync(fixtureFile, "utf8"); + } catch { + errors.push(`${JOB_NAME} fixture missing`); + } + if (fixture) { + if (createHash("sha256").update(fixture).digest("hex") !== SHA) { + errors.push(`${JOB_NAME} fixture must match its trusted SHA-256`); + } + if (/\b(?:install\s+-m|chmod)\s+0?644\b/u.test(fixture)) { + errors.push(`${JOB_NAME} fixture must reject permissive 0644 modes`); + } + if ( + !hasProof( + fixture, + `expected_state_root=/var/lib/nemoclaw-e2e +expected_daemon_json=@docker +validate_daemon_path +@gpu\\.[0-9]+\\.[0-9]+\\.fallback`, + ) + ) { + errors.push(`${JOB_NAME} fixture must pin privileged state and daemon paths`); + } + if ( + !hasProof( + fixture, + `umask 077 +install -m 0600 /dev/null "$state_dir/daemon.json.original" +@bin/jq +sudo stat -c '%a %u %g' @daemon +daemon.json.metadata +sudo install -m "$original_mode" +sudo chown "$original_uid:$original_gid" @daemon +sudo chmod "$original_mode" @daemon +sudo cmp -s "$state_dir/daemon.json.original" @daemon +restored_mode $restored_uid $restored_gid +"$restored_runtime" != "$original_runtime" +rm -rf -- @state`, + false, + true, + ) + ) { + errors.push(`${JOB_NAME} fixture must preserve content, mode, UID, GID, and runtime`); + } + const cleanup = + /^\s{2}rm -rf -- "\$state_dir" \|\| restore_failed=1$/mu.exec(fixture)?.index ?? -1; + if (cleanup < 0 || fixture.indexOf('if [ "$restore_failed" -ne 0 ]', cleanup) < cleanup) { + errors.push(`${JOB_NAME} fixture must clean private state before restore failure`); + } + } + + const upload = asRecord( + steps.find((step) => step.name === "Upload Hermes GPU startup artifacts")?.with, + ); + if ( + upload.name !== "e2e-hermes-gpu-startup-${{ matrix.scenario }}" || + upload.path !== "e2e-artifacts/live/hermes-gpu-startup/${{ matrix.scenario }}/" + ) { + errors.push(`${JOB_NAME} upload needs a scenario artifact path`); } return errors; diff --git a/tools/e2e/operations-workflow-boundary.mts b/tools/e2e/operations-workflow-boundary.mts index a87f8d08d36..c9a93ed6060 100644 --- a/tools/e2e/operations-workflow-boundary.mts +++ b/tools/e2e/operations-workflow-boundary.mts @@ -187,9 +187,15 @@ function validatePrGateDispatch(errors: string[], workflow: OperationsWorkflow): for (const [jobName, job] of Object.entries(workflow.jobs)) { for (const step of job.steps ?? []) { + const trustedHermesFixtureCheckout = + jobName === "hermes-gpu-startup" && + step.name === "Checkout trusted Hermes GPU runtime fixture" && + step.with?.repository === "NVIDIA/NemoClaw" && + step.with?.ref === "${{ github.workflow_sha }}"; if ( step.uses?.startsWith("actions/checkout@") && - step.with?.ref !== "${{ inputs.checkout_sha || github.sha }}" + step.with?.ref !== "${{ inputs.checkout_sha || github.sha }}" && + !trustedHermesFixtureCheckout ) { errors.push(`${jobName} checkout must use the selected PR commit`); } diff --git a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts index 4b7f3ae3f03..99893fe332f 100644 --- a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts +++ b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts @@ -97,6 +97,13 @@ const EXPLICIT_UPLOAD_CONTRACTS = new Map([ path: "e2e-artifacts/live/hermes-inference-switch/${{ matrix.mode }}/", }, ], + [ + "hermes-gpu-startup", + { + name: "e2e-hermes-gpu-startup-${{ matrix.scenario }}", + path: "e2e-artifacts/live/hermes-gpu-startup/${{ matrix.scenario }}/", + }, + ], [ "hermes-slack", { diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index 4e06b2c9e97..c2d7f164519 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -51,6 +51,7 @@ const SELECTOR_ID_PATTERN = /^[A-Za-z0-9_-]+$/; const FREE_STANDING_JOB_MARKER = "E2E_JOB"; const FREE_STANDING_TARGET_MARKER = "E2E_TARGET_ID"; const FREE_STANDING_DEFAULT_ENABLED_MARKER = "E2E_DEFAULT_ENABLED"; +const EXPLICIT_ONLY_JOBS_WITHOUT_ENV_MARKER = new Set(["hermes-gpu-startup"]); const COMMON_SECRET_ENV_NAMES = [ "NVIDIA_API_KEY", "NVIDIA_INFERENCE_API_KEY", @@ -58,7 +59,7 @@ const COMMON_SECRET_ENV_NAMES = [ "DOCKERHUB_TOKEN", "GITHUB_TOKEN", ]; -const FREE_STANDING_SELECTOR_SPECIAL_CASES = new Set(["hermes-e2e"]); +const FREE_STANDING_SELECTOR_SPECIAL_CASES = new Set(["hermes-e2e", "hermes-gpu-startup"]); const PUBLIC_NVIDIA_ENDPOINT_KEY_JOBS = new Set([ "device-auth-health", "model-router-provider-routed-inference", @@ -133,6 +134,8 @@ function deriveFreeStandingJobsInventoryFromJobs(jobs: WorkflowRecord): { } else { explicitOnlyJobs.push(jobId); } + } else if (EXPLICIT_ONLY_JOBS_WITHOUT_ENV_MARKER.has(jobId)) { + explicitOnlyJobs.push(jobId); } if (!hasTargetMarker) continue; @@ -2088,7 +2091,11 @@ function validateDockerHubAuthBoundary(errors: string[], jobs: WorkflowRecord): const authIndex = steps.indexOf(auth); const cleanupIndex = steps.indexOf(cleanup); const expectedAuthIndex = - jobName === "jetson-nvmap-gpu" ? checkoutIndex + 2 : checkoutIndex + 1; + jobName === "jetson-nvmap-gpu" + ? checkoutIndex + 2 + : jobName === "hermes-gpu-startup" + ? checkoutIndex + 3 + : checkoutIndex + 1; if (checkoutIndex < 0 || authIndex !== expectedAuthIndex) { errors.push( jobName === "jetson-nvmap-gpu"