diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 645945038bf..288b96b8fcc 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -14,7 +14,7 @@ on: default: "" type: string jobs: - description: "Optional comma-separated free-standing live E2E job ids. Empty runs default-enabled jobs only when targets is also empty; explicit-only jobs openshell-gateway-auth-contract, jetson-nvmap-gpu, and sandbox-rlimits-connect are skipped unless selected." + description: "Optional comma-separated free-standing live E2E job ids. Empty runs default-enabled jobs only when targets is also empty; explicit-only jobs hermes-gpu-startup, openshell-gateway-auth-contract, jetson-nvmap-gpu, and sandbox-rlimits-connect are skipped unless selected." required: false default: "" type: string @@ -1578,6 +1578,51 @@ jobs: shell: bash run: bash .github/scripts/docker-auth-cleanup.sh + hermes-gpu-startup: + needs: generate-matrix + if: ${{ contains(format(',{0},', inputs.jobs), ',hermes-gpu-startup,') || contains(format(',{0},', inputs.targets), ',hermes-gpu-startup,') }} + runs-on: linux-amd64-gpu-rtxpro6000-latest-1 + timeout-minutes: 75 + env: + E2E_JOB: "1" + E2E_DEFAULT_ENABLED: "0" + E2E_TARGET_ID: "hermes-gpu-startup" + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/hermes-gpu-startup + NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js + NEMOCLAW_RUN_LIVE_E2E: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_AGENT: hermes + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_RECREATE_SANDBOX: "1" + NEMOCLAW_SANDBOX_GPU: "1" + NEMOCLAW_SANDBOX_NAME: e2e-hermes-gpu-startup + NEMOCLAW_ONBOARD_VALIDATION_TIMEOUT_SECONDS: "60" + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - *dockerhub-auth + + - name: Prepare E2E workspace + uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28 + + - name: Run Hermes GPU startup live Vitest test + run: | + set -euo pipefail + npx vitest run --project e2e-live \ + test/e2e/live/hermes-gpu-startup.test.ts \ + --silent=false --reporter=default + + - name: Upload Hermes GPU startup artifacts + if: always() + uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 + + - name: Clean up Docker auth + if: always() + shell: bash + run: bash .github/scripts/docker-auth-cleanup.sh + hermes-dashboard: needs: generate-matrix if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',hermes-dashboard,') || contains(format(',{0},', inputs.targets), ',hermes-dashboard,') }} @@ -4426,6 +4471,7 @@ jobs: sessions-agents-cli, runtime-overrides, hermes-e2e, + hermes-gpu-startup, hermes-dashboard, hermes-slack, hermes-discord, @@ -4639,7 +4685,7 @@ jobs: ? '**Requested jobs:** _(selector rejected by workflow validation)_' : requestedJobs ? `**Requested jobs:** \`${requestedJobs}\`` - : '**Requested jobs:** _(default — all default-enabled free-standing jobs; explicit-only jobs `openshell-gateway-auth-contract`, `jetson-nvmap-gpu`, and `sandbox-rlimits-connect` are skipped unless selected)_', + : '**Requested jobs:** _(default — all default-enabled free-standing jobs; explicit-only jobs `hermes-gpu-startup`, `openshell-gateway-auth-contract`, `jetson-nvmap-gpu`, and `sandbox-rlimits-connect` are skipped unless selected)_', `**Summary:** ${passed.length} passed, ${failed.length} failed, ${cancelled.length} cancelled, ${skipped.length} skipped`, '', '| Job | Result |', diff --git a/agents/hermes/Dockerfile b/agents/hermes/Dockerfile index 5ec250e5fc7..4191e4813d7 100644 --- a/agents/hermes/Dockerfile +++ b/agents/hermes/Dockerfile @@ -168,7 +168,7 @@ RUN test -x /usr/local/lib/nemoclaw/validate-hermes-env-secret-boundary.py \ # also rewrite the Dockerfile-committed hash, which reviewers gate). Regenerate # with `sha256sum agents/hermes/{hermes-wrapper.py,validate-env-secret-boundary.py}`. ARG NEMOCLAW_HERMES_WRAPPER_SHA256=03e0afbe00e352d0dfcf14b99ea1821f9fd29f87dad49ce19add2ec96d1941cc -ARG NEMOCLAW_HERMES_VALIDATOR_SHA256=d8ebac7143ce79061a86fefb79f2be9fb1d73f41229dc7144a1c157384947fd1 +ARG NEMOCLAW_HERMES_VALIDATOR_SHA256=970d7ff03bc409ff1d5ca46bfdbd2a42ac28a32a810ccc147a508301bff38496 # hadolint ignore=DL4006 RUN printf '%s %s\n' \ "$NEMOCLAW_HERMES_VALIDATOR_SHA256" /usr/local/lib/nemoclaw/validate-hermes-env-secret-boundary.py \ diff --git a/agents/hermes/runtime-config-guard.py b/agents/hermes/runtime-config-guard.py index a4c9cb268c1..34550600afe 100755 --- a/agents/hermes/runtime-config-guard.py +++ b/agents/hermes/runtime-config-guard.py @@ -321,7 +321,22 @@ def _pid1_is_nemoclaw_start() -> bool: os.close(proc_pid_fd) if proc_root_fd >= 0: os.close(proc_root_fd) - return _cmdline_is_nemoclaw_start(cmdline) + # SOURCE_OF_TRUTH_REVIEW (#6110): OpenShell owns PID 1 in Docker-driver + # sandboxes and starts the workload as its direct non-root child. NemoClaw's + # GPU recreate used to append that workload to the supervisor argv; the + # source fix is buildDockerGpuCloneRunArgs() in docker-gpu-patch.ts, which + # now preserves an empty Config.Cmd. An attacker controlling an image or + # recreate argv could otherwise append `nemoclaw-start`, impersonate direct + # PID 1 authority, and bypass the startup identity check. Keep this + # exclusion as defense in depth; live proof is in + # assertHermesGpuStartupProof() in + # test/e2e/live/hermes-gpu-startup-proof.ts. The dual-mode authorization + # branch can be removed only after direct-PID1 images are no longer + # supported; this supervisor exclusion itself remains a security invariant. + return ( + not _cmdline_is_openshell_supervisor(cmdline) + and _cmdline_is_nemoclaw_start(cmdline) + ) def _pinned_process_matches_startup_identity( diff --git a/agents/hermes/start.sh b/agents/hermes/start.sh index 0a80598a317..3a245cc1f68 100755 --- a/agents/hermes/start.sh +++ b/agents/hermes/start.sh @@ -1681,8 +1681,8 @@ ensure_hermes_runtime_api_server_key() { rm -f "$result_file" return 1 } - # Keep the guard as PID 1's direct child: --startup-owner is authenticated by - # exact parent identity. The guard's own alarm bounds this startup-only call; + # Keep the guard as the startup owner's direct child: --startup-owner is + # authenticated by exact parent identity. Its own alarm bounds this call; # wrapping it in `timeout` would interpose a different parent process. if "$_HERMES_PYTHON" -I "$_HERMES_RUNTIME_CONFIG_GUARD" ensure-api-key \ --hermes-dir "$HERMES_DIR" \ diff --git a/agents/hermes/validate-env-secret-boundary.py b/agents/hermes/validate-env-secret-boundary.py index e3eef37e597..998e930caff 100755 --- a/agents/hermes/validate-env-secret-boundary.py +++ b/agents/hermes/validate-env-secret-boundary.py @@ -52,6 +52,13 @@ } ) RUNTIME_ALLOWED_RAW_SECRET_KEYS = frozenset({"OPENCLAW_GATEWAY_TOKEN"}) +# OpenShell's Docker/Podman supervisor owns this variable and injects a mounted +# file path, not private-key material. Keep the allowance exact and runtime-only +# so a caller cannot use the secret-shaped name to smuggle an arbitrary value or +# persist it in Hermes' mutable .env file. +RUNTIME_ALLOWED_PLATFORM_PATH_VALUES = frozenset( + {("OPENSHELL_TLS_KEY", "/etc/openshell/tls/client/tls.key")} +) ALLOWED_LITERALS = frozenset({"", "[STRIPPED_BY_MIGRATION]"}) MAX_ENV_BYTES = 4 * 1024 * 1024 MAX_ENV_LINE_BYTES = 256 * 1024 @@ -480,6 +487,8 @@ def validate_runtime_env(env: dict[str, str] | None = None) -> int: key, value ): continue + if (key, value) in RUNTIME_ALLOWED_PLATFORM_PATH_VALUES: + continue if not KEY_NAME_RE.fullmatch(key): continue if not SECRET_KEY_RE.search(key): diff --git a/docs/reference/commands-nemohermes.mdx b/docs/reference/commands-nemohermes.mdx index 3d54a051797..d81db29f620 100644 --- a/docs/reference/commands-nemohermes.mdx +++ b/docs/reference/commands-nemohermes.mdx @@ -360,9 +360,11 @@ Use `--no-gpu` to opt out when you want host-side inference providers only and d Use `--gpu` to require GPU passthrough and fail fast if an NVIDIA GPU is not detected. Use `--sandbox-gpu` or `--no-sandbox-gpu` to control only direct NVIDIA GPU access inside the sandbox. Use `--sandbox-gpu --sandbox-gpu-device ` to pass a specific OpenShell GPU device selector to `openshell sandbox create`; device selectors require explicit sandbox GPU enablement. -On Linux Docker-driver gateways, NemoClaw can create the sandbox first and then recreate the OpenShell-managed Docker container with NVIDIA GPU access when that compatibility path is needed. -When this compatibility path recreates the Docker container, NemoClaw uses an available NVIDIA CDI spec before falling back to Docker `--gpus all` or the NVIDIA runtime. -On Jetson/Tegra hosts, it also adds the host group IDs that own `/dev/nvmap` and `/dev/nvhost-*` so the sandbox user can initialize CUDA. +On ordinary native Linux Docker-driver hosts with usable CDI, NemoClaw uses OpenShell native GPU injection by default. +On Docker Desktop WSL and Jetson/Tegra, NemoClaw creates the sandbox first and then recreates the OpenShell-managed Docker container with NVIDIA GPU access by default. +When you force this compatibility path on ordinary native Linux, NemoClaw uses an available NVIDIA CDI spec before falling back to Docker `--gpus all` or the NVIDIA runtime. +On Docker Desktop WSL, the compatibility path skips CDI and tries Docker `--gpus all` before the NVIDIA runtime. +On Jetson/Tegra hosts, the compatibility path uses the NVIDIA runtime and adds the host group IDs that own `/dev/nvmap` and `/dev/nvhost-*` so the sandbox user can initialize CUDA. If the patch fails, onboarding keeps diagnostics and prints a manual cleanup command rather than deleting the failed sandbox automatically. Prerequisites: @@ -376,7 +378,12 @@ When GPU passthrough is enabled and a gateway already exists without it, onboard If no other registered sandbox depends on that gateway, or if `--recreate-sandbox` is recreating the only registered sandbox with the same name, onboarding cleans up the stale gateway and continues. If other sandboxes depend on the gateway or Docker state is unclear, onboarding exits without cleanup and prints targeted destroy or gateway-removal guidance. To add GPU to an existing sandbox, rerun with `--recreate-sandbox`. -Set `NEMOCLAW_DOCKER_GPU_PATCH=0` only when you need to bypass the Linux Docker-driver compatibility patch during troubleshooting. +Leave `NEMOCLAW_DOCKER_GPU_PATCH` unset or set it to `auto` to use the platform default. +Set `NEMOCLAW_DOCKER_GPU_PATCH=1` to force the legacy Docker container-swap path on ordinary native Linux. +Set `NEMOCLAW_DOCKER_GPU_PATCH=0` to select native OpenShell GPU injection on ordinary native Linux or Jetson/Tegra. +Use `NEMOCLAW_DOCKER_GPU_PATCH=0` on Jetson/Tegra only for troubleshooting because it bypasses Tegra device-group propagation and CUDA may not initialize. +Docker Desktop WSL ignores `NEMOCLAW_DOCKER_GPU_PATCH=0` because GPU passthrough on that runtime requires the compatibility patch. +Use `--no-sandbox-gpu`, `--no-gpu`, or `NEMOCLAW_SANDBOX_GPU=0` when you want to disable sandbox GPU passthrough on Docker Desktop WSL. ### `nemohermes list` @@ -2037,7 +2044,7 @@ Set them before running `nemohermes onboard`. | `NEMOCLAW_RAM` | percentage or Kubernetes memory quantity | Overrides the selected profile's memory size passed to OpenShell `--memory`. Percentages resolve against detected capacity. | | `NEMOCLAW_SANDBOX_GPU` | `auto`, `1`, or `0` | Controls sandbox GPU passthrough during onboarding. `auto` enables GPU passthrough when an NVIDIA GPU is detected, `1` requires GPU passthrough, and `0` forces CPU-only sandbox creation. | | `NEMOCLAW_SANDBOX_GPU_DEVICE` | OpenShell GPU device selector | Selects the GPU device passed with `openshell sandbox create --gpu-device`. Requires explicit sandbox GPU enablement with `NEMOCLAW_SANDBOX_GPU=1` (or `--sandbox-gpu` for CLI-driven onboarding); otherwise onboarding rejects the selector instead of treating it as an implicit opt-in. | -| `NEMOCLAW_DOCKER_GPU_PATCH` | `0` to disable, anything else to keep the default | Controls the Linux Docker-driver GPU sandbox compatibility patch. Set to `0` only as an escape hatch when the patch fails and you need onboarding to continue without patching the GPU sandbox container. | +| `NEMOCLAW_DOCKER_GPU_PATCH` | unset, `auto`, `1`, or `0` | Selects Linux Docker-driver GPU routing. Unset or `auto` uses native OpenShell GPU injection on ordinary native Linux and the compatibility patch on Docker Desktop WSL and Jetson/Tegra. `1` forces the compatibility patch. `0` selects native injection on ordinary native Linux and Jetson/Tegra, but Docker Desktop WSL ignores it. On Jetson/Tegra, use `0` only for troubleshooting because it bypasses the device-group propagation needed for CUDA. | | `NEMOCLAW_OPENSHELL_GATEWAY_CONTAINER_PATCH` | `1` to enable; disabled by default | Explicitly opts into the Linux gateway compatibility container for an older host ABI or a diagnostic run. This mode uses host networking and mounts the Docker socket read-only, but the socket still exposes the privileged Docker API. Use it only on a trusted local host; prefer OpenShell 0.0.71's directly supported glibc 2.28+ path. See the [OpenShell 0.0.71 gateway auth review](../security/openshell-0.0.71-gateway-auth-review#source-of-truth-boundaries). | | `NEMOCLAW_OPENSHELL_GATEWAY_BIN` | path | Advanced override for the `openshell-gateway` binary used by the Linux Docker-driver standalone fallback. Defaults to the binary next to `openshell`, then common install paths. | | `NEMOCLAW_OPENSHELL_SANDBOX_BIN` | path | Advanced override for the `openshell-sandbox` binary used by the Linux Docker-driver standalone fallback. Defaults to the binary next to `openshell`, then common install paths. | diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 926127f9078..03d1bc7ade6 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -454,9 +454,11 @@ Use `--no-gpu` to opt out when you want host-side inference providers only and d Use `--gpu` to require GPU passthrough and fail fast if an NVIDIA GPU is not detected. Use `--sandbox-gpu` or `--no-sandbox-gpu` to control only direct NVIDIA GPU access inside the sandbox. Use `--sandbox-gpu --sandbox-gpu-device ` to pass a specific OpenShell GPU device selector to `openshell sandbox create`; device selectors require explicit sandbox GPU enablement. -On Linux Docker-driver gateways, NemoClaw can create the sandbox first and then recreate the OpenShell-managed Docker container with NVIDIA GPU access when that compatibility path is needed. -When this compatibility path recreates the Docker container, NemoClaw uses an available NVIDIA CDI spec before falling back to Docker `--gpus all` or the NVIDIA runtime. -On Jetson/Tegra hosts, it also adds the host group IDs that own `/dev/nvmap` and `/dev/nvhost-*` so the sandbox user can initialize CUDA. +On ordinary native Linux Docker-driver hosts with usable CDI, NemoClaw uses OpenShell native GPU injection by default. +On Docker Desktop WSL and Jetson/Tegra, NemoClaw creates the sandbox first and then recreates the OpenShell-managed Docker container with NVIDIA GPU access by default. +When you force this compatibility path on ordinary native Linux, NemoClaw uses an available NVIDIA CDI spec before falling back to Docker `--gpus all` or the NVIDIA runtime. +On Docker Desktop WSL, the compatibility path skips CDI and tries Docker `--gpus all` before the NVIDIA runtime. +On Jetson/Tegra hosts, the compatibility path uses the NVIDIA runtime and adds the host group IDs that own `/dev/nvmap` and `/dev/nvhost-*` so the sandbox user can initialize CUDA. If the patch fails, onboarding keeps diagnostics and prints a manual cleanup command rather than deleting the failed sandbox automatically. Prerequisites: @@ -470,7 +472,12 @@ When GPU passthrough is enabled and a gateway already exists without it, onboard If no other registered sandbox depends on that gateway, or if `--recreate-sandbox` is recreating the only registered sandbox with the same name, onboarding cleans up the stale gateway and continues. If other sandboxes depend on the gateway or Docker state is unclear, onboarding exits without cleanup and prints targeted destroy or gateway-removal guidance. To add GPU to an existing sandbox, rerun with `--recreate-sandbox`. -Set `NEMOCLAW_DOCKER_GPU_PATCH=0` only when you need to bypass the Linux Docker-driver compatibility patch during troubleshooting. +Leave `NEMOCLAW_DOCKER_GPU_PATCH` unset or set it to `auto` to use the platform default. +Set `NEMOCLAW_DOCKER_GPU_PATCH=1` to force the legacy Docker container-swap path on ordinary native Linux. +Set `NEMOCLAW_DOCKER_GPU_PATCH=0` to select native OpenShell GPU injection on ordinary native Linux or Jetson/Tegra. +Use `NEMOCLAW_DOCKER_GPU_PATCH=0` on Jetson/Tegra only for troubleshooting because it bypasses Tegra device-group propagation and CUDA may not initialize. +Docker Desktop WSL ignores `NEMOCLAW_DOCKER_GPU_PATCH=0` because GPU passthrough on that runtime requires the compatibility patch. +Use `--no-sandbox-gpu`, `--no-gpu`, or `NEMOCLAW_SANDBOX_GPU=0` when you want to disable sandbox GPU passthrough on Docker Desktop WSL. ### `$$nemoclaw list` @@ -2522,7 +2529,7 @@ Set them before running `$$nemoclaw onboard`. | `NEMOCLAW_RAM` | percentage or Kubernetes memory quantity | Overrides the selected profile's memory size passed to OpenShell `--memory`. Percentages resolve against detected capacity. | | `NEMOCLAW_SANDBOX_GPU` | `auto`, `1`, or `0` | Controls sandbox GPU passthrough during onboarding. `auto` enables GPU passthrough when an NVIDIA GPU is detected, `1` requires GPU passthrough, and `0` forces CPU-only sandbox creation. | | `NEMOCLAW_SANDBOX_GPU_DEVICE` | OpenShell GPU device selector | Selects the GPU device passed with `openshell sandbox create --gpu-device`. Requires explicit sandbox GPU enablement with `NEMOCLAW_SANDBOX_GPU=1` (or `--sandbox-gpu` for CLI-driven onboarding); otherwise onboarding rejects the selector instead of treating it as an implicit opt-in. | -| `NEMOCLAW_DOCKER_GPU_PATCH` | `0` to disable, anything else to keep the default | Controls the Linux Docker-driver GPU sandbox compatibility patch. Set to `0` only as an escape hatch when the patch fails and you need onboarding to continue without patching the GPU sandbox container. | +| `NEMOCLAW_DOCKER_GPU_PATCH` | unset, `auto`, `1`, or `0` | Selects Linux Docker-driver GPU routing. Unset or `auto` uses native OpenShell GPU injection on ordinary native Linux and the compatibility patch on Docker Desktop WSL and Jetson/Tegra. `1` forces the compatibility patch. `0` selects native injection on ordinary native Linux and Jetson/Tegra, but Docker Desktop WSL ignores it. On Jetson/Tegra, use `0` only for troubleshooting because it bypasses the device-group propagation needed for CUDA. | | `NEMOCLAW_OPENSHELL_GATEWAY_CONTAINER_PATCH` | `1` to enable; disabled by default | Explicitly opts into the Linux gateway compatibility container for an older host ABI or a diagnostic run. This mode uses host networking and mounts the Docker socket read-only, but the socket still exposes the privileged Docker API. Use it only on a trusted local host; prefer OpenShell 0.0.71's directly supported glibc 2.28+ path. See the [OpenShell 0.0.71 gateway auth review](../security/openshell-0.0.71-gateway-auth-review#source-of-truth-boundaries). | | `NEMOCLAW_OPENSHELL_GATEWAY_BIN` | path | Advanced override for the `openshell-gateway` binary used by the Linux Docker-driver standalone fallback. Defaults to the binary next to `openshell`, then common install paths. | | `NEMOCLAW_OPENSHELL_SANDBOX_BIN` | path | Advanced override for the `openshell-sandbox` binary used by the Linux Docker-driver standalone fallback. Defaults to the binary next to `openshell`, then common install paths. | diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 55a28ee2d49..81e4425e073 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -1605,7 +1605,9 @@ If GPU passthrough is not required on this host, rerun onboarding with `--no-gpu ### Docker GPU patch failed during sandbox create -On Linux Docker-driver gateways, NemoClaw may create the sandbox first and then recreate the OpenShell-managed Docker container with NVIDIA GPU flags. +On ordinary native Linux Docker-driver hosts with usable CDI, NemoClaw uses OpenShell native GPU injection by default. +On Docker Desktop WSL and Jetson/Tegra, NemoClaw uses the compatibility patch by default, which creates the sandbox first and then recreates the OpenShell-managed Docker container with NVIDIA GPU flags. +Set `NEMOCLAW_DOCKER_GPU_PATCH=1` to force this compatibility path on ordinary native Linux for diagnostics or older host compatibility. If that compatibility patch fails, onboarding leaves the failed sandbox and diagnostic bundle in place so you can inspect the OpenShell and Docker state. @@ -1621,7 +1623,8 @@ openshell sandbox delete Fix the NVIDIA Container Toolkit or CDI configuration reported in the diagnostics, clean up the failed sandbox, then rerun onboarding. If you do not need GPU access inside the sandbox, rerun with `--no-sandbox-gpu`. -Set `NEMOCLAW_DOCKER_GPU_PATCH=0` only when you need to bypass this compatibility path during troubleshooting. +Set `NEMOCLAW_DOCKER_GPU_PATCH=0` to select native OpenShell GPU injection on ordinary native Linux or Jetson/Tegra. +Use this override on Jetson/Tegra only for troubleshooting because it bypasses the compatibility path's `/dev/nvmap` and `/dev/nvhost-*` group propagation, so CUDA may not initialize. On Docker Desktop WSL, the patch is required for GPU passthrough. `NEMOCLAW_DOCKER_GPU_PATCH=0` is ignored on that runtime, and onboarding logs a warning when it is set there. To skip GPU passthrough entirely on Docker Desktop WSL, rerun with `--no-gpu` or set `NEMOCLAW_SANDBOX_GPU=0`. diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index a4b9f265cc3..51fe60f5115 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -1970,10 +1970,10 @@ async function startGatewayWithOptions( if (isLinuxDockerDriverGatewayEnabled()) { return startDockerDriverGateway({ exitOnFailure, - skipSandboxBridgeReachability: - gpuPassthrough && - process.env.NEMOCLAW_DOCKER_GPU_PATCH !== "0" && - dockerGpuPatch.getDockerGpuPatchNetworkMode(process.env) === "host", + skipSandboxBridgeReachability: dockerGpuLocalInference.shouldSkipGpuBridgeProbe( + gpuPassthrough, + _gpu?.platform, + ), }); } diff --git a/src/lib/onboard/docker-gpu-diagnostic-redaction.test.ts b/src/lib/onboard/docker-gpu-diagnostic-redaction.test.ts new file mode 100644 index 00000000000..a40948f408b --- /dev/null +++ b/src/lib/onboard/docker-gpu-diagnostic-redaction.test.ts @@ -0,0 +1,190 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it, vi } from "vitest"; + +import { + buildDockerGpuMode, + collectDockerGpuPatchDiagnostics, + type DockerContainerInspect, +} from "./docker-gpu-patch"; + +describe("Docker GPU diagnostic redaction", () => { + it("redacts opaque conventional and custom-placeholder values from every shared collector sink", () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-gpu-diagnostic-redaction-")); + const canaries = { + error: "opaque-a-7f31", + modeAttempt: "opaque-b-8a42", + headline: "opaque-c-9b53", + sandboxList: "opaque-d-ac64", + containerState: "opaque-e-bd75", + dockerPs: "opaque-f-ce86", + inspect: "opaque-g-df97", + network: "opaque-h-e0a8", + dockerLogs: "opaque-i-f1b9", + openshellGet: "opaque-j-02ca", + openshellList: "opaque-k-13db", + openshellLogs: "opaque-l-24ec", + } as const; + const placeholderEntries = Object.entries(canaries).map( + ([key, value]) => [`COLLECTOR_${key.toUpperCase()}`, value] as const, + ); + const placeholderKeys = placeholderEntries.map(([key]) => key).join(","); + const startupCommand = [ + "env", + `NEMOCLAW_EXTRA_PLACEHOLDER_KEYS=${placeholderKeys}`, + ...placeholderEntries.map(([key, value]) => `${key}=${value}`), + "nemoclaw-start", + ].join(" "); + const suffixCanary = ["redaction", "sentinel"].join("-"); + const inspect: DockerContainerInspect = { + Id: "new-container-id", + Name: `/openshell-alpha-${canaries.inspect}`, + Config: { + Image: "openshell/sandbox:test", + Env: [`OPENSHELL_SANDBOX_COMMAND=${startupCommand}`, `SIGNING_KEY=${suffixCanary}`], + Labels: { + "openshell.ai/sandbox-name": "alpha", + "untrusted.label": canaries.inspect, + }, + Entrypoint: ["/opt/openshell/bin/openshell-sandbox"], + Cmd: ["hidden", canaries.inspect], + User: "1000", + }, + HostConfig: { + NetworkMode: "openshell-docker", + RestartPolicy: { Name: "unless-stopped" }, + GroupAdd: ["1000"], + }, + NetworkSettings: { + Networks: { + "openshell-docker": { + IPAddress: "172.18.0.2", + Gateway: "172.18.0.1", + Aliases: [`alpha-${canaries.network}`], + }, + }, + }, + }; + const dockerResponses = new Map([ + [ + "ps -a --filter label=openshell.ai/managed-by=openshell --filter label=openshell.ai/sandbox-name=alpha --format {{.ID}}", + "new-container-id\n", + ], + ["inspect new-container-id", JSON.stringify([inspect])], + [ + "ps -a --filter label=openshell.ai/managed-by=openshell --filter label=openshell.ai/sandbox-name=alpha", + `new-container-id running ${canaries.dockerPs}\n`, + ], + ]); + const dockerCapture = vi.fn( + (args: readonly string[]) => dockerResponses.get(args.join(" ")) ?? "", + ); + const openshellResponses = new Map([ + ["sandbox get", `Phase: Error\nuseful get context ${canaries.openshellGet}\n`], + ["sandbox list", `alpha Error useful list context ${canaries.openshellList}\n`], + ["doctor logs", `useful gateway log context ${canaries.openshellLogs}\n`], + ]); + const runCaptureOpenshell = vi.fn( + (args: string[]) => openshellResponses.get(`${args[0] ?? ""} ${args[1] ?? ""}`) ?? "", + ); + const writeFileSpy = vi.spyOn(fs, "writeFileSync"); + + try { + const mode = buildDockerGpuMode("gpus"); + const diagnostics = collectDockerGpuPatchDiagnostics( + "alpha", + { + error: new Error(`useful failure context ${canaries.error}`), + context: { + sandboxName: "alpha", + newContainerId: "new-container-id", + selectedMode: mode, + modeAttempts: [ + { mode, ok: false, error: `useful mode context ${canaries.modeAttempt}` }, + ], + }, + selectedMode: mode, + snapshot: { + sandboxPhase: "Error", + sandboxListLine: `alpha Error useful snapshot context ${canaries.sandboxList}`, + patchedContainerState: { + Status: "exited", + ExitCode: 125, + Error: `useful state context ${canaries.containerState}`, + }, + }, + classification: { + kind: "patched_container_failed", + headline: `useful headline context ${canaries.headline}`, + summaryLines: [], + }, + }, + { + dockerCapture, + dockerLogs: vi.fn( + () => `useful docker log context ${canaries.dockerLogs} ${suffixCanary}\n`, + ), + homedir: () => tmpDir, + now: () => new Date("2026-07-02T00:00:00Z"), + runCaptureOpenshell, + }, + ); + + expect(diagnostics?.dir).toBeTruthy(); + const expectedFiles = [ + "summary.txt", + "patched-container-state.json", + "docker-ps.txt", + "docker-inspect.json", + "docker-network-summary.txt", + "docker-logs.txt", + "openshell-sandbox-get.txt", + "openshell-sandbox-list.txt", + "openshell-logs.txt", + ]; + const contents = Object.fromEntries( + expectedFiles.map((name) => [ + name, + fs.readFileSync(path.join(diagnostics?.dir ?? "", name), "utf8"), + ]), + ); + const published = `${diagnostics?.summaryLines.join("\n")}\n${Object.values(contents).join("\n")}`; + for (const canary of Object.values(canaries)) expect(published).not.toContain(canary); + expect(published).not.toContain(suffixCanary); + + expect(contents["summary.txt"]).toContain("failure_kind=patched_container_failed"); + expect(contents["summary.txt"]).toContain("useful failure context "); + expect(contents["docker-logs.txt"]).toContain( + "useful docker log context ", + ); + expect(contents["openshell-sandbox-get.txt"]).toContain("useful get context "); + expect(contents["docker-network-summary.txt"]).toContain("network_mode=openshell-docker"); + const state = JSON.parse(contents["patched-container-state.json"]); + expect(state.Error).toBe("useful state context "); + const inspected = JSON.parse(contents["docker-inspect.json"]); + expect(inspected[0].Config.Env).toEqual([ + "OPENSHELL_SANDBOX_COMMAND=", + "SIGNING_KEY=", + ]); + expect(inspected[0].Config.Labels).toEqual({ "openshell.ai/sandbox-name": "alpha" }); + expect(inspected[0].Config.Cmd).toEqual(["hidden", "<1 additional arguments omitted>"]); + + const fullInspectOrders = dockerCapture.mock.calls + .map(([args], index) => ({ args, order: dockerCapture.mock.invocationCallOrder[index] })) + .filter(({ args }) => args[0] === "inspect" && args[1] !== "--format") + .map(({ order }) => order ?? 0); + expect(fullInspectOrders.length).toBeGreaterThan(0); + expect(Math.max(...fullInspectOrders)).toBeLessThan( + writeFileSpy.mock.invocationCallOrder[0] ?? Number.MAX_SAFE_INTEGER, + ); + } finally { + writeFileSpy.mockRestore(); + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); +}); diff --git a/src/lib/onboard/docker-gpu-diagnostic-redaction.ts b/src/lib/onboard/docker-gpu-diagnostic-redaction.ts new file mode 100644 index 00000000000..16e06f87ecb --- /dev/null +++ b/src/lib/onboard/docker-gpu-diagnostic-redaction.ts @@ -0,0 +1,148 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { redactFull } from "../security/redact"; +import type { DockerContainerInspect } from "./docker-gpu-patch"; + +const SENSITIVE_ENV_KEY = + /(?:api_?key|private_?key|(?:^|_)key$|token|secret|password|credential|authorization|cookie|proxy)/i; +const EXTRA_PLACEHOLDER_KEYS_ENV = "NEMOCLAW_EXTRA_PLACEHOLDER_KEYS"; + +function inspectEnv(inspect: DockerContainerInspect): Map { + const env = new Map(); + for (const assignment of inspect.Config?.Env ?? []) { + const separator = assignment.indexOf("="); + if (separator <= 0) continue; + env.set(assignment.slice(0, separator), assignment.slice(separator + 1)); + } + return env; +} + +function startupCommandEnv(inspect: DockerContainerInspect): Map { + const assignments = new Map(); + const command = inspectEnv(inspect).get("OPENSHELL_SANDBOX_COMMAND") ?? ""; + const tokens = command.trim().split(/\s+/u); + if (tokens.shift() !== "env") return assignments; + for (const token of tokens) { + const separator = token.indexOf("="); + if (separator <= 0) break; + assignments.set(token.slice(0, separator), token.slice(separator + 1)); + } + return assignments; +} + +function summarizeArgv( + value: string[] | string | null | undefined, + redactText: (text: string) => string, +): string[] | string | null | undefined { + if (!Array.isArray(value)) { + return typeof value === "string" ? redactText(value) : value; + } + if (value.length <= 1) return value.map(redactText); + return [redactText(value[0] ?? ""), `<${String(value.length - 1)} additional arguments omitted>`]; +} + +export type DockerGpuDiagnosticRedactor = { + rememberInspect(inspect: DockerContainerInspect): void; + redactText(text: string): string; + redactValue(value: unknown): unknown; + sanitizeInspect(inspect: DockerContainerInspect): DockerContainerInspect; +}; + +export function discoverDockerGpuDiagnosticSensitiveValues( + inspect: DockerContainerInspect, +): string[] { + const env = inspectEnv(inspect); + const startupEnv = startupCommandEnv(inspect); + const extraPlaceholderKeys = new Set( + (startupEnv.get(EXTRA_PLACEHOLDER_KEYS_ENV) ?? env.get(EXTRA_PLACEHOLDER_KEYS_ENV) ?? "") + .split(/[\s,]+/u) + .map((value) => value.trim()) + .filter(Boolean), + ); + return [...env, ...startupEnv] + .filter( + ([key, value]) => + (SENSITIVE_ENV_KEY.test(key) || extraPlaceholderKeys.has(key)) && value.length > 0, + ) + .map(([, value]) => value); +} + +/** + * Owns the redaction state for one diagnostic bundle. Full Docker inspect + * records are observed before any artifact is written so conventionally + * sensitive and custom-placeholder values are removed from every sink. + */ +export function createDockerGpuDiagnosticRedactor( + initialSensitiveValues: Iterable = [], +): DockerGpuDiagnosticRedactor { + const sensitiveValues = new Set([...initialSensitiveValues].filter((value) => value.length > 0)); + const redactText = (text: string): string => { + let redacted = redactFull(text); + for (const value of [...sensitiveValues].sort((left, right) => right.length - left.length)) { + redacted = redacted.split(value).join(""); + } + return redacted; + }; + const rememberInspect = (inspect: DockerContainerInspect): void => { + for (const value of discoverDockerGpuDiagnosticSensitiveValues(inspect)) { + sensitiveValues.add(value); + } + }; + const redactValue = (value: unknown, seen: WeakSet = new WeakSet()): unknown => { + if (typeof value === "string") return redactText(value); + if (value === null || typeof value !== "object") return value; + if (seen.has(value)) return "[Circular]"; + seen.add(value); + if (Array.isArray(value)) return value.map((entry) => redactValue(entry, seen)); + return Object.fromEntries( + Object.entries(value).map(([key, entry]) => [redactText(key), redactValue(entry, seen)]), + ); + }; + const sanitizeInspect = (inspect: DockerContainerInspect): DockerContainerInspect => { + const envKeys = [...inspectEnv(inspect).keys()].sort(); + const labels = Object.fromEntries( + Object.entries(inspect.Config?.Labels ?? {}) + .filter(([key]) => key.startsWith("openshell.ai/")) + .map(([key, value]) => [redactText(key), redactText(value)]), + ); + const networks = Object.fromEntries( + Object.entries(inspect.NetworkSettings?.Networks ?? {}).map(([name, network]) => [ + redactText(name), + { + IPAddress: network.IPAddress ? redactText(network.IPAddress) : network.IPAddress, + Gateway: network.Gateway ? redactText(network.Gateway) : network.Gateway, + Aliases: network.Aliases?.map(redactText), + }, + ]), + ); + return { + Id: inspect.Id ? redactText(inspect.Id) : inspect.Id, + Name: inspect.Name ? redactText(inspect.Name) : inspect.Name, + Config: { + Image: inspect.Config?.Image ? redactText(inspect.Config.Image) : inspect.Config?.Image, + User: inspect.Config?.User ? redactText(inspect.Config.User) : inspect.Config?.User, + Entrypoint: summarizeArgv(inspect.Config?.Entrypoint, redactText), + Cmd: summarizeArgv(inspect.Config?.Cmd, redactText), + Env: envKeys.map((key) => `${redactText(key)}=`), + Labels: labels, + }, + HostConfig: { + NetworkMode: inspect.HostConfig?.NetworkMode + ? redactText(inspect.HostConfig.NetworkMode) + : inspect.HostConfig?.NetworkMode, + RestartPolicy: inspect.HostConfig?.RestartPolicy + ? { + ...inspect.HostConfig.RestartPolicy, + Name: inspect.HostConfig.RestartPolicy.Name + ? redactText(inspect.HostConfig.RestartPolicy.Name) + : inspect.HostConfig.RestartPolicy.Name, + } + : inspect.HostConfig?.RestartPolicy, + GroupAdd: inspect.HostConfig?.GroupAdd?.map(redactText), + }, + NetworkSettings: { Networks: networks }, + }; + }; + return { rememberInspect, redactText, redactValue, sanitizeInspect }; +} diff --git a/src/lib/onboard/docker-gpu-local-inference.test.ts b/src/lib/onboard/docker-gpu-local-inference.test.ts index 59ee505a1e8..f3959d5bf5e 100644 --- a/src/lib/onboard/docker-gpu-local-inference.test.ts +++ b/src/lib/onboard/docker-gpu-local-inference.test.ts @@ -7,12 +7,17 @@ import { enforceDockerGpuPatchPreserveNetwork, getSandboxRuntimeInferenceEndpoint, printDockerGpuSandboxInferenceVerificationFailure, + shouldSkipGpuBridgeProbe, shouldUseDockerGpuPatchHostNetwork, verifyDockerGpuSandboxLocalInference, verifyGpuSandboxAfterReady, } from "./docker-gpu-local-inference"; -const HOST_NETWORK_ENV = { NEMOCLAW_DOCKER_GPU_PATCH_NETWORK: "host" } as NodeJS.ProcessEnv; +const HOST_NETWORK_ENV = { + NEMOCLAW_DOCKER_GPU_PATCH: "1", + NEMOCLAW_DOCKER_GPU_PATCH_NETWORK: "host", +} as NodeJS.ProcessEnv; +const LEGACY_PATCH_ENV = { NEMOCLAW_DOCKER_GPU_PATCH: "1" } as NodeJS.ProcessEnv; const GPU_CONFIG = { sandboxGpuEnabled: true }; function gpuPatchOptions(extra: Record = {}) { @@ -20,7 +25,7 @@ function gpuPatchOptions(extra: Record = {}) { sandboxName: "alpha", dockerDriverGateway: true, platform: "linux" as NodeJS.Platform, - env: {} as NodeJS.ProcessEnv, + env: { ...LEGACY_PATCH_ENV }, ...extra, }; } @@ -60,6 +65,33 @@ describe("shouldUseDockerGpuPatchHostNetwork", () => { }); }); +describe("shouldSkipGpuBridgeProbe", () => { + it("forces the gateway context and skips only for an active legacy host-network patch", () => { + expect( + shouldSkipGpuBridgeProbe(true, "linux", { + dockerDriverGateway: false, + dockerDesktopWsl: false, + env: HOST_NETWORK_ENV, + platform: "linux", + }), + ).toBe(true); + expect( + shouldSkipGpuBridgeProbe(true, "linux", { + dockerDesktopWsl: false, + env: { NEMOCLAW_DOCKER_GPU_PATCH_NETWORK: "host" }, + platform: "linux", + }), + ).toBe(false); + expect( + shouldSkipGpuBridgeProbe(false, "linux", { + dockerDesktopWsl: false, + env: HOST_NETWORK_ENV, + platform: "linux", + }), + ).toBe(false); + }); +}); + describe("enforceDockerGpuPatchPreserveNetwork", () => { it("downgrades a LOCAL provider to preserve and re-checks the bridge (#4509)", async () => { const env = { ...HOST_NETWORK_ENV }; @@ -267,7 +299,7 @@ describe("verifyGpuSandboxAfterReady", () => { sandboxName: "alpha", dockerDriverGateway: true, platform: "linux" as NodeJS.Platform, - env: {} as NodeJS.ProcessEnv, + env: { ...LEGACY_PATCH_ENV }, useDockerGpuPatch: true, verifyDirectSandboxGpu: vi.fn(), selectedMode: () => null, diff --git a/src/lib/onboard/docker-gpu-local-inference.ts b/src/lib/onboard/docker-gpu-local-inference.ts index 3d6baf7cdb3..bbe7934a0ea 100644 --- a/src/lib/onboard/docker-gpu-local-inference.ts +++ b/src/lib/onboard/docker-gpu-local-inference.ts @@ -32,6 +32,7 @@ const SANDBOX_RUNTIME_INFERENCE_ENDPOINT = "https://inference.local/v1/models"; type DockerGpuLocalInferenceConfig = { sandboxGpuEnabled: boolean; sandboxGpuDevice?: string | null; + hostGpuPlatform?: string | null; // Written back by `verifyGpuSandboxAfterReady` with the CUDA-usability proof // result so the registry/`status` can distinguish a configured GPU from a // proven-usable one (#4231). @@ -54,6 +55,20 @@ function isLocalInferenceProvider(provider: string | null | undefined): provider return Boolean(provider && LOCAL_INFERENCE_PROVIDERS.includes(provider)); } +export function shouldSkipGpuBridgeProbe( + gpuPassthrough: boolean, + hostGpuPlatform?: string | null, + options: Partial = {}, +): boolean { + return ( + gpuPassthrough && + shouldUseDockerGpuPatchHostNetwork( + { sandboxGpuEnabled: true, hostGpuPlatform }, + { ...options, dockerDriverGateway: true }, + ) + ); +} + /** * True on the Linux Docker-driver GPU patch path with * `NEMOCLAW_DOCKER_GPU_PATCH_NETWORK=host`, i.e. when the recreated sandbox was diff --git a/src/lib/onboard/docker-gpu-patch-validation.test.ts b/src/lib/onboard/docker-gpu-patch-validation.test.ts new file mode 100644 index 00000000000..01ccd34ddf4 --- /dev/null +++ b/src/lib/onboard/docker-gpu-patch-validation.test.ts @@ -0,0 +1,134 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; +import { + buildDockerGpuCloneRunArgs, + buildDockerGpuMode, + type DockerContainerInspect, + recreateOpenShellDockerSandboxWithGpu, +} from "./docker-gpu-patch"; +import { + appendExtraPlaceholderKeysEnvArg, + EXTRA_PLACEHOLDER_KEYS_ENV, + parseExtraPlaceholderKeys, +} from "./extra-placeholder-keys"; + +function inspectFixture(): DockerContainerInspect { + return { + Id: "old-container-id", + Name: "/openshell-alpha", + Config: { + Image: "openshell/sandbox:abc", + Env: [ + "OPENSHELL_ENDPOINT=http://host.openshell.internal:8080/", + "OPENSHELL_SANDBOX_COMMAND=sleep infinity", + ], + Labels: { + "openshell.ai/managed-by": "openshell", + "openshell.ai/sandbox-name": "alpha", + }, + Entrypoint: ["/opt/openshell/bin/openshell-sandbox"], + Cmd: [], + User: "0", + }, + HostConfig: { + NetworkMode: "openshell-docker", + RestartPolicy: { Name: "unless-stopped" }, + }, + NetworkSettings: { Networks: { "openshell-docker": {} } }, + }; +} + +describe("Docker GPU startup command validation (#6110)", () => { + it.each([ + ["Docker --gpus", buildDockerGpuMode("gpus")], + ["native CDI", buildDockerGpuMode("cdi")], + ["Jetson runtime", buildDockerGpuMode("nvidia-runtime", null, { backend: "jetson" })], + ])("preserves the OpenShell supervisor boundary for %s", (_label, mode) => { + const extraPlaceholderKeys = ["TELEGRAM_BOT_TOKEN_AGENT_A", "SLACK_BOT_TOKEN_AGENT_B"]; + const extraPlaceholderEnv: string[] = []; + appendExtraPlaceholderKeysEnvArg( + extraPlaceholderEnv, + extraPlaceholderKeys, + (key, value) => `${key}=${value}`, + ); + const sandboxCommand = [ + "env", + "CHAT_UI_URL=http://127.0.0.1:8642", + "NEMOCLAW_DASHBOARD_PORT=8642", + "HTTP_PROXY=http://proxy.example:8080", + ...extraPlaceholderEnv, + "nemoclaw-start", + ]; + + const args = buildDockerGpuCloneRunArgs(inspectFixture(), mode, { + openshellSandboxCommand: sandboxCommand, + }); + + expect(args).toEqual( + expect.arrayContaining(["--env", `OPENSHELL_SANDBOX_COMMAND=${sandboxCommand.join(" ")}`]), + ); + expect(args).not.toEqual( + expect.arrayContaining(["--env", "OPENSHELL_SANDBOX_COMMAND=sleep infinity"]), + ); + expect(args).toEqual(expect.arrayContaining(mode.args)); + expect(args.slice(args.indexOf("openshell/sandbox:abc"))).toEqual(["openshell/sandbox:abc"]); + + const serializedCommand = args.find((arg) => arg.startsWith("OPENSHELL_SANDBOX_COMMAND=")); + const commandTokens = serializedCommand + ?.slice("OPENSHELL_SANDBOX_COMMAND=".length) + .split(/[\s\u0085]+/u); + const assignment = commandTokens?.find((token) => + token.startsWith(`${EXTRA_PLACEHOLDER_KEYS_ENV}=`), + ); + expect(assignment).toBe(extraPlaceholderEnv[0]); + expect( + parseExtraPlaceholderKeys( + assignment?.slice(EXTRA_PLACEHOLDER_KEYS_ENV.length + 1), + new Set(["TELEGRAM_BOT_TOKEN", "SLACK_BOT_TOKEN"]), + ), + ).toEqual({ keys: extraPlaceholderKeys, warnings: [] }); + }); + + it.each([ + ["an empty token", ""], + ["ASCII whitespace", "HTTP_PROXY=http://proxy.example/path with space"], + ["U+0085 NEXT LINE", "HTTP_PROXY=http://proxy.example/path\u0085next-line"], + ])("rejects %s before touching the original container", (_label, invalidToken) => { + const dockerCapture = vi.fn((args: readonly string[]) => + args[0] === "ps" + ? "old-container-id\n" + : args[0] === "inspect" + ? JSON.stringify([inspectFixture()]) + : "", + ); + const dockerStop = vi.fn(() => ({ status: 0 })); + const dockerRename = vi.fn(() => ({ status: 0 })); + const dockerRunDetached = vi.fn(() => ({ status: 0, stdout: "new-container-id\n" })); + + expect(() => + recreateOpenShellDockerSandboxWithGpu( + { + sandboxName: "alpha", + timeoutSecs: 1, + openshellSandboxCommand: ["env", invalidToken, "nemoclaw-start"], + }, + { + dockerCapture, + detectSandboxFallbackDns: vi.fn(() => null), + dockerRun: vi.fn(() => ({ status: 0, stdout: "probe-id\n" })), + dockerRunDetached, + dockerRename, + dockerRm: vi.fn(() => ({ status: 0 })), + dockerStop, + readDir: vi.fn(() => null), + readFile: vi.fn(() => null), + }, + ), + ).toThrow("OpenShell sandbox startup command tokens cannot be empty or contain whitespace"); + expect(dockerStop).not.toHaveBeenCalled(); + expect(dockerRename).not.toHaveBeenCalled(); + expect(dockerRunDetached).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/onboard/docker-gpu-patch.test.ts b/src/lib/onboard/docker-gpu-patch.test.ts index 2c67062fbbe..7edb7c25e91 100644 --- a/src/lib/onboard/docker-gpu-patch.test.ts +++ b/src/lib/onboard/docker-gpu-patch.test.ts @@ -77,12 +77,28 @@ function inspectFixture(): DockerContainerInspect { } describe("docker-gpu-patch", () => { - it("detects only the Linux Docker-driver GPU path and honors the opt-out", () => { + it("routes native CDI Linux directly unless the legacy patch is forced", () => { expect( shouldApplyDockerGpuPatch( { sandboxGpuEnabled: true }, { env: {}, platform: "linux", dockerDriverGateway: true }, ), + ).toBe(false); + expect( + shouldApplyDockerGpuPatch( + { sandboxGpuEnabled: true }, + { + env: { NEMOCLAW_DOCKER_GPU_PATCH: "auto" }, + platform: "linux", + dockerDriverGateway: true, + }, + ), + ).toBe(false); + expect( + shouldApplyDockerGpuPatch( + { sandboxGpuEnabled: true }, + { env: { NEMOCLAW_DOCKER_GPU_PATCH: "1" }, platform: "linux", dockerDriverGateway: true }, + ), ).toBe(true); expect( shouldApplyDockerGpuPatch( @@ -104,6 +120,21 @@ describe("docker-gpu-patch", () => { ).toBe(false); }); + it("keeps Jetson on the compatibility patch by default while honoring its opt-out", () => { + expect( + shouldApplyDockerGpuPatch( + { sandboxGpuEnabled: true, hostGpuPlatform: "jetson" }, + { env: {}, platform: "linux", dockerDriverGateway: true }, + ), + ).toBe(true); + expect( + shouldApplyDockerGpuPatch( + { sandboxGpuEnabled: true, hostGpuPlatform: "jetson" }, + { env: { NEMOCLAW_DOCKER_GPU_PATCH: "0" }, platform: "linux", dockerDriverGateway: true }, + ), + ).toBe(false); + }); + it("builds clone args that preserve OpenShell labels and runtime settings", () => { const args = buildDockerGpuCloneRunArgs(inspectFixture(), buildDockerGpuMode("gpus")); @@ -149,33 +180,6 @@ describe("docker-gpu-patch", () => { expect(args).not.toEqual(expect.arrayContaining(["--env", "NVIDIA_VISIBLE_DEVICES=void"])); }); - it("replaces OpenShell's idle sandbox command when recreating a managed container", () => { - const sandboxCommand = [ - "env", - "CHAT_UI_URL=http://127.0.0.1:8642", - "NEMOCLAW_DASHBOARD_PORT=8642", - "nemoclaw-start", - ]; - - const args = buildDockerGpuCloneRunArgs(inspectFixture(), buildDockerGpuMode("gpus"), { - openshellSandboxCommand: sandboxCommand, - }); - - expect(args).toEqual( - expect.arrayContaining([ - "--env", - "OPENSHELL_SANDBOX_COMMAND=env CHAT_UI_URL=http://127.0.0.1:8642 NEMOCLAW_DASHBOARD_PORT=8642 nemoclaw-start", - ]), - ); - expect(args).not.toEqual( - expect.arrayContaining(["--env", "OPENSHELL_SANDBOX_COMMAND=sleep infinity"]), - ); - expect(args.slice(args.indexOf("openshell/sandbox:abc"))).toEqual([ - "openshell/sandbox:abc", - ...sandboxCommand, - ]); - }); - it("adds OpenShell's sandbox command env when the inspected container lacks one", () => { const inspect = inspectFixture(); inspect.Config!.Env = inspect.Config!.Env!.filter( @@ -590,7 +594,12 @@ describe("docker-gpu-patch", () => { if (args[0] === "info") return ""; return ""; }); - const dockerRunDetached = vi.fn(() => ({ status: 0, stdout: "new-container-id\n" })); + const dockerRunDetached = vi.fn( + (_args: readonly string[], _opts?: Record) => ({ + status: 0, + stdout: "new-container-id\n", + }), + ); const dockerRm = vi.fn((_name: string) => ({ status: 0 })); const runOpenshell = vi.fn(() => ({ status: 1, stderr: "phase: Provisioning" })); @@ -625,15 +634,19 @@ describe("docker-gpu-patch", () => { expect( dockerRm.mock.calls.some((call) => String(call[0]).includes("nemoclaw-gpu-backup")), ).toBe(false); - expect(dockerRunDetached).toHaveBeenCalledWith( + const cloneArgs = dockerRunDetached.mock.calls[0]?.[0] ?? []; + expect(cloneArgs).toEqual( expect.arrayContaining([ "--env", "OPENSHELL_SANDBOX_COMMAND=env CHAT_UI_URL=http://127.0.0.1:8642 nemoclaw-start", "openshell/sandbox:abc", - "env", - "CHAT_UI_URL=http://127.0.0.1:8642", - "nemoclaw-start", ]), + ); + expect(cloneArgs.slice(cloneArgs.indexOf("openshell/sandbox:abc"))).toEqual([ + "openshell/sandbox:abc", + ]); + expect(dockerRunDetached).toHaveBeenCalledWith( + cloneArgs, expect.objectContaining({ ignoreError: true }), ); }); diff --git a/src/lib/onboard/docker-gpu-patch.ts b/src/lib/onboard/docker-gpu-patch.ts index 1ba7dfdfe0a..d912b919162 100644 --- a/src/lib/onboard/docker-gpu-patch.ts +++ b/src/lib/onboard/docker-gpu-patch.ts @@ -14,6 +14,7 @@ import { dockerRunDetached, dockerStop, } from "../adapters/docker"; +import { createDockerGpuDiagnosticRedactor } from "./docker-gpu-diagnostic-redaction"; import { reconcileSupervisorReconnect, rollbackDockerGpuPatchOnRecreateFailure, @@ -412,8 +413,14 @@ function replaceEnvValue(entry: string, key: string, value: string | null | unde function openshellSandboxCommandEnvValue( command: readonly string[] | null | undefined, ): string | null { - const parts = (command || []).map((part) => String(part)).filter((part) => part.length > 0); - return parts.length > 0 ? parts.join(" ") : null; + const parts = (command || []).map((part) => String(part)); + if (parts.length === 0) return null; + if (parts.some((part) => part.length === 0 || /[\s\u0085]/u.test(part))) { + throw new Error( + "OpenShell sandbox startup command tokens cannot be empty or contain whitespace.", + ); + } + return parts.join(" "); } function dockerGpuHostEndpointFromOpenShellEndpoint(endpoint: string): string | null { @@ -532,7 +539,7 @@ export function buildDockerGpuModeCandidates( } export function shouldApplyDockerGpuPatch( - config: { sandboxGpuEnabled: boolean }, + config: { sandboxGpuEnabled: boolean; hostGpuPlatform?: string | null }, options: { env?: NodeJS.ProcessEnv; platform?: NodeJS.Platform; @@ -551,7 +558,10 @@ export function shouldApplyDockerGpuPatch( ) { return false; } - const optedOut = String(env.NEMOCLAW_DOCKER_GPU_PATCH || "").trim() === "0"; + const control = String(env.NEMOCLAW_DOCKER_GPU_PATCH || "") + .trim() + .toLowerCase(); + const optedOut = control === "0"; if (optedOut && dockerDesktopWsl) { const log = options.log ?? ((message: string) => console.warn(message)); log( @@ -560,7 +570,12 @@ export function shouldApplyDockerGpuPatch( log(" Skip GPU passthrough entirely with --no-gpu or NEMOCLAW_SANDBOX_GPU=0."); return true; } - return !optedOut; + if (dockerDesktopWsl) return true; + if (config.hostGpuPlatform === "jetson") return !optedOut; + // OpenShell 0.0.71 natively injects CDI devices for ordinary native Linux. + // Keep the container-swap path as an explicit compatibility control while + // WSL and Jetson retain the legacy defaults they still require. + return control === "1"; } export function buildDockerGpuCloneRunOptions( @@ -772,10 +787,16 @@ export function buildDockerGpuCloneRunArgs( const entrypoint = stringArray(config.Entrypoint); if (entrypoint.length > 0) args.push("--entrypoint", entrypoint[0]); - const commandArgs = - options.openshellSandboxCommand && options.openshellSandboxCommand.length > 0 - ? [...options.openshellSandboxCommand] - : [...entrypoint.slice(1), ...stringArray(config.Cmd)]; + // OpenShell 0.0.71's Docker driver deliberately clears Config.Cmd and passes + // the workload only through OPENSHELL_SANDBOX_COMMAND. Keep that contract + // when recreating the container: appending the same workload after the image + // puts `nemoclaw-start` in the supervisor's own argv. The serializer above + // rejects whitespace-bearing tokens because OpenShell uses split_whitespace() + // when reading this environment value (#6110). Remove this compatibility + // rewrite only after the Docker container-swap GPU patch itself is retired. + const commandArgs = openshellSandboxCommandEnv + ? [] + : [...entrypoint.slice(1), ...stringArray(config.Cmd)]; args.push(image, ...commandArgs); return args; } @@ -1081,22 +1102,6 @@ export function recreateOpenShellDockerSandboxWithGpu( const backupContainerName = buildBackupContainerName(originalName, d.now()); context.backupContainerName = backupContainerName; - d.dockerStop(oldContainerId, { - ignoreError: true, - suppressOutput: true, - timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, - }); - const renameResult = d.dockerRename(oldContainerId, backupContainerName, { - ignoreError: true, - suppressOutput: true, - timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, - }); - if (!isZeroStatus(renameResult)) { - throw new Error( - `Could not move original sandbox container aside: ${resultText(renameResult)}`, - ); - } - const cloneOptions = buildDockerGpuCloneRunOptions(inspect); cloneOptions.openshellSandboxCommand = options.openshellSandboxCommand ?? null; const sandboxFallbackDns = d.detectSandboxFallbackDns(); @@ -1121,7 +1126,27 @@ export function recreateOpenShellDockerSandboxWithGpu( ); } } + // Validate and build the replacement command before touching the original + // container. A malformed startup envelope must fail without stopping or + // renaming the user's working sandbox. const cloneArgs = buildDockerGpuCloneRunArgs(inspect, selection.mode, cloneOptions); + + d.dockerStop(oldContainerId, { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }); + const renameResult = d.dockerRename(oldContainerId, backupContainerName, { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }); + if (!isZeroStatus(renameResult)) { + throw new Error( + `Could not move original sandbox container aside: ${resultText(renameResult)}`, + ); + } + const runResult = d.dockerRunDetached(cloneArgs, { ignoreError: true, suppressOutput: true, @@ -1305,7 +1330,10 @@ export function printDockerGpuPatchFailureAndExit( } console.error(" Escape hatches:"); console.error( - " NEMOCLAW_DOCKER_GPU_PATCH=0 skip this Docker GPU patch (Linux native Docker only; ignored on Docker Desktop WSL where the patch is required).", + " NEMOCLAW_DOCKER_GPU_PATCH=1 force the legacy Docker GPU container-swap path.", + ); + console.error( + " NEMOCLAW_DOCKER_GPU_PATCH=0 use native OpenShell GPU injection (ignored on Docker Desktop WSL; Jetson also defaults to the compatibility path).", ); console.error( " NEMOCLAW_SANDBOX_GPU=0 skip GPU passthrough entirely (or rerun with --no-gpu).", @@ -1679,6 +1707,8 @@ export function collectDockerGpuPatchDiagnostics( selectedMode?: DockerGpuPatchMode | null; snapshot?: DockerGpuPatchSandboxSnapshot | null; classification?: DockerGpuPatchFailureClassification | null; + additionalSensitiveValues?: readonly string[]; + dockerTopOutput?: string | null; } = {}, deps: DockerGpuPatchDeps = {}, ): DockerGpuPatchDiagnostics | null { @@ -1697,24 +1727,61 @@ export function collectDockerGpuPatchDiagnostics( } const context = options.context || getDockerGpuPatchFailureContext(options.error) || null; - const cleanupCommands = dockerGpuPatchCleanupCommands(sandboxName); - const errorText = + const redactor = createDockerGpuDiagnosticRedactor(options.additionalSensitiveValues); + let discoveredContainerIds: string[] = []; + try { + discoveredContainerIds = findOpenShellDockerSandboxContainerIds(sandboxName, deps); + } catch { + discoveredContainerIds = []; + } + const containerTargets = uniqueStrings([ + ...(context + ? [context.oldContainerId, context.newContainerId, context.backupContainerName] + : []), + ...discoveredContainerIds, + ]); + const inspectedTargets: Array<{ target: string; entries: DockerContainerInspect[] }> = []; + for (const target of containerTargets) { + try { + const inspect = d.dockerCapture(["inspect", target], { + ignoreError: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }); + if (!inspect.trim()) continue; + const parsed = JSON.parse(inspect); + const entries = (Array.isArray(parsed) ? parsed : [parsed]) as DockerContainerInspect[]; + for (const entry of entries) redactor.rememberInspect(entry); + inspectedTargets.push({ target, entries }); + } catch { + /* best effort */ + } + } + const writeDiagnosticText = (name: string, content: string): void => { + writeTextFile(dir, name, redactor.redactText(content)); + }; + const writeDiagnosticJson = (name: string, value: unknown): void => { + writeTextFile(dir, name, JSON.stringify(redactor.redactValue(value), null, 2)); + }; + + const cleanupCommands = dockerGpuPatchCleanupCommands(sandboxName).map(redactor.redactText); + const errorText = redactor.redactText( options.error instanceof Error ? options.error.message : options.error ? String(options.error) - : "none"; + : "none", + ); const selectedMode = options.selectedMode || context?.selectedMode || null; const snapshot = options.snapshot ?? null; const classification = options.classification ?? null; const summaryLines = [ `created_at=${now.toISOString()}`, - `sandbox_name=${sandboxName}`, + `sandbox_name=${redactor.redactText(sandboxName)}`, `error=${errorText}`, - `selected_gpu_mode=${selectedMode?.label ?? "none"}`, - `old_container_id=${context?.oldContainerId ?? "unknown"}`, - `new_container_id=${context?.newContainerId ?? "unknown"}`, - `backup_container_name=${context?.backupContainerName ?? "none"}`, + `selected_gpu_mode=${redactor.redactText(selectedMode?.label ?? "none")}`, + `old_container_id=${redactor.redactText(context?.oldContainerId ?? "unknown")}`, + `new_container_id=${redactor.redactText(context?.newContainerId ?? "unknown")}`, + `backup_container_name=${redactor.redactText(context?.backupContainerName ?? "none")}`, `rolled_back=${context?.rolledBack === true ? "yes" : context?.rolledBack === false ? "failed" : "no"}`, "cleanup_commands:", ...cleanupCommands.map((command) => ` ${command}`), @@ -1723,26 +1790,35 @@ export function collectDockerGpuPatchDiagnostics( summaryLines.push("gpu_mode_attempts:"); for (const attempt of context.modeAttempts) { summaryLines.push( - ` ${attempt.mode.label}: ${attempt.ok ? "ok" : "failed"}${attempt.error ? `: ${attempt.error}` : ""}`, + redactor.redactText( + ` ${attempt.mode.label}: ${attempt.ok ? "ok" : "failed"}${attempt.error ? `: ${attempt.error}` : ""}`, + ), ); } } if (classification) { - summaryLines.push(`failure_kind=${classification.kind}`); - if (classification.headline) summaryLines.push(`failure_headline=${classification.headline}`); + summaryLines.push(`failure_kind=${redactor.redactText(classification.kind)}`); + if (classification.headline) { + summaryLines.push(`failure_headline=${redactor.redactText(classification.headline)}`); + } } if (snapshot) { - if (snapshot.sandboxPhase) summaryLines.push(`sandbox_phase=${snapshot.sandboxPhase}`); - if (snapshot.sandboxListLine) summaryLines.push(`sandbox_list_row=${snapshot.sandboxListLine}`); - summaryLines.push(...describePatchedContainerState(snapshot.patchedContainerState)); + if (snapshot.sandboxPhase) { + summaryLines.push(`sandbox_phase=${redactor.redactText(snapshot.sandboxPhase)}`); + } + if (snapshot.sandboxListLine) { + summaryLines.push(`sandbox_list_row=${redactor.redactText(snapshot.sandboxListLine)}`); + } + summaryLines.push( + ...describePatchedContainerState(snapshot.patchedContainerState).map(redactor.redactText), + ); } - writeTextFile(dir, "summary.txt", summaryLines.join("\n")); + writeDiagnosticText("summary.txt", summaryLines.join("\n")); if (snapshot?.patchedContainerState) { - writeTextFile( - dir, - "patched-container-state.json", - JSON.stringify(snapshot.patchedContainerState, null, 2), - ); + writeDiagnosticJson("patched-container-state.json", snapshot.patchedContainerState); + } + if (options.dockerTopOutput?.trim()) { + writeDiagnosticText("docker-top.txt", options.dockerTopOutput); } try { @@ -1757,64 +1833,46 @@ export function collectDockerGpuPatchDiagnostics( ], { ignoreError: true, timeout: DOCKER_GPU_PATCH_TIMEOUT_MS }, ); - if (ps.trim()) writeTextFile(dir, "docker-ps.txt", ps); + if (ps.trim()) writeDiagnosticText("docker-ps.txt", ps); } catch { /* best effort */ } - let discoveredContainerIds: string[] = []; - try { - discoveredContainerIds = findOpenShellDockerSandboxContainerIds(sandboxName, deps); - } catch { - discoveredContainerIds = []; - } - const containerTargets = uniqueStrings([ - ...(context - ? [context.oldContainerId, context.newContainerId, context.backupContainerName] - : []), - ...discoveredContainerIds, - ]); if (containerTargets.length > 0) { - const inspectEntries: unknown[] = []; + const inspectEntries: DockerContainerInspect[] = []; const networkSummaries: string[] = []; - for (const target of containerTargets) { - try { - const inspect = d.dockerCapture(["inspect", target], { - ignoreError: true, - timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, - }); - if (!inspect.trim()) continue; - const parsed = JSON.parse(inspect); - const entries = Array.isArray(parsed) ? parsed : [parsed]; - inspectEntries.push(...entries); - for (const [index, entry] of entries.entries()) { - networkSummaries.push( + for (const { target, entries } of inspectedTargets) { + const sanitizedEntries = entries.map(redactor.sanitizeInspect); + inspectEntries.push(...sanitizedEntries); + for (const [index, entry] of sanitizedEntries.entries()) { + networkSummaries.push( + redactor.redactText( formatDockerInspectNetworkSummary( entries.length === 1 ? target : `${target}[${index}]`, entry, ), - ); - } - } catch { - /* best effort */ + ), + ); } } if (inspectEntries.length > 0) { - writeTextFile(dir, "docker-inspect.json", JSON.stringify(inspectEntries, null, 2)); + writeDiagnosticJson("docker-inspect.json", inspectEntries); } if (networkSummaries.length > 0) { - writeTextFile(dir, "docker-network-summary.txt", networkSummaries.join("\n\n")); + writeDiagnosticText("docker-network-summary.txt", networkSummaries.join("\n\n")); } const logs = containerTargets .map((target) => { try { - return [`===== ${target} =====`, d.dockerLogs(target, { tail: 120 })].join("\n"); + return redactor.redactText( + [`===== ${target} =====`, d.dockerLogs(target, { tail: 120 })].join("\n"), + ); } catch { - return `===== ${target} =====\n(unavailable)`; + return redactor.redactText(`===== ${target} =====\n(unavailable)`); } }) .join("\n"); - if (logs.trim()) writeTextFile(dir, "docker-logs.txt", logs); + if (logs.trim()) writeDiagnosticText("docker-logs.txt", logs); } if (deps.runCaptureOpenshell) { @@ -1829,7 +1887,7 @@ export function collectDockerGpuPatchDiagnostics( ignoreError: true, timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, }); - if (output.trim()) writeTextFile(dir, fileName, output); + if (output.trim()) writeDiagnosticText(fileName, output); } catch { /* best effort */ } diff --git a/src/lib/onboard/docker-gpu-pre-rollback-diagnostics.test.ts b/src/lib/onboard/docker-gpu-pre-rollback-diagnostics.test.ts new file mode 100644 index 00000000000..9fa9bd464b6 --- /dev/null +++ b/src/lib/onboard/docker-gpu-pre-rollback-diagnostics.test.ts @@ -0,0 +1,230 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { buildDockerGpuMode, type DockerGpuPatchResult } from "./docker-gpu-patch"; +import { captureDockerGpuPreRollbackDiagnostics } from "./docker-gpu-pre-rollback-diagnostics"; + +function patchResult(): DockerGpuPatchResult { + return { + applied: true, + oldContainerId: "old-container-id", + newContainerId: "new-container-id", + originalName: "openshell-alpha", + backupContainerName: "backup-container", + mode: buildDockerGpuMode("cdi"), + backupRemoved: false, + }; +} + +describe("Docker GPU pre-rollback diagnostics (#6110)", () => { + afterEach(() => vi.restoreAllMocks()); + + it("captures the failed clone state, process topology, and logs before rollback", () => { + vi.spyOn(console, "error").mockImplementation(() => {}); + const writeFileSpy = vi.spyOn(fs, "writeFileSync"); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-gpu-pre-rollback-")); + const secretCanary = "pre-rollback-secret-canary-value"; + const discoveredSecretCanary = "discovered-only-secret-canary-value"; + const inspectOutput = JSON.stringify([ + { + Id: "new-container-id", + Name: "/openshell-alpha", + Config: { + Image: "openshell/sandbox:test", + Cmd: null, + Env: [ + `OPENSHELL_SANDBOX_COMMAND=env NEMOCLAW_EXTRA_PLACEHOLDER_KEYS=CUSTOM_PROVIDER_CREDENTIAL CUSTOM_PROVIDER_CREDENTIAL=${secretCanary} nemoclaw-start`, + ], + Labels: { + "openshell.ai/sandbox-name": "alpha", + "untrusted.secret": secretCanary, + }, + }, + HostConfig: { NetworkMode: "openshell-docker" }, + NetworkSettings: { Networks: { "openshell-docker": {} } }, + }, + ]); + const discoveredInspectOutput = JSON.stringify([ + { + Id: "discovered-container-id", + Config: { + Env: [ + `OPENSHELL_SANDBOX_COMMAND=env NEMOCLAW_EXTRA_PLACEHOLDER_KEYS=DISCOVERED_CUSTOM_VALUE DISCOVERED_CUSTOM_VALUE=${discoveredSecretCanary} nemoclaw-start`, + ], + }, + }, + ]); + const dockerResponses = new Map([ + [ + "ps -a --filter label=openshell.ai/managed-by=openshell --filter label=openshell.ai/sandbox-name=alpha --format {{.ID}}", + "new-container-id\ndiscovered-container-id\n", + ], + [ + "ps -a --filter label=openshell.ai/managed-by=openshell --filter label=openshell.ai/sandbox-name=alpha", + `new-container-id ${secretCanary} ${discoveredSecretCanary}\n`, + ], + [ + "top new-container-id -eo user,pid,ppid,stat,comm", + `USER PID PPID STAT COMMAND\nsandbox 42 1 S nemoclaw-start-${secretCanary}\n`, + ], + [ + "inspect --format {{json .State}} new-container-id", + JSON.stringify({ Status: "running", Running: true, ExitCode: 0 }), + ], + ["inspect new-container-id", inspectOutput], + ["inspect old-container-id", "[]"], + ["inspect backup-container", "[]"], + ["inspect discovered-container-id", discoveredInspectOutput], + ]); + const dockerCapture = vi.fn((args: readonly string[], _options?: Record) => { + return dockerResponses.get(args.join(" ")) ?? ""; + }); + const openshellResponses = new Map([ + ["sandbox get", `Phase: Error\ndetail=${secretCanary} ${discoveredSecretCanary}\n`], + ["sandbox list", `alpha Error ${secretCanary} ${discoveredSecretCanary}\n`], + ]); + const runCaptureOpenshell = vi.fn( + (args: string[], _options?: Record) => + openshellResponses.get(`${args[0] ?? ""} ${args[1] ?? ""}`.trim()) ?? + `gateway reconnect log ${secretCanary}\n`, + ); + const dockerLogs = vi.fn((target: string, _options?: { tail?: number; timeout?: number }) => + target === "new-container-id" ? `failed clone log ${secretCanary}\n` : "", + ); + + try { + const diagnostics = captureDockerGpuPreRollbackDiagnostics("alpha", patchResult(), { + dockerCapture, + dockerLogs, + homedir: () => tmpDir, + now: () => new Date("2026-07-01T23:00:00Z"), + runCaptureOpenshell, + }); + + expect(diagnostics?.dir).toBeTruthy(); + expect( + fs.readFileSync(path.join(diagnostics?.dir ?? "", "docker-top.txt"), "utf-8"), + ).toContain("nemoclaw-start"); + expect( + fs.readFileSync(path.join(diagnostics?.dir ?? "", "docker-logs.txt"), "utf-8"), + ).toContain("failed clone log "); + const inspect = JSON.parse( + fs.readFileSync(path.join(diagnostics?.dir ?? "", "docker-inspect.json"), "utf-8"), + ); + expect(inspect[0]).toMatchObject({ + Id: "new-container-id", + Config: { + Image: "openshell/sandbox:test", + Cmd: null, + Env: ["OPENSHELL_SANDBOX_COMMAND="], + }, + HostConfig: { NetworkMode: "openshell-docker" }, + }); + const diagnosticContents = fs + .readdirSync(diagnostics?.dir ?? "") + .map((name) => fs.readFileSync(path.join(diagnostics?.dir ?? "", name), "utf-8")) + .join("\n"); + expect(diagnosticContents).not.toContain(secretCanary); + expect(diagnosticContents).not.toContain(discoveredSecretCanary); + expect(diagnosticContents).not.toContain("untrusted.secret"); + const fullInspectCalls = dockerCapture.mock.calls + .map(([args], index) => ({ args, order: dockerCapture.mock.invocationCallOrder[index] })) + .filter(({ args }) => args[0] === "inspect" && args[1] !== "--format"); + expect(fullInspectCalls.map(({ args }) => args[1])).toEqual( + expect.arrayContaining([ + "new-container-id", + "old-container-id", + "backup-container", + "discovered-container-id", + ]), + ); + expect(Math.max(...fullInspectCalls.map(({ order }) => order ?? 0))).toBeLessThan( + writeFileSpy.mock.invocationCallOrder[0] ?? Number.MAX_SAFE_INTEGER, + ); + expect(dockerCapture).toHaveBeenCalledWith( + ["top", "new-container-id", "-eo", "user,pid,ppid,stat,comm"], + expect.objectContaining({ ignoreError: true, timeout: expect.any(Number) }), + ); + for (const [, options] of dockerCapture.mock.calls) { + expect(Number(options?.timeout)).toBeLessThanOrEqual(2_000); + } + for (const [, options] of runCaptureOpenshell.mock.calls) { + expect(Number(options?.timeout)).toBeLessThanOrEqual(2_000); + } + for (const [, options] of dockerLogs.mock.calls) { + expect(Number(options?.timeout)).toBeLessThanOrEqual(2_000); + } + expect(console.error).toHaveBeenCalledWith( + expect.stringContaining("Pre-rollback diagnostics saved:"), + ); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); + + it("redacts snapshot values when the shared capture budget expires before collector inspect", () => { + vi.spyOn(console, "error").mockImplementation(() => {}); + const clock = [0, 0, 0, 0, 0, 0, 0, 0]; + vi.spyOn(Date, "now").mockImplementation(() => clock.shift() ?? 10_001); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-gpu-budget-redaction-")); + const canary = "opaque-budget-value-71f4"; + const inspectOutput = JSON.stringify([ + { + Id: "new-container-id", + Config: { + Env: [ + `OPENSHELL_SANDBOX_COMMAND=env NEMOCLAW_EXTRA_PLACEHOLDER_KEYS=BUDGET_VALUE BUDGET_VALUE=${canary} nemoclaw-start`, + ], + }, + }, + ]); + const dockerResponses = new Map([ + [ + "ps -a --filter label=openshell.ai/managed-by=openshell --filter label=openshell.ai/sandbox-name=alpha --format {{.ID}}", + "new-container-id\n", + ], + ["inspect new-container-id", inspectOutput], + ["inspect old-container-id", "[]"], + ["inspect backup-container", "[]"], + [ + "inspect --format {{json .State}} new-container-id", + JSON.stringify({ Status: "exited", ExitCode: 125, Error: `state ${canary}` }), + ], + ]); + const openshellResponses = new Map([ + ["sandbox get", `Phase: Error\ndetail=${canary}\n`], + ["sandbox list", `alpha Error ${canary}\n`], + ]); + + try { + const diagnostics = captureDockerGpuPreRollbackDiagnostics("alpha", patchResult(), { + dockerCapture: vi.fn( + (args: readonly string[]) => dockerResponses.get(args.join(" ")) ?? "", + ), + dockerLogs: vi.fn(() => ""), + homedir: () => tmpDir, + now: () => new Date("2026-07-02T01:00:00Z"), + runCaptureOpenshell: vi.fn( + (args: string[]) => openshellResponses.get(`${args[0] ?? ""} ${args[1] ?? ""}`) ?? "", + ), + }); + + const summary = fs.readFileSync(path.join(diagnostics?.dir ?? "", "summary.txt"), "utf8"); + const state = fs.readFileSync( + path.join(diagnostics?.dir ?? "", "patched-container-state.json"), + "utf8", + ); + expect(`${summary}\n${state}`).not.toContain(canary); + expect(summary).toContain("sandbox_list_row=alpha Error "); + expect(JSON.parse(state).Error).toBe("state "); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); +}); diff --git a/src/lib/onboard/docker-gpu-pre-rollback-diagnostics.ts b/src/lib/onboard/docker-gpu-pre-rollback-diagnostics.ts new file mode 100644 index 00000000000..4e7f53cbe21 --- /dev/null +++ b/src/lib/onboard/docker-gpu-pre-rollback-diagnostics.ts @@ -0,0 +1,158 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + dockerCapture as defaultDockerCapture, + dockerLogs as defaultDockerLogs, +} from "../adapters/docker"; +import { discoverDockerGpuDiagnosticSensitiveValues } from "./docker-gpu-diagnostic-redaction"; +import type { + DockerContainerInspect, + DockerGpuPatchDeps, + DockerGpuPatchDiagnostics, + DockerGpuPatchFailureContext, + DockerGpuPatchResult, +} from "./docker-gpu-patch"; +import { + captureDockerGpuPatchSandboxSnapshot, + classifyDockerGpuPatchFailure, + collectDockerGpuPatchDiagnostics, + findOpenShellDockerSandboxContainerIds, +} from "./docker-gpu-patch"; + +const DOCKER_GPU_PATCH_TIMEOUT_MS = 30_000; +const PRE_ROLLBACK_DIAGNOSTICS_TOTAL_BUDGET_MS = 10_000; +const PRE_ROLLBACK_DIAGNOSTICS_CALL_TIMEOUT_MS = 2_000; + +type PreRollbackDiagnosticsDeps = Pick< + DockerGpuPatchDeps, + "runCaptureOpenshell" | "dockerCapture" | "dockerLogs" | "homedir" | "now" +>; + +// This wrapper owns only the pre-rollback time budget. The shared collector +// is the sole redaction and artifact-publication boundary for every caller. +function boundedDiagnosticsDeps(deps: PreRollbackDiagnosticsDeps): PreRollbackDiagnosticsDeps { + const capture = deps.dockerCapture ?? defaultDockerCapture; + const logs = deps.dockerLogs ?? defaultDockerLogs; + const deadline = Date.now() + PRE_ROLLBACK_DIAGNOSTICS_TOTAL_BUDGET_MS; + const boundedOptions = (options: Record | undefined) => { + const remaining = deadline - Date.now(); + if (remaining <= 0) return null; + return { + ...options, + timeout: Math.min(PRE_ROLLBACK_DIAGNOSTICS_CALL_TIMEOUT_MS, remaining), + }; + }; + return { + ...deps, + dockerCapture: (args, options) => { + const bounded = boundedOptions(options); + if (!bounded) return ""; + return capture(args, bounded); + }, + dockerLogs: (containerName, options) => { + const bounded = boundedOptions(options); + if (!bounded) return ""; + return logs(containerName, bounded); + }, + runCaptureOpenshell: deps.runCaptureOpenshell + ? (args, options) => { + const bounded = boundedOptions(options); + if (!bounded) return ""; + return deps.runCaptureOpenshell?.(args, bounded) ?? ""; + } + : undefined, + }; +} + +function primeSensitiveDiagnosticValues( + sandboxName: string, + result: DockerGpuPatchResult, + deps: PreRollbackDiagnosticsDeps, +): string[] { + let discoveredContainerIds: string[] = []; + try { + discoveredContainerIds = findOpenShellDockerSandboxContainerIds(sandboxName, deps); + } catch { + // Known recreate targets still provide useful values when label discovery + // races with rollback or daemon recovery. + } + const targets = [ + result.newContainerId, + result.oldContainerId, + result.backupContainerName, + ...discoveredContainerIds, + ].filter((target, index, values) => target.length > 0 && values.indexOf(target) === index); + const sensitiveValues = new Set(); + for (const target of targets) { + try { + const output = deps.dockerCapture?.(["inspect", target], { + ignoreError: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }); + if (!output?.trim()) continue; + const parsed = JSON.parse(output); + const entries = (Array.isArray(parsed) ? parsed : [parsed]) as DockerContainerInspect[]; + for (const entry of entries) { + for (const value of discoverDockerGpuDiagnosticSensitiveValues(entry)) { + sensitiveValues.add(value); + } + } + } catch { + // Diagnostics remain best effort when a short-lived target disappears. + } + } + return [...sensitiveValues]; +} + +export function captureDockerGpuPreRollbackDiagnostics( + sandboxName: string, + result: DockerGpuPatchResult, + deps: PreRollbackDiagnosticsDeps = {}, +): DockerGpuPatchDiagnostics | null { + const context: DockerGpuPatchFailureContext = { + sandboxName, + oldContainerId: result.oldContainerId, + newContainerId: result.newContainerId, + backupContainerName: result.backupContainerName, + selectedMode: result.mode, + }; + const diagnosticDeps = boundedDiagnosticsDeps(deps); + const additionalSensitiveValues = primeSensitiveDiagnosticValues( + sandboxName, + result, + diagnosticDeps, + ); + const snapshot = captureDockerGpuPatchSandboxSnapshot( + sandboxName, + { patchedContainerId: result.newContainerId }, + diagnosticDeps, + ); + const classification = classifyDockerGpuPatchFailure(snapshot, result.mode); + let dockerTopOutput: string | null = null; + try { + const dockerCapture = diagnosticDeps.dockerCapture ?? defaultDockerCapture; + dockerTopOutput = dockerCapture( + ["top", result.newContainerId, "-eo", "user,pid,ppid,stat,comm"], + { ignoreError: true, timeout: DOCKER_GPU_PATCH_TIMEOUT_MS }, + ); + } catch { + // The remaining bundle is still useful when the clone exits before top. + } + const diagnostics = collectDockerGpuPatchDiagnostics( + sandboxName, + { + context, + selectedMode: result.mode, + snapshot, + classification, + additionalSensitiveValues, + dockerTopOutput, + }, + diagnosticDeps, + ); + if (!diagnostics) return null; + + console.error(` Pre-rollback diagnostics saved: ${diagnostics.dir}`); + return diagnostics; +} diff --git a/src/lib/onboard/docker-gpu-sandbox-create-diagnostics.test.ts b/src/lib/onboard/docker-gpu-sandbox-create-diagnostics.test.ts new file mode 100644 index 00000000000..ee77ddaccdf --- /dev/null +++ b/src/lib/onboard/docker-gpu-sandbox-create-diagnostics.test.ts @@ -0,0 +1,108 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import type { DockerGpuPatchResult } from "./docker-gpu-patch"; +import { createDockerGpuSandboxCreatePatch } from "./docker-gpu-sandbox-create"; + +const RESULT: DockerGpuPatchResult = { + applied: true, + oldContainerId: "old-container-id", + newContainerId: "new-container-id", + originalName: "openshell-alpha", + backupContainerName: "backup-container", + mode: { + kind: "gpus", + label: "--gpus all", + device: "all", + args: ["--gpus", "all"], + }, + backupRemoved: false, +}; + +describe("Docker GPU create diagnostics fail-safety (#6110)", () => { + afterEach(() => vi.restoreAllMocks()); + + it("still rolls back when pre-rollback diagnostic capture fails", () => { + vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(console, "warn").mockImplementation(() => {}); + const deps = { + runOpenshell: vi.fn(() => ({ status: 0 })), + runCaptureOpenshell: vi.fn(() => ""), + sleep: vi.fn(), + dockerCapture: vi.fn(() => ""), + }; + const finalizeBackup = vi.fn(() => ({ backupRemoved: false, rolledBack: true })); + const onPatchFailureExit = vi.fn(); + const patch = createDockerGpuSandboxCreatePatch({ + enabled: true, + sandboxName: "alpha", + timeoutSecs: 60, + deps, + overrides: { + findContainerIds: vi.fn(() => ["existing-container"]), + recreatePatch: vi.fn(() => RESULT), + waitForSupervisor: vi.fn(() => false), + capturePreRollbackDiagnostics: vi.fn(() => { + throw new Error("disk full"); + }), + finalizeBackup, + onPatchFailureExit, + }, + }); + + patch.maybeApplyDuringCreate(); + patch.waitForSupervisorReconnectIfNeeded(); + + expect(finalizeBackup).toHaveBeenCalledWith({ result: RESULT, supervisorReady: false }, deps); + expect(onPatchFailureExit).toHaveBeenCalledTimes(1); + expect(console.warn).toHaveBeenCalledWith( + expect.stringContaining( + "Could not capture the failed GPU container before rollback: disk full", + ), + ); + }); + + it("captures before rollback when ensureApplied performs the recreate after create exits", () => { + vi.spyOn(console, "log").mockImplementation(() => {}); + const deps = { + runOpenshell: vi.fn(() => ({ status: 0 })), + runCaptureOpenshell: vi.fn(() => ""), + sleep: vi.fn(), + dockerCapture: vi.fn(() => ""), + }; + const recreatePatch = vi.fn(() => RESULT); + const waitForSupervisor = vi.fn(() => false); + const capturePreRollbackDiagnostics = vi.fn(() => null); + const finalizeBackup = vi.fn(() => ({ backupRemoved: false, rolledBack: true })); + const onPatchFailureExit = vi.fn(); + const patch = createDockerGpuSandboxCreatePatch({ + enabled: true, + sandboxName: "alpha", + timeoutSecs: 60, + deps, + overrides: { + recreatePatch, + waitForSupervisor, + capturePreRollbackDiagnostics, + finalizeBackup, + onPatchFailureExit, + }, + }); + + patch.ensureApplied(); + patch.waitForSupervisorReconnectIfNeeded(); + + expect(recreatePatch).toHaveBeenCalledWith( + expect.objectContaining({ waitForSupervisor: false }), + deps, + ); + expect(capturePreRollbackDiagnostics).toHaveBeenCalledWith("alpha", RESULT, deps); + expect(capturePreRollbackDiagnostics.mock.invocationCallOrder[0]).toBeLessThan( + finalizeBackup.mock.invocationCallOrder[0], + ); + expect(finalizeBackup).toHaveBeenCalledWith({ result: RESULT, supervisorReady: false }, deps); + expect(onPatchFailureExit).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/lib/onboard/docker-gpu-sandbox-create.test.ts b/src/lib/onboard/docker-gpu-sandbox-create.test.ts index b5353e8f7e9..e5a28f7eca5 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create.test.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create.test.ts @@ -52,6 +52,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { const recreatePatch = vi.fn(() => result); const waitForSupervisor = vi.fn(() => true); const finalizeBackup = vi.fn(() => ({ backupRemoved: true, rolledBack: false })); + const capturePreRollbackDiagnostics = vi.fn(() => null); const onPatchFailureExit = vi.fn(); const findContainerIds = vi.fn(() => ["existing-container"]); @@ -65,6 +66,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { recreatePatch, waitForSupervisor, finalizeBackup, + capturePreRollbackDiagnostics, onPatchFailureExit, }, }); @@ -83,6 +85,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { expect(waitForSupervisor).toHaveBeenCalledTimes(1); expect(finalizeBackup).toHaveBeenCalledTimes(1); expect(finalizeBackup).toHaveBeenCalledWith({ result, supervisorReady: true }, deps); + expect(capturePreRollbackDiagnostics).not.toHaveBeenCalled(); expect(onPatchFailureExit).not.toHaveBeenCalled(); }); @@ -91,6 +94,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { const result = deferredCreateResult(); const recreatePatch = vi.fn(() => result); const waitForSupervisor = vi.fn(() => false); + const capturePreRollbackDiagnostics = vi.fn(() => null); const finalizeBackup = vi.fn(() => ({ backupRemoved: false, rolledBack: true })); const onPatchFailureExit = vi.fn(); const findContainerIds = vi.fn(() => ["existing-container"]); @@ -105,6 +109,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { recreatePatch, waitForSupervisor, finalizeBackup, + capturePreRollbackDiagnostics, onPatchFailureExit, }, }); @@ -112,6 +117,10 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { patch.maybeApplyDuringCreate(); patch.waitForSupervisorReconnectIfNeeded(); + expect(capturePreRollbackDiagnostics).toHaveBeenCalledWith("alpha", result, deps); + expect(capturePreRollbackDiagnostics.mock.invocationCallOrder[0]).toBeLessThan( + finalizeBackup.mock.invocationCallOrder[0], + ); expect(finalizeBackup).toHaveBeenCalledWith({ result, supervisorReady: false }, deps); expect(onPatchFailureExit).toHaveBeenCalledTimes(1); const [sandboxName, error, exitDeps] = onPatchFailureExit.mock.calls[0]; @@ -129,6 +138,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { const recreatePatch = vi.fn(() => result); const waitForSupervisor = vi.fn(() => false); const finalizeBackup = vi.fn(() => ({ backupRemoved: false, rolledBack: false })); + const capturePreRollbackDiagnostics = vi.fn(() => null); const onPatchFailureExit = vi.fn(); const findContainerIds = vi.fn(() => ["existing-container"]); @@ -142,6 +152,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { recreatePatch, waitForSupervisor, finalizeBackup, + capturePreRollbackDiagnostics, onPatchFailureExit, }, }); @@ -259,6 +270,36 @@ describe("resolveDockerGpuSandboxCreatePlan Docker Desktop WSL handling", () => } }); + it("uses native OpenShell GPU by default and preserves the explicit legacy force", () => { + vi.stubEnv("NEMOCLAW_DOCKER_GPU_PATCH", ""); + try { + expect( + resolveDockerGpuSandboxCreatePlan( + { sandboxGpuEnabled: true }, + { + dockerDriverGateway: true, + detectDockerDesktopWsl: () => false, + platform: "linux", + }, + ).useDockerGpuPatch, + ).toBe(false); + + vi.stubEnv("NEMOCLAW_DOCKER_GPU_PATCH", "1"); + expect( + resolveDockerGpuSandboxCreatePlan( + { sandboxGpuEnabled: true }, + { + dockerDriverGateway: true, + detectDockerDesktopWsl: () => false, + platform: "linux", + }, + ).useDockerGpuPatch, + ).toBe(true); + } finally { + vi.unstubAllEnvs(); + } + }); + it("suppresses the openshell sandbox create --gpu flag on Docker Desktop WSL when the opt-out is ignored", () => { const originalEnv = process.env.NEMOCLAW_DOCKER_GPU_PATCH; process.env.NEMOCLAW_DOCKER_GPU_PATCH = "0"; diff --git a/src/lib/onboard/docker-gpu-sandbox-create.ts b/src/lib/onboard/docker-gpu-sandbox-create.ts index df6d11565c0..300531390d7 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create.ts @@ -11,7 +11,6 @@ import type { DockerGpuPatchResult, } from "./docker-gpu-patch"; import { - applyDockerGpuPatchOrExit, findOpenShellDockerSandboxContainerIds, getDockerGpuSupervisorReconnectTimeoutSecs, printDockerGpuPatchFailureAndExit, @@ -22,6 +21,7 @@ import { waitForOpenShellSupervisorReconnect, } from "./docker-gpu-patch"; import { finalizeDockerGpuPatchBackup } from "./docker-gpu-patch-finalize"; +import { captureDockerGpuPreRollbackDiagnostics } from "./docker-gpu-pre-rollback-diagnostics"; import { detectWslDockerDesktopStatus } from "./wsl-docker-desktop-gpu"; let cachedDockerDesktopWslRuntime: boolean | null = null; @@ -46,6 +46,7 @@ type RecreatePatchFn = typeof recreateOpenShellDockerSandboxWithGpu; type WaitSupervisorFn = typeof waitForOpenShellSupervisorReconnect; type FindContainerIdsFn = typeof findOpenShellDockerSandboxContainerIds; type FinalizeBackupFn = typeof finalizeDockerGpuPatchBackup; +type CapturePreRollbackDiagnosticsFn = typeof captureDockerGpuPreRollbackDiagnostics; // Loosen the override return type from `never` to `void` so tests can pass a // plain `vi.fn()` mock. Production wires `printDockerGpuPatchFailureAndExit` // which has return type `never`; that is assignable to `void`. @@ -80,6 +81,7 @@ type DockerGpuSandboxCreatePatchOptions = { recreatePatch?: RecreatePatchFn; waitForSupervisor?: WaitSupervisorFn; finalizeBackup?: FinalizeBackupFn; + capturePreRollbackDiagnostics?: CapturePreRollbackDiagnosticsFn; onPatchFailureExit?: PatchFailureExitFn; }; }; @@ -133,6 +135,8 @@ export function createDockerGpuSandboxCreatePatch( const waitForSupervisor = options.overrides?.waitForSupervisor ?? waitForOpenShellSupervisorReconnect; const finalizeBackup = options.overrides?.finalizeBackup ?? finalizeDockerGpuPatchBackup; + const captureFailedClone = + options.overrides?.capturePreRollbackDiagnostics ?? captureDockerGpuPreRollbackDiagnostics; const onPatchFailureExit = options.overrides?.onPatchFailureExit ?? printDockerGpuPatchFailureAndExit; @@ -180,7 +184,17 @@ export function createDockerGpuSandboxCreatePatch( ensureApplied() { if (!options.enabled || result) return; - result = applyDockerGpuPatchOrExit(applyOptions, options.deps); + console.log(" Recreating OpenShell Docker sandbox container with NVIDIA GPU access..."); + try { + result = recreatePatch({ ...applyOptions, waitForSupervisor: false }, options.deps); + needsSupervisorWait = true; + console.log(` ✓ Docker GPU mode selected: ${result.mode.label}`); + } catch (error) { + onPatchFailureExit(options.sandboxName, error, { + runCaptureOpenshell: options.deps.runCaptureOpenshell, + dockerCapture: options.deps.dockerCapture, + }); + } }, waitForSupervisorReconnectIfNeeded() { @@ -204,6 +218,15 @@ export function createDockerGpuSandboxCreatePatch( sleep: options.deps.sleep, }, ); + if (!supervisorReady && result) { + try { + captureFailedClone(options.sandboxName, result, options.deps); + } catch (error) { + console.warn( + ` ⚠ Could not capture the failed GPU container before rollback: ${error instanceof Error ? error.message : String(error)}`, + ); + } + } const finalizeOutcome = result ? finalizeBackup({ result, supervisorReady }, options.deps) : null; @@ -312,12 +335,14 @@ export function shouldUseDockerGpuPatchForCreate( options: { dockerDriverGateway: boolean; dockerDesktopWsl?: boolean; + platform?: NodeJS.Platform; log?: (message: string) => void; }, ): boolean { const enabled = shouldApplyDockerGpuPatch(config, { dockerDriverGateway: options.dockerDriverGateway, dockerDesktopWsl: options.dockerDesktopWsl, + platform: options.platform, log: options.log, }); if (enabled) { @@ -336,6 +361,7 @@ export function resolveDockerGpuSandboxCreatePlan( dockerDriverGateway: boolean; dockerDesktopWsl?: boolean; detectDockerDesktopWsl?: () => boolean; + platform?: NodeJS.Platform; }, ): DockerGpuSandboxCreatePlan { const dockerDesktopWsl = @@ -343,6 +369,7 @@ export function resolveDockerGpuSandboxCreatePlan( const useDockerGpuPatch = shouldUseDockerGpuPatchForCreate(config, { dockerDriverGateway: options.dockerDriverGateway, dockerDesktopWsl, + platform: options.platform, }); const logMessage = config.sandboxGpuEnabled ? useDockerGpuPatch diff --git a/src/lib/onboard/extra-placeholder-keys.test.ts b/src/lib/onboard/extra-placeholder-keys.test.ts index 91bb0aa120a..7f5ab49616c 100644 --- a/src/lib/onboard/extra-placeholder-keys.test.ts +++ b/src/lib/onboard/extra-placeholder-keys.test.ts @@ -290,7 +290,7 @@ describe("appendExtraPlaceholderKeysEnvArg", () => { return `${key}=${value}`; } - it("appends one whitespace-joined env arg containing only the key names, never their token values", () => { + it("appends one comma-joined env arg containing only the key names, never their token values", () => { const envArgs: string[] = []; appendExtraPlaceholderKeysEnvArg( envArgs, @@ -298,7 +298,7 @@ describe("appendExtraPlaceholderKeysEnvArg", () => { formatEnvAssignment, ); expect(envArgs).toEqual([ - `${EXTRA_PLACEHOLDER_KEYS_ENV}=TELEGRAM_BOT_TOKEN_AGENT_A SLACK_BOT_TOKEN_AGENT_B`, + `${EXTRA_PLACEHOLDER_KEYS_ENV}=TELEGRAM_BOT_TOKEN_AGENT_A,SLACK_BOT_TOKEN_AGENT_B`, ]); // The emitted env arg holds only the key list, not the resolved token // value. Operators who set the credential see openshell:resolve:env: @@ -309,6 +309,29 @@ describe("appendExtraPlaceholderKeysEnvArg", () => { } }); + it("survives the OpenShell split_whitespace command round trip", () => { + const envArgs: string[] = []; + appendExtraPlaceholderKeysEnvArg( + envArgs, + ["TELEGRAM_BOT_TOKEN_AGENT_A", "SLACK_BOT_TOKEN_AGENT_B"], + formatEnvAssignment, + ); + + const commandTokens = ["env", ...envArgs, "nemoclaw-start"].join(" ").split(/\s+/u); + const assignment = commandTokens.find((token) => + token.startsWith(`${EXTRA_PLACEHOLDER_KEYS_ENV}=`), + ); + expect(assignment).toBe( + `${EXTRA_PLACEHOLDER_KEYS_ENV}=TELEGRAM_BOT_TOKEN_AGENT_A,SLACK_BOT_TOKEN_AGENT_B`, + ); + + const rawValue = assignment?.slice(EXTRA_PLACEHOLDER_KEYS_ENV.length + 1); + expect(parseExtraPlaceholderKeys(rawValue, CANONICAL_ENVKEYS_FIXTURE)).toEqual({ + keys: ["TELEGRAM_BOT_TOKEN_AGENT_A", "SLACK_BOT_TOKEN_AGENT_B"], + warnings: [], + }); + }); + it("emits no env arg when the extras list is empty", () => { const envArgs: string[] = []; appendExtraPlaceholderKeysEnvArg(envArgs, [], formatEnvAssignment); diff --git a/src/lib/onboard/extra-placeholder-keys.ts b/src/lib/onboard/extra-placeholder-keys.ts index 8cb88b6e462..059d7100821 100644 --- a/src/lib/onboard/extra-placeholder-keys.ts +++ b/src/lib/onboard/extra-placeholder-keys.ts @@ -2,8 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 import { getCredential, normalizeCredentialValue } from "../credentials/store"; -import { getChannelTokenKeys, listChannels } from "../sandbox/channels"; import * as webSearch from "../inference/web-search"; +import { getChannelTokenKeys, listChannels } from "../sandbox/channels"; interface MessagingTokenDefShape { name: string; @@ -122,5 +122,9 @@ export function appendExtraPlaceholderKeysEnvArg( formatEnvAssignment: (key: string, value: string) => string, ): void { if (extraKeys.length === 0) return; - envArgs.push(formatEnvAssignment(EXTRA_PLACEHOLDER_KEYS_ENV, extraKeys.join(" "))); + // OpenShell's Docker supervisor deserializes OPENSHELL_SANDBOX_COMMAND with + // split_whitespace(). Commas preserve this list as one env assignment when + // the Docker GPU compatibility path transports the command through that + // variable; both host and sandbox parsers accept comma separators. + envArgs.push(formatEnvAssignment(EXTRA_PLACEHOLDER_KEYS_ENV, extraKeys.join(","))); } diff --git a/test/e2e/fixtures/fake-openai-compatible.ts b/test/e2e/fixtures/fake-openai-compatible.ts index 74179ca70b1..8bf4c622390 100644 --- a/test/e2e/fixtures/fake-openai-compatible.ts +++ b/test/e2e/fixtures/fake-openai-compatible.ts @@ -17,6 +17,7 @@ export interface FakeOpenAiCompatibleRequest { readonly auth?: string; readonly model?: string; readonly stream?: boolean; + readonly forbiddenMarkerMatches?: number; } export interface FakeOpenAiCompatibleServer { @@ -30,6 +31,7 @@ export interface FakeOpenAiCompatibleServer { export interface FakeOpenAiCompatibleServerOptions { readonly apiKey?: string; readonly chatContent?: string; + readonly forbiddenMarkers?: readonly string[]; readonly host?: string; readonly model?: string; readonly port?: number; @@ -124,6 +126,7 @@ export async function startFakeOpenAiCompatibleServer( ...process.env, NEMOCLAW_FAKE_OPENAI_API_KEY: options.apiKey ?? "", NEMOCLAW_FAKE_OPENAI_CHAT_CONTENT: options.chatContent ?? "ok", + NEMOCLAW_FAKE_OPENAI_FORBIDDEN_MARKERS: JSON.stringify(options.forbiddenMarkers ?? []), NEMOCLAW_FAKE_OPENAI_HOST: host, NEMOCLAW_FAKE_OPENAI_LOG_FILE: logFile, NEMOCLAW_FAKE_OPENAI_MODEL: options.model ?? "test-model", diff --git a/test/e2e/lib/fake-openai-compatible-api.mts b/test/e2e/lib/fake-openai-compatible-api.mts index 1aaa8f156e2..454b96142dd 100755 --- a/test/e2e/lib/fake-openai-compatible-api.mts +++ b/test/e2e/lib/fake-openai-compatible-api.mts @@ -17,6 +17,16 @@ const apiKey = process.env.NEMOCLAW_FAKE_OPENAI_API_KEY || ""; const requireAuth = process.env.NEMOCLAW_FAKE_OPENAI_REQUIRE_AUTH === "1"; const chatContent = process.env.NEMOCLAW_FAKE_OPENAI_CHAT_CONTENT || "ok"; const responseText = process.env.NEMOCLAW_FAKE_OPENAI_RESPONSE_TEXT || chatContent; +const forbiddenMarkers = (() => { + try { + const parsed = JSON.parse(process.env.NEMOCLAW_FAKE_OPENAI_FORBIDDEN_MARKERS || "[]"); + return Array.isArray(parsed) + ? parsed.filter((value): value is string => typeof value === "string" && value.length > 0) + : []; + } catch { + return []; + } +})(); function log(message: string): void { if (logFile) { @@ -102,12 +112,23 @@ function parseJsonBody(raw: Buffer): JsonObject { } } +function forbiddenMarkerMatches(req: IncomingMessage, raw: Buffer): number { + const headerValues = Object.values(req.headers).flatMap((value) => value ?? []); + const requestMaterial = [req.url ?? "", ...headerValues, raw.toString("utf8")].join("\n"); + return forbiddenMarkers.filter((marker) => requestMaterial.includes(marker)).length; +} + const server = createServer(async (req, res) => { const path = requestPath(req); if (req.method === "GET" && ["/v1/models", "/models"].includes(path)) { log(`GET ${path}`); - recordRequest({ method: "GET", path, bodyBytes: 0 }); + recordRequest({ + method: "GET", + path, + bodyBytes: 0, + forbiddenMarkerMatches: forbiddenMarkerMatches(req, Buffer.alloc(0)), + }); sendJson(res, 200, { object: "list", data: [{ id: model, object: "model" }] }); return; } @@ -122,6 +143,7 @@ const server = createServer(async (req, res) => { auth, model: payload.model, stream: Boolean(payload.stream), + forbiddenMarkerMatches: forbiddenMarkerMatches(req, raw), }); if (req.method === "POST" && ["/v1/chat/completions", "/chat/completions"].includes(path)) { diff --git a/test/e2e/live/hermes-gpu-startup-integrity.ts b/test/e2e/live/hermes-gpu-startup-integrity.ts new file mode 100644 index 00000000000..20e09fc6e3f --- /dev/null +++ b/test/e2e/live/hermes-gpu-startup-integrity.ts @@ -0,0 +1,158 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export interface HermesManagedStartupIntegrityPaths { + configPath: string; + envPath: string; + strictHashPath: string; + compatHashPath: string; + startupLogPath: string; + strictHashUid: number; + strictHashGid: number; +} + +const DEFAULT_PATHS: HermesManagedStartupIntegrityPaths = { + configPath: "/sandbox/.hermes/config.yaml", + envPath: "/sandbox/.hermes/.env", + strictHashPath: "/etc/nemoclaw/hermes.config-hash", + compatHashPath: "/sandbox/.hermes/.config-hash", + startupLogPath: "/tmp/nemoclaw-start.log", + strictHashUid: 0, + strictHashGid: 0, +}; + +function pythonLiteral(value: string | number): string { + return JSON.stringify(value); +} + +/** + * Build an independent read-only proof for the managed non-root Hermes startup hash contract. + * The strict anchor intentionally describes the build-time environment before PID 1 mints one API key. + */ +export function buildHermesManagedStartupIntegrityScript( + overrides: Partial = {}, +): string { + const paths = { ...DEFAULT_PATHS, ...overrides }; + return `set -eu +/usr/bin/python3 -I - <<'PY' +import hashlib +import os +from pathlib import Path +import re +import secrets +import stat + +config_path = Path(${pythonLiteral(paths.configPath)}) +env_path = Path(${pythonLiteral(paths.envPath)}) +strict_hash_path = Path(${pythonLiteral(paths.strictHashPath)}) +compat_hash_path = Path(${pythonLiteral(paths.compatHashPath)}) +startup_log_path = Path(${pythonLiteral(paths.startupLogPath)}) +strict_hash_uid = ${pythonLiteral(paths.strictHashUid)} +strict_hash_gid = ${pythonLiteral(paths.strictHashGid)} + +def fail(message): + raise SystemExit(message) + +def read_regular(path, label, max_bytes): + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) + try: + fd = os.open(path, flags) + except OSError: + fail(f"{label} is not a readable regular file") + try: + metadata = os.fstat(fd) + if not stat.S_ISREG(metadata.st_mode) or metadata.st_nlink != 1: + fail(f"{label} is not a single-link regular file") + with os.fdopen(fd, "rb", closefd=False) as stream: + data = stream.read(max_bytes + 1) + if len(data) > max_bytes: + fail(f"{label} exceeds the proof size limit") + return data, metadata + finally: + os.close(fd) + +def parse_hash(data, label): + try: + text = data.decode("ascii") + except UnicodeDecodeError: + fail(f"{label} is not ASCII") + if not text.endswith("\\n"): + fail(f"{label} is missing its final newline") + lines = text.splitlines() + expected_paths = (str(config_path), str(env_path)) + if len(lines) != len(expected_paths): + fail(f"{label} does not contain exactly two records") + digests = [] + for line, expected_path in zip(lines, expected_paths): + match = re.fullmatch(r"([0-9a-f]{64}) (.+)", line) + if match is None or match.group(2) != expected_path: + fail(f"{label} contains an unexpected record") + digests.append(match.group(1)) + return tuple(digests) + +def digest(data): + return hashlib.sha256(data).hexdigest() + +config_bytes, _config_metadata = read_regular(config_path, "Hermes config", 4 * 1024 * 1024) +env_bytes, _env_metadata = read_regular(env_path, "Hermes environment", 1024 * 1024) +strict_hash_bytes, strict_hash_metadata = read_regular( + strict_hash_path, "Hermes strict hash", 4096 +) +compat_hash_bytes, _compat_hash_metadata = read_regular( + compat_hash_path, "Hermes compatibility hash", 4096 +) +startup_log_bytes, _startup_log_metadata = read_regular( + startup_log_path, "Hermes startup log", 8 * 1024 * 1024 +) + +if ( + strict_hash_metadata.st_uid != strict_hash_uid + or strict_hash_metadata.st_gid != strict_hash_gid + or stat.S_IMODE(strict_hash_metadata.st_mode) & 0o222 +): + fail("Hermes strict hash is not the expected read-only owner anchor") + +try: + env_text = env_bytes.decode("utf-8") +except UnicodeDecodeError: + fail("Hermes environment is not UTF-8") +base_env_lines = [] +api_key_lines = 0 +for line in env_text.splitlines(keepends=True): + candidate = line.rstrip("\\n") + if candidate.startswith("export "): + candidate = candidate[len("export "):].lstrip() + key = candidate.split("=", 1)[0] if "=" in candidate else None + if key == "API_SERVER_KEY": + if api_key_lines != 0 or re.fullmatch(r"API_SERVER_KEY=[0-9a-f]{64}\\n", line) is None: + fail("Hermes environment contains an unexpected API key assignment") + api_key_lines += 1 + else: + base_env_lines.append(line) +if api_key_lines != 1: + fail("Hermes environment does not contain exactly one canonical generated API key") +base_env_bytes = "".join(base_env_lines).encode("utf-8") + +strict_config_digest, strict_env_digest = parse_hash(strict_hash_bytes, "Hermes strict hash") +compat_config_digest, compat_env_digest = parse_hash( + compat_hash_bytes, "Hermes compatibility hash" +) +if not secrets.compare_digest(strict_config_digest, digest(config_bytes)): + fail("Hermes config differs from the strict startup base") +if not secrets.compare_digest(strict_env_digest, digest(base_env_bytes)): + fail("Hermes environment differs from the strict startup base beyond the generated API key") +if not secrets.compare_digest(compat_config_digest, digest(config_bytes)): + fail("Hermes compatibility hash does not match the current config") +if not secrets.compare_digest(compat_env_digest, digest(env_bytes)): + fail("Hermes compatibility hash does not match the current environment") + +startup_log = startup_log_bytes.decode("utf-8", "replace") +if ( + "ensure-api-key is restricted to the Hermes PID 1 startup transaction" in startup_log + or "Hermes runtime config guard refuses mutation under a foreign PID 1" in startup_log +): + fail("Hermes startup log contains a runtime config guard refusal") + +print("OK") +PY`; +} diff --git a/test/e2e/live/hermes-gpu-startup-proof.ts b/test/e2e/live/hermes-gpu-startup-proof.ts new file mode 100644 index 00000000000..16e0f9c32d5 --- /dev/null +++ b/test/e2e/live/hermes-gpu-startup-proof.ts @@ -0,0 +1,263 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { + type HostCliClient, + resultText, + type SandboxClient, + trustedSandboxShellScript, +} from "../fixtures/clients/index.ts"; +import { expect } from "../fixtures/e2e-test.ts"; +import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; +import { buildHermesManagedStartupIntegrityScript } from "./hermes-gpu-startup-integrity.ts"; +import { stripAnsi } from "./json-envelope.ts"; + +export const HERMES_GPU_EXTRA_PLACEHOLDER_KEYS = [ + "TELEGRAM_BOT_TOKEN_AGENT_A", + "SLACK_BOT_TOKEN_AGENT_B", +] as const; + +interface HermesGpuStartupProofOptions { + env: NodeJS.ProcessEnv; + gpuRoute: "legacy-patch" | "native-openshell"; + host: HostCliClient; + install: Pick; + sandbox: SandboxClient; + sandboxName: string; + status: Pick; +} + +export async function assertHermesGpuStartupProof({ + env, + gpuRoute, + host, + install, + sandbox, + sandboxName, + status, +}: HermesGpuStartupProofOptions): Promise { + expect(resultText(install)).toContain("Starting OpenShell Docker-driver gateway..."); + expect(resultText(install)).toContain("Docker-driver gateway is healthy"); + expect(resultText(install)).not.toContain("Reusing healthy NemoClaw gateway."); + expect(resultText(install)).not.toContain("Reusing existing Docker-driver gateway"); + expect(resultText(install)).not.toContain("[reuse] Skipping gateway (running)"); + if (gpuRoute === "legacy-patch") { + expect(resultText(install)).toContain( + "Recreating OpenShell Docker sandbox container with NVIDIA GPU access", + ); + expect(resultText(install)).toContain("Docker GPU mode selected:"); + } else { + expect(resultText(install)).toContain( + "Direct sandbox GPU enabled; allowing OpenShell GPU policy enrichment.", + ); + expect(resultText(install)).not.toContain( + "Recreating OpenShell Docker sandbox container with NVIDIA GPU access", + ); + expect(resultText(install)).not.toContain("Docker GPU mode selected:"); + } + const plainStatus = stripAnsi(resultText(status)); + expect(plainStatus).toMatch(/Phase:\s*Ready/i); + expect(plainStatus).toContain("Sandbox GPU: enabled"); + expect(plainStatus).toContain("CUDA verified"); + expect(plainStatus).not.toMatch(/last CUDA proof failed|CUDA unverified/i); + + const openshellState = await sandbox.openshell(["sandbox", "get", sandboxName], { + artifactName: "phase-4-openshell-sandbox-ready-gpu-startup", + env, + timeoutMs: 30_000, + }); + expect(openshellState.exitCode, resultText(openshellState)).toBe(0); + expect(stripAnsi(resultText(openshellState))).toMatch(/Phase:\s*Ready/i); + + const pid1Topology = await sandbox.execShell( + sandboxName, + trustedSandboxShellScript( + String.raw`python3 -c 'import json; from pathlib import Path; argv=[item.decode("utf-8", "strict") for item in Path("/proc/1/cmdline").read_bytes().split(b"\0") if item]; print(json.dumps({"argv0": argv[0] if argv else "", "has_nemoclaw_start": any(item in ("nemoclaw-start", "/usr/local/bin/nemoclaw-start") for item in argv)}))'`, + ), + { + artifactName: "phase-4-gpu-startup-pid1-topology", + env, + timeoutMs: 30_000, + }, + ); + expect(pid1Topology.exitCode, resultText(pid1Topology)).toBe(0); + expect(JSON.parse(pid1Topology.stdout)).toEqual({ + argv0: "/opt/openshell/bin/openshell-sandbox", + has_nemoclaw_start: false, + }); + + const runningContainers = await host.command( + "docker", + [ + "ps", + "--filter", + `label=openshell.ai/sandbox-name=${sandboxName}`, + "--format", + "{{.ID}} {{.Names}}", + ], + { + artifactName: "phase-4-gpu-startup-running-containers", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }, + ); + expect(runningContainers.exitCode, resultText(runningContainers)).toBe(0); + const containerRows = runningContainers.stdout + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean); + expect( + containerRows, + `expected one running container, got ${runningContainers.stdout}`, + ).toHaveLength(1); + const [containerId = ""] = containerRows[0].split(/\s+/, 1); + expect(containerId).not.toBe(""); + + const expectedExtraPlaceholderAssignment = `NEMOCLAW_EXTRA_PLACEHOLDER_KEYS=${HERMES_GPU_EXTRA_PLACEHOLDER_KEYS.join(",")}`; + const extraPlaceholderEnv = await host.command( + "docker", + [ + "exec", + "--user", + "0", + containerId, + "python3", + "-c", + String.raw`import os +from pathlib import Path + +expected = ${JSON.stringify(expectedExtraPlaceholderAssignment)}.encode("utf-8") +for proc in Path("/proc").iterdir(): + if not proc.name.isdigit() or int(proc.name) == os.getpid(): + continue + try: + argv = [item.decode("utf-8", "strict") for item in (proc / "cmdline").read_bytes().split(b"\0") if item] + if not any(Path(item).name == "nemoclaw-start" for item in argv): + continue + entries = (proc / "environ").read_bytes().split(b"\0") + except (OSError, UnicodeDecodeError): + continue + if expected in entries: + print(expected.decode("utf-8")) + raise SystemExit(0) +raise SystemExit(1)`, + ], + { + artifactName: "phase-4-gpu-startup-extra-placeholder-env", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }, + ); + expect(extraPlaceholderEnv.exitCode, resultText(extraPlaceholderEnv)).toBe(0); + expect(extraPlaceholderEnv.stdout.trim()).toBe(expectedExtraPlaceholderAssignment); + + const guardWithoutStartupOwner = await sandbox.execShell( + sandboxName, + trustedSandboxShellScript( + "python3 -I /usr/local/lib/nemoclaw/hermes-runtime-config-guard.py ensure-api-key --hermes-dir /sandbox/.hermes", + ), + { + artifactName: "phase-4-gpu-startup-guard-without-startup-owner", + env, + timeoutMs: 30_000, + }, + ); + expect(guardWithoutStartupOwner.exitCode).not.toBe(0); + expect(resultText(guardWithoutStartupOwner)).toContain( + "Hermes runtime config guard refuses mutation under a foreign PID 1", + ); + + const guardFromNonStartupChild = await sandbox.execShell( + sandboxName, + trustedSandboxShellScript( + "python3 -I /usr/local/lib/nemoclaw/hermes-runtime-config-guard.py ensure-api-key --hermes-dir /sandbox/.hermes --startup-owner", + ), + { + artifactName: "phase-4-gpu-startup-owner-from-non-startup-child", + env, + timeoutMs: 30_000, + }, + ); + expect(guardFromNonStartupChild.exitCode).not.toBe(0); + expect(resultText(guardFromNonStartupChild)).toContain( + "Hermes runtime config guard refuses mutation under a foreign PID 1", + ); + + const startupConfig = await sandbox.execShell( + sandboxName, + trustedSandboxShellScript(buildHermesManagedStartupIntegrityScript()), + { + artifactName: "phase-4-gpu-startup-config-and-guard", + env, + timeoutMs: 30_000, + }, + ); + expect(startupConfig.exitCode, resultText(startupConfig)).toBe(0); + expect(startupConfig.stdout.trim()).toBe("OK"); + + const dockerCommandBoundary = await host.command( + "bash", + [ + "-lc", + String.raw`docker inspect "$1" | python3 -c 'import json, sys; config=json.load(sys.stdin)[0]["Config"]; env=dict(item.split("=", 1) for item in (config.get("Env") or []) if "=" in item); command=env.get("OPENSHELL_SANDBOX_COMMAND", ""); tokens=command.split(); print(json.dumps({"cmd": config.get("Cmd"), "entrypoint": config.get("Entrypoint"), "has_openshell_sandbox_command": bool(command), "command_is_sleep_infinity": tokens == ["sleep", "infinity"], "command_ends_with_nemoclaw_start": bool(tokens) and tokens[-1] in ("nemoclaw-start", "/usr/local/bin/nemoclaw-start")}))'`, + "hermes-gpu-command-boundary", + containerId, + ], + { + artifactName: "phase-4-gpu-startup-docker-command-boundary", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }, + ); + expect(dockerCommandBoundary.exitCode, resultText(dockerCommandBoundary)).toBe(0); + const commandBoundary = JSON.parse(dockerCommandBoundary.stdout); + expect([null, []]).toContainEqual(commandBoundary.cmd); + expect(commandBoundary).toMatchObject({ + entrypoint: ["/opt/openshell/bin/openshell-sandbox"], + has_openshell_sandbox_command: true, + }); + if (gpuRoute === "legacy-patch") { + expect(commandBoundary.command_ends_with_nemoclaw_start).toBe(true); + expect(commandBoundary.command_is_sleep_infinity).toBe(false); + } else { + expect(commandBoundary.command_is_sleep_infinity).toBe(true); + expect(commandBoundary.command_ends_with_nemoclaw_start).toBe(false); + } + + const containerState = await host.command( + "docker", + ["inspect", "--format", "{{.State.Status}} {{.RestartCount}}", containerId], + { + artifactName: "phase-4-gpu-startup-container-state", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }, + ); + expect(containerState.exitCode, resultText(containerState)).toBe(0); + expect(containerState.stdout.trim()).toBe("running 0"); + + const allContainers = await host.command( + "docker", + [ + "ps", + "-a", + "--filter", + `label=openshell.ai/sandbox-name=${sandboxName}`, + "--format", + "{{.Names}}", + ], + { + artifactName: "phase-4-gpu-startup-all-containers", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }, + ); + expect(allContainers.exitCode, resultText(allContainers)).toBe(0); + expect( + allContainers.stdout + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean), + ).toHaveLength(1); +} diff --git a/test/e2e/live/hermes-gpu-startup.test.ts b/test/e2e/live/hermes-gpu-startup.test.ts new file mode 100644 index 00000000000..b8c43d8bb69 --- /dev/null +++ b/test/e2e/live/hermes-gpu-startup.test.ts @@ -0,0 +1,315 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import path from "node:path"; + +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { + type HostCliClient, + resultText, + type SandboxClient, + validateSandboxName, +} from "../fixtures/clients/index.ts"; +import { expect, test } from "../fixtures/e2e-test.ts"; +import { startFakeOpenAiCompatibleServer } from "../fixtures/fake-openai-compatible.ts"; +import { shouldRunLiveE2E } from "../fixtures/live-project-gate.ts"; +import { + assertHermesGpuStartupProof, + HERMES_GPU_EXTRA_PLACEHOLDER_KEYS, +} from "./hermes-gpu-startup-proof.ts"; + +const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); +const GATEWAY_CLEANUP_MODULE = path.join(REPO_ROOT, "dist/lib/actions/sandbox/destroy-gateway.js"); +// Clean runners do not have OpenShell until install.sh runs. Tool absence is +// accepted here only because the bind probe below and the later no-reuse log +// assertions still reject an orphaned runtime or stale registration. +const GATEWAY_CLEANUP_SCRIPT = String.raw` +command -v openshell >/dev/null 2>&1 || exit 0 +exec node -e 'const { cleanupGatewayAfterLastSandbox } = require(process.argv[1]); cleanupGatewayAfterLastSandbox(process.argv[2]);' "$@" +`; +const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-hermes-gpu-startup"; +const FAKE_API_KEY = "e2e-hermes-gpu-startup-key"; +const FAKE_MODEL = "test-model"; +const EXTRA_PLACEHOLDER_TOKEN_A = "e2e-hermes-gpu-extra-telegram-token"; +const EXTRA_PLACEHOLDER_TOKEN_B = "e2e-hermes-gpu-extra-slack-token"; +const LIVE_TIMEOUT_MS = 70 * 60_000; +const FORCE_LEGACY_GPU_PATCH = process.env.NEMOCLAW_DOCKER_GPU_PATCH === "1"; +const GPU_ROUTE = FORCE_LEGACY_GPU_PATCH ? "legacy-patch" : "native-openshell"; +validateSandboxName(SANDBOX_NAME); + +function commandEnv(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { + return { + ...buildAvailabilityProbeEnv(), + ...extra, + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + NEMOCLAW_AGENT: "hermes", + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_RECREATE_SANDBOX: "1", + NEMOCLAW_SANDBOX_GPU: "1", + NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, + NEMOCLAW_ONBOARD_VALIDATION_TIMEOUT_SECONDS: "60", + ...(FORCE_LEGACY_GPU_PATCH ? { NEMOCLAW_DOCKER_GPU_PATCH: "1" } : {}), + }; +} + +async function bestEffort(run: () => Promise): Promise { + try { + await run(); + } catch { + // Cleanup and failure diagnostics must not mask the primary live-test result. + } +} + +async function cleanupHermes( + host: HostCliClient, + sandbox: SandboxClient, + label: string, +): Promise { + await bestEffort(() => + host.nemoclaw([SANDBOX_NAME, "destroy", "--yes", "--cleanup-gateway"], { + artifactName: `${label}-nemoclaw-destroy`, + env: commandEnv(), + timeoutMs: 120_000, + }), + ); + await bestEffort(() => + sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { + artifactName: `${label}-openshell-sandbox-delete`, + env: commandEnv(), + timeoutMs: 60_000, + }), + ); + const runtimeCleanup = await host.command( + "bash", + ["-c", GATEWAY_CLEANUP_SCRIPT, "gateway-runtime-cleanup", GATEWAY_CLEANUP_MODULE, "nemoclaw"], + { + artifactName: `${label}-gateway-runtime-cleanup`, + env: commandEnv(), + timeoutMs: 60_000, + }, + ); + expect( + runtimeCleanup.exitCode, + `owned gateway runtime cleanup failed: ${resultText(runtimeCleanup)}`, + ).toBe(0); + await host + .cleanupGatewayRegistration("nemoclaw", { + artifactName: `${label}-openshell-gateway`, + env: commandEnv(), + timeoutMs: 60_000, + }) + .catch((error: unknown) => { + expect(error).toMatchObject({ message: "spawn openshell ENOENT" }); + }); + const gatewayPort = process.env.NEMOCLAW_GATEWAY_PORT ?? "8080"; + const portAvailable = await host.command( + "node", + [ + "-e", + 'const net=require("node:net"); const server=net.createServer(); server.once("error", error => { console.error(error.code || "bind failed"); process.exit(1); }); server.listen(Number(process.argv[1]), "127.0.0.1", () => server.close(error => { if (error) { console.error(error.message); process.exit(1); } console.log("available"); }));', + gatewayPort, + ], + { + artifactName: `${label}-gateway-port-available`, + env: commandEnv(), + timeoutMs: 30_000, + }, + ); + expect( + portAvailable.exitCode, + `gateway port ${gatewayPort} remains occupied after cleanup: ${resultText(portAvailable)}`, + ).toBe(0); +} + +async function captureFailedGpuContainer( + host: HostCliClient, + preRollbackDiagnosticsDir: string, +): Promise { + const sandboxFilter = `label=openshell.ai/sandbox-name=${SANDBOX_NAME}`; + const script = String.raw`set -u +diagnostics_dir="$2" +if [ -n "$diagnostics_dir" ] && [ -d "$diagnostics_dir" ]; then + printf '%s\n' "== pre-rollback diagnostics $diagnostics_dir ==" + for name in summary.txt patched-container-state.json docker-inspect.json docker-network-summary.txt docker-top.txt docker-logs.txt openshell-sandbox-get.txt openshell-sandbox-list.txt openshell-logs.txt; do + file="$diagnostics_dir/$name" + if [ -f "$file" ]; then + printf '%s\n' "== $name ==" + if [ "$name" = openshell-logs.txt ]; then + tail -n 800 "$file" + else + sed -n '1,800p' "$file" + fi + fi + done +else + printf '%s\n' "pre-rollback diagnostics directory unavailable: $diagnostics_dir" +fi +ids="$(docker ps -aq --filter "$1")" +if [ -z "$ids" ]; then + printf '%s\n' "no Docker container found for $1" + exit 0 +fi +for id in $ids; do + printf '%s\n' "== container $id inspect ==" + docker inspect --format '{{json .Name}} {{json .Config.User}} {{json .Config.Entrypoint}} {{json .Config.Cmd}} {{json .State}} {{json .HostConfig.RestartPolicy}}' "$id" 2>&1 || true + printf '%s\n' "== container $id top ==" + docker top "$id" -eo user,pid,ppid,stat,args 2>&1 || true + printf '%s\n' "== container $id logs ==" + docker logs --tail 300 "$id" 2>&1 || true +done`; + await bestEffort(() => + host.command( + "bash", + ["-lc", script, "hermes-gpu-failure-diagnostics", sandboxFilter, preRollbackDiagnosticsDir], + { + artifactName: "phase-2-hermes-gpu-startup-failure-diagnostics", + env: buildAvailabilityProbeEnv(), + redactionValues: [FAKE_API_KEY, EXTRA_PLACEHOLDER_TOKEN_A, EXTRA_PLACEHOLDER_TOKEN_B], + timeoutMs: 30_000, + }, + ), + ); +} + +test.skipIf(!shouldRunLiveE2E())( + "hermes-gpu-startup: selected OpenShell GPU route reaches stable Ready state", + { timeout: LIVE_TIMEOUT_MS }, + async ({ artifacts, cleanup, host, sandbox }) => { + await artifacts.writeJson("target.json", { + id: "hermes-gpu-startup", + runner: "vitest", + boundary: "install.sh --non-interactive --fresh + Hermes GPU-supervised startup", + sandboxName: SANDBOX_NAME, + inference: "hermetic fake OpenAI-compatible endpoint", + gpuRoute: GPU_ROUTE, + }); + + await cleanupHermes(host, sandbox, "pre-cleanup"); + + const dockerInfo = await host.command("docker", ["info"], { + artifactName: "phase-1-docker-info", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + expect(dockerInfo.exitCode, resultText(dockerInfo)).toBe(0); + + const hostAddressProbe = await host.command( + "bash", + [ + "-lc", + [ + 'ip_addr="$(ip route get 1.1.1.1 2>/dev/null | awk \'{for (i=1;i<=NF;i++) if ($i=="src") {print $(i+1); exit}}\')"', + 'test -n "$ip_addr" || ip_addr="$(hostname -I 2>/dev/null | awk \'{print $1}\')"', + 'test -n "$ip_addr"', + 'printf "%s\\n" "$ip_addr"', + ].join("\n"), + ], + { + artifactName: "phase-1-sandbox-reachable-host-address", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }, + ); + expect(hostAddressProbe.exitCode, resultText(hostAddressProbe)).toBe(0); + const hostAddress = hostAddressProbe.stdout.trim().split(/\s+/)[0]; + expect(hostAddress).toBeTruthy(); + + const fake = await startFakeOpenAiCompatibleServer({ + apiKey: FAKE_API_KEY, + forbiddenMarkers: [EXTRA_PLACEHOLDER_TOKEN_A, EXTRA_PLACEHOLDER_TOKEN_B], + host: "0.0.0.0", + model: FAKE_MODEL, + publicHost: hostAddress, + requireAuth: true, + }); + cleanup.add("close fake OpenAI-compatible endpoint", async () => { + await artifacts.writeJson("fake-openai-compatible-requests.json", fake.requests()); + await fake.close(); + }); + cleanup.add(`destroy Hermes sandbox ${SANDBOX_NAME}`, async () => { + await cleanupHermes(host, sandbox, "cleanup"); + }); + await artifacts.writeJson("fake-openai-compatible.json", { + baseUrl: fake.baseUrl, + model: FAKE_MODEL, + publicHost: hostAddress, + }); + + const env = commandEnv({ + COMPATIBLE_API_KEY: FAKE_API_KEY, + NEMOCLAW_COMPAT_MODEL: FAKE_MODEL, + NEMOCLAW_ENDPOINT_URL: fake.baseUrl, + NEMOCLAW_MODEL: FAKE_MODEL, + NEMOCLAW_EXTRA_PLACEHOLDER_KEYS: HERMES_GPU_EXTRA_PLACEHOLDER_KEYS.join(","), + NEMOCLAW_POLICY_MODE: "suggested", + NEMOCLAW_PREFERRED_API: "openai-completions", + NEMOCLAW_PROVIDER: "custom", + [HERMES_GPU_EXTRA_PLACEHOLDER_KEYS[0]]: EXTRA_PLACEHOLDER_TOKEN_A, + [HERMES_GPU_EXTRA_PLACEHOLDER_KEYS[1]]: EXTRA_PLACEHOLDER_TOKEN_B, + }); + const install = await host.command("bash", ["install.sh", "--non-interactive", "--fresh"], { + artifactName: "phase-2-install-hermes-gpu-startup", + cwd: REPO_ROOT, + env, + redactionValues: [FAKE_API_KEY, EXTRA_PLACEHOLDER_TOKEN_A, EXTRA_PLACEHOLDER_TOKEN_B], + timeoutMs: 60 * 60_000, + }); + const preRollbackDiagnosticsDir = + resultText(install).match(/Pre-rollback diagnostics saved:\s*(\S+)/)?.[1] ?? ""; + await (install.exitCode !== 0 + ? captureFailedGpuContainer(host, preRollbackDiagnosticsDir) + : Promise.resolve()); + expect(install.exitCode, resultText(install)).toBe(0); + + const status = await host.command("nemoclaw", [SANDBOX_NAME, "status"], { + artifactName: "phase-3-nemoclaw-status", + env: commandEnv(), + timeoutMs: 60_000, + }); + expect(status.exitCode, resultText(status)).toBe(0); + + await assertHermesGpuStartupProof({ + env: commandEnv(), + gpuRoute: GPU_ROUTE, + host, + install, + sandbox, + sandboxName: SANDBOX_NAME, + status, + }); + + const fakeRequests = fake.requests(); + const inferencePosts = fakeRequests.filter( + (request) => + request.method === "POST" && + ["/v1/chat/completions", "/chat/completions", "/v1/responses", "/responses"].includes( + request.path, + ), + ); + expect( + inferencePosts.length, + `expected authenticated fake inference POST, got ${JSON.stringify(fakeRequests)}`, + ).toBeGreaterThan(0); + expect(inferencePosts.filter((request) => request.auth !== "ok")).toEqual([]); + expect(inferencePosts.filter((request) => (request.forbiddenMarkerMatches ?? 0) > 0)).toEqual( + [], + ); + expect(JSON.stringify(fakeRequests)).not.toContain(EXTRA_PLACEHOLDER_TOKEN_A); + expect(JSON.stringify(fakeRequests)).not.toContain(EXTRA_PLACEHOLDER_TOKEN_B); + + await artifacts.writeJson("target-result.json", { + id: "hermes-gpu-startup", + assertions: { + selectedGpuRouteVerified: true, + openshellReady: true, + sandboxCudaVerified: true, + extraPlaceholderCommandRoundTripValid: true, + stableSingleContainer: true, + startupConfigHashesValid: true, + supervisorTopologyValid: true, + authenticatedInferenceRequestVerified: true, + placeholderTokensAbsentFromInference: true, + }, + }); + }, +); diff --git a/test/e2e/support/hermes-gpu-startup-integrity.test.ts b/test/e2e/support/hermes-gpu-startup-integrity.test.ts new file mode 100644 index 00000000000..37ecf2db9de --- /dev/null +++ b/test/e2e/support/hermes-gpu-startup-integrity.test.ts @@ -0,0 +1,190 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; + +import { buildHermesManagedStartupIntegrityScript } from "../live/hermes-gpu-startup-integrity.ts"; + +interface IntegrityFixture { + root: string; + configPath: string; + envPath: string; + strictHashPath: string; + compatHashPath: string; + startupLogPath: string; + baseEnv: string; + generatedKey: string; +} + +const roots: string[] = []; + +afterEach(() => { + for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); +}); + +function digest(text: string): string { + return createHash("sha256").update(text).digest("hex"); +} + +function writeHash( + hashPath: string, + configPath: string, + config: string, + envPath: string, + env: string, +): void { + fs.writeFileSync(hashPath, `${digest(config)} ${configPath}\n${digest(env)} ${envPath}\n`); +} + +function createFixture(): IntegrityFixture { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-gpu-integrity-")); + roots.push(root); + const hermesDir = path.join(root, ".hermes"); + const fixture: IntegrityFixture = { + root, + configPath: path.join(hermesDir, "config.yaml"), + envPath: path.join(hermesDir, ".env"), + strictHashPath: path.join(root, "hermes.config-hash"), + compatHashPath: path.join(hermesDir, ".config-hash"), + startupLogPath: path.join(root, "nemoclaw-start.log"), + baseEnv: "API_SERVER_PORT=18642\nSAFE_SETTING=trusted\n", + generatedKey: "a".repeat(64), + }; + const config = "model:\n default: test-model\n"; + const liveEnv = `${fixture.baseEnv}API_SERVER_KEY=${fixture.generatedKey}\n`; + fs.mkdirSync(hermesDir, { recursive: true }); + fs.writeFileSync(fixture.configPath, config); + fs.writeFileSync(fixture.envPath, liveEnv); + writeHash(fixture.strictHashPath, fixture.configPath, config, fixture.envPath, fixture.baseEnv); + writeHash(fixture.compatHashPath, fixture.configPath, config, fixture.envPath, liveEnv); + fs.chmodSync(fixture.strictHashPath, 0o444); + fs.writeFileSync(fixture.startupLogPath, "[config] managed startup complete\n"); + return fixture; +} + +function runProof(fixture: IntegrityFixture, extraEnv: NodeJS.ProcessEnv = {}) { + const strictMetadata = fs.statSync(fixture.strictHashPath); + return spawnSync( + "/bin/bash", + [ + "-c", + buildHermesManagedStartupIntegrityScript({ + configPath: fixture.configPath, + envPath: fixture.envPath, + strictHashPath: fixture.strictHashPath, + compatHashPath: fixture.compatHashPath, + startupLogPath: fixture.startupLogPath, + strictHashUid: strictMetadata.uid, + strictHashGid: strictMetadata.gid, + }), + ], + { + encoding: "utf-8", + timeout: 5000, + env: { + LANG: "C", + LC_ALL: "C", + ...extraEnv, + }, + }, + ); +} + +describe("Hermes managed startup integrity proof", () => { + it("accepts a current compatibility hash and one generated API key beyond the strict base", () => { + const fixture = createFixture(); + const rawStrictCheck = spawnSync("sha256sum", ["-c", fixture.strictHashPath, "--status"], { + encoding: "utf-8", + timeout: 5000, + }); + + expect(rawStrictCheck.error).toBeUndefined(); + expect(rawStrictCheck.status).not.toBeNull(); + expect(rawStrictCheck.status).not.toBe(0); + const proof = runProof(fixture); + expect(proof.status, proof.stderr).toBe(0); + expect(proof.stdout).toBe("OK\n"); + }); + + it("rejects non-key environment drift even when the compatibility hash accepts it", () => { + const fixture = createFixture(); + const config = fs.readFileSync(fixture.configPath, "utf-8"); + const driftedEnv = `${fs.readFileSync(fixture.envPath, "utf-8")}UNEXPECTED=drift\n`; + fs.writeFileSync(fixture.envPath, driftedEnv); + writeHash(fixture.compatHashPath, fixture.configPath, config, fixture.envPath, driftedEnv); + + const proof = runProof(fixture); + expect(proof.status).not.toBe(0); + expect(proof.stderr).toContain( + "Hermes environment differs from the strict startup base beyond the generated API key", + ); + }); + + it("rejects duplicate generated keys", () => { + const duplicate = createFixture(); + fs.appendFileSync(duplicate.envPath, `API_SERVER_KEY=${"b".repeat(64)}\n`); + const proof = runProof(duplicate); + expect(proof.status).not.toBe(0); + expect(proof.stderr).toContain("Hermes environment contains an unexpected API key assignment"); + }); + + it("rejects a stale compatibility hash", () => { + const stale = createFixture(); + const config = fs.readFileSync(stale.configPath, "utf-8"); + writeHash(stale.compatHashPath, stale.configPath, config, stale.envPath, stale.baseEnv); + const proof = runProof(stale); + expect(proof.status).not.toBe(0); + expect(proof.stderr).toContain( + "Hermes compatibility hash does not match the current environment", + ); + }); + + it("rejects a noncanonical API key assignment even when it belongs to the strict base", () => { + const fixture = createFixture(); + const config = fs.readFileSync(fixture.configPath, "utf-8"); + const pollutedBase = `${fixture.baseEnv}export API_SERVER_KEY=${"b".repeat(64)}\n`; + const liveEnv = `${pollutedBase}API_SERVER_KEY=${fixture.generatedKey}\n`; + fs.writeFileSync(fixture.envPath, liveEnv); + fs.chmodSync(fixture.strictHashPath, 0o644); + writeHash(fixture.strictHashPath, fixture.configPath, config, fixture.envPath, pollutedBase); + fs.chmodSync(fixture.strictHashPath, 0o444); + writeHash(fixture.compatHashPath, fixture.configPath, config, fixture.envPath, liveEnv); + + const proof = runProof(fixture); + expect(proof.status).not.toBe(0); + expect(proof.stderr).toContain("Hermes environment contains an unexpected API key assignment"); + }); + + it("ignores ambient Python module shadowing", () => { + const fixture = createFixture(); + const shadowDir = path.join(fixture.root, "python-shadow"); + fs.mkdirSync(shadowDir); + fs.writeFileSync(path.join(shadowDir, "hashlib.py"), 'raise SystemExit("shadowed hashlib")\n'); + + const proof = runProof(fixture, { PYTHONPATH: shadowDir }); + expect(proof.status, proof.stderr).toBe(0); + expect(proof.stdout).toBe("OK\n"); + }); + + it("rejects a writable strict anchor and startup guard refusal", () => { + const writable = createFixture(); + fs.chmodSync(writable.strictHashPath, 0o644); + let proof = runProof(writable); + expect(proof.status).not.toBe(0); + expect(proof.stderr).toContain("Hermes strict hash is not the expected read-only owner anchor"); + + const refused = createFixture(); + fs.writeFileSync( + refused.startupLogPath, + "Hermes runtime config guard refuses mutation under a foreign PID 1\n", + ); + proof = runProof(refused); + expect(proof.status).not.toBe(0); + expect(proof.stderr).toContain("Hermes startup log contains a runtime config guard refusal"); + }); +}); diff --git a/test/e2e/support/hermes-workflow-boundary.test.ts b/test/e2e/support/hermes-workflow-boundary.test.ts index 833c259ce2a..2985a8f55e6 100644 --- a/test/e2e/support/hermes-workflow-boundary.test.ts +++ b/test/e2e/support/hermes-workflow-boundary.test.ts @@ -10,15 +10,52 @@ import YAML from "yaml"; import { validateE2eWorkflowBoundary } from "../../../tools/e2e/workflow-boundary.mts"; describe("Hermes E2E workflow boundary", () => { - it("rejects pinned Hermes Vitest model overrides", () => { + it("rejects hosted Hermes model and hermetic GPU-startup boundary drift", () => { const workflow = YAML.parse(fs.readFileSync(".github/workflows/e2e.yaml", "utf8")); workflow.jobs["hermes-e2e"].env.NEMOCLAW_MODEL = "minimaxai/minimax-m2.7"; + const gpuJob = workflow.jobs["hermes-gpu-startup"]; + gpuJob["runs-on"] = "ubuntu-latest"; + gpuJob.if = "${{ always() }}"; + gpuJob.env.NEMOCLAW_DOCKER_GPU_PATCH = "1"; + gpuJob.env.NEMOCLAW_E2E_USE_HOSTED_INFERENCE = "1"; + gpuJob.env.UNRELATED_SECRET = "${{ github.ref == 'refs/heads/main' && secrets.FOO || '' }}"; + const gpuRun = gpuJob.steps.find( + (step: { name?: string }) => step.name === "Run Hermes GPU startup live Vitest test", + ); + gpuRun.env = { + COMPATIBLE_API_KEY: "${{ secrets.NVIDIA_INFERENCE_API_KEY }}", + NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1", + NVIDIA_API_KEY: "${{ secrets.NVIDIA_INFERENCE_API_KEY }}", + NVIDIA_INFERENCE_API_KEY: "${{ secrets.NVIDIA_INFERENCE_API_KEY }}", + }; + gpuRun.run = "npx vitest run --project e2e-live test/e2e/live/hermes-e2e.test.ts"; + gpuJob.steps.push({ + name: "Unexpected hosted test", + run: "npx vitest run --project e2e-live test/e2e/live/hermes-e2e.test.ts", + with: { token: "${{ github.ref == 'refs/heads/main' && secrets.FOO || '' }}" }, + }); const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-hermes-workflow-")); const workflowPath = path.join(tmp, "workflow.yaml"); try { fs.writeFileSync(workflowPath, YAML.stringify(workflow)); - expect(validateE2eWorkflowBoundary(workflowPath)).toContain( - "hermes-e2e job must use the shared hosted-compatible model default", + expect(validateE2eWorkflowBoundary(workflowPath)).toEqual( + expect.arrayContaining([ + "hermes-e2e job must use the shared hosted-compatible model default", + "hermes-gpu-startup job must run on the native RTX PRO 6000 GPU runner", + "hermes-gpu-startup job must remain explicit-only behind generate-matrix", + "hermes-gpu-startup job must leave NEMOCLAW_DOCKER_GPU_PATCH unset to exercise auto routing", + "hermes-gpu-startup job env must not expose NEMOCLAW_E2E_USE_HOSTED_INFERENCE", + "hermes-gpu-startup job env must not consume repository secrets", + "hermes-gpu-startup step 'Run Hermes GPU startup live Vitest test' must not expose COMPATIBLE_API_KEY", + "hermes-gpu-startup step 'Run Hermes GPU startup live Vitest test' must not expose NEMOCLAW_E2E_USE_HOSTED_INFERENCE", + "hermes-gpu-startup step 'Run Hermes GPU startup live Vitest test' must not expose NVIDIA_API_KEY", + "hermes-gpu-startup step 'Run Hermes GPU startup live Vitest test' must not expose NVIDIA_INFERENCE_API_KEY", + "hermes-gpu-startup step 'Run Hermes GPU startup live Vitest test' must not consume repository secrets", + "hermes-gpu-startup step must run the dedicated Hermes GPU startup test", + "hermes-gpu-startup step 'Run Hermes GPU startup live Vitest test' must not run the hosted Hermes E2E test", + "hermes-gpu-startup step 'Unexpected hosted test' must not run the hosted Hermes E2E test", + "hermes-gpu-startup step 'Unexpected hosted test' must not consume repository secrets", + ]), ); } finally { fs.rmSync(tmp, { recursive: true, force: true }); diff --git a/test/e2e/support/hosted-inference.test.ts b/test/e2e/support/hosted-inference.test.ts index f7a727f4146..1cb5d67f691 100644 --- a/test/e2e/support/hosted-inference.test.ts +++ b/test/e2e/support/hosted-inference.test.ts @@ -9,13 +9,13 @@ import { describe, expect, it } from "vitest"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import { ProviderClient, trustedProviderEndpoint } from "../fixtures/clients/provider.ts"; +import { startFakeOpenAiCompatibleServer } from "../fixtures/fake-openai-compatible.ts"; +import { requireHostedInferenceConfig } from "../fixtures/hosted-inference.ts"; import type { ShellProbeResult, ShellProbeRunOptions, TrustedShellCommand, } from "../fixtures/shell-probe.ts"; -import { startFakeOpenAiCompatibleServer } from "../fixtures/fake-openai-compatible.ts"; -import { requireHostedInferenceConfig } from "../fixtures/hosted-inference.ts"; const COMPAT_HELPER = path.join( import.meta.dirname, @@ -371,6 +371,7 @@ describe("hosted inference E2E config", () => { const fake = await startFakeOpenAiCompatibleServer({ apiKey: "fake-compatible-key", chatContent: "CHAT_OK", + forbiddenMarkers: ["FORBIDDEN_REQUEST_MARKER"], model: "nvidia/nvidia/fake-model", requireAuth: true, responseText: "RESP_OK", @@ -384,7 +385,10 @@ describe("hosted inference E2E config", () => { }); const unauthenticatedChat = await fetch(`${fake.baseUrl}/chat/completions`, { - body: JSON.stringify({ messages: [], model: "nvidia/nvidia/fake-model" }), + body: JSON.stringify({ + messages: [{ content: "FORBIDDEN_REQUEST_MARKER", role: "user" }], + model: "nvidia/nvidia/fake-model", + }), headers: { "Content-Type": "application/json" }, method: "POST", }); @@ -419,12 +423,18 @@ describe("hosted inference E2E config", () => { expect(responsesText).toContain("event: response.output_text.delta"); expect(responsesText).toContain('data: {"delta":"RESP_OK"}'); - expect(fake.requests()).toEqual( + const requests = fake.requests(); + expect(requests).toEqual( expect.arrayContaining([ expect.objectContaining({ method: "GET", path: "/v1/models" }), - expect.objectContaining({ auth: "missing", path: "/v1/chat/completions" }), + expect.objectContaining({ + auth: "missing", + forbiddenMarkerMatches: 1, + path: "/v1/chat/completions", + }), expect.objectContaining({ auth: "ok", + forbiddenMarkerMatches: 0, model: "nvidia/nvidia/fake-model", path: "/v1/chat/completions", stream: false, @@ -432,6 +442,10 @@ describe("hosted inference E2E config", () => { expect.objectContaining({ auth: "ok", path: "/v1/responses", stream: true }), ]), ); + expect( + requests.reduce((total, request) => total + (request.forbiddenMarkerMatches ?? 0), 0), + ).toBe(1); + expect(JSON.stringify(requests)).not.toContain("FORBIDDEN_REQUEST_MARKER"); } finally { await fake.close(); } diff --git a/test/e2e/support/jetson-workflow-boundary.test.ts b/test/e2e/support/jetson-workflow-boundary.test.ts index edd79b120c1..906e90bbe6f 100644 --- a/test/e2e/support/jetson-workflow-boundary.test.ts +++ b/test/e2e/support/jetson-workflow-boundary.test.ts @@ -23,7 +23,7 @@ describe("Jetson nvmap GPU E2E workflow boundary", () => { expect(inventory.allowedJobs).toContain("jetson-nvmap-gpu"); expect(inventory.explicitOnlyJobs).toContain("jetson-nvmap-gpu"); expect(formatFreeStandingJobsInventoryForShell(inventory)).toContain( - "explicit_only_jobs_csv=openshell-gateway-auth-contract,sandbox-rlimits-connect,jetson-nvmap-gpu", + "explicit_only_jobs_csv=openshell-gateway-auth-contract,hermes-gpu-startup,sandbox-rlimits-connect,jetson-nvmap-gpu", ); expect(inventory.targetToJob.get("jetson-nvmap-gpu")).toBe("jetson-nvmap-gpu"); expect(evaluateE2eWorkflowDispatchSelectors({}).selectedFreeStandingJobs).not.toContain( diff --git a/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts b/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts index 8124194d562..86eb4b818c4 100644 --- a/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts +++ b/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts @@ -77,7 +77,7 @@ function validateActionMutation(mutate: (action: MutableAction) => void): string } describe("upload-e2e-artifacts workflow boundary", () => { - it("binds one canonical uploader to all 73 E2E execution jobs", () => { + it("binds one canonical uploader to all 74 E2E execution jobs", () => { expect(validateUploadE2eArtifactsAction()).toEqual([]); expect(validateUploadE2eArtifactsInvocations(readWorkflow())).toEqual([]); }); @@ -175,8 +175,8 @@ describe("upload-e2e-artifacts workflow boundary", () => { expect(validateUploadE2eArtifactsInvocations(workflow)).toEqual( expect.arrayContaining([ - "upload-e2e-artifacts must cover exactly 73 live and E2E_JOB execution jobs", - "upload-e2e-artifacts must keep exactly 64 default callers", + "upload-e2e-artifacts must cover exactly 74 live and E2E_JOB execution jobs", + "upload-e2e-artifacts must keep exactly 65 default callers", ]), ); }); diff --git a/test/hermes-gateway-wrapper.test.ts b/test/hermes-gateway-wrapper.test.ts index 322637810d3..0a085b6f184 100644 --- a/test/hermes-gateway-wrapper.test.ts +++ b/test/hermes-gateway-wrapper.test.ts @@ -182,6 +182,9 @@ describe.skipIf(!canRun)("agents/hermes/hermes-wrapper.py", () => { SLACK_BOT_TOKEN: "xoxb-OPENSHELL-RESOLVE-ENV-SLACK_BOT_TOKEN", TELEGRAM_BOT_TOKEN: "openshell:resolve:env:TELEGRAM_BOT_TOKEN", OPENCLAW_GATEWAY_TOKEN: "raw-gateway-token", + OPENSHELL_TLS_CA: "/etc/openshell/tls/client/ca.crt", + OPENSHELL_TLS_CERT: "/etc/openshell/tls/client/tls.crt", + OPENSHELL_TLS_KEY: "/etc/openshell/tls/client/tls.key", }); expect(run.status).toBe(0); @@ -190,6 +193,17 @@ describe.skipIf(!canRun)("agents/hermes/hermes-wrapper.py", () => { expect(run.realArgs).toBe("gateway run"); }); + it("refuses `gateway` with a noncanonical OpenShell TLS key path", () => { + const value = "/tmp/not-openshell/tls.key"; + const run = runWrapper(["gateway", "run"], { OPENSHELL_TLS_KEY: value }); + + expect(run.status).toBe(1); + expect(run.stderr).toContain("process environment"); + expect(run.stderr).toContain("OPENSHELL_TLS_KEY"); + expect(run.stderr).not.toContain(value); + expect(run.realInvoked).toBe(false); + }); + it("passes non-gateway subcommands straight through, even with raw secrets present", () => { // The guard scopes to gateway startup; other subcommands must not be blocked. const run = runWrapper(["dashboard"], { SLACK_BOT_TOKEN: "xoxb-real-1234567890" }); diff --git a/test/hermes-openshell-runtime-env-boundary.test.ts b/test/hermes-openshell-runtime-env-boundary.test.ts new file mode 100644 index 00000000000..f463df2ec5e --- /dev/null +++ b/test/hermes-openshell-runtime-env-boundary.test.ts @@ -0,0 +1,107 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; + +const VALIDATOR = path.join( + import.meta.dirname, + "..", + "agents", + "hermes", + "validate-env-secret-boundary.py", +); +const CANONICAL_TLS_KEY_PATH = "/etc/openshell/tls/client/tls.key"; + +function runRuntimeEnvValidator(envOverrides: Record) { + return spawnSync("python3", [VALIDATOR, "runtime-env"], { + encoding: "utf-8", + timeout: 5000, + env: { + HOME: os.tmpdir(), + PATH: process.env.PATH ?? "", + ...envOverrides, + }, + }); +} + +function runEnvFileValidator(envFileContent: string) { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-openshell-boundary-")); + const envFile = path.join(tmpDir, ".env"); + fs.writeFileSync(envFile, envFileContent); + + try { + return spawnSync("python3", [VALIDATOR, "env-file", envFile], { + encoding: "utf-8", + timeout: 5000, + env: { HOME: os.tmpdir(), PATH: process.env.PATH ?? "" }, + }); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } +} + +describe("Hermes OpenShell runtime environment boundary", () => { + it("accepts the driver-owned OpenShell transport environment", () => { + const result = runRuntimeEnvValidator({ + OPENSHELL_ENDPOINT: "https://gateway.openshell.internal:8080", + OPENSHELL_LOG_LEVEL: "info", + OPENSHELL_SANDBOX: "hermes-gpu", + OPENSHELL_SANDBOX_ID: "sandbox-id", + OPENSHELL_TLS_CA: "/etc/openshell/tls/client/ca.crt", + OPENSHELL_TLS_CERT: "/etc/openshell/tls/client/tls.crt", + OPENSHELL_TLS_KEY: CANONICAL_TLS_KEY_PATH, + }); + + expect(result.status, result.stderr).toBe(0); + expect(result.stderr).toBe(""); + }); + + it("rejects noncanonical OpenShell TLS key values without printing them", () => { + const pemValue = [ + "-----BEGIN PRIVATE ", + "KEY-----\nraw-private-key\n-----END PRIVATE ", + "KEY-----", + ].join(""); + for (const value of [ + "raw-private-key", + pemValue, + "relative/tls.key", + "/tmp/tls.key", + `${CANONICAL_TLS_KEY_PATH}.bak`, + ]) { + const result = runRuntimeEnvValidator({ OPENSHELL_TLS_KEY: value }); + + expect(result.status, `${value}: ${result.stderr}`).toBe(1); + expect(result.stderr).toContain("process environment"); + expect(result.stderr).toContain("OPENSHELL_TLS_KEY"); + expect(result.stderr).not.toContain(value); + } + }); + + it("keeps the canonical OpenShell TLS key path out of mutable Hermes .env", () => { + const result = runEnvFileValidator(`OPENSHELL_TLS_KEY=${CANONICAL_TLS_KEY_PATH}\n`); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("OPENSHELL_TLS_KEY (line 1)"); + expect(result.stderr).not.toContain(CANONICAL_TLS_KEY_PATH); + }); + + it("continues to reject OpenShell supervisor identity credentials", () => { + const values = { + OPENSHELL_K8S_SA_TOKEN_FILE: "/var/run/secrets/openshell/token", + OPENSHELL_SANDBOX_TOKEN: "raw-sandbox-token", + OPENSHELL_SANDBOX_TOKEN_FILE: "/etc/openshell/token", + }; + const result = runRuntimeEnvValidator(values); + + expect(result.status).toBe(1); + for (const [key, value] of Object.entries(values)) { + expect(result.stderr).toContain(key); + expect(result.stderr).not.toContain(value); + } + }); +}); diff --git a/test/hermes-runtime-config-guard.test.ts b/test/hermes-runtime-config-guard.test.ts index 6d0000c2c79..8a42410c8e2 100644 --- a/test/hermes-runtime-config-guard.test.ts +++ b/test/hermes-runtime-config-guard.test.ts @@ -835,6 +835,45 @@ with tempfile.TemporaryDirectory() as tmp: }); describe("Hermes startup readiness lease", () => { + it("rejects a supervisor argv polluted with the appended startup command (#6110)", () => { + const result = runPythonHarness(`${loadGuardModule} +import json + +polluted_supervisor = ( + b"/opt/openshell/bin/openshell-sandbox\\0" + b"env\\0CHAT_UI_URL=http://127.0.0.1:18789\\0nemoclaw-start\\0" +) +guard.__file__ = guard.INSTALLED_RUNTIME_CONFIG_GUARD +guard._open_proc_root = lambda: 101 +guard._open_proc_pid = lambda _root, _pid: 102 +guard._read_proc_pid_file = lambda _fd, _name, _display: polluted_supervisor +guard.os.close = lambda _fd: None +guard.os.getppid = lambda: 1 +guard.pwd.getpwnam = lambda _name: type("User", (), {"pw_uid": 1000})() +guard._startup_ready_marker_absent = lambda: True +guard._openshell_supervised_nonroot_start_is_live = lambda *_args: False + +classification = guard._pid1_is_nemoclaw_start() + +try: + guard._validate_action_readiness("ensure-api-key", True) + error = None +except guard.UnsafePathError as exc: + error = str(exc) + +print(json.dumps({ + "classification": classification, + "error": error, +})) +`); + + expect(result.status, result.stderr).toBe(0); + expect(JSON.parse(result.stdout)).toEqual({ + classification: false, + error: "Hermes runtime config guard refuses mutation under a foreign PID 1", + }); + }); + it("fails closed under foreign PID 1 only for the installed guard entrypoint", () => { const result = runPythonHarness(`${loadGuardModule} import json diff --git a/tools/e2e/hermes-gpu-startup-workflow-boundary.mts b/tools/e2e/hermes-gpu-startup-workflow-boundary.mts new file mode 100644 index 00000000000..d19fbebd0ea --- /dev/null +++ b/tools/e2e/hermes-gpu-startup-workflow-boundary.mts @@ -0,0 +1,124 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +import YAML from "yaml"; + +const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); +const DEFAULT_WORKFLOW_PATH = join(REPO_ROOT, ".github", "workflows", "e2e.yaml"); +const JOB_NAME = "hermes-gpu-startup"; +const RUN_STEP_NAME = "Run Hermes GPU startup live Vitest test"; +const DOCKER_AUTH_STEP_NAME = "Authenticate to Docker Hub"; +const HOSTED_PROVIDER_ENV_NAMES = [ + "COMPATIBLE_API_KEY", + "NEMOCLAW_E2E_USE_HOSTED_INFERENCE", + "NVIDIA_API_KEY", + "NVIDIA_INFERENCE_API_KEY", +] as const; +const SECRET_REFERENCE_PATTERN = /\bsecrets\.[A-Za-z0-9_]+\b/u; +const EXPECTED_SELECTOR = + "${{ contains(format(',{0},', inputs.jobs), ',hermes-gpu-startup,') || contains(format(',{0},', inputs.targets), ',hermes-gpu-startup,') }}"; + +type WorkflowRecord = Record; +type WorkflowStep = WorkflowRecord & { + name?: string; + run?: string; +}; + +function asRecord(value: unknown): WorkflowRecord { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as WorkflowRecord) + : {}; +} + +function asSteps(value: unknown): WorkflowStep[] { + return Array.isArray(value) ? (value as WorkflowStep[]) : []; +} + +function stringValue(value: unknown): string { + return typeof value === "string" ? value : ""; +} + +export function validateHermesGpuStartupWorkflowBoundary( + workflowPath = DEFAULT_WORKFLOW_PATH, +): string[] { + const workflow = asRecord(YAML.parse(readFileSync(workflowPath, "utf8"))); + const job = asRecord(asRecord(workflow.jobs)[JOB_NAME]); + const errors: string[] = []; + if (Object.keys(job).length === 0) { + return [`workflow missing ${JOB_NAME} job`]; + } + + if (job["runs-on"] !== "linux-amd64-gpu-rtxpro6000-latest-1") { + errors.push(`${JOB_NAME} job must run on the native RTX PRO 6000 GPU runner`); + } + if (job.needs !== "generate-matrix" || job.if !== EXPECTED_SELECTOR) { + errors.push(`${JOB_NAME} job must remain explicit-only behind generate-matrix`); + } + if (job["timeout-minutes"] !== 75) { + errors.push(`${JOB_NAME} job must keep the 75 minute timeout`); + } + + const jobEnv = asRecord(job.env); + const requiredEnv = { + E2E_DEFAULT_ENABLED: "0", + E2E_JOB: "1", + E2E_TARGET_ID: JOB_NAME, + NEMOCLAW_AGENT: "hermes", + NEMOCLAW_RUN_LIVE_E2E: "1", + NEMOCLAW_SANDBOX_GPU: "1", + NEMOCLAW_SANDBOX_NAME: "e2e-hermes-gpu-startup", + } as const; + for (const [name, expected] of Object.entries(requiredEnv)) { + if (jobEnv[name] !== expected) { + errors.push(`${JOB_NAME} job must set ${name}=${expected}`); + } + } + if (Object.hasOwn(jobEnv, "NEMOCLAW_DOCKER_GPU_PATCH")) { + errors.push( + `${JOB_NAME} job must leave NEMOCLAW_DOCKER_GPU_PATCH unset to exercise auto routing`, + ); + } + for (const name of HOSTED_PROVIDER_ENV_NAMES) { + if (Object.hasOwn(jobEnv, name)) { + errors.push(`${JOB_NAME} job env must not expose ${name}`); + } + } + if (SECRET_REFERENCE_PATTERN.test(JSON.stringify(jobEnv))) { + errors.push(`${JOB_NAME} job env must not consume repository secrets`); + } + + const steps = asSteps(job.steps); + for (const step of steps) { + const stepName = step.name ?? ""; + const stepEnv = asRecord(step.env); + for (const name of HOSTED_PROVIDER_ENV_NAMES) { + if (Object.hasOwn(stepEnv, name)) { + errors.push(`${JOB_NAME} step '${stepName}' must not expose ${name}`); + } + } + if (stepName !== DOCKER_AUTH_STEP_NAME && SECRET_REFERENCE_PATTERN.test(JSON.stringify(step))) { + errors.push(`${JOB_NAME} step '${stepName}' must not consume repository secrets`); + } + if (stringValue(step.run).includes("test/e2e/live/hermes-e2e.test.ts")) { + errors.push(`${JOB_NAME} step '${stepName}' must not run the hosted Hermes E2E test`); + } + } + const runStep = steps.find((step) => step.name === RUN_STEP_NAME); + if (!runStep) { + errors.push(`${JOB_NAME} job missing step: ${RUN_STEP_NAME}`); + return errors; + } + const runScript = stringValue(runStep.run); + if (!runScript.includes("npx vitest run --project e2e-live")) { + errors.push(`${JOB_NAME} step must run the e2e-live Vitest project`); + } + if (!runScript.includes("test/e2e/live/hermes-gpu-startup.test.ts")) { + errors.push(`${JOB_NAME} step must run the dedicated Hermes GPU startup test`); + } + + return errors; +} diff --git a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts index 19752b7ea42..a4caa2044bf 100644 --- a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts +++ b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts @@ -32,8 +32,8 @@ const UPLOAD_ARTIFACT_ACTION_PREFIX = "actions/upload-artifact@"; const INNER_ALWAYS = "${{ always() }}"; const CALLER_ALWAYS = "always()"; const TARGET_ID_PATTERN = /^[A-Za-z0-9_-]+$/; -const EXPECTED_UPLOAD_JOB_COUNT = 73; -const EXPECTED_DEFAULT_CALLER_COUNT = 64; +const EXPECTED_UPLOAD_JOB_COUNT = 74; +const EXPECTED_DEFAULT_CALLER_COUNT = 65; type WorkflowRecord = Record; type WorkflowStep = WorkflowRecord & { diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index 08c0e632dd1..7bad543e0ad 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -7,6 +7,7 @@ import { fileURLToPath } from "node:url"; import YAML from "yaml"; import { validateDocsValidationWorkflowBoundary } from "./docs-validation-workflow-boundary.mts"; import { validateHermesDashboardWorkflowBoundary } from "./hermes-dashboard-workflow-boundary.mts"; +import { validateHermesGpuStartupWorkflowBoundary } from "./hermes-gpu-startup-workflow-boundary.mts"; import { validateInferenceSwitchWorkflowBoundary } from "./inference-switch-workflow-boundary.mts"; import { validateE2eOperationsWorkflowBoundary } from "./operations-workflow-boundary.mts"; import { validatePrepareE2eWorkflowBoundary } from "./prepare-e2e-workflow-boundary.mts"; @@ -3642,6 +3643,7 @@ export function validateE2eWorkflowBoundary(workflowPath = DEFAULT_E2E_WORKFLOW_ errors.push(...validatePrepareE2eWorkflowBoundary(workflow)); errors.push(...validateUploadE2eArtifactsWorkflowBoundary(workflow)); errors.push(...validateHermesDashboardWorkflowBoundary(workflowPath)); + errors.push(...validateHermesGpuStartupWorkflowBoundary(workflowPath)); errors.push(...validateInferenceSwitchWorkflowBoundary(workflowPath)); errors.push(...validateE2eOperationsWorkflowBoundary(workflowPath)); errors.push(...validateDocsValidationWorkflowBoundary(workflowPath));