diff --git a/ci/platform-matrix.json b/ci/platform-matrix.json index 5c7d94c53b8..64ccf1864de 100644 --- a/ci/platform-matrix.json +++ b/ci/platform-matrix.json @@ -149,7 +149,7 @@ "name": "Local NVIDIA NIM", "status": "experimental", "endpoint_type": "Local OpenAI-compatible", - "notes": "Requires `NEMOCLAW_EXPERIMENTAL=1` and a NIM-capable NVIDIA GPU. Host must have the NVIDIA Container Toolkit installed and a CDI spec present (`onboard` asserts CDI presence with `assertCdiNvidiaGpuSpecPresent`, `src/lib/onboard/fatal-runtime-preflight.ts`). NIM images pull from `nvcr.io` and require NGC registry login. NemoClaw gates this path behind the experimental flag because it does not auto-select a NIM image for the host today. You must explicitly pick from the validated image list. On Linux arm64 DGX Spark and DGX Station hosts, onboarding warns that some NIM images may not publish a `linux/arm64` manifest; the warning is advisory, and the selected image pull can still fail when the registry has no matching platform manifest. Managed vLLM has host-specific default models and is not gated on the same boxes. Validated images referenced in `src/lib/inference/config.ts` and `nemoclaw/src/index.ts`: `nvidia/nemotron-3-super-120b-a12b` (default cloud model), `nvidia/nemotron-3-nano-30b-a3b`, `nvidia/llama-3.3-nemotron-super-49b-v1.5`." + "notes": "Requires `NEMOCLAW_EXPERIMENTAL=1` and a NIM-capable NVIDIA GPU. Host must have the NVIDIA Container Toolkit installed and a healthy CDI spec. Onboarding evaluates the canonical `host.gpu.nvidia_available`, `host.gpu.container_toolkit_available`, and `host.gpu.cdi_healthy` readiness capabilities before gateway, image, or sandbox lifecycle effects. NIM images pull from `nvcr.io` and require NGC registry login. NemoClaw gates this path behind the experimental flag because it does not auto-select a NIM image for the host today. You must explicitly pick from the validated image list. On Linux arm64 DGX Spark and DGX Station hosts, onboarding warns that some NIM images may not publish a `linux/arm64` manifest; the warning is advisory, and the selected image pull can still fail when the registry has no matching platform manifest. Managed vLLM has host-specific default models and is not gated on the same boxes. Validated images referenced in `src/lib/inference/config.ts` and `nemoclaw/src/index.ts`: `nvidia/nemotron-3-super-120b-a12b` (default cloud model), `nvidia/nemotron-3-nano-30b-a3b`, `nvidia/llama-3.3-nemotron-super-49b-v1.5`." }, { "name": "Local vLLM (already running)", @@ -223,9 +223,9 @@ "notes": "OpenClaw, Hermes, and LangChain Deep Agents Code support authenticated HTTPS Streamable HTTP MCP server lifecycle through `$$nemoclaw mcp`. NemoClaw stores only credential names and ownership metadata, while OpenShell keeps raw credential values outside the sandbox and replaces credential placeholders at approved egress boundaries." }, { - "name": "Host readiness reporting", + "name": "System readiness reporting", "status": "tested", - "notes": "`$$nemoclaw host probe` reports host observations, capabilities, qualifications, findings, evidence, and CLI provenance without changing host, gateway, provider, policy, credential, or sandbox state. The human-readable and JSON formats use the same schema-versioned report and deterministic exit codes." + "notes": "`$$nemoclaw host probe` reports host and gateway observations, capabilities, qualifications, findings, evidence, and CLI provenance without changing host, Docker, gateway, provider, policy, credential, or sandbox state. The human-readable and JSON formats use the same schema-versioned report and deterministic exit codes." }, { "name": "Deep Agents trace export", @@ -283,7 +283,7 @@ { "name": "Non-NVIDIA GPUs (AMD/ROCm, Intel Arc, Apple Metal)", "status": "unsupported", - "notes": "Local vLLM and NIM paths assert NVIDIA CDI presence with `assertCdiNvidiaGpuSpecPresent` (`src/lib/onboard/fatal-runtime-preflight.ts`). NemoClaw does not install non-NVIDIA accelerator drivers." + "notes": "GPU-backed local vLLM and NIM onboarding requires the canonical `host.gpu.nvidia_available`, `host.gpu.container_toolkit_available`, and `host.gpu.cdi_healthy` readiness capabilities. NemoClaw does not install non-NVIDIA accelerator drivers." }, { "name": "Other LangChain, AutoGen, CrewAI, or non-listed agent harnesses", diff --git a/ci/source-architecture-budget.json b/ci/source-architecture-budget.json index 474c485bf0e..f9b27a9d0dd 100644 --- a/ci/source-architecture-budget.json +++ b/ci/source-architecture-budget.json @@ -13,7 +13,7 @@ "src/lib/agent/defs.ts": 32, "src/lib/cli/branding.ts": 86, "src/lib/cli/nemoclaw-oclif-command.ts": 106, - "src/lib/cli/terminal-style.ts": 44, + "src/lib/cli/terminal-style.ts": 43, "src/lib/core/json-types.ts": 37, "src/lib/core/ports.ts": 87, "src/lib/core/shell-quote.ts": 26, diff --git a/ci/source-shape-test-budget.json b/ci/source-shape-test-budget.json index 7ae023b4644..b9339837762 100644 --- a/ci/source-shape-test-budget.json +++ b/ci/source-shape-test-budget.json @@ -36,6 +36,11 @@ "test": "bounds and redacts successful probe text before schema validation", "category": "compatibility" }, + { + "file": "src/lib/readiness/system.test.ts", + "test": "publishes a schema-valid host and gateway report with resolved references (#7411)", + "category": "compatibility" + }, { "file": "src/lib/onboard/inference-providers/compatible-endpoint-gateway-route.test.ts", "test": "matches the bundled local-inference host-gateway ports (#5744)", diff --git a/ci/test-file-size-budget.json b/ci/test-file-size-budget.json index 787ba05d16f..eedc783553f 100644 --- a/ci/test-file-size-budget.json +++ b/ci/test-file-size-budget.json @@ -4,9 +4,9 @@ "legacyMaxLines": { "nemoclaw/src/commands/migration-state.test.ts": 1301, "src/lib/inference/nim.test.ts": 2068, - "src/lib/onboard/preflight.test.ts": 1904, + "src/lib/onboard/preflight.test.ts": 1875, "test/generate-openclaw-config.test.ts": 1915, - "test/install-preflight.test.ts": 3908, + "test/install-preflight.test.ts": 3906, "test/nemoclaw-start.test.ts": 4791, "test/onboard-messaging.test.ts": 2036, "test/onboard-selection.test.ts": 4767 diff --git a/docs/about/how-it-works.mdx b/docs/about/how-it-works.mdx index afcac70704b..3fcf4c7b6cf 100644 --- a/docs/about/how-it-works.mdx +++ b/docs/about/how-it-works.mdx @@ -123,15 +123,19 @@ This separation keeps host orchestration, agent-specific assets, and the sandbox ## Readiness and Sandbox Creation -Run `$$nemoclaw host probe` when you need a read-only host readiness report before onboarding. -The report describes host observations, capabilities, qualifications, findings, evidence, and CLI provenance without changing host state. +Run `$$nemoclaw host probe` when you need a read-only system readiness report before onboarding. +The report combines host and gateway observations, capabilities, qualifications, findings, evidence, and CLI provenance without changing system state. +Onboarding consumes the same stable host and gateway entities and applies its explicit admission policy. +It revalidates live facts after permitted preparation and when a saved onboarding session resumes. When you run `$$nemoclaw onboard`, the host CLI and blueprint runner complete these operations: -1. NemoClaw resolves the blueprint, checks version compatibility, and verifies the digest. -2. Onboarding validates the selected inference provider, credentials, agent settings, and platform requirements. -3. The runner determines which gateway, provider, policy, sandbox, and integration resources to create or update. -4. NemoClaw records progress so a supported interruption can resume or report a specific recovery action. +1. NemoClaw resolves gateway lifecycle authority and rejects blocking system readiness results before managed resource effects. + A container-backed WSL GPU proof can run only after this admission check; explicit CPU-only intent skips it. +2. NemoClaw resolves the blueprint, checks version compatibility, and verifies the digest. +3. Onboarding validates the selected inference provider, credentials, agent settings, and platform requirements. +4. The runner determines which gateway, provider, policy, sandbox, and integration resources to create or update. +5. NemoClaw records progress so a supported interruption can resume or report a specific recovery action. After the sandbox starts, the selected agent uses its managed configuration and the controls supported by the host. @@ -141,7 +145,7 @@ NemoClaw operates the sandbox and its manifest-declared state through host-side | Operation | Result | |---|---| -| Inspect | `host probe`, `status`, and `logs` report host, sandbox, agent-runtime, inference, and recovery information without replacing the sandbox. | +| Inspect | `host probe`, `status`, and `logs` report system, sandbox, agent-runtime, inference, and recovery information without replacing the sandbox. | | Configure | Inference, policy, managed MCP, and supported agent-runtime integration commands update the applicable managed resources. | | Rebuild | Recreates the sandbox from the recorded configuration and restores supported agent state through a recorded transaction. | | Recover | Repairs a stopped or degraded agent runtime and its sandbox-scoped forwards when the recorded identities still match. | diff --git a/docs/get-started/windows-preparation.mdx b/docs/get-started/windows-preparation.mdx index 0a160894412..63cd0b5bba4 100644 --- a/docs/get-started/windows-preparation.mdx +++ b/docs/get-started/windows-preparation.mdx @@ -173,8 +173,10 @@ If the installer offers express install on WSL, accepting it selects a local Oll The Windows-host Ollama path requires Docker Desktop WSL integration; the express prompt appears on WSL regardless of the container runtime. When the installer must read the Docker configuration file to determine the effective Docker context and Node.js is unavailable, it defers provider selection until after it installs Node.js. It then reads the configuration and applies the same runtime check. -With native Docker Engine inside WSL, or when the Docker runtime is unavailable or cannot be probed, express install configures WSL-local Ollama instead of aborting. -An installed Windows-host Ollama that the container runtime cannot reach leaves the WSL-local install available, in both the onboarding menu and a requested `install-ollama` provider. +The express selection does not waive system readiness. +With native Docker Engine inside WSL, or when the Docker runtime is unavailable or cannot be probed, the installer skips onboarding until Docker Desktop WSL integration is available. +After you fix the runtime, rerun the installer or run `$$nemoclaw onboard`. +An installed Windows-host Ollama that Docker Desktop cannot reach leaves the WSL-local install available, in both the onboarding menu and a requested `install-ollama` provider. When containers cannot reach host loopback, onboarding fronts that WSL-local daemon with the sandbox auth proxy. You can still decline the express prompt (or set `NEMOCLAW_NO_EXPRESS=1`) to choose a provider manually; the onboarding menu labels the Windows-host actions as requiring Docker Desktop integration. When Ollama runs on the Windows host, NemoClaw checks it from Docker Desktop through `host.docker.internal` and pulls missing models through the Ollama HTTP API. diff --git a/docs/inference/choose-inference-provider.mdx b/docs/inference/choose-inference-provider.mdx index b6106f48d5e..b75fe47fd7a 100644 --- a/docs/inference/choose-inference-provider.mdx +++ b/docs/inference/choose-inference-provider.mdx @@ -30,7 +30,7 @@ Use this status table to distinguish validated provider integrations from adapte | Local Ollama | Tested with limitations | Local Ollama API | Available when Ollama is installed or running on the host. Validated default models: `qwen3.6:35b` (high VRAM), `nemotron-3-nano:30b` (medium VRAM), `qwen3.5:9b` (low VRAM fallback). | | Local llama.cpp (already running) | Experimental | Local OpenAI-compatible | Attaches only to an authenticated operator-managed server on loopback port `8081` that passes cooperative native llama.cpp fingerprinting. NemoClaw does not own its server, model, or lifecycle. | | Local llama.cpp (managed DGX Spark) | Experimental | Local OpenAI-compatible | Uses the repository-owned YAML recipe for NVIDIA Nemotron 3 Nano 30B-A3B, exact GGUF verification, and Docker lifecycle on a qualified DGX Spark host. The recipe remains unqualified for agent support. | -| Local NVIDIA NIM | Experimental | Local OpenAI-compatible | Requires `NEMOCLAW_EXPERIMENTAL=1` and a NIM-capable NVIDIA GPU. Host must have the NVIDIA Container Toolkit installed and a CDI spec present (`onboard` asserts CDI presence with `assertCdiNvidiaGpuSpecPresent`, `src/lib/onboard/fatal-runtime-preflight.ts`). NIM images pull from `nvcr.io` and require NGC registry login. NemoClaw gates this path behind the experimental flag because it does not auto-select a NIM image for the host today. You must explicitly pick from the validated image list. On Linux arm64 DGX Spark and DGX Station hosts, onboarding warns that some NIM images may not publish a `linux/arm64` manifest; the warning is advisory, and the selected image pull can still fail when the registry has no matching platform manifest. Managed vLLM has host-specific default models and is not gated on the same boxes. Validated images referenced in `src/lib/inference/config.ts` and `nemoclaw/src/index.ts`: `nvidia/nemotron-3-super-120b-a12b` (default cloud model), `nvidia/nemotron-3-nano-30b-a3b`, `nvidia/llama-3.3-nemotron-super-49b-v1.5`. | +| Local NVIDIA NIM | Experimental | Local OpenAI-compatible | Requires `NEMOCLAW_EXPERIMENTAL=1` and a NIM-capable NVIDIA GPU. Host must have the NVIDIA Container Toolkit installed and a healthy CDI spec. Onboarding evaluates the canonical `host.gpu.nvidia_available`, `host.gpu.container_toolkit_available`, and `host.gpu.cdi_healthy` readiness capabilities before gateway, image, or sandbox lifecycle effects. NIM images pull from `nvcr.io` and require NGC registry login. NemoClaw gates this path behind the experimental flag because it does not auto-select a NIM image for the host today. You must explicitly pick from the validated image list. On Linux arm64 DGX Spark and DGX Station hosts, onboarding warns that some NIM images may not publish a `linux/arm64` manifest; the warning is advisory, and the selected image pull can still fail when the registry has no matching platform manifest. Managed vLLM has host-specific default models and is not gated on the same boxes. Validated images referenced in `src/lib/inference/config.ts` and `nemoclaw/src/index.ts`: `nvidia/nemotron-3-super-120b-a12b` (default cloud model), `nvidia/nemotron-3-nano-30b-a3b`, `nvidia/llama-3.3-nemotron-super-49b-v1.5`. | | Local vLLM (already running) | Tested with limitations | Local OpenAI-compatible | Appears in the onboarding menu when NemoClaw detects a server already on `localhost:8000`. No flag required. Model is whatever the existing server serves. | | Local vLLM (managed install/start) | Tested with limitations | Local OpenAI-compatible | Appears by default on DGX Spark and qualifying DGX Station GB300 hosts. DGX Station is Tested with limitations across qualified profiles on one physical DGX Station GB300. Full Station Express end-to-end qualification for the accepted no-OTA DGX OS `7.6.x` profile is pending. Dual-Station configurations are not yet validated, and dedicated CI coverage is not available. For canonical Station qualification and host preparation, see the Additional Setup page for [OpenClaw](/user-guide/openclaw/get-started/additional-setup/dgx-station-preparation), [Hermes](/user-guide/hermes/get-started/additional-setup/dgx-station-preparation), or [Deep Agents](/user-guide/deepagents/get-started/additional-setup/dgx-station-preparation). Generic Linux NVIDIA GPU hosts require `NEMOCLAW_EXPERIMENTAL=1` or `NEMOCLAW_PROVIDER=install-vllm`, NVIDIA Container Toolkit, and CDI. NemoClaw pins runtime images to immutable digests. Station Express defaults to `nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4`; `--station-deepseek` selects `deepseek-ai/DeepSeek-V4-Flash`. Direct managed-vLLM defaults are `nvidia/Qwen3.6-35B-A3B-NVFP4` on DGX Spark, `deepseek-ai/DeepSeek-V4-Flash` on DGX Station, and `nvidia/NVIDIA-Nemotron-3-Nano-4B-FP8` on generic Linux NVIDIA GPU hosts. Image pulls from `nvcr.io` require NGC registry login. | {/* provider-status:end */} diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index f1956c936ff..63253bd4974 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -212,7 +212,7 @@ If the gateway is not running, Kubernetes allocatable fields are omitted and hos ### `$$nemoclaw host probe` -Inspect host readiness before onboarding without changing host, Docker, gateway, credential, policy, or sandbox state. +Inspect host capabilities and gateway authority before onboarding without changing host, Docker, gateway, credential, policy, or sandbox state. Use `--json` for the schema-versioned report. The command exits with `0` for `supported`, `2` for `incompatible`, and `3` for `inconclusive`. @@ -756,7 +756,7 @@ Kanban backup does not include named boards, attachments, worker logs, scratch w Before creating the gateway, the wizard runs preflight checks. -It verifies that Docker is reachable, warns on untested runtimes such as Podman, and prints host remediation guidance when prerequisites are missing. +It verifies that Docker is reachable, rejects unsupported runtimes such as Podman, and prints host remediation guidance when prerequisites are missing. The preflight also enforces the OpenShell version range declared in the blueprint (`min_openshell_version` and `max_openshell_version`). If the installed OpenShell version falls outside this range, onboarding exits with an actionable error and a link to compatible releases. For fresh OpenShell installs, NemoClaw queries published OpenShell releases and asks the installer to use a release that fits the blueprint range. diff --git a/docs/reference/platform-support.mdx b/docs/reference/platform-support.mdx index fb7b2c7fdf0..e1906e52d17 100644 --- a/docs/reference/platform-support.mdx +++ b/docs/reference/platform-support.mdx @@ -104,7 +104,7 @@ NemoClaw routes inference through the OpenShell gateway. Each row below is a pro | Local Ollama | Tested with limitations | Local Ollama API | Available when Ollama is installed or running on the host. Validated default models: `qwen3.6:35b` (high VRAM), `nemotron-3-nano:30b` (medium VRAM), `qwen3.5:9b` (low VRAM fallback). | | Local llama.cpp (already running) | Experimental | Local OpenAI-compatible | Attaches only to an authenticated operator-managed server on loopback port `8081` that passes cooperative native llama.cpp fingerprinting. NemoClaw does not own its server, model, or lifecycle. | | Local llama.cpp (managed DGX Spark) | Experimental | Local OpenAI-compatible | Uses the repository-owned YAML recipe for NVIDIA Nemotron 3 Nano 30B-A3B, exact GGUF verification, and Docker lifecycle on a qualified DGX Spark host. The recipe remains unqualified for agent support. | -| Local NVIDIA NIM | Experimental | Local OpenAI-compatible | Requires `NEMOCLAW_EXPERIMENTAL=1` and a NIM-capable NVIDIA GPU. Host must have the NVIDIA Container Toolkit installed and a CDI spec present (`onboard` asserts CDI presence with `assertCdiNvidiaGpuSpecPresent`, `src/lib/onboard/fatal-runtime-preflight.ts`). NIM images pull from `nvcr.io` and require NGC registry login. NemoClaw gates this path behind the experimental flag because it does not auto-select a NIM image for the host today. You must explicitly pick from the validated image list. On Linux arm64 DGX Spark and DGX Station hosts, onboarding warns that some NIM images may not publish a `linux/arm64` manifest; the warning is advisory, and the selected image pull can still fail when the registry has no matching platform manifest. Managed vLLM has host-specific default models and is not gated on the same boxes. Validated images referenced in `src/lib/inference/config.ts` and `nemoclaw/src/index.ts`: `nvidia/nemotron-3-super-120b-a12b` (default cloud model), `nvidia/nemotron-3-nano-30b-a3b`, `nvidia/llama-3.3-nemotron-super-49b-v1.5`. | +| Local NVIDIA NIM | Experimental | Local OpenAI-compatible | Requires `NEMOCLAW_EXPERIMENTAL=1` and a NIM-capable NVIDIA GPU. Host must have the NVIDIA Container Toolkit installed and a healthy CDI spec. Onboarding evaluates the canonical `host.gpu.nvidia_available`, `host.gpu.container_toolkit_available`, and `host.gpu.cdi_healthy` readiness capabilities before gateway, image, or sandbox lifecycle effects. NIM images pull from `nvcr.io` and require NGC registry login. NemoClaw gates this path behind the experimental flag because it does not auto-select a NIM image for the host today. You must explicitly pick from the validated image list. On Linux arm64 DGX Spark and DGX Station hosts, onboarding warns that some NIM images may not publish a `linux/arm64` manifest; the warning is advisory, and the selected image pull can still fail when the registry has no matching platform manifest. Managed vLLM has host-specific default models and is not gated on the same boxes. Validated images referenced in `src/lib/inference/config.ts` and `nemoclaw/src/index.ts`: `nvidia/nemotron-3-super-120b-a12b` (default cloud model), `nvidia/nemotron-3-nano-30b-a3b`, `nvidia/llama-3.3-nemotron-super-49b-v1.5`. | | Local vLLM (already running) | Tested with limitations | Local OpenAI-compatible | Appears in the onboarding menu when NemoClaw detects a server already on `localhost:8000`. No flag required. Model is whatever the existing server serves. | | Local vLLM (managed install/start) | Tested with limitations | Local OpenAI-compatible | Appears by default on DGX Spark and qualifying DGX Station GB300 hosts. DGX Station is Tested with limitations across qualified profiles on one physical DGX Station GB300. Full Station Express end-to-end qualification for the accepted no-OTA DGX OS `7.6.x` profile is pending. Dual-Station configurations are not yet validated, and dedicated CI coverage is not available. For canonical Station qualification and host preparation, see the Additional Setup page for [OpenClaw](/user-guide/openclaw/get-started/additional-setup/dgx-station-preparation), [Hermes](/user-guide/hermes/get-started/additional-setup/dgx-station-preparation), or [Deep Agents](/user-guide/deepagents/get-started/additional-setup/dgx-station-preparation). Generic Linux NVIDIA GPU hosts require `NEMOCLAW_EXPERIMENTAL=1` or `NEMOCLAW_PROVIDER=install-vllm`, NVIDIA Container Toolkit, and CDI. NemoClaw pins runtime images to immutable digests. Station Express defaults to `nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4`; `--station-deepseek` selects `deepseek-ai/DeepSeek-V4-Flash`. Direct managed-vLLM defaults are `nvidia/Qwen3.6-35B-A3B-NVFP4` on DGX Spark, `deepseek-ai/DeepSeek-V4-Flash` on DGX Station, and `nvidia/NVIDIA-Nemotron-3-Nano-4B-FP8` on generic Linux NVIDIA GPU hosts. Image pulls from `nvcr.io` require NGC registry login. | {/* provider-status-full:end */} @@ -139,7 +139,7 @@ Each row below is a launch-facing capability claim that NemoClaw makes in docs, | Snapshot and restore | Tested | Create, list, and restore named snapshots of sandbox state with the `$$nemoclaw snapshot` subcommands (`create`, `list`, `restore`). Credential stripping is enforced on capture. Unsafe symlinks are rejected on restore. | | Agent skills | Tested | Packaged agent skills are discoverable by Cursor, Claude Code, and other coding assistants under `.agents/skills/`. Skills also install into the sandbox with `$$nemoclaw skill install`. | | Managed MCP servers | Tested | OpenClaw, Hermes, and LangChain Deep Agents Code support authenticated HTTPS Streamable HTTP MCP server lifecycle through `$$nemoclaw mcp`. NemoClaw stores only credential names and ownership metadata, while OpenShell keeps raw credential values outside the sandbox and replaces credential placeholders at approved egress boundaries. | -| Host readiness reporting | Tested | `$$nemoclaw host probe` reports host observations, capabilities, qualifications, findings, evidence, and CLI provenance without changing host, gateway, provider, policy, credential, or sandbox state. The human-readable and JSON formats use the same schema-versioned report and deterministic exit codes. | +| System readiness reporting | Tested | `$$nemoclaw host probe` reports host and gateway observations, capabilities, qualifications, findings, evidence, and CLI provenance without changing host, Docker, gateway, provider, policy, credential, or sandbox state. The human-readable and JSON formats use the same schema-versioned report and deterministic exit codes. | | Deep Agents trace export | Tested with limitations | LangChain Deep Agents Code can opt into bounded-content OpenTelemetry Protocol (OTLP) over HTTP trace export to an operator-managed host collector. The sandbox receives no remote-backend credential, the local receiver has no authentication, trace content can contain sensitive application data, and delivery failures do not stop agent work. | | State migration | Tested | Sandbox state migrates across rebuilds with credentials intentionally excluded. Hermes excludes `auth.json` and restores its SQLite session DB through the backup API. OpenClaw config merge prevents stale state from overwriting fresh values. | | Blueprint versioning | Tested | Versioned, digest-verified, and reproducible blueprint lifecycle. Drives `$$nemoclaw rebuild` and the migration safeguards above. | @@ -172,7 +172,7 @@ They are listed here so launch material, sales conversations, and support triage | Native Kubernetes or OpenShift deployments | Unsupported | NemoClaw runs the sandbox as a Docker container, not a Kubernetes pod. The default Docker-driver topology does not embed k3s. Operator-managed K8s/OpenShift deployments are out of scope; see issue #407 (community OpenShift through agent-sandbox CRD). | | Air-gapped / offline installs | Unsupported | Onboard assumes network reachability for package fetches, container pulls, and provider validation. See open issues #4872 and #2218 (production-deployment epic covering air-gapped support, China network guidance, multi-host topology). | | Windows-on-ARM GPU passthrough | Unsupported | Windows-on-ARM CPU paths run under WSL2 'tested with limitations', but GPU passthrough on WOA is denylisted (`src/lib/onboard/wsl-docker-desktop-gpu.ts:214`, `src/lib/inference/gpu-trust.test.ts:70`). See closed issue #4565. | -| Non-NVIDIA GPUs (AMD/ROCm, Intel Arc, Apple Metal) | Unsupported | Local vLLM and NIM paths assert NVIDIA CDI presence with `assertCdiNvidiaGpuSpecPresent` (`src/lib/onboard/fatal-runtime-preflight.ts`). NemoClaw does not install non-NVIDIA accelerator drivers. | +| Non-NVIDIA GPUs (AMD/ROCm, Intel Arc, Apple Metal) | Unsupported | GPU-backed local vLLM and NIM onboarding requires the canonical `host.gpu.nvidia_available`, `host.gpu.container_toolkit_available`, and `host.gpu.cdi_healthy` readiness capabilities. NemoClaw does not install non-NVIDIA accelerator drivers. | | Other LangChain, AutoGen, CrewAI, or non-listed agent harnesses | Unsupported | LangChain Deep Agents Code is the only integrated LangChain-family agent runtime (refer to the Agents section above; status `Tested`). Other LangChain agent runtimes, AutoGen, CrewAI, and any agent runtime not listed in the Agents table are not integrated. Bringing more agent runtimes is tracked as a research epic (refer to open issue #4861) but is not on the current roadmap. | | Multi-user host sharing | Unsupported | Sandboxes are scoped to a single host user. NemoClaw treats multi-user hosts as a risk and warns at onboard; see `docs/security/openclaw-controls.mdx` Multi-user detection. | | Hosted SaaS / managed NemoClaw | Unsupported | There is no managed offering. Supported deployment paths are local CLI onboarding and headless Linux server onboarding. | diff --git a/docs/reference/system-readiness.mdx b/docs/reference/system-readiness.mdx index c8bd9b44c6f..049bcebbe98 100644 --- a/docs/reference/system-readiness.mdx +++ b/docs/reference/system-readiness.mdx @@ -3,14 +3,14 @@ # SPDX-License-Identifier: Apache-2.0 title: "System Readiness" sidebar-title: "System Readiness" -description: "Check host readiness before NemoClaw onboarding without changing system state." -description-agent: "Documents the read-only host readiness command, JSON contract, exit codes, redaction boundary, and mutation guarantees. Use when checking a host before onboarding or consuming readiness JSON." +description: "Check host and gateway readiness before NemoClaw onboarding without changing system state." +description-agent: "Documents the read-only system readiness command, host and gateway contract, onboarding admission, exit codes, redaction boundary, and mutation guarantees. Use when checking a system before onboarding or consuming readiness JSON." keywords: ["nemoclaw host probe", "system readiness", "readiness json", "pre-onboarding check"] content: type: "reference" --- -Use the system readiness report to inspect a host before you run onboarding. +Use the system readiness report to inspect host prerequisites and OpenShell gateway authority before you run onboarding. The command works without a registered gateway or sandbox. ## Run the readiness check @@ -61,7 +61,7 @@ The identity is present for `supported`, `incompatible`, and `inconclusive` repo | --- | --- | | `provenance.nemoclawVersion` | Public version of the executing CLI build. | | `provenance.sourceRevision` | Full immutable source revision of that build. | -| `provenance.observedAt` | Time when the host observation started. | +| `provenance.observedAt` | Time when the system observation started. | A Git-described version can use a value such as `0.0.96-35-g8bfff4526`. The version must begin with three dot-separated digit groups. @@ -97,6 +97,11 @@ Do not use `observedAt` as build identity. ## Interpret Docker storage +The report rejects an unsupported `DOCKER_HOST` before it uses Docker daemon evidence. +`host.docker.host_invalid` records whether the configured endpoint is invalid, `host.docker.endpoint_supported` is absent when a configured endpoint is not an absolute local `unix://` socket, and the blocking `host.docker.host_invalid` finding identifies that condition. +TCP, SSH, relative Unix socket, and unsafe path values do not fall back to the default Docker socket. +Unset `DOCKER_HOST` continues to use Docker's default local socket. + The Docker storage capabilities separate the current host configuration from NemoClaw's supported remediation path. | Capability ID | Result represented | @@ -130,10 +135,14 @@ Each capability uses `present`, `absent`, or `unknown`. | `host.platform.dgx_spark` | DGX Spark identity on ARM64 with an available NVIDIA GPU. | | `host.platform.dgx_station` | Qualified DGX Station GB300 identity, Linux ARM64 runtime, Ubuntu 24.04, hardware, and software profile. | +If the combined platform capability is absent and no more specific platform qualification finding applies, the blocking `host.platform.unsupported` finding identifies an unsupported host and runtime combination. + The report adds `host.platform.wsl`, `host.platform.dgx_spark`, or `host.platform.dgx_station` qualifications when those platforms apply. For WSL, Docker Desktop is `qualified`, native Docker is `unqualified`, and an unrecognized reachable runtime is `unknown`. The `host.platform.wsl_gpu_passthrough` capability is `present` only after the Docker Desktop proof passes. It is `absent` when the proof fails or the host has no NVIDIA GPU, and `unknown` when the proof cannot run conclusively. +The read-only `host probe` command does not pull or start the container-backed WSL GPU proof, so a host that needs that proof reports this capability as `unknown`. +Onboarding can run the bounded proof only after its current host and gateway readiness checks admit the run. DGX Station qualification fails closed. The `host.platform.dgx_station` capability is `present` only when the host runs Linux ARM64 on Ubuntu 24.04, reports an effective NVIDIA GPU, and the product identity, an exact display-class PCI device with NVIDIA vendor ID (`0x10de`) and GB300 device ID (`0x31c2` or `0x31c3`), and the software profile all qualify. @@ -144,10 +153,156 @@ An unreadable marker or missing required OS or PCI evidence produces an `unknown Both outcomes prevent `host.platform.supported` from becoming `present`. When available, bounded Station identity appears under evidence ID `host.platform.identity`. -## Understand the read-only boundary +## Interpret NVIDIA GPU Integration + +GPU readiness distinguishes Jetson/Tegra Docker runtime integration from CDI-based NVIDIA GPU integration on other Linux hosts. + +| Stable ID | Kind | Result represented | +| --- | --- | --- | +| `host.gpu.nvidia_runtime` | Observation | Whether Docker advertises an NVIDIA runtime when the detected GPU platform is Jetson/Tegra. It is `present` when advertised, `absent` when missing or when the check does not apply, and `unknown` when an applicable check lacks enough evidence. The `host.gpu.nvidia_runtime_missing` finding distinguishes a missing Jetson/Tegra runtime from an inapplicable check. | +| `host.gpu.nvidia_available` | Capability | `present` when NemoClaw detects an NVIDIA GPU and `absent` otherwise. | +| `host.gpu.container_toolkit_available` | Capability | NVIDIA container integration is available. On Jetson/Tegra, this capability is derived from Docker NVIDIA runtime evidence rather than CDI files. It is `present` when container integration checks do not apply. | +| `host.gpu.cdi_healthy` | Capability | Applicable CDI specifications are present and current. It is `present` when CDI checks do not apply, including Jetson/Tegra and Docker Desktop GPU integration in WSL. | +| `host.gpu.nvidia_runtime_missing` | Finding | Blocking: Docker NVIDIA runtime support is missing for Jetson/Tegra sandbox GPU. | +| `host.gpu.container_toolkit_missing` | Finding | Blocking: NVIDIA Container Toolkit is missing where toolkit integration applies. | +| `host.gpu.cdi_missing` | Finding | Blocking: an applicable NVIDIA CDI specification is missing. | +| `host.gpu.cdi_stale` | Finding | Blocking: an applicable NVIDIA CDI specification is stale. | + +Jetson/Tegra detection can use platform firmware and device evidence even when `nvidia-smi` is unavailable. +Its remediation configures the Docker NVIDIA runtime and does not instruct the user to generate CDI files. +An explicit CPU-only onboarding run can waive GPU integration findings for that run; `host probe` still reports the observed host state. + +## Interpret Gateway Readiness + +Gateway readiness establishes one lifecycle authority before onboarding can create or change managed resources. +The report covers both NemoClaw-managed and externally supervised gateways. + +The gateway observations use these stable IDs: + +| Observation ID | Values and meaning | +| --- | --- | +| `gateway.management.mode` | `nemoclaw-managed` lets NemoClaw manage the gateway lifecycle. `externally-supervised` limits NemoClaw to attachment and validation. | +| `gateway.owner.name` | The canonical OpenShell gateway registration for later operations. | +| `gateway.owner.source` | `declared`, `packaged-service`, or `standalone`, according to how NemoClaw resolved the authority. | +| `gateway.owner.port` | The host gateway port paired with the resolved registration. | +| `gateway.attachment` | `verified`, `rejected`, or `unknown` for external supervision. NemoClaw-managed gateways report `not-applicable`. | +| `gateway.reuse` | `healthy`, `missing`, `stale`, `active-unnamed`, `foreign-active`, or `unknown` for managed gateways. External supervision reports `not-applicable`. | +| `gateway.version_drift` | `detected`, `not-detected`, or `unknown` for managed gateways. External supervision reports `not-applicable`. | +| `gateway.port_conflict` | `none`, `occupied`, `multiple-owners`, `owner-mismatch`, or `unknown`. | + +Each gateway capability uses `present`, `absent`, or `unknown`: + +| Capability ID | Result represented | +| --- | --- | +| `gateway.authority.resolved` | NemoClaw resolved one lifecycle authority for the run. | +| `gateway.attachment.valid` | The external attachment passed, or attachment does not apply to the managed mode. | +| `gateway.reuse.ready` | The managed gateway is reusable, no gateway exists yet, or reuse does not apply. An absent value can require managed reconciliation. | +| `gateway.version.compatible` | A positively identified managed gateway has source-bound version evidence that matches the installed OpenShell version, no managed gateway is running, or the check does not apply. Missing or unbound version evidence is unknown. | +| `gateway.port.uncontested` | The gateway port has no incompatible or ambiguous owner. | + +Use finding IDs to handle gateway failures without parsing summaries: + +| Finding ID | Severity | Meaning | +| --- | --- | --- | +| `gateway.authority.invalid` | Blocking | NemoClaw could not resolve one lifecycle authority. | +| `gateway.probe.inconclusive` | Warning | Required gateway observations could not be collected. | +| `gateway.attachment.effect_forbidden` | Blocking | An operation conflicts with external lifecycle authority. | +| `gateway.attachment.unreachable` | Blocking | The declared external gateway did not answer its health probe. | +| `gateway.supervisor.inactive` | Blocking | The declared external supervisor was not active. | +| `gateway.ownership.mismatch` | Blocking | The listener identity did not match the declared supervisor. | +| `gateway.ownership.unverified` | Blocking | NemoClaw could not verify the listener identity. | +| `gateway.ownership.multiple` | Blocking | More than one process could own the declared gateway port. | +| `gateway.endpoint.port_mismatch` | Blocking | The declared endpoint port did not match the configured gateway port. | +| `gateway.registration.failed` | Blocking | NemoClaw could not verify the required OpenShell registration. | +| `gateway.capability.unsupported` | Blocking | This NemoClaw build does not implement a capability required by the declaration. | +| `gateway.attachment.inconclusive` | Warning | The external attachment probe did not establish a result. | +| `gateway.reuse.stale` | Warning | Managed gateway metadata requires reconciliation before reuse. | +| `gateway.reuse.active_unnamed` | Warning | A managed gateway is active without the expected named registration. | +| `gateway.reuse.foreign_active` | Blocking | A different active gateway conflicts with the configured gateway. | +| `gateway.reuse.inconclusive` | Warning | NemoClaw could not determine managed gateway reuse state. | +| `gateway.version.drift` | Blocking | The running gateway version does not match the installed OpenShell version. | +| `gateway.version.inconclusive` | Warning | NemoClaw could not determine gateway version drift. | +| `gateway.port.multiple_owners` | Blocking | Multiple listeners claim the managed gateway port. | +| `gateway.port.owner_mismatch` | Blocking | A listener on the managed gateway port failed identity verification. | +| `gateway.port.occupied` | Blocking | An incompatible listener occupies the managed gateway port. | +| `gateway.port.inconclusive` | Warning | NemoClaw could not establish gateway port ownership. | + +The report includes bounded evidence for the resolved owner and applicable attachment, port, collection, or stale-observation failures. +It omits the gateway state directory, removes process environments, redacts credential-shaped content, bounds diagnostic length, and renders control characters visibly. + +Managed gateway metadata is reusable only when its endpoint is bound to loopback on the configured gateway port. +A legacy Docker proxy is accepted only when the active cluster container and its published host port establish the same target. +For a direct Linux host-process gateway, version comparison runs only against an independently resolved binary whose exact path matches the listener's kernel-backed executable identity. +For a package service, the validated systemd `ExecStart` or official Homebrew formula binary must match the live executable before NemoClaw runs that trusted binary's version command. +NemoClaw samples the listener executable on both sides of identity checks and samples an external listener on both sides of its health request. +A PID, executable, service, or listener change during that interval makes the result inconclusive instead of combining evidence from different processes. +A service PID with missing or ambiguous executable evidence produces unknown version compatibility instead of comparing an unrelated CLI sibling. + +An externally supervised declaration is authoritative. +NemoClaw validates its endpoint, supervisor, listener identity, port, and required capabilities. +NemoClaw does not start, stop, replace, or use a standalone fallback for that gateway. +An owner mismatch, multiple listeners, or missing identity evidence blocks onboarding before managed lifecycle effects. + +## Understand Onboarding Admission + +Onboarding consumes the same stable host and gateway entities as `host probe`. +It applies an onboarding policy instead of treating the public report status as the only decision. + +Fresh onboarding and authoritative rebuilds use this order: + +1. Resolve and validate gateway authority. +2. Collect canonical host and gateway facts and reject blocking findings or unknown required capabilities. +3. On a resumed agent change, retain the old router and durable provider state until the full preflight phase succeeds; only then stop the old router and commit the projected agent state. +4. Apply only an explicitly permitted host remediation, if required, and replace the earlier observations. +5. Run a bounded WSL GPU proof when required, then revalidate live host and gateway facts. +6. Validate the effective GPU configuration before Docker bridge and container DNS probes that can start a short-lived container. +7. Install or upgrade OpenShell when required. A successful install can change gateway authority only from managed `standalone` to `packaged-service`. NemoClaw records that transition before later effects. +8. Revalidate gateway authority immediately before gateway selection, recovery, reconciliation, or other lifecycle effects. + +The readiness gate runs before model-router cleanup, provider selection, credential registration, policy changes, image builds, or sandbox lifecycle effects. +Host and gateway observations have a 30-second reuse window. +If collection itself takes too long, onboarding rejects the stale composite instead of assigning a fresh timestamp to old facts. + +The policy permits only these narrow exceptions: + +- An explicit CPU-only sandbox mode makes NVIDIA GPU passthrough findings irrelevant to that run. +- The experimental portable profile can prepare its rootless runtime after an initial admission check. +- A NemoClaw-managed gateway can apply the documented Docker storage remediation when the report says it is available. +- A NemoClaw-managed gateway can reconcile version drift after its ownership and port checks pass. +- An explicit runtime path that does not use the standard Docker driver can admit its documented runtime exception. + +The portable profile discards its earlier host and gateway observations after preparation. +It collects both again before the runtime probes or later lifecycle effects. +No other blocking finding or unknown required capability is waived. + +Resume does not trust a cached preflight result for live host or gateway facts. +It collects both again before a container-backed WSL GPU proof, GPU validation, or bridge and DNS probes. +Explicit CPU-only intent skips the WSL GPU proof. +The gateway phase validates authority again immediately before managed reconciliation or external attachment. + +Authoritative rebuild preflight pins read-only probes to the recorded gateway without selecting, starting, or recovering it. +Immediately before source deletion, the rebuild journal requires the complete gateway authority to match that preflight handoff and carries the journaled authority into replacement onboarding. +Gateway recovery remains after the canonical readiness gate. + +For external supervision, onboarding skips managed gateway selection, reuse refresh, cleanup, start, stop, replace, and standalone fallback paths. +It attaches only after the current external authority passes the readiness checks. + +## Distinguish Sandbox Doctor + +`$$nemoclaw doctor` remains a separate diagnostic workflow for an existing sandbox and its dependencies. +Its report and exit behavior are not the system readiness JSON contract. +An agent runtime can expose an explicit `doctor --fix` repair mode, while `host probe` cannot change state. + +The Docker host check in `doctor` and the system readiness report share the same daemon-reachability semantics. +Both require positive server-version evidence from JSON `docker info` output. +They reject an exit-zero response that contains empty or error-only daemon data. + +## Understand the Read-Only Boundary `host probe` observes the host and does not run remediation. Repeated runs do not change filesystem, service, Docker image or container, gateway, provider, credential, policy, or sandbox state. +Read-only child processes receive a replacement environment containing only exact system variables, the selected OpenShell gateway, its validated local TLS directory when present, and a supported local Unix Docker socket when configured. +Ambient provider, OpenShell, sandbox, and gateway credentials are not forwarded to those children. A future command that changes the host must use a separate mutating mode. That mode cannot run with `--json` probing. diff --git a/scripts/install.sh b/scripts/install.sh index e60a7e5bf1a..5956b6625e8 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -3030,7 +3030,12 @@ repair_installer_nvidia_cdi_spec() { run_installer_host_preflight() { local preflight_module="${NEMOCLAW_SOURCE_ROOT}/dist/lib/onboard/preflight.js" - if ! command_exists node || [[ ! -f "$preflight_module" ]]; then + local host_readiness_module="${NEMOCLAW_SOURCE_ROOT}/dist/lib/readiness/host.js" + local onboard_admission_module="${NEMOCLAW_SOURCE_ROOT}/dist/lib/readiness/onboard-admission.js" + if ! command_exists node \ + || [[ ! -f "$preflight_module" ]] \ + || [[ ! -f "$host_readiness_module" ]] \ + || [[ ! -f "$onboard_admission_module" ]]; then return 0 fi @@ -3041,17 +3046,35 @@ run_installer_host_preflight() { # shellcheck disable=SC2016 node -e ' const preflightPath = process.argv[1]; + const hostReadinessPath = process.argv[2]; + const onboardAdmissionPath = process.argv[3]; try { - const { assessHost, planHostRemediation } = require(preflightPath); + const { assessHost, planHostAdvisories } = require(preflightPath); + const { createHostReadinessReport } = require(hostReadinessPath); + const { evaluateOnboardReadinessAdmission } = require(onboardAdmissionPath); const host = assessHost(); - const actions = planHostRemediation(host); - const blockingActions = actions.filter((action) => action && action.blocking); + const actions = planHostAdvisories(host); + const readiness = createHostReadinessReport( + { nemoclawVersion: "installer", sourceRevision: "installer" }, + { + assess: () => host, + detectGpu: () => null, + detectHostGpuPlatform: () => host.hostGpuPlatform, + detectNvidiaDriverVersion: () => host.nvidiaDriverVersion, + collectPlatformIdentity: () => ({}), + } + ); + const admission = evaluateOnboardReadinessAdmission(readiness, { + explicitlyOptedOutGpuPassthrough: false, + allowUnsupportedRuntime: false, + allowStorageRemediation: false, + }); const infoLines = []; const actionLines = []; if (host.runtime && host.runtime !== "unknown") { infoLines.push(`Detected container runtime: ${host.runtime}`); } - if (host.notes && host.notes.includes("Running under WSL")) { + if (host.isWsl) { infoLines.push("Running under WSL"); } for (const action of actions) { @@ -3066,11 +3089,11 @@ run_installer_host_preflight() { if (actionLines.length > 0) { process.stdout.write(`__ACTIONS__\n${actionLines.join("\n")}`); } - process.exit(blockingActions.length > 0 ? 10 : 0); + process.exit(admission.admitted ? 0 : 10); } catch { process.exit(0); } - ' "$preflight_module" + ' "$preflight_module" "$host_readiness_module" "$onboard_admission_module" )"; then status=0 else diff --git a/src/commands/host/probe.test.ts b/src/commands/host/probe.test.ts index 0730af4b11f..c949e360386 100644 --- a/src/commands/host/probe.test.ts +++ b/src/commands/host/probe.test.ts @@ -9,18 +9,23 @@ import systemReadinessSchema from "../../../schemas/system-readiness.schema.json import type { SystemReadinessReport } from "../../lib/readiness/types"; const mocks = vi.hoisted(() => ({ - createHostReadinessReport: vi.fn(), + createProductionGatewayReadinessDependencies: vi.fn(), createPublicReadinessReport: vi.fn(), + createSystemReadinessReport: vi.fn(), getBuildIdentity: vi.fn(), renderReadinessReport: vi.fn(), })); vi.mock("../../lib/readiness/index", () => ({ - createHostReadinessReport: mocks.createHostReadinessReport, createPublicReadinessReport: mocks.createPublicReadinessReport, + createSystemReadinessReport: mocks.createSystemReadinessReport, renderReadinessReport: mocks.renderReadinessReport, })); +vi.mock("../../lib/readiness/gateway-production", () => ({ + createProductionGatewayReadinessDependencies: mocks.createProductionGatewayReadinessDependencies, +})); + vi.mock("../../lib/core/version", () => ({ getBuildIdentity: mocks.getBuildIdentity, })); @@ -68,13 +73,20 @@ function report(outcome: ReadinessOutcome): SystemReadinessReport { } describe("host probe command (#7412)", () => { + const gatewayDependencies = { + resolveOwner: vi.fn(), + probeAttachment: vi.fn(), + observeManagedGateway: vi.fn(), + }; + beforeEach(() => { vi.clearAllMocks(); mocks.getBuildIdentity.mockReturnValue({ nemoclawVersion: "0.0.96-35-g8bfff4526", sourceRevision: `8bfff4526${"a".repeat(31)}`, }); - mocks.createHostReadinessReport.mockReturnValue(report(READINESS_OUTCOMES[0])); + mocks.createProductionGatewayReadinessDependencies.mockReturnValue(gatewayDependencies); + mocks.createSystemReadinessReport.mockResolvedValue(report(READINESS_OUTCOMES[0])); mocks.createPublicReadinessReport.mockImplementation((value) => value); mocks.renderReadinessReport.mockReturnValue("System readiness: supported"); }); @@ -86,7 +98,7 @@ describe("host probe command (#7412)", () => { it.each(READINESS_OUTCOMES)("returns the deterministic $status exit code", async (outcome) => { const previousExitCode = process.exitCode; process.exitCode = undefined; - mocks.createHostReadinessReport.mockReturnValueOnce(report(outcome)); + mocks.createSystemReadinessReport.mockResolvedValueOnce(report(outcome)); try { await HostProbeCommand.run([], process.cwd()); @@ -98,7 +110,7 @@ describe("host probe command (#7412)", () => { it.each(READINESS_OUTCOMES)("emits schema-valid JSON for $status hosts", async (outcome) => { const expectedReport = report(outcome); - mocks.createHostReadinessReport.mockReturnValueOnce(expectedReport); + mocks.createSystemReadinessReport.mockResolvedValueOnce(expectedReport); const log = vi.spyOn(console, "log").mockImplementation(() => undefined); const previousExitCode = process.exitCode; process.exitCode = undefined; @@ -123,12 +135,16 @@ describe("host probe command (#7412)", () => { await HostProbeCommand.run([], process.cwd()); - expect(mocks.createHostReadinessReport).toHaveBeenCalledWith({ - nemoclawVersion: "0.0.96-35-g8bfff4526", - sourceRevision: `8bfff4526${"a".repeat(31)}`, - }); + expect(mocks.createSystemReadinessReport).toHaveBeenCalledWith( + { + nemoclawVersion: "0.0.96-35-g8bfff4526", + sourceRevision: `8bfff4526${"a".repeat(31)}`, + }, + { gateway: gatewayDependencies }, + ); + expect(mocks.createProductionGatewayReadinessDependencies).toHaveBeenCalledOnce(); expect(mocks.createPublicReadinessReport).toHaveBeenCalledWith( - mocks.createHostReadinessReport.mock.results[0]?.value, + await mocks.createSystemReadinessReport.mock.results[0]?.value, ); expect(mocks.renderReadinessReport).toHaveBeenCalledWith(publicReport); expect(log).toHaveBeenCalledWith("System readiness: supported"); @@ -140,7 +156,8 @@ describe("host probe command (#7412)", () => { await HostProbeCommand.run([], process.cwd()); await HostProbeCommand.run(["--json"], process.cwd()); - expect(mocks.createHostReadinessReport).toHaveBeenCalledTimes(2); + expect(mocks.createSystemReadinessReport).toHaveBeenCalledTimes(2); + expect(mocks.createProductionGatewayReadinessDependencies).toHaveBeenCalledTimes(2); expect(mocks.createPublicReadinessReport).toHaveBeenCalledTimes(2); expect(mocks.renderReadinessReport).toHaveBeenCalledTimes(1); }); diff --git a/src/commands/host/probe.ts b/src/commands/host/probe.ts index 3d6b210b507..11f8bef8564 100644 --- a/src/commands/host/probe.ts +++ b/src/commands/host/probe.ts @@ -4,25 +4,26 @@ import { NemoClawCommand } from "../../lib/cli/nemoclaw-oclif-command"; import { getBuildIdentity } from "../../lib/core/version"; import { - createHostReadinessReport, createPublicReadinessReport, + createSystemReadinessReport, renderReadinessReport, } from "../../lib/readiness"; +import { createProductionGatewayReadinessDependencies } from "../../lib/readiness/gateway-production"; export default class HostProbeCommand extends NemoClawCommand { static id = "host:probe"; static strict = true; static enableJsonFlag = true; - static summary = "Report host readiness without changing the system"; + static summary = "Report host and gateway readiness without changing the system"; static description = - "Inspect host capabilities before onboarding. The command does not remediate findings or change host, Docker, gateway, credential, policy, or sandbox state."; + "Inspect host capabilities and gateway authority before onboarding. The command does not remediate findings or change host, Docker, gateway, credential, policy, or sandbox state."; static usage = ["host probe [--json]"]; static examples = ["<%= config.bin %> host probe", "<%= config.bin %> host probe --json"]; static flags = {}; static publicDisplay = [ { usage: "nemoclaw host probe", - description: "Report host readiness without changing the system", + description: "Report host and gateway readiness without changing the system", flags: "[--json]", group: "Troubleshooting", scope: "global", @@ -32,7 +33,11 @@ export default class HostProbeCommand extends NemoClawCommand { public async run(): Promise { await this.parse(HostProbeCommand); - const report = createPublicReadinessReport(createHostReadinessReport(getBuildIdentity())); + const report = createPublicReadinessReport( + await createSystemReadinessReport(getBuildIdentity(), { + gateway: createProductionGatewayReadinessDependencies(), + }), + ); this.setExitCode(report.exitCode); if (this.jsonEnabled()) return report; this.log(renderReadinessReport(report)); diff --git a/src/lib/actions/sandbox/gateway-target.ts b/src/lib/actions/sandbox/gateway-target.ts index cd125ceb8b7..c3500d1bea6 100644 --- a/src/lib/actions/sandbox/gateway-target.ts +++ b/src/lib/actions/sandbox/gateway-target.ts @@ -2,7 +2,11 @@ // SPDX-License-Identifier: Apache-2.0 import { GATEWAY_PORT } from "../../core/ports"; -import { resolveGatewayName, resolveSandboxGatewayName } from "../../onboard/gateway-binding"; +import { + resolveGatewayName, + resolveSandboxGatewayName, + type SandboxGatewayBinding, +} from "../../onboard/gateway-binding"; import * as registry from "../../state/registry"; export function getKnownSandboxTargetGatewayName(sandboxName = ""): string | null { @@ -14,6 +18,11 @@ export function getSandboxTargetGatewayName(sandboxName = ""): string { return getKnownSandboxTargetGatewayName(sandboxName) ?? resolveGatewayName(GATEWAY_PORT); } +/** Resolve a gateway directly from the already-authoritative persisted row. */ +export function getPersistedSandboxTargetGatewayName(sandbox: SandboxGatewayBinding): string { + return resolveSandboxGatewayName(sandbox); +} + export function gatewayNamePattern(gatewayName: string): RegExp { return new RegExp( `Gateway:\\s+${gatewayName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}(?=\\s|$)`, diff --git a/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts b/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts index 411b9b9b17f..7407778b1c2 100644 --- a/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts +++ b/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts @@ -79,6 +79,16 @@ function stubRecreateJournal(): RebuildRecreateJournal { id: "journal-1", acceptedTarget: false, sourceConfirmedAbsent: false, + gatewayAuthority: { + gatewayName: "nemoclaw", + gatewayPort: 8080, + mode: "nemoclaw-managed", + source: "standalone", + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], + }, targetGeneration: "generation-1", targetIntentFingerprint: "intent-1", markDeleting: vi.fn(), diff --git a/src/lib/actions/sandbox/rebuild-gpu-opt-out.ts b/src/lib/actions/sandbox/rebuild-gpu-opt-out.ts index 0218c1379a9..e0f4000ff97 100644 --- a/src/lib/actions/sandbox/rebuild-gpu-opt-out.ts +++ b/src/lib/actions/sandbox/rebuild-gpu-opt-out.ts @@ -29,6 +29,7 @@ import { normalizeSandboxGpuMode } from "../../onboard/sandbox-gpu-mode"; import type { ManagedWorkloadRebuildHandoff } from "../../onboard/workload/rebuild"; import { getTier } from "../../policy/tiers"; import type { SandboxBaseImageResolutionMetadata } from "../../sandbox-base-image"; +import type { CheckpointGatewayAuthority } from "../../state/onboard-checkpoint-types"; import type { PreservedEnvFile } from "../../state/preserved-env"; import { type ToolDisclosure, toolDisclosureOrDefault } from "../../tool-disclosure"; @@ -124,6 +125,8 @@ export type RebuildRecreateOnboardOpts = { rebuildRegistryInferenceRoute?: RebuildRouteHandoff; rebuildProviderReconfigure?: RebuildProviderReconfigureHandoff; providerRecoveryReceipt?: ProviderRecoveryReceipt; + /** Target-scoped authority admitted by the authoritative rebuild preflight. */ + rebuildGatewayAuthority?: CheckpointGatewayAuthority; preparedImageRebuild?: PreparedImageRebuildHandoff; managedWorkloadRebuild?: ManagedWorkloadRebuildHandoff; rebuildPreservedEnv?: readonly PreservedEnvFile[]; diff --git a/src/lib/actions/sandbox/rebuild-hermes-accepted-target.test.ts b/src/lib/actions/sandbox/rebuild-hermes-accepted-target.test.ts index 3b42e2b330d..f5a8e2b0688 100644 --- a/src/lib/actions/sandbox/rebuild-hermes-accepted-target.test.ts +++ b/src/lib/actions/sandbox/rebuild-hermes-accepted-target.test.ts @@ -14,6 +14,17 @@ const phaseMocks = vi.hoisted(() => ({ runShields: vi.fn(), })); +const gatewayAuthority = { + gatewayName: "nemoclaw", + gatewayPort: 8080, + mode: "nemoclaw-managed", + source: "standalone", + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], +} as const; + vi.mock("./rebuild-recreate-journal", () => ({ fingerprintRebuildRecreateTargetIntent: () => "intent-1", openRebuildRecreateJournal: phaseMocks.openRecreateJournal, @@ -82,6 +93,7 @@ describe("Hermes accepted replacement recovery", () => { observabilityEnabled: false, targetGatewayName: "nemoclaw", targetGatewayPort: 8080, + rebuildGatewayAuthority: gatewayAuthority, }, messagingPlan: null, baseImagePreflight: null, @@ -116,6 +128,7 @@ describe("Hermes accepted replacement recovery", () => { id: "journal-1", acceptedTarget: true, sourceConfirmedAbsent: true, + gatewayAuthority, targetGeneration: "generation-1", targetIntentFingerprint: "intent-1", completeAcceptedTarget, diff --git a/src/lib/actions/sandbox/rebuild-onboard-dependencies.ts b/src/lib/actions/sandbox/rebuild-onboard-dependencies.ts index 392cdacc186..f93d591bc14 100644 --- a/src/lib/actions/sandbox/rebuild-onboard-dependencies.ts +++ b/src/lib/actions/sandbox/rebuild-onboard-dependencies.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import type { CheckpointGatewayAuthority } from "../../state/onboard-checkpoint-types"; import type { RebuildDurableConfig } from "./rebuild-durable-config"; import type { RebuildRecreateOnboardOpts } from "./rebuild-gpu-opt-out"; @@ -20,7 +21,7 @@ type RebuildOnboardModule = { onboard: (options: RebuildRecreateOnboardOpts) => Promise; preflightAuthoritativeRebuildTarget: ( options: RebuildAuthoritativePreflightOptions, - ) => Promise; + ) => Promise; }; function loadOnboardModule(): RebuildOnboardModule { @@ -48,7 +49,7 @@ export const rebuildOnboardDependencies = { }, preflightAuthoritativeRebuildTarget( options: RebuildAuthoritativePreflightOptions, - ): Promise { + ): Promise { return loadOnboardModule().preflightAuthoritativeRebuildTarget(options); }, }; diff --git a/src/lib/actions/sandbox/rebuild-pipeline.ts b/src/lib/actions/sandbox/rebuild-pipeline.ts index f14c7aaed81..3f1c140de4c 100644 --- a/src/lib/actions/sandbox/rebuild-pipeline.ts +++ b/src/lib/actions/sandbox/rebuild-pipeline.ts @@ -280,17 +280,24 @@ async function rebuildSandboxUnlocked( return; } + const expectedGatewayAuthority = recreateOptions.rebuildGatewayAuthority; + if (!expectedGatewayAuthority) { + bail("Authoritative rebuild gateway readiness did not produce an authority handoff."); + return; + } const recreateJournal = openRebuildRecreateJournal({ target: { sandboxName, gatewayName: recreateOptions.targetGatewayName, gatewayPort: recreateOptions.targetGatewayPort, }, + expectedGatewayAuthority, agentName: rebuildAgent || "openclaw", targetIntentFingerprint: fingerprintRebuildRecreateTargetIntent(recreateOptions), log, onAuthorityRefusal: (lines) => bail(lines.join("\n")), }); + recreateOptions.rebuildGatewayAuthority = recreateJournal.gatewayAuthority; // An earlier run of this rebuild already registered and proved the // replacement. Retire its journal and stop before the destroy phase so a diff --git a/src/lib/actions/sandbox/rebuild-preflight-target-phase.test.ts b/src/lib/actions/sandbox/rebuild-preflight-target-phase.test.ts index 38fe099572b..86b7a30144c 100644 --- a/src/lib/actions/sandbox/rebuild-preflight-target-phase.test.ts +++ b/src/lib/actions/sandbox/rebuild-preflight-target-phase.test.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import type { ProviderRecoveryReceipt, @@ -9,10 +9,72 @@ import type { } from "../../onboard/rebuild-route-handoff"; import type { SandboxBaseImageResolutionMetadata } from "../../sandbox-base-image"; import { + pinRebuildTargetGatewayForReadiness, + runRebuildGatewayRecoveryAfterReadiness, stageRebuildBaseImageResolutionHandoff, stageRegistryProviderRecoveryReceipt, } from "./rebuild-preflight-target-phase"; +const originalGateway = process.env.OPENSHELL_GATEWAY; + +afterEach(() => { + originalGateway === undefined + ? Reflect.deleteProperty(process.env, "OPENSHELL_GATEWAY") + : (process.env.OPENSHELL_GATEWAY = originalGateway); +}); + +describe("rebuild readiness gateway pin", () => { + it("pins the recorded target without selecting or recovering a gateway (#7411)", () => { + const log = vi.fn(); + + expect( + pinRebuildTargetGatewayForReadiness( + "alpha", + { gatewayName: "nemoclaw-9443", gatewayPort: 9443 } as never, + log, + ), + ).toBe("nemoclaw-9443"); + expect(process.env.OPENSHELL_GATEWAY).toBe("nemoclaw-9443"); + expect(log).toHaveBeenCalledWith( + "Pinned rebuild readiness probes for 'alpha' to target gateway 'nemoclaw-9443'", + ); + }); + + it("does not select or recover the gateway when readiness rejects (#7411)", async () => { + const afterReadiness = vi.fn(); + const recoverGateway = vi.fn(async () => true); + + await expect( + runRebuildGatewayRecoveryAfterReadiness({ + assertReadiness: async () => false, + afterReadiness, + recoverGateway, + }), + ).resolves.toBe(false); + expect(afterReadiness).not.toHaveBeenCalled(); + expect(recoverGateway).not.toHaveBeenCalled(); + }); + + it("recovers the gateway only after readiness and post-admission staging (#7411)", async () => { + const calls: string[] = []; + + await expect( + runRebuildGatewayRecoveryAfterReadiness({ + assertReadiness: async () => { + calls.push("readiness"); + return true; + }, + afterReadiness: () => calls.push("stage-receipt"), + recoverGateway: async () => { + calls.push("recover-gateway"); + return true; + }, + }), + ).resolves.toBe(true); + expect(calls).toEqual(["readiness", "stage-receipt", "recover-gateway"]); + }); +}); + const target = { sandboxName: "alpha", gatewayName: "nemoclaw", diff --git a/src/lib/actions/sandbox/rebuild-preflight-target-phase.ts b/src/lib/actions/sandbox/rebuild-preflight-target-phase.ts index 473a128439c..9ccdf93d231 100644 --- a/src/lib/actions/sandbox/rebuild-preflight-target-phase.ts +++ b/src/lib/actions/sandbox/rebuild-preflight-target-phase.ts @@ -23,7 +23,10 @@ import { readSandboxBaseImageResolutionMetadata } from "../../sandbox-base-image import * as registry from "../../state/registry"; import type { ToolDisclosure } from "../../tool-disclosure"; import { prepareManagedRebuildProfileHandoff } from "./agents/managed-workload-rebuild-profile"; -import { getSandboxTargetGatewayName } from "./gateway-target"; +import { + getPersistedSandboxTargetGatewayName, + getSandboxTargetGatewayName, +} from "./gateway-target"; import type { RebuildBail, RebuildLog } from "./rebuild-credential-preflight"; import type { PreparedRebuildImage } from "./rebuild-custom-image-preflight"; import { isDcodeRebuildAgent } from "./rebuild-dcode-orchestrator"; @@ -89,6 +92,29 @@ export interface RebuildPreparedTarget { routePreflightReceipt: RebuildRoutePreflightReceipt; } +/** Pin read-only rebuild probes without selecting, starting, or repairing a gateway. */ +export function pinRebuildTargetGatewayForReadiness( + sandboxName: string, + sandboxEntry: RebuildSandboxEntry, + log: RebuildLog, +): string { + const gatewayName = getPersistedSandboxTargetGatewayName(sandboxEntry); + process.env.OPENSHELL_GATEWAY = gatewayName; + log(`Pinned rebuild readiness probes for '${sandboxName}' to target gateway '${gatewayName}'`); + return gatewayName; +} + +/** Keep gateway selection and recovery behind authoritative readiness. */ +export async function runRebuildGatewayRecoveryAfterReadiness(options: { + assertReadiness(): Promise; + afterReadiness(): void; + recoverGateway(): Promise; +}): Promise { + if (!(await options.assertReadiness())) return false; + options.afterReadiness(); + return options.recoverGateway(); +} + /** Carry the outer resolver's verified provenance into the inner onboard build. */ export function stageRebuildBaseImageResolutionHandoff( recreateOptions: Pick, @@ -144,8 +170,7 @@ export async function prepareRebuildTargetPreflights(args: { bail, } = args; hydrateMessagingConfigForRebuild(sandboxName, log); - if (!(await ensureRebuildTargetGatewaySelected(sandboxName, sandboxEntry, log, bail))) - return null; + pinRebuildTargetGatewayForReadiness(sandboxName, sandboxEntry, log); const targetConfig = prepareRebuildTargetConfig( sandboxName, @@ -250,29 +275,29 @@ export async function prepareRebuildTargetPreflights(args: { error: (message) => console.error(message), bail, }); - if ( - !(await preflightAuthoritativeOnboardRuntime( - sandboxName, - resumeConfig, - recreateOptions, - bail, - preparedBackupRecovery ? { deferInferenceRouteUntilOnboard: true } : {}, - )) - ) { - return null; - } - stageRegistryProviderRecoveryReceipt( - recreateOptions, - { - sandboxName, - gatewayName: recreateOptions.targetGatewayName, - provider: resumeConfig.provider, - model: resumeConfig.model, - }, - resumeConfig.registryInferenceRoute, - ); - if (!(await ensureRebuildTargetGatewaySelected(sandboxName, sandboxEntry, log, bail))) - return null; + const gatewayRecovered = await runRebuildGatewayRecoveryAfterReadiness({ + assertReadiness: () => + preflightAuthoritativeOnboardRuntime( + sandboxName, + resumeConfig, + recreateOptions, + bail, + preparedBackupRecovery ? { deferInferenceRouteUntilOnboard: true } : {}, + ), + afterReadiness: () => + stageRegistryProviderRecoveryReceipt( + recreateOptions, + { + sandboxName, + gatewayName: recreateOptions.targetGatewayName, + provider: resumeConfig.provider, + model: resumeConfig.model, + }, + resumeConfig.registryInferenceRoute, + ), + recoverGateway: () => ensureRebuildTargetGatewaySelected(sandboxName, sandboxEntry, log, bail), + }); + if (!gatewayRecovered) return null; if (!checkRebuildGatewaySchemaPreflight(sandboxName, sandboxEntry, bail)) return null; const rebuildsDcodeSandbox = isDcodeRebuildAgent(rebuildAgent); diff --git a/src/lib/actions/sandbox/rebuild-recreate-journal.test.ts b/src/lib/actions/sandbox/rebuild-recreate-journal.test.ts index 9479e7fff4b..a8e813c524c 100644 --- a/src/lib/actions/sandbox/rebuild-recreate-journal.test.ts +++ b/src/lib/actions/sandbox/rebuild-recreate-journal.test.ts @@ -13,10 +13,12 @@ vi.mock("../../adapters/openshell/runtime", () => ({ runOpenshell: vi.fn(), })); -vi.mock("../../onboard/gateway-teardown-authority", () => ({ +vi.mock("../../onboard/gateway-teardown-authority", async (importOriginal) => ({ + ...(await importOriginal()), resolveGatewayRebuildAuthority: mocks.resolveGatewayRebuildAuthority, })); +import type { CheckpointGatewayAuthority } from "../../state/onboard-checkpoint-types"; import type { Session } from "../../state/onboard-session"; import * as onboardSession from "../../state/onboard-session"; import * as registry from "../../state/registry"; @@ -35,6 +37,17 @@ const NON_DEFAULT_TARGET = { gatewayPort: 9090, }; +const STANDALONE_GATEWAY_AUTHORITY: CheckpointGatewayAuthority = { + gatewayName: "nemoclaw-9090", + gatewayPort: 9090, + mode: "nemoclaw-managed", + source: "standalone", + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], +}; + const recreateOptions: RebuildRecreateOnboardOpts = { resume: true, nonInteractive: true, @@ -212,16 +225,7 @@ describe("rebuild replacement journal", () => { gatewayName: "nemoclaw-9090", gatewayPort: 9090, } as registry.SandboxEntry); - mocks.resolveGatewayRebuildAuthority.mockReturnValue({ - gatewayName: "nemoclaw-9090", - gatewayPort: 9090, - mode: "nemoclaw-managed", - source: "standalone", - endpoint: null, - stateDir: null, - supervisor: null, - requiredCapabilities: [], - }); + mocks.resolveGatewayRebuildAuthority.mockReturnValue(STANDALONE_GATEWAY_AUTHORITY); mocks.captureOpenshell.mockReturnValue(livePresentProbe()); }); @@ -232,6 +236,7 @@ describe("rebuild replacement journal", () => { function open() { return openRebuildRecreateJournal({ target: NON_DEFAULT_TARGET, + expectedGatewayAuthority: STANDALONE_GATEWAY_AUTHORITY, agentName: "langchain-deepagents-code", targetIntentFingerprint: fingerprintRebuildRecreateTargetIntent(recreateOptions), log: vi.fn(), @@ -251,7 +256,7 @@ describe("rebuild replacement journal", () => { }); it("selects the exact gateway authority the journal records", () => { - open(); + const journal = open(); expect(mocks.resolveGatewayRebuildAuthority).toHaveBeenCalledWith({ gatewayName: "nemoclaw-9090", @@ -261,6 +266,29 @@ describe("rebuild replacement journal", () => { expect(authority?.kind).toBe("selected"); expect(authority?.kind === "selected" && authority.value.gatewayPort).toBe(9090); expect(authority?.kind === "selected" && authority.value.source).toBe("standalone"); + expect(authority?.kind === "selected" && authority.value).toBe(journal.gatewayAuthority); + }); + + it("refuses an authority change after preflight before writing the journal (#7411)", () => { + const onAuthorityRefusal = vi.fn(); + + expect(() => + openRebuildRecreateJournal({ + target: NON_DEFAULT_TARGET, + expectedGatewayAuthority: { + ...STANDALONE_GATEWAY_AUTHORITY, + source: "packaged-service", + }, + agentName: "langchain-deepagents-code", + targetIntentFingerprint: fingerprintRebuildRecreateTargetIntent(recreateOptions), + log: vi.fn(), + onAuthorityRefusal, + }), + ).toThrow(/authority changed after authoritative rebuild preflight/); + + expect(onAuthorityRefusal).toHaveBeenCalledOnce(); + expect(onboardSession.updateSession).not.toHaveBeenCalled(); + expect(session.checkpoint?.sandboxRecreate ?? null).toBeNull(); }); it("starts at deleted when the source sandbox is already absent", () => { @@ -320,6 +348,7 @@ describe("rebuild replacement journal", () => { expect(() => openRebuildRecreateJournal({ target: NON_DEFAULT_TARGET, + expectedGatewayAuthority: STANDALONE_GATEWAY_AUTHORITY, agentName: "langchain-deepagents-code", targetIntentFingerprint: fingerprintRebuildRecreateTargetIntent({ ...recreateOptions, @@ -336,6 +365,7 @@ describe("rebuild replacement journal", () => { expect(() => openRebuildRecreateJournal({ target: NON_DEFAULT_TARGET, + expectedGatewayAuthority: STANDALONE_GATEWAY_AUTHORITY, agentName: "langchain-deepagents-code", targetIntentFingerprint: fingerprintRebuildRecreateTargetIntent({ ...recreateOptions, @@ -356,6 +386,7 @@ describe("rebuild replacement journal", () => { expect(() => openRebuildRecreateJournal({ target: NON_DEFAULT_TARGET, + expectedGatewayAuthority: STANDALONE_GATEWAY_AUTHORITY, agentName: "langchain-deepagents-code", targetIntentFingerprint: fingerprintRebuildRecreateTargetIntent({ ...recreateOptions, diff --git a/src/lib/actions/sandbox/rebuild-recreate-journal.ts b/src/lib/actions/sandbox/rebuild-recreate-journal.ts index e059e84433a..27bc6617e32 100644 --- a/src/lib/actions/sandbox/rebuild-recreate-journal.ts +++ b/src/lib/actions/sandbox/rebuild-recreate-journal.ts @@ -1,7 +1,11 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { checkpointGatewayAuthority } from "../../onboard/gateway-authority-checkpoint"; +import { + checkpointGatewayAuthority, + gatewayOwnerFromCheckpoint, +} from "../../onboard/gateway-authority-checkpoint"; +import { describeGatewayOwnerForError, sameGatewayOwner } from "../../onboard/gateway-ownership"; import { GatewayAuthorityError, gatewayAuthorityFailureLines, @@ -23,7 +27,10 @@ import { } from "../../onboard/sandbox-recreate-transaction"; import { decisionSelected } from "../../state/onboard-checkpoint-decision"; import { deriveCheckpointFromSession } from "../../state/onboard-checkpoint-migrate"; -import type { CheckpointSandboxRecreatePhase } from "../../state/onboard-checkpoint-types"; +import type { + CheckpointGatewayAuthority, + CheckpointSandboxRecreatePhase, +} from "../../state/onboard-checkpoint-types"; import * as onboardSession from "../../state/onboard-session"; import * as registry from "../../state/registry"; import type { RebuildRecreateOnboardOpts } from "./rebuild-gpu-opt-out"; @@ -38,6 +45,7 @@ export interface RebuildRecreateJournal { readonly id: string; readonly acceptedTarget: boolean; readonly sourceConfirmedAbsent: boolean; + readonly gatewayAuthority: CheckpointGatewayAuthority; readonly targetGeneration: string; readonly targetIntentFingerprint: string; markDeleting(): void; @@ -90,6 +98,7 @@ export const observeRebuildSandbox = observeSandboxOnGateway; export interface OpenRebuildRecreateJournalInput { readonly target: RebuildRecreateJournalTarget; + readonly expectedGatewayAuthority: CheckpointGatewayAuthority; readonly agentName: string; readonly targetIntentFingerprint: string; readonly log: (message: string) => void; @@ -116,11 +125,20 @@ export function openRebuildRecreateJournal( gatewayName: target.gatewayName, gatewayPort: target.gatewayPort, }); + const expectedAuthority = gatewayOwnerFromCheckpoint(input.expectedGatewayAuthority); + if (!sameGatewayOwner(expectedAuthority, authority)) { + throw new GatewayAuthorityError( + "Gateway lifecycle authority changed after authoritative rebuild preflight " + + `(${describeGatewayOwnerForError(expectedAuthority)} -> ${describeGatewayOwnerForError(authority)}). ` + + "Retry the rebuild; the current run will not delete the source sandbox.", + ); + } } catch (error) { if (!(error instanceof GatewayAuthorityError)) throw error; input.onAuthorityRefusal?.(gatewayAuthorityFailureLines(error, "sandbox rebuild")); throw error; } + const gatewayAuthority = checkpointGatewayAuthority(authority); const sourceEntry = registry.getSandbox(target.sandboxName); const observation = observe(target); const active = onboardSession.loadSession()?.checkpoint?.sandboxRecreate ?? null; @@ -144,7 +162,7 @@ export function openRebuildRecreateJournal( machineState: current.machine.state, updatedAt: new Date().toISOString(), sandboxIdentity: decisionSelected({ name: target.sandboxName, agent: agentName }), - gatewayAuthority: decisionSelected(checkpointGatewayAuthority(authority)), + gatewayAuthority: decisionSelected(gatewayAuthority), }; beginSandboxRecreateTransaction(current, { sandboxName: target.sandboxName, @@ -179,6 +197,7 @@ export function openRebuildRecreateJournal( id: transaction.id, acceptedTarget, sourceConfirmedAbsent: recovery.action === "continue_create", + gatewayAuthority, targetGeneration: transaction.targetGeneration, targetIntentFingerprint: transaction.targetIntentFingerprint, markDeleting: () => { diff --git a/src/lib/actions/sandbox/rebuild-recreate-observability.test.ts b/src/lib/actions/sandbox/rebuild-recreate-observability.test.ts index 6ca11f89afd..957d641e681 100644 --- a/src/lib/actions/sandbox/rebuild-recreate-observability.test.ts +++ b/src/lib/actions/sandbox/rebuild-recreate-observability.test.ts @@ -7,6 +7,7 @@ import { restoreEnv } from "../../../../test/helpers/env-test-helpers"; import * as shields from "../../shields"; import { decisionSelected } from "../../state/onboard-checkpoint-decision"; import { deriveCheckpointFromSession } from "../../state/onboard-checkpoint-migrate"; +import type { CheckpointGatewayAuthority } from "../../state/onboard-checkpoint-types"; import type { Session } from "../../state/onboard-session"; import * as onboardSession from "../../state/onboard-session"; import type { RebuildDurableConfig } from "./rebuild-durable-config"; @@ -17,6 +18,22 @@ import type { RebuildResumeConfig } from "./rebuild-resume-config"; const DCODE_AGENT = "langchain-deepagents-code"; +const STANDALONE_GATEWAY_AUTHORITY: CheckpointGatewayAuthority = { + gatewayName: "nemoclaw", + gatewayPort: 8080, + mode: "nemoclaw-managed", + source: "standalone", + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], +}; + +const PACKAGED_GATEWAY_AUTHORITY: CheckpointGatewayAuthority = { + ...STANDALONE_GATEWAY_AUTHORITY, + source: "packaged-service", +}; + const durableConfig: RebuildDurableConfig = { dcodeAutoApprovalMode: "disabled", dcodeAutoApprovalModeError: null, @@ -70,8 +87,37 @@ const recreateOptions: RebuildRecreateOnboardOpts = { observabilityRequestedExplicitly: true, policyTier: "restricted", baseImageResolutionHint: null, + rebuildGatewayAuthority: STANDALONE_GATEWAY_AUTHORITY, }; +function seedRecreateJournalCheckpoint( + session: Session, + gatewayAuthority: CheckpointGatewayAuthority = STANDALONE_GATEWAY_AUTHORITY, +): void { + session.checkpoint = { + ...deriveCheckpointFromSession(session), + sandboxIdentity: decisionSelected({ name: "alpha", agent: DCODE_AGENT }), + gatewayAuthority: decisionSelected(gatewayAuthority), + sandboxRecreate: { + version: 1, + id: "journal-1", + revision: 3, + sandboxName: "alpha", + gatewayName: "nemoclaw", + gatewayPort: 8080, + sourceRegistryFingerprint: "source-registry", + sourceLiveIdentityFingerprint: "source-identity", + sourceWorkload: null, + targetIntentFingerprint: "intent-1", + targetGeneration: "generation-1", + targetLiveIdentityFingerprint: null, + phase: "deleted", + startedAt: "2026-07-28T00:00:00.000Z", + updatedAt: "2026-07-28T00:00:01.000Z", + }, + }; +} + function makeInput(overrides: Partial = {}): RebuildRecreatePhaseInput { return { sandboxName: "alpha", @@ -93,6 +139,7 @@ function makeInput(overrides: Partial = {}): RebuildR id: "journal-1", acceptedTarget: false, sourceConfirmedAbsent: false, + gatewayAuthority: STANDALONE_GATEWAY_AUTHORITY, targetGeneration: "generation-1", targetIntentFingerprint: "intent-1", markDeleting: vi.fn(), @@ -132,6 +179,7 @@ describe("runRebuildRecreatePhase handoff", () => { sandboxName: "alpha", observabilityEnabled: false, }); + seedRecreateJournalCheckpoint(session); vi.spyOn(console, "log").mockImplementation(() => undefined); vi.spyOn(console, "error").mockImplementation(() => undefined); vi.spyOn(onboardSession, "loadSession").mockImplementation(() => session); @@ -192,37 +240,6 @@ describe("runRebuildRecreatePhase handoff", () => { }); it("carries the replacement journal and its target fingerprint into inner onboard (#7734)", async () => { - session.checkpoint = { - ...deriveCheckpointFromSession(session), - sandboxIdentity: decisionSelected({ name: "alpha", agent: DCODE_AGENT }), - gatewayAuthority: decisionSelected({ - gatewayName: "nemoclaw", - gatewayPort: 8080, - mode: "nemoclaw-managed", - source: "standalone", - endpoint: null, - stateDir: null, - supervisor: null, - requiredCapabilities: [], - }), - sandboxRecreate: { - version: 1, - id: "journal-1", - revision: 3, - sandboxName: "alpha", - gatewayName: "nemoclaw", - gatewayPort: 8080, - sourceRegistryFingerprint: "source-registry", - sourceLiveIdentityFingerprint: "source-identity", - sourceWorkload: null, - targetIntentFingerprint: "intent-1", - targetGeneration: "generation-1", - targetLiveIdentityFingerprint: null, - phase: "deleted", - startedAt: "2026-07-28T00:00:00.000Z", - updatedAt: "2026-07-28T00:00:01.000Z", - }, - }; const retiredSessionId = session.sessionId; let observedFingerprint: string | null | undefined; let observedJournalPhase: string | undefined; @@ -249,6 +266,40 @@ describe("runRebuildRecreatePhase handoff", () => { } }); + it("carries the journal authority instead of a stale preflight option (#7411)", async () => { + let observedAuthority: unknown; + vi.spyOn(rebuildOnboardDependencies, "onboard").mockImplementation(async (options) => { + observedAuthority = options.rebuildGatewayAuthority; + }); + + await expect( + runRebuildRecreatePhase( + makeInput({ + recreateOptions: { + ...recreateOptions, + rebuildGatewayAuthority: PACKAGED_GATEWAY_AUTHORITY, + }, + }), + ), + ).resolves.toBe(true); + + expect(observedAuthority).toBe(STANDALONE_GATEWAY_AUTHORITY); + const selected = onboardSession.loadSession()?.checkpoint?.gatewayAuthority; + expect(selected?.kind === "selected" && selected.value).toBe(STANDALONE_GATEWAY_AUTHORITY); + }); + + it("refuses a changed checkpoint authority before recreating the sandbox (#7411)", async () => { + seedRecreateJournalCheckpoint(session, PACKAGED_GATEWAY_AUTHORITY); + const onboardSpy = vi.spyOn(rebuildOnboardDependencies, "onboard"); + + await expect(runRebuildRecreatePhase(makeInput())).rejects.toThrow( + "bail: Authoritative rebuild journal authority changed before sandbox recreation.", + ); + + expect(onboardSession.updateSession).not.toHaveBeenCalled(); + expect(onboardSpy).not.toHaveBeenCalled(); + }); + it("pins the authoritative restricted tier during recreate and restores ambient policy input", async () => { const previousPolicyTier = process.env.NEMOCLAW_POLICY_TIER; process.env.NEMOCLAW_POLICY_TIER = "open"; @@ -432,6 +483,7 @@ describe("rebuild recreate shields state", () => { sandboxName: "alpha", observabilityEnabled: false, }); + seedRecreateJournalCheckpoint(session); vi.spyOn(console, "log").mockImplementation(() => undefined); vi.spyOn(console, "error").mockImplementation(() => undefined); vi.spyOn(onboardSession, "loadSession").mockImplementation(() => session); diff --git a/src/lib/actions/sandbox/rebuild-recreate-phase.ts b/src/lib/actions/sandbox/rebuild-recreate-phase.ts index 9139b383e0a..0433fd958b5 100644 --- a/src/lib/actions/sandbox/rebuild-recreate-phase.ts +++ b/src/lib/actions/sandbox/rebuild-recreate-phase.ts @@ -5,8 +5,11 @@ import { CLI_NAME } from "../../cli/branding"; import { RD as _RD, R } from "../../cli/terminal-style"; import { MessagingSetupApplier, type SandboxMessagingPlan } from "../../messaging"; import { markLastStartedStepFailed } from "../../onboard/exit-step-failure"; +import { gatewayOwnerFromCheckpoint } from "../../onboard/gateway-authority-checkpoint"; +import { sameGatewayOwner } from "../../onboard/gateway-ownership"; import { applyReasoningEffortEnv } from "../../onboard/reasoning-mode"; import * as shields from "../../shields"; +import { decisionSelected, isDecisionSelected } from "../../state/onboard-checkpoint-decision"; import { deriveCheckpointFromSession } from "../../state/onboard-checkpoint-migrate"; import type { Session } from "../../state/onboard-session"; import * as onboardSession from "../../state/onboard-session"; @@ -106,8 +109,46 @@ export async function runRebuildRecreatePhase(input: RebuildRecreatePhaseInput): `Session before update: sandboxName=${sessionBefore?.sandboxName}, status=${sessionBefore?.status}, resumable=${sessionBefore?.resumable}, provider=${sessionBefore?.provider}, model=${sessionBefore?.model}, sessionMatch=${sessionMatchesSandbox}`, ); + const rebuildGatewayAuthority = recreateJournal.gatewayAuthority; + if ( + rebuildGatewayAuthority.gatewayName !== recreateOptions.targetGatewayName || + rebuildGatewayAuthority.gatewayPort !== recreateOptions.targetGatewayPort + ) { + return bail("Authoritative rebuild journal authority does not match the target gateway."); + } + const journaledCheckpoint = onboardSession.loadSession()?.checkpoint ?? null; + if (!journaledCheckpoint) { + return bail("Authoritative rebuild journal does not identify the target sandbox."); + } + const journaledSandboxIdentity = journaledCheckpoint.sandboxIdentity; + if (!isDecisionSelected(journaledSandboxIdentity)) { + return bail("Authoritative rebuild journal does not identify the target sandbox."); + } + const journaledRecreate = journaledCheckpoint.sandboxRecreate; + if ( + journaledSandboxIdentity.value.name !== sandboxName || + !journaledRecreate || + journaledRecreate.id !== recreateJournal.id || + journaledRecreate.sandboxName !== sandboxName || + journaledRecreate.gatewayName !== recreateOptions.targetGatewayName || + journaledRecreate.gatewayPort !== recreateOptions.targetGatewayPort || + journaledRecreate.targetGeneration !== recreateJournal.targetGeneration || + journaledRecreate.targetIntentFingerprint !== recreateJournal.targetIntentFingerprint + ) { + return bail("Authoritative rebuild journal does not match the target replacement."); + } + const journaledGatewayAuthority = journaledCheckpoint.gatewayAuthority; + if ( + !isDecisionSelected(journaledGatewayAuthority) || + !sameGatewayOwner( + gatewayOwnerFromCheckpoint(journaledGatewayAuthority.value), + gatewayOwnerFromCheckpoint(rebuildGatewayAuthority), + ) + ) { + return bail("Authoritative rebuild journal authority changed before sandbox recreation."); + } + onboardSession.updateSession((s: Session) => { - const journaledCheckpoint = s.checkpoint; Object.assign( s, onboardSession.createSession({ @@ -161,14 +202,12 @@ export async function runRebuildRecreatePhase(input: RebuildRecreatePhaseInput): s.observabilityRequestedExplicitly = recreateOptions.observabilityRequestedExplicitly; // The journal outlives this reset, but the retired session owns its effect // receipts and bindings. Rebind only the values the journal invariant needs. - s.checkpoint = journaledCheckpoint - ? { - ...deriveCheckpointFromSession(s), - sandboxIdentity: journaledCheckpoint.sandboxIdentity, - gatewayAuthority: journaledCheckpoint.gatewayAuthority, - sandboxRecreate: journaledCheckpoint.sandboxRecreate, - } - : null; + s.checkpoint = { + ...deriveCheckpointFromSession(s), + sandboxIdentity: journaledSandboxIdentity, + gatewayAuthority: decisionSelected(rebuildGatewayAuthority), + sandboxRecreate: journaledRecreate, + }; return s; }); const sessionAfter = onboardSession.loadSession(); @@ -225,6 +264,7 @@ export async function runRebuildRecreatePhase(input: RebuildRecreatePhaseInput): try { await rebuildOnboardDependencies.onboard({ ...recreateOptions, + rebuildGatewayAuthority, ...(rebuildsHermesSandbox && backupManifest?.preservedEnv ? { rebuildPreservedEnv: backupManifest.preservedEnv } : {}), diff --git a/src/lib/actions/sandbox/rebuild-recreate-reasoning.test.ts b/src/lib/actions/sandbox/rebuild-recreate-reasoning.test.ts index cff587bff4d..55b1355942a 100644 --- a/src/lib/actions/sandbox/rebuild-recreate-reasoning.test.ts +++ b/src/lib/actions/sandbox/rebuild-recreate-reasoning.test.ts @@ -5,6 +5,9 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { restoreEnv } from "../../../../test/helpers/env-test-helpers"; import * as shields from "../../shields"; +import { decisionSelected } from "../../state/onboard-checkpoint-decision"; +import { deriveCheckpointFromSession } from "../../state/onboard-checkpoint-migrate"; +import type { CheckpointGatewayAuthority } from "../../state/onboard-checkpoint-types"; import type { Session } from "../../state/onboard-session"; import * as onboardSession from "../../state/onboard-session"; import type { RebuildDurableConfig } from "./rebuild-durable-config"; @@ -16,6 +19,17 @@ import type { RebuildResumeConfig } from "./rebuild-resume-config"; const SANDBOX_NAME = "rebuild-reasoning-effort"; +const GATEWAY_AUTHORITY: CheckpointGatewayAuthority = { + gatewayName: "nemoclaw", + gatewayPort: 8080, + mode: "nemoclaw-managed", + source: "standalone", + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], +}; + const durableConfig: RebuildDurableConfig = { dcodeAutoApprovalMode: "disabled", dcodeAutoApprovalModeError: null, @@ -69,12 +83,14 @@ const recreateOptions: RebuildRecreateOnboardOpts = { observabilityRequestedExplicitly: false, policyTier: null, baseImageResolutionHint: null, + rebuildGatewayAuthority: GATEWAY_AUTHORITY, }; const recreateJournal: RebuildRecreateJournal = { id: "11111111-1111-4111-8111-111111111111", acceptedTarget: false, sourceConfirmedAbsent: true, + gatewayAuthority: GATEWAY_AUTHORITY, targetGeneration: "22222222-2222-4222-8222-222222222222", targetIntentFingerprint: "rebuild-reasoning-target", markDeleting: vi.fn(), @@ -126,6 +142,28 @@ describe("rebuild recreate compatible-endpoint reasoning handoff (#7940)", () => previousReasoning = process.env.NEMOCLAW_REASONING; previousReasoningEffort = process.env.NEMOCLAW_REASONING_EFFORT; session = onboardSession.createSession({ sandboxName: SANDBOX_NAME }); + session.checkpoint = { + ...deriveCheckpointFromSession(session), + sandboxIdentity: decisionSelected({ name: SANDBOX_NAME, agent: "openclaw" }), + gatewayAuthority: decisionSelected(GATEWAY_AUTHORITY), + sandboxRecreate: { + version: 1, + id: recreateJournal.id, + revision: 3, + sandboxName: SANDBOX_NAME, + gatewayName: GATEWAY_AUTHORITY.gatewayName, + gatewayPort: GATEWAY_AUTHORITY.gatewayPort, + sourceRegistryFingerprint: "source-registry", + sourceLiveIdentityFingerprint: null, + sourceWorkload: null, + targetIntentFingerprint: recreateJournal.targetIntentFingerprint, + targetGeneration: recreateJournal.targetGeneration, + targetLiveIdentityFingerprint: null, + phase: "deleted", + startedAt: "2026-08-07T00:00:00.000Z", + updatedAt: "2026-08-07T00:00:01.000Z", + }, + }; vi.spyOn(console, "log").mockImplementation(() => undefined); vi.spyOn(console, "error").mockImplementation(() => undefined); vi.spyOn(shields, "clearShieldsState").mockImplementation(() => undefined); diff --git a/src/lib/actions/sandbox/rebuild-resume-snapshot.test.ts b/src/lib/actions/sandbox/rebuild-resume-snapshot.test.ts index 5477e4fc444..0709b09b695 100644 --- a/src/lib/actions/sandbox/rebuild-resume-snapshot.test.ts +++ b/src/lib/actions/sandbox/rebuild-resume-snapshot.test.ts @@ -88,22 +88,32 @@ describe("rebuild resume snapshot repair", () => { session = cloneSession((mutator as (value: Session) => Session | void)(current) ?? current); return loadSession(); }; + const resolveGatewayAuthority = ({ + gatewayName, + gatewayPort, + }: { + gatewayName: string; + gatewayPort: number; + }) => ({ + gatewayName, + gatewayPort, + mode: "nemoclaw-managed" as const, + source: "standalone" as const, + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], + }); spies.push( vi.spyOn(gatewayDrift, "detectOpenShellStateRpcPreflightIssue").mockReturnValue(null), vi.spyOn(gatewayDrift, "detectOpenShellStateRpcResultIssue").mockReturnValue(null), vi .spyOn(gatewayTeardownAuthority, "resolveGatewayTeardownAuthority") - .mockImplementation(({ gatewayName, gatewayPort }) => ({ - gatewayName, - gatewayPort, - mode: "nemoclaw-managed", - source: "standalone", - endpoint: null, - stateDir: null, - supervisor: null, - requiredCapabilities: [], - })), + .mockImplementation(resolveGatewayAuthority), + vi + .spyOn(gatewayTeardownAuthority, "resolveGatewayRebuildAuthority") + .mockImplementation(resolveGatewayAuthority), vi.spyOn(gatewayRuntime, "recoverNamedGatewayRuntime").mockResolvedValue({ recovered: true, before: { state: "healthy_named", status: "", gatewayInfo: "", activeGateway: null }, @@ -211,7 +221,16 @@ describe("rebuild resume snapshot repair", () => { vi.spyOn(nim, "detectGpu").mockReturnValue(null), vi .spyOn(rebuildOnboardDependencies, "preflightAuthoritativeRebuildTarget") - .mockResolvedValue(undefined), + .mockResolvedValue({ + gatewayName: "nemoclaw", + gatewayPort: 8080, + mode: "nemoclaw-managed", + source: "standalone", + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], + }), vi.spyOn(rebuildImagePreflight, "preflightRebuildImage").mockResolvedValue({ ok: true, imageTag: null, diff --git a/src/lib/actions/sandbox/rebuild-shields-finally.test.ts b/src/lib/actions/sandbox/rebuild-shields-finally.test.ts index 86966bfa125..732b22ee444 100644 --- a/src/lib/actions/sandbox/rebuild-shields-finally.test.ts +++ b/src/lib/actions/sandbox/rebuild-shields-finally.test.ts @@ -11,6 +11,17 @@ const phaseMocks = vi.hoisted(() => ({ openRecreateJournal: vi.fn(), })); +const gatewayAuthority = { + gatewayName: "nemoclaw", + gatewayPort: 8080, + mode: "nemoclaw-managed", + source: "standalone", + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], +} as const; + vi.mock("./rebuild-recreate-journal", () => ({ fingerprintRebuildRecreateTargetIntent: () => "intent-1", openRebuildRecreateJournal: phaseMocks.openRecreateJournal, @@ -55,6 +66,7 @@ describe("rebuild shields relock guard", () => { observabilityEnabled: false, targetGatewayName: "nemoclaw", targetGatewayPort: 8080, + rebuildGatewayAuthority: gatewayAuthority, }, liveState: { staleRecovery: false, staleRegistrySnapshot: null }, recoveryManifest: null, @@ -77,6 +89,7 @@ describe("rebuild shields relock guard", () => { }); phaseMocks.openRecreateJournal.mockReturnValue({ id: "journal-1", + gatewayAuthority, targetGeneration: "generation-1", targetIntentFingerprint: "intent-1", markDeleting: vi.fn(), diff --git a/src/lib/actions/sandbox/rebuild-target-runtime.ts b/src/lib/actions/sandbox/rebuild-target-runtime.ts index e92f7d3da04..3a073990fcd 100644 --- a/src/lib/actions/sandbox/rebuild-target-runtime.ts +++ b/src/lib/actions/sandbox/rebuild-target-runtime.ts @@ -280,13 +280,14 @@ export async function preflightAuthoritativeOnboardRuntime( options: { deferInferenceRouteUntilOnboard?: true } = {}, ): Promise { try { - await rebuildOnboardDependencies.preflightAuthoritativeRebuildTarget({ + const gatewayAuthority = await rebuildOnboardDependencies.preflightAuthoritativeRebuildTarget({ ...recreateOptions, ...options, model: resumeConfig.model, provider: resumeConfig.provider, sandboxName, }); + recreateOptions.rebuildGatewayAuthority = gatewayAuthority; return true; } catch (err) { printRebuildPreflightFailure( diff --git a/src/lib/adapters/openshell/gateway-drift.test.ts b/src/lib/adapters/openshell/gateway-drift.test.ts index 9d1bf9ab98a..3008874cdeb 100644 --- a/src/lib/adapters/openshell/gateway-drift.test.ts +++ b/src/lib/adapters/openshell/gateway-drift.test.ts @@ -12,6 +12,8 @@ const { formatOpenShellStateRpcIssue, getGatewayClusterImageDrift, getGatewayHostProcessDrift, + isGatewayClusterActiveForGateway, + observeOpenShellGatewayVersionCompatibility, parseGatewayClusterImageVersion, } = requireDist("./gateway-drift.ts") as typeof import("./gateway-drift"); @@ -114,6 +116,8 @@ describe("OpenShell gateway drift preflight", () => { currentVersion: "0.0.36", expectedVersion: "0.0.37", }); + expect(isGatewayClusterActiveForGateway("nemoclaw", { expectedGatewayPort: 8080 })).toBe(true); + expect(isGatewayClusterActiveForGateway("nemoclaw", { expectedGatewayPort: 9090 })).toBe(false); }); it("ignores stale cluster containers whose published port is not the active gateway endpoint", () => { @@ -269,6 +273,79 @@ describe("OpenShell gateway drift preflight", () => { ).toBeNull(); }); + it.each([ + ["0.0.44", "compatible"], + ["0.0.43", "drift"], + [null, "unknown"], + ] as const)("reports host-process running version %s as %s readiness evidence", (runningVersion, expected) => { + expect( + observeOpenShellGatewayVersionCompatibility({ + source: "host-process", + deps: { + getInstalledOpenshellVersion: () => "0.0.44", + getGatewayClusterImageRef: () => null, + getHostProcessGatewayRuntime: () => ({ + gatewayBin: "/home/u/.local/bin/openshell-gateway", + runningVersion, + }), + }, + }), + ).toBe(expected); + }); + + it("keeps version compatibility unknown without an installed version", () => { + expect( + observeOpenShellGatewayVersionCompatibility({ + source: "host-process", + deps: { + getInstalledOpenshellVersion: () => null, + getGatewayClusterImageRef: () => null, + getHostProcessGatewayRuntime: () => ({ + gatewayBin: "/home/u/.local/bin/openshell-gateway", + runningVersion: "0.0.44", + }), + }, + }), + ).toBe("unknown"); + }); + + it("does not fall from an unproven legacy cluster to an unrelated host binary", () => { + const getHostProcessGatewayRuntime = vi.fn(() => ({ + gatewayBin: "/home/u/.local/bin/openshell-gateway", + runningVersion: "0.0.44", + })); + + expect( + observeOpenShellGatewayVersionCompatibility({ + source: "legacy-cluster", + deps: { + getInstalledOpenshellVersion: () => "0.0.44", + getGatewayClusterImageRef: () => null, + isGatewayClusterActive: () => false, + getHostProcessGatewayRuntime, + }, + }), + ).toBe("unknown"); + expect(getHostProcessGatewayRuntime).not.toHaveBeenCalled(); + }); + + it.each([ + ["ghcr.io/nvidia/openshell/cluster:0.0.44", "compatible"], + ["ghcr.io/nvidia/openshell/cluster:0.0.43", "drift"], + ["example.com/cluster:latest", "unknown"], + ] as const)("reports bound legacy cluster image %s as %s", (image, expected) => { + expect( + observeOpenShellGatewayVersionCompatibility({ + source: "legacy-cluster", + deps: { + getInstalledOpenshellVersion: () => "0.0.44", + getGatewayClusterImageRef: () => image, + isGatewayClusterActive: () => true, + }, + }), + ).toBe(expected); + }); + it("surfaces host-process drift as a preflight issue when cluster image drift is absent", () => { const issue = detectOpenShellStateRpcPreflightIssue({ deps: { diff --git a/src/lib/adapters/openshell/gateway-drift.ts b/src/lib/adapters/openshell/gateway-drift.ts index 024f610f46a..c5fbdad6952 100644 --- a/src/lib/adapters/openshell/gateway-drift.ts +++ b/src/lib/adapters/openshell/gateway-drift.ts @@ -16,13 +16,28 @@ import { resolveDockerDriverGatewayStateDir, } from "../../onboard/host-gateway-process"; import { isOpenShellProtobufSchemaMismatch } from "../../runtime-recovery"; -import { isGatewayHealthy } from "../../state/gateway"; +import { + type GatewayReuseState, + getGatewayReuseState, + isGatewayHealthy, +} from "../../state/gateway"; import { dockerContainerInspectFormat } from "../docker"; import { parseVersionFromText, stripAnsi } from "./client"; import { resolveOpenshell } from "./resolve"; import { captureOpenshell, getInstalledOpenshellVersionOrNull } from "./runtime"; import { OPENSHELL_PROBE_TIMEOUT_MS } from "./timeouts"; +export type { GatewayReuseState }; +// Gateway observation consumers use the established drift adapter as their +// OpenShell parsing and state boundary. +export { + getGatewayReuseState, + isGatewayHealthy, + OPENSHELL_PROBE_TIMEOUT_MS, + parseVersionFromText, + stripAnsi, +}; + const DEFAULT_GATEWAY_NAME = "nemoclaw"; export type GatewayClusterImageDrift = { @@ -62,7 +77,7 @@ export type OpenShellStateRpcIssue = output: string; }; -type GatewayDriftDeps = { +export type GatewayDriftDeps = { getInstalledOpenshellVersion?: () => string | null; getGatewayClusterImageRef?: (gatewayName: string) => string | null; isGatewayClusterActive?: (gatewayName: string) => boolean; @@ -75,12 +90,19 @@ export function isHostProcessGatewayDrift( return !!drift && "gatewayBin" in drift; } -type GatewayDriftOptions = { +export type GatewayDriftOptions = { gatewayName?: string; deps?: GatewayDriftDeps; timeoutMs?: number; }; +export type GatewayVersionCompatibility = "compatible" | "drift" | "unknown"; +export type GatewayVersionSource = "host-process" | "legacy-cluster"; + +export type GatewayVersionCompatibilityOptions = GatewayDriftOptions & { + source: GatewayVersionSource; +}; + type StateRpcResult = { status?: number | null; output?: string | null; @@ -185,7 +207,10 @@ function getGatewayClusterPublishedHostPorts( export function isGatewayClusterActiveForGateway( gatewayName = DEFAULT_GATEWAY_NAME, - { timeoutMs = OPENSHELL_PROBE_TIMEOUT_MS }: { timeoutMs?: number } = {}, + { + expectedGatewayPort, + timeoutMs = OPENSHELL_PROBE_TIMEOUT_MS, + }: { expectedGatewayPort?: number; timeoutMs?: number } = {}, ): boolean { const status = captureOpenshell(["status"], { ignoreError: true, @@ -207,6 +232,9 @@ export function isGatewayClusterActiveForGateway( parseGatewayEndpointPort(activeGatewayInfo.output) ?? parseGatewayEndpointPort(gatewayInfo.output); if (!endpointPort) return false; + if (expectedGatewayPort !== undefined && endpointPort !== String(expectedGatewayPort)) { + return false; + } const containerName = getGatewayClusterContainerName(gatewayName); const running = dockerContainerInspectFormat("{{.State.Running}}", containerName, { @@ -227,8 +255,9 @@ export function getGatewayClusterImageDrift({ return null; } const expectedVersion = - deps.getInstalledOpenshellVersion?.() ?? - getInstalledOpenshellVersionOrNull({ timeout: timeoutMs }); + typeof deps.getInstalledOpenshellVersion === "function" + ? deps.getInstalledOpenshellVersion() + : getInstalledOpenshellVersionOrNull({ timeout: timeoutMs }); const clusterActive = deps.isGatewayClusterActive?.(gatewayName) ?? (typeof deps.getGatewayClusterImageRef === "function" @@ -408,7 +437,9 @@ export function getGatewayHostProcessDrift({ return null; } const runtime = - deps.getHostProcessGatewayRuntime?.() ?? getHostProcessGatewayRuntimeOrNull({ timeoutMs }); + typeof deps.getHostProcessGatewayRuntime === "function" + ? deps.getHostProcessGatewayRuntime() + : getHostProcessGatewayRuntimeOrNull({ timeoutMs }); if (!runtime || !runtime.runningVersion || runtime.runningVersion === expectedVersion) { return null; } @@ -419,6 +450,50 @@ export function getGatewayHostProcessDrift({ }; } +/** + * Observe managed gateway version compatibility without conflating a missing + * version with a proven match. Drift-only helpers return `null` for both cases; + * readiness needs this three-state result so it can fail closed when either + * side of the comparison is unavailable. + */ +export function observeOpenShellGatewayVersionCompatibility({ + source, + gatewayName = DEFAULT_GATEWAY_NAME, + deps = {}, + timeoutMs = OPENSHELL_PROBE_TIMEOUT_MS, +}: GatewayVersionCompatibilityOptions): GatewayVersionCompatibility { + if (isGatewayDriftPreflightDisabled(deps)) return "unknown"; + const expectedVersion = + typeof deps.getInstalledOpenshellVersion === "function" + ? deps.getInstalledOpenshellVersion() + : getInstalledOpenshellVersionOrNull({ timeout: timeoutMs }); + if (!expectedVersion) return "unknown"; + + if (source === "legacy-cluster") { + const clusterActive = + deps.isGatewayClusterActive?.(gatewayName) ?? + isGatewayClusterActiveForGateway(gatewayName, { timeoutMs }); + if (!clusterActive) return "unknown"; + const clusterImage = + typeof deps.getGatewayClusterImageRef === "function" + ? deps.getGatewayClusterImageRef(gatewayName) + : getGatewayClusterImageRef(gatewayName, { timeoutMs }); + const currentVersion = parseGatewayClusterImageVersion(clusterImage); + if (!currentVersion) return "unknown"; + return currentVersion === expectedVersion ? "compatible" : "drift"; + } + + // The caller must first bind this source to a positively identified live + // host-process listener. Do not inspect a legacy cluster or fall between + // runtime authorities here: missing host-process evidence stays unknown. + const runtime = + typeof deps.getHostProcessGatewayRuntime === "function" + ? deps.getHostProcessGatewayRuntime() + : getHostProcessGatewayRuntimeOrNull({ timeoutMs }); + if (!runtime?.runningVersion) return "unknown"; + return runtime.runningVersion === expectedVersion ? "compatible" : "drift"; +} + export function detectOpenShellStateRpcPreflightIssue({ gatewayName = DEFAULT_GATEWAY_NAME, deps = {}, diff --git a/src/lib/advisories/checks/host/docker.test.ts b/src/lib/advisories/checks/host/docker.test.ts index c8443b4e7d3..87c3a5db7f1 100644 --- a/src/lib/advisories/checks/host/docker.test.ts +++ b/src/lib/advisories/checks/host/docker.test.ts @@ -73,6 +73,16 @@ describe("Docker host advisories (#3213)", () => { expect(result.advisories.map((advisory) => advisory.id)).toEqual(["invalid_docker_host"]); }); + it("reports an invalid DOCKER_HOST in WSL before runtime reachability guidance (#7411)", () => { + const result = runAdvisories( + DOCKER_HOST_ADVISORY_CHECKS, + host({ isWsl: true, dockerHostInvalid: true }), + { phase: "preflight.host" }, + ); + + expect(result.advisories.map((advisory) => advisory.id)).toEqual(["invalid_docker_host"]); + }); + it("re-evaluates Docker state on resume", () => { const context = host({ dockerInstalled: false }); const cachedResults = new Map([["install_docker", null]]); diff --git a/src/lib/advisories/checks/host/docker.ts b/src/lib/advisories/checks/host/docker.ts index a1203edb149..04d3042cf31 100644 --- a/src/lib/advisories/checks/host/docker.ts +++ b/src/lib/advisories/checks/host/docker.ts @@ -24,7 +24,7 @@ export const enableDockerDesktopWslIntegration: AdvisoryCheck = severity: "blocking", resumeSafe: false, check(host) { - if (!wslDockerBlocksRemainingChecks(host)) return null; + if (host.dockerHostInvalid || !wslDockerBlocksRemainingChecks(host)) return null; const dockerMissing = !host.dockerInstalled; return hostAdvisory(enableDockerDesktopWslIntegration, { title: "Enable Docker Desktop WSL integration", @@ -72,7 +72,7 @@ export const invalidDockerHost: AdvisoryCheck = { severity: "blocking", resumeSafe: false, check(host) { - if (!host.dockerHostInvalid || !host.dockerInstalled || host.isWsl) { + if (!host.dockerHostInvalid || !host.dockerInstalled) { return null; } return hostAdvisory(invalidDockerHost, { diff --git a/src/lib/advisories/checks/host/index.test.ts b/src/lib/advisories/checks/host/index.test.ts index 12c017a9d36..e7682f6c122 100644 --- a/src/lib/advisories/checks/host/index.test.ts +++ b/src/lib/advisories/checks/host/index.test.ts @@ -3,7 +3,7 @@ import { describe, expect, it } from "vitest"; import type { HostAssessment } from "../../../onboard/preflight"; -import { planHostAdvisories, planHostRemediation } from "../../../onboard/preflight"; +import { planHostAdvisories } from "../../../onboard/preflight"; import { HOST_ADVISORY_CHECKS } from "."; function host(overrides: Partial = {}): HostAssessment { @@ -48,7 +48,6 @@ describe("host advisory registry (#3213)", () => { "docker_group_permission", "start_docker", "container_runtime_under_provisioned", - "unsupported_runtime_warning", "install_nodejs", "install_openshell", "headless_remote_hint", @@ -74,17 +73,15 @@ describe("host advisory registry (#3213)", () => { expect(planHostAdvisories(assessment).map((advisory) => advisory.id)).toEqual([ "container_runtime_under_provisioned", - "unsupported_runtime_warning", "install_nodejs", "install_openshell", "generate_nvidia_cdi_spec", ]); - expect(planHostRemediation(assessment).map(({ id, blocking }) => ({ id, blocking }))).toEqual([ - { id: "container_runtime_under_provisioned", blocking: false }, - { id: "unsupported_runtime_warning", blocking: false }, - { id: "install_nodejs", blocking: false }, - { id: "install_openshell", blocking: false }, - { id: "generate_nvidia_cdi_spec", blocking: true }, + expect(planHostAdvisories(assessment).map(({ id, severity }) => ({ id, severity }))).toEqual([ + { id: "container_runtime_under_provisioned", severity: "warning" }, + { id: "install_nodejs", severity: "warning" }, + { id: "install_openshell", severity: "warning" }, + { id: "generate_nvidia_cdi_spec", severity: "blocking" }, ]); }); @@ -97,7 +94,7 @@ describe("host advisory registry (#3213)", () => { cdiNvidiaGpuSpecMissing: true, }); - expect(planHostRemediation(assessment).map((action) => action.id)).toEqual([ + expect(planHostAdvisories(assessment).map((action) => action.id)).toEqual([ "enable_docker_desktop_wsl_integration", ]); }); diff --git a/src/lib/advisories/checks/host/runtime.test.ts b/src/lib/advisories/checks/host/runtime.test.ts index 8e6c6ad5f5a..43e4ee446c3 100644 --- a/src/lib/advisories/checks/host/runtime.test.ts +++ b/src/lib/advisories/checks/host/runtime.test.ts @@ -46,8 +46,8 @@ describe("container runtime host advisories (#3213)", () => { expect(result.advisories.map((advisory) => advisory.id)).toEqual([ "container_runtime_under_provisioned", - "unsupported_runtime_warning", ]); + expect(result.advisories.some(({ id }) => id.includes("unsupported"))).toBe(false); expect(result.advisories[0]?.commands).toContain("colima start --cpu 4 --memory 8 --disk 100"); }); }); diff --git a/src/lib/advisories/checks/host/runtime.ts b/src/lib/advisories/checks/host/runtime.ts index 9cf44994c97..be4986e2c4c 100644 --- a/src/lib/advisories/checks/host/runtime.ts +++ b/src/lib/advisories/checks/host/runtime.ts @@ -49,28 +49,4 @@ export const increaseContainerRuntimeResources: AdvisoryCheck = }, }; -export const warnUnsupportedContainerRuntime: AdvisoryCheck = { - id: "unsupported_runtime_warning", - phase: "preflight.host", - severity: "warning", - resumeSafe: false, - skipIf: wslDockerBlocksRemainingChecks, - check(host) { - if (!host.isUnsupportedRuntime) return null; - return hostAdvisory(warnUnsupportedContainerRuntime, { - title: "Use a supported Docker runtime if problems appear", - kind: "manual", - reason: - "OpenShell officially documents Docker-based runtimes. Podman may work in some environments, but it is not a supported runtime and behavior may vary.", - commands: - host.platform === "darwin" - ? ["If onboarding or sandbox lifecycle fails, switch to Docker Desktop or Colima."] - : ["If onboarding or sandbox lifecycle fails, switch to a Docker-supported runtime."], - }); - }, -}; - -export const RUNTIME_HOST_ADVISORY_CHECKS = Object.freeze([ - increaseContainerRuntimeResources, - warnUnsupportedContainerRuntime, -]); +export const RUNTIME_HOST_ADVISORY_CHECKS = Object.freeze([increaseContainerRuntimeResources]); diff --git a/src/lib/advisories/registry.test.ts b/src/lib/advisories/registry.test.ts index a3754c7e1fb..99f55f57056 100644 --- a/src/lib/advisories/registry.test.ts +++ b/src/lib/advisories/registry.test.ts @@ -22,7 +22,9 @@ describe("defineAdvisoryRegistry", () => { expect(registry.map((entry) => entry.id)).toEqual(["first", "second"]); expect(Object.isFrozen(registry)).toBe(true); - expect(ADVISORY_CHECKS).toEqual([]); + expect(ADVISORY_CHECKS.length).toBeGreaterThan(0); + expect(ADVISORY_CHECKS.every((entry) => entry.phase === "preflight.host")).toBe(true); + expect(new Set(ADVISORY_CHECKS.map((entry) => entry.id)).size).toBe(ADVISORY_CHECKS.length); }); it("rejects duplicate stable IDs", () => { diff --git a/src/lib/advisories/registry.ts b/src/lib/advisories/registry.ts index d948428eb3e..f8fd253fff2 100644 --- a/src/lib/advisories/registry.ts +++ b/src/lib/advisories/registry.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { HOST_ADVISORY_CHECKS } from "./checks/host"; import type { AdvisoryCheck } from "./types"; /** Creates an explicit, immutable advisory registry with globally unique IDs. */ @@ -16,4 +17,4 @@ export function defineAdvisoryRegistry( } /** Checks are imported and registered explicitly as migration slices land (#3213). */ -export const ADVISORY_CHECKS = defineAdvisoryRegistry([]); +export const ADVISORY_CHECKS = defineAdvisoryRegistry(HOST_ADVISORY_CHECKS); diff --git a/src/lib/domain/docker-host.test.ts b/src/lib/domain/docker-host.test.ts index d092abcb3eb..49dbc1b6164 100644 --- a/src/lib/domain/docker-host.test.ts +++ b/src/lib/domain/docker-host.test.ts @@ -3,7 +3,51 @@ import { describe, expect, it } from "vitest"; -import { isSupportedGatewayDockerHost } from "./docker-host"; +import { + isDockerDaemonReachable, + isSupportedGatewayDockerHost, + parseDockerDaemonObservation, +} from "./docker-host"; + +describe("parseDockerDaemonObservation", () => { + it("requires positive daemon evidence from Docker JSON", () => { + expect( + parseDockerDaemonObservation( + JSON.stringify({ ServerVersion: "29.3.1", OperatingSystem: "Ubuntu 24.04" }), + ), + ).toEqual({ reachable: true, serverVersion: "29.3.1" }); + + expect( + parseDockerDaemonObservation( + JSON.stringify({ + ServerVersion: "", + ServerErrors: ["Cannot connect to the Docker daemon"], + }), + ), + ).toEqual({ reachable: false }); + expect(isDockerDaemonReachable(JSON.stringify({ ServerVersion: "" }))).toBe(false); + }); + + it("recognizes native Podman version evidence used by a docker alias", () => { + expect( + parseDockerDaemonObservation( + JSON.stringify({ version: { APIVersion: "5.3.1", Version: "5.3.1" } }), + ), + ).toEqual({ reachable: true, serverVersion: "5.3.1" }); + }); + + it("keeps the plain-text compatibility path without accepting connection errors", () => { + expect(parseDockerDaemonObservation("25.0.0")).toEqual({ + reachable: true, + serverVersion: "25.0.0", + }); + expect( + isDockerDaemonReachable( + "Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?", + ), + ).toBe(false); + }); +}); describe("isSupportedGatewayDockerHost (#7731)", () => { it.each([ diff --git a/src/lib/domain/docker-host.ts b/src/lib/domain/docker-host.ts index 9c6d7e282cf..8205fc30142 100644 --- a/src/lib/domain/docker-host.ts +++ b/src/lib/domain/docker-host.ts @@ -3,6 +3,68 @@ import path from "node:path"; +export interface DockerDaemonObservation { + readonly reachable: boolean; + readonly serverVersion?: string; +} + +/** + * Parse `docker info` output into the stable daemon facts shared by host + * readiness and installed-sandbox diagnostics. + * + * Docker can exit successfully while emitting a zero-value JSON object when + * its daemon is unavailable. Positive server-version evidence is therefore + * required for JSON output. Plain text remains supported for older callers and + * test doubles, except for the well-known daemon connection failures. + */ +export function parseDockerDaemonObservation(rawOutput = ""): DockerDaemonObservation { + const text = String(rawOutput).trim(); + if (!text) return { reachable: false }; + + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch { + const lowered = text.toLowerCase(); + const reachable = !( + lowered.includes("cannot connect to the docker daemon") || + lowered.includes("error during connect") || + lowered.includes("is the docker daemon running") + ); + return { + reachable, + ...(reachable && !/\s/u.test(text) ? { serverVersion: text } : {}), + }; + } + + if (!parsed || typeof parsed !== "object") return { reachable: false }; + const observation = parsed as Record; + if (Array.isArray(observation.ServerErrors) && observation.ServerErrors.length > 0) { + return { reachable: false }; + } + + if ( + typeof observation.ServerVersion === "string" && + observation.ServerVersion.trim().length > 0 + ) { + return { reachable: true, serverVersion: observation.ServerVersion.trim() }; + } + + const version = observation.version; + if (version && typeof version === "object") { + const nativeVersion = (version as Record).Version; + if (typeof nativeVersion === "string" && nativeVersion.trim().length > 0) { + return { reachable: true, serverVersion: nativeVersion.trim() }; + } + } + + return { reachable: false }; +} + +export function isDockerDaemonReachable(rawOutput = ""): boolean { + return parseDockerDaemonObservation(rawOutput).reachable; +} + // A DOCKER_HOST value onboarding can use. Unset means Docker's default socket, // which is supported; a set value must be an absolute `unix://` socket that can // be written to the gateway environment file. TCP and SSH endpoints and diff --git a/src/lib/inference/nim.ts b/src/lib/inference/nim.ts index 0b055dc9aa7..775144a6976 100644 --- a/src/lib/inference/nim.ts +++ b/src/lib/inference/nim.ts @@ -101,6 +101,8 @@ export interface DetectGpuDeps { // in production `detectGpu()` lazily builds the default prover only when it // is about to reject a denylisted ARM64 name. proveArm64WslDockerDesktopGpu?: Arm64WslDockerDesktopGpuProver | null; + /** Read-only command transport used by observation-only readiness callers. */ + runCaptureImpl?: typeof runCapture; } // Lazily construct the default ARM64 Linux GPU prover. Keep it behind a require @@ -205,9 +207,9 @@ function readPlatformModel(): string { return ""; } -function readHostMemoryMB(): number { +function readHostMemoryMB(runCaptureImpl: typeof runCapture = runCapture): number { try { - const freeOut = runCapture(["free", "-m"], { ignoreError: true }); + const freeOut = runCaptureImpl(["free", "-m"], { ignoreError: true }); if (freeOut) { const memLine = freeOut.split("\n").find((l: string) => l.includes("Mem:")); if (memLine) { @@ -227,9 +229,9 @@ function readHostMemoryMB(): number { // a 22 GB model right now?" signal the unified-memory Linux path uses. // Returns 0 when any expected field is missing so the caller can treat // the figure as "unknown" and fall back to total memory. -function readMacOsAvailableMemoryMB(): number { +function readMacOsAvailableMemoryMB(runCaptureImpl: typeof runCapture = runCapture): number { try { - const out = runCapture(["vm_stat"], { ignoreError: true }); + const out = runCaptureImpl(["vm_stat"], { ignoreError: true }); if (!out) return 0; const pageMatch = out.match(/page size of (\d+) bytes/); if (!pageMatch) return 0; @@ -253,9 +255,9 @@ function readMacOsAvailableMemoryMB(): number { // 22 GB Ollama load right now?" on unified-memory hosts. Returns 0 when // the column cannot be parsed; the caller treats 0 as "unknown" and falls // back to total memory. -function readHostAvailableMemoryMB(): number { +function readHostAvailableMemoryMB(runCaptureImpl: typeof runCapture = runCapture): number { try { - const freeOut = runCapture(["free", "-m"], { ignoreError: true }); + const freeOut = runCaptureImpl(["free", "-m"], { ignoreError: true }); if (freeOut) { const memLine = freeOut.split("\n").find((l: string) => l.includes("Mem:")); if (memLine) { @@ -411,12 +413,13 @@ export function adoptServedModelId(catalogModel: string | null, port = VLLM_PORT } export function detectGpu(deps: DetectGpuDeps = {}): GpuDetection | null { + const runCaptureImpl = deps.runCaptureImpl ?? runCapture; // Try NVIDIA first — query name, total, and free VRAM in a single call so // the preflight line can show the GPU model alongside the memory size and // the bootstrap-model selector can pick a model that fits currently // available memory, not just the headline total. try { - const output = runCapture( + const output = runCaptureImpl( ["nvidia-smi", "--query-gpu=name,memory.total,memory.free", "--format=csv,noheader,nounits"], { ignoreError: true }, ); @@ -529,7 +532,7 @@ export function detectGpu(deps: DetectGpuDeps = {}): GpuDetection | null { // Fallback: unified-memory NVIDIA devices try { - const nameOutput = runCapture( + const nameOutput = runCaptureImpl( ["nvidia-smi", "--query-gpu=name", "--format=csv,noheader,nounits"], { ignoreError: true }, ); @@ -569,7 +572,7 @@ export function detectGpu(deps: DetectGpuDeps = {}): GpuDetection | null { ? gpuNames : []; if (unifiedGpuNames.length > 0) { - const totalMemoryMB = readHostMemoryMB(); + const totalMemoryMB = readHostMemoryMB(runCaptureImpl); const count = unifiedGpuNames.length; const perGpuMB = count > 0 ? Math.floor(totalMemoryMB / count) : totalMemoryMB; const firstUnifiedName = unifiedGpuNames[0] ?? ""; @@ -595,7 +598,7 @@ export function detectGpu(deps: DetectGpuDeps = {}): GpuDetection | null { // `availableMemoryMB` mirrors that approximation using MemAvailable so // the bootstrap-model selector reacts to concurrent GPU workloads // eating into the shared system pool. - const availableMemoryMB = readHostAvailableMemoryMB(); + const availableMemoryMB = readHostAvailableMemoryMB(runCaptureImpl); return { type: "nvidia", ...(allUnifiedSameName ? { name: firstUnifiedName } : {}), @@ -619,8 +622,8 @@ export function detectGpu(deps: DetectGpuDeps = {}): GpuDetection | null { // integrated NVIDIA GPU through firmware and Tegra device nodes. const tegraGpu = detectTegraHostGpu(); if (tegraGpu) { - const totalMemoryMB = readHostMemoryMB(); - const availableMemoryMB = readHostAvailableMemoryMB(); + const totalMemoryMB = readHostMemoryMB(runCaptureImpl); + const availableMemoryMB = readHostAvailableMemoryMB(runCaptureImpl); return { type: "nvidia", name: tegraGpu.name, @@ -640,7 +643,7 @@ export function detectGpu(deps: DetectGpuDeps = {}): GpuDetection | null { // macOS: detect Apple Silicon or discrete GPU if (process.platform === "darwin") { try { - const spOutput = runCapture(["system_profiler", "SPDisplaysDataType"], { + const spOutput = runCaptureImpl(["system_profiler", "SPDisplaysDataType"], { ignoreError: true, }); if (spOutput) { @@ -657,14 +660,16 @@ export function detectGpu(deps: DetectGpuDeps = {}): GpuDetection | null { if (vramMatch[2].toUpperCase() === "GB") memoryMB *= 1024; } else { try { - const memBytes = runCapture(["sysctl", "-n", "hw.memsize"], { ignoreError: true }); + const memBytes = runCaptureImpl(["sysctl", "-n", "hw.memsize"], { + ignoreError: true, + }); if (memBytes) memoryMB = Math.floor(parseInt(memBytes, 10) / 1024 / 1024); } catch { /* ignored */ } } - const availableMemoryMB = readMacOsAvailableMemoryMB(); + const availableMemoryMB = readMacOsAvailableMemoryMB(runCaptureImpl); return { type: "apple", name, @@ -686,8 +691,10 @@ export function detectGpu(deps: DetectGpuDeps = {}): GpuDetection | null { } /** Return one consistent NVIDIA driver version from the read-only host GPU inventory. */ -export function detectNvidiaDriverVersion(): string | undefined { - const output = runCapture( +export function detectNvidiaDriverVersion( + deps: { runCaptureImpl?: typeof runCapture } = {}, +): string | undefined { + const output = (deps.runCaptureImpl ?? runCapture)( ["nvidia-smi", "--query-gpu=driver_version", "--format=csv,noheader,nounits"], { ignoreError: true }, ); diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index f97501bfb40..28238911a95 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -502,8 +502,6 @@ const authoritativeRebuildTarget: typeof import("./onboard/authoritative-rebuild require("./onboard/authoritative-rebuild-target"); const { assertDashboardPortNotReserved, buildRequiredPreflightPorts } = require("./onboard/preflight-ports") as typeof import("./onboard/preflight-ports"); -const { failFastOnForeignGatewayPortConflict } = - require("./onboard/gateway-port-conflict") as typeof import("./onboard/gateway-port-conflict"); const { printPortConflictReport } = require("./onboard/port-conflict-report") as typeof import("./onboard/port-conflict-report"); const { tryCleanupOrphanedDashboardForward } = @@ -538,6 +536,8 @@ const dockerDriverGatewayRuntimeMarker: typeof import("./onboard/docker-driver-g const gatewayBinding: typeof import("./onboard/gateway-binding") = require("./onboard/gateway-binding"); const fatalRuntimePreflight: typeof import("./onboard/fatal-runtime-preflight") = require("./onboard/fatal-runtime-preflight"); +const preflightGatewayAuthority: typeof import("./onboard/machine/preflight-gateway-authority") = + require("./onboard/machine/preflight-gateway-authority"); const preflightUtils: typeof import("./onboard/preflight") = require("./onboard/preflight"); const clusterImagePatch: typeof import("./cluster-image-patch") = require("./cluster-image-patch"); const overlayfsAutoFix: typeof import("./onboard/overlayfs-auto-fix") = require("./onboard/overlayfs-auto-fix"); @@ -574,7 +574,8 @@ import { type MessagingChannelConfig, } from "./messaging-channel-config"; import { streamGatewayStart } from "./onboard/gateway"; -import { bindGatewayAuthorityToCheckpoint } from "./onboard/gateway-authority-checkpoint"; +// biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. +import { adoptPackagedGatewayAuthorityAfterTrustedInstall, bindGatewayAuthorityToCheckpoint, checkpointGatewayAuthority } from "./onboard/gateway-authority-checkpoint"; import { createGatewayHostRuntime } from "./onboard/gateway-host-runtime"; import { mergeRequiredHermesToolGatewayPolicyPresets, @@ -1119,8 +1120,10 @@ function areRequiredDockerDriverBinariesPresent( function ensureOpenshellForOnboard( exitProcess: (code: number) => never = (code) => process.exit(code), + persistTrustedGatewayOwner?: (owner: import("./onboard/gateway-ownership").GatewayOwner) => void, ): OpenShellInstallResult { - return openshellInstallFlow.ensureOpenshellForOnboard(getOpenShellInstallDeps(exitProcess)); + // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. + return openshellInstallFlow.ensureOpenshellForOnboard(getOpenShellInstallDeps(exitProcess), { afterSuccessfulInstall: () => adoptPackagedGatewayOwnerAfterTrustedInstall(persistTrustedGatewayOwner) }); } function getOpenShellInstallDeps( @@ -1399,17 +1402,15 @@ function attachGatewayMetadataIfNeeded({ // ── Step 1: Preflight ──────────────────────────────────────────── type PreflightOptions = import("./onboard/fatal-runtime-preflight").FatalRuntimePreflightOptions; +// biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. +async function collectOnboardGatewayReadiness() { return preflightGatewayAuthority.collectOnboardGatewayReadiness({ gatewayName: () => GATEWAY_NAME, gatewayPort: () => GATEWAY_PORT, resolveOwner: getGatewayOwner, probeAttachment: machineGatewayOwnerDeps.probeGatewayAttachment }); } async function preflight( preflightOpts: PreflightOptions = {}, ): Promise> { step(1, 8, "Preflight checks"); - const { gpu, host, sandboxGpuConfig } = fatalRuntimePreflight.runFatalOnboardRuntimePreflight( - preflightOpts, - { - nonInteractive: isNonInteractive(), - }, - ); + // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. + const { gpu, host, sandboxGpuConfig } = await fatalRuntimePreflight.runReadinessGatedRuntimePreflight(preflightOpts, { nonInteractive: isNonInteractive(), collectGatewayReadiness: collectOnboardGatewayReadiness }); await preflightUtils.checkContainerRuntimeResources(host, { ignored: process.env.NEMOCLAW_IGNORE_RUNTIME_RESOURCES === "1", @@ -1417,25 +1418,9 @@ async function preflight( confirm: () => promptYesNoOrDefault(" Continue with onboarding?", null, false), }); - ensureOpenshellForOnboard(); - // Bind the one lifecycle authority before applying any legacy listener-name - // heuristic. An external supervisor may intentionally use an arbitrary - // executable name; its listener is validated exactly by the downstream FSM. - const gatewayExternallySupervised = isGatewayExternallySupervised(); - await failFastOnForeignGatewayPortConflict({ - gatewayPort: GATEWAY_PORT, - externallySupervised: gatewayExternallySupervised, - checkPortAvailable, - getGatewayPortCheckOptions: dockerDriverGatewayEnv.getGatewayPortCheckOptions, - isDockerDriverGatewayPortListener, - exitProcess: (code) => process.exit(code), - }); - - // Classify gateway state before port checks; the legacy path destroys - // stale/unnamed gateways here while the Docker-driver path defers (#2020). - const gatewaySnapshot = selectNamedGatewayForReuseIfNeeded(getGatewayReuseSnapshot()); - let gatewayReuseState = gatewaySnapshot.gatewayReuseState; - gatewayReuseState = await refreshDockerDriverGatewayReuseState(gatewayReuseState); + // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. + const { externallySupervised: gatewayExternallySupervised, gatewayReuseState: initialGatewayReuseState } = await preflightGatewayAuthority.preparePreflightGatewayAuthority({ collectGatewayReadiness: collectOnboardGatewayReadiness, ensureOpenshell: (persistTrustedGatewayOwner) => ensureOpenshellForOnboard((code) => process.exit(code), persistTrustedGatewayOwner), persistTrustedGatewayOwner: (owner) => { onboardSession.updateSession((session) => { adoptPackagedGatewayAuthorityAfterTrustedInstall(session, owner); }); }, gatewayPort: GATEWAY_PORT, portConflict: { checkPortAvailable, getGatewayPortCheckOptions: dockerDriverGatewayEnv.getGatewayPortCheckOptions, isDockerDriverGatewayPortListener, exitProcess: (code) => process.exit(code) }, getGatewayReuseSnapshot, selectNamedGatewayForReuseIfNeeded, refreshDockerDriverGatewayReuseState }); + let gatewayReuseState = initialGatewayReuseState; // Verify the legacy gateway container is actually running — openshell CLI // metadata can be stale after a manual `docker rm`. See #2020. Newer @@ -2044,6 +2029,7 @@ const applyOverlayfsAutoFix = overlayfsAutoFix.createOverlayfsAutoFix({ }); const { + adoptPackagedGatewayOwnerAfterTrustedInstall, assertGatewayStartAllowed, bindGatewayOwner, getGatewayLocalEndpoint, @@ -3689,10 +3675,10 @@ const recordStepSkipped = onboardRuntimeBoundary.recordStepSkipped.bind(onboardR const recordStepFailed = onboardRuntimeBoundary.recordStepFailed.bind(onboardRuntimeBoundary); const recordStateSkipped = onboardRuntimeBoundary.recordStateSkipped.bind(onboardRuntimeBoundary); const recordRepairEvent = onboardRuntimeBoundary.recordRepairEvent.bind(onboardRuntimeBoundary); -/** Run only non-mutating fatal onboard gates while the rebuild target is still intact. */ +/** Run readiness gates while the authoritative rebuild target is still intact. */ async function preflightAuthoritativeRebuildTarget( opts: import("./onboard/authoritative-rebuild-target").AuthoritativeRebuildPreflightOptions, -): Promise { +): Promise { const authoritativeGateway = authoritativeRebuildTarget.resolveAuthoritativeOnboardGatewayBinding(opts); if (!authoritativeGateway) throw new Error("Authoritative rebuild preflight has no gateway"); @@ -3706,6 +3692,7 @@ async function preflightAuthoritativeRebuildTarget( GATEWAY_PORT = authoritativeGateway.port; NON_INTERACTIVE = true; _preflightDashboardPort = opts.controlUiPort ?? null; + resetGatewayOwnerBinding(); const fail = (message: string): never => { throw new Error(message); }; @@ -3714,8 +3701,9 @@ async function preflightAuthoritativeRebuildTarget( { ...opts, controlUiPort: opts.controlUiPort ?? null }, { resolveBaselinePolicy: resolveSandboxBaselinePolicy, - runFatalRuntimePreflight: () => - fatalRuntimePreflight.runFatalOnboardRuntimePreflight( + bindGatewayAuthority: () => bindGatewayOwner(getGatewayOwner()), + runFatalRuntimePreflight: async () => + fatalRuntimePreflight.runReadinessGatedRuntimePreflight( { sandboxGpu: opts.sandboxGpu, sandboxGpuDevice: opts.sandboxGpuDevice, @@ -3723,6 +3711,7 @@ async function preflightAuthoritativeRebuildTarget( }, { nonInteractive: true, + collectGatewayReadiness: collectOnboardGatewayReadiness, exitProcess: (code) => fail(`onboard runtime preflight exited with code ${String(code)}`), }, @@ -3731,12 +3720,15 @@ async function preflightAuthoritativeRebuildTarget( ensureOpenshellForOnboard((code) => fail(`OpenShell component preflight exited with code ${String(code)}`), ), + assertGatewayReadiness: collectOnboardGatewayReadiness, inferenceRouteReady: (p, m) => isInferenceRouteReady(authoritativeGateway.name, p, m), captureForwardList: () => runCaptureOpenshell(["forward", "list"], { ignoreError: true }), checkPort: (port) => checkPortAvailable(port), }, ); + return checkpointGatewayAuthority(getGatewayOwner()); } finally { + resetGatewayOwnerBinding(); GATEWAY_NAME = previous.gatewayName; GATEWAY_PORT = previous.gatewayPort; NON_INTERACTIVE = previous.nonInteractive; @@ -3944,17 +3936,6 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { resume, canPrompt: !cannotPrompt, }); - const selectedAgentTransition = await runtimeControlFlow.applySelectedAgentTransition({ - resume, - session, - selectedAgentName: agent?.name, - routerPort: loadBlueprintProfile("routed")?.router.port || 4000, - note, - }); - session = selectedAgentTransition.session; - const resumeAgentChanged = selectedAgentTransition.resumeAgentChanged; - const forceProviderSelectionForAgentChange = resumeAgentChanged; - const recordedSandboxName = session?.steps?.sandbox?.status === "complete" ? session?.sandboxName || null : null; // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. @@ -3976,6 +3957,17 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { ); }); bindGatewayOwner(checkpointedGatewayOwner); + const selectedAgentTransition = runtimeControlFlow.planSelectedAgentTransition({ + resume, + session, + selectedAgentName: agent?.name, + routerPort: loadBlueprintProfile("routed")?.router.port || 4000, + note, + }); + setOnboardBrandingAgent(agent?.name || "openclaw"); + session = selectedAgentTransition.session; + const resumeAgentChanged = selectedAgentTransition.resumeAgentChanged; + const forceProviderSelectionForAgentChange = resumeAgentChanged; console.log(""); console.log(` ${cliDisplayName()} Onboarding`); if (isNonInteractive()) note(" (non-interactive mode)"); @@ -4027,17 +4019,22 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { noGpu: opts.noGpu === true, env: process.env, recordedGpuPassthroughBeforePreflight, + commitSelectedAgentTransition: selectedAgentTransition.commit, ensureResumePreflightDashboardPortAvailable: () => { if (_preflightDashboardPort === null) preflightDashboardPortRangeAvailability(); }, preflightDeps: { getSandbox: registry.getSandbox.bind(registry), getResumeSandboxGpuOverrides, + detectGpuForReadiness: () => nim.detectGpu({ proveArm64WslDockerDesktopGpu: null }), detectGpu: nim.detectGpu, runPreflight: (preflightOptions) => preflight({ ...opts, ...preflightOptions }), assessHost, - assertCdiNvidiaGpuSpecPresent: preflightUtils.assertCdiNvidiaGpuSpecPresent, - rejectUnsupportedContainerRuntime: fatalRuntimePreflight.rejectUnsupportedContainerRuntime, + assertOnboardHostReadiness: (host, gpu, options) => + fatalRuntimePreflight.assertOnboardHostReadiness(host, gpu ?? null, { + ...options, + allowStorageRemediation: !isGatewayExternallySupervised(), + }), assertDockerBridgeAndContainerDnsHealthy, resolveSandboxGpuConfig, validateSandboxGpuPreflight, @@ -4049,6 +4046,9 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { }, getInitialGatewayReuseState: () => selectNamedGatewayForReuseIfNeeded(getGatewayReuseSnapshot()).gatewayReuseState, + assertGatewayReadiness: async () => { + await collectOnboardGatewayReadiness(); + }, gatewayName: GATEWAY_NAME, recreateSandbox: isRecreateSandbox, gatewayDeps: { diff --git a/src/lib/onboard/authoritative-rebuild-target.test.ts b/src/lib/onboard/authoritative-rebuild-target.test.ts index 9b7852686e9..52893b9fc96 100644 --- a/src/lib/onboard/authoritative-rebuild-target.test.ts +++ b/src/lib/onboard/authoritative-rebuild-target.test.ts @@ -26,8 +26,10 @@ const originalGateway = process.env.OPENSHELL_GATEWAY; function deps(overrides: Partial = {}) { return { resolveBaselinePolicy: vi.fn(() => ({})), + bindGatewayAuthority: vi.fn(), runFatalRuntimePreflight: vi.fn(), ensureOpenshell: vi.fn(), + assertGatewayReadiness: vi.fn(), inferenceRouteReady: vi.fn(() => true), captureForwardList: vi.fn(() => "alpha 127.0.0.1 18789 42 active"), checkPort: vi.fn(async () => ({ ok: true })), @@ -199,6 +201,22 @@ describe("prepared provider reconfiguration handoff", () => { }); describe("authoritative rebuild target preflight", () => { + it("binds process-local gateway authority before readiness and install (#7411)", async () => { + const calls: string[] = []; + const targetDeps = deps({ + bindGatewayAuthority: vi.fn(() => calls.push("bind")), + runFatalRuntimePreflight: vi.fn(() => calls.push("readiness")), + ensureOpenshell: vi.fn(() => calls.push("install")), + assertGatewayReadiness: vi.fn(() => calls.push("post-install-readiness")), + }); + + await preflightAuthoritativeRebuildTarget(target, targetDeps); + + expect(calls).toEqual(["bind", "readiness", "install", "post-install-readiness"]); + expect(targetDeps.bindGatewayAuthority).toHaveBeenCalledOnce(); + expect(targetDeps.assertGatewayReadiness).toHaveBeenCalledOnce(); + }); + it("rejects an unreadable replacement baseline before runtime probes (#7194)", async () => { const targetDeps = deps({ resolveBaselinePolicy: vi.fn(() => null) }); @@ -207,7 +225,9 @@ describe("authoritative rebuild target preflight", () => { ); expect(targetDeps.runFatalRuntimePreflight).not.toHaveBeenCalled(); + expect(targetDeps.bindGatewayAuthority).not.toHaveBeenCalled(); expect(targetDeps.ensureOpenshell).not.toHaveBeenCalled(); + expect(targetDeps.assertGatewayReadiness).not.toHaveBeenCalled(); expect(targetDeps.inferenceRouteReady).not.toHaveBeenCalled(); }); @@ -294,4 +314,48 @@ describe("authoritative rebuild target preflight", () => { ).rejects.toThrow("fatal runtime gate"); expect(process.env.OPENSHELL_GATEWAY).toBe("before"); }); + + it("awaits async runtime readiness before OpenShell and route checks", async () => { + let releaseRuntime!: () => void; + const runtimeReady = new Promise((resolve) => { + releaseRuntime = resolve; + }); + const calls: string[] = []; + const targetDeps = deps({ + runFatalRuntimePreflight: vi.fn(async () => { + calls.push("runtime-start"); + await runtimeReady; + calls.push("runtime-end"); + }), + ensureOpenshell: vi.fn(() => calls.push("openshell")), + assertGatewayReadiness: vi.fn(() => calls.push("gateway")), + inferenceRouteReady: vi.fn(() => { + calls.push("route"); + return true; + }), + }); + + const pending = preflightAuthoritativeRebuildTarget(target, targetDeps); + await vi.waitFor(() => expect(calls).toEqual(["runtime-start"])); + expect(targetDeps.ensureOpenshell).not.toHaveBeenCalled(); + + releaseRuntime(); + await pending; + expect(calls).toEqual(["runtime-start", "runtime-end", "openshell", "gateway", "route"]); + }); + + it("stops rebuild checks when async runtime readiness rejects", async () => { + const targetDeps = deps({ + runFatalRuntimePreflight: vi.fn(async () => { + throw new Error("gateway readiness changed"); + }), + }); + + await expect(preflightAuthoritativeRebuildTarget(target, targetDeps)).rejects.toThrow( + "gateway readiness changed", + ); + expect(targetDeps.ensureOpenshell).not.toHaveBeenCalled(); + expect(targetDeps.assertGatewayReadiness).not.toHaveBeenCalled(); + expect(targetDeps.inferenceRouteReady).not.toHaveBeenCalled(); + }); }); diff --git a/src/lib/onboard/authoritative-rebuild-target.ts b/src/lib/onboard/authoritative-rebuild-target.ts index 558b4c40f13..ad527349f99 100644 --- a/src/lib/onboard/authoritative-rebuild-target.ts +++ b/src/lib/onboard/authoritative-rebuild-target.ts @@ -177,15 +177,17 @@ export function rebuildProviderFlowOptions( export type AuthoritativeRebuildTargetDeps = { resolveBaselinePolicy(sandboxName: string): unknown | null; - runFatalRuntimePreflight(): unknown; + bindGatewayAuthority(): void; + runFatalRuntimePreflight(): unknown | Promise; ensureOpenshell(): unknown; + assertGatewayReadiness(): unknown | Promise; inferenceRouteReady(provider: string, model: string): boolean; captureForwardList(): string | null; checkPort(port: number): Promise; env?: NodeJS.ProcessEnv; }; -/** Run non-mutating target checks under an exact process-local gateway scope. */ +/** Run target-bound readiness and installer checks under an exact process-local gateway scope. */ export async function preflightAuthoritativeRebuildTarget( target: AuthoritativeRebuildTarget, deps: AuthoritativeRebuildTargetDeps, @@ -200,8 +202,10 @@ export async function preflightAuthoritativeRebuildTarget( if (!deps.resolveBaselinePolicy(target.sandboxName)) { fail(`Could not read the baseline policy for sandbox '${target.sandboxName}'.`); } - deps.runFatalRuntimePreflight(); + deps.bindGatewayAuthority(); + await deps.runFatalRuntimePreflight(); deps.ensureOpenshell(); + await deps.assertGatewayReadiness(); // Prepared-backup recovery can run after the installer has replaced a // legacy gateway. That fresh gateway has no inference route to validate // yet; authoritative onboarding configures and verifies the pinned route diff --git a/src/lib/onboard/bridge-dns-preflight.ts b/src/lib/onboard/bridge-dns-preflight.ts index 08e3f39193d..1e2e963a05d 100644 --- a/src/lib/onboard/bridge-dns-preflight.ts +++ b/src/lib/onboard/bridge-dns-preflight.ts @@ -136,7 +136,7 @@ export function printDockerBridgeContainerStartFailure( * Docker/DNS state, and the original attempt that wrote the cache may * have aborted later at sandbox build with exactly the #3508/#3630 * failure modes. Resuming without re-checking would walk into the same - * wall (mirroring the [[assertCdiNvidiaGpuSpecPresent]] resume backstop + * wall (mirroring the host-readiness resume backstop * pattern at #3152). */ export function assertDockerBridgeAndContainerDnsHealthy( diff --git a/src/lib/onboard/docker-cdi.ts b/src/lib/onboard/docker-cdi.ts index 986b4e78440..f305629192e 100644 --- a/src/lib/onboard/docker-cdi.ts +++ b/src/lib/onboard/docker-cdi.ts @@ -64,6 +64,7 @@ const NVIDIA_CDI_KIND_YAML_RE = /^[ \t]*kind[ \t]*:[ \t]*(?:"nvidia\.com\/gpu"|'nvidia\.com\/gpu'|nvidia\.com\/gpu)[ \t]*(?:#.*)?$/im; const NVIDIA_CDI_KIND_JSON_RE = /"kind"\s*:\s*"nvidia\.com\/gpu"/; const NVIDIA_CDI_REFRESH_SPEC_PATH = "/var/run/cdi/nvidia.yaml"; +export const DEFAULT_DOCKER_CDI_SPEC_DIRS = ["/etc/cdi", "/var/run/cdi"] as const; export function parseDockerCdiSpecDirs(value: string | null | undefined): string[] { const raw = String(value || "").trim(); @@ -308,9 +309,13 @@ function parseSystemctlFailedState(value = ""): boolean | null { } export function assessNvidiaCdiHost(opts: NvidiaCdiHostAssessmentOpts): NvidiaCdiHostAssessment { - const dockerCdiSpecDirs = opts.dockerReachable + const reportedDockerCdiSpecDirs = opts.dockerReachable ? parseDockerCdiSpecDirs(opts.dockerInfoOutput) : []; + const dockerCdiSpecDirs = + opts.dockerReachable && reportedDockerCdiSpecDirs.length === 0 + ? [...DEFAULT_DOCKER_CDI_SPEC_DIRS] + : reportedDockerCdiSpecDirs; const cdiSpecPresenceApplies = opts.platform === "linux" && opts.hasNvidiaGpu && dockerCdiSpecDirs.length > 0; const cdiSpecRepairApplies = diff --git a/src/lib/onboard/docker-driver-gateway-env.ts b/src/lib/onboard/docker-driver-gateway-env.ts index f77ffd8bd8b..5e2aa1d46a2 100644 --- a/src/lib/onboard/docker-driver-gateway-env.ts +++ b/src/lib/onboard/docker-driver-gateway-env.ts @@ -33,6 +33,11 @@ import { isPortableExperimentalProfile, PORTABLE_HOST_GATEWAY_IP } from "./docke export { getGatewayHttpsEndpoint, startPackageManagedDockerDriverGateway }; +/** Return the configured gateway port used by Docker-driver runtime helpers. */ +export function getConfiguredGatewayPort(): number { + return GATEWAY_PORT; +} + export const DOCKER_DRIVER_GATEWAY_RUNTIME_ENV_KEYS = [ "CONTAINERS_CONF", "DOCKER_HOST", diff --git a/src/lib/onboard/docker-driver-gateway-process-identity.test.ts b/src/lib/onboard/docker-driver-gateway-process-identity.test.ts index 1a4cf6418fb..814fd60f574 100644 --- a/src/lib/onboard/docker-driver-gateway-process-identity.test.ts +++ b/src/lib/onboard/docker-driver-gateway-process-identity.test.ts @@ -3,7 +3,11 @@ import { describe, expect, it } from "vitest"; -import { getDockerDriverGatewayTargetIdentityDrift } from "./docker-driver-gateway-process-identity"; +import { + getDockerDriverGatewayTargetIdentityDrift, + hasDockerDriverGatewayEnvironment, + isDockerDriverGatewayProcessIdentity, +} from "./docker-driver-gateway-process-identity"; const normalizeGatewayExecutablePath = (value: string | null | undefined) => value ?? null; @@ -29,4 +33,42 @@ describe("Docker-driver gateway target identity", () => { }), ).toBeNull(); }); + + it("requires both gateway executable identity and Linux Docker-driver environment proof", () => { + const input = { + pid: 999_999, + gatewayBin: "/opt/openshell/openshell-gateway", + captureProcessArgs: () => "/opt/openshell/openshell-gateway --name nemoclaw --port 8080", + processIdentityMatchesGatewayBinary: () => true, + requireDockerDriverEnv: true, + hasDockerDriverGatewayEnv: () => false, + }; + + expect(isDockerDriverGatewayProcessIdentity(input)).toBe(false); + expect( + isDockerDriverGatewayProcessIdentity({ + ...input, + hasDockerDriverGatewayEnv: () => true, + }), + ).toBe(true); + expect( + isDockerDriverGatewayProcessIdentity({ + ...input, + processIdentityMatchesGatewayBinary: () => false, + hasDockerDriverGatewayEnv: () => true, + }), + ).toBe(false); + }); + + it("recognizes only documented Docker-driver environment markers", () => { + expect(hasDockerDriverGatewayEnvironment({ OPENSHELL_DRIVERS: "docker" }, "tcp://x")).toBe( + true, + ); + expect( + hasDockerDriverGatewayEnvironment({ OPENSHELL_GRPC_ENDPOINT: "tcp://x" }, "tcp://x"), + ).toBe(true); + expect( + hasDockerDriverGatewayEnvironment({ OPENSHELL_GRPC_ENDPOINT: "tcp://other" }, "tcp://x"), + ).toBe(false); + }); }); diff --git a/src/lib/onboard/docker-driver-gateway-process-identity.ts b/src/lib/onboard/docker-driver-gateway-process-identity.ts index ca8ce3e2dc2..b8b262d8814 100644 --- a/src/lib/onboard/docker-driver-gateway-process-identity.ts +++ b/src/lib/onboard/docker-driver-gateway-process-identity.ts @@ -27,6 +27,53 @@ export function readDockerDriverGatewayProcessIdentity( return captureProcessArgs(pid); } +export function readDockerDriverGatewayProcessEnvironment( + pid: number, +): Record | null { + const procEnvPath = `/proc/${pid}/environ`; + const env: Record = {}; + try { + if (!fs.existsSync(procEnvPath)) return null; + for (const entry of fs.readFileSync(procEnvPath, "utf-8").split("\0")) { + if (!entry) continue; + const separator = entry.indexOf("="); + if (separator <= 0) continue; + env[entry.slice(0, separator)] = entry.slice(separator + 1); + } + } catch { + return null; + } + return env; +} + +export function hasDockerDriverGatewayEnvironment( + env: Readonly> | null, + expectedEndpoint: string, +): boolean { + if (!env) return false; + return ( + env.OPENSHELL_DRIVERS === "docker" || + env.OPENSHELL_DRIVERS === "podman" || + Boolean(env.OPENSHELL_DOCKER_SUPERVISOR_IMAGE) || + env.OPENSHELL_GRPC_ENDPOINT === expectedEndpoint + ); +} + +/** Apply the shared positive identity decision used by runtime and readiness scans. */ +export function isDockerDriverGatewayProcessIdentity(input: { + pid: number; + gatewayBin?: string | null; + captureProcessArgs(pid: number): string; + processIdentityMatchesGatewayBinary(identity: string, gatewayBin?: string | null): boolean; + requireDockerDriverEnv: boolean; + hasDockerDriverGatewayEnv(pid: number): boolean; +}): boolean { + const identity = readDockerDriverGatewayProcessIdentity(input.pid, input.captureProcessArgs); + if (!identity) return false; + if (!input.processIdentityMatchesGatewayBinary(identity, input.gatewayBin)) return false; + return !input.requireDockerDriverEnv || input.hasDockerDriverGatewayEnv(input.pid); +} + export function getDockerDriverGatewayTargetIdentityDrift(input: { gatewayBin?: string | null; gatewayPort: number; diff --git a/src/lib/onboard/docker-driver-gateway-runtime.ts b/src/lib/onboard/docker-driver-gateway-runtime.ts index 309185345e1..c9a1a143942 100644 --- a/src/lib/onboard/docker-driver-gateway-runtime.ts +++ b/src/lib/onboard/docker-driver-gateway-runtime.ts @@ -5,7 +5,6 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { resolveOpenshell } from "../adapters/openshell/resolve"; import { isErrnoException } from "../core/errno"; import { isSupportedGatewayDockerHost } from "../domain/docker-host"; import { @@ -18,6 +17,9 @@ import { } from "./docker-driver-gateway-port-listener"; import { getDockerDriverGatewayTargetIdentityDrift, + hasDockerDriverGatewayEnvironment, + isDockerDriverGatewayProcessIdentity, + readDockerDriverGatewayProcessEnvironment, readDockerDriverGatewayProcessIdentity, } from "./docker-driver-gateway-process-identity"; import * as dockerDriverGatewayRuntimeMarker from "./docker-driver-gateway-runtime-marker"; @@ -27,6 +29,7 @@ import { gatewayProcessCmdlineMatches, OPENSHELL_GATEWAY_PROCESS_NAMES, } from "./gateway-process-identity"; +import { resolveOpenshell } from "./openshell-cli"; import type { PortProbeResult } from "./preflight"; // Keep the listener option type on the established runtime facade while the @@ -41,6 +44,16 @@ const OPENSHELL_SUPERVISOR_MANIFEST_DIGESTS: Readonly> = "0.0.101": "sha256:b58be5e40c788977ffa0e8305a8cad9c656efdf1a3fe182582a00ca870bb0edb", }; +/** Resolve the canonical gateway name without bypassing the binding owner. */ +export function resolveDockerDriverGatewayName(gatewayPort: number): string { + return gatewayBinding.resolveGatewayName(gatewayPort); +} + +/** Resolve the canonical state-directory name without bypassing the binding owner. */ +export function resolveDockerDriverGatewayStateDirName(gatewayPort: number): string { + return gatewayBinding.resolveGatewayStateDirName(gatewayPort); +} + export type DockerDriverGatewayRuntimeDrift = { reason: string }; type RunCapture = (args: string[], opts?: { ignoreError?: boolean }) => string; @@ -272,31 +285,13 @@ export function createDockerDriverGatewayRuntimeHelpers(deps: DockerDriverGatewa } function readProcessEnv(pid: number): Record | null { - const procEnvPath = `/proc/${pid}/environ`; - const env: Record = {}; - try { - if (!fs.existsSync(procEnvPath)) return null; - for (const entry of fs.readFileSync(procEnvPath, "utf-8").split("\0")) { - if (!entry) continue; - const idx = entry.indexOf("="); - if (idx <= 0) continue; - env[entry.slice(0, idx)] = entry.slice(idx + 1); - } - } catch { - return null; - } - return env; + return readDockerDriverGatewayProcessEnvironment(pid); } function hasDockerDriverGatewayEnv(pid: number): boolean { - const env = readProcessEnv(pid); - if (!env) return false; - return ( - env.OPENSHELL_DRIVERS === "docker" || - env.OPENSHELL_DRIVERS === "podman" || - Boolean(env.OPENSHELL_DOCKER_SUPERVISOR_IMAGE) || - env.OPENSHELL_GRPC_ENDPOINT === - dockerDriverGatewayEnv.getDockerDriverGatewayEndpoint(currentGatewayPort()) + return hasDockerDriverGatewayEnvironment( + readProcessEnv(pid), + dockerDriverGatewayEnv.getDockerDriverGatewayEndpoint(currentGatewayPort()), ); } @@ -458,12 +453,14 @@ export function createDockerDriverGatewayRuntimeHelpers(deps: DockerDriverGatewa gatewayBin?: string | null, opts: { requireDockerDriverEnv?: boolean } = {}, ): boolean { - const identity = readDockerDriverGatewayProcessIdentity(pid, captureProcessArgs); - if (!identity) return false; - const matchesGatewayBinary = processIdentityMatchesGatewayBinary(identity, gatewayBin); - if (!matchesGatewayBinary) return false; - if (opts.requireDockerDriverEnv && !hasDockerDriverGatewayEnv(pid)) return false; - return true; + return isDockerDriverGatewayProcessIdentity({ + pid, + gatewayBin, + captureProcessArgs, + processIdentityMatchesGatewayBinary, + requireDockerDriverEnv: opts.requireDockerDriverEnv === true, + hasDockerDriverGatewayEnv, + }); } function isDockerDriverGatewayProcessAlive(): boolean { diff --git a/src/lib/onboard/docker-driver-gateway-service-version-gate.test.ts b/src/lib/onboard/docker-driver-gateway-service-version-gate.test.ts index 5790a5d92d8..029a47f422d 100644 --- a/src/lib/onboard/docker-driver-gateway-service-version-gate.test.ts +++ b/src/lib/onboard/docker-driver-gateway-service-version-gate.test.ts @@ -148,6 +148,24 @@ describe("package-managed gateway version gate (#8094)", () => { expect(warn).toHaveBeenCalledTimes(1); }); + it("lets observation-only callers decline the package unit without consuming the warning", () => { + const suppressedWarn = vi.fn(); + const laterWarn = vi.fn(); + + expect( + hasOpenShellGatewayUserService( + resolveOptions("0.0.91", { + suppressUnsupportedVersionWarning: true, + warn: suppressedWarn, + }), + ), + ).toBe(false); + expect(suppressedWarn).not.toHaveBeenCalled(); + + hasOpenShellGatewayUserService(resolveOptions("0.0.91", { warn: laterWarn })); + expect(laterWarn).toHaveBeenCalledOnce(); + }); + it("probes every documented package binary location", () => { const localBinary = "/usr/local/bin/openshell-gateway"; expect(checkUpstreamGatewayVersion(localBinary, resolveOptions("0.0.91"))).toMatchObject({ diff --git a/src/lib/onboard/docker-driver-gateway-service.test.ts b/src/lib/onboard/docker-driver-gateway-service.test.ts index bb39624646c..d6b72dc9f70 100644 --- a/src/lib/onboard/docker-driver-gateway-service.test.ts +++ b/src/lib/onboard/docker-driver-gateway-service.test.ts @@ -10,6 +10,7 @@ import { getOpenShellGatewayUserServiceBinaryPaths, getOpenShellGatewayUserServicePaths, getOpenShellUserConfigHome, + getTrustedActiveOpenShellGatewayUserServiceIdentity, getTrustedActiveOpenShellGatewayUserServicePid, hasOpenShellGatewayUserService, NEMOCLAW_OPENSHELL_GATEWAY_USER_SERVICE_MARKER, @@ -278,7 +279,7 @@ describe("docker-driver-gateway-service", () => { ); expect( - getTrustedActiveOpenShellGatewayUserServicePid({ + getTrustedActiveOpenShellGatewayUserServiceIdentity({ commandExists: (command) => command === "systemctl", env: { HOME: home }, existsSync: (candidate) => candidate === servicePath, @@ -288,7 +289,7 @@ describe("docker-driver-gateway-service", () => { readFileSync: () => `# ${NEMOCLAW_OPENSHELL_GATEWAY_USER_SERVICE_MARKER}\n`, spawnSyncImpl, }), - ).toBe(4242); + ).toEqual({ pid: 4242, executablePath: gatewayBin }); expect(spawnSyncImpl).toHaveBeenCalledWith( "systemctl", [ @@ -330,19 +331,25 @@ describe("docker-driver-gateway-service", () => { }); it("identifies the active official Homebrew gateway process (#6903)", () => { + const formulaPrefix = "/opt/homebrew/opt/openshell"; + const gatewayBin = `${formulaPrefix}/bin/openshell-gateway`; const spawnSyncImpl = vi.fn((_command: string, args: string[]) => { - if (args[0] === "info") return officialFormulaInfo(); - if (args[0] === "services") return officialRunningServiceInfo(); - return spawnResult(); + const responses = { + info: officialFormulaInfo(), + services: officialRunningServiceInfo(), + "--prefix": spawnResult(0, "", formulaPrefix), + }; + return responses[args[0] as keyof typeof responses] ?? spawnResult(); }); expect( - getTrustedActiveOpenShellGatewayUserServicePid({ + getTrustedActiveOpenShellGatewayUserServiceIdentity({ commandExists: (command) => command === "brew", + existsSync: (candidate) => candidate === gatewayBin, platform: "darwin", spawnSyncImpl, }), - ).toBe(4242); + ).toEqual({ pid: 4242, executablePath: gatewayBin }); expect(spawnSyncImpl).toHaveBeenCalledWith( "brew", ["services", "info", "openshell", "--json"], diff --git a/src/lib/onboard/docker-driver-gateway-service.ts b/src/lib/onboard/docker-driver-gateway-service.ts index 9db5fd4d5f3..ab305166d58 100644 --- a/src/lib/onboard/docker-driver-gateway-service.ts +++ b/src/lib/onboard/docker-driver-gateway-service.ts @@ -42,6 +42,8 @@ export interface OpenShellGatewayUserServiceOptions { platform?: NodeJS.Platform; /** Sink for the one-shot notice emitted when a package unit is declined. */ warn?: (message: string) => void; + /** Keep observation-only callers from emitting or consuming the warning latch. */ + suppressUnsupportedVersionWarning?: boolean; preparePortForServiceStart?: () => void; prepareServiceEnv?: () => void; readFileSync?: (filePath: string, encoding: BufferEncoding) => string; @@ -520,7 +522,9 @@ function resolveOpenShellGatewayUserService( if (verdict.supported) { return upstreamService; } - warnUnsupportedUpstreamGateway(verdict, opts); + if (!opts.suppressUnsupportedVersionWarning) { + warnUnsupportedUpstreamGateway(verdict, opts); + } } } @@ -630,9 +634,26 @@ function validateSystemdServiceIdentityFromProperties( }; } -export function getTrustedActiveOpenShellGatewayUserServicePid( +export interface TrustedActiveOpenShellGatewayUserServiceIdentity { + pid: number; + /** Exact validated systemd ExecStart or official Homebrew formula binary path. */ + executablePath: string | null; +} + +function resolveOfficialHomebrewGatewayBinary( + opts: Required>, +): string | null { + const prefix = runBrew(["--prefix", OPENSHELL_GATEWAY_HOMEBREW_SERVICE], opts); + if (!prefix.ok) return null; + const value = prefix.stdout?.trim() ?? ""; + if (!path.isAbsolute(value)) return null; + const gatewayBinary = path.normalize(path.join(value, "bin", "openshell-gateway")); + return opts.existsSync(gatewayBinary) ? gatewayBinary : null; +} + +export function getTrustedActiveOpenShellGatewayUserServiceIdentity( opts: OpenShellGatewayUserServiceOptions = {}, -): number | null { +): TrustedActiveOpenShellGatewayUserServiceIdentity | null { const platform = opts.platform ?? process.platform; if (platform !== "linux" && platform !== "darwin") return null; const env = opts.env ?? process.env; @@ -666,12 +687,22 @@ export function getTrustedActiveOpenShellGatewayUserServicePid( candidate.name === service.serviceName && candidate.service_name === `homebrew.mxcl.${service.serviceName}`, ); - return record?.running === true && + const pid = + record?.running === true && record.loaded === true && Number.isSafeInteger(record.pid) && Number(record.pid) > 0 - ? Number(record.pid) - : null; + ? Number(record.pid) + : null; + if (pid === null) return null; + return { + pid, + executablePath: resolveOfficialHomebrewGatewayBinary({ + env, + existsSync: opts.existsSync ?? fs.existsSync, + spawnSyncImpl, + }), + }; } catch { return null; } @@ -690,14 +721,20 @@ export function getTrustedActiveOpenShellGatewayUserServicePid( ); if (!result.ok) return null; const properties = parseSystemctlShow(result.stdout ?? ""); - if ( - properties.ActiveState !== "active" || - !validateSystemdServiceIdentityFromProperties(service, properties).ok - ) { + const identity = validateSystemdServiceIdentityFromProperties(service, properties); + if (properties.ActiveState !== "active" || !identity.ok) { return null; } const mainPid = Number(properties.MainPID); - return Number.isSafeInteger(mainPid) && mainPid > 0 ? mainPid : null; + return Number.isSafeInteger(mainPid) && mainPid > 0 + ? { pid: mainPid, executablePath: identity.execStartPath } + : null; +} + +export function getTrustedActiveOpenShellGatewayUserServicePid( + opts: OpenShellGatewayUserServiceOptions = {}, +): number | null { + return getTrustedActiveOpenShellGatewayUserServiceIdentity(opts)?.pid ?? null; } function removeCompetingNemoclawUnit( diff --git a/src/lib/onboard/fatal-runtime-preflight.test.ts b/src/lib/onboard/fatal-runtime-preflight.test.ts index 821308599e5..075019305de 100644 --- a/src/lib/onboard/fatal-runtime-preflight.test.ts +++ b/src/lib/onboard/fatal-runtime-preflight.test.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it, vi } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; const mocks = vi.hoisted(() => ({ preparePortableExperimentalHost: vi.fn() })); @@ -9,16 +9,21 @@ vi.mock("./experimental/portable-host-preparation", () => ({ preparePortableExperimentalHost: mocks.preparePortableExperimentalHost, })); +import type { DetectGpuDeps, GpuDetection } from "../inference/nim"; +import type { GatewayReadinessProjection } from "../readiness/gateway"; import { isLinuxDockerDriverGatewayEnabled } from "./docker-driver-platform"; import { - rejectUnsupportedContainerRuntime, + assertOnboardGatewayReadiness, + assertOnboardHostReadiness, runFatalOnboardRuntimePreflight, + runOnboardRuntimeEffectfulPreflightChecks, + runReadinessGatedRuntimePreflight, } from "./fatal-runtime-preflight"; import type { HostAssessment } from "./preflight"; function hostWithRuntime(runtime: HostAssessment["runtime"]): HostAssessment { return { - platform: process.platform, + platform: "linux", isWsl: false, runtime, dockerInstalled: true, @@ -39,7 +44,120 @@ function hostWithRuntime(runtime: HostAssessment["runtime"]): HostAssessment { }; } -describe("rejectUnsupportedContainerRuntime (#7320)", () => { +function hostWithMissingGpuIntegration(): HostAssessment { + return { + ...hostWithRuntime("docker"), + hasNvidiaGpu: true, + dockerCdiSpecDirs: ["/etc/cdi"], + cdiNvidiaGpuSpecMissing: true, + cdiNvidiaGpuSpecStale: false, + cdiNvidiaGpuSpecNeedsRepair: true, + nvidiaContainerToolkitInstalled: false, + }; +} + +function wslDockerDesktopHost(): HostAssessment { + return { + ...hostWithRuntime("docker-desktop"), + isWsl: true, + hasNvidiaGpu: true, + nvidiaContainerToolkitInstalled: false, + }; +} + +function managedGatewayReadiness( + overrides: Partial = {}, +): GatewayReadinessProjection { + return { + observations: [{ id: "gateway.management.mode", state: "present", value: "nemoclaw-managed" }], + capabilities: [ + { id: "gateway.authority.resolved", state: "present" }, + { id: "gateway.attachment.valid", state: "present" }, + { id: "gateway.reuse.ready", state: "present" }, + { id: "gateway.version.compatible", state: "present" }, + { id: "gateway.port.uncontested", state: "present" }, + ], + findings: [], + evidence: [], + ...overrides, + }; +} + +afterEach(() => { + vi.unstubAllEnvs(); + vi.restoreAllMocks(); +}); + +describe("report-backed runtime readiness (#7411)", () => { + it("rejects ambiguous gateway ownership before the caller can run effects", () => { + const exit = vi.fn(() => { + throw new Error("exit"); + }); + const gateway: GatewayReadinessProjection = { + observations: [], + capabilities: [ + { id: "gateway.authority.resolved", state: "present" }, + { id: "gateway.attachment.valid", state: "absent" }, + { id: "gateway.reuse.ready", state: "present" }, + { id: "gateway.version.compatible", state: "present" }, + { id: "gateway.port.uncontested", state: "absent" }, + ], + findings: [ + { + id: "gateway.ownership.multiple", + severity: "blocking", + summary: "Multiple gateway owners were observed.", + }, + ], + evidence: [ + { + id: "gateway.attachment.failure", + summary: "Stop the competing listener on port 8080 and retry.", + }, + ], + }; + const error = vi.spyOn(console, "error").mockImplementation(() => undefined); + + expect(() => assertOnboardGatewayReadiness(gateway, exit as never)).toThrow("exit"); + expect(exit).toHaveBeenCalledWith(1); + expect(error).toHaveBeenCalledWith(" Stop the competing listener on port 8080 and retry."); + }); + + it("preserves actionable managed port evidence at the readiness gate", () => { + const exit = vi.fn(() => { + throw new Error("exit"); + }); + const gateway = managedGatewayReadiness({ + capabilities: [ + { id: "gateway.authority.resolved", state: "present" }, + { id: "gateway.attachment.valid", state: "present" }, + { id: "gateway.reuse.ready", state: "present" }, + { id: "gateway.version.compatible", state: "present" }, + { id: "gateway.port.uncontested", state: "absent" }, + ], + findings: [ + { + id: "gateway.port.owner_mismatch", + severity: "blocking", + summary: "The gateway port has an incompatible owner.", + evidenceIds: ["gateway.port.conflict"], + }, + ], + evidence: [ + { + id: "gateway.port.conflict", + summary: "Inspect port 8080 and stop only its owning process before retrying.", + }, + ], + }); + const error = vi.spyOn(console, "error").mockImplementation(() => undefined); + + expect(() => assertOnboardGatewayReadiness(gateway, exit as never)).toThrow("exit"); + expect(error).toHaveBeenCalledWith( + " Inspect port 8080 and stop only its owning process before retrying.", + ); + }); + // The Docker-driver gateway path is forced on Linux and Apple Silicon macOS; // the reject gate only fires there. Gate the test on the same predicate via // it.skipIf (not an in-body `if`) so it runs on the Linux CI runner. @@ -50,7 +168,10 @@ describe("rejectUnsupportedContainerRuntime (#7320)", () => { throw new Error("exit"); }); expect(() => - rejectUnsupportedContainerRuntime(hostWithRuntime("podman"), exit as never), + assertOnboardHostReadiness(hostWithRuntime("podman"), null, { + explicitlyOptedOutGpuPassthrough: false, + exitProcess: exit as never, + }), ).toThrow("exit"); expect(exit).toHaveBeenCalledWith(1); }, @@ -58,31 +179,494 @@ describe("rejectUnsupportedContainerRuntime (#7320)", () => { it("does not exit for a supported Docker runtime", () => { const exit = vi.fn(); - rejectUnsupportedContainerRuntime(hostWithRuntime("docker"), exit as never); + assertOnboardHostReadiness(hostWithRuntime("docker"), null, { + explicitlyOptedOutGpuPassthrough: false, + exitProcess: exit as never, + }); expect(exit).not.toHaveBeenCalled(); }); + it("rejects an unsupported DOCKER_HOST before runtime probe effects (#7411)", async () => { + const bridge = vi.fn(); + const validateGpu = vi.fn(); + const exitProcess = vi.fn((_code: number): never => { + throw new Error("invalid Docker endpoint"); + }); + + await expect( + runReadinessGatedRuntimePreflight( + {}, + { + nonInteractive: true, + collectGatewayReadiness: async () => managedGatewayReadiness(), + assessHost: () => ({ + ...hostWithRuntime("docker"), + dockerHostInvalid: true, + }), + detectGpu: () => null, + warnIfHostProxyMissesLoopback: vi.fn(), + assertDockerBridgeAndContainerDnsHealthy: bridge, + validateSandboxGpuPreflight: validateGpu, + exitProcess, + }, + ), + ).rejects.toThrow("invalid Docker endpoint"); + + expect(exitProcess).toHaveBeenCalledWith(1); + expect(bridge).not.toHaveBeenCalled(); + expect(validateGpu).not.toHaveBeenCalled(); + }); + + it("preserves Jetson NVIDIA runtime remediation before GPU-enabled effects (#7411)", () => { + const exit = vi.fn((_code: number): never => { + throw new Error("exit"); + }); + const error = vi.spyOn(console, "error").mockImplementation(() => undefined); + const host = { + ...hostWithRuntime("docker"), + hasNvidiaGpu: true, + dockerNvidiaRuntimeAvailable: false, + cdiNvidiaGpuSpecMissing: true, + cdiNvidiaGpuSpecNeedsRepair: true, + nvidiaContainerToolkitInstalled: true, + }; + const gpu: GpuDetection = { + type: "nvidia", + platform: "jetson", + count: 1, + totalMemoryMB: 8_192, + perGpuMB: 8_192, + nimCapable: true, + }; + + expect(() => + assertOnboardHostReadiness(host, gpu, { + explicitlyOptedOutGpuPassthrough: false, + exitProcess: exit, + }), + ).toThrow("exit"); + const output = error.mock.calls.map(([line]) => line).join("\n"); + expect(output).toContain("sudo nvidia-ctk runtime configure --runtime=docker"); + expect(output).not.toContain("Generate NVIDIA CDI device specs"); + expect(output).not.toContain("nvidia-ctk cdi generate"); + + const optOutExit = vi.fn(); + assertOnboardHostReadiness(host, gpu, { + explicitlyOptedOutGpuPassthrough: true, + exitProcess: optOutExit as never, + }); + expect(optOutExit).not.toHaveBeenCalled(); + }); + it.skipIf(!isLinuxDockerDriverGatewayEnabled())( "allows Podman only when the portable profile is explicit", () => { vi.stubEnv("NEMOCLAW_EXPERIMENTAL_PROFILE", "portable"); const exit = vi.fn(); - rejectUnsupportedContainerRuntime(hostWithRuntime("podman"), exit as never); + assertOnboardHostReadiness(hostWithRuntime("podman"), null, { + explicitlyOptedOutGpuPassthrough: false, + exitProcess: exit as never, + }); expect(exit).not.toHaveBeenCalled(); - vi.unstubAllEnvs(); }, ); }); describe("runFatalOnboardRuntimePreflight", () => { - it("prepares the portable host before probing the container runtime", () => { + it.each([ + ["the CLI disable flag", { sandboxGpu: "disable" as const }, {}], + ["the environment CPU-only mode", {}, { NEMOCLAW_SANDBOX_GPU: "0" }], + ])("treats %s as explicit CPU-only intent", (_label, options, env) => { + for (const [name, value] of Object.entries(env)) vi.stubEnv(name, value); + const result = runFatalOnboardRuntimePreflight(options, { + nonInteractive: true, + assessHost: () => hostWithMissingGpuIntegration(), + detectGpu: () => ({ + type: "nvidia", + count: 1, + totalMemoryMB: 24_576, + perGpuMB: 24_576, + nimCapable: true, + }), + warnIfHostProxyMissesLoopback: vi.fn(), + assertDockerBridgeAndContainerDnsHealthy: vi.fn(), + validateSandboxGpuPreflight: vi.fn(), + exitProcess: vi.fn(() => { + throw new Error("unexpected exit"); + }) as never, + }); + + expect(result.sandboxGpuConfig.mode).toBe("0"); + }); + + it("admits the read-only host report before portable preparation effects", () => { + vi.stubEnv("NEMOCLAW_EXPERIMENTAL_PROFILE", "portable"); mocks.preparePortableExperimentalHost.mockImplementationOnce(() => { throw new Error("portable host prepared"); }); + const assess = vi.fn(() => hostWithRuntime("docker")); - expect(() => runFatalOnboardRuntimePreflight({}, { nonInteractive: true })).toThrow( - "portable host prepared", - ); + expect(() => + runFatalOnboardRuntimePreflight( + {}, + { nonInteractive: true, assessHost: assess, detectGpu: () => null }, + ), + ).toThrow("portable host prepared"); + expect(assess).toHaveBeenCalledOnce(); expect(mocks.preparePortableExperimentalHost).toHaveBeenCalledWith(process.env); + expect(assess.mock.invocationCallOrder[0]).toBeLessThan( + mocks.preparePortableExperimentalHost.mock.invocationCallOrder[0] ?? Number.POSITIVE_INFINITY, + ); + }); + + it("defers image and container checks until the caller explicitly runs them", () => { + const bridge = vi.fn(); + const gpu = vi.fn(); + const context = { + nonInteractive: true, + deferEffectfulChecks: true, + assessHost: () => hostWithRuntime("docker"), + detectGpu: () => null, + warnIfHostProxyMissesLoopback: vi.fn(), + assertDockerBridgeAndContainerDnsHealthy: bridge, + validateSandboxGpuPreflight: gpu, + }; + + const result = runFatalOnboardRuntimePreflight({}, context); + expect(bridge).not.toHaveBeenCalled(); + expect(gpu).not.toHaveBeenCalled(); + + runOnboardRuntimeEffectfulPreflightChecks(result, context); + expect(bridge).toHaveBeenCalledOnce(); + expect(gpu).toHaveBeenCalledOnce(); + }); + + it("disables the container-backed WSL GPU prover during host admission", () => { + const detect = vi.fn((_deps?: DetectGpuDeps): GpuDetection | null => null); + + runFatalOnboardRuntimePreflight( + {}, + { + nonInteractive: true, + deferEffectfulChecks: true, + assessHost: wslDockerDesktopHost, + detectGpu: detect, + }, + ); + + expect(detect).toHaveBeenCalledOnce(); + expect(detect).toHaveBeenCalledWith({ proveArm64WslDockerDesktopGpu: null }); + }); + + it("rejects known GPU configuration errors before bridge or GPU container probes (#7411)", () => { + const bridge = vi.fn(); + const validateGpu = vi.fn(); + const exitProcess = vi.fn((_code: number): never => { + throw new Error("invalid GPU configuration"); + }); + const context = { + nonInteractive: true, + deferEffectfulChecks: true, + assessHost: () => hostWithRuntime("docker"), + detectGpu: () => null, + warnIfHostProxyMissesLoopback: vi.fn(), + assertDockerBridgeAndContainerDnsHealthy: bridge, + validateSandboxGpuPreflight: validateGpu, + exitProcess, + }; + const result = runFatalOnboardRuntimePreflight({ sandboxGpu: "enable" }, context); + + expect(() => runOnboardRuntimeEffectfulPreflightChecks(result, context)).toThrow( + "invalid GPU configuration", + ); + expect(bridge).not.toHaveBeenCalled(); + expect(validateGpu).not.toHaveBeenCalled(); + }); +}); + +describe("readiness-gated runtime preflight", () => { + it("rejects a host assessment that exceeds the freshness window before effects (#7411)", async () => { + let currentTime = Date.parse("2026-08-07T12:00:00.000Z"); + const bridge = vi.fn(); + const validateGpu = vi.fn(); + const exitProcess = vi.fn((_code: number): never => { + throw new Error("stale host blocked"); + }); + + await expect( + runReadinessGatedRuntimePreflight( + {}, + { + nonInteractive: true, + now: () => new Date(currentTime), + collectGatewayReadiness: async () => managedGatewayReadiness(), + assessHost: () => { + currentTime += 30_001; + return hostWithRuntime("docker"); + }, + detectGpu: () => null, + assertDockerBridgeAndContainerDnsHealthy: bridge, + validateSandboxGpuPreflight: validateGpu, + exitProcess, + }, + ), + ).rejects.toThrow("stale host blocked"); + expect(bridge).not.toHaveBeenCalled(); + expect(validateGpu).not.toHaveBeenCalled(); + }); + + it("rejects the initial gateway snapshot before collecting host facts", async () => { + const assessHost = vi.fn(wslDockerDesktopHost); + const detectGpu = vi.fn((_deps?: DetectGpuDeps): GpuDetection | null => null); + const exitProcess = vi.fn(() => { + throw new Error("gateway blocked"); + }); + const blocked = managedGatewayReadiness({ + capabilities: [ + { id: "gateway.authority.resolved", state: "absent" }, + { id: "gateway.attachment.valid", state: "absent" }, + { id: "gateway.reuse.ready", state: "absent" }, + { id: "gateway.version.compatible", state: "unknown" }, + { id: "gateway.port.uncontested", state: "unknown" }, + ], + findings: [ + { + id: "gateway.ownership.multiple", + severity: "blocking", + summary: "Multiple gateway owners were observed.", + }, + ], + }); + + await expect( + runReadinessGatedRuntimePreflight( + {}, + { + nonInteractive: true, + collectGatewayReadiness: async () => blocked, + assessHost, + detectGpu, + exitProcess: exitProcess as never, + }, + ), + ).rejects.toThrow("gateway blocked"); + expect(assessHost).not.toHaveBeenCalled(); + expect(detectGpu).not.toHaveBeenCalled(); + }); + + it("runs the bounded WSL GPU proof only after host and gateway admission", async () => { + const calls: string[] = []; + const detectGpu = vi.fn((deps?: DetectGpuDeps): GpuDetection | null => { + const isObservation = deps?.proveArm64WslDockerDesktopGpu === null; + calls.push(isObservation ? "gpu-observation" : "gpu-runtime-proof"); + return isObservation + ? null + : { + type: "nvidia", + count: 1, + totalMemoryMB: 32_768, + perGpuMB: 32_768, + nimCapable: true, + wslDockerDesktopGpuProofPassed: true, + }; + }); + + const result = await runReadinessGatedRuntimePreflight( + {}, + { + nonInteractive: true, + collectGatewayReadiness: async () => { + calls.push("gateway-admission"); + return managedGatewayReadiness(); + }, + assessHost: () => { + calls.push("host-observation"); + return wslDockerDesktopHost(); + }, + detectGpu, + warnIfHostProxyMissesLoopback: vi.fn(), + assertDockerBridgeAndContainerDnsHealthy: () => calls.push("bridge-dns"), + validateSandboxGpuPreflight: () => calls.push("gpu-validation"), + }, + ); + + expect(calls).toEqual([ + "gateway-admission", + "host-observation", + "gpu-observation", + "gateway-admission", + "host-observation", + "gpu-observation", + "gpu-runtime-proof", + "gateway-admission", + "host-observation", + "gpu-validation", + "bridge-dns", + ]); + expect(result.gpu).toMatchObject({ wslDockerDesktopGpuProofPassed: true }); + }); + + it("preserves a failed bounded WSL GPU proof as an absent readiness capability (#7411)", async () => { + const detectGpu = vi.fn((_deps?: DetectGpuDeps): GpuDetection | null => null); + + const result = await runReadinessGatedRuntimePreflight( + {}, + { + nonInteractive: true, + collectGatewayReadiness: async () => managedGatewayReadiness(), + assessHost: wslDockerDesktopHost, + detectGpu, + warnIfHostProxyMissesLoopback: vi.fn(), + assertDockerBridgeAndContainerDnsHealthy: vi.fn(), + validateSandboxGpuPreflight: vi.fn(), + }, + ); + + expect(result.readinessReport.capabilities).toContainEqual({ + id: "host.platform.wsl_gpu_passthrough", + state: "absent", + }); + expect(result.readinessReport.findings).toContainEqual( + expect.objectContaining({ id: "host.platform.wsl_gpu_passthrough_unavailable" }), + ); + }); + + it("rejects an explicit GPU request after a failed WSL proof before later container probes (#7411)", async () => { + const bridge = vi.fn(); + const validateGpu = vi.fn(); + const exitProcess = vi.fn((_code: number): never => { + throw new Error("GPU proof rejected"); + }); + + await expect( + runReadinessGatedRuntimePreflight( + { sandboxGpu: "enable" }, + { + nonInteractive: true, + collectGatewayReadiness: async () => managedGatewayReadiness(), + assessHost: wslDockerDesktopHost, + detectGpu: () => null, + warnIfHostProxyMissesLoopback: vi.fn(), + assertDockerBridgeAndContainerDnsHealthy: bridge, + validateSandboxGpuPreflight: validateGpu, + exitProcess, + }, + ), + ).rejects.toThrow("GPU proof rejected"); + + expect(exitProcess).toHaveBeenCalledWith(1); + expect(bridge).not.toHaveBeenCalled(); + expect(validateGpu).not.toHaveBeenCalled(); + }); + + it("recollects gateway facts before bridge and GPU probe effects", async () => { + const calls: string[] = []; + const collectGatewayReadiness = vi.fn(async () => { + calls.push("gateway"); + return managedGatewayReadiness(); + }); + + await runReadinessGatedRuntimePreflight( + {}, + { + nonInteractive: true, + collectGatewayReadiness, + assessHost: () => { + calls.push("host"); + return hostWithRuntime("docker"); + }, + detectGpu: () => null, + warnIfHostProxyMissesLoopback: vi.fn(), + assertDockerBridgeAndContainerDnsHealthy: () => calls.push("bridge"), + validateSandboxGpuPreflight: () => calls.push("gpu"), + }, + ); + + expect(calls).toEqual(["gateway", "host", "gateway", "host", "gpu", "bridge"]); + }); + + it("replaces portable host and gateway facts before runtime probe effects", async () => { + vi.stubEnv("NEMOCLAW_EXPERIMENTAL_PROFILE", "portable"); + const calls: string[] = []; + mocks.preparePortableExperimentalHost.mockImplementationOnce(() => { + calls.push("portable"); + }); + + await runReadinessGatedRuntimePreflight( + {}, + { + nonInteractive: true, + collectGatewayReadiness: async () => { + calls.push("gateway"); + return managedGatewayReadiness(); + }, + assessHost: () => { + calls.push("host"); + return hostWithRuntime("docker"); + }, + detectGpu: () => null, + warnIfHostProxyMissesLoopback: vi.fn(), + assertDockerBridgeAndContainerDnsHealthy: () => calls.push("bridge"), + validateSandboxGpuPreflight: () => calls.push("gpu"), + }, + ); + + expect(calls).toEqual([ + "gateway", + "host", + "portable", + "host", + "gateway", + "host", + "gpu", + "bridge", + ]); + }); + + it("does not run image or container checks when refreshed gateway facts block", async () => { + const bridge = vi.fn(); + const gpu = vi.fn(); + const exit = vi.fn(() => { + throw new Error("blocked"); + }); + const blocked = managedGatewayReadiness({ + capabilities: [ + { id: "gateway.authority.resolved", state: "present" }, + { id: "gateway.attachment.valid", state: "present" }, + { id: "gateway.reuse.ready", state: "present" }, + { id: "gateway.version.compatible", state: "present" }, + { id: "gateway.port.uncontested", state: "absent" }, + ], + findings: [ + { + id: "gateway.port.owner_mismatch", + severity: "blocking", + summary: "The gateway port owner changed.", + }, + ], + }); + const collectGatewayReadiness = vi + .fn<() => Promise>() + .mockResolvedValueOnce(managedGatewayReadiness()) + .mockResolvedValueOnce(blocked); + + await expect( + runReadinessGatedRuntimePreflight( + {}, + { + nonInteractive: true, + collectGatewayReadiness, + assessHost: () => hostWithRuntime("docker"), + detectGpu: () => null, + warnIfHostProxyMissesLoopback: vi.fn(), + assertDockerBridgeAndContainerDnsHealthy: bridge, + validateSandboxGpuPreflight: gpu, + exitProcess: exit as never, + }, + ), + ).rejects.toThrow("blocked"); + expect(collectGatewayReadiness).toHaveBeenCalledTimes(2); + expect(bridge).not.toHaveBeenCalled(); + expect(gpu).not.toHaveBeenCalled(); }); }); diff --git a/src/lib/onboard/fatal-runtime-preflight.ts b/src/lib/onboard/fatal-runtime-preflight.ts index ee1184f467d..ea7fffd626b 100644 --- a/src/lib/onboard/fatal-runtime-preflight.ts +++ b/src/lib/onboard/fatal-runtime-preflight.ts @@ -1,7 +1,24 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { getBuildIdentity } from "../core/version"; import { detectGpu, type GpuDetection } from "../inference/nim"; +import { + createGatewayReadinessProjection, + type GatewayReadinessProjection, + refreshGatewayReadinessProjection, +} from "../readiness/gateway"; +import { + createProductionGatewayReadinessDependencies, + type ProductionGatewayReadinessOptions, +} from "../readiness/gateway-production"; +import { collectHostObservations, projectHostReadiness } from "../readiness/host"; +import { + evaluateOnboardGatewayReadinessAdmission, + evaluateOnboardReadinessAdmission, +} from "../readiness/onboard-admission"; +import { composeSystemReadinessReport } from "../readiness/system"; +import type { SystemReadinessReport } from "../readiness/types"; import { assertDockerBridgeAndContainerDnsHealthy } from "./bridge-dns-preflight"; import { isLinuxDockerDriverGatewayEnabled, @@ -9,16 +26,17 @@ import { } from "./docker-driver-platform"; import { preparePortableExperimentalHost } from "./experimental/portable-host-preparation"; import { warnIfHostProxyMissesLoopback } from "./http-proxy-preflight"; +import { assessHost, type HostAssessment, planHostAdvisories } from "./preflight"; import { - assertCdiNvidiaGpuSpecPresent, - assessHost, - type HostAssessment, - planHostRemediation, -} from "./preflight"; -import { printDockerNotReachableError, printUnsupportedRuntimeError } from "./preflight-messages"; + printCdiSpecUnavailableError, + printDockerNotReachableError, + printUnsupportedRuntimeError, +} from "./preflight-messages"; import { printRemediationActions } from "./remediation"; import { resolveSandboxGpuConfig, type SandboxGpuConfig } from "./sandbox-gpu-mode"; import { + exitOnSandboxGpuConfigErrors, + printJetsonNvidiaRuntimeUnavailableError, resolveSandboxGpuFlagFromOptions, validateSandboxGpuPreflight, } from "./sandbox-gpu-preflight"; @@ -33,27 +51,354 @@ export type FatalRuntimePreflightOptions = Pick< export interface FatalRuntimePreflightContext { nonInteractive: boolean; + resuming?: boolean; + allowStorageRemediation?: boolean; + deferEffectfulChecks?: boolean; exitProcess?: (code: number) => never; + assessHost?: typeof assessHost; + /** + * GPU detector used in both phases. Readiness collection passes an explicit + * null WSL prover; the post-admission runtime phase calls it without that + * override so the bounded Docker proof can run when needed. + */ + detectGpu?: typeof detectGpu; + preparePortableExperimentalHost?: typeof preparePortableExperimentalHost; + warnIfHostProxyMissesLoopback?: typeof warnIfHostProxyMissesLoopback; + assertDockerBridgeAndContainerDnsHealthy?: typeof assertDockerBridgeAndContainerDnsHealthy; + validateSandboxGpuPreflight?: typeof validateSandboxGpuPreflight; + now?: () => Date; } export interface FatalRuntimePreflightResult { gpu: GpuDetection | null; host: HostAssessment; + readinessReport: SystemReadinessReport; sandboxGpuConfig: SandboxGpuConfig; } +export type ReadinessGatedRuntimePreflightContext = Omit< + FatalRuntimePreflightContext, + "allowStorageRemediation" | "deferEffectfulChecks" +> & { + collectGatewayReadiness(): Promise; +}; + +export interface ReadinessGatedRuntimePreflightResult extends FatalRuntimePreflightResult { + gatewayReadiness: GatewayReadinessProjection; +} + const exitProcessByDefault = (code: number): never => process.exit(code); +const JETSON_INAPPLICABLE_CDI_ADVISORY_IDS = new Set([ + "warn_nvidia_cdi_refresh_unhealthy", + "wsl_docker_desktop_gpu_compatibility", + "generate_nvidia_cdi_spec", + "refresh_nvidia_cdi_spec", + "install_nvidia_container_toolkit", +]); + +export interface OnboardHostReadinessOptions { + explicitlyOptedOutGpuPassthrough: boolean; + /** Preserve the outcome of a bounded WSL Docker Desktop GPU proof. */ + wslDockerDesktopGpuProofPassed?: boolean; + resuming?: boolean; + allowStorageRemediation?: boolean; + allowPortableHostPreparation?: boolean; + exitProcess?: (code: number) => never; + observedAt?: string; + now?: () => Date; +} -/** Reject runtimes that cannot support the OpenShell Docker-driver integration. */ -export function rejectUnsupportedContainerRuntime( +function printReadinessFailure( + report: Pick, + findingIds: readonly string[], + capabilityIds: readonly string[], +): void { + const findings = new Map(report.findings.map((finding) => [finding.id, finding])); + for (const findingId of findingIds) { + const summary = findings.get(findingId)?.summary ?? findingId; + console.error(` ✗ ${summary}`); + } + if (capabilityIds.length > 0) { + console.error( + ` ✗ System readiness could not confirm required capabilities: ${capabilityIds.join(", ")}.`, + ); + } +} + +function printGatewayReadinessEvidence(gateway: GatewayReadinessProjection): void { + const actionableEvidenceIds = new Set([ + "gateway.attachment.failure", + "gateway.port.conflict", + "gateway.probe.failure", + "gateway.probe.stale", + ]); + for (const entry of gateway.evidence) { + if (actionableEvidenceIds.has(entry.id)) console.error(` ${entry.summary}`); + } +} + +/** Apply onboarding policy to one canonical system readiness report. */ +export function assertOnboardSystemReadiness( + readinessReport: SystemReadinessReport, host: HostAssessment, + options: OnboardHostReadinessOptions, +): SystemReadinessReport { + const exitProcess = options.exitProcess ?? exitProcessByDefault; + const admission = evaluateOnboardReadinessAdmission(readinessReport, { + explicitlyOptedOutGpuPassthrough: options.explicitlyOptedOutGpuPassthrough, + allowUnsupportedRuntime: + isPortableExperimentalProfile() || !isLinuxDockerDriverGatewayEnabled(), + allowStorageRemediation: options.allowStorageRemediation === true, + allowPortableHostPreparation: options.allowPortableHostPreparation, + }); + if (admission.admitted) return readinessReport; + const jetsonRuntimeMissing = admission.findingIds.includes("host.gpu.nvidia_runtime_missing"); + + if ( + admission.findingIds.includes("host.docker.unavailable") || + admission.findingIds.includes("host.docker.daemon_unreachable") + ) { + printDockerNotReachableError(); + } else if (admission.findingIds.includes("host.docker.runtime_unsupported")) { + printUnsupportedRuntimeError(); + } else if ( + admission.findingIds.includes("host.gpu.cdi_missing") || + admission.findingIds.includes("host.gpu.cdi_stale") + ) { + printCdiSpecUnavailableError(); + } else if (jetsonRuntimeMissing) { + printJetsonNvidiaRuntimeUnavailableError(); + } else { + printReadinessFailure(readinessReport, admission.findingIds, admission.capabilityIds); + } + const advisories = planHostAdvisories(host, { resuming: options.resuming }); + printRemediationActions( + jetsonRuntimeMissing + ? advisories.filter(({ id }) => !JETSON_INAPPLICABLE_CDI_ADVISORY_IDS.has(id)) + : advisories, + ); + exitProcess(1); + throw new Error("Onboarding continued after a blocking system readiness result."); +} + +/** Fail closed on the canonical gateway projection before onboarding effects. */ +export function assertOnboardGatewayReadiness( + gateway: GatewayReadinessProjection, exitProcess: (code: number) => never = exitProcessByDefault, ): void { - if (isLinuxDockerDriverGatewayEnabled() && host.runtime === "podman") { - if (isPortableExperimentalProfile()) return; - printUnsupportedRuntimeError(); - exitProcess(1); + const admission = evaluateOnboardGatewayReadinessAdmission(gateway); + if (admission.admitted) return; + printReadinessFailure(gateway, admission.findingIds, admission.capabilityIds); + printGatewayReadinessEvidence(gateway); + exitProcess(1); + throw new Error("Onboarding continued after an unsafe gateway readiness result."); +} + +/** Collect and admit the production gateway projection before onboarding effects. */ +export async function collectOnboardGatewayReadiness( + options: ProductionGatewayReadinessOptions, +): Promise { + const gatewayReadiness = await createGatewayReadinessProjection( + createProductionGatewayReadinessDependencies(options), + ); + assertOnboardGatewayReadiness(gatewayReadiness); + return gatewayReadiness; +} + +function isManagedGatewayReadiness(gateway: GatewayReadinessProjection): boolean { + return gateway.observations.some( + ({ id, state, value }) => + id === "gateway.management.mode" && state === "present" && value === "nemoclaw-managed", + ); +} + +function requiresRuntimeGpuProof( + result: FatalRuntimePreflightResult, + options: FatalRuntimePreflightOptions, +): boolean { + return ( + result.host.isWsl && + result.host.runtime === "docker-desktop" && + result.host.dockerReachable && + result.host.hasNvidiaGpu && + result.gpu?.wslDockerDesktopGpuProofPassed !== true && + result.sandboxGpuConfig.mode !== "0" && + options.optedOutGpuPassthrough !== true + ); +} + +function refreshOnboardHostReadiness( + options: FatalRuntimePreflightOptions, + context: FatalRuntimePreflightContext, + allowStorageRemediation: boolean, + runtimeGpu?: { + value: GpuDetection | null; + wslDockerDesktopGpuProofPassed?: boolean; + }, +): FatalRuntimePreflightResult { + const now = context.now ?? (() => new Date()); + const observedAt = now().toISOString(); + const host = (context.assessHost ?? assessHost)(); + const gpu = runtimeGpu + ? runtimeGpu.value + : (context.detectGpu ?? detectGpu)({ proveArm64WslDockerDesktopGpu: null }); + const sandboxGpuConfig = resolveSandboxGpuConfig(gpu, { + flag: resolveSandboxGpuFlagFromOptions(options), + device: options.sandboxGpuDevice ?? null, + }); + const readinessReport = assertOnboardHostReadiness(host, gpu, { + explicitlyOptedOutGpuPassthrough: + sandboxGpuConfig.mode === "0" || options.optedOutGpuPassthrough === true, + wslDockerDesktopGpuProofPassed: runtimeGpu?.wslDockerDesktopGpuProofPassed, + resuming: context.resuming, + allowStorageRemediation, + exitProcess: context.exitProcess, + observedAt, + now, + }); + return { gpu, host, readinessReport, sandboxGpuConfig }; +} + +/** Resolve the bounded WSL GPU proof only after canonical readiness admission. */ +function resolveRuntimeGpuProof( + result: FatalRuntimePreflightResult, + options: FatalRuntimePreflightOptions, + context: FatalRuntimePreflightContext, +): { result: FatalRuntimePreflightResult; proofRan: boolean } { + if (!requiresRuntimeGpuProof(result, options)) return { result, proofRan: false }; + const gpu = (context.detectGpu ?? detectGpu)(); + const sandboxGpuConfig = resolveSandboxGpuConfig(gpu, { + flag: resolveSandboxGpuFlagFromOptions(options), + device: options.sandboxGpuDevice ?? null, + }); + return { + result: { ...result, gpu, sandboxGpuConfig }, + proofRan: true, + }; +} + +/** Apply onboarding policy to one freshly collected host readiness report. */ +export function assertOnboardHostReadiness( + host: HostAssessment, + gpu: GpuDetection | null, + options: OnboardHostReadinessOptions, +): SystemReadinessReport { + const now = options.now ?? (() => new Date()); + const observedAt = options.observedAt; + const snapshot = collectHostObservations({ + assess: () => host, + detectGpu: () => gpu, + wslDockerDesktopGpuProofPassed: options.wslDockerDesktopGpuProofPassed, + now: observedAt ? () => new Date(observedAt) : now, + }); + const readinessReport = projectHostReadiness(snapshot, { ...getBuildIdentity(), now }); + return assertOnboardSystemReadiness(readinessReport, host, options); +} + +/** Run runtime probes that may pull an image or start a short-lived container. */ +export function runOnboardRuntimeEffectfulPreflightChecks( + result: FatalRuntimePreflightResult, + context: FatalRuntimePreflightContext, +): void { + const exitProcess = context.exitProcess ?? exitProcessByDefault; + exitOnSandboxGpuConfigErrors(result.sandboxGpuConfig, exitProcess); + console.log(" ✓ Docker is running"); + (context.warnIfHostProxyMissesLoopback ?? warnIfHostProxyMissesLoopback)(); + (context.validateSandboxGpuPreflight ?? validateSandboxGpuPreflight)( + result.sandboxGpuConfig, + {}, + exitProcess, + ); + (context.assertDockerBridgeAndContainerDnsHealthy ?? assertDockerBridgeAndContainerDnsHealthy)( + result.host, + context.nonInteractive, + exitProcess, + ); + if (result.host.runtime !== "unknown") { + console.log(` ✓ Container runtime: ${result.host.runtime}`); + } + if (result.host.notes.includes("Running under WSL")) console.log(" ⓘ Running under WSL"); +} + +/** Revalidate gateway facts after host preparation and before runtime probe effects. */ +export async function runReadinessGatedRuntimePreflight( + options: FatalRuntimePreflightOptions, + context: ReadinessGatedRuntimePreflightContext, +): Promise { + const exitProcess = context.exitProcess ?? exitProcessByDefault; + const gatewayBeforePreparation = await context.collectGatewayReadiness(); + assertOnboardGatewayReadiness(gatewayBeforePreparation, exitProcess); + runFatalOnboardRuntimePreflight(options, { + ...context, + allowStorageRemediation: isManagedGatewayReadiness(gatewayBeforePreparation), + deferEffectfulChecks: true, + }); + let gatewayReadiness = await context.collectGatewayReadiness(); + assertOnboardGatewayReadiness(gatewayReadiness, exitProcess); + let managedGatewayReadiness = isManagedGatewayReadiness(gatewayReadiness); + // Gateway collection can be slow. Replace the earlier host observation so + // the composite gate never stamps an old assessment with a fresh timestamp. + let refreshedResult = refreshOnboardHostReadiness(options, context, managedGatewayReadiness); + gatewayReadiness = refreshGatewayReadinessProjection(gatewayReadiness); + assertOnboardGatewayReadiness(gatewayReadiness, exitProcess); + managedGatewayReadiness = isManagedGatewayReadiness(gatewayReadiness); + let readinessReport = composeSystemReadinessReport( + refreshedResult.readinessReport, + gatewayReadiness, + ); + assertOnboardSystemReadiness(readinessReport, refreshedResult.host, { + explicitlyOptedOutGpuPassthrough: + refreshedResult.sandboxGpuConfig.mode === "0" || options.optedOutGpuPassthrough === true, + resuming: context.resuming, + allowStorageRemediation: managedGatewayReadiness, + exitProcess, + }); + // The only GPU detection path that may pull or start a container is delayed + // until both canonical host and gateway reports have admitted the run. + const runtimeGpu = resolveRuntimeGpuProof(refreshedResult, options, { + ...context, + allowStorageRemediation: managedGatewayReadiness, + }); + refreshedResult = runtimeGpu.result; + if (runtimeGpu.proofRan) { + // An explicit GPU request cannot fall back to CPU after a failed proof. + // Reject that known configuration error before any later container probe. + exitOnSandboxGpuConfigErrors(refreshedResult.sandboxGpuConfig, exitProcess); + // The bounded proof may pull an image or start a short-lived container. + // Replace both host and gateway observations again before later probes. + gatewayReadiness = await context.collectGatewayReadiness(); + assertOnboardGatewayReadiness(gatewayReadiness, exitProcess); + managedGatewayReadiness = isManagedGatewayReadiness(gatewayReadiness); + refreshedResult = refreshOnboardHostReadiness(options, context, managedGatewayReadiness, { + value: runtimeGpu.result.gpu, + // `detectGpu()` rejects a failed bounded proof by returning null. Keep + // that negative outcome distinct from the observation-only phase's + // intentionally unknown result. A normal trusted WSL GPU has no proof + // marker and remains unknown because no bounded proof was necessary. + wslDockerDesktopGpuProofPassed: + runtimeGpu.result.gpu === null + ? false + : runtimeGpu.result.gpu.wslDockerDesktopGpuProofPassed, + }); } + gatewayReadiness = refreshGatewayReadinessProjection(gatewayReadiness); + assertOnboardGatewayReadiness(gatewayReadiness, exitProcess); + managedGatewayReadiness = isManagedGatewayReadiness(gatewayReadiness); + readinessReport = composeSystemReadinessReport(refreshedResult.readinessReport, gatewayReadiness); + assertOnboardSystemReadiness(readinessReport, refreshedResult.host, { + explicitlyOptedOutGpuPassthrough: + refreshedResult.sandboxGpuConfig.mode === "0" || options.optedOutGpuPassthrough === true, + resuming: context.resuming, + allowStorageRemediation: managedGatewayReadiness, + exitProcess, + }); + const gatedResult = { + ...refreshedResult, + readinessReport, + gatewayReadiness, + }; + runOnboardRuntimeEffectfulPreflightChecks(gatedResult, context); + return gatedResult; } /** Prepare and run the runtime gates shared by fresh, resume, and rebuild onboarding. */ @@ -62,32 +407,71 @@ export function runFatalOnboardRuntimePreflight( context: FatalRuntimePreflightContext, ): FatalRuntimePreflightResult { const exitProcess = context.exitProcess ?? exitProcessByDefault; - preparePortableExperimentalHost(process.env); - const host = assessHost(); - if (!host.dockerReachable) { - printDockerNotReachableError(); - printRemediationActions(planHostRemediation(host)); - exitProcess(1); - } - rejectUnsupportedContainerRuntime(host, exitProcess); - console.log(" ✓ Docker is running"); - warnIfHostProxyMissesLoopback(); - const gpu = detectGpu(); - const sandboxGpuConfig = resolveSandboxGpuConfig(gpu, { + const assess = context.assessHost ?? assessHost; + const detect = context.detectGpu ?? detectGpu; + const preparePortable = + context.preparePortableExperimentalHost ?? preparePortableExperimentalHost; + const now = context.now ?? (() => new Date()); + let observedAt = now().toISOString(); + let host = assess(); + let gpu = detect({ proveArm64WslDockerDesktopGpu: null }); + let sandboxGpuConfig = resolveSandboxGpuConfig(gpu, { flag: resolveSandboxGpuFlagFromOptions(options), device: options.sandboxGpuDevice ?? null, }); - const explicitlyOptedOutGpuPassthrough = - options.optedOutGpuPassthrough === true || options.noGpu === true; - assertCdiNvidiaGpuSpecPresent( - host, + let explicitlyOptedOutGpuPassthrough = + sandboxGpuConfig.mode === "0" || options.optedOutGpuPassthrough === true; + + if (isPortableExperimentalProfile()) { + // Portable setup is an explicit remediation. Admit only its narrow, + // pre-mutation exception set, apply it, then replace every observation + // with a fresh canonical host report before continuing. + assertOnboardHostReadiness(host, gpu, { + explicitlyOptedOutGpuPassthrough, + resuming: context.resuming, + allowPortableHostPreparation: true, + exitProcess, + observedAt, + now, + }); + preparePortable(process.env); + observedAt = now().toISOString(); + host = assess(); + gpu = detect({ proveArm64WslDockerDesktopGpu: null }); + sandboxGpuConfig = resolveSandboxGpuConfig(gpu, { + flag: resolveSandboxGpuFlagFromOptions(options), + device: options.sandboxGpuDevice ?? null, + }); + explicitlyOptedOutGpuPassthrough = + sandboxGpuConfig.mode === "0" || options.optedOutGpuPassthrough === true; + } + const readinessReport = assertOnboardHostReadiness(host, gpu, { explicitlyOptedOutGpuPassthrough, - sandboxGpuConfig.hostGpuPlatform, + resuming: context.resuming, + allowStorageRemediation: context.allowStorageRemediation, exitProcess, - ); - assertDockerBridgeAndContainerDnsHealthy(host, context.nonInteractive, exitProcess); - validateSandboxGpuPreflight(sandboxGpuConfig, {}, exitProcess); - if (host.runtime !== "unknown") console.log(` ✓ Container runtime: ${host.runtime}`); - if (host.notes.includes("Running under WSL")) console.log(" ⓘ Running under WSL"); - return { gpu, host, sandboxGpuConfig }; + observedAt, + now, + }); + let result = { gpu, host, readinessReport, sandboxGpuConfig }; + if (!context.deferEffectfulChecks) { + const runtimeGpu = resolveRuntimeGpuProof(result, options, context); + if (runtimeGpu.proofRan) { + exitOnSandboxGpuConfigErrors( + runtimeGpu.result.sandboxGpuConfig, + context.exitProcess ?? exitProcessByDefault, + ); + } + result = runtimeGpu.proofRan + ? refreshOnboardHostReadiness(options, context, context.allowStorageRemediation === true, { + value: runtimeGpu.result.gpu, + wslDockerDesktopGpuProofPassed: + runtimeGpu.result.gpu === null + ? false + : runtimeGpu.result.gpu.wslDockerDesktopGpuProofPassed, + }) + : runtimeGpu.result; + runOnboardRuntimeEffectfulPreflightChecks(result, context); + } + return result; } diff --git a/src/lib/onboard/gateway-authority-checkpoint.test.ts b/src/lib/onboard/gateway-authority-checkpoint.test.ts index fe3565c47a3..4ca6cad0587 100644 --- a/src/lib/onboard/gateway-authority-checkpoint.test.ts +++ b/src/lib/onboard/gateway-authority-checkpoint.test.ts @@ -6,6 +6,7 @@ import { describe, expect, it } from "vitest"; import { isDecisionSelected } from "../state/onboard-checkpoint-decision"; import { createSession, normalizeSession } from "../state/onboard-session"; import { + adoptPackagedGatewayAuthorityAfterTrustedInstall, bindGatewayAuthorityToCheckpoint, checkpointGatewayAuthority, } from "./gateway-authority-checkpoint"; @@ -43,6 +44,15 @@ function managedOwner() { }); } +function packagedManagedOwner() { + return resolveGatewayOwner({ + gatewayName: "nemoclaw", + gatewayPort: 8080, + declaration: null, + hasPackagedService: true, + }); +} + describe("durable gateway lifecycle authority", () => { it("records the resolved authority before gateway effects (#6576)", () => { const session = createSession({ sessionId: "authority-session" }); @@ -91,4 +101,28 @@ describe("durable gateway lifecycle authority", () => { /authority changed since this onboarding attempt was checkpointed/, ); }); + + it("persists a trusted packaged-service install across process resume (#7411)", () => { + const firstProcess = createSession({ sessionId: "authority-session" }); + bindGatewayAuthorityToCheckpoint(firstProcess, managedOwner()); + + adoptPackagedGatewayAuthorityAfterTrustedInstall(firstProcess, packagedManagedOwner()); + const resumed = normalizeSession(JSON.parse(JSON.stringify(firstProcess)) as never); + + expect(resumed).not.toBeNull(); + expect(bindGatewayAuthorityToCheckpoint(resumed!, packagedManagedOwner())).toEqual( + packagedManagedOwner(), + ); + }); + + it("leaves the checkpoint unchanged when trusted install sees declaration drift (#7411)", () => { + const session = createSession({ sessionId: "authority-session" }); + bindGatewayAuthorityToCheckpoint(session, managedOwner()); + const before = structuredClone(session.checkpoint); + + expect(() => + adoptPackagedGatewayAuthorityAfterTrustedInstall(session, externalOwner()), + ).toThrow(/changed during trusted OpenShell installation/); + expect(session.checkpoint).toEqual(before); + }); }); diff --git a/src/lib/onboard/gateway-authority-checkpoint.ts b/src/lib/onboard/gateway-authority-checkpoint.ts index d29da93dc41..418cda94ad5 100644 --- a/src/lib/onboard/gateway-authority-checkpoint.ts +++ b/src/lib/onboard/gateway-authority-checkpoint.ts @@ -71,3 +71,37 @@ export function bindGatewayAuthorityToCheckpoint( }; return resolvedOwner; } + +/** + * Persist the one authority transition produced by NemoClaw's own successful + * OpenShell install. All other checkpoint drift remains an error. + */ +export function adoptPackagedGatewayAuthorityAfterTrustedInstall( + session: Session, + resolvedOwner: GatewayOwner, +): GatewayOwner { + const checkpoint = session.checkpoint ?? deriveCheckpointFromSession(session); + const decision = checkpoint.gatewayAuthority; + if (!isDecisionSelected(decision)) { + throw new Error("Trusted gateway installation requires a previously bound gateway authority."); + } + const recordedOwner = gatewayOwnerFromCheckpoint(decision.value); + const expectedPackagedOwner = { ...recordedOwner, source: "packaged-service" as const }; + if ( + recordedOwner.mode !== "nemoclaw-managed" || + recordedOwner.source !== "standalone" || + resolvedOwner.source !== "packaged-service" || + !sameGatewayOwner(expectedPackagedOwner, resolvedOwner) + ) { + throw new Error( + "Gateway lifecycle authority changed during trusted OpenShell installation " + + `(${describeGatewayOwnerForError(recordedOwner)} -> ${describeGatewayOwnerForError(resolvedOwner)}).`, + ); + } + session.checkpoint = { + ...checkpoint, + updatedAt: new Date().toISOString(), + gatewayAuthority: decisionSelected(checkpointGatewayAuthority(resolvedOwner)), + }; + return resolvedOwner; +} diff --git a/src/lib/onboard/gateway-host-runtime-https.test.ts b/src/lib/onboard/gateway-host-runtime-https.test.ts index a2974ebe4f9..7654f2a8da4 100644 --- a/src/lib/onboard/gateway-host-runtime-https.test.ts +++ b/src/lib/onboard/gateway-host-runtime-https.test.ts @@ -8,13 +8,16 @@ import { afterEach, describe, expect, it, vi } from "vitest"; const readiness = vi.hoisted(() => ({ http: vi.fn(async () => false), https: vi.fn(async () => true), + wait: vi.fn( + async (options: { probe?: () => Promise; recordTrace?: boolean } = {}) => + options.probe?.() ?? false, + ), })); vi.mock("./gateway-http-readiness", () => ({ isGatewayHttpReady: readiness.http, isDockerDriverGatewayHttpReady: readiness.https, - waitForGatewayHttpReady: async (options: { probe?: () => Promise } = {}) => - options.probe?.() ?? false, + waitForGatewayHttpReady: readiness.wait, })); import { createGatewayHostRuntime, type GatewayHostRuntimeDeps } from "./gateway-host-runtime"; @@ -31,6 +34,7 @@ afterEach(() => { vi.restoreAllMocks(); readiness.http.mockClear(); readiness.https.mockClear(); + readiness.wait.mockClear(); }); function createDeps(): GatewayHostRuntimeDeps { @@ -41,6 +45,7 @@ function createDeps(): GatewayHostRuntimeDeps { gatewayPort: () => 8080, getGatewayPortListenerRawScan: () => ({ pids: [4242], complete: true }), getInstalledOpenshellVersion: () => "0.0.90", + hasOpenShellGatewayUserService: () => false, isGatewayHealthy: () => true, readProcCgroup: () => `0::/system.slice/${SERVICE_NAME}\n`, readProcExe: () => EXEC_PATH, @@ -74,6 +79,8 @@ function declareHttpsExternalSupervision() { describe("externally supervised HTTPS gateway readiness", () => { it("uses the production mTLS gRPC probe and the declared TLS bundle (#6576)", async () => { declareHttpsExternalSupervision(); + const originalTlsDir = process.env.OPENSHELL_LOCAL_TLS_DIR; + process.env.OPENSHELL_GATEWAY_AUTH_TOKEN = "must-not-cross-readiness-boundary"; const runtime = createGatewayHostRuntime(createDeps()); const owner = runtime.getGatewayOwner(); @@ -84,14 +91,36 @@ describe("externally supervised HTTPS gateway readiness", () => { expect(readiness.https).toHaveBeenCalledWith( undefined, "https://127.0.0.1:8080/openshell.v1.OpenShell/Health", + { OPENSHELL_LOCAL_TLS_DIR: `${STATE_DIR}/tls` }, + { recordTrace: true }, ); - expect(process.env.OPENSHELL_LOCAL_TLS_DIR).toBe(`${STATE_DIR}/tls`); + expect(readiness.wait).toHaveBeenCalledWith(expect.objectContaining({ recordTrace: true })); + expect(process.env.OPENSHELL_LOCAL_TLS_DIR).toBe(originalTlsDir); expect(fs.statSync).toHaveBeenCalledTimes(3); expect(fs.accessSync).toHaveBeenCalledTimes(3); }); + it("disables both readiness trace spans for an observation-only probe (#7411)", async () => { + declareHttpsExternalSupervision(); + const runtime = createGatewayHostRuntime({ + ...createDeps(), + recordHttpReadinessTrace: false, + }); + + await runtime.probeGatewayAttachment(runtime.getGatewayOwner()); + + expect(readiness.https).toHaveBeenCalledWith( + undefined, + "https://127.0.0.1:8080/openshell.v1.OpenShell/Health", + expect.objectContaining({ OPENSHELL_LOCAL_TLS_DIR: `${STATE_DIR}/tls` }), + { recordTrace: false }, + ); + expect(readiness.wait).toHaveBeenCalledWith(expect.objectContaining({ recordTrace: false })); + }); + it("fails before probing when the declared client TLS bundle is unreadable (#6576)", async () => { declareHttpsExternalSupervision(); + const originalTlsDir = process.env.OPENSHELL_LOCAL_TLS_DIR; vi.mocked(fs.accessSync).mockImplementation(() => { throw new Error("EACCES"); }); @@ -101,5 +130,6 @@ describe("externally supervised HTTPS gateway readiness", () => { /TLS file is missing or unreadable/, ); expect(readiness.https).not.toHaveBeenCalled(); + expect(process.env.OPENSHELL_LOCAL_TLS_DIR).toBe(originalTlsDir); }); }); diff --git a/src/lib/onboard/gateway-host-runtime.test.ts b/src/lib/onboard/gateway-host-runtime.test.ts index dd927ce50ab..db452c7196e 100644 --- a/src/lib/onboard/gateway-host-runtime.test.ts +++ b/src/lib/onboard/gateway-host-runtime.test.ts @@ -177,6 +177,73 @@ describe("gateway host runtime ownership", () => { expect(() => runtime.getGatewayOwner()).toThrow(/authority changed during this run/); }); + + it("adopts only a trusted standalone-to-packaged-service install transition (#7411)", () => { + let hasPackagedService = false; + const runtime = createGatewayHostRuntime( + createDeps({ hasOpenShellGatewayUserService: () => hasPackagedService }), + ); + expect(runtime.getGatewayOwner()).toMatchObject({ source: "standalone" }); + + hasPackagedService = true; + + const persistOwner = vi.fn(); + expect(runtime.adoptPackagedGatewayOwnerAfterTrustedInstall(persistOwner)).toMatchObject({ + source: "packaged-service", + }); + expect(persistOwner).toHaveBeenCalledWith( + expect.objectContaining({ source: "packaged-service" }), + ); + expect(runtime.getGatewayOwner()).toMatchObject({ source: "packaged-service" }); + }); + + it("keeps the old binding when trusted-install persistence fails (#7411)", () => { + let hasPackagedService = false; + const runtime = createGatewayHostRuntime( + createDeps({ hasOpenShellGatewayUserService: () => hasPackagedService }), + ); + runtime.getGatewayOwner(); + hasPackagedService = true; + + expect(() => + runtime.adoptPackagedGatewayOwnerAfterTrustedInstall(() => { + throw new Error("checkpoint write failed"); + }), + ).toThrow("checkpoint write failed"); + expect(() => runtime.getGatewayOwner()).toThrow(/authority changed during this run/); + }); + + it("keeps normal owner revalidation strict after a packaged service appears (#7411)", () => { + let hasPackagedService = false; + const runtime = createGatewayHostRuntime( + createDeps({ hasOpenShellGatewayUserService: () => hasPackagedService }), + ); + runtime.getGatewayOwner(); + hasPackagedService = true; + + expect(() => runtime.getGatewayOwner()).toThrow(/authority changed during this run/); + }); + + it("rejects trusted-install adoption when a declaration changes the authority (#7411)", () => { + const runtime = createGatewayHostRuntime(createDeps()); + runtime.getGatewayOwner(); + declareExternalSupervision(); + + expect(() => runtime.adoptPackagedGatewayOwnerAfterTrustedInstall()).toThrow( + /authority changed during this run/, + ); + expect(() => runtime.getGatewayOwner()).toThrow(/authority changed during this run/); + }); + + it("rejects trusted-install adoption without an existing owner binding (#7411)", () => { + const runtime = createGatewayHostRuntime( + createDeps({ hasOpenShellGatewayUserService: () => true }), + ); + + expect(() => runtime.adoptPackagedGatewayOwnerAfterTrustedInstall()).toThrow( + /before this run has bound/, + ); + }); }); describe("gateway host runtime attachment probe", () => { @@ -354,6 +421,35 @@ describe("gateway host runtime attachment probe", () => { }); }); + it("does not combine a foreign health response with a later declared listener (#7411)", async () => { + declareExternalSupervision(); + let healthProbeRan = false; + const getGatewayPortListenerRawScan = vi.fn(() => ({ + pids: [healthProbeRan ? SYSTEMD_GATEWAY_PID : 4343], + complete: true, + })); + const runtime = createGatewayHostRuntime( + createDeps({ + getGatewayPortListenerRawScan, + probeGatewayHttpReady: async () => { + healthProbeRan = true; + return true; + }, + }), + ); + const owner = runtime.getGatewayOwner(); + + const probe = await runtime.probeGatewayAttachment(owner); + + expect(probe.httpReady).toBe(true); + expect(probe.listenerPids).toEqual([SYSTEMD_GATEWAY_PID]); + expect(probe.listenerExecPath).toBeNull(); + expect(evaluateGatewayAttachment(owner, probe)).toMatchObject({ + ok: false, + code: "unknown_listener", + }); + }); + it("fails closed when a PID is reused while its identity is read (#6576)", async () => { declareExternalSupervision(); const readProcStartTime = vi.fn().mockReturnValueOnce("710024").mockReturnValueOnce("710025"); @@ -370,6 +466,26 @@ describe("gateway host runtime attachment probe", () => { }); }); + it("fails closed when a listener execs another binary without changing PID (#7411)", async () => { + declareExternalSupervision(); + const readProcExe = vi + .fn() + .mockReturnValueOnce(SYSTEMD_GATEWAY_EXEC) + .mockReturnValueOnce("/opt/platform/impostor-gateway") + .mockReturnValue(SYSTEMD_GATEWAY_EXEC); + const runtime = createGatewayHostRuntime(createDeps({ readProcExe })); + const owner = runtime.getGatewayOwner(); + + const probe = await runtime.probeGatewayAttachment(owner); + + expect(probe.listenerExecPath).toBeNull(); + expect(probe.listenerSupervisorMatch).toBeNull(); + expect(evaluateGatewayAttachment(owner, probe)).toMatchObject({ + ok: false, + code: "unknown_listener", + }); + }); + it("fails closed when the listener PID is reused between identity and port confirmation (#6576)", async () => { declareExternalSupervision(); const readProcStartTime = vi @@ -409,6 +525,27 @@ describe("gateway host runtime attachment probe", () => { }); }); + it("uses the caller-supplied credential-free supervisor probe environment (#7411)", async () => { + declareExternalSupervision(); + const spawnSyncImpl = vi.fn(() => ({ status: 0, stdout: "active\n", stderr: "" })); + const supervisorProbeEnv = { + PATH: "/usr/bin", + XDG_RUNTIME_DIR: "/run/user/1000", + DBUS_SESSION_BUS_ADDRESS: "unix:path=/run/user/1000/bus", + }; + const runtime = createGatewayHostRuntime( + createDeps({ spawnSyncImpl: spawnSyncImpl as never, supervisorProbeEnv }), + ); + + await runtime.probeGatewayAttachment(runtime.getGatewayOwner()); + + expect(spawnSyncImpl).toHaveBeenCalledWith( + "systemctl", + ["is-active", "openshell-gateway.service"], + expect.objectContaining({ env: supervisorProbeEnv }), + ); + }); + it("reads the authoritative gateway port lazily, not at construction (#6576)", () => { let port = 8080; const runtime = createGatewayHostRuntime(createDeps({ gatewayPort: () => port })); diff --git a/src/lib/onboard/gateway-host-runtime.ts b/src/lib/onboard/gateway-host-runtime.ts index 3418a90db9d..749a971e8bf 100644 --- a/src/lib/onboard/gateway-host-runtime.ts +++ b/src/lib/onboard/gateway-host-runtime.ts @@ -48,6 +48,7 @@ const SUPERVISOR_PROBE_TIMEOUT_MS = 5_000; export interface GatewayHostRuntimeDeps { applyOverlayfsAutoFix(clusterImage: string): string | null; checkGatewayPortAvailable(): Promise; + hasOpenShellGatewayUserService?: typeof hasOpenShellGatewayUserService; /** * Read lazily: the onboarding entrypoint rebinds its gateway port at runtime * when an authoritative gateway is selected, so a captured value goes stale. @@ -65,6 +66,7 @@ export interface GatewayHostRuntimeDeps { ): GatewayPortListenerRawScan; getInstalledOpenshellVersion(): string | null; isGatewayHealthy?: typeof isGatewayHealthy; + loadGatewayManagementDeclaration?: typeof loadGatewayManagementDeclaration; runCaptureOpenshell(args: string[], opts?: { ignoreError?: boolean }): string; runOpenshell( args: string[], @@ -72,8 +74,14 @@ export interface GatewayHostRuntimeDeps { ): { status: number | null }; resolveOpenShellGatewayBinary(): string | null; spawnSyncImpl?: typeof import("node:child_process").spawnSync; + /** Explicit environment for local supervisor status probes. */ + supervisorProbeEnv?: NodeJS.ProcessEnv; + /** Credential-free base environment for external HTTPS client probes. */ + clientProbeEnv?: NodeJS.ProcessEnv; /** Overrides the readiness request; defaults to a real probe of `endpoint`. */ probeGatewayHttpReady?(endpoint: string | null): Promise; + /** Defaults to true; public observation-only readiness disables trace initialization. */ + recordHttpReadinessTrace?: boolean; /** Overrides `/proc//exe` resolution; defaults to the real symlink. */ readProcExe?(pid: number): string | null; /** Overrides `/proc//cgroup` reads; defaults to the real file. */ @@ -93,6 +101,13 @@ export interface GatewayHostRuntime { target?: { gatewayName: string; gatewayPort: number }, ): void; attachGateway(owner: GatewayOwner, expectedProbe: GatewayAttachmentProbe): Promise; + /** + * Adopt the packaged managed service only after NemoClaw's installer has + * successfully installed it during this run. + */ + adoptPackagedGatewayOwnerAfterTrustedInstall( + persistOwner?: (owner: GatewayOwner) => void, + ): GatewayOwner; bindGatewayOwner(owner: GatewayOwner): void; /** Local endpoint of the gateway this process operates. */ getGatewayLocalEndpoint(): string; @@ -114,7 +129,7 @@ export function createGatewayHostRuntime(deps: GatewayHostRuntimeDeps): GatewayH let boundOwner: GatewayOwner | null = null; function resolveCurrentGatewayOwner(gatewayName: string, gatewayPort: number): GatewayOwner { - const loaded = loadGatewayManagementDeclaration(); + const loaded = (deps.loadGatewayManagementDeclaration ?? loadGatewayManagementDeclaration)(); if (!loaded.ok) { throw invalidGatewayManagementDeclarationError(loaded.reason); } @@ -122,7 +137,7 @@ export function createGatewayHostRuntime(deps: GatewayHostRuntimeDeps): GatewayH gatewayName, gatewayPort, declaration: loaded.declaration, - hasPackagedService: hasOpenShellGatewayUserService(), + hasPackagedService: (deps.hasOpenShellGatewayUserService ?? hasOpenShellGatewayUserService)(), }); } @@ -152,7 +167,7 @@ export function createGatewayHostRuntime(deps: GatewayHostRuntimeDeps): GatewayH throw new Error( "Gateway lifecycle authority changed during this run " + `(${describeGatewayOwnerForError(boundOwner)} -> ${describeGatewayOwnerForError(resolved)}). ` + - "Exactly one component owns the gateway per run; re-run onboarding to adopt the new authority.", + "Exactly one component owns the gateway per run; rerun onboarding to adopt the new authority.", ); } return boundOwner; @@ -165,6 +180,33 @@ export function createGatewayHostRuntime(deps: GatewayHostRuntimeDeps): GatewayH return getGatewayOwnerForTarget(); } + function adoptPackagedGatewayOwnerAfterTrustedInstall( + persistOwner?: (owner: GatewayOwner) => void, + ): GatewayOwner { + if (!boundOwner) { + throw new Error( + "Cannot adopt a packaged gateway owner before this run has bound its lifecycle authority.", + ); + } + const resolved = resolveCurrentGatewayOwner(boundOwner.gatewayName, boundOwner.gatewayPort); + if (sameGatewayOwner(boundOwner, resolved)) return boundOwner; + const expectedPackagedOwner = { ...boundOwner, source: "packaged-service" as const }; + if ( + boundOwner.mode !== "nemoclaw-managed" || + boundOwner.source !== "standalone" || + !sameGatewayOwner(expectedPackagedOwner, resolved) + ) { + throw new Error( + "Gateway lifecycle authority changed during this run " + + `(${describeGatewayOwnerForError(boundOwner)} -> ${describeGatewayOwnerForError(resolved)}). ` + + "Exactly one component owns the gateway per run; re-run onboarding to adopt the new authority.", + ); + } + persistOwner?.(resolved); + boundOwner = resolved; + return boundOwner; + } + function isSupervisorUnitActive(owner: GatewayOwner): boolean | null { const supervisor = owner.supervisor; if (!supervisor) return null; @@ -172,6 +214,7 @@ export function createGatewayHostRuntime(deps: GatewayHostRuntimeDeps): GatewayH const scope = supervisor.kind === "systemd-user" ? ["--user"] : []; const result = spawnSyncImpl("systemctl", [...scope, "is-active", supervisor.serviceName], { encoding: "utf-8", + env: deps.supervisorProbeEnv, // spawnSync blocks the event loop, so a wedged systemd/D-Bus session would // otherwise stall onboarding indefinitely. A timeout surfaces as an error, // which the unknown-supervisor path below already treats as "cannot tell". @@ -236,8 +279,9 @@ export function createGatewayHostRuntime(deps: GatewayHostRuntimeDeps): GatewayH const startTime = readProcStartTime(pid); const execPath = readListenerExecPath(pid); const supervisorMatch = readListenerSupervisorMatch(owner, pid); + const confirmedExecPath = readListenerExecPath(pid); const endTime = readProcStartTime(pid); - return startTime !== null && endTime === startTime + return startTime !== null && endTime === startTime && confirmedExecPath === execPath ? { startTime, execPath, supervisorMatch } : null; } @@ -253,15 +297,26 @@ export function createGatewayHostRuntime(deps: GatewayHostRuntimeDeps): GatewayH if (!owner.endpoint) return deps.waitForGatewayHttpReady(); // The endpoint is constrained to a supported loopback origin at parse time, // so this cannot be pointed at an arbitrary host. - prepareExternalGatewayClient(owner); + const clientEnv = getExternalGatewayClientEnv(owner); const endpoint = owner.endpoint; + const recordTrace = deps.recordHttpReadinessTrace !== false; if (new URL(endpoint).protocol === "https:") { return waitForGatewayHttpReady({ probe: () => - isDockerDriverGatewayHttpReady(undefined, `${endpoint}/openshell.v1.OpenShell/Health`), + isDockerDriverGatewayHttpReady( + undefined, + `${endpoint}/openshell.v1.OpenShell/Health`, + clientEnv, + { recordTrace }, + ), + recordTrace, }); } - return waitForGatewayHttpReady({ probe: () => isGatewayHttpReady(undefined, `${endpoint}/`) }); + return waitForGatewayHttpReady({ + probe: () => + isGatewayHttpReady(undefined, `${endpoint}/`, undefined, undefined, { recordTrace }), + recordTrace, + }); } /** @@ -274,8 +329,6 @@ export function createGatewayHostRuntime(deps: GatewayHostRuntimeDeps): GatewayH throw new GatewayOwnershipError(configuration.code, configuration.message, owner); } const portCheck = await deps.checkGatewayPortAvailable(); - const httpReady = await waitForDeclaredGatewayHttpReady(owner); - const supervisorActive = isSupervisorUnitActive(owner); const gatewayBin = deps.resolveOpenShellGatewayBinary(); const scan = deps.getGatewayPortListenerRawScan(portCheck, { gatewayBin, @@ -285,6 +338,11 @@ export function createGatewayHostRuntime(deps: GatewayHostRuntimeDeps): GatewayH scan.complete && scan.pids.length === 1 && typeof firstPid === "number" ? readStableListenerIdentity(owner, firstPid) : null; + // Bind the health response to the same process identity observed on both + // sides of the request. Otherwise a foreign listener can answer health, + // exit, and be replaced by the declared service before identity sampling. + const httpReady = await waitForDeclaredGatewayHttpReady(owner); + const supervisorActive = isSupervisorUnitActive(owner); const verifiedScan = typeof firstPid === "number" ? deps.getGatewayPortListenerRawScan(portCheck, { @@ -338,7 +396,7 @@ export function createGatewayHostRuntime(deps: GatewayHostRuntimeDeps): GatewayH } } - function prepareExternalGatewayClient(owner: GatewayOwner): void { + function getExternalGatewayClientEnv(owner: GatewayOwner): NodeJS.ProcessEnv | undefined { if (!isExternallySupervised(owner) || !owner.endpoint) return; if (new URL(owner.endpoint).protocol !== "https:") return; if (!owner.stateDir) { @@ -356,7 +414,14 @@ export function createGatewayHostRuntime(deps: GatewayHostRuntimeDeps): GatewayH ); } } - process.env.OPENSHELL_LOCAL_TLS_DIR = localTlsDir; + return { ...(deps.clientProbeEnv ?? {}), OPENSHELL_LOCAL_TLS_DIR: localTlsDir }; + } + + function prepareExternalGatewayClient(owner: GatewayOwner): void { + const clientEnv = getExternalGatewayClientEnv(owner); + if (clientEnv?.OPENSHELL_LOCAL_TLS_DIR) { + process.env.OPENSHELL_LOCAL_TLS_DIR = clientEnv.OPENSHELL_LOCAL_TLS_DIR; + } } function sameAttachmentEvidence( @@ -494,6 +559,7 @@ export function createGatewayHostRuntime(deps: GatewayHostRuntimeDeps): GatewayH } return { + adoptPackagedGatewayOwnerAfterTrustedInstall, assertGatewayStartAllowed, attachGateway, bindGatewayOwner, diff --git a/src/lib/onboard/gateway-http-readiness.ts b/src/lib/onboard/gateway-http-readiness.ts index 4aeccb34175..ee9faa8408f 100644 --- a/src/lib/onboard/gateway-http-readiness.ts +++ b/src/lib/onboard/gateway-http-readiness.ts @@ -38,8 +38,24 @@ export type WaitForGatewayHttpReadyOpts = { sleeper?: (seconds: number) => void; maxAttempts?: number; intervalSeconds?: number; + /** Keep observation-only callers from initializing the onboard trace sink. */ + recordTrace?: boolean; }; +export type GatewayHttpReadinessTraceOptions = { + /** Defaults to true so onboarding keeps its existing trace coverage. */ + recordTrace?: boolean; +}; + +function withOptionalTraceSpan( + options: GatewayHttpReadinessTraceOptions, + name: string, + attributes: Record, + fn: () => T, +): T { + return options.recordTrace === false ? fn() : withTraceSpan(name, attributes, fn); +} + /** * Resolve raw poll count and interval (seconds) for the reuse-time gateway * HTTP readiness wait, from `NEMOCLAW_REUSE_HEALTH_POLL_COUNT` and @@ -78,9 +94,13 @@ export function isGatewayHttpReady( url = `${getGatewayHttpEndpoint(GATEWAY_PORT)}/`, method: "GET" | "POST" = "GET", signal?: AbortSignal, + traceOptions: GatewayHttpReadinessTraceOptions = {}, ): Promise { - return withTraceSpan("nemoclaw.gateway.http_probe", { timeout_ms: timeoutMs, url, method }, () => - isGatewayHttpReadyImpl(timeoutMs, url, method, signal), + return withOptionalTraceSpan( + traceOptions, + "nemoclaw.gateway.http_probe", + { timeout_ms: timeoutMs, url, method }, + () => isGatewayHttpReadyImpl(timeoutMs, url, method, signal), ); } @@ -126,8 +146,10 @@ export function isDockerDriverGatewayHttpReady( timeoutMs = ISGATEWAY_HTTP_READY_DEFAULT_TIMEOUT_MS, url = `${getGatewayHttpsEndpoint(GATEWAY_PORT)}/openshell.v1.OpenShell/Health`, env: NodeJS.ProcessEnv = process.env, + traceOptions: GatewayHttpReadinessTraceOptions = {}, ): Promise { - return withTraceSpan( + return withOptionalTraceSpan( + traceOptions, "nemoclaw.gateway.docker_driver_http_probe", { timeout_ms: timeoutMs, url }, () => isDockerDriverGatewayHttpReadyImpl(timeoutMs, url, env), @@ -270,8 +292,14 @@ function dockerDriverGatewayHttp2ConnectOptions( export async function waitForGatewayHttpReady( opts: WaitForGatewayHttpReadyOpts = {}, ): Promise { - return withTraceSpan("nemoclaw.gateway.http_readiness_wait", {}, async () => { - const probe = opts.probe ?? (() => isGatewayHttpReady()); + return withOptionalTraceSpan(opts, "nemoclaw.gateway.http_readiness_wait", {}, async () => { + const recordTrace = opts.recordTrace !== false; + const probe = + opts.probe ?? + (() => + isGatewayHttpReady(undefined, undefined, undefined, undefined, { + recordTrace, + })); const sleeper = opts.sleeper ?? sleepSeconds; const config = getGatewayReuseHealthWaitConfig(); // Always probe at least once, even if the caller passed a non-positive @@ -282,7 +310,12 @@ export async function waitForGatewayHttpReady( const maxAttempts = Number.isFinite(rawAttempts) ? Math.max(1, Math.round(rawAttempts)) : 1; const rawInterval = opts.intervalSeconds ?? config.interval; const intervalSeconds = Number.isFinite(rawInterval) ? Math.max(0, rawInterval) : 0; - addTraceEvent("wait_config", { max_attempts: maxAttempts, interval_seconds: intervalSeconds }); + if (recordTrace) { + addTraceEvent("wait_config", { + max_attempts: maxAttempts, + interval_seconds: intervalSeconds, + }); + } // The default probe (isGatewayHttpReady) never rejects, but injected probes // can. Treat a rejection as "not ready this attempt" so we exhaust the @@ -300,7 +333,7 @@ export async function waitForGatewayHttpReady( async () => { attempt += 1; if (!(await safeProbe())) return false; - addTraceEvent("ready", { attempt }); + if (recordTrace) addTraceEvent("ready", { attempt }); return true; }, { @@ -314,7 +347,7 @@ export async function waitForGatewayHttpReady( if (ready) { return true; } - addTraceEvent("not_ready", { attempts: maxAttempts }); + if (recordTrace) addTraceEvent("not_ready", { attempts: maxAttempts }); return false; }); } diff --git a/src/lib/onboard/machine/handlers/preflight.test.ts b/src/lib/onboard/machine/handlers/preflight.test.ts index 15296f63695..2d7f487cf5a 100644 --- a/src/lib/onboard/machine/handlers/preflight.test.ts +++ b/src/lib/onboard/machine/handlers/preflight.test.ts @@ -4,8 +4,8 @@ import { describe, expect, it, vi } from "vitest"; import type { GpuDetection } from "../../../inference/nim"; -import { resolveSandboxGpuConfig } from "../../sandbox-gpu-mode"; import { createSession, type Session } from "../../../state/onboard-session"; +import { resolveSandboxGpuConfig } from "../../sandbox-gpu-mode"; import { handlePreflightState, type PreflightStateOptions } from "./preflight"; type Gpu = GpuDetection | null; @@ -50,16 +50,18 @@ function createDeps( } return { flag: "enable" as const, device: "0" }; }, + detectGpuForReadiness: () => ({ type: "nvidia" }) as Gpu, detectGpu: () => ({ type: "nvidia" }) as Gpu, runPreflight: async () => ({ type: "nvidia" }) as Gpu, assessHost: () => ({ cdiNvidiaGpuSpecMissing: false }), - assertCdiNvidiaGpuSpecPresent: vi.fn(), + assertOnboardHostReadiness: vi.fn(), + assertGatewayReadiness: vi.fn(async () => undefined), resolveSandboxGpuConfig: ( _gpu: Gpu, opts: { flag: "enable" | "disable" | null; device: string | null | undefined }, ) => ({ sandboxGpuEnabled: opts.flag === "enable", - mode: opts.flag === "enable" ? "1" : "0", + mode: opts.flag === "enable" ? "1" : opts.flag === "disable" ? "0" : "auto", sandboxGpuDevice: opts.device, }), validateSandboxGpuPreflight: vi.fn(), @@ -157,13 +159,13 @@ describe("handlePreflightState", () => { expect(result.gpuPassthrough).toBe(false); }); - it("skips full preflight on resume but re-detects GPU and revalidates CDI/sandbox GPU", async () => { + it("skips recorded preflight on resume but re-runs live host readiness (#7411)", async () => { const session = createSession(); session.steps.preflight.status = "complete"; session.gpuPassthrough = false; const harness = createDeps({ detectGpu: vi.fn(() => ({ type: "nvidia" }) as Gpu), - assertCdiNvidiaGpuSpecPresent: vi.fn(), + assertOnboardHostReadiness: vi.fn(), validateSandboxGpuPreflight: vi.fn(), skippedStepMessage: vi.fn(), startRecordedStep: vi.fn(async () => undefined), @@ -179,26 +181,99 @@ describe("handlePreflightState", () => { expect(harness.deps.skippedStepMessage).toHaveBeenCalledWith("preflight", "cached"); expect(harness.deps.recordStateSkipped).toHaveBeenCalledWith("preflight", { reason: "resume", - validation: "gpu-cdi", + validation: "host-readiness", }); expect(harness.deps.detectGpu).toHaveBeenCalledOnce(); expect(harness.deps.runPreflight).not.toHaveBeenCalled(); expect(harness.deps.startRecordedStep).not.toHaveBeenCalled(); - expect(harness.deps.assertCdiNvidiaGpuSpecPresent).toHaveBeenCalledWith( + expect(harness.deps.assertOnboardHostReadiness).toHaveBeenCalledWith( { cdiNvidiaGpuSpecMissing: false }, - true, - undefined, + { type: "nvidia" }, + expect.objectContaining({ explicitlyOptedOutGpuPassthrough: false, resuming: true }), ); expect(harness.deps.validateSandboxGpuPreflight).toHaveBeenCalledOnce(); expect(result.resumePreflight).toBe(true); }); - it("passes host GPU platform into the resumed CDI guard", async () => { + it("rejects changed gateway ownership before cached resume probe effects (#7411)", async () => { + const session = createSession(); + session.steps.preflight.status = "complete"; + const assertDockerBridgeAndContainerDnsHealthy = vi.fn(); + const detectGpu = vi.fn(() => ({ type: "nvidia" }) as Gpu); + const harness = createDeps({ + assertGatewayReadiness: vi.fn(async () => { + throw new Error("gateway ownership changed"); + }), + detectGpu, + assertDockerBridgeAndContainerDnsHealthy, + }); + + await expect( + handlePreflightState({ + ...baseOptions(harness.deps, session), + resume: true, + }), + ).rejects.toThrow("gateway ownership changed"); + expect(detectGpu).not.toHaveBeenCalled(); + expect(assertDockerBridgeAndContainerDnsHealthy).not.toHaveBeenCalled(); + }); + + it("admits live host and gateway facts before resolving a cached resume GPU proof (#7411)", async () => { + const session = createSession(); + session.steps.preflight.status = "complete"; + const calls: string[] = []; + const harness = createDeps({ + assessHost: () => { + calls.push("host-observation"); + return { cdiNvidiaGpuSpecMissing: false }; + }, + detectGpuForReadiness: () => { + calls.push("gpu-observation"); + return null; + }, + assertOnboardHostReadiness: () => { + calls.push("host-admission"); + }, + assertGatewayReadiness: async () => { + calls.push("gateway-admission"); + }, + detectGpu: () => { + calls.push("gpu-runtime-proof"); + return { type: "nvidia" } as Gpu; + }, + validateSandboxGpuPreflight: () => { + calls.push("gpu-validation"); + }, + assertDockerBridgeAndContainerDnsHealthy: () => { + calls.push("bridge-dns"); + }, + }); + + await handlePreflightState({ + ...baseOptions(harness.deps, session), + resume: true, + }); + + expect(calls).toEqual([ + "host-observation", + "gpu-observation", + "gateway-admission", + "host-admission", + "gpu-runtime-proof", + "host-observation", + "gateway-admission", + "host-admission", + "gpu-validation", + "bridge-dns", + ]); + }); + + it("passes the fresh host and GPU observations into resumed readiness (#7411)", async () => { const session = createSession(); session.steps.preflight.status = "complete"; - const assertCdiNvidiaGpuSpecPresent = vi.fn(); + const assertOnboardHostReadiness = vi.fn(); const harness = createDeps({ - assertCdiNvidiaGpuSpecPresent, + assertOnboardHostReadiness, resolveSandboxGpuConfig: vi.fn( ( _gpu: Gpu, @@ -218,13 +293,116 @@ describe("handlePreflightState", () => { explicitSandboxGpuFlag: "enable", }); - expect(assertCdiNvidiaGpuSpecPresent).toHaveBeenCalledWith( + expect(assertOnboardHostReadiness).toHaveBeenCalledWith( { cdiNvidiaGpuSpecMissing: false }, - true, - "jetson", + { type: "nvidia" }, + expect.objectContaining({ explicitlyOptedOutGpuPassthrough: false, resuming: true }), ); }); + it("preserves a failed resumed WSL GPU proof for canonical readiness (#7411)", async () => { + const session = createSession(); + session.steps.preflight.status = "complete"; + const assertOnboardHostReadiness = vi.fn(); + const harness = createDeps({ + detectGpuForReadiness: () => null, + detectGpu: () => null, + assertOnboardHostReadiness, + resolveSandboxGpuConfig, + }); + + await handlePreflightState({ + ...baseOptions(harness.deps, session), + resume: true, + explicitSandboxGpuFlag: "enable", + }); + + expect(assertOnboardHostReadiness).toHaveBeenLastCalledWith( + { cdiNvidiaGpuSpecMissing: false }, + null, + expect.objectContaining({ + explicitlyOptedOutGpuPassthrough: false, + wslDockerDesktopGpuProofPassed: false, + resuming: true, + }), + ); + }); + + it("reuses environment CPU-only intent when validating a cached preflight (#7411)", async () => { + const session = createSession(); + session.steps.preflight.status = "complete"; + const assertOnboardHostReadiness = vi.fn(); + const harness = createDeps({ + assertOnboardHostReadiness, + resolveSandboxGpuConfig, + }); + + await handlePreflightState({ + ...baseOptions(harness.deps, session), + resume: true, + env: { NEMOCLAW_SANDBOX_GPU: "0" }, + }); + + expect(assertOnboardHostReadiness).toHaveBeenCalledWith( + { cdiNvidiaGpuSpecMissing: false }, + { type: "nvidia" }, + expect.objectContaining({ explicitlyOptedOutGpuPassthrough: true, resuming: true }), + ); + }); + + it("does not treat an old auto-disabled session outcome as explicit CPU-only intent (#7411)", async () => { + const session = createSession(); + session.steps.preflight.status = "complete"; + session.gpuPassthrough = false; + const assertOnboardHostReadiness = vi.fn(); + const harness = createDeps({ + assertOnboardHostReadiness, + getResumeSandboxGpuOverrides: () => ({ flag: null, device: null }), + resolveSandboxGpuConfig, + }); + + await handlePreflightState({ + ...baseOptions(harness.deps, session), + resume: true, + env: {}, + }); + + expect(assertOnboardHostReadiness).toHaveBeenCalledWith( + { cdiNvidiaGpuSpecMissing: false }, + { type: "nvidia" }, + expect.objectContaining({ explicitlyOptedOutGpuPassthrough: false, resuming: true }), + ); + }); + + it("rejects a cached resume when host collection exceeds the freshness window (#7411)", async () => { + const session = createSession(); + session.steps.preflight.status = "complete"; + let currentTime = Date.parse("2026-08-07T12:00:00.000Z"); + const detectGpu = vi.fn(() => ({ type: "nvidia" }) as Gpu); + const bridge = vi.fn(); + const harness = createDeps({ + now: () => new Date(currentTime), + assessHost: () => { + currentTime += 30_001; + return { cdiNvidiaGpuSpecMissing: false }; + }, + assertOnboardHostReadiness: (_host, _gpu, options) => { + expect(options.observedAt).toBe("2026-08-07T12:00:00.000Z"); + const age = currentTime - Date.parse(options.observedAt as string); + expect(age).toBeGreaterThan(30_000); + throw new Error("host observations are stale"); + }, + detectGpu, + assertDockerBridgeAndContainerDnsHealthy: bridge, + }); + + await expect( + handlePreflightState({ ...baseOptions(harness.deps, session), resume: true }), + ).rejects.toThrow("host observations are stale"); + expect(detectGpu).not.toHaveBeenCalled(); + expect(bridge).not.toHaveBeenCalled(); + }); + it("restores saved sandbox GPU intent only when resume has no explicit override", async () => { const session = createSession(); session.steps.preflight.status = "complete"; diff --git a/src/lib/onboard/machine/handlers/preflight.ts b/src/lib/onboard/machine/handlers/preflight.ts index 10c7d0f2351..72c031c9c09 100644 --- a/src/lib/onboard/machine/handlers/preflight.ts +++ b/src/lib/onboard/machine/handlers/preflight.ts @@ -41,14 +41,26 @@ export interface PreflightStateOptions< sandbox: SandboxEntry | null, sessionGpuPassthrough: boolean | null | undefined, ): PreflightSandboxGpuOverrides; + /** Observe GPU state without running a container-backed proof. */ + detectGpuForReadiness(): Gpu; + /** Resolve any runtime-backed GPU proof after readiness admission. */ detectGpu(): Gpu; runPreflight(options: { optedOutGpuPassthrough?: boolean }): Promise; assessHost(): Host; - assertCdiNvidiaGpuSpecPresent( + assertOnboardHostReadiness( host: Host, - optedOutGpuPassthrough: boolean, - hostGpuPlatform?: string | null, + gpu: Gpu, + options: { + explicitlyOptedOutGpuPassthrough: boolean; + observedAt?: string; + now?: () => Date; + wslDockerDesktopGpuProofPassed?: boolean; + resuming: true; + }, ): void; + /** Revalidate canonical gateway ownership before resume probe effects. */ + assertGatewayReadiness(): Promise; + now?: () => Date; /** * Resume backstop for #3508/#3630. Runs the same bridge+DNS fatal * gate that `preflight()` does, so a cached preflight step cannot @@ -57,16 +69,13 @@ export interface PreflightStateOptions< * that haven't been updated yet. */ assertDockerBridgeAndContainerDnsHealthy?(host: Host): void; - /** - * Resume backstop for unsupported container runtimes (e.g. Podman - * with the Linux Docker-driver gateway). Must run before the bridge/ - * DNS backstop above so Podman hosts see the unsupported-runtime - * message instead of Docker-specific diagnostics. - */ - rejectUnsupportedContainerRuntime?(host: Host): void; resolveSandboxGpuConfig( gpu: Gpu, - options: { flag: PreflightSandboxGpuFlag; device: string | null | undefined }, + options: { + flag: PreflightSandboxGpuFlag; + device: string | null | undefined; + env?: NodeJS.ProcessEnv; + }, ): Config; validateSandboxGpuPreflight(config: Config): void; skippedStepMessage(stepName: string, detail?: string | null): void; @@ -97,6 +106,15 @@ function envHasSandboxGpuOverride(env: NodeJS.ProcessEnv): boolean { return env.NEMOCLAW_SANDBOX_GPU !== undefined || env.NEMOCLAW_SANDBOX_GPU_DEVICE !== undefined; } +function resolvedWslDockerDesktopGpuProof(gpu: unknown): boolean | undefined { + if (gpu === null) return false; + if (!gpu || typeof gpu !== "object") return undefined; + return (gpu as { wslDockerDesktopGpuProofPassed?: boolean }).wslDockerDesktopGpuProofPassed === + true + ? true + : undefined; +} + export async function handlePreflightState< Gpu, SandboxEntry, @@ -135,28 +153,54 @@ export async function handlePreflightState< let gpu: Gpu; if (resumePreflight) { deps.skippedStepMessage("preflight", "cached"); - await deps.recordStateSkipped("preflight", { reason: "resume", validation: "gpu-cdi" }); - gpu = deps.detectGpu(); - const resumeSandboxGpuConfig = deps.resolveSandboxGpuConfig(gpu, { + await deps.recordStateSkipped("preflight", { + reason: "resume", + validation: "host-readiness", + }); + const now = deps.now ?? (() => new Date()); + // Collect host facts, then require a live gateway result immediately + // before any runtime-backed probe. A successful gateway collection is + // younger than its reuse window, so the preceding host facts are current + // at the effect edge too. + let hostObservedAt = now().toISOString(); + let resumeHost = deps.assessHost(); + gpu = deps.detectGpuForReadiness(); + let resumeSandboxGpuConfig = deps.resolveSandboxGpuConfig(gpu, { flag: effectiveSandboxGpuFlag, device: effectiveSandboxGpuDevice, + env, + }); + await deps.assertGatewayReadiness(); + deps.assertOnboardHostReadiness(resumeHost, gpu, { + explicitlyOptedOutGpuPassthrough: resumeSandboxGpuConfig.mode === "0", + observedAt: hostObservedAt, + now, + resuming: true, }); + // A full detector can run the bounded ARM64 WSL Docker GPU proof. Keep it + // behind both live readiness gates, and skip it entirely for CPU-only + // intent. Replace gateway and host facts after that effect before any + // later runtime probe. + if (resumeSandboxGpuConfig.mode !== "0") { + gpu = deps.detectGpu(); + hostObservedAt = now().toISOString(); + resumeHost = deps.assessHost(); + resumeSandboxGpuConfig = deps.resolveSandboxGpuConfig(gpu, { + flag: effectiveSandboxGpuFlag, + device: effectiveSandboxGpuDevice, + env, + }); + await deps.assertGatewayReadiness(); + const wslDockerDesktopGpuProofPassed = resolvedWslDockerDesktopGpuProof(gpu); + deps.assertOnboardHostReadiness(resumeHost, gpu, { + explicitlyOptedOutGpuPassthrough: false, + observedAt: hostObservedAt, + now, + ...(wslDockerDesktopGpuProofPassed === undefined ? {} : { wslDockerDesktopGpuProofPassed }), + resuming: true, + }); + } deps.validateSandboxGpuPreflight(resumeSandboxGpuConfig); - const resumeOptedOutGpuPassthrough = - noGpu || - (!gpuRequested && session?.gpuPassthrough === false) || - !resumeSandboxGpuConfig.sandboxGpuEnabled; - const resumeHost = deps.assessHost(); - // Reject unsupported runtimes (Podman) BEFORE the CDI GPU-spec - // backstop and the Docker-specific bridge/DNS probes so Podman - // hosts always hit the unsupported-runtime message (#3630 - // CodeRabbit). - deps.rejectUnsupportedContainerRuntime?.(resumeHost); - deps.assertCdiNvidiaGpuSpecPresent( - resumeHost, - resumeOptedOutGpuPassthrough, - resumeSandboxGpuConfig.hostGpuPlatform, - ); // Resume backstop for #3508/#3630. Cached preflight does not capture // host Docker/DNS state, and a session written by an older NemoClaw // may have skipped the new bridge/DNS fatal checks. @@ -170,6 +214,7 @@ export async function handlePreflightState< const sandboxGpuConfig = deps.resolveSandboxGpuConfig(gpu, { flag: effectiveSandboxGpuFlag, device: effectiveSandboxGpuDevice, + env, }); const gpuPassthrough = sandboxGpuConfig.sandboxGpuEnabled; if (session && session.gpuPassthrough !== gpuPassthrough) { diff --git a/src/lib/onboard/machine/initial-flow-phases.test.ts b/src/lib/onboard/machine/initial-flow-phases.test.ts index 4ae83d80b05..2f8e55e3984 100644 --- a/src/lib/onboard/machine/initial-flow-phases.test.ts +++ b/src/lib/onboard/machine/initial-flow-phases.test.ts @@ -7,6 +7,7 @@ import { createSession, type Session } from "../../state/onboard-session"; import { resolveGatewayOwner } from "../gateway-ownership"; import { createInitialOnboardFlowPhases, + getInitialGatewayReuseStateForOwner, type InitialOnboardFlowContext, runInitialOnboardFlowSlice, } from "./initial-flow-phases"; @@ -100,9 +101,35 @@ function completeStep(): Session["steps"][string] { } describe("initial onboard flow phases", () => { + it("does not run managed gateway selection for an external owner (#7411)", () => { + const owner = resolveGatewayOwner({ + gatewayName: "nemoclaw", + gatewayPort: 31818, + declaration: { + version: 1, + mode: "externally-supervised", + endpoint: "http://127.0.0.1:31818", + stateDir: "/var/lib/openshell/gateway", + supervisor: { + kind: "systemd-system", + serviceName: "openshell-gateway.service", + execPath: "/usr/local/bin/openshell-gateway", + }, + requiredCapabilities: [], + }, + hasPackagedService: false, + }); + const getManagedReuseState = vi.fn(() => "healthy" as const); + + expect(getInitialGatewayReuseStateForOwner(owner, getManagedReuseState)).toBe("missing"); + expect(getManagedReuseState).not.toHaveBeenCalled(); + }); + it("carries preflight GPU output into the gateway phase", async () => { const notes: string[] = []; const gpu: Gpu = { type: "nvidia", platform: "linux" }; + let preflightFailure: Error | null = null; + const commitSelectedAgentTransition = vi.fn(async () => createSession()); const phases = createInitialOnboardFlowPhases({ explicitSandboxGpuFlag: null, sandboxGpuDevice: null, @@ -111,15 +138,16 @@ describe("initial onboard flow phases", () => { env: {}, platform: "darwin", recordedGpuPassthroughBeforePreflight: false, + commitSelectedAgentTransition, ensureResumePreflightDashboardPortAvailable: vi.fn(), preflightDeps: { getSandbox: () => null, getResumeSandboxGpuOverrides: () => ({ flag: null, device: null }), + detectGpuForReadiness: () => gpu, detectGpu: () => gpu, - runPreflight: async () => gpu, + runPreflight: async () => (preflightFailure ? Promise.reject(preflightFailure) : gpu), assessHost: () => ({}), - assertCdiNvidiaGpuSpecPresent: vi.fn(), - rejectUnsupportedContainerRuntime: vi.fn(), + assertOnboardHostReadiness: vi.fn(), assertDockerBridgeAndContainerDnsHealthy: vi.fn(), resolveSandboxGpuConfig: config, validateSandboxGpuPreflight: vi.fn(), @@ -133,6 +161,7 @@ describe("initial onboard flow phases", () => { }, }, getInitialGatewayReuseState: () => "healthy", + assertGatewayReadiness: vi.fn(async () => undefined), gatewayName: "nemoclaw", recreateSandbox: () => false, gatewayDeps: { @@ -209,6 +238,12 @@ describe("initial onboard flow phases", () => { expect(notes).toContain( " GPU passthrough requested; passing --gpu to OpenShell gateway and sandbox creation.", ); + expect(commitSelectedAgentTransition).toHaveBeenCalledOnce(); + + commitSelectedAgentTransition.mockClear(); + preflightFailure = new Error("readiness blocked"); + await expect(phases[0].run(context())).rejects.toThrow("readiness blocked"); + expect(commitSelectedAgentTransition).not.toHaveBeenCalled(); }); it("repairs preflight before strict gateway entry", async () => { @@ -336,6 +371,10 @@ describe("initial onboard flow phases", () => { env: {}, platform: "darwin", recordedGpuPassthroughBeforePreflight: true, + commitSelectedAgentTransition: async () => { + calls.push("commit-agent-transition"); + return session; + }, ensureResumePreflightDashboardPortAvailable, preflightDeps: { getSandbox: vi.fn(() => { @@ -346,6 +385,10 @@ describe("initial onboard flow phases", () => { calls.push("resume-gpu-overrides"); return { flag: "enable" as const, device: null }; }), + detectGpuForReadiness: vi.fn(() => { + calls.push("detect-gpu-readiness"); + return gpu; + }), detectGpu: vi.fn(() => { calls.push("detect-gpu"); return gpu; @@ -357,11 +400,8 @@ describe("initial onboard flow phases", () => { calls.push("assess-host"); return { docker: true }; }), - assertCdiNvidiaGpuSpecPresent: vi.fn(() => { - calls.push("assert-cdi"); - }), - rejectUnsupportedContainerRuntime: vi.fn(() => { - calls.push("reject-unsupported-runtime"); + assertOnboardHostReadiness: vi.fn(() => { + calls.push("assert-host-readiness"); }), assertDockerBridgeAndContainerDnsHealthy: vi.fn(() => { calls.push("assert-bridge-dns"); @@ -390,6 +430,9 @@ describe("initial onboard flow phases", () => { calls.push("initial-gateway-reuse-state"); return "healthy"; }, + assertGatewayReadiness: vi.fn(async () => { + calls.push("assert-gateway-readiness"); + }), gatewayName: "nemoclaw", recreateSandbox: () => false, gatewayDeps: { @@ -484,15 +527,22 @@ describe("initial onboard flow phases", () => { "resume-gpu-overrides", "skip-preflight", "record-preflight-skipped", + "assess-host", + "detect-gpu-readiness", + "resolve-gpu-config", + "assert-gateway-readiness", + "assert-host-readiness", "detect-gpu", + "assess-host", "resolve-gpu-config", + "assert-gateway-readiness", + "assert-host-readiness", "validate-gpu-preflight", - "assess-host", - "reject-unsupported-runtime", - "assert-cdi", "assert-bridge-dns", "resolve-gpu-config", "ensure-resume-preflight-port", + "commit-agent-transition", + "assert-gateway-readiness", "initial-gateway-reuse-state", "refresh-gateway-reuse", "gateway-lifecycle-support", diff --git a/src/lib/onboard/machine/initial-flow-phases.ts b/src/lib/onboard/machine/initial-flow-phases.ts index 498ac29e84b..6c89314a043 100644 --- a/src/lib/onboard/machine/initial-flow-phases.ts +++ b/src/lib/onboard/machine/initial-flow-phases.ts @@ -3,8 +3,10 @@ import { spawnSync } from "node:child_process"; import type { GatewayReuseState } from "../../state/gateway"; +import { type GatewayOwner, isExternallySupervised } from "../gateway-ownership"; import { formatSandboxGpuPassthroughNote } from "../sandbox-gpu-notes"; import type { OnboardFlowContext } from "./flow-context"; +import { UnexpectedOnboardFlowSliceStateError } from "./flow-slice-error"; import { runInitialOnboardFlowSequence } from "./flow-slices"; import { type GatewayStateOptions, handleGatewayState } from "./handlers/gateway"; import { @@ -13,7 +15,6 @@ import { type PreflightSandboxGpuFlag, type PreflightStateOptions, } from "./handlers/preflight"; -import { UnexpectedOnboardFlowSliceStateError } from "./flow-slice-error"; import { type OnboardPrerequisiteRepairEventRecorder, runOnboardPrerequisiteRepair, @@ -33,6 +34,13 @@ export type InitialOnboardFlowContext< type SpawnSync = typeof spawnSync; +export function getInitialGatewayReuseStateForOwner( + owner: GatewayOwner, + getManagedReuseState: () => GatewayReuseState, +): GatewayReuseState { + return isExternallySupervised(owner) ? "missing" : getManagedReuseState(); +} + export interface InitialOnboardFlowPhaseOptions< Context extends InitialOnboardFlowContext, Agent, @@ -48,9 +56,15 @@ export interface InitialOnboardFlowPhaseOptions< env: NodeJS.ProcessEnv; platform?: NodeJS.Platform; recordedGpuPassthroughBeforePreflight: boolean; + /** Commit any provider lifecycle transition only after preflight admission. */ + commitSelectedAgentTransition?(): Promise; ensureResumePreflightDashboardPortAvailable(): void; - preflightDeps: PreflightStateOptions["deps"]; + preflightDeps: Omit< + PreflightStateOptions["deps"], + "assertGatewayReadiness" + >; getInitialGatewayReuseState(): GatewayReuseState; + assertGatewayReadiness(): Promise; gatewayName: string; recreateSandbox(): boolean; gatewayDeps: GatewayStateOptions["deps"]; @@ -130,9 +144,15 @@ export function createInitialOnboardFlowPhases< gpuRequested: options.gpuRequested, noGpu: options.noGpu, env: options.env, - deps: options.preflightDeps, + deps: { + ...options.preflightDeps, + assertGatewayReadiness: options.assertGatewayReadiness, + }, }); if (context.resume) options.ensureResumePreflightDashboardPortAvailable(); + const transitionedSession = options.commitSelectedAgentTransition + ? await options.commitSelectedAgentTransition() + : preflightResult.session; const preflightGpu = preflightResult.gpu ?? null; emitPreflightGpuNote({ @@ -150,7 +170,7 @@ export function createInitialOnboardFlowPhases< return { context: { ...context, - session: preflightResult.session, + session: transitionedSession, gpu: preflightGpu, sandboxGpuConfig: preflightResult.sandboxGpuConfig, gpuPassthrough: preflightResult.gpuPassthrough, @@ -165,10 +185,18 @@ export function createInitialOnboardFlowPhases< const gatewayPhase: OnboardSequencePhase = { state: "gateway", async run(context) { + // Resolve authority before the managed-only reuse helper can select a + // gateway or mutate OPENSHELL_GATEWAY. External attachment revalidates + // the same owner again at the effect edge. + const owner = options.gatewayDeps.resolveGatewayOwner(); + await options.assertGatewayReadiness(); const gatewayResult = await handleGatewayState({ resume: context.resume, session: context.session, - initialGatewayReuseState: options.getInitialGatewayReuseState(), + initialGatewayReuseState: getInitialGatewayReuseStateForOwner( + owner, + options.getInitialGatewayReuseState, + ), gpu: context.gpu as Gpu, gpuPassthrough: context.gpuPassthrough, gatewayName: options.gatewayName, diff --git a/src/lib/onboard/machine/preflight-gateway-authority.test.ts b/src/lib/onboard/machine/preflight-gateway-authority.test.ts new file mode 100644 index 00000000000..818a4bdf7dd --- /dev/null +++ b/src/lib/onboard/machine/preflight-gateway-authority.test.ts @@ -0,0 +1,63 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { GatewayReadinessProjection } from "../../readiness/gateway"; +import { preparePreflightGatewayAuthority } from "./preflight-gateway-authority"; + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe("preflight gateway authority", () => { + it("rejects a blocked refreshed projection before reuse or lifecycle handling (#7411)", async () => { + const gatewayReadiness: GatewayReadinessProjection = { + observations: [], + capabilities: [], + findings: [ + { + id: "gateway.authority.conflict", + severity: "blocking", + summary: "Gateway authority conflicts with the selected endpoint.", + }, + ], + evidence: [], + }; + const getGatewayReuseSnapshot = vi.fn(() => ({ + gatewayStatus: "", + gwInfo: "", + activeGatewayInfo: "", + gatewayReuseState: "healthy" as const, + })); + const selectNamedGatewayForReuseIfNeeded = vi.fn(); + const refreshDockerDriverGatewayReuseState = vi.fn(); + const checkPortAvailable = vi.fn(); + vi.spyOn(console, "error").mockImplementation(() => undefined); + vi.spyOn(process, "exit").mockImplementation(((code: number) => { + throw new Error(`exit ${code}`); + }) as never); + + await expect( + preparePreflightGatewayAuthority({ + collectGatewayReadiness: async () => gatewayReadiness, + ensureOpenshell: vi.fn(), + persistTrustedGatewayOwner: vi.fn(), + gatewayPort: 8080, + portConflict: { + checkPortAvailable, + getGatewayPortCheckOptions: () => ({}), + isDockerDriverGatewayPortListener: vi.fn(), + exitProcess: process.exit, + }, + getGatewayReuseSnapshot, + selectNamedGatewayForReuseIfNeeded, + refreshDockerDriverGatewayReuseState, + }), + ).rejects.toThrow("exit 1"); + + expect(checkPortAvailable).not.toHaveBeenCalled(); + expect(getGatewayReuseSnapshot).not.toHaveBeenCalled(); + expect(selectNamedGatewayForReuseIfNeeded).not.toHaveBeenCalled(); + expect(refreshDockerDriverGatewayReuseState).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/onboard/machine/preflight-gateway-authority.ts b/src/lib/onboard/machine/preflight-gateway-authority.ts new file mode 100644 index 00000000000..72ab05e463d --- /dev/null +++ b/src/lib/onboard/machine/preflight-gateway-authority.ts @@ -0,0 +1,75 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { GatewayReadinessProjection } from "../../readiness/gateway"; +import type { GatewayReuseState } from "../../state/gateway"; +import * as fatalRuntimePreflight from "../fatal-runtime-preflight"; +import type { GatewayOwner } from "../gateway-ownership"; +import { + failFastOnForeignGatewayPortConflict, + type GatewayPortConflictDeps, +} from "../gateway-port-conflict"; +import type { GatewayReuseSnapshot } from "../gateway-reuse"; + +export interface OnboardGatewayReadinessCollectorDeps { + gatewayName(): string; + gatewayPort(): number; + resolveOwner(): GatewayOwner; + probeAttachment: Parameters< + typeof fatalRuntimePreflight.collectOnboardGatewayReadiness + >[0]["probeAttachment"]; +} + +export interface PreparePreflightGatewayAuthorityDeps { + collectGatewayReadiness(): Promise; + ensureOpenshell(persistTrustedGatewayOwner: (owner: GatewayOwner) => void): void; + persistTrustedGatewayOwner(owner: GatewayOwner): void; + gatewayPort: number; + portConflict: Omit; + getGatewayReuseSnapshot(): GatewayReuseSnapshot; + selectNamedGatewayForReuseIfNeeded(snapshot: GatewayReuseSnapshot): GatewayReuseSnapshot; + refreshDockerDriverGatewayReuseState(state: GatewayReuseState): Promise; +} + +export interface PreflightGatewayAuthority { + externallySupervised: boolean; + gatewayReuseState: GatewayReuseState; +} + +export function collectOnboardGatewayReadiness( + deps: OnboardGatewayReadinessCollectorDeps, +): Promise { + return fatalRuntimePreflight.collectOnboardGatewayReadiness(deps); +} + +function isManagedGateway(readiness: GatewayReadinessProjection): boolean { + return readiness.observations.some( + ({ id, state, value }) => + id === "gateway.management.mode" && state === "present" && value === "nemoclaw-managed", + ); +} + +export async function preparePreflightGatewayAuthority( + deps: PreparePreflightGatewayAuthorityDeps, +): Promise { + deps.ensureOpenshell(deps.persistTrustedGatewayOwner); + + // Installation and portable preparation can replace binaries, services, or + // runtime endpoints. Refresh authority before any lifecycle effect consumes it. + const gatewayReadiness = await deps.collectGatewayReadiness(); + fatalRuntimePreflight.assertOnboardGatewayReadiness(gatewayReadiness); + const externallySupervised = !isManagedGateway(gatewayReadiness); + await failFastOnForeignGatewayPortConflict({ + gatewayPort: deps.gatewayPort, + externallySupervised, + ...deps.portConflict, + }); + + const observedSnapshot = deps.getGatewayReuseSnapshot(); + const gatewayReuseState = externallySupervised + ? observedSnapshot.gatewayReuseState + : await deps.refreshDockerDriverGatewayReuseState( + deps.selectNamedGatewayForReuseIfNeeded(observedSnapshot).gatewayReuseState, + ); + return { externallySupervised, gatewayReuseState }; +} diff --git a/src/lib/onboard/openshell-cli.ts b/src/lib/onboard/openshell-cli.ts index e4c85024119..69bf9ef0fca 100644 --- a/src/lib/onboard/openshell-cli.ts +++ b/src/lib/onboard/openshell-cli.ts @@ -9,6 +9,11 @@ import { captureSandboxRecreateOpenshellCommand, } from "./sandbox-recreate-probe"; +// Keep OpenShell binary resolution behind the established onboarding adapter. +// Readiness and runtime callers reuse this export instead of creating parallel +// dependencies on the low-level resolver. +export { resolveOpenshell }; + export interface OpenshellCliDeps { getCachedBinary(): string | null; setCachedBinary(binary: string): void; diff --git a/src/lib/onboard/openshell-install.test.ts b/src/lib/onboard/openshell-install.test.ts index 47a567515db..c8582d3a8f1 100644 --- a/src/lib/onboard/openshell-install.test.ts +++ b/src/lib/onboard/openshell-install.test.ts @@ -48,6 +48,30 @@ function makeDeps(overrides: Partial = {}) { } describe("ensureOpenshellForOnboard", () => { + it("runs trusted post-install reconciliation only after a successful install", () => { + const afterSuccessfulInstall = vi.fn(); + const deps = makeDeps({ + isOpenshellInstalled: () => false, + installOpenshell: () => ({ + installed: true, + localBin: "/tmp/openshell", + futureShellPathHint: null, + }), + }); + + ensureOpenshellForOnboard(deps, { afterSuccessfulInstall }); + + expect(afterSuccessfulInstall).toHaveBeenCalledOnce(); + }); + + it("does not run post-install reconciliation when no install was needed", () => { + const afterSuccessfulInstall = vi.fn(); + + ensureOpenshellForOnboard(makeDeps(), { afterSuccessfulInstall }); + + expect(afterSuccessfulInstall).not.toHaveBeenCalled(); + }); + it("reinstalls when the installed OpenShell lacks messaging rewrite or MCP L7 support", () => { const hasFeatures = vi.fn().mockReturnValueOnce(false).mockReturnValue(true); const deps = makeDeps({ diff --git a/src/lib/onboard/openshell-install.ts b/src/lib/onboard/openshell-install.ts index da4ea9ca8e2..3a683e10479 100644 --- a/src/lib/onboard/openshell-install.ts +++ b/src/lib/onboard/openshell-install.ts @@ -150,7 +150,10 @@ export function areRequiredDockerDriverBinariesPresent( return true; } -export function ensureOpenshellForOnboard(deps: OpenShellInstallDeps): OpenShellInstallResult { +export function ensureOpenshellForOnboard( + deps: OpenShellInstallDeps, + options: { afterSuccessfulInstall?(): void } = {}, +): OpenShellInstallResult { const platform = deps.platform ?? process.platform; const arch = deps.arch ?? process.arch; let openshellInstall: OpenShellInstallResult = { @@ -286,5 +289,6 @@ export function ensureOpenshellForOnboard(deps: OpenShellInstallDeps): OpenShell " Add that export to your shell profile, or open a new terminal before running openshell directly.", ); } + if (openshellInstall.installed === true) options.afterSuccessfulInstall?.(); return openshellInstall; } diff --git a/src/lib/onboard/preflight-cdi.test.ts b/src/lib/onboard/preflight-cdi.test.ts index 25d89e2aada..c72693b7407 100644 --- a/src/lib/onboard/preflight-cdi.test.ts +++ b/src/lib/onboard/preflight-cdi.test.ts @@ -3,14 +3,13 @@ import { describe, expect, it, vi } from "vitest"; // Import source directly so tests cannot pass against a stale build. -import { - assertCdiNvidiaGpuSpecPresent, - assessHost, - planHostRemediation, - shouldEnforceCdiNvidiaGpuSpec, -} from "./preflight"; +import { assessHost, planHostAdvisories } from "./preflight"; -type HostAssessment = Parameters[0]; +type HostAssessment = Parameters[0]; + +function advisoryCommands(advisory: { commands?: readonly string[] } | undefined) { + return advisory?.commands ?? []; +} function baseAssessment(overrides: Partial = {}): HostAssessment { return { @@ -42,21 +41,6 @@ function baseAssessment(overrides: Partial = {}): HostAssessment }; } -function withStderrColorDepth(colorDepth: number, noColor: string, callback: () => T): T { - const stderr = Object.assign(Object.create(process.stderr), { - getColorDepth: () => colorDepth, - isTTY: true, - }) as typeof process.stderr; - const getStderr = vi.spyOn(process, "stderr", "get").mockReturnValue(stderr); - vi.stubEnv("NO_COLOR", noColor); - try { - return callback(); - } finally { - getStderr.mockRestore(); - vi.unstubAllEnvs(); - } -} - function runCaptureWithLspci(lspciOutput: string): (command: readonly string[]) => string { const resultByCmd: Record = { "nvidia-smi": "", lspci: lspciOutput }; return (command) => { @@ -253,7 +237,7 @@ describe("assessHost — CDI", () => { expect(result.nvidiaCdiRefreshServiceEnabled).toBe(false); }); - it("does not apply CDI checks without an NVIDIA Linux CDI context", () => { + it("uses default CDI directories only when the NVIDIA Linux check applies (#7411)", () => { const linuxWithoutGpu = assessHost({ platform: "linux", env: {}, @@ -278,25 +262,29 @@ describe("assessHost — CDI", () => { }); expect(linuxWithoutGpu.cdiNvidiaGpuSpecMissing).toBe(false); - expect(noCdiDirs.dockerCdiSpecDirs).toEqual([]); - expect(noCdiDirs.cdiNvidiaGpuSpecMissing).toBe(false); + expect(noCdiDirs.dockerCdiSpecDirs).toEqual(["/etc/cdi", "/var/run/cdi"]); + expect(noCdiDirs.cdiNvidiaGpuSpecMissing).toBe(true); }); }); -describe("planHostRemediation — CDI", () => { +describe("planHostAdvisories — CDI", () => { it("emits a blocking generate action for missing nvidia.com/gpu specs", () => { - const actions = planHostRemediation(baseAssessment({ cdiNvidiaGpuSpecMissing: true })); + const actions = planHostAdvisories(baseAssessment({ cdiNvidiaGpuSpecMissing: true })); const action = actions.find((entry: { id: string }) => entry.id === "generate_nvidia_cdi_spec"); expect(action).toBeTruthy(); expect(action?.kind).toBe("sudo"); - expect(action?.blocking).toBe(true); - expect(action?.commands.some((command) => command.includes("--output='/etc/cdi"))).toBe(true); - expect(action?.commands.some((command) => command.includes("nvidia-ctk cdi list"))).toBe(true); + expect(action?.severity).toBe("blocking"); + expect(advisoryCommands(action).some((command) => command.includes("--output='/etc/cdi"))).toBe( + true, + ); + expect( + advisoryCommands(action).some((command) => command.includes("nvidia-ctk cdi list")), + ).toBe(true); }); it("emits service-refresh commands for stale nvidia.com/gpu specs", () => { - const actions = planHostRemediation( + const actions = planHostAdvisories( baseAssessment({ cdiNvidiaGpuSpecStale: true, cdiNvidiaGpuSpecNeedsRepair: true, @@ -306,20 +294,26 @@ describe("planHostRemediation — CDI", () => { const action = actions.find((entry: { id: string }) => entry.id === "refresh_nvidia_cdi_spec"); expect(action).toBeTruthy(); - expect(action?.blocking).toBe(true); - expect(action?.commands[0]).toBe( + expect(action?.severity).toBe("blocking"); + expect(advisoryCommands(action)[0]).toBe( "sudo systemctl enable --now nvidia-cdi-refresh.path nvidia-cdi-refresh.service", ); - expect(action?.commands[1]).toBe("sudo systemctl start nvidia-cdi-refresh.service"); + expect(advisoryCommands(action)[1]).toBe("sudo systemctl start nvidia-cdi-refresh.service"); expect( - action?.commands.some((command) => command.includes("sudo rm -f '/etc/cdi/nvidia.yaml'")), + advisoryCommands(action).some((command) => + command.includes("sudo rm -f '/etc/cdi/nvidia.yaml'"), + ), ).toBe(true); - expect(action?.commands.some((command) => command.includes("--output=/etc/cdi"))).toBe(false); - expect(action?.commands.some((command) => command.includes("nvidia-ctk cdi list"))).toBe(false); + expect(advisoryCommands(action).some((command) => command.includes("--output=/etc/cdi"))).toBe( + false, + ); + expect( + advisoryCommands(action).some((command) => command.includes("nvidia-ctk cdi list")), + ).toBe(false); }); it("emits manual stale-spec guidance without systemctl on non-systemd hosts", () => { - const actions = planHostRemediation( + const actions = planHostAdvisories( baseAssessment({ systemctlAvailable: false, cdiNvidiaGpuSpecStale: true, @@ -330,14 +324,14 @@ describe("planHostRemediation — CDI", () => { const action = actions.find((entry: { id: string }) => entry.id === "refresh_nvidia_cdi_spec"); expect(action).toBeTruthy(); - expect(action?.blocking).toBe(true); + expect(action?.severity).toBe("blocking"); expect(action?.kind).toBe("manual"); - expect(action?.commands.join("\n")).toContain("/var/run/cdi/nvidia.yaml"); - expect(action?.commands.join("\n")).not.toContain("systemctl"); + expect(advisoryCommands(action).join("\n")).toContain("/var/run/cdi/nvidia.yaml"); + expect(advisoryCommands(action).join("\n")).not.toContain("systemctl"); }); it("emits a non-blocking refresh-service warning when refresh units are unhealthy", () => { - const actions = planHostRemediation( + const actions = planHostAdvisories( baseAssessment({ dockerCdiSpecDirs: ["/etc/cdi"], cdiNvidiaGpuRefreshUnhealthy: true, @@ -351,7 +345,7 @@ describe("planHostRemediation — CDI", () => { ); expect(action).toBeTruthy(); - expect(action?.blocking).toBe(false); + expect(action?.severity).toBe("warning"); expect(action?.title).toBe("Enable NVIDIA CDI refresh service"); expect(action?.reason).toContain("path disabled"); }); @@ -383,17 +377,17 @@ describe("planHostRemediation — CDI", () => { expect(result.hasNvidiaGpu).toBe(true); expect(result.cdiNvidiaGpuSpecMissing).toBe(true); - const actions = planHostRemediation(result); + const actions = planHostAdvisories(result); const action = actions.find((entry) => entry.id === "install_nvidia_container_toolkit"); expect(action).toBeTruthy(); - expect(action?.blocking).toBe(true); + expect(action?.severity).toBe("blocking"); expect( - action?.commands.some( + advisoryCommands(action).some( (command) => command === "sudo apt-get install -y nvidia-container-toolkit", ), ).toBe(true); expect( - action?.commands.some((command) => + advisoryCommands(action).some((command) => command.startsWith("sudo nvidia-ctk cdi generate --output="), ), ).toBe(true); @@ -426,14 +420,14 @@ describe("planHostRemediation — CDI", () => { expect(result.hasNvidiaGpu).toBe(false); - const action = planHostRemediation(result).find( + const action = planHostAdvisories(result).find( (entry) => entry.id === "install_nvidia_container_toolkit", ); expect(action).toBeFalsy(); }); it("bootstraps nvidia-container-toolkit before missing-spec generation", () => { - const actions = planHostRemediation( + const actions = planHostAdvisories( baseAssessment({ cdiNvidiaGpuSpecMissing: true, nvidiaContainerToolkitInstalled: false, @@ -443,105 +437,14 @@ describe("planHostRemediation — CDI", () => { expect(action).toBeTruthy(); expect( - action?.commands.some( + advisoryCommands(action).some( (command) => command === "sudo apt-get install -y nvidia-container-toolkit", ), ).toBe(true); expect( - action?.commands.some((command) => + advisoryCommands(action).some((command) => command.startsWith("sudo nvidia-ctk cdi generate --output="), ), ).toBe(true); }); }); - -describe("shouldEnforceCdiNvidiaGpuSpec enforcement gate (#5489)", () => { - it("enforces when the spec is missing and the operator did not explicitly opt out", () => { - // The #5489 scenario: GPU hardware present (so cdiNvidiaGpuSpecMissing is - // true) with sandbox GPU AUTO-disabled (nvidia-smi unavailable). Auto-disable - // must NOT be treated as an opt-out, so the gate enforces. - expect( - shouldEnforceCdiNvidiaGpuSpec({ - cdiNvidiaGpuSpecMissing: true, - cdiNvidiaGpuSpecNeedsRepair: false, - explicitlyOptedOutGpuPassthrough: false, - }), - ).toBe(true); - }); - - it("enforces when the spec needs repair (stale) and not explicitly opted out", () => { - expect( - shouldEnforceCdiNvidiaGpuSpec({ - cdiNvidiaGpuSpecMissing: false, - cdiNvidiaGpuSpecNeedsRepair: true, - explicitlyOptedOutGpuPassthrough: false, - }), - ).toBe(true); - }); - - it("does NOT enforce when the operator explicitly opted out of GPU passthrough (--no-gpu)", () => { - // Escape hatch: a host with an unusable GPU can still onboard CPU-only. - expect( - shouldEnforceCdiNvidiaGpuSpec({ - cdiNvidiaGpuSpecMissing: true, - cdiNvidiaGpuSpecNeedsRepair: true, - explicitlyOptedOutGpuPassthrough: true, - }), - ).toBe(false); - }); - - it("does NOT enforce when the CDI spec is present and healthy", () => { - expect( - shouldEnforceCdiNvidiaGpuSpec({ - cdiNvidiaGpuSpecMissing: false, - cdiNvidiaGpuSpecNeedsRepair: false, - explicitlyOptedOutGpuPassthrough: false, - }), - ).toBe(false); - }); -}); - -describe("assertCdiNvidiaGpuSpecPresent severity (#6004)", () => { - it("colors the fatal missing-CDI line red before exiting", () => { - withStderrColorDepth(24, "", () => { - const error = vi.spyOn(console, "error").mockImplementation(() => undefined); - const exitProcess = vi.fn((code: number): never => { - throw new Error(`exit ${code}`); - }); - - expect(() => - assertCdiNvidiaGpuSpecPresent( - baseAssessment({ cdiNvidiaGpuSpecMissing: true }), - false, - null, - exitProcess, - ), - ).toThrow("exit 1"); - expect(error.mock.calls[0]?.[0]).toBe( - " \x1b[31m✗ Docker is configured for CDI device injection (CDISpecDirs is set), but the NVIDIA GPU CDI spec is missing or stale. OpenShell GPU startup can fail until the CDI spec is refreshed.\x1b[39m", - ); - expect(exitProcess).toHaveBeenCalledWith(1); - error.mockRestore(); - }); - }); - - it("keeps the fatal missing-CDI line plain under NO_COLOR", () => { - withStderrColorDepth(24, "1", () => { - const error = vi.spyOn(console, "error").mockImplementation(() => undefined); - - expect(() => - assertCdiNvidiaGpuSpecPresent( - baseAssessment({ cdiNvidiaGpuSpecNeedsRepair: true }), - false, - null, - (code): never => { - throw new Error(`exit ${code}`); - }, - ), - ).toThrow("exit 1"); - expect(String(error.mock.calls[0]?.[0])).toContain(" ✗ Docker is configured for CDI"); - expect(String(error.mock.calls[0]?.[0])).not.toContain("\x1b["); - error.mockRestore(); - }); - }); -}); diff --git a/src/lib/onboard/preflight-docker-host.test.ts b/src/lib/onboard/preflight-docker-host.test.ts index 05a828e892a..d97ef017548 100644 --- a/src/lib/onboard/preflight-docker-host.test.ts +++ b/src/lib/onboard/preflight-docker-host.test.ts @@ -3,7 +3,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import { assessHost, planHostRemediation } from "./preflight"; +import { assessHost, planHostAdvisories } from "./preflight"; import { printRemediationActions } from "./remediation"; // Regression: NemoClaw #7731. This invalid TCP endpoint makes `docker info` @@ -27,7 +27,7 @@ describe("assessHost invalid DOCKER_HOST (#7731)", () => { }); const err = vi.spyOn(console, "error").mockImplementation(() => undefined); - printRemediationActions(planHostRemediation(assessment)); + printRemediationActions(planHostAdvisories(assessment)); const output = err.mock.calls.map((call: unknown[]) => String(call[0])).join("\n"); expect(output).toContain("Fix the DOCKER_HOST endpoint (invalid_docker_host):"); @@ -52,7 +52,7 @@ describe("assessHost invalid DOCKER_HOST (#7731)", () => { expect(assessment.dockerReachable).toBe(false); expect(assessment.dockerServiceActive).toBe(true); - const ids = planHostRemediation(assessment).map((action) => action.id); + const ids = planHostAdvisories(assessment).map((action) => action.id); expect(ids).toContain("invalid_docker_host"); expect(ids).not.toContain("docker_group_permission"); }); diff --git a/src/lib/onboard/preflight-messages.test.ts b/src/lib/onboard/preflight-messages.test.ts index 0bfd678e3a5..62b7cecb53e 100644 --- a/src/lib/onboard/preflight-messages.test.ts +++ b/src/lib/onboard/preflight-messages.test.ts @@ -4,6 +4,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { + printCdiSpecUnavailableError, printDockerNotReachableError, printLowMemoryWarning, printMessagingProviderMissing, @@ -77,6 +78,15 @@ describe("onboard preflight severity messages (#6004)", () => { expect(lines(err).join("\n")).toContain("Colima"); }); + it("preserves the CDI failure message through the shared presenter (#7411)", () => { + const err = vi.spyOn(console, "error").mockImplementation(() => undefined); + + printCdiSpecUnavailableError(); + + expect(lines(err)[0]).toContain("✗ Docker is configured for CDI device injection"); + expect(lines(err)[0]).toContain("NVIDIA GPU CDI spec is missing or stale"); + }); + it("prints the under-provisioned warning to stderr with a ⚠ marker and colima resize", () => { const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); printUnderProvisionedRuntimeWarning({ diff --git a/src/lib/onboard/preflight-messages.ts b/src/lib/onboard/preflight-messages.ts index 6f2dc8af087..91f1e0aee37 100644 --- a/src/lib/onboard/preflight-messages.ts +++ b/src/lib/onboard/preflight-messages.ts @@ -27,6 +27,15 @@ export function printUnsupportedRuntimeError(): void { console.error(" Switch to Docker Engine, Docker Desktop, or Colima, then rerun onboarding."); } +/** NVIDIA CDI state cannot support GPU passthrough for this onboarding run. */ +export function printCdiSpecUnavailableError(): void { + console.error( + failLine( + "Docker is configured for CDI device injection (CDISpecDirs is set), but the NVIDIA GPU CDI spec is missing or stale. OpenShell GPU startup can fail until the CDI spec is refreshed.", + ), + ); +} + export interface UnderProvisionedRuntimeWarning { /** Human-readable detected resources, e.g. "2 vCPU / 2.0 GiB". */ detectedStr: string; diff --git a/src/lib/onboard/preflight-readiness.test.ts b/src/lib/onboard/preflight-readiness.test.ts new file mode 100644 index 00000000000..466f2409e07 --- /dev/null +++ b/src/lib/onboard/preflight-readiness.test.ts @@ -0,0 +1,44 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; +import { assessHost, parseDockerNvidiaRuntimeAvailable, planHostAdvisories } from "./preflight"; + +describe("readiness-facing host preflight", () => { + it("accepts only an absolute OpenShell command path from an injected host transport (#7411)", () => { + const assessWithCommandVResult = (commandVResult: string) => + assessHost({ + platform: "darwin", + env: {}, + dockerInfoOutput: "", + gpuProbeImpl: () => false, + runCaptureImpl: (command: readonly string[]) => + command.at(-1) === "openshell" ? commandVResult : "", + }); + + expect(assessWithCommandVResult("/usr/local/bin/openshell").openshellInstalled).toBe(true); + expect(assessWithCommandVResult("openshell").openshellInstalled).toBe(false); + expect(assessWithCommandVResult("alias openshell='echo unexpected'").openshellInstalled).toBe( + false, + ); + }); + + it("distinguishes configured, absent, and unreported NVIDIA runtimes (#7411)", () => { + expect(parseDockerNvidiaRuntimeAvailable('{"Runtimes":{"runc":{},"nvidia":{}}}')).toBe(true); + expect(parseDockerNvidiaRuntimeAvailable('{"Runtimes":{"runc":{}}}')).toBe(false); + expect(parseDockerNvidiaRuntimeAvailable('{"ServerVersion":"27.0"}')).toBeUndefined(); + }); + + it("does not duplicate the canonical unsupported-runtime finding as an advisory (#7411)", () => { + const host = assessHost({ + platform: "darwin", + env: {}, + dockerInfoOutput: '{"ServerVersion":"5.0.0","Name":"Podman Engine"}', + gpuProbeImpl: () => false, + runCaptureImpl: () => "", + }); + + expect(host.isUnsupportedRuntime).toBe(true); + expect(planHostAdvisories(host).some(({ id }) => id.includes("unsupported"))).toBe(false); + }); +}); diff --git a/src/lib/onboard/preflight.test.ts b/src/lib/onboard/preflight.test.ts index cda831ad8cf..c59df0e9b93 100644 --- a/src/lib/onboard/preflight.test.ts +++ b/src/lib/onboard/preflight.test.ts @@ -19,7 +19,7 @@ import { parseDockerInfoMemTotalBytes, parseDockerStorageDriver, parseDockerUsesContainerdSnapshotter, - planHostRemediation, + planHostAdvisories, probeContainerDns, probeDockerBridgeContainerStart, } from "./preflight"; @@ -32,6 +32,10 @@ function requireMemoryInfo(result: ReturnType) { return result; } +function advisoryCommands(advisory: { commands?: readonly string[] } | undefined) { + return advisory?.commands ?? []; +} + describe("checkPortAvailable", () => { it("falls through to the probe when lsof output is empty", async () => { let probedPort: number | null = null; @@ -636,10 +640,10 @@ describe("parseDockerUsesContainerdSnapshotter", () => { }); }); -describe("planHostRemediation", () => { +describe("planHostAdvisories", () => { function baseAssessment( - overrides: Partial[0]> = {}, - ): Parameters[0] { + overrides: Partial[0]> = {}, + ): Parameters[0] { return { platform: "linux", isWsl: false, @@ -670,7 +674,7 @@ describe("planHostRemediation", () => { } it("recommends starting docker when installed but unreachable and service inactive", () => { - const actions = planHostRemediation({ + const actions = planHostAdvisories({ platform: "linux", isWsl: false, runtime: "unknown", @@ -698,12 +702,12 @@ describe("planHostRemediation", () => { }); expect(actions[0].id).toBe("start_docker"); - expect(actions[0].blocking).toBe(true); - expect(actions[0].commands).toContain("sudo systemctl start docker"); + expect(actions[0].severity).toBe("blocking"); + expect(advisoryCommands(actions[0])).toContain("sudo systemctl start docker"); }); it("recommends Docker Desktop WSL integration when docker is missing inside WSL", () => { - const actions = planHostRemediation( + const actions = planHostAdvisories( baseAssessment({ isWsl: true, dockerInstalled: false, @@ -713,16 +717,16 @@ describe("planHostRemediation", () => { expect(actions[0].id).toBe("enable_docker_desktop_wsl_integration"); expect(actions[0].title).toBe("Enable Docker Desktop WSL integration"); - expect(actions[0].blocking).toBe(true); - expect(actions[0].commands.join("\n")).toContain( + expect(actions[0].severity).toBe("blocking"); + expect(advisoryCommands(actions[0]).join("\n")).toContain( "Docker Desktop → Settings → Resources → WSL integration", ); - expect(actions[0].commands.join("\n")).toContain("wsl --shutdown"); - expect(actions[0].commands.join("\n")).toContain("docker info"); + expect(advisoryCommands(actions[0]).join("\n")).toContain("wsl --shutdown"); + expect(advisoryCommands(actions[0]).join("\n")).toContain("docker info"); }); it("recommends Docker Desktop WSL integration when docker is unreachable inside WSL", () => { - const actions = planHostRemediation( + const actions = planHostAdvisories( baseAssessment({ isWsl: true, dockerInstalled: true, @@ -733,13 +737,13 @@ describe("planHostRemediation", () => { expect(actions[0].id).toBe("enable_docker_desktop_wsl_integration"); expect(actions[0].reason).toContain("WSL distro cannot reach the Docker daemon"); - expect(actions[0].commands.join("\n")).toContain("Start Docker Desktop"); - expect(actions[0].commands.join("\n")).toContain("wsl --shutdown"); - expect(actions[0].commands.join("\n")).not.toContain("sudo systemctl start docker"); + expect(advisoryCommands(actions[0]).join("\n")).toContain("Start Docker Desktop"); + expect(advisoryCommands(actions[0]).join("\n")).toContain("wsl --shutdown"); + expect(advisoryCommands(actions[0]).join("\n")).not.toContain("sudo systemctl start docker"); }); it("suggests usermod when docker service is active but daemon is unreachable", () => { - const actions = planHostRemediation({ + const actions = planHostAdvisories({ platform: "linux", isWsl: false, runtime: "unknown", @@ -768,50 +772,15 @@ describe("planHostRemediation", () => { expect(actions[0].id).toBe("docker_group_permission"); expect(actions[0].kind).toBe("sudo"); - expect(actions[0].blocking).toBe(true); - expect(actions[0].commands[0]).toBe("sudo usermod -aG docker $USER"); - expect(actions[0].commands[1]).toContain("newgrp docker"); - expect(actions[0].commands[2]).toBe("nemoclaw onboard"); + expect(actions[0].severity).toBe("blocking"); + expect(advisoryCommands(actions[0])[0]).toBe("sudo usermod -aG docker $USER"); + expect(advisoryCommands(actions[0])[1]).toContain("newgrp docker"); + expect(advisoryCommands(actions[0])[2]).toBe("nemoclaw onboard"); expect(actions[0].reason).toContain("docker group"); }); - it("warns that podman is unsupported on macOS without blocking onboarding", () => { - const actions = planHostRemediation({ - platform: "darwin", - isWsl: false, - runtime: "podman", - packageManager: "brew", - systemctlAvailable: false, - dockerServiceActive: null, - dockerServiceEnabled: null, - dockerInstalled: true, - dockerRunning: true, - dockerReachable: true, - nodeInstalled: true, - openshellInstalled: true, - dockerCgroupVersion: "unknown", - dockerDefaultCgroupnsMode: "unknown", - isContainerRuntimeUnderProvisioned: false, - hasNestedOverlayConflict: false, - requiresHostCgroupnsFix: false, - isUnsupportedRuntime: true, - isHeadlessLikely: false, - hasNvidiaGpu: false, - dockerCdiSpecDirs: [], - cdiNvidiaGpuSpecMissing: false, - nvidiaContainerToolkitInstalled: true, - notes: [], - }); - - const action = actions.find( - (entry: { id: string }) => entry.id === "unsupported_runtime_warning", - ); - expect(action).toBeTruthy(); - expect(action?.blocking).toBe(false); - }); - it("recommends installing Docker with a generic Linux hint when it is missing", () => { - const actions = planHostRemediation({ + const actions = planHostAdvisories({ platform: "linux", isWsl: false, runtime: "unknown", @@ -839,11 +808,11 @@ describe("planHostRemediation", () => { }); expect(actions[0].id).toBe("install_docker"); - expect(actions[0].commands[0]).toContain("Install Docker Engine"); + expect(advisoryCommands(actions[0])[0]).toContain("Install Docker Engine"); }); it("recommends installing openshell when missing", () => { - const actions = planHostRemediation({ + const actions = planHostAdvisories({ platform: "linux", isWsl: false, runtime: "docker", @@ -1848,7 +1817,7 @@ describe("assessHost container runtime resource detection (#2514)", () => { }); }); -describe("planHostRemediation — under-provisioned runtime", () => { +describe("planHostAdvisories — under-provisioned runtime", () => { it("emits a Colima-specific resize action when runtime is colima", () => { const assessment = assessHost({ platform: "darwin", @@ -1861,11 +1830,11 @@ describe("planHostRemediation — under-provisioned runtime", () => { }), commandExistsImpl: (name: string) => name === "docker", }); - const actions = planHostRemediation(assessment); + const actions = planHostAdvisories(assessment); const action = actions.find((a) => a.id === "container_runtime_under_provisioned"); expect(action).toBeDefined(); - expect(action?.blocking).toBe(false); - expect(action?.commands.some((c) => c.startsWith("colima start"))).toBe(true); + expect(action?.severity).toBe("warning"); + expect(advisoryCommands(action).some((c) => c.startsWith("colima start"))).toBe(true); }); it("emits a Docker Desktop hint when runtime is docker-desktop", () => { @@ -1880,10 +1849,12 @@ describe("planHostRemediation — under-provisioned runtime", () => { }), commandExistsImpl: (name: string) => name === "docker", }); - const actions = planHostRemediation(assessment); + const actions = planHostAdvisories(assessment); const action = actions.find((a) => a.id === "container_runtime_under_provisioned"); expect(action).toBeDefined(); - expect(action?.commands.some((c) => c.toLowerCase().includes("docker desktop"))).toBe(true); + expect(advisoryCommands(action).some((c) => c.toLowerCase().includes("docker desktop"))).toBe( + true, + ); }); it("emits no resource action when runtime is properly sized", () => { @@ -1898,7 +1869,7 @@ describe("planHostRemediation — under-provisioned runtime", () => { }), commandExistsImpl: (name: string) => name === "docker", }); - const actions = planHostRemediation(assessment); + const actions = planHostAdvisories(assessment); expect(actions.find((a) => a.id === "container_runtime_under_provisioned")).toBeUndefined(); }); }); diff --git a/src/lib/onboard/preflight.ts b/src/lib/onboard/preflight.ts index dc3137e62a7..fe0c35ca630 100644 --- a/src/lib/onboard/preflight.ts +++ b/src/lib/onboard/preflight.ts @@ -15,18 +15,17 @@ import net from "node:net"; import os from "node:os"; import path from "node:path"; -import { HOST_ADVISORY_CHECKS } from "../advisories/checks/host"; +import { ADVISORY_CHECKS } from "../advisories/registry"; import { runAdvisories } from "../advisories/runner"; -import { failLine } from "../cli/terminal-style"; import { DASHBOARD_PORT } from "../core/ports"; -import { isSupportedGatewayDockerHost } from "../domain/docker-host"; +import { isDockerDaemonReachable, isSupportedGatewayDockerHost } from "../domain/docker-host"; +import { resolveOpenshell } from "../readiness/openshell-resolver"; import { MIN_RECOMMENDED_DOCKER_CPUS, MIN_RECOMMENDED_DOCKER_MEM_GIB, } from "./container-runtime-resources"; import { assessNvidiaCdiHost } from "./docker-cdi"; import { printUnderProvisionedRuntimeWarning } from "./preflight-messages"; -import { printRemediationActions } from "./remediation"; import { isWslDockerDesktopRuntime } from "./wsl-docker-desktop-gpu"; export { @@ -109,8 +108,6 @@ export type ContainerRuntime = "docker" | "docker-desktop" | "colima" | "podman" export type PackageManager = "apt" | "dnf" | "yum" | "brew" | "pacman" | "unknown"; -export type RemediationKind = "info" | "manual" | "auto" | "sudo"; - export interface HostAssessment { platform: NodeJS.Platform | string; isWsl: boolean; @@ -130,6 +127,7 @@ export interface HostAssessment { dockerDefaultCgroupnsMode?: "host" | "private" | "unknown"; dockerStorageDriver?: string; dockerUsesContainerdSnapshotter?: boolean; + dockerNvidiaRuntimeAvailable?: boolean; dockerCpus?: number; dockerMemTotalBytes?: number; isContainerRuntimeUnderProvisioned: boolean; @@ -152,15 +150,6 @@ export interface HostAssessment { notes: string[]; } -export interface RemediationAction { - id: string; - title: string; - kind: RemediationKind; - reason: string; - commands: string[]; - blocking: boolean; -} - export const DOCKER_DESKTOP_WSL_INTEGRATION_HINT = "If you use Docker Desktop from WSL, open Docker Desktop > Settings > Resources > WSL integration and enable integration for this distro."; @@ -175,6 +164,7 @@ export interface AssessHostOpts { readFileImpl?: (filePath: string, encoding: BufferEncoding) => string; readdirImpl?: (dir: string) => string[]; runCaptureImpl?: RunCaptureFn; + resolveOpenshellImpl?: () => string | null; commandExistsImpl?: (commandName: string) => boolean; gpuProbeImpl?: () => boolean; } @@ -192,6 +182,39 @@ function commandExists(commandName: string, runCaptureImpl: RunCaptureFn): boole } } +function isOpenshellAvailable(opts: AssessHostOpts, runCaptureImpl: RunCaptureFn): boolean { + if (opts.resolveOpenshellImpl) return opts.resolveOpenshellImpl() !== null; + // Preserve the explicit dependency-injection seam used by existing host + // assessment tests. Production callers use the secure resolver below. + if (opts.commandExistsImpl) return opts.commandExistsImpl("openshell"); + + // A caller-provided command transport may represent another host. Validate + // its `command -v` evidence through the same resolver without probing local + // fallback paths on behalf of that remote transport. + if (opts.runCaptureImpl) { + let commandVResult: string | null = null; + try { + commandVResult = + String( + runCaptureImpl(buildCommandVArgv("openshell"), { + ignoreError: true, + }) || "", + ).trim() || null; + } catch { + commandVResult = null; + } + return ( + resolveOpenshell({ + commandVResult, + checkExecutable: () => false, + home: "", + }) !== null + ); + } + + return resolveOpenshell() !== null; +} + function detectWsl(opts: { platform: NodeJS.Platform | string; env: NodeJS.ProcessEnv; @@ -322,62 +345,6 @@ function parseDockerInfoSummary(info = ""): string | undefined { return parts.length > 0 ? parts.join(" · ") : undefined; } -/** - * Decide whether `docker info --format '{{json .}}'` output reflects an - * actually responding daemon, not just the Docker CLI emitting a zero-value - * client-side struct. - * - * NemoClaw #2348: when the daemon is unreachable (for example after - * `colima stop`), Docker CLI can still exit 0 and print a JSON struct with - * `ServerVersion: ""` plus `ServerErrors`. A naive non-empty-output check - * misreads that as "daemon reachable". - */ -function isDockerDaemonReachable(rawOutput = ""): boolean { - const text = String(rawOutput).trim(); - if (!text) return false; - - let parsed: unknown; - try { - parsed = JSON.parse(text); - } catch { - // Backward-compatible fallback for callers that still inject plain-text - // docker info output, but do not let plain-text daemon connection errors - // recreate the false positive that this check is meant to prevent. - const lowered = text.toLowerCase(); - return !( - lowered.includes("cannot connect to the docker daemon") || - lowered.includes("error during connect") || - lowered.includes("is the docker daemon running") - ); - } - - if (!parsed || typeof parsed !== "object") return false; - const obj = parsed as Record; - - // Explicit negative signal: Docker CLI fills ServerErrors when it could - // not reach the daemon, even when exit code is 0 under `--format`. - if (Array.isArray(obj.ServerErrors) && obj.ServerErrors.length > 0) { - return false; - } - - // Canonical positive signal: Docker CLI and podman's docker-compat layer - // both populate ServerVersion from the running daemon. - if (typeof obj.ServerVersion === "string" && obj.ServerVersion.trim().length > 0) { - return true; - } - - // podman-docker alias path: `docker info --format '{{json .}}'` actually - // runs `podman info`, whose native schema has no top-level ServerVersion - // but nests a `version.Version` instead. - const version = obj.version; - if (version && typeof version === "object") { - const v = (version as Record).Version; - if (typeof v === "string" && v.trim().length > 0) return true; - } - - return false; -} - export function parseDockerStorageDriver(info = ""): string | undefined { // JSON form (`docker info --format '{{json .}}'`) is the canonical caller // path inside this file, but accept the plain-text `Storage Driver: ` @@ -397,6 +364,18 @@ export function parseDockerUsesContainerdSnapshotter(info = ""): boolean { return /io\.containerd\.snapshotter\.v1/.test(info); } +export function parseDockerNvidiaRuntimeAvailable(info = ""): boolean | undefined { + const text = String(info || "").trim(); + if (!text) return undefined; + try { + const parsed = JSON.parse(text) as { Runtimes?: unknown }; + if (!parsed.Runtimes || typeof parsed.Runtimes !== "object") return undefined; + return Object.hasOwn(parsed.Runtimes, "nvidia"); + } catch { + return undefined; + } +} + export function parseDockerInfoCpus(info = ""): number | undefined { const jsonMatch = info.match(/"NCPU"\s*:\s*(\d+)/); if (jsonMatch) { @@ -604,19 +583,19 @@ export function assessHost(opts: AssessHostOpts = {}): HostAssessment { const dockerInstalled = opts.commandExistsImpl?.("docker") ?? commandExists("docker", runCaptureImpl); const nodeInstalled = opts.commandExistsImpl?.("node") ?? commandExists("node", runCaptureImpl); - const openshellInstalled = - opts.commandExistsImpl?.("openshell") ?? commandExists("openshell", runCaptureImpl); + const openshellInstalled = isOpenshellAvailable(opts, runCaptureImpl); const hasNvidiaGpu = opts.gpuProbeImpl?.() ?? detectNvidiaGpu(runCaptureImpl); const nvidiaContainerToolkitInstalled = opts.commandExistsImpl?.("nvidia-ctk") ?? commandExists("nvidia-ctk", runCaptureImpl); const packageManager = detectPackageManager(runCaptureImpl); const systemctlAvailable = opts.commandExistsImpl?.("systemctl") ?? commandExists("systemctl", runCaptureImpl); + const dockerHostInvalid = !isSupportedGatewayDockerHost(env.DOCKER_HOST); let dockerInfoOutput = opts.dockerInfoOutput; let dockerReachable = false; let dockerRunning = false; - if (dockerInstalled && dockerInfoOutput === undefined) { + if (dockerInstalled && !dockerHostInvalid && dockerInfoOutput === undefined) { dockerInfoOutput = runCaptureImpl(["docker", "info", "--format", "{{json .}}"], { ignoreError: true, }); @@ -630,7 +609,7 @@ export function assessHost(opts: AssessHostOpts = {}): HostAssessment { // socket is reclassified below. Only probed when the daemon is reachable so a // down/absent Docker never pays for the extra call (#7320). let dockerVersionOutput = opts.dockerVersionOutput; - if (dockerReachable && dockerVersionOutput === undefined) { + if (dockerReachable && !dockerHostInvalid && dockerVersionOutput === undefined) { dockerVersionOutput = runCaptureImpl(["docker", "version", "--format", "{{json .}}"], { ignoreError: true, }); @@ -672,6 +651,9 @@ export function assessHost(opts: AssessHostOpts = {}): HostAssessment { const dockerUsesContainerdSnapshotter = dockerReachable ? parseDockerUsesContainerdSnapshotter(dockerInfoOutput) : false; + const dockerNvidiaRuntimeAvailable = dockerReachable + ? parseDockerNvidiaRuntimeAvailable(dockerInfoOutput) + : undefined; const dockerCpus = dockerReachable ? parseDockerInfoCpus(dockerInfoOutput) : undefined; const dockerMemTotalBytes = dockerReachable ? parseDockerInfoMemTotalBytes(dockerInfoOutput) @@ -734,7 +716,7 @@ export function assessHost(opts: AssessHostOpts = {}): HostAssessment { systemctlAvailable, dockerServiceActive, dockerServiceEnabled, - dockerHostInvalid: !isSupportedGatewayDockerHost(env.DOCKER_HOST), + dockerHostInvalid, dockerInstalled, dockerRunning, dockerReachable, @@ -745,6 +727,7 @@ export function assessHost(opts: AssessHostOpts = {}): HostAssessment { dockerDefaultCgroupnsMode, dockerStorageDriver, dockerUsesContainerdSnapshotter, + dockerNvidiaRuntimeAvailable, dockerCpus, dockerMemTotalBytes, isContainerRuntimeUnderProvisioned, @@ -772,67 +755,16 @@ export function assessHost(opts: AssessHostOpts = {}): HostAssessment { return assessment; } -/** - * Decide whether onboarding must enforce a present-and-configured NVIDIA CDI - * spec (i.e. block on a missing/stale spec). The fix for #5489 makes - * `assessHost().hasNvidiaGpu` true via an lspci hardware probe when the driver - * is unloaded, which is what flags `cdiNvidiaGpuSpecMissing`. The onboard gate - * must enforce based on whether the operator *explicitly* opted out of GPU - * passthrough — NOT on whether sandbox GPU was *auto*-disabled because - * `nvidia-smi` is unavailable. Auto-disable was the bypass that let onboard skip - * the toolkit/CDI remediation in #5489; an explicit `--no-gpu` still skips it so - * a host with an unusable GPU can still onboard CPU-only. - */ -export function shouldEnforceCdiNvidiaGpuSpec(opts: { - cdiNvidiaGpuSpecMissing: boolean; - cdiNvidiaGpuSpecNeedsRepair: boolean; - explicitlyOptedOutGpuPassthrough: boolean; -}): boolean { - if (opts.explicitlyOptedOutGpuPassthrough) return false; - return opts.cdiNvidiaGpuSpecNeedsRepair || opts.cdiNvidiaGpuSpecMissing; -} - -export function assertCdiNvidiaGpuSpecPresent( - host: HostAssessment, - explicitlyOptedOutGpuPassthrough: boolean, - hostGpuPlatform: string | null | undefined = null, - exitProcess: (code: number) => never = (code) => process.exit(code), -): void { - if (hostGpuPlatform === "jetson" || isWslDockerDesktopRuntime(host)) return; - if ( - !shouldEnforceCdiNvidiaGpuSpec({ - cdiNvidiaGpuSpecMissing: host.cdiNvidiaGpuSpecMissing, - cdiNvidiaGpuSpecNeedsRepair: host.cdiNvidiaGpuSpecNeedsRepair ?? false, - explicitlyOptedOutGpuPassthrough, - }) - ) - return; - console.error( - failLine( - "Docker is configured for CDI device injection (CDISpecDirs is set), but the NVIDIA GPU CDI spec is missing or stale. OpenShell GPU startup can fail until the CDI spec is refreshed.", - ), - ); - printRemediationActions(planHostRemediation(host)); - exitProcess(1); -} - -export function planHostAdvisories(assessment: HostAssessment) { - return runAdvisories(HOST_ADVISORY_CHECKS, assessment, { +export function planHostAdvisories( + assessment: HostAssessment, + options: { resuming?: boolean } = {}, +) { + return runAdvisories(ADVISORY_CHECKS, assessment, { phase: "preflight.host", + resuming: options.resuming, }).advisories; } -export function planHostRemediation(assessment: HostAssessment): RemediationAction[] { - return planHostAdvisories(assessment).map((advisory) => ({ - id: advisory.id, - title: advisory.title, - kind: advisory.kind ?? "manual", - reason: advisory.reason, - commands: [...(advisory.commands ?? [])], - blocking: advisory.severity === "fatal" || advisory.severity === "blocking", - })); -} - // ── Port availability ──────────────────────────────────────────── export async function probePortAvailability( diff --git a/src/lib/onboard/remediation.ts b/src/lib/onboard/remediation.ts index 6ecc2514529..90a9732affd 100644 --- a/src/lib/onboard/remediation.ts +++ b/src/lib/onboard/remediation.ts @@ -7,7 +7,12 @@ const OPENCLAW_LAUNCH_AGENT_PLIST = "~/Library/LaunchAgents/ai.openclaw.gateway. export function printRemediationActions( actions: - | Array<{ id: string; title: string; reason: string; commands?: string[] }> + | readonly { + id: string; + title: string; + reason: string; + commands?: readonly string[]; + }[] | null | undefined, ): void { diff --git a/src/lib/onboard/runtime-control-flow.test.ts b/src/lib/onboard/runtime-control-flow.test.ts index 5cdc6f4d68a..2a33397f4f3 100644 --- a/src/lib/onboard/runtime-control-flow.test.ts +++ b/src/lib/onboard/runtime-control-flow.test.ts @@ -6,7 +6,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { createSession } from "../state/onboard-session"; import { applyOnboardRuntimeControlRequests, - applySelectedAgentTransition, + planSelectedAgentTransition, updateSessionAgent, } from "./runtime-control-flow"; @@ -76,7 +76,7 @@ describe("onboard runtime control flow", () => { expect(session).toEqual(before); }); - it("rejects an invalid resumed agent transition before router or session mutation", async () => { + it("rejects an invalid resumed agent transition before router or session mutation", () => { const session = createSession({ agent: "langchain-deepagents-code", observabilityEnabled: true, @@ -86,7 +86,6 @@ describe("onboard runtime control flow", () => { const before = structuredClone(session); const stopTrackedModelRouterForAgentChange = vi.fn(async () => undefined); const clearAgentScopedResumeState = vi.fn((current) => current); - const setOnboardBrandingAgent = vi.fn(); const updateSession = vi.fn((mutator) => mutator(session) ?? session); const note = vi.fn(); const error = vi.fn(); @@ -94,8 +93,8 @@ describe("onboard runtime control flow", () => { throw new Error("exit 1"); }); - await expect( - applySelectedAgentTransition( + expect(() => + planSelectedAgentTransition( { resume: true, session, @@ -106,19 +105,58 @@ describe("onboard runtime control flow", () => { { stopTrackedModelRouterForAgentChange, clearAgentScopedResumeState, - setOnboardBrandingAgent, updateSession, error, exitProcess, }, ), - ).rejects.toThrow("exit 1"); + ).toThrow("exit 1"); expect(stopTrackedModelRouterForAgentChange).not.toHaveBeenCalled(); expect(clearAgentScopedResumeState).not.toHaveBeenCalled(); - expect(setOnboardBrandingAgent).not.toHaveBeenCalled(); expect(updateSession).not.toHaveBeenCalled(); expect(note).not.toHaveBeenCalled(); expect(session).toEqual(before); }); + + it("plans without effects and commits router cleanup before durable session clearing (#7411)", async () => { + const session = createSession({ + agent: "langchain-deepagents-code", + provider: "nvidia", + routerPid: 1234, + }); + const before = structuredClone(session); + const effects: string[] = []; + const updateSession = vi.fn((mutator) => { + effects.push("update-session"); + return mutator(session) ?? session; + }); + + const plan = planSelectedAgentTransition( + { + resume: true, + session, + selectedAgentName: "openclaw", + routerPort: 4000, + note: () => undefined, + }, + { + stopTrackedModelRouterForAgentChange: async () => { + effects.push("stop-router"); + }, + updateSession, + }, + ); + + expect(plan.resumeAgentChanged).toBe(true); + expect(plan.session.routerPid).toBeNull(); + expect(effects).toEqual([]); + expect(updateSession).not.toHaveBeenCalled(); + expect(session).toEqual(before); + + await plan.commit(); + + expect(effects).toEqual(["stop-router", "update-session"]); + expect(session.routerPid).toBeNull(); + }); }); diff --git a/src/lib/onboard/runtime-control-flow.ts b/src/lib/onboard/runtime-control-flow.ts index 2cbe1fa9dd0..2b16c8f3b92 100644 --- a/src/lib/onboard/runtime-control-flow.ts +++ b/src/lib/onboard/runtime-control-flow.ts @@ -3,7 +3,6 @@ import { type Session, updateSession } from "../state/onboard-session"; import { clearAgentScopedResumeState } from "./agent-resume-state"; -import { setOnboardBrandingAgent } from "./branding"; import { isDcodeAutoApprovalMode } from "./dcode-auto-approval"; import { managedSandboxFeatureIssue } from "./managed-sandbox-feature"; import { stopTrackedModelRouterForAgentChange } from "./model-router-process"; @@ -23,7 +22,6 @@ export interface SelectedAgentTransitionDeps extends RuntimeControlAgentDeps { note(message: string): void; stopTrackedModelRouterForAgentChange(session: Session, routerPort: number): Promise; clearAgentScopedResumeState(session: Session, selectedAgentName: string): Session; - setOnboardBrandingAgent(agentName: string): void; updateSession(mutator: (session: Session) => Session | void): Session; } @@ -85,7 +83,14 @@ export function validateSessionAgentObservability( } } -export async function applySelectedAgentTransition( +export interface SelectedAgentTransitionPlan { + session: Session; + resumeAgentChanged: boolean; + commit(): Promise; +} + +/** Plan an agent transition without changing durable state or stopping a router. */ +export function planSelectedAgentTransition( input: { resume: boolean; session: Session | null; @@ -94,34 +99,50 @@ export async function applySelectedAgentTransition( note(message: string): void; }, overrides: SelectedAgentTransitionOverrides = {}, -): Promise<{ session: Session; resumeAgentChanged: boolean }> { +): SelectedAgentTransitionPlan { const deps: SelectedAgentTransitionDeps = { note: input.note, stopTrackedModelRouterForAgentChange, clearAgentScopedResumeState, - setOnboardBrandingAgent, updateSession, error: console.error, exitProcess: (code) => process.exit(code), ...overrides, }; validateSessionAgentObservability(input.session, input.selectedAgentName, deps); + if (!input.session) throw new Error("Agent transition requires an active onboarding session."); const selectedAgentName = normalizeSandboxAgentName(input.selectedAgentName); const recordedAgentName = normalizeSandboxAgentName(input.session?.agent); const resumeAgentChanged = Boolean( input.resume && input.session && recordedAgentName !== selectedAgentName, ); - if (resumeAgentChanged && input.session) { - deps.note( - ` Agent changed from ${formatSandboxAgentName(recordedAgentName)} to ${formatSandboxAgentName(selectedAgentName)}; refreshing provider selection.`, - ); - await deps.stopTrackedModelRouterForAgentChange(input.session, input.routerPort); - deps.updateSession((current) => deps.clearAgentScopedResumeState(current, selectedAgentName)); + const originalSession = structuredClone(input.session); + let projectedSession = structuredClone(input.session); + if (resumeAgentChanged) { + projectedSession = deps.clearAgentScopedResumeState(projectedSession, selectedAgentName); } - deps.setOnboardBrandingAgent(input.selectedAgentName || "openclaw"); - const session = deps.updateSession((current) => - updateSessionAgent(current, input.selectedAgentName, deps), - ); - return { session, resumeAgentChanged }; + projectedSession = updateSessionAgent(projectedSession, input.selectedAgentName, deps); + let committed: Promise | null = null; + return { + session: projectedSession, + resumeAgentChanged, + commit() { + committed ??= (async () => { + if (resumeAgentChanged) { + deps.note( + ` Agent changed from ${formatSandboxAgentName(recordedAgentName)} to ${formatSandboxAgentName(selectedAgentName)}; refreshing provider selection.`, + ); + await deps.stopTrackedModelRouterForAgentChange(originalSession, input.routerPort); + } + return deps.updateSession((current) => { + const transitioned = resumeAgentChanged + ? deps.clearAgentScopedResumeState(current, selectedAgentName) + : current; + return updateSessionAgent(transitioned, input.selectedAgentName, deps); + }); + })(); + return committed; + }, + }; } diff --git a/src/lib/onboard/runtime-provider/docker.test.ts b/src/lib/onboard/runtime-provider/docker.test.ts new file mode 100644 index 00000000000..97922b67895 --- /dev/null +++ b/src/lib/onboard/runtime-provider/docker.test.ts @@ -0,0 +1,72 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { createDockerRuntimeProviderBundle } from "./docker"; + +function inspectDockerHost(stdout: string, status = 0, stderr = "") { + const captureHostCommand = vi.fn(() => ({ status, stdout, stderr })); + const provider = createDockerRuntimeProviderBundle({ captureHostCommand }); + expect(provider.preflightDoctor.supported).toBe(true); + const preflightDoctor = provider.preflightDoctor as Extract< + typeof provider.preflightDoctor, + { supported: true } + >; + + return { + captureHostCommand, + check: preflightDoctor.inspectHost(), + }; +} + +describe("Docker runtime provider host doctor", () => { + it("reports the daemon version from the shared reachability observation (#7411)", () => { + const { captureHostCommand, check } = inspectDockerHost( + JSON.stringify({ ServerVersion: "29.3.1", OperatingSystem: "Ubuntu 24.04" }), + ); + + expect(captureHostCommand).toHaveBeenCalledWith( + "docker", + ["info", "--format", "{{json .}}"], + 8000, + ); + expect(check).toEqual({ + group: "Host", + label: "Docker daemon", + status: "ok", + detail: "server 29.3.1", + hint: undefined, + }); + }); + + it.each([ + ["empty output", ""], + ["zero-value JSON", JSON.stringify({ ServerVersion: "" })], + [ + "daemon error JSON", + JSON.stringify({ + ServerVersion: "", + ServerErrors: ["Cannot connect to the Docker daemon"], + }), + ], + ])("rejects exit-zero %s without positive daemon evidence (#7411)", (_case, stdout) => { + expect(inspectDockerHost(stdout).check).toEqual({ + group: "Host", + label: "Docker daemon", + status: "fail", + detail: "docker info failed", + hint: "start Docker and verify your user can access the daemon", + }); + }); + + it("preserves the captured Docker error when the command fails", () => { + expect(inspectDockerHost("", 1, "Cannot connect to the Docker daemon\n").check).toEqual({ + group: "Host", + label: "Docker daemon", + status: "fail", + detail: "Cannot connect to the Docker daemon", + hint: "start Docker and verify your user can access the daemon", + }); + }); +}); diff --git a/src/lib/onboard/runtime-provider/docker.ts b/src/lib/onboard/runtime-provider/docker.ts index 5863bdcaee7..0331ea1a4eb 100644 --- a/src/lib/onboard/runtime-provider/docker.ts +++ b/src/lib/onboard/runtime-provider/docker.ts @@ -6,6 +6,7 @@ import { isDockerRuntimeDown, printDockerRuntimeDownGuidance, } from "../../actions/sandbox/gateway-failure-classifier"; +import { parseDockerDaemonObservation } from "../../domain/docker-host"; import { cliName } from "../branding"; import { findLabeledSandboxContainers, @@ -103,21 +104,17 @@ function oneLine(value = ""): string { } function inspectDockerHost(deps: DockerRuntimeProviderDependencies): RuntimeProviderDoctorCheck { - const result = deps.captureHostCommand( - "docker", - ["info", "--format", "{{.ServerVersion}}"], - 8000, - ); + const result = deps.captureHostCommand("docker", ["info", "--format", "{{json .}}"], 8000); + const observation = parseDockerDaemonObservation(result.stdout); + const reachable = result.status === 0 && observation.reachable; return { group: "Host", label: "Docker daemon", - status: result.status === 0 ? "ok" : "fail", - detail: - result.status === 0 - ? `server ${result.stdout.trim() || "unknown"}` - : oneLine(result.stderr || result.error?.message || "docker info failed"), - hint: - result.status === 0 ? undefined : "start Docker and verify your user can access the daemon", + status: reachable ? "ok" : "fail", + detail: reachable + ? `server ${observation.serverVersion ?? "unknown"}` + : oneLine(result.stderr || result.error?.message || "docker info failed"), + hint: reachable ? undefined : "start Docker and verify your user can access the daemon", }; } diff --git a/src/lib/onboard/sandbox-gpu-preflight.ts b/src/lib/onboard/sandbox-gpu-preflight.ts index 5fbbde0fe0d..65cc1f8479f 100644 --- a/src/lib/onboard/sandbox-gpu-preflight.ts +++ b/src/lib/onboard/sandbox-gpu-preflight.ts @@ -5,7 +5,11 @@ import { dockerInfoFormat } from "../adapters/docker"; import { failLine, warnLine } from "../cli/terminal-style"; import type { GpuDetection } from "../inference/nim"; import type { SandboxGpuProofResult } from "../state/registry"; -import { findReadableNvidiaCdiSpecFiles, getDockerCdiSpecDirs } from "./docker-cdi"; +import { + DEFAULT_DOCKER_CDI_SPEC_DIRS, + findReadableNvidiaCdiSpecFiles, + getDockerCdiSpecDirs, +} from "./docker-cdi"; import type { SandboxGpuConfig, SandboxGpuFlag } from "./sandbox-gpu-mode"; import { detectWslDockerDesktopStatus, @@ -22,8 +26,6 @@ const SANDBOX_GPU_PREFLIGHT_TIMEOUT_MS = 30_000; // CDISpecDirs (daemon unreachable from this process, or an engine that omits // the field) even though CDI works, so the preflight falls back to these // before declaring CDI unsupported (#7330). -const FALLBACK_DOCKER_CDI_SPEC_DIRS = ["/etc/cdi", "/var/run/cdi"]; - export type SandboxGpuPreflightDeps = WslDockerDesktopDetectionDeps & { getDockerCdiSpecDirs?: () => string[]; findReadableNvidiaCdiSpecFiles?: (dirs: string[]) => string[]; @@ -133,20 +135,22 @@ export function dockerNvidiaRuntimeAvailable(deps: SandboxGpuPreflightDeps = {}) } } +export function printJetsonNvidiaRuntimeUnavailableError(): void { + console.error(""); + console.error(failLine("Docker NVIDIA runtime was not detected for Jetson/Tegra sandbox GPU.")); + console.error(" Jetson sandbox GPU uses NVIDIA Container Runtime semantics, not CDI."); + console.error(" Install/configure NVIDIA Container Toolkit for Docker, then restart Docker:"); + console.error(" sudo nvidia-ctk runtime configure --runtime=docker"); + console.error(" sudo systemctl restart docker"); + console.error(" Or force CPU sandbox behavior with NEMOCLAW_SANDBOX_GPU=0."); +} + function validateJetsonSandboxGpuPreflight( deps: SandboxGpuPreflightDeps, exitProcess: (code: number) => never, ): void { if (!dockerNvidiaRuntimeAvailable(deps)) { - console.error(""); - console.error(failLine("Docker NVIDIA runtime was not detected for Jetson/Tegra sandbox GPU.")); - console.error(" Jetson sandbox GPU uses NVIDIA Container Runtime semantics, not CDI."); - console.error( - " Install/configure NVIDIA Container Toolkit for Docker, then restart Docker:", - ); - console.error(" sudo nvidia-ctk runtime configure --runtime=docker"); - console.error(" sudo systemctl restart docker"); - console.error(" Or force CPU sandbox behavior with NEMOCLAW_SANDBOX_GPU=0."); + printJetsonNvidiaRuntimeUnavailableError(); exitProcess(1); } console.log(" ✓ Docker NVIDIA runtime detected for Jetson/Tegra sandbox GPU"); @@ -376,7 +380,7 @@ export function validateSandboxGpuPreflight( const reportedCdiSpecDirs = (deps.getDockerCdiSpecDirs ?? getDockerCdiSpecDirs)(); const cdiSpecDirs = - reportedCdiSpecDirs.length > 0 ? reportedCdiSpecDirs : FALLBACK_DOCKER_CDI_SPEC_DIRS; + reportedCdiSpecDirs.length > 0 ? reportedCdiSpecDirs : [...DEFAULT_DOCKER_CDI_SPEC_DIRS]; const cdiSpecFiles = (deps.findReadableNvidiaCdiSpecFiles ?? findReadableNvidiaCdiSpecFiles)( cdiSpecDirs, ); diff --git a/src/lib/readiness/effects.test.ts b/src/lib/readiness/effects.test.ts index 32d7b15e2bc..2803a11e6ca 100644 --- a/src/lib/readiness/effects.test.ts +++ b/src/lib/readiness/effects.test.ts @@ -4,6 +4,8 @@ import { describe, expect, it, vi } from "vitest"; import { assessHost } from "../onboard/preflight"; import { createHostReadinessReport } from "./host"; +import { getSystemReadinessReferenceErrors } from "./references"; +import { createSystemReadinessReport } from "./system"; const NOW = new Date("2026-06-01T12:00:00Z"); const SOURCE_REVISION = "21e60ae287e8c2a184f71406ac8b418f046330d1"; @@ -83,6 +85,7 @@ describe("readiness process effects (#7412)", () => { expect(first).toEqual(second); expect(first.mutated).toBe(false); + expect(getSystemReadinessReferenceErrors(first)).toEqual([]); expect(assess).toHaveBeenCalledTimes(2); expect(commands.length).toBeGreaterThan(0); expect(commands.every((command) => Object.hasOwn(COMMAND_OUTPUTS, command.join(" ")))).toBe( @@ -130,4 +133,68 @@ describe("readiness process effects (#7412)", () => { ).toBe(true); expect(directories.every((path) => path === "/etc/cdi" || path === "/var/run/cdi")).toBe(true); }); + + it("repeats the composite probe without invoking gateway lifecycle effects (#7411)", async () => { + const resolveOwner = vi.fn(() => ({ + gatewayName: "nemoclaw", + gatewayPort: 8080, + mode: "nemoclaw-managed" as const, + source: "standalone" as const, + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], + })); + const observeManagedGateway = vi.fn(async () => ({ + reuseState: "missing" as const, + driftState: "not-detected" as const, + portConflictState: "none" as const, + })); + const assess = vi.fn(() => + assessHost({ + gpuProbeImpl: () => false, + platform: "linux", + readFileImpl: (path) => FILE_CONTENTS[path] ?? "", + readdirImpl: () => [], + runCaptureImpl: (command) => COMMAND_OUTPUTS[command.join(" ")] ?? "", + }), + ); + const reportOptions = { + nemoclawVersion: "0.1.0", + sourceRevision: SOURCE_REVISION, + now: () => NOW, + }; + const collectionOptions = { + gateway: { + resolveOwner, + probeAttachment: vi.fn(async () => { + throw new Error("managed gateway attachment must not run"); + }), + observeManagedGateway, + }, + host: { + assess, + collectPlatformIdentity: () => ({ + productName: null, + nvidiaPlatform: null, + stationProfile: null, + stationGb300PciGpu: null, + }), + }, + }; + const beforeEnv = { ...process.env }; + + const first = await createSystemReadinessReport(reportOptions, collectionOptions); + const second = await createSystemReadinessReport(reportOptions, collectionOptions); + + expect(first).toEqual(second); + expect(first.mutated).toBe(false); + expect(getSystemReadinessReferenceErrors(first)).toEqual([]); + expect(first.observations).toContainEqual( + expect.objectContaining({ id: "gateway.reuse", value: "missing" }), + ); + expect(resolveOwner).toHaveBeenCalledTimes(2); + expect(observeManagedGateway).toHaveBeenCalledTimes(2); + expect(process.env).toEqual(beforeEnv); + }); }); diff --git a/src/lib/readiness/gateway-production.test.ts b/src/lib/readiness/gateway-production.test.ts new file mode 100644 index 00000000000..807e12a7239 --- /dev/null +++ b/src/lib/readiness/gateway-production.test.ts @@ -0,0 +1,337 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { GatewayOwner } from "../onboard/gateway-ownership"; +import { resetTraceForTests, TRACE_FILE_ENV } from "../trace"; + +const subprocess = vi.hoisted(() => ({ + spawnSync: vi.fn(), +})); + +vi.mock("node:child_process", async (importOriginal) => ({ + ...(await importOriginal()), + spawnSync: subprocess.spawnSync, +})); + +afterEach(() => { + resetTraceForTests(); + vi.unstubAllEnvs(); + vi.restoreAllMocks(); +}); + +import { + buildGatewayReadinessProbeEnv, + classifyManagedGatewayEndpointBinding, + classifyManagedGatewayPortConflict, + classifyManagedGatewayVersionDrift, + classifyManagedGatewayVersionSource, + createProductionGatewayReadinessDependencies, + gatewayExecutableSamplesMatchTrustedBinary, + gatewayProcessIdentityMatchesTrustedBinary, + parseDarwinLsofExecutable, +} from "./gateway-production"; + +function commandResult(stdout = "", status = 1) { + return { status, stdout, stderr: "", signal: null, pid: 1, output: [] }; +} + +function managedOwner(gatewayPort: number): GatewayOwner { + return { + gatewayName: "nemoclaw-readiness-test", + gatewayPort, + mode: "nemoclaw-managed", + source: "standalone", + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], + }; +} + +function externalOwner(gatewayPort: number): GatewayOwner { + return { + gatewayName: "nemoclaw-readiness-test", + gatewayPort, + mode: "externally-supervised", + source: "declared", + endpoint: `http://127.0.0.1:${gatewayPort}`, + stateDir: "/var/lib/openshell/gateway", + supervisor: { + kind: "systemd-system", + serviceName: "openshell-gateway.service", + execPath: "/opt/platform/gatewayd", + }, + requiredCapabilities: ["gateway.health"], + }; +} + +describe("managed gateway port readiness (#7411)", () => { + it("omits ambient credentials from every read-only gateway probe environment", () => { + const env = buildGatewayReadinessProbeEnv( + { + HOME: "/home/test", + PATH: "/usr/bin", + DOCKER_HOST: "unix:///run/user/1000/docker.sock", + XDG_RUNTIME_DIR: "/run/user/1000", + DBUS_SESSION_BUS_ADDRESS: "unix:path=/run/user/1000/bus", + GITHUB_TOKEN: "github-secret", + NVIDIA_INFERENCE_API_KEY: "nvidia-secret", + OPENSHELL_GATEWAY: "ambient-wrong-gateway", + OPENSHELL_GATEWAY_AUTH_TOKEN: "gateway-secret", + OPENSHELL_OIDC_CLIENT_SECRET: "oidc-secret", + OPENSHELL_SANDBOX_TOKEN: "sandbox-secret", + OPENSHELL_TOKEN: "openshell-secret", + LC_CLIENT_SECRET: "locale-prefix-secret", + XDG_API_TOKEN: "xdg-prefix-secret", + }, + { + gatewayName: "nemoclaw-readiness-test", + localTlsDir: "/var/lib/nemoclaw/gateway/tls", + }, + ); + + expect(env).toEqual({ + HOME: "/home/test", + PATH: "/usr/bin", + DOCKER_HOST: "unix:///run/user/1000/docker.sock", + XDG_RUNTIME_DIR: "/run/user/1000", + DBUS_SESSION_BUS_ADDRESS: "unix:path=/run/user/1000/bus", + OPENSHELL_GATEWAY: "nemoclaw-readiness-test", + OPENSHELL_LOCAL_TLS_DIR: "/var/lib/nemoclaw/gateway/tls", + }); + }); + + it("omits unsupported remote Docker endpoints from readiness children", () => { + expect( + buildGatewayReadinessProbeEnv({ DOCKER_HOST: "tcp://attacker.example:2375" }).DOCKER_HOST, + ).toBeUndefined(); + }); + + it("rejects a spoofed listener path without executing its binary", () => { + subprocess.spawnSync.mockClear(); + const trusted = "/opt/openshell/bin/openshell-gateway"; + const spoofed = "/tmp/spoof/openshell-gateway --name nemoclaw-readiness-test --port 8080"; + + expect( + gatewayProcessIdentityMatchesTrustedBinary(spoofed, trusted, "nemoclaw-readiness-test", 8080), + ).toBe(false); + expect(subprocess.spawnSync).not.toHaveBeenCalled(); + }); + + it("rejects trusted-looking argv when the Linux process executable is foreign", () => { + subprocess.spawnSync.mockClear(); + const trusted = "/opt/openshell/bin/openshell-gateway"; + const spoofedArgv = `${trusted} --name nemoclaw-readiness-test --port 8080`; + + expect( + gatewayProcessIdentityMatchesTrustedBinary( + spoofedArgv, + trusted, + "nemoclaw-readiness-test", + 8080, + "/tmp/foreign-gateway", + "linux", + ), + ).toBe(false); + expect( + gatewayProcessIdentityMatchesTrustedBinary( + spoofedArgv, + trusted, + "nemoclaw-readiness-test", + 8080, + trusted, + "linux", + ), + ).toBe(true); + expect(subprocess.spawnSync).not.toHaveBeenCalled(); + }); + + it("rejects trusted-looking argv on macOS without package-service identity", () => { + const trusted = "/opt/homebrew/opt/openshell/bin/openshell-gateway"; + const spoofedArgv = `${trusted} --name nemoclaw-readiness-test --port 8080`; + + expect( + gatewayProcessIdentityMatchesTrustedBinary( + spoofedArgv, + trusted, + "nemoclaw-readiness-test", + 8080, + trusted, + "darwin", + ), + ).toBe(false); + }); + + it("rejects an executable change while a listener PID remains stable", () => { + const trusted = "/opt/openshell/bin/openshell-gateway"; + + expect(gatewayExecutableSamplesMatchTrustedBinary(trusted, trusted, trusted)).toBe(true); + expect( + gatewayExecutableSamplesMatchTrustedBinary(trusted, "/tmp/foreign-gateway", trusted), + ).toBe(false); + expect( + gatewayExecutableSamplesMatchTrustedBinary("/tmp/foreign-gateway", trusted, trusted), + ).toBe(false); + }); + + it("uses the main macOS executable vnode before dyld or later mappings", () => { + expect( + parseDarwinLsofExecutable( + "p4242\nftxt\nn/opt/homebrew/Cellar/openshell/1/bin/gateway\nftxt\nn/usr/lib/dyld\n", + ), + ).toBe("/opt/homebrew/Cellar/openshell/1/bin/gateway"); + expect( + parseDarwinLsofExecutable( + "p4242\nftxt\nn/tmp/foreign-gateway\nftxt\nn/opt/homebrew/Cellar/openshell/1/bin/gateway\n", + ), + ).toBe("/tmp/foreign-gateway"); + }); + + it("does not compare a package-service listener against a different CLI sibling binary", () => { + const listenerScan = { pids: [41], unverifiedPids: [] }; + + expect(classifyManagedGatewayVersionSource(false, listenerScan, new Set())).toBeNull(); + expect(classifyManagedGatewayVersionSource(false, listenerScan, new Set([41]))).toBe( + "host-process", + ); + }); + + it.each([ + [true, { pids: [], unverifiedPids: [], complete: true }, "missing", "none"], + [true, { pids: [], unverifiedPids: [], complete: true }, "healthy", "unknown"], + [true, { pids: [], unverifiedPids: [], complete: true }, "active-unnamed", "unknown"], + [false, { pids: [41], unverifiedPids: [], complete: true }, "healthy", "none"], + [false, { pids: [41], unverifiedPids: [], complete: true }, "stale", "none"], + [false, { pids: [41], unverifiedPids: [], complete: true }, "active-unnamed", "none"], + [false, { pids: [41], unverifiedPids: [], complete: true }, "missing", "none"], + [false, { pids: [41], unverifiedPids: [], complete: true }, "foreign-active", "owner-mismatch"], + [false, { pids: [41, 42], unverifiedPids: [], complete: true }, "healthy", "multiple-owners"], + [false, { pids: [], unverifiedPids: [41], complete: true }, "stale", "owner-mismatch"], + [false, { pids: [], unverifiedPids: [], complete: false }, "healthy", "unknown"], + ] as const)("maps portAvailable=%s, listeners=%o, reuse=%s to %s", (portAvailable, listeners, reuseState, expected) => { + expect(classifyManagedGatewayPortConflict(portAvailable, listeners, reuseState)).toBe(expected); + }); + + it.each([ + ["https://127.0.0.1:8080", 8080, "match"], + ["http://localhost:8080", 8080, "match"], + ["https://127.0.0.1:9090", 8080, "mismatch"], + ["https://gateway.example:8080", 8080, "mismatch"], + ] as const)("binds managed endpoint %s to port %s as %s", (endpoint, port, expected) => { + expect(classifyManagedGatewayEndpointBinding([`Gateway endpoint: ${endpoint}`], port)).toBe( + expected, + ); + }); + + it("rejects healthy managed metadata bound to another endpoint", () => { + const listener = { pids: [41], unverifiedPids: [], complete: true }; + + expect(classifyManagedGatewayPortConflict(false, listener, "healthy", false, "mismatch")).toBe( + "owner-mismatch", + ); + expect(classifyManagedGatewayPortConflict(false, listener, "healthy", false, "unknown")).toBe( + "unknown", + ); + }); + + it("accepts one legacy Docker proxy only when the exact cluster endpoint owns the port", () => { + const proxy = { pids: [], unverifiedPids: [41], complete: true }; + + expect(classifyManagedGatewayPortConflict(false, proxy, "healthy", true)).toBe("none"); + expect(classifyManagedGatewayPortConflict(false, proxy, "healthy", false)).toBe( + "owner-mismatch", + ); + expect( + classifyManagedGatewayPortConflict( + false, + { pids: [], unverifiedPids: [41, 42], complete: true }, + "healthy", + true, + ), + ).toBe("multiple-owners"); + }); + + it("rejects simultaneous legacy-cluster and host-process ownership evidence", () => { + expect( + classifyManagedGatewayPortConflict( + false, + { pids: [41], unverifiedPids: [], complete: true }, + "healthy", + true, + ), + ).toBe("owner-mismatch"); + }); + + it.each([ + [false, "healthy", "compatible", "not-detected"], + [false, "healthy", "drift", "detected"], + [false, "healthy", "unknown", "unknown"], + [true, "healthy", null, "unknown"], + [true, "stale", null, "not-detected"], + [false, "missing", "compatible", "not-detected"], + [false, "missing", "drift", "detected"], + [false, "missing", null, "unknown"], + [false, "foreign-active", null, "not-detected"], + ] as const)("maps portAvailable=%s, reuse=%s, version=%s to %s", (portAvailable, reuseState, compatibility, expected) => { + expect(classifyManagedGatewayVersionDrift(portAvailable, reuseState, compatibility)).toBe( + expected, + ); + }); + + it("collects production port evidence without attempting sudo", async () => { + vi.stubEnv("GITHUB_TOKEN", "github-secret"); + vi.stubEnv("OPENSHELL_GATEWAY_AUTH_TOKEN", "gateway-secret"); + subprocess.spawnSync.mockImplementation((command: string, args: readonly string[] = []) => { + const resolvesLsof = command === "sh" && args.includes('command -v "$1"'); + return resolvesLsof ? commandResult("/usr/bin/lsof\n", 0) : commandResult(); + }); + + const gatewayPort = 0; + const deps = createProductionGatewayReadinessDependencies({ + gatewayName: () => "nemoclaw-readiness-test", + gatewayPort: () => gatewayPort, + }); + + await deps.observeManagedGateway(managedOwner(gatewayPort)); + + expect(subprocess.spawnSync.mock.calls.some(([command]) => command === "lsof")).toBe(true); + expect(subprocess.spawnSync.mock.calls.some(([command]) => command === "sudo")).toBe(false); + for (const [, , options] of subprocess.spawnSync.mock.calls) { + const env = options?.env as NodeJS.ProcessEnv | undefined; + expect(env).toBeDefined(); + expect(env?.GITHUB_TOKEN).toBeUndefined(); + expect(env?.OPENSHELL_GATEWAY_AUTH_TOKEN).toBeUndefined(); + expect(env?.OPENSHELL_GATEWAY).toBe("nemoclaw-readiness-test"); + } + }); + + it("does not write an onboard trace for a public external attachment probe (#7411)", async () => { + const traceDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-readiness-trace-")); + const tracePath = path.join(traceDir, "unexpected.json"); + vi.stubEnv(TRACE_FILE_ENV, tracePath); + vi.stubEnv("NEMOCLAW_REUSE_HEALTH_POLL_COUNT", "1"); + vi.stubEnv("NEMOCLAW_REUSE_HEALTH_POLL_INTERVAL", "0"); + resetTraceForTests(); + subprocess.spawnSync.mockImplementation(() => commandResult()); + + try { + const gatewayPort = 65_534; + const deps = createProductionGatewayReadinessDependencies({ + gatewayName: () => "nemoclaw-readiness-test", + gatewayPort: () => gatewayPort, + }); + + await deps.probeAttachment(externalOwner(gatewayPort)); + expect(fs.existsSync(tracePath)).toBe(false); + vi.stubEnv(TRACE_FILE_ENV, ""); + } finally { + resetTraceForTests(); + fs.rmSync(traceDir, { force: true, recursive: true }); + } + }); +}); diff --git a/src/lib/readiness/gateway-production.ts b/src/lib/readiness/gateway-production.ts new file mode 100644 index 00000000000..c2b0dd8e49f --- /dev/null +++ b/src/lib/readiness/gateway-production.ts @@ -0,0 +1,751 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { + type GatewayReuseState, + type GatewayVersionCompatibility, + type GatewayVersionSource, + getGatewayClusterContainerName, + getGatewayReuseState, + isGatewayHealthy, + OPENSHELL_PROBE_TIMEOUT_MS, + observeOpenShellGatewayVersionCompatibility, + parseVersionFromText, + stripAnsi, +} from "../adapters/openshell/gateway-drift"; +import { + getConfiguredGatewayPort, + getDockerDriverGatewayEndpoint, + getGatewayPortCheckOptions, +} from "../onboard/docker-driver-gateway-env"; +import { getDockerDriverGatewayLocalTlsDir } from "../onboard/docker-driver-gateway-local-tls"; +import { createDockerDriverGatewayPortListenerHelpers } from "../onboard/docker-driver-gateway-port-listener"; +import { + hasDockerDriverGatewayEnvironment, + isDockerDriverGatewayProcessIdentity, + readDockerDriverGatewayProcessEnvironment, +} from "../onboard/docker-driver-gateway-process-identity"; +import { + resolveDockerDriverGatewayName, + resolveDockerDriverGatewayStateDirName, +} from "../onboard/docker-driver-gateway-runtime"; +import { + getTrustedActiveOpenShellGatewayUserServiceIdentity, + hasOpenShellGatewayUserService, +} from "../onboard/docker-driver-gateway-service"; +import { createGatewayHostRuntime } from "../onboard/gateway-host-runtime"; +import { loadGatewayManagementDeclaration } from "../onboard/gateway-management"; +import { + cleanGatewayProcessToken, + gatewayProcessCmdlineMatches, + OPENSHELL_GATEWAY_PROCESS_NAMES, +} from "../onboard/gateway-process-identity"; +import { resolveOpenshell } from "../onboard/openshell-cli"; +import { checkPortAvailable } from "../onboard/preflight"; +import type { + GatewayPortConflictState, + GatewayReadinessDependencies, + ManagedGatewayObservations, +} from "./gateway"; +import { buildSystemReadinessProbeEnv, type ReadinessProbeEnvironmentControls } from "./probe-env"; + +export interface ProductionGatewayReadinessOptions { + gatewayName?: () => string; + gatewayPort?: () => number; + resolveOwner?: GatewayReadinessDependencies["resolveOwner"]; + probeAttachment?: GatewayReadinessDependencies["probeAttachment"]; + isLegacyClusterBound?: () => boolean; + observeVersionCompatibility?: ( + source: GatewayVersionSource, + hostProcessPid: number | null, + ) => GatewayVersionCompatibility; +} + +interface ReadonlyCaptureResult { + stdout: string; + stderr: string; + exitCode: number | null; + timedOut: boolean; +} + +/** Environment for read-only readiness children; intentionally excludes every credential family. */ +export function buildGatewayReadinessProbeEnv( + source: NodeJS.ProcessEnv = process.env, + controls: ReadinessProbeEnvironmentControls = {}, +): NodeJS.ProcessEnv { + return buildSystemReadinessProbeEnv(source, controls); +} + +function captureReadonly( + args: readonly string[], + env: NodeJS.ProcessEnv = buildGatewayReadinessProbeEnv(), +): ReadonlyCaptureResult { + const [file, ...argv] = args; + if (!file) return { stdout: "", stderr: "", exitCode: null, timedOut: false }; + const result = spawnSync(file, argv, { + encoding: "utf-8", + env, + shell: false, + stdio: ["ignore", "pipe", "pipe"], + timeout: OPENSHELL_PROBE_TIMEOUT_MS, + }); + return { + stdout: String(result.stdout ?? "").trim(), + stderr: String(result.stderr ?? "").trim(), + exitCode: result.status, + timedOut: + (result.error as NodeJS.ErrnoException | undefined)?.code === "ETIMEDOUT" || + result.status === 28, + }; +} + +function isPidAlive(pid: number): boolean { + if (!Number.isInteger(pid) || pid <= 0) return false; + try { + process.kill(pid, 0); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code === "EPERM"; + } +} + +function combinedOutput(result: ReadonlyCaptureResult): string { + return [result.stdout, result.stderr].filter(Boolean).join("\n"); +} + +function normalizeExecutablePath(value: string): string | null { + try { + return fs.realpathSync.native(value); + } catch { + return path.isAbsolute(value) ? path.normalize(value) : null; + } +} + +function resolveManagedGatewayProbeTlsDir( + gatewayPort: number, + source: NodeJS.ProcessEnv, +): string | undefined { + const configuredStateDir = source.NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR?.trim(); + const stateDir = configuredStateDir + ? path.resolve(configuredStateDir) + : path.join( + source.HOME || os.homedir(), + ".local", + "state", + "nemoclaw", + resolveDockerDriverGatewayStateDirName(gatewayPort), + ); + const localTlsDir = getDockerDriverGatewayLocalTlsDir(stateDir); + return ["ca.crt", "client/tls.crt", "client/tls.key"].every((relativePath) => + fs.existsSync(path.join(localTlsDir, relativePath)), + ) + ? localTlsDir + : undefined; +} + +/** Require both executable observations to remain the independently trusted binary. */ +export function gatewayExecutableSamplesMatchTrustedBinary( + before: string | null, + after: string | null, + trustedBinary: string | null, +): boolean { + if (!before || !after || !trustedBinary) return false; + const expected = normalizeExecutablePath(trustedBinary); + return ( + expected !== null && + normalizeExecutablePath(before) === expected && + normalizeExecutablePath(after) === expected + ); +} + +function resolveTrustedOpenshellBinary(env: NodeJS.ProcessEnv): string | null { + const commandV = captureReadonly(["sh", "-c", 'command -v "$1"', "--", "openshell"], env); + return resolveOpenshell({ + commandVResult: commandV.exitCode === 0 ? commandV.stdout || null : null, + home: env.HOME, + }); +} + +function resolveTrustedGatewayBinary(openshell: string | null): string | null { + const configured = process.env.NEMOCLAW_OPENSHELL_GATEWAY_BIN?.trim(); + const candidates = [ + ...(configured ? [path.resolve(configured)] : []), + ...(openshell ? [path.join(path.dirname(openshell), "openshell-gateway")] : []), + path.join(os.homedir(), ".local", "bin", "openshell-gateway"), + "/opt/homebrew/bin/openshell-gateway", + "/usr/local/bin/openshell-gateway", + "/usr/bin/openshell-gateway", + ]; + for (const candidate of candidates) { + if (!fs.existsSync(candidate)) continue; + return normalizeExecutablePath(candidate); + } + return null; +} + +/** Require the observed argv0 to resolve to the independently selected binary. */ +export function gatewayProcessIdentityMatchesTrustedBinary( + identity: string, + trustedGatewayBin: string | null, + gatewayName: string, + gatewayPort: number, + actualExecutablePath: string | null = null, + platform: NodeJS.Platform = process.platform, +): boolean { + // Linux procfs supplies a kernel-backed executable identity for direct + // processes. On macOS, argv0 is user-controlled, so direct listeners remain + // untrusted and only the positively identified Homebrew service is eligible. + if (!trustedGatewayBin || platform !== "linux") return false; + const argv0 = cleanGatewayProcessToken(identity.trim().split(/\s+/, 1)[0] ?? ""); + const actual = argv0 ? normalizeExecutablePath(argv0) : null; + const expected = normalizeExecutablePath(trustedGatewayBin); + if (!actual || !expected || actual !== expected) return false; + if (!actualExecutablePath || normalizeExecutablePath(actualExecutablePath) !== expected) { + return false; + } + return gatewayProcessCmdlineMatches(identity, trustedGatewayBin, { + expectedOpenShellGateway: { name: gatewayName, port: gatewayPort }, + processNames: OPENSHELL_GATEWAY_PROCESS_NAMES, + resolveExecutablePath: normalizeExecutablePath, + }); +} + +function readLinuxProcessStartTime(pid: number): string | null { + try { + const stat = fs.readFileSync(`/proc/${pid}/stat`, "utf-8"); + const commandEnd = stat.lastIndexOf(")"); + if (commandEnd < 0) return null; + return ( + stat + .slice(commandEnd + 1) + .trim() + .split(/\s+/)[19] ?? null + ); + } catch { + return null; + } +} + +function readLinuxProcessExecutable(pid: number): string | null { + try { + return fs.realpathSync.native(`/proc/${pid}/exe`); + } catch { + return null; + } +} + +export function parseDarwinLsofExecutable(output: string): string | null { + // macOS reports the process executable first, followed by other text vnodes + // such as /usr/lib/dyld. Selecting the first record prevents a process from + // satisfying identity by mapping the trusted binary as a later text vnode. + for (const line of output.split(/\r?\n/)) { + if (!line.startsWith("n/")) continue; + const candidate = line.slice(1).trim(); + return path.isAbsolute(candidate) ? candidate : null; + } + return null; +} + +function readDarwinProcessExecutable(pid: number, env: NodeJS.ProcessEnv): string | null { + const result = captureReadonly( + ["/usr/sbin/lsof", "-a", "-p", String(pid), "-d", "txt", "-Fn"], + env, + ); + if (result.exitCode !== 0 || result.timedOut) return null; + return parseDarwinLsofExecutable(result.stdout); +} + +function getTrustedHostProcessGatewayRuntime( + trustedGatewayBin: string | null, + env: NodeJS.ProcessEnv, +): { gatewayBin: string | null; runningVersion: string | null } | null { + if (!trustedGatewayBin) return null; + const version = captureReadonly([trustedGatewayBin, "--version"], env); + if (version.exitCode !== 0 || version.timedOut) { + return { gatewayBin: trustedGatewayBin, runningVersion: null }; + } + return { + gatewayBin: trustedGatewayBin, + runningVersion: parseVersionFromText(combinedOutput(version), `${trustedGatewayBin} --version`), + }; +} + +type ManagedGatewayEndpointBinding = "match" | "mismatch" | "not-applicable" | "unknown"; + +export function classifyManagedGatewayEndpointBinding( + outputs: readonly string[], + expectedGatewayPort: number, +): Exclude { + for (const output of outputs) { + const match = stripAnsi(output).match(/^\s*Gateway endpoint:\s+(\S+)\s*$/m); + if (!match?.[1]) continue; + try { + const endpoint = new URL(match[1]); + const localHost = + endpoint.hostname === "127.0.0.1" || + endpoint.hostname === "localhost" || + endpoint.hostname === "[::1]"; + const endpointPort = + endpoint.port || + (endpoint.protocol === "https:" ? "443" : endpoint.protocol === "http:" ? "80" : ""); + return localHost && endpointPort === String(expectedGatewayPort) ? "match" : "mismatch"; + } catch { + return "mismatch"; + } + } + return "unknown"; +} + +function observeReuseState( + gatewayName: string, + gatewayPort: number, + openshell: string | null, + env: NodeJS.ProcessEnv, +): { endpointBinding: ManagedGatewayEndpointBinding; reuseState: GatewayReuseState | "unknown" } { + if (!openshell) return { endpointBinding: "not-applicable", reuseState: "missing" }; + + const status = captureReadonly([openshell, "status"], env); + const named = captureReadonly([openshell, "gateway", "info", "-g", gatewayName], env); + const active = captureReadonly([openshell, "gateway", "info"], env); + if ([status, named, active].some(({ exitCode, timedOut }) => timedOut || exitCode === null)) { + return { endpointBinding: "unknown", reuseState: "unknown" }; + } + + const statusOutput = combinedOutput(status); + let reuseState: GatewayReuseState | "unknown" = getGatewayReuseState( + statusOutput, + combinedOutput(named), + combinedOutput(active), + gatewayName, + ); + if (status.exitCode !== 0 && reuseState === "missing") { + reuseState = /\bNo active gateway\b|\bNo gateway metadata found\b/i.test(statusOutput) + ? "missing" + : "unknown"; + } + const liveManagedState = reuseState === "healthy" || reuseState === "active-unnamed"; + return { + reuseState, + endpointBinding: liveManagedState + ? classifyManagedGatewayEndpointBinding( + [combinedOutput(active), statusOutput, combinedOutput(named)], + gatewayPort, + ) + : "not-applicable", + }; +} + +function inspectLegacyCluster( + gatewayName: string, + gatewayPort: number, + openshell: string | null, + env: NodeJS.ProcessEnv, +): { active: boolean; imageRef: string | null } { + if (!openshell) return { active: false, imageRef: null }; + const status = captureReadonly([openshell, "status"], env); + const named = captureReadonly([openshell, "gateway", "info", "-g", gatewayName], env); + const active = captureReadonly([openshell, "gateway", "info"], env); + if ( + [status, named, active].some(({ exitCode, timedOut }) => timedOut || exitCode === null) || + !isGatewayHealthy( + combinedOutput(status), + combinedOutput(named), + combinedOutput(active), + gatewayName, + ) || + classifyManagedGatewayEndpointBinding( + [combinedOutput(active), combinedOutput(named)], + gatewayPort, + ) !== "match" + ) { + return { active: false, imageRef: null }; + } + + const containerName = getGatewayClusterContainerName(gatewayName); + const running = captureReadonly( + ["docker", "inspect", "--format", "{{.State.Running}}", containerName], + env, + ); + const ports = captureReadonly( + ["docker", "inspect", "--format", "{{json .NetworkSettings.Ports}}", containerName], + env, + ); + if ( + running.exitCode !== 0 || + running.timedOut || + running.stdout !== "true" || + ports.exitCode !== 0 || + ports.timedOut + ) { + return { active: false, imageRef: null }; + } + try { + const bindings = JSON.parse(ports.stdout) as Record< + string, + Array<{ HostPort?: string | number | null }> | null + >; + const expectedPort = String(gatewayPort); + const portBound = Object.values(bindings).some((values) => + values?.some(({ HostPort }) => String(HostPort ?? "").trim() === expectedPort), + ); + if (!portBound) return { active: false, imageRef: null }; + } catch { + return { active: false, imageRef: null }; + } + + const image = captureReadonly( + ["docker", "inspect", "--format", "{{.Config.Image}}", containerName], + env, + ); + return { + active: true, + imageRef: image.exitCode === 0 && !image.timedOut ? image.stdout || null : null, + }; +} + +function observeInstalledOpenshellVersion( + openshell: string | null, + env: NodeJS.ProcessEnv, +): string | null { + if (!openshell) return null; + const result = captureReadonly([openshell, "--version"], env); + if (result.exitCode !== 0 || result.timedOut) return null; + return parseVersionFromText(combinedOutput(result), `${openshell} --version`); +} + +export function classifyManagedGatewayPortConflict( + portAvailable: boolean, + listenerScan: { + pids: readonly number[]; + unverifiedPids: readonly number[]; + complete: boolean; + }, + reuseState: GatewayReuseState | "unknown", + legacyClusterBound = false, + endpointBinding: ManagedGatewayEndpointBinding = "not-applicable", +): GatewayPortConflictState { + const listenerCount = listenerScan.pids.length + listenerScan.unverifiedPids.length; + if (listenerCount > 1) return "multiple-owners"; + const liveManagedState = reuseState === "healthy" || reuseState === "active-unnamed"; + if (liveManagedState && endpointBinding === "mismatch") return "owner-mismatch"; + if (liveManagedState && endpointBinding === "unknown") return "unknown"; + if (portAvailable) { + if (listenerCount > 0 || liveManagedState) return "unknown"; + return "none"; + } + if (reuseState === "foreign-active") return "owner-mismatch"; + if (legacyClusterBound) { + // Docker's running container plus the exact OpenShell endpoint and + // published host-port binding positively own this listener. A separately + // verified host gateway at the same time is an authority contradiction. + return listenerScan.pids.length > 0 ? "owner-mismatch" : "none"; + } + if (!listenerScan.complete) return "unknown"; + if (listenerScan.unverifiedPids.length > 0) return "owner-mismatch"; + const recognizedManagedState = + reuseState === "healthy" || + reuseState === "stale" || + reuseState === "active-unnamed" || + reuseState === "missing"; + return recognizedManagedState && listenerScan.pids.length === 1 ? "none" : "occupied"; +} + +export function classifyManagedGatewayVersionDrift( + portAvailable: boolean, + reuseState: GatewayReuseState | "unknown", + compatibility: GatewayVersionCompatibility | null, +): ManagedGatewayObservations["driftState"] { + const managedGatewayCanBeRunning = + reuseState === "healthy" || reuseState === "stale" || reuseState === "active-unnamed"; + if (portAvailable) { + return managedGatewayCanBeRunning && reuseState !== "stale" ? "unknown" : "not-detected"; + } + if (compatibility === "compatible") return "not-detected"; + if (compatibility === "drift") return "detected"; + if (reuseState === "foreign-active") return "not-detected"; + return "unknown"; +} + +export function classifyManagedGatewayVersionSource( + legacyClusterBound: boolean, + listenerScan: { pids: readonly number[]; unverifiedPids: readonly number[] }, + trustedBinaryPids: { has(pid: number): boolean }, +): GatewayVersionSource | null { + if (legacyClusterBound) return "legacy-cluster"; + if ( + listenerScan.pids.length === 1 && + listenerScan.unverifiedPids.length === 0 && + trustedBinaryPids.has(listenerScan.pids[0]) + ) { + return "host-process"; + } + return null; +} + +function gatewayPortConflictDetail( + gatewayPort: number, + portCheck: Awaited>, + state: GatewayPortConflictState, +): string | undefined { + if (state === "none") return undefined; + const owner = portCheck.process + ? `${portCheck.process}${portCheck.pid ? ` (PID ${portCheck.pid})` : ""}` + : "an unknown listener"; + const condition = + state === "multiple-owners" + ? "has multiple listeners" + : state === "unknown" + ? "could not be observed completely" + : `is occupied by ${owner}`; + return ( + `Gateway port ${gatewayPort} ${condition}. ` + + `Inspect port ${gatewayPort} and stop only its owning process before retrying.` + ); +} + +function rejectUnexpectedGatewayEffect(): never { + throw new Error("The public readiness probe cannot perform gateway lifecycle effects."); +} + +/** Bind the public collector to local, read-only production probes. */ +export function createProductionGatewayReadinessDependencies( + options: ProductionGatewayReadinessOptions = {}, +): GatewayReadinessDependencies { + const gatewayPort = options.gatewayPort?.() ?? getConfiguredGatewayPort(); + const gatewayName = options.gatewayName?.() ?? resolveDockerDriverGatewayName(gatewayPort); + const probeEnv = buildGatewayReadinessProbeEnv(process.env, { + gatewayName, + localTlsDir: resolveManagedGatewayProbeTlsDir(gatewayPort, process.env), + }); + const openshellBin = resolveTrustedOpenshellBinary(probeEnv); + const trustedGatewayBin = resolveTrustedGatewayBinary(openshellBin); + const trustedVersionBinaryByPid = new Map(); + + function observeDirectGatewayBinary(pid: number): string | null { + if (process.platform !== "linux" || !trustedGatewayBin) return null; + const generationBefore = readLinuxProcessStartTime(pid); + const executableBefore = readLinuxProcessExecutable(pid); + const exactTrustedBinary = isDockerDriverGatewayProcessIdentity({ + pid, + gatewayBin: trustedGatewayBin, + captureProcessArgs: (candidatePid) => { + const result = captureReadonly(["ps", "-p", String(candidatePid), "-o", "args="], probeEnv); + return result.exitCode === 0 ? result.stdout : ""; + }, + processIdentityMatchesGatewayBinary: (identity) => + gatewayProcessIdentityMatchesTrustedBinary( + identity, + trustedGatewayBin, + gatewayName, + gatewayPort, + executableBefore, + process.platform, + ), + requireDockerDriverEnv: true, + hasDockerDriverGatewayEnv: (candidatePid) => + hasDockerDriverGatewayEnvironment( + readDockerDriverGatewayProcessEnvironment(candidatePid), + getDockerDriverGatewayEndpoint(gatewayPort), + ), + }); + const executableAfter = readLinuxProcessExecutable(pid); + const generationAfter = readLinuxProcessStartTime(pid); + return exactTrustedBinary && + generationBefore !== null && + generationAfter === generationBefore && + gatewayExecutableSamplesMatchTrustedBinary( + executableBefore, + executableAfter, + trustedGatewayBin, + ) + ? trustedGatewayBin + : null; + } + + function observePackagedServiceGatewayBinary(pid: number): string | null { + if (process.platform !== "linux" && process.platform !== "darwin") return null; + const serviceBefore = getTrustedActiveOpenShellGatewayUserServiceIdentity({ + env: probeEnv, + suppressUnsupportedVersionWarning: true, + }); + if (serviceBefore?.pid !== pid || !serviceBefore.executablePath) return null; + const generationBefore = process.platform === "linux" ? readLinuxProcessStartTime(pid) : null; + const executableBefore = + process.platform === "linux" + ? readLinuxProcessExecutable(pid) + : readDarwinProcessExecutable(pid, probeEnv); + const serviceAfter = getTrustedActiveOpenShellGatewayUserServiceIdentity({ + env: probeEnv, + suppressUnsupportedVersionWarning: true, + }); + // Bracket the complete service-identity probe so PID reuse or re-exec + // cannot preserve a stale trusted executable sample. + const executableAfter = + process.platform === "linux" + ? readLinuxProcessExecutable(pid) + : readDarwinProcessExecutable(pid, probeEnv); + const generationAfter = process.platform === "linux" ? readLinuxProcessStartTime(pid) : null; + const expected = normalizeExecutablePath(serviceBefore.executablePath); + const confirmed = serviceAfter?.executablePath + ? normalizeExecutablePath(serviceAfter.executablePath) + : null; + const stableGeneration = + process.platform !== "linux" || + (generationBefore !== null && generationAfter === generationBefore); + if ( + serviceAfter?.pid !== pid || + !expected || + confirmed !== expected || + !stableGeneration || + !gatewayExecutableSamplesMatchTrustedBinary(executableBefore, executableAfter, expected) + ) { + return null; + } + return expected; + } + + const listenerHelpers = createDockerDriverGatewayPortListenerHelpers({ + gatewayPort, + runCaptureEx: (args) => captureReadonly(args, probeEnv), + isPidAlive, + isDockerDriverGatewayProcess: (pid) => { + const trustedBinary = + observeDirectGatewayBinary(pid) ?? observePackagedServiceGatewayBinary(pid); + if (!trustedBinary) return false; + trustedVersionBinaryByPid.set(pid, trustedBinary); + return true; + }, + }); + const checkGatewayPortAvailable = () => + checkPortAvailable(gatewayPort, { + ...getGatewayPortCheckOptions(), + // Public readiness is observation-only. The independent bind probe tells + // us whether the port is occupied, and the unprivileged listener scan + // below supplies any ownership evidence available to this user. If that + // scan cannot see the listener, classification remains unknown and fails + // closed instead of attempting `sudo -n lsof`. + skipLsof: true, + }); + const runtime = createGatewayHostRuntime({ + applyOverlayfsAutoFix: () => null, + checkGatewayPortAvailable, + gatewayName: () => gatewayName, + gatewayPort: () => gatewayPort, + getGatewayPortListenerRawScan: (portCheck, options) => + listenerHelpers.getGatewayPortListenerRawScan(portCheck, options), + getInstalledOpenshellVersion: () => null, + hasOpenShellGatewayUserService: () => + hasOpenShellGatewayUserService({ + env: probeEnv, + suppressUnsupportedVersionWarning: true, + }), + loadGatewayManagementDeclaration, + recordHttpReadinessTrace: false, + clientProbeEnv: probeEnv, + resolveOpenShellGatewayBinary: () => null, + runCaptureOpenshell: rejectUnexpectedGatewayEffect, + runOpenshell: rejectUnexpectedGatewayEffect, + waitForGatewayHttpReady: async () => false, + supervisorProbeEnv: probeEnv, + }); + + async function observeManagedGateway(): Promise { + const { endpointBinding, reuseState } = observeReuseState( + gatewayName, + gatewayPort, + openshellBin, + probeEnv, + ); + const portCheck = await checkGatewayPortAvailable(); + trustedVersionBinaryByPid.clear(); + const listenerScan = listenerHelpers.getDockerDriverGatewayPortListenerScan(portCheck, { + gatewayBin: trustedGatewayBin, + }); + const managedGatewayCanBeRunning = + reuseState === "healthy" || reuseState === "stale" || reuseState === "active-unnamed"; + let legacyClusterBound = false; + let legacyClusterImageRef: string | null = null; + if (!portCheck.ok && managedGatewayCanBeRunning) { + try { + if (options.isLegacyClusterBound) { + legacyClusterBound = options.isLegacyClusterBound(); + } else { + const legacyCluster = inspectLegacyCluster( + gatewayName, + gatewayPort, + openshellBin, + probeEnv, + ); + legacyClusterBound = legacyCluster.active; + legacyClusterImageRef = legacyCluster.imageRef; + } + } catch { + legacyClusterBound = false; + } + } + let compatibility: GatewayVersionCompatibility | null = null; + if (!portCheck.ok) { + const source = classifyManagedGatewayVersionSource( + legacyClusterBound, + listenerScan, + trustedVersionBinaryByPid, + ); + try { + if (source) { + const hostProcessPid = source === "host-process" ? listenerScan.pids[0] : null; + const installedVersion = observeInstalledOpenshellVersion(openshellBin, probeEnv); + compatibility = + options.observeVersionCompatibility?.(source, hostProcessPid) ?? + observeOpenShellGatewayVersionCompatibility({ + gatewayName, + source, + deps: + source === "legacy-cluster" + ? { + getInstalledOpenshellVersion: () => installedVersion, + getGatewayClusterImageRef: () => legacyClusterImageRef, + isGatewayClusterActive: () => true, + } + : { + getInstalledOpenshellVersion: () => installedVersion, + getGatewayClusterImageRef: () => null, + getHostProcessGatewayRuntime: () => + hostProcessPid === null + ? null + : getTrustedHostProcessGatewayRuntime( + trustedVersionBinaryByPid.get(hostProcessPid) ?? null, + probeEnv, + ), + }, + }); + } + } catch { + compatibility = "unknown"; + } + } + const driftState = classifyManagedGatewayVersionDrift(portCheck.ok, reuseState, compatibility); + const portConflictState = classifyManagedGatewayPortConflict( + portCheck.ok, + listenerScan, + reuseState, + legacyClusterBound, + endpointBinding, + ); + return { + reuseState, + driftState, + portConflictState, + portConflictDetail: gatewayPortConflictDetail(gatewayPort, portCheck, portConflictState), + }; + } + + return { + resolveOwner: options.resolveOwner ?? runtime.getGatewayOwner, + probeAttachment: options.probeAttachment ?? runtime.probeGatewayAttachment, + observeManagedGateway, + }; +} diff --git a/src/lib/readiness/gateway.test.ts b/src/lib/readiness/gateway.test.ts new file mode 100644 index 00000000000..6091c1b8015 --- /dev/null +++ b/src/lib/readiness/gateway.test.ts @@ -0,0 +1,265 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; +import { + type GatewayAttachmentProbe, + type GatewayOwner, + GatewayOwnershipError, +} from "../onboard/gateway-ownership"; +import { + collectGatewayObservations, + createGatewayReadinessProjection, + type GatewayReadinessDependencies, + projectGatewayReadiness, +} from "./gateway"; + +const NOW = new Date("2026-08-07T12:00:00.000Z"); + +function managedOwner(): GatewayOwner { + return { + gatewayName: "nemoclaw", + gatewayPort: 8080, + mode: "nemoclaw-managed", + source: "standalone", + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], + }; +} + +function externalOwner(): GatewayOwner { + return { + gatewayName: "nemoclaw", + gatewayPort: 8080, + mode: "externally-supervised", + source: "declared", + endpoint: "https://127.0.0.1:8080", + stateDir: "/var/lib/private-gateway-state", + supervisor: { + kind: "systemd-system", + serviceName: "openshell-gateway.service", + execPath: "/opt/platform/gatewayd", + }, + requiredCapabilities: ["gateway.health"], + }; +} + +function attachment(overrides: Partial = {}): GatewayAttachmentProbe { + return { + gatewayPort: 8080, + httpReady: true, + portOccupied: true, + listenerPids: [4242], + listenerScanComplete: true, + listenerStartTime: "12345", + supervisorActive: true, + listenerExecPath: "/opt/platform/gatewayd", + listenerSupervisorMatch: true, + ...overrides, + }; +} + +function dependencies(owner: GatewayOwner): GatewayReadinessDependencies { + return { + resolveOwner: vi.fn(() => owner), + probeAttachment: vi.fn(async () => attachment()), + observeManagedGateway: vi.fn(async () => ({ + reuseState: "healthy" as const, + driftState: "not-detected" as const, + portConflictState: "none" as const, + })), + }; +} + +describe("gateway readiness projection (#7411)", () => { + it("rejects a snapshot made stale by a slow onboarding collection", async () => { + let currentTime = NOW.getTime(); + const deps = dependencies(managedOwner()); + vi.mocked(deps.observeManagedGateway).mockImplementationOnce(async () => { + currentTime += 30_001; + return { + reuseState: "healthy", + driftState: "not-detected", + portConflictState: "none", + }; + }); + + const projection = await createGatewayReadinessProjection(deps, { + now: () => new Date(currentTime), + }); + + expect(projection.capabilities.every(({ state }) => state === "unknown")).toBe(true); + expect(projection.evidence).toContainEqual( + expect.objectContaining({ id: "gateway.probe.stale" }), + ); + }); + + it("collects managed reuse, drift, and port ownership without attachment effects", async () => { + const deps = dependencies(managedOwner()); + const snapshot = await collectGatewayObservations(deps, { now: () => NOW }); + const projection = projectGatewayReadiness(snapshot, { now: () => NOW }); + + expect(deps.observeManagedGateway).toHaveBeenCalledOnce(); + expect(deps.probeAttachment).not.toHaveBeenCalled(); + expect(projection.observations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: "gateway.management.mode", + state: "present", + value: "nemoclaw-managed", + }), + expect.objectContaining({ id: "gateway.reuse", value: "healthy" }), + expect.objectContaining({ id: "gateway.version_drift", value: "not-detected" }), + expect.objectContaining({ id: "gateway.port_conflict", value: "none" }), + ]), + ); + expect(projection.capabilities.every(({ state }) => state === "present")).toBe(true); + expect(projection.findings).toEqual([]); + }); + + it("verifies an externally supervised attachment and does not run managed reuse", async () => { + const owner = externalOwner(); + const deps = dependencies(owner); + const snapshot = await collectGatewayObservations(deps, { now: () => NOW }); + const projection = projectGatewayReadiness(snapshot, { now: () => NOW }); + + expect(deps.probeAttachment).toHaveBeenCalledWith(owner); + expect(deps.observeManagedGateway).not.toHaveBeenCalled(); + expect(projection.observations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ id: "gateway.attachment", value: "verified" }), + expect.objectContaining({ id: "gateway.reuse", value: "not-applicable" }), + ]), + ); + expect(projection.findings).toEqual([]); + expect(JSON.stringify(projection)).not.toContain(owner.stateDir); + }); + + it.each([ + [{ listenerPids: [4242, 4343] }, "gateway.ownership.multiple", "multiple-owners"], + [{ listenerSupervisorMatch: false }, "gateway.ownership.mismatch", "owner-mismatch"], + ] as const)("rejects ambiguous external ownership before any managed operation", async (probeOverrides, findingId, conflictState) => { + const owner = externalOwner(); + const deps = dependencies(owner); + vi.mocked(deps.probeAttachment).mockResolvedValueOnce(attachment(probeOverrides)); + + const projection = projectGatewayReadiness( + await collectGatewayObservations(deps, { now: () => NOW }), + { now: () => NOW }, + ); + + expect(deps.observeManagedGateway).not.toHaveBeenCalled(); + expect(projection.findings).toContainEqual( + expect.objectContaining({ id: findingId, severity: "blocking" }), + ); + expect(projection.observations).toContainEqual( + expect.objectContaining({ id: "gateway.port_conflict", value: conflictState }), + ); + expect(projection.capabilities).toContainEqual( + expect.objectContaining({ id: "gateway.attachment.valid", state: "absent" }), + ); + }); + + it("redacts probe failures and omits private gateway state", async () => { + const token = `nvapi-${"a".repeat(24)}`; + const owner = externalOwner(); + const deps = dependencies(owner); + vi.mocked(deps.probeAttachment).mockRejectedValueOnce( + new GatewayOwnershipError( + "gateway_unreachable", + `Authorization: Bearer ${token} request failed at ${owner.stateDir}/tls/client/tls.key`, + owner, + ), + ); + + const projection = projectGatewayReadiness( + await collectGatewayObservations(deps, { now: () => NOW }), + { now: () => NOW }, + ); + const serialized = JSON.stringify(projection); + + expect(serialized).not.toContain(token); + expect(serialized).not.toContain(owner.stateDir); + expect(serialized).toContain(""); + expect(serialized).toContain(""); + }); + + it("fails closed when lifecycle authority cannot be resolved", async () => { + const deps = dependencies(managedOwner()); + vi.mocked(deps.resolveOwner).mockImplementationOnce(() => { + throw new Error("invalid declaration"); + }); + + const projection = projectGatewayReadiness( + await collectGatewayObservations(deps, { now: () => NOW }), + { now: () => NOW }, + ); + + expect(deps.probeAttachment).not.toHaveBeenCalled(); + expect(deps.observeManagedGateway).not.toHaveBeenCalled(); + expect(projection.findings).toContainEqual( + expect.objectContaining({ id: "gateway.authority.invalid", severity: "blocking" }), + ); + expect(projection.capabilities.every(({ state }) => state === "unknown")).toBe(true); + }); + + it("rejects observations older than 30 seconds unless collection marked them reusable", async () => { + const snapshot = await collectGatewayObservations(dependencies(managedOwner()), { + now: () => NOW, + }); + const stale = { ...snapshot, observedAt: "2026-08-07T11:00:00.000Z", reusable: false }; + + const projection = projectGatewayReadiness(stale, { now: () => NOW }); + + expect(projection.capabilities.every(({ state }) => state === "unknown")).toBe(true); + expect(projection.evidence).toContainEqual( + expect.objectContaining({ id: "gateway.probe.stale" }), + ); + }); + + it("carries redacted managed port diagnostics as bounded evidence", async () => { + const deps = dependencies(managedOwner()); + const token = `nvapi-${"b".repeat(24)}`; + vi.mocked(deps.observeManagedGateway).mockResolvedValueOnce({ + reuseState: "stale", + driftState: "not-detected", + portConflictState: "owner-mismatch", + portConflictDetail: `Port 8080 belongs to Bearer ${token}`, + }); + + const projection = projectGatewayReadiness( + await collectGatewayObservations(deps, { now: () => NOW }), + { now: () => NOW }, + ); + const detail = projection.evidence.find(({ id }) => id === "gateway.port.conflict"); + + expect(detail?.summary).toContain(""); + expect(detail?.summary).not.toContain(token); + expect(projection.findings).toContainEqual( + expect.objectContaining({ id: "gateway.port.owner_mismatch", severity: "blocking" }), + ); + }); + + it("escapes terminal controls before onboarding can print gateway evidence", async () => { + const deps = dependencies(managedOwner()); + vi.mocked(deps.observeManagedGateway).mockResolvedValueOnce({ + reuseState: "stale", + driftState: "not-detected", + portConflictState: "owner-mismatch", + portConflictDetail: "forged\n\u001b[31merror \u202efailure", + }); + + const projection = projectGatewayReadiness( + await collectGatewayObservations(deps, { now: () => NOW }), + { now: () => NOW }, + ); + const detail = projection.evidence.find(({ id }) => id === "gateway.port.conflict"); + + expect(detail?.summary).toBe("forged\\u000a\\u001b[31merror \\u202efailure"); + expect(detail?.summary).not.toMatch( + /[\u0000-\u001f\u007f-\u009f\u061c\u200e\u200f\u2028-\u202e\u2066-\u2069]/u, + ); + }); +}); diff --git a/src/lib/readiness/gateway.ts b/src/lib/readiness/gateway.ts new file mode 100644 index 00000000000..f08900f12fa --- /dev/null +++ b/src/lib/readiness/gateway.ts @@ -0,0 +1,547 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + describeGatewayOwner, + evaluateGatewayAttachment, + type GatewayAttachmentProbe, + type GatewayOwner, + type GatewayOwnerDescription, + GatewayOwnershipError, + type GatewayOwnershipFailureCode, + isExternallySupervised, +} from "../onboard/gateway-ownership"; +import type { GatewayReuseState } from "../state/gateway"; +import { sanitizeReadinessText } from "./sanitize"; +import type { + EvidenceScalar, + ReadinessCapability, + ReadinessEvidence, + ReadinessFinding, + ReadinessObservation, + ReadinessState, +} from "./types"; + +const DEFAULT_MAX_AGE_MS = 30_000; +const MAX_REPORT_TEXT_LENGTH = 1024; +const projectionSnapshots = new WeakMap< + GatewayReadinessProjection, + Readonly +>(); + +export type GatewayAttachmentState = "verified" | "rejected" | "not-applicable" | "unknown"; +export type GatewayDriftState = "detected" | "not-detected" | "not-applicable" | "unknown"; +export type GatewayPortConflictState = + | "none" + | "occupied" + | "multiple-owners" + | "owner-mismatch" + | "unknown"; +export type GatewayReuseObservation = GatewayReuseState | "not-applicable" | "unknown"; + +export interface ManagedGatewayObservations { + reuseState: GatewayReuseState | "unknown"; + driftState: Exclude; + portConflictState: GatewayPortConflictState; + portConflictDetail?: string; +} + +export interface GatewayReadinessDependencies { + resolveOwner(): GatewayOwner; + probeAttachment(owner: GatewayOwner): Promise; + observeManagedGateway(owner: GatewayOwner): Promise; +} + +export interface GatewayObservations { + owner: GatewayOwnerDescription; + attachmentState: GatewayAttachmentState; + attachmentFailureCode?: GatewayOwnershipFailureCode; + attachmentFailure?: string; + reuseState: GatewayReuseObservation; + driftState: GatewayDriftState; + portConflictState: GatewayPortConflictState; + portConflictDetail?: string; +} + +export interface GatewayObservationSnapshot { + observedAt: string; + observations?: Readonly; + failure?: string; + authorityFailure?: boolean; + reusable?: boolean; +} + +export interface CollectGatewayObservationsOptions { + now?: () => Date; +} + +export interface ProjectGatewayReadinessOptions { + now?: () => Date; + maxObservationAgeMs?: number; +} + +export type CreateGatewayReadinessProjectionOptions = CollectGatewayObservationsOptions & + ProjectGatewayReadinessOptions; + +export interface GatewayReadinessProjection { + observations: ReadinessObservation[]; + capabilities: ReadinessCapability[]; + findings: ReadinessFinding[]; + evidence: ReadinessEvidence[]; +} + +function safeReportText(value: string): string { + return sanitizeReadinessText(value, MAX_REPORT_TEXT_LENGTH); +} + +function safeOwnerFailureText(value: string, owner: GatewayOwner): string { + const withoutPrivateState = owner.stateDir + ? value.split(owner.stateDir).join("") + : value; + return safeReportText(withoutPrivateState); +} + +function conflictFromAttachmentFailure( + code: GatewayOwnershipFailureCode, +): GatewayPortConflictState { + if (code === "multiple_owners") return "multiple-owners"; + if (code === "identity_mismatch") return "owner-mismatch"; + return "unknown"; +} + +function rejectedAttachment( + owner: GatewayOwner, + code: GatewayOwnershipFailureCode, + message: string, +): GatewayObservations { + return { + owner: describeGatewayOwner(owner), + attachmentState: "rejected", + attachmentFailureCode: code, + attachmentFailure: safeOwnerFailureText(message, owner), + reuseState: "not-applicable", + driftState: "not-applicable", + portConflictState: conflictFromAttachmentFailure(code), + }; +} + +export async function collectGatewayObservations( + deps: GatewayReadinessDependencies, + options: CollectGatewayObservationsOptions = {}, +): Promise { + const observedAt = (options.now ?? (() => new Date()))().toISOString(); + let owner: GatewayOwner; + try { + owner = deps.resolveOwner(); + } catch { + return { + observedAt, + failure: "Gateway lifecycle authority could not be resolved before lifecycle effects.", + authorityFailure: true, + reusable: false, + }; + } + + if (isExternallySupervised(owner)) { + try { + const result = evaluateGatewayAttachment(owner, await deps.probeAttachment(owner)); + if (!result.ok) { + return { + observedAt, + observations: rejectedAttachment(owner, result.code, result.message), + reusable: false, + }; + } + return { + observedAt, + observations: { + owner: describeGatewayOwner(owner), + attachmentState: "verified", + reuseState: "not-applicable", + driftState: "not-applicable", + portConflictState: "none", + }, + reusable: false, + }; + } catch (error) { + if (error instanceof GatewayOwnershipError) { + return { + observedAt, + observations: rejectedAttachment(owner, error.code, error.message), + reusable: false, + }; + } + return { + observedAt, + observations: { + owner: describeGatewayOwner(owner), + attachmentState: "unknown", + reuseState: "not-applicable", + driftState: "not-applicable", + portConflictState: "unknown", + }, + failure: "The externally supervised gateway attachment probe failed safely.", + reusable: false, + }; + } + } + + try { + const managed = await deps.observeManagedGateway(owner); + return { + observedAt, + observations: { + owner: describeGatewayOwner(owner), + attachmentState: "not-applicable", + reuseState: managed.reuseState, + driftState: managed.driftState, + portConflictState: managed.portConflictState, + portConflictDetail: managed.portConflictDetail + ? safeReportText(managed.portConflictDetail) + : undefined, + }, + reusable: false, + }; + } catch { + return { + observedAt, + observations: { + owner: describeGatewayOwner(owner), + attachmentState: "not-applicable", + reuseState: "unknown", + driftState: "unknown", + portConflictState: "unknown", + }, + failure: "Managed gateway observations could not be collected safely.", + reusable: false, + }; + } +} + +function observation( + id: string, + value: EvidenceScalar | undefined, + evidenceIds: readonly string[], +): ReadinessObservation { + return value === undefined || value === "unknown" + ? { id, state: "unknown", evidenceIds } + : { id, state: "present", value, evidenceIds }; +} + +function capability( + id: string, + state: ReadinessState, + evidenceIds: readonly string[], +): ReadinessCapability { + return { id, state, evidenceIds }; +} + +function finding( + id: string, + severity: ReadinessFinding["severity"], + summary: string, + capabilityIds: readonly string[], + evidenceIds: readonly string[], +): ReadinessFinding { + return { id, severity, summary, capabilityIds, evidenceIds }; +} + +const ATTACHMENT_FINDING_IDS: Record = { + external_supervision_forbids_effect: "gateway.attachment.effect_forbidden", + gateway_unreachable: "gateway.attachment.unreachable", + supervisor_inactive: "gateway.supervisor.inactive", + identity_mismatch: "gateway.ownership.mismatch", + unknown_listener: "gateway.ownership.unverified", + multiple_owners: "gateway.ownership.multiple", + endpoint_port_mismatch: "gateway.endpoint.port_mismatch", + gateway_registration_failed: "gateway.registration.failed", + capability_unsupported: "gateway.capability.unsupported", +}; + +function unknownProjection( + snapshot: Readonly, + stale = false, +): GatewayReadinessProjection { + const evidenceIds = [ + ...(snapshot.failure ? ["gateway.probe.failure"] : []), + ...(stale ? ["gateway.probe.stale"] : []), + ]; + const capabilityIds = [ + "gateway.authority.resolved", + "gateway.attachment.valid", + "gateway.reuse.ready", + "gateway.version.compatible", + "gateway.port.uncontested", + ]; + const projection: GatewayReadinessProjection = { + observations: [ + "gateway.management.mode", + "gateway.owner.name", + "gateway.owner.source", + "gateway.owner.port", + "gateway.attachment", + "gateway.reuse", + "gateway.version_drift", + "gateway.port_conflict", + ].map((id) => ({ id, state: "unknown", evidenceIds })), + capabilities: capabilityIds.map((id) => ({ id, state: "unknown", evidenceIds })), + findings: [ + finding( + snapshot.authorityFailure ? "gateway.authority.invalid" : "gateway.probe.inconclusive", + snapshot.authorityFailure ? "blocking" : "warning", + snapshot.authorityFailure + ? "Gateway lifecycle authority could not be resolved safely." + : "Gateway observations could not be collected safely.", + snapshot.authorityFailure ? ["gateway.authority.resolved"] : capabilityIds, + evidenceIds, + ), + ], + evidence: [ + ...(snapshot.failure + ? [{ id: "gateway.probe.failure", summary: safeReportText(snapshot.failure) }] + : []), + ...(stale + ? [ + { + id: "gateway.probe.stale", + summary: "Gateway observations exceeded their safe reuse window.", + }, + ] + : []), + ], + }; + projectionSnapshots.set(projection, snapshot); + return projection; +} + +function ownerEvidence(owner: GatewayOwnerDescription): ReadinessEvidence { + return { + id: "gateway.owner", + summary: "Resolved gateway lifecycle authority.", + details: { + gatewayName: safeReportText(owner.gatewayName), + gatewayPort: owner.gatewayPort, + mode: owner.mode, + source: owner.source, + endpoint: owner.endpoint ? safeReportText(owner.endpoint) : null, + supervisorKind: owner.supervisor?.kind ?? null, + supervisorService: owner.supervisor ? safeReportText(owner.supervisor.serviceName) : null, + supervisorExecPath: owner.supervisor ? safeReportText(owner.supervisor.execPath) : null, + requiredCapabilities: safeReportText(owner.requiredCapabilities.join(",")), + }, + }; +} + +/** Re-evaluate the original snapshot after another readiness collection. */ +export function refreshGatewayReadinessProjection( + projection: GatewayReadinessProjection, + options: ProjectGatewayReadinessOptions = {}, +): GatewayReadinessProjection { + const snapshot = projectionSnapshots.get(projection); + return snapshot ? projectGatewayReadiness(snapshot, options) : projection; +} + +export function projectGatewayReadiness( + snapshot: Readonly, + options: ProjectGatewayReadinessOptions = {}, +): GatewayReadinessProjection { + const now = (options.now ?? (() => new Date()))(); + const age = now.getTime() - Date.parse(snapshot.observedAt); + const stale = + !Number.isFinite(age) || age < 0 || age > (options.maxObservationAgeMs ?? DEFAULT_MAX_AGE_MS); + if (stale && snapshot.reusable !== true) return unknownProjection(snapshot, true); + + const gateway = snapshot.observations; + if (!gateway) return unknownProjection(snapshot); + + const evidence: ReadinessEvidence[] = [ownerEvidence(gateway.owner)]; + const evidenceIds = ["gateway.owner"]; + if (snapshot.failure) { + evidence.push({ id: "gateway.probe.failure", summary: safeReportText(snapshot.failure) }); + evidenceIds.push("gateway.probe.failure"); + } + if (gateway.attachmentFailure) { + evidence.push({ + id: "gateway.attachment.failure", + summary: safeReportText(gateway.attachmentFailure), + }); + evidenceIds.push("gateway.attachment.failure"); + } + if (gateway.portConflictDetail) { + evidence.push({ + id: "gateway.port.conflict", + summary: safeReportText(gateway.portConflictDetail), + }); + evidenceIds.push("gateway.port.conflict"); + } + + const external = gateway.owner.mode === "externally-supervised"; + const attachmentState: ReadinessState = + gateway.attachmentState === "unknown" + ? "unknown" + : gateway.attachmentState === "rejected" + ? "absent" + : "present"; + const reuseState: ReadinessState = + gateway.reuseState === "unknown" + ? "unknown" + : gateway.reuseState === "healthy" || + gateway.reuseState === "missing" || + gateway.reuseState === "not-applicable" + ? "present" + : "absent"; + const versionState: ReadinessState = + gateway.driftState === "unknown" + ? "unknown" + : gateway.driftState === "detected" + ? "absent" + : "present"; + const portState: ReadinessState = + gateway.portConflictState === "unknown" + ? "unknown" + : gateway.portConflictState === "none" + ? "present" + : "absent"; + + const findings: ReadinessFinding[] = []; + if (gateway.attachmentState === "rejected" && gateway.attachmentFailureCode) { + findings.push( + finding( + ATTACHMENT_FINDING_IDS[gateway.attachmentFailureCode], + "blocking", + "The externally supervised gateway attachment was rejected.", + ["gateway.attachment.valid", "gateway.port.uncontested"], + evidenceIds, + ), + ); + } else if (gateway.attachmentState === "unknown") { + findings.push( + finding( + "gateway.attachment.inconclusive", + "warning", + "The externally supervised gateway attachment could not be verified.", + ["gateway.attachment.valid", "gateway.port.uncontested"], + evidenceIds, + ), + ); + } + + if (!external) { + if (gateway.reuseState === "active-unnamed" || gateway.reuseState === "stale") { + findings.push( + finding( + `gateway.reuse.${gateway.reuseState.replace("-", "_")}`, + "warning", + "The managed gateway requires reconciliation before reuse.", + ["gateway.reuse.ready"], + evidenceIds, + ), + ); + } else if (gateway.reuseState === "foreign-active") { + findings.push( + finding( + "gateway.reuse.foreign_active", + "blocking", + "A different active gateway conflicts with the configured NemoClaw gateway.", + ["gateway.reuse.ready", "gateway.port.uncontested"], + evidenceIds, + ), + ); + } else if (gateway.reuseState === "unknown") { + findings.push( + finding( + "gateway.reuse.inconclusive", + "warning", + "Managed gateway reuse state could not be determined.", + ["gateway.reuse.ready"], + evidenceIds, + ), + ); + } + + if (gateway.driftState === "detected") { + findings.push( + finding( + "gateway.version.drift", + "blocking", + "The running gateway version does not match the installed OpenShell version.", + ["gateway.version.compatible"], + evidenceIds, + ), + ); + } else if (gateway.driftState === "unknown") { + findings.push( + finding( + "gateway.version.inconclusive", + "warning", + "Gateway version drift could not be determined.", + ["gateway.version.compatible"], + evidenceIds, + ), + ); + } + + if (gateway.portConflictState !== "none" && gateway.portConflictState !== "unknown") { + findings.push( + finding( + gateway.portConflictState === "multiple-owners" + ? "gateway.port.multiple_owners" + : gateway.portConflictState === "owner-mismatch" + ? "gateway.port.owner_mismatch" + : "gateway.port.occupied", + "blocking", + "The gateway port is held by an incompatible or ambiguous owner.", + ["gateway.port.uncontested"], + evidenceIds, + ), + ); + } else if (gateway.portConflictState === "unknown") { + findings.push( + finding( + "gateway.port.inconclusive", + "warning", + "Gateway port ownership could not be determined.", + ["gateway.port.uncontested"], + evidenceIds, + ), + ); + } + } + + const projection: GatewayReadinessProjection = { + observations: [ + observation("gateway.management.mode", gateway.owner.mode, evidenceIds), + observation("gateway.owner.name", gateway.owner.gatewayName, evidenceIds), + observation("gateway.owner.source", gateway.owner.source, evidenceIds), + observation("gateway.owner.port", gateway.owner.gatewayPort, evidenceIds), + observation("gateway.attachment", gateway.attachmentState, evidenceIds), + observation("gateway.reuse", gateway.reuseState, evidenceIds), + observation("gateway.version_drift", gateway.driftState, evidenceIds), + observation("gateway.port_conflict", gateway.portConflictState, evidenceIds), + ], + capabilities: [ + capability("gateway.authority.resolved", "present", evidenceIds), + capability("gateway.attachment.valid", attachmentState, evidenceIds), + capability("gateway.reuse.ready", reuseState, evidenceIds), + capability("gateway.version.compatible", versionState, evidenceIds), + capability("gateway.port.uncontested", portState, evidenceIds), + ], + findings, + evidence, + }; + projectionSnapshots.set(projection, snapshot); + return projection; +} + +/** Collect and project gateway facts against the live post-collection clock. */ +export async function createGatewayReadinessProjection( + deps: GatewayReadinessDependencies, + options: CreateGatewayReadinessProjectionOptions = {}, +): Promise { + const now = options.now ?? (() => new Date()); + const snapshot = await collectGatewayObservations(deps, { now }); + return projectGatewayReadiness(snapshot, { + now, + maxObservationAgeMs: options.maxObservationAgeMs, + }); +} diff --git a/src/lib/readiness/host-production.test.ts b/src/lib/readiness/host-production.test.ts new file mode 100644 index 00000000000..326d015a5f8 --- /dev/null +++ b/src/lib/readiness/host-production.test.ts @@ -0,0 +1,63 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, describe, expect, it, vi } from "vitest"; + +const subprocess = vi.hoisted(() => ({ spawnSync: vi.fn() })); + +vi.mock("node:child_process", async (importOriginal) => ({ + ...(await importOriginal()), + spawnSync: subprocess.spawnSync, +})); + +import { collectHostObservations } from "./host"; + +afterEach(() => { + vi.unstubAllEnvs(); + vi.restoreAllMocks(); +}); + +describe("production host readiness collection (#7411)", () => { + it("runs every child with a credential-free replacement environment", () => { + vi.stubEnv("OPENSHELL_GATEWAY_AUTH_TOKEN", "gateway-secret"); + vi.stubEnv("OPENSHELL_SANDBOX_TOKEN", "sandbox-secret"); + vi.stubEnv("XDG_API_TOKEN", "prefix-secret"); + subprocess.spawnSync.mockReturnValue({ + error: undefined, + status: 1, + stdout: "", + stderr: "not found", + }); + + collectHostObservations({ now: () => new Date("2026-08-07T12:00:00.000Z") }); + + expect(subprocess.spawnSync.mock.calls.length).toBeGreaterThan(0); + for (const [, , options] of subprocess.spawnSync.mock.calls) { + const env = options?.env as NodeJS.ProcessEnv | undefined; + expect(env).toBeDefined(); + expect(env?.OPENSHELL_GATEWAY_AUTH_TOKEN).toBeUndefined(); + expect(env?.OPENSHELL_SANDBOX_TOKEN).toBeUndefined(); + expect(env?.XDG_API_TOKEN).toBeUndefined(); + } + }); + + it("does not contact Docker through an unsupported configured endpoint", () => { + vi.stubEnv("DOCKER_HOST", "tcp://attacker.example:2375"); + subprocess.spawnSync.mockImplementation((command: string, args: readonly string[] = []) => + command === "sh" && args.at(-1) === "docker" + ? { error: undefined, status: 0, stdout: "/usr/bin/docker\n", stderr: "" } + : { error: undefined, status: 1, stdout: "", stderr: "not found" }, + ); + + const snapshot = collectHostObservations({ + now: () => new Date("2026-08-07T12:00:00.000Z"), + }); + + expect(snapshot.observations).toMatchObject({ + dockerHostInvalid: true, + dockerInstalled: true, + dockerReachable: false, + }); + expect(subprocess.spawnSync.mock.calls.some(([command]) => command === "docker")).toBe(false); + }); +}); diff --git a/src/lib/readiness/host.test.ts b/src/lib/readiness/host.test.ts index 71223d5ccc5..e04cd12f2a3 100644 --- a/src/lib/readiness/host.test.ts +++ b/src/lib/readiness/host.test.ts @@ -11,7 +11,7 @@ import type { HostAssessment } from "../onboard/preflight"; import { collectHostObservations, createHostReadinessReport, projectHostReadiness } from "./host"; const { detectGpu, detectNvidiaDriverVersion, detectNvidiaPlatform } = vi.hoisted(() => ({ - detectGpu: vi.fn<() => GpuDetection | null>(() => null), + detectGpu: vi.fn<(_deps?: unknown) => GpuDetection | null>(() => null), detectNvidiaDriverVersion: vi.fn<() => string | undefined>(() => undefined), detectNvidiaPlatform: vi.fn<() => NvidiaPlatform>(() => "linux"), })); @@ -51,6 +51,7 @@ function host(overrides: Partial = {}): HostAssessment { dockerDefaultCgroupnsMode: "private", dockerStorageDriver: "overlay2", dockerUsesContainerdSnapshotter: false, + dockerNvidiaRuntimeAvailable: true, dockerCpus: 8, dockerMemTotalBytes: 16 * 1024 ** 3, isContainerRuntimeUnderProvisioned: false, @@ -177,16 +178,73 @@ describe("host readiness projection (#7408)", () => { expect(findingIds(result)).toContain(findingId); }); + it("blocks a reachable but unsupported DOCKER_HOST before using daemon evidence (#7411)", () => { + const result = report({ dockerHostInvalid: true, dockerReachable: true }); + + expect(result.status).toBe("incompatible"); + expect(state(result, "host.docker.endpoint_supported")).toBe("absent"); + expect(state(result, "host.docker.daemon_reachable")).toBe("unknown"); + expect(findingIds(result)).toContain("host.docker.host_invalid"); + expect(result.observations.find(({ id }) => id === "host.docker.runtime")?.state).toBe( + "unknown", + ); + }); + + it("blocks an unsupported DOCKER_HOST in WSL (#7411)", () => { + const result = report({ + dockerHostInvalid: true, + dockerReachable: true, + isWsl: true, + runtime: "docker-desktop", + }); + + expect(state(result, "host.docker.endpoint_supported")).toBe("absent"); + expect(findingIds(result)).toContain("host.docker.host_invalid"); + }); + + it("classifies an unsupported runtime as a blocking public finding (#7411)", () => { + const result = report({ isUnsupportedRuntime: true, runtime: "podman" }); + + expect(result.status).toBe("incompatible"); + expect(result.exitCode).toBe(2); + expect(result.findings).toContainEqual( + expect.objectContaining({ + id: "host.docker.runtime_unsupported", + severity: "blocking", + }), + ); + }); + + it("classifies an unsupported platform as incompatible instead of supported (#7411)", () => { + const result = report({ platform: "darwin", runtime: "docker-desktop" }); + + expect(result.status).toBe("incompatible"); + expect(result.exitCode).toBe(2); + expect(result.findings).toContainEqual( + expect.objectContaining({ id: "host.platform.unsupported", severity: "blocking" }), + ); + }); + it.each([ - [{ cdiNvidiaGpuSpecMissing: true }, { detectHostGpuPlatform: () => "jetson" as const }], [ - { isWsl: true, runtime: "docker-desktop", cdiNvidiaGpuSpecStale: true }, + { cdiNvidiaGpuSpecMissing: true, nvidiaContainerToolkitInstalled: false }, + { detectHostGpuPlatform: () => "jetson" as const }, + ], + [ + { + isWsl: true, + runtime: "docker-desktop", + cdiNvidiaGpuSpecStale: true, + nvidiaContainerToolkitInstalled: false, + }, { wslDockerDesktopGpuProofPassed: true }, ], ] as const)("preserves CDI enforcement exclusions for %s", (overrides, collectionOptions) => { const result = report(overrides, collectionOptions); expect(state(result, "host.gpu.cdi_healthy")).toBe("present"); + expect(state(result, "host.gpu.container_toolkit_available")).toBe("present"); + expect(findingIds(result)).not.toContain("host.gpu.container_toolkit_missing"); expect(findingIds(result)).not.toContain("host.gpu.cdi_missing"); expect(findingIds(result)).not.toContain("host.gpu.cdi_stale"); expect(result.status).toBe("supported"); @@ -210,6 +268,49 @@ describe("host readiness projection (#7408)", () => { expect(result.status).toBe("supported"); }); + it("recognizes a Jetson GPU through the canonical detector without nvidia-smi", () => { + const result = createHostReadinessReport( + { nemoclawVersion: "0.1.0", sourceRevision: SOURCE_REVISION, now: () => NOW }, + { + assess: () => + host({ + hasNvidiaGpu: false, + nvidiaContainerToolkitInstalled: false, + cdiNvidiaGpuSpecMissing: true, + }), + architecture: "arm64", + collectPlatformIdentity: emptyPlatformIdentity, + detectGpu: () => ({ count: 1, platform: "jetson", type: "nvidia" }), + }, + ); + + expect(state(result, "host.gpu.nvidia_available")).toBe("present"); + expect(state(result, "host.gpu.container_toolkit_available")).toBe("present"); + expect(state(result, "host.gpu.cdi_healthy")).toBe("present"); + expect(findingIds(result)).not.toContain("host.gpu.container_toolkit_missing"); + expect(findingIds(result)).not.toContain("host.gpu.cdi_missing"); + }); + + it("blocks Jetson GPU admission when Docker lacks the NVIDIA runtime", () => { + const result = report( + { dockerNvidiaRuntimeAvailable: false }, + { detectHostGpuPlatform: () => "jetson" }, + ); + + expect(state(result, "host.gpu.container_toolkit_available")).toBe("absent"); + expect(result.observations).toContainEqual({ + id: "host.gpu.nvidia_runtime", + state: "absent", + value: false, + }); + expect(result.findings).toContainEqual( + expect.objectContaining({ + id: "host.gpu.nvidia_runtime_missing", + severity: "blocking", + }), + ); + }); + it("projects bounded GPU count and driver observations for managed serving", () => { const result = createHostReadinessReport( { nemoclawVersion: "0.1.0", sourceRevision: SOURCE_REVISION, now: () => NOW }, @@ -234,16 +335,7 @@ describe("host readiness projection (#7408)", () => { ); }); - it("uses the canonical WSL Docker Desktop GPU proof in the default report creator", () => { - detectGpu.mockReturnValue({ - type: "nvidia", - count: 1, - totalMemoryMB: 32_768, - perGpuMB: 32_768, - nimCapable: true, - wslDockerDesktopGpuProofPassed: true, - }); - + it("keeps the default WSL Docker Desktop GPU observation container-free", () => { const result = createHostReadinessReport( { nemoclawVersion: "0.1.0", sourceRevision: SOURCE_REVISION, now: () => NOW }, { @@ -253,8 +345,13 @@ describe("host readiness projection (#7408)", () => { }, ); - expect(detectGpu).toHaveBeenCalledOnce(); - expect(state(result, "host.platform.wsl_gpu_passthrough")).toBe("present"); + expect(detectGpu).toHaveBeenCalledWith( + expect.objectContaining({ + proveArm64WslDockerDesktopGpu: null, + runCaptureImpl: expect.any(Function), + }), + ); + expect(state(result, "host.platform.wsl_gpu_passthrough")).toBe("unknown"); }); it("skips the WSL Docker Desktop GPU proof when Docker is unreachable", () => { diff --git a/src/lib/readiness/host.ts b/src/lib/readiness/host.ts index fb1a4b35cc5..dd6c563d13e 100644 --- a/src/lib/readiness/host.ts +++ b/src/lib/readiness/host.ts @@ -10,13 +10,15 @@ import { } from "../inference/nim.js"; import type { HostAssessment } from "../onboard/preflight.js"; import { assessHost } from "../onboard/preflight.js"; -import { redactFull } from "../security/redact.js"; +import { resolveOpenshell } from "./openshell-resolver.js"; import { type CollectPlatformIdentityOptions, collectPlatformIdentity, type PlatformIdentity, projectPlatformQualification, } from "./platform-qualification.js"; +import { buildSystemReadinessProbeEnv, createSystemReadinessCapture } from "./probe-env.js"; +import { sanitizeReadinessText } from "./sanitize.js"; import { type EvidenceScalar, type FindingSeverity, @@ -39,11 +41,13 @@ export interface HostObservations { isHeadlessLikely: boolean; dockerInstalled: boolean; dockerReachable: boolean; + dockerHostInvalid: boolean; runtime: string; dockerCgroupVersion?: string; dockerDefaultCgroupnsMode?: string; dockerStorageDriver?: string; dockerUsesContainerdSnapshotter?: boolean; + dockerNvidiaRuntimeAvailable?: boolean; dockerCpus?: number; dockerMemTotalBytes?: number; isContainerRuntimeUnderProvisioned: boolean; @@ -73,7 +77,10 @@ export interface HostObservationSnapshot { export interface CollectHostObservationsOptions { assess?: () => HostAssessment; architecture?: string; - detectGpu?: () => Pick | null; + detectGpu?: () => + | (Pick & + Partial>) + | null; detectNvidiaDriverVersion?: () => string | undefined; detectHostGpuPlatform?: () => NvidiaPlatform; wslDockerDesktopGpuProofPassed?: boolean; @@ -90,12 +97,13 @@ export interface CreateHostReadinessReportOptions { } function safeReportText(value: string): string { - return redactFull(value).slice(0, MAX_REPORT_TEXT_LENGTH); + return sanitizeReadinessText(value, MAX_REPORT_TEXT_LENGTH); } function adaptHostAssessment( host: Readonly, architecture: string, + hasNvidiaGpu: boolean, hostGpuPlatform?: NvidiaPlatform, nvidiaGpuCount?: number, nvidiaDriverVersion?: string, @@ -109,11 +117,13 @@ function adaptHostAssessment( isHeadlessLikely: host.isHeadlessLikely, dockerInstalled: host.dockerInstalled, dockerReachable: host.dockerReachable, + dockerHostInvalid: host.dockerHostInvalid === true, runtime: host.runtime, dockerCgroupVersion: host.dockerCgroupVersion, dockerDefaultCgroupnsMode: host.dockerDefaultCgroupnsMode, dockerStorageDriver: host.dockerStorageDriver, dockerUsesContainerdSnapshotter: host.dockerUsesContainerdSnapshotter, + dockerNvidiaRuntimeAvailable: host.dockerNvidiaRuntimeAvailable, dockerCpus: host.dockerCpus, dockerMemTotalBytes: host.dockerMemTotalBytes, isContainerRuntimeUnderProvisioned: host.isContainerRuntimeUnderProvisioned, @@ -121,7 +131,7 @@ function adaptHostAssessment( isUnsupportedRuntime: host.isUnsupportedRuntime, nodeInstalled: host.nodeInstalled, openshellInstalled: host.openshellInstalled, - hasNvidiaGpu: host.hasNvidiaGpu, + hasNvidiaGpu, nvidiaGpuCount, nvidiaDriverVersion, hostGpuPlatform, @@ -141,17 +151,44 @@ export function collectHostObservations( ): HostObservationSnapshot { const observedAt = (options.now ?? (() => new Date()))().toISOString(); try { - const assessment = (options.assess ?? assessHost)(); + const probeEnv = buildSystemReadinessProbeEnv(); + const runCaptureImpl = createSystemReadinessCapture(probeEnv); + const resolveReadinessOpenshell = () => { + const commandVResult = runCaptureImpl(["sh", "-c", 'command -v "$1"', "--", "openshell"], { + ignoreError: true, + }); + return resolveOpenshell({ + commandVResult: commandVResult || null, + home: probeEnv.HOME, + }); + }; + const assessment = options.assess + ? options.assess() + : assessHost({ + env: process.env, + resolveOpenshellImpl: resolveReadinessOpenshell, + runCaptureImpl, + }); const gpuProbeAllowed = - assessment.hasNvidiaGpu && - (!assessment.isWsl || assessment.runtime !== "docker-desktop" || assessment.dockerReachable); - const gpu = gpuProbeAllowed ? (options.detectGpu ?? detectGpu)() : null; + !assessment.isWsl || assessment.runtime !== "docker-desktop" || assessment.dockerReachable; + // Public readiness collection is observation-only. In particular, the + // ARM64 WSL Docker Desktop accept path must not pull or start the bounded + // CUDA proof container while producing a `mutated: false` report. The + // onboarding admission path runs that proof explicitly after host and + // gateway readiness have both admitted the run. + const gpu = gpuProbeAllowed + ? options.detectGpu + ? options.detectGpu() + : detectGpu({ proveArm64WslDockerDesktopGpu: null, runCaptureImpl }) + : null; + const hasNvidiaGpu = + assessment.hasNvidiaGpu || gpu?.type === "nvidia" || gpu?.platform === "jetson"; const wslDockerDesktopGpuProofPassed = options.wslDockerDesktopGpuProofPassed ?? (assessment.isWsl && assessment.runtime === "docker-desktop" && assessment.dockerReachable && - assessment.hasNvidiaGpu + hasNvidiaGpu ? gpu?.wslDockerDesktopGpuProofPassed : undefined); return { @@ -159,12 +196,15 @@ export function collectHostObservations( observations: adaptHostAssessment( assessment, options.architecture ?? process.arch, - assessment.hasNvidiaGpu - ? (options.detectHostGpuPlatform ?? detectNvidiaPlatform)() + hasNvidiaGpu, + hasNvidiaGpu + ? (gpu?.platform ?? (options.detectHostGpuPlatform ?? detectNvidiaPlatform)()) : undefined, gpu?.count, - gpuProbeAllowed - ? (options.detectNvidiaDriverVersion ?? detectNvidiaDriverVersion)() + hasNvidiaGpu + ? options.detectNvidiaDriverVersion + ? options.detectNvidiaDriverVersion() + : detectNvidiaDriverVersion({ runCaptureImpl }) : undefined, ( options.collectPlatformIdentity ?? @@ -219,6 +259,7 @@ function unknownProjection(evidenceIds: readonly string[]): { "host.session.headless", "host.docker.installed", "host.docker.reachable", + "host.docker.host_invalid", "host.docker.runtime", "host.docker.cpus", "host.docker.memory_bytes", @@ -232,12 +273,14 @@ function unknownProjection(evidenceIds: readonly string[]): { "host.gpu.count", "host.gpu.driver_version", "host.gpu.container_toolkit", + "host.gpu.nvidia_runtime", "host.gpu.cdi", "host.gpu.cdi_stale", ]; const capabilityIds = [ "host.docker.available", "host.docker.daemon_reachable", + "host.docker.endpoint_supported", "host.docker.runtime_supported", "host.docker.resources_sufficient", "host.docker.storage_compatible", @@ -300,12 +343,21 @@ export function projectHostReadiness( const projected = unknownProjection(evidence.map(({ id }) => id)); ({ observations, capabilities, findings } = projected); } else { + const dockerHostBlocks = host.dockerHostInvalid; + const dockerEvidenceUsable = host.dockerReachable && !dockerHostBlocks; const cdiApplies = host.platform === "linux" && + dockerEvidenceUsable && host.hasNvidiaGpu && host.dockerCdiSpecDirs.length > 0 && host.hostGpuPlatform !== "jetson" && !(host.isWsl && host.runtime === "docker-desktop"); + const containerToolkitApplies = + host.hasNvidiaGpu && + host.hostGpuPlatform !== "jetson" && + !(host.isWsl && host.runtime === "docker-desktop"); + const jetsonRuntimeApplies = + dockerEvidenceUsable && host.hasNvidiaGpu && host.hostGpuPlatform === "jetson"; const cdiHealthy = !cdiApplies || (!host.cdiNvidiaGpuSpecMissing && @@ -314,6 +366,7 @@ export function projectHostReadiness( const storageRemediationAvailable = host.platform === "linux" && !host.isWsl && + dockerEvidenceUsable && host.runtime === "docker" && host.hasNestedOverlayConflict && host.dockerStorageDriver === "overlayfs" && @@ -324,28 +377,29 @@ export function projectHostReadiness( observation("host.os.wsl", host.isWsl), observation("host.session.headless", host.isHeadlessLikely), observation("host.docker.installed", host.dockerInstalled), - observation("host.docker.reachable", host.dockerReachable), - observation("host.docker.runtime", host.dockerReachable ? host.runtime : undefined), - observation("host.docker.cpus", host.dockerReachable ? host.dockerCpus : undefined), + observation("host.docker.reachable", dockerHostBlocks ? undefined : host.dockerReachable), + observation("host.docker.host_invalid", host.dockerHostInvalid), + observation("host.docker.runtime", dockerEvidenceUsable ? host.runtime : undefined), + observation("host.docker.cpus", dockerEvidenceUsable ? host.dockerCpus : undefined), observation( "host.docker.memory_bytes", - host.dockerReachable ? host.dockerMemTotalBytes : undefined, + dockerEvidenceUsable ? host.dockerMemTotalBytes : undefined, ), observation( "host.docker.cgroup_version", - host.dockerReachable ? host.dockerCgroupVersion : undefined, + dockerEvidenceUsable ? host.dockerCgroupVersion : undefined, ), observation( "host.docker.cgroupns_mode", - host.dockerReachable ? host.dockerDefaultCgroupnsMode : undefined, + dockerEvidenceUsable ? host.dockerDefaultCgroupnsMode : undefined, ), observation( "host.docker.storage_driver", - host.dockerReachable ? host.dockerStorageDriver : undefined, + dockerEvidenceUsable ? host.dockerStorageDriver : undefined, ), observation( "host.docker.containerd_snapshotter", - host.dockerReachable ? host.dockerUsesContainerdSnapshotter : undefined, + dockerEvidenceUsable ? host.dockerUsesContainerdSnapshotter : undefined, ), observation("host.toolchain.node", host.nodeInstalled), observation("host.toolchain.openshell", host.openshellInstalled), @@ -359,6 +413,10 @@ export function projectHostReadiness( "host.gpu.container_toolkit", host.hasNvidiaGpu ? host.nvidiaContainerToolkitInstalled : false, ), + observation( + "host.gpu.nvidia_runtime", + jetsonRuntimeApplies ? host.dockerNvidiaRuntimeAvailable : false, + ), observation("host.gpu.cdi", cdiApplies ? cdiHealthy : false), observation("host.gpu.cdi_stale", cdiApplies ? host.cdiNvidiaGpuSpecStale : false), ]; @@ -367,7 +425,7 @@ export function projectHostReadiness( architecture: host.architecture, isWsl: host.isWsl, dockerInstalled: host.dockerInstalled, - dockerReachable: host.dockerReachable, + dockerReachable: dockerEvidenceUsable, runtime: host.runtime, hasNvidiaGpu: host.hasNvidiaGpu, ...host.platformIdentity, @@ -377,32 +435,41 @@ export function projectHostReadiness( capabilities = [ ...platform.capabilities, capability("host.docker.available", stateOf(host.dockerInstalled)), + capability("host.docker.endpoint_supported", stateOf(!host.dockerHostInvalid)), capability( "host.docker.daemon_reachable", - host.dockerInstalled ? stateOf(host.dockerReachable) : "absent", + dockerHostBlocks + ? "unknown" + : host.dockerInstalled + ? stateOf(host.dockerReachable) + : "absent", ), capability( "host.docker.runtime_supported", - host.dockerReachable ? stateOf(!host.isUnsupportedRuntime) : "unknown", + dockerEvidenceUsable ? stateOf(!host.isUnsupportedRuntime) : "unknown", ), capability( "host.docker.resources_sufficient", - host.dockerReachable ? stateOf(!host.isContainerRuntimeUnderProvisioned) : "unknown", + dockerEvidenceUsable ? stateOf(!host.isContainerRuntimeUnderProvisioned) : "unknown", ), capability( "host.docker.storage_compatible", - host.dockerReachable ? stateOf(!host.hasNestedOverlayConflict) : "unknown", + dockerEvidenceUsable ? stateOf(!host.hasNestedOverlayConflict) : "unknown", ), capability( "host.docker.storage_remediation_available", - host.dockerReachable ? stateOf(storageRemediationAvailable) : "unknown", + dockerEvidenceUsable ? stateOf(storageRemediationAvailable) : "unknown", ), capability("host.toolchain.node_available", stateOf(host.nodeInstalled)), capability("host.toolchain.openshell_available", stateOf(host.openshellInstalled)), capability("host.gpu.nvidia_available", stateOf(host.hasNvidiaGpu)), capability( "host.gpu.container_toolkit_available", - host.hasNvidiaGpu ? stateOf(host.nvidiaContainerToolkitInstalled) : "present", + jetsonRuntimeApplies + ? stateOf(host.dockerNvidiaRuntimeAvailable) + : containerToolkitApplies + ? stateOf(host.nvidiaContainerToolkitInstalled) + : "present", ), capability("host.gpu.cdi_healthy", cdiApplies ? stateOf(cdiHealthy) : "present"), ]; @@ -413,13 +480,22 @@ export function projectHostReadiness( "host.docker.available", ]), ); - else if (!host.dockerReachable) + if (dockerHostBlocks) + findings.push( + finding( + "host.docker.host_invalid", + "blocking", + "DOCKER_HOST is not a supported absolute local Unix socket endpoint.", + ["host.docker.endpoint_supported"], + ), + ); + else if (host.dockerInstalled && !host.dockerReachable) findings.push( finding("host.docker.daemon_unreachable", "blocking", "The Docker daemon is unreachable.", [ "host.docker.daemon_reachable", ]), ); - if (host.isContainerRuntimeUnderProvisioned) + if (dockerEvidenceUsable && host.isContainerRuntimeUnderProvisioned) findings.push( finding( "host.docker.resources_insufficient", @@ -428,16 +504,16 @@ export function projectHostReadiness( ["host.docker.resources_sufficient"], ), ); - if (host.isUnsupportedRuntime) + if (dockerEvidenceUsable && host.isUnsupportedRuntime) findings.push( finding( "host.docker.runtime_unsupported", - "warning", + "blocking", "The detected container runtime is unsupported.", ["host.docker.runtime_supported"], ), ); - if (host.hasNestedOverlayConflict) + if (dockerEvidenceUsable && host.hasNestedOverlayConflict) findings.push( finding( "host.docker.storage_incompatible", @@ -446,7 +522,7 @@ export function projectHostReadiness( ["host.docker.storage_compatible"], ), ); - if (host.hasNvidiaGpu && !host.nvidiaContainerToolkitInstalled) + if (containerToolkitApplies && !host.nvidiaContainerToolkitInstalled) findings.push( finding( "host.gpu.container_toolkit_missing", @@ -455,6 +531,15 @@ export function projectHostReadiness( ["host.gpu.container_toolkit_available"], ), ); + if (jetsonRuntimeApplies && host.dockerNvidiaRuntimeAvailable === false) + findings.push( + finding( + "host.gpu.nvidia_runtime_missing", + "blocking", + "Docker NVIDIA runtime support is missing for Jetson/Tegra sandbox GPU.", + ["host.gpu.container_toolkit_available"], + ), + ); if (cdiApplies && host.cdiNvidiaGpuSpecMissing) findings.push( finding("host.gpu.cdi_missing", "blocking", "The NVIDIA CDI specification is missing.", [ diff --git a/src/lib/readiness/index.ts b/src/lib/readiness/index.ts index 43b9a77b843..dc9497306e4 100644 --- a/src/lib/readiness/index.ts +++ b/src/lib/readiness/index.ts @@ -3,6 +3,26 @@ export type { SchemaCompatibility } from "./compatibility.js"; export { checkSystemReadinessSchemaVersion } from "./compatibility.js"; +export type { + CollectGatewayObservationsOptions, + CreateGatewayReadinessProjectionOptions, + GatewayAttachmentState, + GatewayDriftState, + GatewayObservationSnapshot, + GatewayObservations, + GatewayPortConflictState, + GatewayReadinessDependencies, + GatewayReadinessProjection, + GatewayReuseObservation, + ManagedGatewayObservations, + ProjectGatewayReadinessOptions, +} from "./gateway.js"; +export { + collectGatewayObservations, + createGatewayReadinessProjection, + projectGatewayReadiness, + refreshGatewayReadinessProjection, +} from "./gateway.js"; export type { CollectHostObservationsOptions, CreateHostReadinessReportOptions, @@ -14,6 +34,18 @@ export { createHostReadinessReport, projectHostReadiness, } from "./host.js"; +export type { + OnboardReadinessAdmissionDecision, + OnboardReadinessAdmissionOptions, + OnboardReadinessAdmissionReasonId, +} from "./onboard-admission.js"; +export { + evaluateOnboardGatewayReadinessAdmission, + evaluateOnboardReadinessAdmission, + ONBOARD_READINESS_ADMISSION_REASON_IDS, + ONBOARD_READINESS_FINDING_IDS, + ONBOARD_REQUIRED_CAPABILITY_IDS, +} from "./onboard-admission.js"; export type { CollectPlatformIdentityOptions, PlatformIdentity, @@ -27,6 +59,8 @@ export { } from "./platform-qualification.js"; export { createPublicReadinessReport, renderReadinessReport } from "./presentation.js"; export { getSystemReadinessReferenceErrors } from "./references.js"; +export type { CollectSystemReadinessOptions } from "./system.js"; +export { composeSystemReadinessReport, createSystemReadinessReport } from "./system.js"; export type { EvidenceScalar, FindingSeverity, diff --git a/src/lib/readiness/onboard-admission.test.ts b/src/lib/readiness/onboard-admission.test.ts new file mode 100644 index 00000000000..0bb00bf4067 --- /dev/null +++ b/src/lib/readiness/onboard-admission.test.ts @@ -0,0 +1,450 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; +import { + evaluateOnboardGatewayReadinessAdmission, + evaluateOnboardReadinessAdmission, + ONBOARD_READINESS_ADMISSION_REASON_IDS, + ONBOARD_READINESS_FINDING_IDS, + ONBOARD_REQUIRED_CAPABILITY_IDS, + type OnboardReadinessAdmissionOptions, +} from "./onboard-admission"; +import type { + ReadinessCapability, + ReadinessFinding, + ReadinessObservation, + SystemReadinessReport, +} from "./types"; + +const DEFAULT_OPTIONS: OnboardReadinessAdmissionOptions = { + explicitlyOptedOutGpuPassthrough: false, + allowUnsupportedRuntime: false, + allowStorageRemediation: true, +}; + +function capability(id: string, state: ReadinessCapability["state"]): ReadinessCapability { + return { id, state }; +} + +function finding( + id: string, + severity: ReadinessFinding["severity"] = "blocking", +): ReadinessFinding { + return { id, severity, summary: id }; +} + +function requiredCapabilities(): ReadinessCapability[] { + return [ + capability(ONBOARD_REQUIRED_CAPABILITY_IDS.dockerAvailable, "present"), + capability(ONBOARD_REQUIRED_CAPABILITY_IDS.dockerDaemonReachable, "present"), + capability(ONBOARD_REQUIRED_CAPABILITY_IDS.dockerRuntimeSupported, "present"), + capability(ONBOARD_REQUIRED_CAPABILITY_IDS.dockerStorageCompatible, "present"), + capability(ONBOARD_REQUIRED_CAPABILITY_IDS.dockerStorageRemediationAvailable, "absent"), + capability(ONBOARD_REQUIRED_CAPABILITY_IDS.nvidiaGpuAvailable, "absent"), + capability(ONBOARD_REQUIRED_CAPABILITY_IDS.nvidiaContainerToolkitAvailable, "present"), + capability(ONBOARD_REQUIRED_CAPABILITY_IDS.nvidiaCdiHealthy, "present"), + capability(ONBOARD_REQUIRED_CAPABILITY_IDS.platformSupported, "present"), + ]; +} + +function gatewayCapabilities( + overrides: Partial> = {}, +): ReadinessCapability[] { + return [ + ONBOARD_REQUIRED_CAPABILITY_IDS.gatewayAuthorityResolved, + ONBOARD_REQUIRED_CAPABILITY_IDS.gatewayAttachmentValid, + ONBOARD_REQUIRED_CAPABILITY_IDS.gatewayReuseReady, + ONBOARD_REQUIRED_CAPABILITY_IDS.gatewayVersionCompatible, + ONBOARD_REQUIRED_CAPABILITY_IDS.gatewayPortUncontested, + ].map((id) => capability(id, overrides[id] ?? "present")); +} + +function report( + overrides: { + capabilities?: ReadinessCapability[]; + findings?: ReadinessFinding[]; + observations?: ReadinessObservation[]; + status?: SystemReadinessReport["status"]; + } = {}, +): SystemReadinessReport { + const status = overrides.status ?? "supported"; + const outcome = + status === "supported" + ? ({ status, exitCode: 0 } as const) + : status === "incompatible" + ? ({ status, exitCode: 2 } as const) + : ({ status, exitCode: 3 } as const); + return { + schemaVersion: "1.1.0", + ...outcome, + mutated: false, + provenance: { + nemoclawVersion: "0.1.0", + sourceRevision: "a".repeat(40), + observedAt: "2026-08-07T12:00:00.000Z", + }, + observations: overrides.observations ?? [], + capabilities: overrides.capabilities ?? requiredCapabilities(), + qualifications: [], + findings: overrides.findings ?? [], + evidence: [], + }; +} + +function withCapabilityState( + capabilities: ReadinessCapability[], + id: string, + state: ReadinessCapability["state"], +): ReadinessCapability[] { + return capabilities.map((entry) => (entry.id === id ? { ...entry, state } : entry)); +} + +describe("onboarding readiness admission (#7411)", () => { + it("admits known required capabilities without relying on the report status", () => { + expect( + evaluateOnboardReadinessAdmission(report({ status: "inconclusive" }), DEFAULT_OPTIONS), + ).toEqual({ admitted: true, waivedFindingIds: [] }); + }); + + it("fails on every unwaived blocking or fatal finding and retains report order", () => { + const decision = evaluateOnboardReadinessAdmission( + report({ + status: "incompatible", + findings: [ + finding("host.example.warning", "warning"), + finding("host.example.blocked"), + finding("host.example.fatal", "fatal"), + ], + }), + DEFAULT_OPTIONS, + ); + + expect(decision).toEqual({ + admitted: false, + reasonIds: [ONBOARD_READINESS_ADMISSION_REASON_IDS.blockingFindings], + findingIds: ["host.example.blocked", "host.example.fatal"], + capabilityIds: [], + waivedFindingIds: [], + }); + }); + + it("waives only the known NVIDIA blockers for explicit CPU-only intent", () => { + const capabilities = withCapabilityState( + withCapabilityState( + withCapabilityState( + requiredCapabilities(), + ONBOARD_REQUIRED_CAPABILITY_IDS.nvidiaGpuAvailable, + "unknown", + ), + ONBOARD_REQUIRED_CAPABILITY_IDS.nvidiaContainerToolkitAvailable, + "unknown", + ), + ONBOARD_REQUIRED_CAPABILITY_IDS.nvidiaCdiHealthy, + "unknown", + ); + const gpuFindings = [ + finding(ONBOARD_READINESS_FINDING_IDS.containerToolkitMissing), + finding(ONBOARD_READINESS_FINDING_IDS.cdiMissing), + finding(ONBOARD_READINESS_FINDING_IDS.cdiStale), + finding(ONBOARD_READINESS_FINDING_IDS.nvidiaRuntimeMissing), + ]; + + expect( + evaluateOnboardReadinessAdmission( + report({ capabilities, findings: gpuFindings, status: "incompatible" }), + { ...DEFAULT_OPTIONS, explicitlyOptedOutGpuPassthrough: true }, + ), + ).toEqual({ + admitted: true, + waivedFindingIds: gpuFindings.map(({ id }) => id), + }); + + expect( + evaluateOnboardReadinessAdmission( + report({ + capabilities, + findings: [...gpuFindings, finding("host.example.blocked")], + status: "incompatible", + }), + { ...DEFAULT_OPTIONS, explicitlyOptedOutGpuPassthrough: true }, + ), + ).toMatchObject({ admitted: false, findingIds: ["host.example.blocked"] }); + + expect( + evaluateOnboardReadinessAdmission( + report({ + capabilities, + findings: [finding(ONBOARD_READINESS_FINDING_IDS.nvidiaRuntimeMissing)], + status: "incompatible", + }), + DEFAULT_OPTIONS, + ), + ).toMatchObject({ + admitted: false, + findingIds: [ONBOARD_READINESS_FINDING_IDS.nvidiaRuntimeMissing], + }); + }); + + it("waives a known unsupported runtime only for the explicit portable profile", () => { + const capabilities = withCapabilityState( + requiredCapabilities(), + ONBOARD_REQUIRED_CAPABILITY_IDS.dockerRuntimeSupported, + "absent", + ); + const runtimeFinding = finding(ONBOARD_READINESS_FINDING_IDS.runtimeUnsupported); + + expect( + evaluateOnboardReadinessAdmission( + report({ capabilities, findings: [runtimeFinding], status: "incompatible" }), + { ...DEFAULT_OPTIONS, allowUnsupportedRuntime: true }, + ), + ).toEqual({ + admitted: true, + waivedFindingIds: [ONBOARD_READINESS_FINDING_IDS.runtimeUnsupported], + }); + expect( + evaluateOnboardReadinessAdmission( + report({ capabilities, findings: [runtimeFinding], status: "incompatible" }), + DEFAULT_OPTIONS, + ), + ).toMatchObject({ + admitted: false, + findingIds: [ONBOARD_READINESS_FINDING_IDS.runtimeUnsupported], + }); + }); + + it("does not let the portable exception hide an unknown runtime", () => { + const capabilities = withCapabilityState( + requiredCapabilities(), + ONBOARD_REQUIRED_CAPABILITY_IDS.dockerRuntimeSupported, + "unknown", + ); + + expect( + evaluateOnboardReadinessAdmission(report({ capabilities }), { + ...DEFAULT_OPTIONS, + allowUnsupportedRuntime: true, + }), + ).toEqual({ + admitted: false, + reasonIds: [ONBOARD_READINESS_ADMISSION_REASON_IDS.requiredCapabilitiesUnknown], + findingIds: [], + capabilityIds: [ONBOARD_REQUIRED_CAPABILITY_IDS.dockerRuntimeSupported], + waivedFindingIds: [], + }); + }); + + it("admits only the pre-mutation facts that portable host preparation can replace", () => { + let capabilities = withCapabilityState( + requiredCapabilities(), + ONBOARD_REQUIRED_CAPABILITY_IDS.dockerDaemonReachable, + "absent", + ); + for (const id of [ + ONBOARD_REQUIRED_CAPABILITY_IDS.dockerRuntimeSupported, + ONBOARD_REQUIRED_CAPABILITY_IDS.dockerStorageCompatible, + ONBOARD_REQUIRED_CAPABILITY_IDS.dockerStorageRemediationAvailable, + ]) { + capabilities = withCapabilityState(capabilities, id, "unknown"); + } + + expect( + evaluateOnboardReadinessAdmission( + report({ + capabilities, + findings: [finding(ONBOARD_READINESS_FINDING_IDS.dockerDaemonUnreachable)], + status: "incompatible", + }), + { ...DEFAULT_OPTIONS, allowPortableHostPreparation: true }, + ), + ).toEqual({ + admitted: true, + waivedFindingIds: [ONBOARD_READINESS_FINDING_IDS.dockerDaemonUnreachable], + }); + + expect( + evaluateOnboardReadinessAdmission( + report({ + capabilities: withCapabilityState( + capabilities, + ONBOARD_REQUIRED_CAPABILITY_IDS.dockerAvailable, + "absent", + ), + findings: [finding("host.docker.unavailable")], + status: "incompatible", + }), + { ...DEFAULT_OPTIONS, allowPortableHostPreparation: true }, + ), + ).toMatchObject({ admitted: false, findingIds: ["host.docker.unavailable"] }); + }); + + it("admits the documented storage exception only when remediation is present", () => { + let capabilities = withCapabilityState( + requiredCapabilities(), + ONBOARD_REQUIRED_CAPABILITY_IDS.dockerStorageCompatible, + "absent", + ); + capabilities = withCapabilityState( + capabilities, + ONBOARD_REQUIRED_CAPABILITY_IDS.dockerStorageRemediationAvailable, + "present", + ); + const storageFinding = finding(ONBOARD_READINESS_FINDING_IDS.storageIncompatible); + + expect( + evaluateOnboardReadinessAdmission( + report({ capabilities, findings: [storageFinding], status: "incompatible" }), + DEFAULT_OPTIONS, + ), + ).toEqual({ + admitted: true, + waivedFindingIds: [ONBOARD_READINESS_FINDING_IDS.storageIncompatible], + }); + expect( + evaluateOnboardReadinessAdmission( + report({ capabilities, findings: [storageFinding], status: "incompatible" }), + { ...DEFAULT_OPTIONS, allowStorageRemediation: false }, + ), + ).toMatchObject({ + admitted: false, + findingIds: [ONBOARD_READINESS_FINDING_IDS.storageIncompatible], + }); + }); + + it("accepts present storage remediation when direct compatibility is unknown", () => { + let capabilities = withCapabilityState( + requiredCapabilities(), + ONBOARD_REQUIRED_CAPABILITY_IDS.dockerStorageCompatible, + "unknown", + ); + capabilities = withCapabilityState( + capabilities, + ONBOARD_REQUIRED_CAPABILITY_IDS.dockerStorageRemediationAvailable, + "present", + ); + + expect(evaluateOnboardReadinessAdmission(report({ capabilities }), DEFAULT_OPTIONS)).toEqual({ + admitted: true, + waivedFindingIds: [], + }); + }); + + it("fails closed when a required capability is unknown or missing", () => { + const capabilities = withCapabilityState( + requiredCapabilities().filter( + ({ id }) => id !== ONBOARD_REQUIRED_CAPABILITY_IDS.platformSupported, + ), + ONBOARD_REQUIRED_CAPABILITY_IDS.dockerDaemonReachable, + "unknown", + ); + + expect(evaluateOnboardReadinessAdmission(report({ capabilities }), DEFAULT_OPTIONS)).toEqual({ + admitted: false, + reasonIds: [ONBOARD_READINESS_ADMISSION_REASON_IDS.requiredCapabilitiesUnknown], + findingIds: [], + capabilityIds: [ + ONBOARD_REQUIRED_CAPABILITY_IDS.dockerDaemonReachable, + ONBOARD_REQUIRED_CAPABILITY_IDS.platformSupported, + ], + waivedFindingIds: [], + }); + }); + + it("reports both stable failure reasons when blockers and unknown requirements coexist", () => { + const capabilities = withCapabilityState( + requiredCapabilities(), + ONBOARD_REQUIRED_CAPABILITY_IDS.dockerAvailable, + "unknown", + ); + + expect( + evaluateOnboardReadinessAdmission( + report({ capabilities, findings: [finding("host.example.blocked")] }), + DEFAULT_OPTIONS, + ), + ).toEqual({ + admitted: false, + reasonIds: [ + ONBOARD_READINESS_ADMISSION_REASON_IDS.blockingFindings, + ONBOARD_READINESS_ADMISSION_REASON_IDS.requiredCapabilitiesUnknown, + ], + findingIds: ["host.example.blocked"], + capabilityIds: [ONBOARD_REQUIRED_CAPABILITY_IDS.dockerAvailable], + waivedFindingIds: [], + }); + }); + + it("fails the gateway-only boundary on blockers and inconclusive required facts", () => { + expect( + evaluateOnboardGatewayReadinessAdmission({ + observations: [], + capabilities: gatewayCapabilities({ + [ONBOARD_REQUIRED_CAPABILITY_IDS.gatewayPortUncontested]: "unknown", + }), + findings: [finding("gateway.ownership.multiple")], + }), + ).toEqual({ + admitted: false, + reasonIds: [ + ONBOARD_READINESS_ADMISSION_REASON_IDS.blockingFindings, + ONBOARD_READINESS_ADMISSION_REASON_IDS.requiredCapabilitiesUnknown, + ], + findingIds: ["gateway.ownership.multiple"], + capabilityIds: [ONBOARD_REQUIRED_CAPABILITY_IDS.gatewayPortUncontested], + waivedFindingIds: [], + }); + }); + + it("waives version drift only for the managed gateway reconciliation path", () => { + const versionDrift = finding(ONBOARD_READINESS_FINDING_IDS.gatewayVersionDrift); + const managed = { + observations: [ + { + id: "gateway.management.mode", + state: "present" as const, + value: "nemoclaw-managed", + }, + ], + capabilities: gatewayCapabilities(), + findings: [versionDrift], + }; + + expect(evaluateOnboardGatewayReadinessAdmission(managed)).toEqual({ + admitted: true, + waivedFindingIds: [ONBOARD_READINESS_FINDING_IDS.gatewayVersionDrift], + }); + expect( + evaluateOnboardGatewayReadinessAdmission({ + ...managed, + observations: [ + { + id: "gateway.management.mode", + state: "present", + value: "externally-supervised", + }, + ], + }), + ).toMatchObject({ + admitted: false, + findingIds: [ONBOARD_READINESS_FINDING_IDS.gatewayVersionDrift], + }); + }); + + it("requires every gateway capability once the public projection is present", () => { + const capabilities = [ + ...requiredCapabilities(), + ...gatewayCapabilities({ + [ONBOARD_REQUIRED_CAPABILITY_IDS.gatewayAttachmentValid]: "unknown", + }), + ]; + + expect(evaluateOnboardReadinessAdmission(report({ capabilities }), DEFAULT_OPTIONS)).toEqual({ + admitted: false, + reasonIds: [ONBOARD_READINESS_ADMISSION_REASON_IDS.requiredCapabilitiesUnknown], + findingIds: [], + capabilityIds: [ONBOARD_REQUIRED_CAPABILITY_IDS.gatewayAttachmentValid], + waivedFindingIds: [], + }); + }); +}); diff --git a/src/lib/readiness/onboard-admission.ts b/src/lib/readiness/onboard-admission.ts new file mode 100644 index 00000000000..75b3a483b5d --- /dev/null +++ b/src/lib/readiness/onboard-admission.ts @@ -0,0 +1,299 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { ReadinessCapability, ReadinessFinding, SystemReadinessReport } from "./types"; + +export const ONBOARD_READINESS_ADMISSION_REASON_IDS = { + blockingFindings: "onboard.readiness.blocking_findings", + requiredCapabilitiesUnknown: "onboard.readiness.required_capabilities_unknown", +} as const; + +export type OnboardReadinessAdmissionReasonId = + (typeof ONBOARD_READINESS_ADMISSION_REASON_IDS)[keyof typeof ONBOARD_READINESS_ADMISSION_REASON_IDS]; + +export const ONBOARD_READINESS_FINDING_IDS = { + containerToolkitMissing: "host.gpu.container_toolkit_missing", + cdiMissing: "host.gpu.cdi_missing", + cdiStale: "host.gpu.cdi_stale", + nvidiaRuntimeMissing: "host.gpu.nvidia_runtime_missing", + dockerDaemonUnreachable: "host.docker.daemon_unreachable", + runtimeUnsupported: "host.docker.runtime_unsupported", + storageIncompatible: "host.docker.storage_incompatible", + gatewayVersionDrift: "gateway.version.drift", +} as const; + +export const ONBOARD_REQUIRED_CAPABILITY_IDS = { + dockerAvailable: "host.docker.available", + dockerDaemonReachable: "host.docker.daemon_reachable", + dockerRuntimeSupported: "host.docker.runtime_supported", + dockerStorageCompatible: "host.docker.storage_compatible", + dockerStorageRemediationAvailable: "host.docker.storage_remediation_available", + nvidiaGpuAvailable: "host.gpu.nvidia_available", + nvidiaContainerToolkitAvailable: "host.gpu.container_toolkit_available", + nvidiaCdiHealthy: "host.gpu.cdi_healthy", + platformSupported: "host.platform.supported", + gatewayAuthorityResolved: "gateway.authority.resolved", + gatewayAttachmentValid: "gateway.attachment.valid", + gatewayReuseReady: "gateway.reuse.ready", + gatewayVersionCompatible: "gateway.version.compatible", + gatewayPortUncontested: "gateway.port.uncontested", +} as const; + +export interface OnboardReadinessAdmissionOptions { + /** CPU-only intent makes NVIDIA host integration irrelevant to this run. */ + explicitlyOptedOutGpuPassthrough: boolean; + /** This run has an explicit runtime path that does not require the standard Docker driver. */ + allowUnsupportedRuntime: boolean; + /** The lifecycle may build the documented patched gateway image for the storage conflict. */ + allowStorageRemediation: boolean; + /** The explicit portable profile may prepare its rootless runtime before revalidation. */ + allowPortableHostPreparation?: boolean; +} + +export type OnboardReadinessAdmissionDecision = + | { + admitted: true; + waivedFindingIds: readonly string[]; + } + | { + admitted: false; + reasonIds: readonly OnboardReadinessAdmissionReasonId[]; + findingIds: readonly string[]; + capabilityIds: readonly string[]; + waivedFindingIds: readonly string[]; + }; + +const GPU_FINDINGS = new Set([ + ONBOARD_READINESS_FINDING_IDS.containerToolkitMissing, + ONBOARD_READINESS_FINDING_IDS.cdiMissing, + ONBOARD_READINESS_FINDING_IDS.cdiStale, + ONBOARD_READINESS_FINDING_IDS.nvidiaRuntimeMissing, +]); + +const ALWAYS_REQUIRED_CAPABILITIES = [ + ONBOARD_REQUIRED_CAPABILITY_IDS.dockerAvailable, + ONBOARD_REQUIRED_CAPABILITY_IDS.dockerDaemonReachable, + ONBOARD_REQUIRED_CAPABILITY_IDS.dockerRuntimeSupported, + ONBOARD_REQUIRED_CAPABILITY_IDS.platformSupported, +] as const; + +const GATEWAY_REQUIRED_CAPABILITIES = [ + ONBOARD_REQUIRED_CAPABILITY_IDS.gatewayAuthorityResolved, + ONBOARD_REQUIRED_CAPABILITY_IDS.gatewayAttachmentValid, + ONBOARD_REQUIRED_CAPABILITY_IDS.gatewayReuseReady, + ONBOARD_REQUIRED_CAPABILITY_IDS.gatewayVersionCompatible, + ONBOARD_REQUIRED_CAPABILITY_IDS.gatewayPortUncontested, +] as const; + +function capabilityState( + capabilities: ReadonlyMap, + id: string, +): ReadinessCapability["state"] | "missing" { + return capabilities.get(id)?.state ?? "missing"; +} + +function isBlocking(finding: ReadinessFinding): boolean { + return finding.severity === "blocking" || finding.severity === "fatal"; +} + +function canWaiveFinding( + finding: ReadinessFinding, + capabilities: ReadonlyMap, + options: Readonly, + managedGateway: boolean, +): boolean { + if (options.explicitlyOptedOutGpuPassthrough && GPU_FINDINGS.has(finding.id)) return true; + if ( + options.allowUnsupportedRuntime && + finding.id === ONBOARD_READINESS_FINDING_IDS.runtimeUnsupported + ) { + return true; + } + if ( + options.allowPortableHostPreparation && + (finding.id === ONBOARD_READINESS_FINDING_IDS.dockerDaemonUnreachable || + finding.id === ONBOARD_READINESS_FINDING_IDS.storageIncompatible) + ) { + return true; + } + if (managedGateway && finding.id === ONBOARD_READINESS_FINDING_IDS.gatewayVersionDrift) { + return true; + } + return ( + options.allowStorageRemediation && + finding.id === ONBOARD_READINESS_FINDING_IDS.storageIncompatible && + capabilityState( + capabilities, + ONBOARD_REQUIRED_CAPABILITY_IDS.dockerStorageRemediationAvailable, + ) === "present" + ); +} + +function addUnknownCapability( + target: string[], + capabilities: ReadonlyMap, + id: string, +): void { + const state = capabilityState(capabilities, id); + if (state === "unknown" || state === "missing") target.push(id); +} + +function requiredUnknownCapabilityIds( + capabilities: ReadonlyMap, + options: Readonly, +): readonly string[] { + const unknown: string[] = []; + for (const id of ALWAYS_REQUIRED_CAPABILITIES) { + if ( + options.allowPortableHostPreparation && + (id === ONBOARD_REQUIRED_CAPABILITY_IDS.dockerDaemonReachable || + id === ONBOARD_REQUIRED_CAPABILITY_IDS.dockerRuntimeSupported) + ) { + continue; + } + addUnknownCapability(unknown, capabilities, id); + } + + const storageCompatible = capabilityState( + capabilities, + ONBOARD_REQUIRED_CAPABILITY_IDS.dockerStorageCompatible, + ); + const storageRemediation = capabilityState( + capabilities, + ONBOARD_REQUIRED_CAPABILITY_IDS.dockerStorageRemediationAvailable, + ); + const storageRequirementSatisfied = + options.allowPortableHostPreparation === true || + storageCompatible === "present" || + (options.allowStorageRemediation && storageRemediation === "present"); + if (!storageRequirementSatisfied) { + if (storageCompatible === "unknown" || storageCompatible === "missing") { + addUnknownCapability( + unknown, + capabilities, + ONBOARD_REQUIRED_CAPABILITY_IDS.dockerStorageCompatible, + ); + } + if ( + options.allowStorageRemediation && + storageRemediation !== "present" && + (storageRemediation === "unknown" || storageRemediation === "missing") + ) { + addUnknownCapability( + unknown, + capabilities, + ONBOARD_REQUIRED_CAPABILITY_IDS.dockerStorageRemediationAvailable, + ); + } + } + + if (!options.explicitlyOptedOutGpuPassthrough) { + addUnknownCapability(unknown, capabilities, ONBOARD_REQUIRED_CAPABILITY_IDS.nvidiaGpuAvailable); + addUnknownCapability( + unknown, + capabilities, + ONBOARD_REQUIRED_CAPABILITY_IDS.nvidiaContainerToolkitAvailable, + ); + addUnknownCapability(unknown, capabilities, ONBOARD_REQUIRED_CAPABILITY_IDS.nvidiaCdiHealthy); + } + if (GATEWAY_REQUIRED_CAPABILITIES.some((id) => capabilities.has(id))) { + for (const id of GATEWAY_REQUIRED_CAPABILITIES) addUnknownCapability(unknown, capabilities, id); + } + return unknown; +} + +type OnboardReadinessInput = Pick< + SystemReadinessReport, + "observations" | "capabilities" | "findings" +>; + +function hasManagedGateway(report: OnboardReadinessInput): boolean { + return report.observations.some( + ({ id, state, value }) => + id === "gateway.management.mode" && state === "present" && value === "nemoclaw-managed", + ); +} + +function decisionFor( + findingIds: readonly string[], + capabilityIds: readonly string[], + waivedFindingIds: readonly string[], +): OnboardReadinessAdmissionDecision { + if (findingIds.length === 0 && capabilityIds.length === 0) { + return { admitted: true, waivedFindingIds }; + } + + const reasonIds: OnboardReadinessAdmissionReasonId[] = []; + if (findingIds.length > 0) { + reasonIds.push(ONBOARD_READINESS_ADMISSION_REASON_IDS.blockingFindings); + } + if (capabilityIds.length > 0) { + reasonIds.push(ONBOARD_READINESS_ADMISSION_REASON_IDS.requiredCapabilitiesUnknown); + } + return { + admitted: false, + reasonIds, + findingIds, + capabilityIds, + waivedFindingIds, + }; +} + +/** + * Decide whether one onboarding run may proceed from the stable public + * readiness entities. Consumer-specific exceptions are explicit and narrow; + * every other blocking finding and every unknown required capability fails + * closed before lifecycle effects begin. + */ +export function evaluateOnboardReadinessAdmission( + report: Readonly, + options: Readonly, +): OnboardReadinessAdmissionDecision { + const capabilities = new Map(report.capabilities.map((entry) => [entry.id, entry])); + const managedGateway = hasManagedGateway(report); + const waivedFindingIds: string[] = []; + const findingIds: string[] = []; + for (const finding of report.findings) { + if (!isBlocking(finding)) continue; + if (canWaiveFinding(finding, capabilities, options, managedGateway)) { + waivedFindingIds.push(finding.id); + } else findingIds.push(finding.id); + } + + const capabilityIds = [...requiredUnknownCapabilityIds(capabilities, options)]; + return decisionFor(findingIds, capabilityIds, waivedFindingIds); +} + +/** Fail closed on gateway facts before any onboarding effect can run. */ +export function evaluateOnboardGatewayReadinessAdmission( + report: OnboardReadinessInput, +): OnboardReadinessAdmissionDecision { + const capabilities = new Map(report.capabilities.map((entry) => [entry.id, entry])); + const managedGateway = hasManagedGateway(report); + const waivedFindingIds: string[] = []; + const findingIds: string[] = []; + for (const finding of report.findings) { + if (!isBlocking(finding)) continue; + if ( + canWaiveFinding( + finding, + capabilities, + { + explicitlyOptedOutGpuPassthrough: false, + allowUnsupportedRuntime: false, + allowStorageRemediation: false, + }, + managedGateway, + ) + ) { + waivedFindingIds.push(finding.id); + } else { + findingIds.push(finding.id); + } + } + const capabilityIds: string[] = []; + for (const id of GATEWAY_REQUIRED_CAPABILITIES) { + addUnknownCapability(capabilityIds, capabilities, id); + } + return decisionFor(findingIds, capabilityIds, waivedFindingIds); +} diff --git a/src/lib/readiness/openshell-resolver.ts b/src/lib/readiness/openshell-resolver.ts new file mode 100644 index 00000000000..49b328bae68 --- /dev/null +++ b/src/lib/readiness/openshell-resolver.ts @@ -0,0 +1,4 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export { resolveOpenshell } from "../adapters/openshell/resolve.js"; diff --git a/src/lib/readiness/platform-qualification.test.ts b/src/lib/readiness/platform-qualification.test.ts index a784af587ce..2d945776c4a 100644 --- a/src/lib/readiness/platform-qualification.test.ts +++ b/src/lib/readiness/platform-qualification.test.ts @@ -182,6 +182,30 @@ describe("platform readiness qualification (#7410)", () => { expect(capability(result, "host.platform.supported")).toBe(expected ? "present" : "absent"); }); + it("blocks an unsupported Intel macOS platform combination", () => { + const result = projectPlatformQualification( + input({ platform: "darwin", architecture: "x64", runtime: "docker-desktop" }), + ); + + expect(capability(result, "host.platform.supported")).toBe("absent"); + expect(result.findings).toContainEqual( + expect.objectContaining({ id: "host.platform.unsupported", severity: "blocking" }), + ); + }); + + it.each([ + ["native Docker", "docker", "host.platform.wsl_native_docker_unqualified"], + ["an inconclusive runtime", "unknown", "host.platform.wsl_runtime_inconclusive"], + ] as const)("blocks WSL with %s using its specific finding", (_scenario, runtime, findingId) => { + const result = projectPlatformQualification(input({ isWsl: true, runtime })); + + expect(capability(result, "host.platform.supported")).toBe("absent"); + expect(result.findings).toContainEqual( + expect.objectContaining({ id: findingId, severity: "blocking" }), + ); + expect(result.findings.some(({ id }) => id === "host.platform.unsupported")).toBe(false); + }); + it("qualifies DGX Spark while retaining product identity as bounded evidence", () => { const result = projectPlatformQualification( input({ diff --git a/src/lib/readiness/platform-qualification.ts b/src/lib/readiness/platform-qualification.ts index 5f297e7d93a..30ddf07dd74 100644 --- a/src/lib/readiness/platform-qualification.ts +++ b/src/lib/readiness/platform-qualification.ts @@ -455,14 +455,14 @@ export function projectPlatformQualification( } else if (nativeDocker) { findings.push({ id: "host.platform.wsl_native_docker_unqualified", - severity: "warning", + severity: "blocking", summary: "Native Docker Engine inside WSL is not the qualified Docker Desktop integration.", capabilityIds: ["host.platform.wsl_native_docker", "host.platform.supported"], }); } else if (input.isWsl && input.dockerReachable && !dockerDesktop) { findings.push({ id: "host.platform.wsl_runtime_inconclusive", - severity: "warning", + severity: "blocking", summary: "WSL Docker runtime identity is inconclusive.", capabilityIds: ["host.platform.wsl_runtime_available"], }); @@ -508,5 +508,16 @@ export function projectPlatformQualification( ...(evidence.length ? { evidenceIds: ["host.platform.identity"] } : {}), }); } + if ( + !platformSupported && + !findings.some(({ severity }) => severity === "blocking" || severity === "fatal") + ) { + findings.push({ + id: "host.platform.unsupported", + severity: "blocking", + summary: "The detected host platform and container runtime are not supported.", + capabilityIds: ["host.platform.supported"], + }); + } return { capabilities, qualifications, findings, evidence }; } diff --git a/src/lib/readiness/presentation.test.ts b/src/lib/readiness/presentation.test.ts index efeae3a65b8..f0cee4706ea 100644 --- a/src/lib/readiness/presentation.test.ts +++ b/src/lib/readiness/presentation.test.ts @@ -187,6 +187,23 @@ describe("public readiness presentation (#7412)", () => { expect(String(publicReport.evidence[0]?.details?.stderr).length).toBeLessThanOrEqual(1024); }); + it("neutralizes terminal and bidirectional controls across the public report", () => { + const unsafe = "trusted\n\u001b[31mforged\u202efailure"; + const publicReport = createPublicReadinessReport( + report({ + observations: [{ id: "host.identity", state: "present", value: unsafe }], + findings: [{ id: "host.finding", severity: "warning", summary: unsafe }], + evidence: [{ id: "host.evidence", summary: unsafe, details: { productName: unsafe } }], + }), + ); + const serialized = JSON.stringify(publicReport); + + expect(serialized).toContain("trusted\\\\u000a\\\\u001b[31mforged\\\\u202efailure"); + expect(serialized).not.toMatch( + /[\u0000-\u001f\u007f-\u009f\u061c\u200e\u200f\u2028-\u202e\u2066-\u2069]/u, + ); + }); + it("excludes camel-cased and suffixed process environment keys", () => { const publicReport = createPublicReadinessReport( report({ diff --git a/src/lib/readiness/presentation.ts b/src/lib/readiness/presentation.ts index 65323294c27..1ab54c8dd90 100644 --- a/src/lib/readiness/presentation.ts +++ b/src/lib/readiness/presentation.ts @@ -3,6 +3,7 @@ import { validateBuildIdentity } from "../core/version.js"; import { redactForLog } from "../security/redact.js"; +import { sanitizeReadinessText } from "./sanitize.js"; import type { EvidenceScalar, ReadinessCapability, @@ -33,9 +34,10 @@ const ENVIRONMENT_DETAIL_KEYS = new Set([ ]); function bounded(value: string, maxLength: number): string { - return String(redactForLog(value)) - .replace(/([a-z][a-z0-9+.-]*:\/\/)[^/@\s]+@/gi, "$1@") - .slice(0, maxLength); + return sanitizeReadinessText( + String(redactForLog(value)).replace(/([a-z][a-z0-9+.-]*:\/\/)[^/@\s]+@/gi, "$1@"), + maxLength, + ); } function scalar(value: EvidenceScalar): EvidenceScalar { diff --git a/src/lib/readiness/probe-env.test.ts b/src/lib/readiness/probe-env.test.ts new file mode 100644 index 00000000000..036aa8c7754 --- /dev/null +++ b/src/lib/readiness/probe-env.test.ts @@ -0,0 +1,82 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, describe, expect, it, vi } from "vitest"; + +const subprocess = vi.hoisted(() => ({ spawnSync: vi.fn() })); + +vi.mock("node:child_process", async (importOriginal) => ({ + ...(await importOriginal()), + spawnSync: subprocess.spawnSync, +})); + +import { buildSystemReadinessProbeEnv, createSystemReadinessCapture } from "./probe-env"; + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe("system readiness child environment (#7411)", () => { + it("uses a replacement environment even when a caller supplies another env", () => { + subprocess.spawnSync.mockReturnValue({ + error: undefined, + status: 0, + stdout: "ready\n", + stderr: "", + }); + const env = buildSystemReadinessProbeEnv( + { + HOME: "/home/test", + PATH: "/usr/bin", + OPENSHELL_TOKEN: "ambient-secret", + XDG_API_TOKEN: "prefix-secret", + }, + { gatewayName: "target-gateway" }, + ); + const capture = createSystemReadinessCapture(env); + + expect( + capture(["probe", "--readonly"], { + env: { OPENSHELL_GATEWAY_AUTH_TOKEN: "caller-secret" }, + }), + ).toBe("ready"); + expect(subprocess.spawnSync).toHaveBeenCalledWith( + "probe", + ["--readonly"], + expect.objectContaining({ + env: { + HOME: "/home/test", + PATH: "/usr/bin", + OPENSHELL_GATEWAY: "target-gateway", + }, + maxBuffer: 1024 * 1024, + shell: false, + timeout: 15_000, + }), + ); + }); + + it("allows callers to lower the bounded readiness probe limits", () => { + subprocess.spawnSync.mockReturnValue({ status: 0, stdout: "ready", stderr: "" }); + const capture = createSystemReadinessCapture({ PATH: "/usr/bin" }); + + capture(["probe"], { maxBuffer: 2048, timeout: 2500 }); + + expect(subprocess.spawnSync).toHaveBeenCalledWith( + "probe", + [], + expect.objectContaining({ maxBuffer: 2048, timeout: 2500 }), + ); + }); + + it.each([ + "gateway\0spoof", + "gateway\rspoof", + "gateway\nspoof", + ])("rejects an invalid gateway name before spawning: %j", (gatewayName) => { + expect(() => buildSystemReadinessProbeEnv({}, { gatewayName })).toThrow( + "Readiness gateway name contains an invalid character.", + ); + expect(subprocess.spawnSync).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/readiness/probe-env.ts b/src/lib/readiness/probe-env.ts new file mode 100644 index 00000000000..a6a67a99a88 --- /dev/null +++ b/src/lib/readiness/probe-env.ts @@ -0,0 +1,138 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import path from "node:path"; + +import { isSupportedGatewayDockerHost } from "../domain/docker-host"; + +const READINESS_PROBE_TIMEOUT_MS = 15_000; +const READINESS_PROBE_MAX_BUFFER_BYTES = 1024 * 1024; + +const READINESS_PROBE_ENV_NAMES = new Set([ + "HOME", + "USER", + "LOGNAME", + "SHELL", + "PATH", + "TERM", + "HOSTNAME", + "NODE_ENV", + "TMPDIR", + "TMP", + "TEMP", + "LANG", + "LANGUAGE", + "LC_ALL", + "LC_COLLATE", + "LC_CTYPE", + "LC_MESSAGES", + "LC_MONETARY", + "LC_NUMERIC", + "LC_TIME", + "LC_PAPER", + "LC_NAME", + "LC_ADDRESS", + "LC_TELEPHONE", + "LC_MEASUREMENT", + "LC_IDENTIFICATION", + "TZ", + "DBUS_SESSION_BUS_ADDRESS", + "XDG_CONFIG_HOME", + "XDG_BIN_HOME", + "XDG_CACHE_HOME", + "XDG_DATA_HOME", + "XDG_STATE_HOME", + "XDG_RUNTIME_DIR", + "XDG_DATA_DIRS", + "XDG_CONFIG_DIRS", + "SSL_CERT_FILE", + "SSL_CERT_DIR", + "NODE_EXTRA_CA_CERTS", + "HOMEBREW_NO_AUTO_UPDATE", + "HOMEBREW_NO_ENV_HINTS", +]); + +export interface ReadinessProbeEnvironmentControls { + /** Select the exact gateway inspected by unqualified OpenShell status commands. */ + gatewayName?: string; + /** Trusted local mTLS bundle directory for the selected gateway. */ + localTlsDir?: string; +} + +/** + * Replacement environment for read-only readiness children. + * + * Names are exact rather than prefix-based: an ambient variable such as + * `XDG_API_TOKEN` or `LC_CLIENT_SECRET` must never cross this boundary. The + * only product control is the caller-selected gateway name. A Docker endpoint + * is retained solely when it is the supported absolute local Unix socket form. + */ +export function buildSystemReadinessProbeEnv( + source: NodeJS.ProcessEnv = process.env, + controls: ReadinessProbeEnvironmentControls = {}, +): NodeJS.ProcessEnv { + const env: NodeJS.ProcessEnv = {}; + for (const name of READINESS_PROBE_ENV_NAMES) { + const value = source[name]; + if (value !== undefined) env[name] = value; + } + if (source.DOCKER_HOST && isSupportedGatewayDockerHost(source.DOCKER_HOST)) { + env.DOCKER_HOST = source.DOCKER_HOST.trim(); + } + if (controls.gatewayName) { + if (/[\0\r\n]/u.test(controls.gatewayName)) { + throw new Error("Readiness gateway name contains an invalid character."); + } + env.OPENSHELL_GATEWAY = controls.gatewayName; + } + if ( + controls.localTlsDir && + path.isAbsolute(controls.localTlsDir) && + !/[\0\r\n]/u.test(controls.localTlsDir) + ) { + env.OPENSHELL_LOCAL_TLS_DIR = path.normalize(controls.localTlsDir); + } + return env; +} + +type ReadinessRunCapture = typeof import("../runner").runCapture; + +/** Build a runCapture-compatible executor whose environment is replaced, not merged. */ +export function createSystemReadinessCapture(env: NodeJS.ProcessEnv): ReadinessRunCapture { + return (cmd, opts = {}) => { + if (!Array.isArray(cmd) || cmd.length === 0) { + throw new Error("Readiness probe command must be a non-empty argv array."); + } + const [file, ...args] = cmd.map(String); + if (!file || file.includes("\0") || args.some((arg) => arg.includes("\0"))) { + throw new Error("Readiness probe command contains an invalid executable or argument."); + } + const { + ignoreError, + includeStderr, + env: _ignoredCallerEnv, + stdio: _ignoredStdio, + shell, + ...spawnOptions + } = opts; + if (shell) throw new Error("Readiness probe commands cannot enable shell interpretation."); + const result = spawnSync(file, args, { + timeout: READINESS_PROBE_TIMEOUT_MS, + maxBuffer: READINESS_PROBE_MAX_BUFFER_BYTES, + ...spawnOptions, + encoding: "utf-8", + env, + shell: false, + stdio: ["ignore", "pipe", "pipe"], + }); + const stdout = String(result.stdout ?? "").trim(); + const stderr = String(result.stderr ?? "").trim(); + const output = includeStderr ? [stdout, stderr].filter(Boolean).join("\n") : stdout; + if (result.error || result.status !== 0) { + if (ignoreError) return includeStderr ? output : ""; + throw new Error("Readiness probe command failed."); + } + return output; + }; +} diff --git a/src/lib/readiness/sanitize.ts b/src/lib/readiness/sanitize.ts new file mode 100644 index 00000000000..4266d1170f2 --- /dev/null +++ b/src/lib/readiness/sanitize.ts @@ -0,0 +1,17 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { redactFull } from "../security/redact.js"; + +const UNSAFE_TERMINAL_CONTROL_PATTERN = + /[\u0000-\u001f\u007f-\u009f\u061c\u200e\u200f\u2028-\u202e\u2066-\u2069]/gu; + +/** Redact and render terminal/bidirectional controls as inert visible text. */ +export function sanitizeReadinessText(value: string, maxLength: number): string { + return redactFull(value) + .replace( + UNSAFE_TERMINAL_CONTROL_PATTERN, + (character) => `\\u${character.charCodeAt(0).toString(16).padStart(4, "0")}`, + ) + .slice(0, maxLength); +} diff --git a/src/lib/readiness/system.test.ts b/src/lib/readiness/system.test.ts new file mode 100644 index 00000000000..b837ab12d50 --- /dev/null +++ b/src/lib/readiness/system.test.ts @@ -0,0 +1,180 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import Ajv2020, { type AnySchema } from "ajv/dist/2020.js"; +import { describe, expect, it } from "vitest"; +import systemReadinessSchema from "../../../schemas/system-readiness.schema.json" with { + type: "json", +}; +import type { HostAssessment } from "../onboard/preflight"; +import { type GatewayObservationSnapshot, projectGatewayReadiness } from "./gateway"; +import { createPublicReadinessReport } from "./presentation"; +import { getSystemReadinessReferenceErrors } from "./references"; +import { composeSystemReadinessReport, createSystemReadinessReport } from "./system"; +import type { SystemReadinessReport } from "./types"; + +const NOW = new Date("2026-08-07T12:00:00.000Z"); + +function hostReport(): SystemReadinessReport { + return { + schemaVersion: "1.1.0", + status: "supported", + exitCode: 0, + mutated: false, + provenance: { + nemoclawVersion: "0.1.0", + sourceRevision: "a".repeat(40), + observedAt: NOW.toISOString(), + }, + observations: [{ id: "host.os.platform", state: "present", value: "linux" }], + capabilities: [{ id: "host.platform.supported", state: "present" }], + qualifications: [], + findings: [], + evidence: [], + }; +} + +function gatewaySnapshot(): GatewayObservationSnapshot { + return { + observedAt: NOW.toISOString(), + reusable: false, + observations: { + owner: { + gatewayName: "nemoclaw", + gatewayPort: 8080, + mode: "nemoclaw-managed", + source: "standalone", + endpoint: null, + supervisor: null, + requiredCapabilities: [], + }, + attachmentState: "not-applicable", + reuseState: "healthy", + driftState: "not-detected", + portConflictState: "none", + }, + }; +} + +function supportedHost(): HostAssessment { + return { + platform: "linux", + isWsl: false, + runtime: "docker", + dockerInstalled: true, + dockerRunning: true, + dockerReachable: true, + nodeInstalled: true, + openshellInstalled: true, + isContainerRuntimeUnderProvisioned: false, + hasNestedOverlayConflict: false, + requiresHostCgroupnsFix: false, + isUnsupportedRuntime: false, + isHeadlessLikely: false, + hasNvidiaGpu: false, + dockerCdiSpecDirs: [], + cdiNvidiaGpuSpecMissing: false, + nvidiaContainerToolkitInstalled: false, + notes: [], + }; +} + +describe("composite system readiness (#7411)", () => { + // source-shape-contract: compatibility -- Composite readiness output must conform to the published schema before external consumers receive it + it("publishes a schema-valid host and gateway report with resolved references (#7411)", () => { + const report = composeSystemReadinessReport( + hostReport(), + projectGatewayReadiness(gatewaySnapshot(), { now: () => NOW }), + ); + const publicReport = createPublicReadinessReport(report); + const ajv = new Ajv2020({ allErrors: true, strict: true }); + ajv.addFormat("date-time", true); + const validate = ajv.compile(systemReadinessSchema as AnySchema); + + expect(validate(publicReport), JSON.stringify(validate.errors)).toBe(true); + expect(getSystemReadinessReferenceErrors(publicReport)).toEqual([]); + expect(publicReport.observations).toContainEqual( + expect.objectContaining({ + id: "gateway.management.mode", + value: "nemoclaw-managed", + }), + ); + }); + + it("rejects duplicate entity IDs before onboarding can consume them", () => { + const host = hostReport(); + const gateway = projectGatewayReadiness(gatewaySnapshot(), { now: () => NOW }); + gateway.observations[0] = { id: "host.os.platform", state: "present" }; + + expect(() => composeSystemReadinessReport(host, gateway)).toThrow( + "System readiness report failed internal reference validation.", + ); + }); + + it("rejects dangling gateway references at composition", () => { + const gateway = projectGatewayReadiness(gatewaySnapshot(), { now: () => NOW }); + gateway.capabilities[0] = { + ...gateway.capabilities[0], + evidenceIds: ["gateway.evidence.missing"], + }; + + expect(() => composeSystemReadinessReport(hostReport(), gateway)).toThrow( + "System readiness report failed internal reference validation.", + ); + }); + + it("marks host and gateway facts stale when collection exceeds the reuse window", async () => { + let currentTime = NOW.getTime(); + const report = await createSystemReadinessReport( + { + nemoclawVersion: "0.1.0", + sourceRevision: "a".repeat(40), + now: () => new Date(currentTime), + }, + { + host: { + assess: supportedHost, + collectPlatformIdentity: () => ({ + productName: null, + nvidiaPlatform: null, + stationProfile: null, + stationGb300PciGpu: null, + }), + }, + gateway: { + resolveOwner: () => ({ + gatewayName: "nemoclaw", + gatewayPort: 8080, + mode: "nemoclaw-managed", + source: "standalone", + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], + }), + probeAttachment: async () => { + throw new Error("managed attachment probe must not run"); + }, + observeManagedGateway: async () => { + currentTime += 30_001; + return { + reuseState: "healthy", + driftState: "not-detected", + portConflictState: "none", + }; + }, + }, + }, + ); + + expect(report.status).toBe("inconclusive"); + expect(report.evidence).toEqual( + expect.arrayContaining([ + expect.objectContaining({ id: "host.probe.stale" }), + expect.objectContaining({ id: "gateway.probe.stale" }), + ]), + ); + expect(report.capabilities.length).toBeGreaterThan(0); + expect(report.capabilities.every(({ state }) => state === "unknown")).toBe(true); + }); +}); diff --git a/src/lib/readiness/system.ts b/src/lib/readiness/system.ts new file mode 100644 index 00000000000..beada3d95b7 --- /dev/null +++ b/src/lib/readiness/system.ts @@ -0,0 +1,78 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + collectGatewayObservations, + type GatewayReadinessDependencies, + type GatewayReadinessProjection, + projectGatewayReadiness, +} from "./gateway.js"; +import { + type CollectHostObservationsOptions, + type CreateHostReadinessReportOptions, + collectHostObservations, + projectHostReadiness, +} from "./host.js"; +import { getSystemReadinessReferenceErrors } from "./references.js"; +import type { SystemReadinessReport } from "./types.js"; + +export interface CollectSystemReadinessOptions { + gateway: GatewayReadinessDependencies; + host?: CollectHostObservationsOptions; +} + +type ReadinessOutcome = + | { status: "supported"; exitCode: 0 } + | { status: "incompatible"; exitCode: 2 } + | { status: "inconclusive"; exitCode: 3 }; + +function outcomeFor( + capabilities: SystemReadinessReport["capabilities"], + findings: SystemReadinessReport["findings"], +): ReadinessOutcome { + if (findings.some(({ severity }) => severity === "blocking" || severity === "fatal")) { + return { status: "incompatible", exitCode: 2 }; + } + if (capabilities.some(({ state }) => state === "unknown")) { + return { status: "inconclusive", exitCode: 3 }; + } + return { status: "supported", exitCode: 0 }; +} + +/** Combine one host snapshot with the canonical gateway projection. */ +export function composeSystemReadinessReport( + host: Readonly, + gateway: Readonly, +): SystemReadinessReport { + const capabilities = [...host.capabilities, ...gateway.capabilities]; + const findings = [...host.findings, ...gateway.findings]; + + const report: SystemReadinessReport = { + ...host, + ...outcomeFor(capabilities, findings), + mutated: false, + observations: [...host.observations, ...gateway.observations], + capabilities, + findings, + evidence: [...host.evidence, ...gateway.evidence], + }; + if (getSystemReadinessReferenceErrors(report).length > 0) { + throw new Error("System readiness report failed internal reference validation."); + } + return report; +} + +export async function createSystemReadinessReport( + options: CreateHostReadinessReportOptions, + collectionOptions: CollectSystemReadinessOptions, +): Promise { + const now = options.now ?? collectionOptions.host?.now ?? (() => new Date()); + const hostSnapshot = collectHostObservations({ ...collectionOptions.host, now }); + const gatewaySnapshot = await collectGatewayObservations(collectionOptions.gateway, { now }); + // Project both snapshots against the live clock after collection. Freezing + // evaluation at collection start would let a slow gateway probe hide stale + // host or gateway evidence from the 30-second reuse boundary. + const host = projectHostReadiness(hostSnapshot, { ...options, now }); + const gateway = projectGatewayReadiness(gatewaySnapshot, { now }); + return composeSystemReadinessReport(host, gateway); +} diff --git a/src/lib/shields/mcp-policy-transition.test.ts b/src/lib/shields/mcp-policy-transition.test.ts index 74b4b312d80..182a8d82062 100644 --- a/src/lib/shields/mcp-policy-transition.test.ts +++ b/src/lib/shields/mcp-policy-transition.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it } from "vitest"; import YAML from "yaml"; +import { testTimeout } from "../../../test/helpers/timeouts"; import { hasManagedMcpPolicyClaims, @@ -418,33 +419,37 @@ describe("managed MCP Shields policy transitions (#7952)", () => { ]); }); - it("reconciles 257 current managed policies without losing entries (#7952)", () => { - const policies = Array.from({ length: 257 }, (_, index) => - registeredPolicy(`server${index}`, "8.8.8.8"), - ); - const current = inspectExactManagedMcpPolicies( - sandboxWithPolicies(policies), - livePolicy( - policies.map((policy, index) => ({ - content: policy.content, - server: `server${index}`, - })), - ), - ); - const snapshot = YAML.stringify({ - version: 1, - network_policies: { restrictive_baseline: {} }, - }); + it( + "reconciles 257 current managed policies without losing entries (#7952)", + () => { + const policies = Array.from({ length: 257 }, (_, index) => + registeredPolicy(`server${index}`, "8.8.8.8"), + ); + const current = inspectExactManagedMcpPolicies( + sandboxWithPolicies(policies), + livePolicy( + policies.map((policy, index) => ({ + content: policy.content, + server: `server${index}`, + })), + ), + ); + const snapshot = YAML.stringify({ + version: 1, + network_policies: { restrictive_baseline: {} }, + }); - const restored = YAML.parse(composeManagedMcpPolicies(snapshot, current)); + const restored = YAML.parse(composeManagedMcpPolicies(snapshot, current)); - expect(Object.keys(restored.network_policies).sort()).toEqual( - ["restrictive_baseline", ...current.map(({ key }) => key)].sort(), - ); - expect(restored.network_policies.mcp_bridge_server256).toEqual( - current.find(({ key }) => key === "mcp_bridge_server256")?.networkPolicy, - ); - }); + expect(Object.keys(restored.network_policies).sort()).toEqual( + ["restrictive_baseline", ...current.map(({ key }) => key)].sort(), + ); + expect(restored.network_policies.mcp_bridge_server256).toEqual( + current.find(({ key }) => key === "mcp_bridge_server256")?.networkPolicy, + ); + }, + testTimeout(15_000), + ); it("does not restore a managed MCP policy removed during the shields-down window", () => { const alpha = registeredPolicy("alpha", "8.8.8.8"); diff --git a/test/e2e/live/managed-image-activation-e2e-helpers.ts b/test/e2e/live/managed-image-activation-e2e-helpers.ts index ceba5496d6f..e2556d833cb 100644 --- a/test/e2e/live/managed-image-activation-e2e-helpers.ts +++ b/test/e2e/live/managed-image-activation-e2e-helpers.ts @@ -228,6 +228,7 @@ async function runAgentTurn( async function preclean( host: HostCliClient, + lifecycle: LifecyclePhaseFixture, sandbox: SandboxClient, sandboxName: string, env: NodeJS.ProcessEnv, @@ -242,6 +243,12 @@ async function preclean( env, timeoutMs: 60_000, }); + await lifecycle.stopGatewayRuntime(); + await host.cleanupGatewayRegistration(GATEWAY, { + artifactName: `pre-cleanup-gateway-${sandboxName}`, + env, + timeoutMs: 60_000, + }); } async function verifyExactCleanup( @@ -335,7 +342,7 @@ async function qualifyAgent( sandbox.cleanupSandbox(sandboxName, { env, timeoutMs: 60_000 }), ); cleanup.trackSandbox(host, sandboxName, { env, timeoutMs: 3 * 60_000 }); - await preclean(host, sandbox, sandboxName, env); + await preclean(host, lifecycle, sandbox, sandboxName, env); enterOnboardPhase(progress, agent); const onboard = await host.nemoclaw( diff --git a/test/exit-code-user-error-surfaces.test.ts b/test/exit-code-user-error-surfaces.test.ts index 2cea085afce..2e7864a2ccd 100644 --- a/test/exit-code-user-error-surfaces.test.ts +++ b/test/exit-code-user-error-surfaces.test.ts @@ -264,6 +264,8 @@ describe("onboard dashboard-port exhaustion exits non-zero (#5974)", () => { [ "#!/usr/bin/env bash", "# openshell capabilities: request-body-credential-rewrite websocket-credential-rewrite allow_all_known_mcp_methods", + 'if [ "${1:-}" = status ]; then printf "No active gateway\\n"; exit 1; fi', + 'if [ "${1:-}" = gateway ] && [ "${2:-}" = info ]; then printf "No gateway metadata found\\n"; exit 1; fi', 'case "${1:-}" in', ' -V|--version) printf "%s 0.0.101\\n" "${0##*/}"; exit 0;;', "esac", @@ -279,7 +281,7 @@ describe("onboard dashboard-port exhaustion exits non-zero (#5974)", () => { path.join(binDir, "docker"), [ "#!/usr/bin/env bash", - 'if [ "$1" = info ]; then echo "Server Version: 24.0.0"; exit 0; fi', + `if [ "$1" = info ]; then printf '%s\\n' '${JSON.stringify({ ServerVersion: "24.0.0", OperatingSystem: "Docker Desktop", NCPU: 8, MemTotal: 17_179_869_184 })}'; exit 0; fi`, 'if [ "$1" = ps ]; then exit 0; fi', "exit 0", ].join("\n"), diff --git a/test/gateway-state-reconcile-2276.test.ts b/test/gateway-state-reconcile-2276.test.ts index 72de5543a01..09630410dd2 100644 --- a/test/gateway-state-reconcile-2276.test.ts +++ b/test/gateway-state-reconcile-2276.test.ts @@ -13,6 +13,7 @@ import assert from "node:assert/strict"; import { spawnSync } from "node:child_process"; import fs from "node:fs"; +import { createServer } from "node:net"; import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, it } from "vitest"; @@ -23,11 +24,11 @@ const SANDBOX_NAME = "my-assistant"; const OPENSHELL_FIXTURE_VERSION = "0.0.101"; // Output fixtures that mirror real OpenShell CLI output. -const GATEWAY_INFO_NEMOCLAW = - "Gateway Info\n\nGateway: nemoclaw\nGateway endpoint: https://127.0.0.1:8080/\n"; +const gatewayInfoNemoclaw = (gatewayName: string, port: number) => + `Gateway Info\n\nGateway: ${gatewayName}\nGateway endpoint: https://127.0.0.1:${port}/\n`; -const STATUS_CONNECTED_NEMOCLAW = - "Server Status\n\nGateway: nemoclaw\nServer: https://127.0.0.1:8080/\nStatus: Connected\n"; +const statusConnectedNemoclaw = (gatewayName: string, port: number) => + `Server Status\n\nGateway: ${gatewayName}\nServer: https://127.0.0.1:${port}/\nStatus: Connected\n`; const SANDBOX_GET_NOT_FOUND = "Error: sandbox my-assistant not found"; interface ScenarioScript { @@ -61,8 +62,10 @@ let openshellPath: string; let stateFile: string; let scriptFile: string; let installerInvocationsFile: string; +let dockerInvocationsFile: string; +let gatewayListener: ReturnType | null; -function writeDefaultRegistry() { +function writeDefaultRegistry(gatewayName: string, gatewayPort: number) { fs.writeFileSync( path.join(registryDir, "sandboxes.json"), JSON.stringify({ @@ -74,8 +77,8 @@ function writeDefaultRegistry() { provider: "nvidia-prod", gpuEnabled: false, sandboxGpuMode: "0", - gatewayName: "nemoclaw", - gatewayPort: 8080, + gatewayName, + gatewayPort, dashboardPort: 28790, fromDockerfile: null, policies: [], @@ -86,13 +89,14 @@ function writeDefaultRegistry() { ); } -function writeDefaultSession() { +function writeDefaultSession(gatewayName: string) { fs.writeFileSync( path.join(registryDir, "onboard-session.json"), JSON.stringify({ version: 1, sandboxName: SANDBOX_NAME, provider: "nvidia-prod", + metadata: { gatewayName, fromDockerfile: null }, }), { mode: 0o600 }, ); @@ -192,6 +196,7 @@ process.exit(0); function runCli(action: string, extraEnv: Record = {}): HarnessResult { const repoRoot = path.join(import.meta.dirname, ".."); + const nodeBinDir = path.dirname(process.execPath); const result = spawnSync( process.execPath, [path.join(repoRoot, "bin", "nemoclaw.js"), SANDBOX_NAME, action], @@ -202,7 +207,7 @@ function runCli(action: string, extraEnv: Record = { env: { ...process.env, HOME: tmpDir, - PATH: `${homeLocalBin}:/usr/bin:/bin`, + PATH: `${homeLocalBin}:${nodeBinDir}:/usr/bin:/bin`, // Keep output deterministic. NO_COLOR: "1", ...extraEnv, @@ -243,7 +248,58 @@ function registrySandboxPresent(r: HarnessResult): boolean { ); } +function writeDockerStub(gatewayName: string, gatewayPort: number) { + fs.writeFileSync( + path.join(homeLocalBin, "docker"), + `#!${process.execPath} +const a = process.argv.slice(2); +const { isOpenClawSecurityInventoryProbe } = require(${JSON.stringify( + path.join(import.meta.dirname, "helpers", "onboard-script-mocks.cjs"), + )}); +const invocationLog = ${JSON.stringify(dockerInvocationsFile)}; +require("fs").appendFileSync(invocationLog, JSON.stringify(a) + "\\n"); +if (a[0] === "info") { + process.stdout.write(JSON.stringify({ServerVersion:"27.0.0", OperatingSystem:"Docker Desktop", NCPU:8, MemTotal:17179869184}) + "\\n"); + process.exit(0); +} +if (a[0] === "inspect") { + const expectedTarget = ${JSON.stringify(`openshell-cluster-${gatewayName}`)}; + const target = a.at(-1); + const formatIndex = a.indexOf("--format"); + const format = formatIndex >= 0 ? a[formatIndex + 1] : ""; + const gatewayPort = ${JSON.stringify(String(gatewayPort))}; + const responses = new Map([ + ["{{.State.Running}}", "true\\n"], + ["{{json .NetworkSettings.Ports}}", JSON.stringify({[gatewayPort + "/tcp"]:[{HostPort:gatewayPort}]}) + "\\n"], + ["{{.Config.Image}}", "nvcr.io/nvidia/openshell/cluster:0.0.101\\n"], + ]); + if (target !== expectedTarget || !responses.has(format)) process.exit(64); + process.stdout.write(responses.get(format)); + process.exit(0); +} +if (a[0] === "build") process.exit(0); +if (a[0] === "image" && a[1] === "inspect") { + const formatIndex = a.indexOf("--format"); + const format = formatIndex >= 0 ? a[formatIndex + 1] : ""; + if (format === "{{.Id}}") process.stdout.write("sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\\n"); + if (format === "{{json .RepoDigests}}") process.stdout.write("[]\\n"); + process.exit(0); +} +if (a[0] === "tag" || a[0] === "rmi") process.exit(0); +if (a[0] === "run") { + if (a.includes("nslookup")) process.stdout.write("Server: 127.0.0.11\\n** server can't find nemoclaw.invalid: NXDOMAIN\\n"); + else if (isOpenClawSecurityInventoryProbe(a)) process.stdout.write("nemoclaw-security-inventory-ok\\n"); + else if (a.includes("/usr/bin/ldd")) process.stdout.write("ldd (GNU libc) 2.41\\n"); + process.exit(0); +} +process.exit(0); +`, + { mode: 0o755 }, + ); +} + beforeEach(() => { + gatewayListener = null; tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-2276-")); homeLocalBin = path.join(tmpDir, ".local", "bin"); registryDir = path.join(tmpDir, ".nemoclaw"); @@ -251,10 +307,12 @@ beforeEach(() => { stateFile = path.join(tmpDir, "state.json"); scriptFile = path.join(tmpDir, "script.json"); installerInvocationsFile = path.join(tmpDir, "installer-invocations.log"); + dockerInvocationsFile = path.join(tmpDir, "docker-invocations.log"); fs.mkdirSync(homeLocalBin, { recursive: true }); fs.mkdirSync(registryDir, { recursive: true }); fs.writeFileSync(installerInvocationsFile, ""); + fs.writeFileSync(dockerInvocationsFile, ""); fs.writeFileSync( path.join(homeLocalBin, "bash"), `#!${process.execPath} @@ -273,41 +331,13 @@ process.exit(result.status ?? 1); `, { mode: 0o755 }, ); - writeDefaultRegistry(); - writeDefaultSession(); - fs.writeFileSync( - path.join(homeLocalBin, "docker"), - `#!${process.execPath} -const a = process.argv.slice(2); -const { isOpenClawSecurityInventoryProbe } = require(${JSON.stringify( - path.join(import.meta.dirname, "helpers", "onboard-script-mocks.cjs"), - )}); -if (a[0] === "info") { - process.stdout.write(JSON.stringify({ServerVersion:"27.0.0", OperatingSystem:"Docker Engine", NCPU:8, MemTotal:17179869184}) + "\\n"); - process.exit(0); -} -if (a[0] === "build") process.exit(0); -if (a[0] === "image" && a[1] === "inspect") { - const formatIndex = a.indexOf("--format"); - const format = formatIndex >= 0 ? a[formatIndex + 1] : ""; - if (format === "{{.Id}}") process.stdout.write("sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\\n"); - if (format === "{{json .RepoDigests}}") process.stdout.write("[]\\n"); - process.exit(0); -} -if (a[0] === "tag" || a[0] === "rmi") process.exit(0); -if (a[0] === "run") { - if (a.includes("nslookup")) process.stdout.write("Server: 127.0.0.11\\n** server can't find nemoclaw.invalid: NXDOMAIN\\n"); - else if (isOpenClawSecurityInventoryProbe(a)) process.stdout.write("nemoclaw-security-inventory-ok\\n"); - else if (a.includes("/usr/bin/ldd")) process.stdout.write("ldd (GNU libc) 2.41\\n"); - process.exit(0); -} -process.exit(0); -`, - { mode: 0o755 }, - ); + fs.writeFileSync(path.join(homeLocalBin, "lsof"), "#!/usr/bin/env bash\nexit 1\n", { + mode: 0o755, + }); }); -afterEach(() => { +afterEach(async () => { + await new Promise((resolve) => gatewayListener?.close(() => resolve()) ?? resolve()); fs.rmSync(tmpDir, { recursive: true, force: true }); }); @@ -326,11 +356,23 @@ afterEach(() => { describe("connect preserves the registry so rebuild can recover in scenario 14 (#4497)", () => { it("after a non-destructive connect, `rebuild --yes` recovers the stale sandbox", { timeout: TIMEOUT_MS, - }, () => { + }, async () => { + gatewayListener = createServer(); + await new Promise((resolve, reject) => { + gatewayListener?.once("error", reject); + gatewayListener?.listen(0, "127.0.0.1", resolve); + }); + const gatewayAddress = gatewayListener.address(); + assert.ok(gatewayAddress && typeof gatewayAddress !== "string"); + const gatewayPort = gatewayAddress.port; + const gatewayName = `nemoclaw-${gatewayPort}`; + writeDefaultRegistry(gatewayName, gatewayPort); + writeDefaultSession(gatewayName); + writeDockerStub(gatewayName, gatewayPort); writeStubOpenshell({ sandboxGet: [{ output: SANDBOX_GET_NOT_FOUND, exit: 1 }], - status: [{ output: STATUS_CONNECTED_NEMOCLAW, exit: 0 }], - gatewayInfo: [{ output: GATEWAY_INFO_NEMOCLAW, exit: 0 }], + status: [{ output: statusConnectedNemoclaw(gatewayName, gatewayPort), exit: 0 }], + gatewayInfo: [{ output: gatewayInfoNemoclaw(gatewayName, gatewayPort), exit: 0 }], gatewaySelect: { output: "", exit: 0 }, selectFlipsActive: false, sandboxList: "", @@ -353,6 +395,7 @@ describe("connect preserves the registry so rebuild can recover in scenario 14 ( // impossible backup (instead of dead-ending at "Cannot back up state"), // and proceeds to recreate from the preserved metadata. const repoRoot = path.join(import.meta.dirname, ".."); + const nodeBinDir = path.dirname(process.execPath); const rebuild = spawnSync( process.execPath, [path.join(repoRoot, "bin", "nemoclaw.js"), SANDBOX_NAME, "rebuild", "--yes"], @@ -363,7 +406,7 @@ describe("connect preserves the registry so rebuild can recover in scenario 14 ( env: { ...process.env, HOME: tmpDir, - PATH: `${homeLocalBin}:/usr/bin:/bin`, + PATH: `${homeLocalBin}:${nodeBinDir}:/usr/bin:/bin`, NO_COLOR: "1", NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", NEMOCLAW_SKIP_HOST_DNS_PREFLIGHT: "1", @@ -382,6 +425,10 @@ describe("connect preserves the registry so rebuild can recover in scenario 14 ( .readFileSync(installerInvocationsFile, "utf8") .split("\n") .filter(Boolean); + const dockerInvocations = fs + .readFileSync(dockerInvocationsFile, "utf8") + .split("\n") + .filter(Boolean); assert.equal( installerInvocations.length, @@ -423,7 +470,7 @@ describe("connect preserves the registry so rebuild can recover in scenario 14 ( assert.match( rebuildOut, /absent from the live OpenShell gateway/, - `rebuild must report the stale-recovery state (#4497), got:\n${rebuildOut}`, + `rebuild must report the stale-recovery state (#4497), got:\n${rebuildOut}\nDocker invocations:\n${dockerInvocations.join("\n")}`, ); assert.match( rebuildOut, diff --git a/test/helpers/ephemeral-gateway-listener.ts b/test/helpers/ephemeral-gateway-listener.ts new file mode 100644 index 00000000000..2220f312cf7 --- /dev/null +++ b/test/helpers/ephemeral-gateway-listener.ts @@ -0,0 +1,22 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import net from "node:net"; + +const markerPath = process.argv[2]; +if (!markerPath) throw new Error("A readiness marker path is required."); + +const server = net.createServer((socket) => socket.end()); +server.once("error", (error) => { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exit(1); +}); +server.listen(0, "127.0.0.1", () => { + const address = server.address(); + if (!address || typeof address === "string") { + process.stderr.write("The gateway fixture did not receive a TCP port.\n"); + process.exit(1); + } + fs.writeFileSync(markerPath, `${address.port}\n`, { mode: 0o600 }); +}); diff --git a/test/helpers/installer-readiness-stubs.ts b/test/helpers/installer-readiness-stubs.ts new file mode 100644 index 00000000000..9c0e3e23065 --- /dev/null +++ b/test/helpers/installer-readiness-stubs.ts @@ -0,0 +1,22 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; + +export function writeInstallerReadinessModuleStubs(readinessDir: string): void { + fs.mkdirSync(readinessDir, { recursive: true }); + fs.writeFileSync( + `${readinessDir}/host.js`, + `exports.createHostReadinessReport = (_options, collection) => ({ host: collection.assess() });\n`, + ); + fs.writeFileSync( + `${readinessDir}/onboard-admission.js`, + `exports.evaluateOnboardReadinessAdmission = (report) => { + const host = report.host; + const unsupportedRuntime = host.runtime === "podman" || host.isUnsupportedRuntime === true; + const cdiBlocks = host.cdiNvidiaGpuSpecNeedsRepair && !(host.isWsl && host.runtime === "docker-desktop"); + return { admitted: !unsupportedRuntime && !cdiBlocks }; +}; +`, + ); +} diff --git a/test/helpers/rebuild-flow-harness.ts b/test/helpers/rebuild-flow-harness.ts index ea64af4382a..89d7280d451 100644 --- a/test/helpers/rebuild-flow-harness.ts +++ b/test/helpers/rebuild-flow-harness.ts @@ -323,20 +323,30 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): dockerfileBasePath: "/tmp/Dockerfile.base", runtime: { kind: "terminal" }, }; + const resolveGatewayAuthority = ({ + gatewayName, + gatewayPort, + }: { + gatewayName: string; + gatewayPort: number; + }) => ({ + gatewayName, + gatewayPort, + mode: "nemoclaw-managed" as const, + source: "standalone" as const, + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], + }); vi.spyOn(gatewayDrift, "detectOpenShellStateRpcPreflightIssue").mockReturnValue(null); vi.spyOn(gatewayDrift, "detectOpenShellStateRpcResultIssue").mockReturnValue(null); vi.spyOn(gatewayTeardownAuthority, "resolveGatewayTeardownAuthority").mockImplementation( - ({ gatewayName, gatewayPort }: { gatewayName: string; gatewayPort: number }) => ({ - gatewayName, - gatewayPort, - mode: "nemoclaw-managed", - source: "standalone", - endpoint: null, - stateDir: null, - supervisor: null, - requiredCapabilities: [], - }), + resolveGatewayAuthority, + ); + vi.spyOn(gatewayTeardownAuthority, "resolveGatewayRebuildAuthority").mockImplementation( + resolveGatewayAuthority, ); vi.spyOn(sandboxList, "captureSandboxListWithGatewayRecovery").mockResolvedValue({ result: { status: 0, output: overrides.sandboxListOutput ?? "alpha Ready" }, @@ -674,9 +684,18 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): const preflightAuthoritativeRebuildTargetSpy = vi .spyOn(rebuildOnboardDependencies, "preflightAuthoritativeRebuildTarget") .mockImplementation(async (options: unknown) => { - await overrides.preflightAuthoritativeRebuildTarget?.( - (options ?? {}) as Record, - ); + const preflightOptions = (options ?? {}) as Record; + await overrides.preflightAuthoritativeRebuildTarget?.(preflightOptions); + return { + gatewayName: String(preflightOptions.targetGatewayName ?? "nemoclaw"), + gatewayPort: Number(preflightOptions.targetGatewayPort ?? 8080), + mode: "nemoclaw-managed", + source: "standalone", + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], + }; }); const livePolicyPresets = new Set(overrides.gatewayPresets ?? []); const managedObservabilityPreset = "observability-otlp-local"; diff --git a/test/helpers/rebuild-flow-test-harness.ts b/test/helpers/rebuild-flow-test-harness.ts index a3f5b3ba95c..144a1ab0545 100644 --- a/test/helpers/rebuild-flow-test-harness.ts +++ b/test/helpers/rebuild-flow-test-harness.ts @@ -93,20 +93,30 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): expectedVersion: "0.2.0", policyAdditionsPath, }; + const resolveGatewayAuthority = ({ + gatewayName, + gatewayPort, + }: { + gatewayName: string; + gatewayPort: number; + }) => ({ + gatewayName, + gatewayPort, + mode: "nemoclaw-managed" as const, + source: "standalone" as const, + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], + }); vi.spyOn(gatewayDrift, "detectOpenShellStateRpcPreflightIssue").mockReturnValue(null); vi.spyOn(gatewayDrift, "detectOpenShellStateRpcResultIssue").mockReturnValue(null); vi.spyOn(gatewayTeardownAuthority, "resolveGatewayTeardownAuthority").mockImplementation( - ({ gatewayName, gatewayPort }: { gatewayName: string; gatewayPort: number }) => ({ - gatewayName, - gatewayPort, - mode: "nemoclaw-managed", - source: "standalone", - endpoint: null, - stateDir: null, - supervisor: null, - requiredCapabilities: [], - }), + resolveGatewayAuthority, + ); + vi.spyOn(gatewayTeardownAuthority, "resolveGatewayRebuildAuthority").mockImplementation( + resolveGatewayAuthority, ); vi.spyOn(sandboxList, "captureSandboxListWithGatewayRecovery").mockResolvedValue({ result: { @@ -549,6 +559,16 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): policies.resolveSandboxBaselinePolicy(String(preflightOptions.sandboxName ?? "")); } await overrides.preflightAuthoritativeRebuildTarget?.(preflightOptions); + return { + gatewayName: String(preflightOptions.targetGatewayName ?? "nemoclaw"), + gatewayPort: Number(preflightOptions.targetGatewayPort ?? 8080), + mode: "nemoclaw-managed", + source: "standalone", + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], + }; }, ); const ensureValidatedBraveSearchCredentialSpy = vi diff --git a/test/install-preflight.test.ts b/test/install-preflight.test.ts index 02ef8ac48d8..517d489d143 100644 --- a/test/install-preflight.test.ts +++ b/test/install-preflight.test.ts @@ -6,13 +6,14 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; +import { writeInstallerReadinessModuleStubs } from "./helpers/installer-readiness-stubs"; +import { writeNpmStub } from "./helpers/installer-run-fixture"; import { INSTALLER_PAYLOAD, readShellConstant, TEST_SYSTEM_PATH, writeExecutable, } from "./helpers/installer-sourced-env"; -import { writeNpmStub } from "./helpers/installer-run-fixture"; const INSTALLER = path.join(import.meta.dirname, "..", "install.sh"); const CURL_PIPE_INSTALLER = path.join(import.meta.dirname, "..", "install.sh"); @@ -784,7 +785,7 @@ exit 0 path.join(fakeBin, "docker"), `#!/usr/bin/env bash if [ "$1" = "info" ]; then - echo '{"ServerVersion":"29.3.1","OperatingSystem":"Ubuntu 26.04 LTS","CgroupVersion":"2"}' + echo '{"ServerVersion":"29.3.1","Name":"Docker Desktop","OperatingSystem":"Ubuntu 26.04 LTS","CgroupVersion":"2"}' exit 0 fi exit 0 @@ -886,7 +887,7 @@ fi`, path.join(fakeBin, "docker"), `#!/usr/bin/env bash if [ "$1" = "info" ]; then - echo '{"ServerVersion":"29.3.1","OperatingSystem":"Ubuntu 24.04","CgroupVersion":"2"}' + echo '{"ServerVersion":"29.3.1","Name":"Docker Desktop","OperatingSystem":"Ubuntu 24.04","CgroupVersion":"2"}' exit 0 fi exit 0 @@ -993,7 +994,7 @@ fi`, path.join(fakeBin, "docker"), `#!/usr/bin/env bash if [ "$1" = "info" ]; then - echo '{"ServerVersion":"29.3.1","OperatingSystem":"Ubuntu 24.04","CgroupVersion":"2"}' + echo '{"ServerVersion":"29.3.1","Name":"Docker Desktop","OperatingSystem":"Ubuntu 24.04","CgroupVersion":"2"}' exit 0 fi exit 0 @@ -1200,31 +1201,32 @@ exports.getNvidiaCdiSpecPath = (host) => String(host.dockerCdiSpecDirs[0]).replace(/\\/+$/, "") + "/nvidia.yaml"; exports.isWslDockerDesktopRuntime = (host) => Boolean(host && host.isWsl && host.runtime === "docker-desktop"); -exports.planHostRemediation = (host) => +exports.planHostAdvisories = (host) => host.cdiNvidiaGpuSpecMissing ? host.isWsl && host.runtime === "docker-desktop" ? [{ title: "Use Docker Desktop WSL GPU compatibility path", reason: "missing nvidia.com/gpu CDI; using Docker --gpus", commands: ["verify Docker --gpus support from WSL"], - blocking: false, + severity: "info", }] : [{ title: "Generate NVIDIA CDI device specs", reason: "missing nvidia.com/gpu", commands: ["sudo nvidia-ctk cdi generate --output=" + exports.getNvidiaCdiSpecPath(host)], - blocking: true, + severity: "blocking", }] : host.cdiNvidiaGpuSpecStale && !host.nvidiaContainerToolkitInstalled ? [{ title: "Install NVIDIA Container Toolkit and refresh CDI device specs", reason: "nvidia-container-toolkit missing", commands: ["sudo apt-get install -y nvidia-container-toolkit"], - blocking: true, + severity: "blocking", }] : []; `, ); + writeInstallerReadinessModuleStubs(path.join(sourceRoot, "dist", "lib", "readiness")); writeNodeStub(fakeBin); writeExecutable( path.join(fakeBin, "sudo"), @@ -1444,7 +1446,7 @@ exit 0 expect(sudoLog).toBe(""); }); - it("warns on Podman but still runs onboarding", () => { + it("rejects Podman through canonical installer admission (#7411)", () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-install-podman-warning-")); const fakeBin = path.join(tmp, "bin"); const prefix = path.join(tmp, "prefix"); @@ -1513,15 +1515,11 @@ exit 0 }); const output = `${result.stdout}${result.stderr}`; - expect(result.status).toBe(0); - expect(output).toMatch(/Host preflight found warnings\./); + expect(result.status).toBe(1); + expect(output).toMatch(/Host preflight found issues that will prevent onboarding right now\./); expect(output).toMatch(/Detected container runtime: podman/); - expect(output).toMatch( - /Podman may work in some environments, but it is not a supported runtime/, - ); - expect(fs.readFileSync(onboardLog, "utf-8")).toMatch( - /^onboard --non-interactive --yes-i-accept-third-party-software --yes$/m, - ); + expect(output).toMatch(/Skipping onboarding until the host prerequisites above are fixed\./); + expect(fs.existsSync(onboardLog)).toBe(false); }); it("requires explicit terms acceptance in non-interactive install mode", () => { @@ -1537,7 +1535,7 @@ exit 0 path.join(fakeBin, "docker"), `#!/usr/bin/env bash if [ "$1" = "info" ]; then - echo '{"ServerVersion":"29.3.1","OperatingSystem":"Ubuntu 24.04","CgroupVersion":"2"}' + echo '{"ServerVersion":"29.3.1","Name":"Docker Desktop","OperatingSystem":"Ubuntu 24.04","CgroupVersion":"2"}' exit 0 fi exit 0 @@ -1608,7 +1606,7 @@ fi`, path.join(fakeBin, "docker"), `#!/usr/bin/env bash if [ "$1" = "info" ]; then - echo '{"ServerVersion":"29.3.1","OperatingSystem":"Ubuntu 24.04","CgroupVersion":"2"}' + echo '{"ServerVersion":"29.3.1","Name":"Docker Desktop","OperatingSystem":"Ubuntu 24.04","CgroupVersion":"2"}' exit 0 fi exit 0 @@ -2005,7 +2003,7 @@ exit 0 path.join(fakeBin, "docker"), `#!/usr/bin/env bash if [ "$1" = "info" ]; then - echo '{"ServerVersion":"29.3.1","OperatingSystem":"Ubuntu 24.04","CgroupVersion":"2"}' + echo '{"ServerVersion":"29.3.1","Name":"Docker Desktop","OperatingSystem":"Ubuntu 24.04","CgroupVersion":"2"}' exit 0 fi exit 0 @@ -3881,7 +3879,7 @@ function writeDockerOkStub(fakeBin: string) { path.join(fakeBin, "docker"), `#!/usr/bin/env bash if [ "$1" = "info" ]; then - echo '{"ServerVersion":"29.3.1","OperatingSystem":"Ubuntu 24.04","CgroupVersion":"2"}' + echo '{"ServerVersion":"29.3.1","Name":"Docker Desktop","OperatingSystem":"Ubuntu 24.04","CgroupVersion":"2"}' exit 0 fi exit 0 diff --git a/test/managed-image-publication-workflow.test.ts b/test/managed-image-publication-workflow.test.ts index 44b6c9cdc69..4778e43db75 100644 --- a/test/managed-image-publication-workflow.test.ts +++ b/test/managed-image-publication-workflow.test.ts @@ -545,6 +545,8 @@ describe("complete managed-image publication workflow", () => { expect(source).toContain("await runAgentTurn("); expect(source).toContain("await sandbox.cleanupSandbox("); expect(source).toContain("managed activation attempted a forbidden Dockerfile build"); + expect(source).toContain("await lifecycle.stopGatewayRuntime()"); + expect(source).toContain("await host.cleanupGatewayRegistration(GATEWAY"); expect(source).toContain("startFakeOpenAiCompatibleServer"); expect(source).not.toContain("runSandboxGpuCreateFlow"); expect(source).not.toContain("createDockerManagedBootstrapAdapter"); diff --git a/test/mcp-destroy-lifecycle.test.ts b/test/mcp-destroy-lifecycle.test.ts index 7503424a21e..27e9c6e7a6b 100644 --- a/test/mcp-destroy-lifecycle.test.ts +++ b/test/mcp-destroy-lifecycle.test.ts @@ -99,6 +99,16 @@ function stubRecreateJournal(): RebuildRecreateJournal { id: "journal-1", acceptedTarget: false, sourceConfirmedAbsent: false, + gatewayAuthority: { + gatewayName: "nemoclaw", + gatewayPort: 8080, + mode: "nemoclaw-managed", + source: "standalone", + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], + }, targetGeneration: "generation-1", targetIntentFingerprint: "intent-1", markDeleting: vi.fn(), diff --git a/test/onboard-gateway-port-conflict-fast-fail.test.ts b/test/onboard-gateway-port-conflict-fast-fail.test.ts index 56221ba95c5..d1b06fb504d 100644 --- a/test/onboard-gateway-port-conflict-fast-fail.test.ts +++ b/test/onboard-gateway-port-conflict-fast-fail.test.ts @@ -1,7 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import fs from "node:fs"; +import type { AddressInfo } from "node:net"; +import net from "node:net"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; @@ -14,15 +15,20 @@ import { import { testTimeoutOptions } from "./helpers/timeouts"; const CLI = path.join(import.meta.dirname, "..", "bin", "nemoclaw.js"); -const GATEWAY_PORT = "18080"; -describe("onboard gateway port conflict fast-fail (#6752)", () => { +describe("onboard gateway port conflict readiness (#6752)", () => { let workspace: OnboardProcessWorkspace; - let openshellCallLog: string; + let gatewayPort: number; + let gatewayServer: net.Server; - beforeEach(() => { + beforeEach(async () => { workspace = createOnboardProcessWorkspace("nemoclaw-6752-"); - openshellCallLog = workspace.path("openshell-calls.log"); + gatewayServer = net.createServer(); + await new Promise((resolve, reject) => { + gatewayServer.once("error", reject); + gatewayServer.listen(0, "127.0.0.1", resolve); + }); + gatewayPort = (gatewayServer.address() as AddressInfo).port; for (const component of ["openshell", "openshell-gateway", "openshell-sandbox"]) { workspace.writeExecutable( @@ -30,10 +36,10 @@ describe("onboard gateway port conflict fast-fail (#6752)", () => { [ "#!/usr/bin/env bash", "# openshell capabilities: request-body-credential-rewrite websocket-credential-rewrite allow_all_known_mcp_methods", - `printf '%s\\n' "$*" >> ${JSON.stringify(openshellCallLog)}`, 'case "$*" in', ' --version|-V) printf "%s 0.0.101\\n" "${0##*/}"; exit 0;;', - ' status|"gateway info"|"gateway info -g nemoclaw"*) sleep 20; exit 0;;', + ' status) printf "No active gateway\\n"; exit 1;;', + ' "gateway info"|"gateway info -g nemoclaw"*) printf "No gateway metadata found\\n"; exit 1;;', "esac", "exit 1", ].join("\n"), @@ -60,8 +66,9 @@ describe("onboard gateway port conflict fast-fail (#6752)", () => { 'for arg in "$@"; do', ' case "$arg" in :*) port="${arg#:}";; esac', "done", - `if [ "$port" = ${JSON.stringify(GATEWAY_PORT)} ]; then`, - ` echo "python3 1234 test 1u IPv4 TCP 127.0.0.1:${GATEWAY_PORT} (LISTEN)"`, + `if [ "$port" = ${JSON.stringify(String(gatewayPort))} ]; then`, + ' if [ "$1" = -ti ]; then printf "%s\\n" "$PPID"; exit 0; fi', + ` echo "python3 $PPID test 1u IPv4 TCP 127.0.0.1:${String(gatewayPort)} (LISTEN)"`, " exit 0", "fi", "exit 1", @@ -69,21 +76,22 @@ describe("onboard gateway port conflict fast-fail (#6752)", () => { ); }); - afterEach(() => { + afterEach(async () => { + await new Promise((resolve) => gatewayServer.close(() => resolve())); workspace.remove(); }); it( - "reports a foreign listener before OpenShell gateway inspection can hang", - testTimeoutOptions(10_000), + "rejects a foreign listener without waiting on lifecycle inspection", + testTimeoutOptions(15_000), () => { const result = runOnboardProcess( [CLI, "onboard", "--name", "foreign-port", "--no-gpu", "--non-interactive"], { - timeoutMs: 5_000, + timeoutMs: 10_000, env: workspaceEnv(workspace, { NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", - NEMOCLAW_GATEWAY_PORT: GATEWAY_PORT, + NEMOCLAW_GATEWAY_PORT: String(gatewayPort), NEMOCLAW_OPENSHELL_BIN: path.join(workspace.binDir, "openshell"), NEMOCLAW_OPENSHELL_CHANNEL: "stable", NEMOCLAW_OPENSHELL_GATEWAY_BIN: path.join(workspace.binDir, "openshell-gateway"), @@ -95,16 +103,15 @@ describe("onboard gateway port conflict fast-fail (#6752)", () => { ); const combined = result.output; - const calls = fs.existsSync(openshellCallLog) - ? fs.readFileSync(openshellCallLog, "utf8") - : ""; expect(result.error).toBeUndefined(); expect(result.signal).toBeNull(); expect(result.status).toBeGreaterThan(0); - expect(combined).toContain(`Port ${GATEWAY_PORT} is not available.`); - expect(combined).toContain("Blocked by: python3 (PID 1234)"); - expect(combined).toContain("NEMOCLAW_GATEWAY_PORT= nemoclaw onboard"); - expect(calls).not.toMatch(/^(status|gateway info(?: -g nemoclaw)?)$/m); + expect(combined).toMatch( + new RegExp(`(?:Port|Gateway port) ${String(gatewayPort)} (?:is not available|is occupied)`), + ); + expect(combined).toMatch( + /The gateway port is held by an incompatible or ambiguous owner|OpenShell gateway needs this port/, + ); }, ); }); diff --git a/test/onboard-inference-reconciliation.test.ts b/test/onboard-inference-reconciliation.test.ts index a44375fbd12..16b991d34da 100644 --- a/test/onboard-inference-reconciliation.test.ts +++ b/test/onboard-inference-reconciliation.test.ts @@ -183,6 +183,15 @@ describe("onboard helpers", () => { const onboardProbesPath = JSON.stringify( path.join(repoRoot, "src", "lib", "inference", "onboard-probes.ts"), ); + const preflightGatewayAuthorityPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "onboard", "machine", "preflight-gateway-authority.ts"), + ); + const preflightPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "onboard", "preflight.ts"), + ); + const bridgeDnsPreflightPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "onboard", "bridge-dns-preflight.ts"), + ); fs.mkdirSync(fakeBin, { recursive: true }); writeOkOpenshell(fakeBin); @@ -197,6 +206,30 @@ const gatewayState = require(${gatewayStatePath}); const dockerDriverPlatform = require(${dockerDriverPlatformPath}); const gatewayGpuPassthrough = require(${gatewayGpuPassthroughPath}); const onboardProbes = require(${onboardProbesPath}); +const preflight = require(${preflightPath}); +preflight.assessHost = () => ({ + platform: "linux", + isWsl: false, + runtime: "docker", + dockerInstalled: true, + dockerRunning: true, + dockerReachable: true, + nodeInstalled: true, + openshellInstalled: true, + isContainerRuntimeUnderProvisioned: false, + hasNestedOverlayConflict: false, + requiresHostCgroupnsFix: false, + isUnsupportedRuntime: false, + isHeadlessLikely: false, + hasNvidiaGpu: false, + dockerCdiSpecDirs: [], + cdiNvidiaGpuSpecMissing: false, + nvidiaContainerToolkitInstalled: false, + notes: [], +}); +const bridgeDnsPreflight = require(${bridgeDnsPreflightPath}); +bridgeDnsPreflight.assertDockerBridgeAndContainerDnsHealthy = () => {}; +const preflightGatewayAuthority = require(${preflightGatewayAuthorityPath}); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); const commands = []; @@ -276,6 +309,18 @@ gatewayState.isGatewayHealthy = () => true; dockerDriverPlatform.isLinuxDockerDriverGatewayEnabled = () => false; gatewayGpuPassthrough.reconcileGatewayGpuReuseForGpuIntent = ({ gatewayReuseState }) => gatewayReuseState; onboardProbes.verifyOnboardInferenceSmoke = () => {}; +preflightGatewayAuthority.collectOnboardGatewayReadiness = async () => ({ + observations: [{ id: "gateway.management.mode", state: "present", value: "nemoclaw-managed" }], + capabilities: [ + "gateway.authority.resolved", + "gateway.attachment.valid", + "gateway.reuse.ready", + "gateway.version.compatible", + "gateway.port.uncontested", + ].map((id) => ({ id, state: "present" })), + findings: [], + evidence: [], +}); const complete = () => ({ status: "complete", diff --git a/test/rebuild-credential-preflight.test.ts b/test/rebuild-credential-preflight.test.ts index f3fb38a192f..41273429072 100644 --- a/test/rebuild-credential-preflight.test.ts +++ b/test/rebuild-credential-preflight.test.ts @@ -10,7 +10,7 @@ * preflight, and the CLI exit status. */ -import { spawnSync } from "node:child_process"; +import { spawn, spawnSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -19,9 +19,14 @@ import { execTimeout } from "./helpers/timeouts"; const REPO_ROOT = path.join(import.meta.dirname, ".."); const NODE_BIN = path.dirname(process.execPath); +const DOCKER_OPERATING_SYSTEM = + ({ darwin: "Docker Desktop" } as Partial>)[process.platform] ?? + "Docker Engine"; const tmpFixtures: string[] = []; +const gatewayProcesses: ReturnType[] = []; afterEach(() => { + for (const child of gatewayProcesses.splice(0)) child.kill(); for (const dir of tmpFixtures.splice(0)) { try { fs.rmSync(dir, { recursive: true, force: true }); @@ -51,6 +56,37 @@ function createFixture(opts: { const nemoclawDir = path.join(tmpDir, ".nemoclaw"); fs.mkdirSync(nemoclawDir, { recursive: true, mode: 0o700 }); + const gatewayReadyMarker = path.join(tmpDir, "gateway-ready"); + const gatewayProcess = spawn( + process.execPath, + [ + "--experimental-strip-types", + path.join(REPO_ROOT, "test", "helpers", "ephemeral-gateway-listener.ts"), + gatewayReadyMarker, + ], + { stdio: "ignore" }, + ); + gatewayProcesses.push(gatewayProcess); + const gatewayWait = spawnSync( + process.execPath, + [ + "-e", + `const fs = require("node:fs"); +const marker = process.argv[1]; +const deadline = Date.now() + 5_000; +const wait = () => fs.existsSync(marker) ? process.exit(0) : Date.now() >= deadline ? process.exit(1) : setTimeout(wait, 10); +wait();`, + gatewayReadyMarker, + ], + { stdio: "ignore", timeout: 6_000 }, + ); + expect(gatewayWait.status).toBe(0); + const gatewayPortText = fs.readFileSync(gatewayReadyMarker, "utf-8").trim(); + expect(gatewayPortText).toMatch(/^[1-9][0-9]{0,4}$/); + const gatewayPort = Number(gatewayPortText); + expect(gatewayPort).toBeLessThanOrEqual(65_535); + const gatewayName = `nemoclaw-${gatewayPort}`; + fs.writeFileSync( path.join(nemoclawDir, "sandboxes.json"), JSON.stringify({ @@ -62,8 +98,8 @@ function createFixture(opts: { provider, gpuEnabled: false, sandboxGpuMode: "0", - gatewayName: "nemoclaw", - gatewayPort: 8080, + gatewayName, + gatewayPort, dashboardPort: agent === "langchain-deepagents-code" ? 0 : 18789, fromDockerfile: null, policies: [], @@ -106,7 +142,7 @@ function createFixture(opts: { webSearchConfig: null, policyPresets: [], messagingPlan: null, - metadata: { gatewayName: "nemoclaw", fromDockerfile: null }, + metadata: { gatewayName, fromDockerfile: null }, steps: { preflight: { status: "complete", startedAt: null, completedAt: null, error: null }, gateway: { status: "complete", startedAt: null, completedAt: null, error: null }, @@ -175,8 +211,8 @@ if (a[0] === "sandbox" && a[1] === "exec") { } process.exit(0); } -if (a[0] === "status") { process.stdout.write("Server Status\\n Gateway: nemoclaw\\n Status: Connected\\n"); process.exit(0); } -if (a[0] === "gateway" && a[1] === "info") { process.stdout.write("Gateway Info\\n\\nGateway: nemoclaw\\n"); process.exit(0); } +if (a[0] === "status") { process.stdout.write("Server Status\\n Gateway: ${gatewayName}\\n Status: Connected\\n"); process.exit(0); } +if (a[0] === "gateway" && a[1] === "info") { process.stdout.write("Gateway Info\\n\\nGateway: ${gatewayName}\\nGateway endpoint: https://127.0.0.1:${gatewayPort}\\n"); process.exit(0); } if (a[0] === "gateway" && a[1] === "select") process.exit(0); if (a[0] === "inference" && a[1] === "get") { process.stdout.write("Gateway inference:\\n Provider: ${provider}\\n Model: meta/llama-3.3-70b-instruct\\n"); process.exit(0); } if (a[0] === "inference" && a[1] === "set") process.exit(0); @@ -204,6 +240,10 @@ process.exit(0); ); } + fs.writeFileSync(path.join(tmpDir, "lsof"), "#!/usr/bin/env node\nprocess.exit(1);\n", { + mode: 0o755, + }); + fs.writeFileSync(path.join(tmpDir, "ps"), "#!/usr/bin/env node\nprocess.exit(0);\n", { mode: 0o755, }); @@ -218,7 +258,7 @@ const { isOpenClawSecurityInventoryProbe } = require(${JSON.stringify( )}); const provenancePath = ${JSON.stringify(path.join(tmpDir, "docker-base-provenance"))}; const readProvenance = () => fs.existsSync(provenancePath) ? JSON.parse(fs.readFileSync(provenancePath, "utf8")) : {}; -if (a[0] === "info") { process.stdout.write(JSON.stringify({ServerVersion:"27.0.0", OperatingSystem:"Docker Engine", NCPU:8, MemTotal:17179869184}) + "\\n"); process.exit(0); } +if (a[0] === "info") { process.stdout.write(JSON.stringify({ServerVersion:"27.0.0", OperatingSystem:"${DOCKER_OPERATING_SYSTEM}", NCPU:8, MemTotal:17179869184}) + "\\n"); process.exit(0); } if (a[0] === "build") { const labelIndex = a.indexOf("--label"); const tagIndex = a.indexOf("-t"); @@ -257,7 +297,14 @@ if (a[0] === "run") { else process.stdout.write("nemoclaw-hermes-mcp-runtime-ok\\n"); process.exit(0); } -if (a[0] === "inspect") { process.stdout.write("true\\n"); process.exit(0); } +if (a[0] === "inspect") { + const formatIndex = a.indexOf("--format"); + const format = formatIndex >= 0 ? a[formatIndex + 1] : ""; + if (format === "{{.State.Running}}") process.stdout.write("true\\n"); + if (format === "{{json .NetworkSettings.Ports}}") process.stdout.write(JSON.stringify({"${gatewayPort}/tcp":[{HostPort:"${gatewayPort}"}]}) + "\\n"); + if (format === "{{.Config.Image}}") process.stdout.write("nvcr.io/nvidia/openshell/cluster:0.0.101\\n"); + process.exit(0); +} if (a[0] === "ps") process.exit(0); process.stderr.write("unexpected docker call: " + a.join(" ") + "\\n"); process.exit(1);