diff --git a/ci/platform-matrix.json b/ci/platform-matrix.json index c133f0b098e..d6abe12757d 100644 --- a/ci/platform-matrix.json +++ b/ci/platform-matrix.json @@ -129,7 +129,7 @@ "name": "Local NVIDIA NIM", "status": "experimental", "endpoint_type": "Local OpenAI-compatible", - "notes": "Requires `NEMOCLAW_EXPERIMENTAL=1` and a NIM-capable NVIDIA GPU. Host must have the NVIDIA Container Toolkit installed and a CDI spec present (`onboard` asserts CDI presence with `assertCdiNvidiaGpuSpecPresent`, `src/lib/onboard/fatal-runtime-preflight.ts:78`). NIM images pull from `nvcr.io` and require NGC registry login. NemoClaw gates this path behind the experimental flag because it does not auto-select a NIM image for the host today. You must explicitly pick from the validated image list. Managed vLLM has host-specific default models and is not gated on the same boxes. Validated images referenced in `src/lib/inference/config.ts` and `nemoclaw/src/index.ts`: `nvidia/nemotron-3-super-120b-a12b` (default cloud model), `nvidia/nemotron-3-nano-30b-a3b`, `nvidia/llama-3.3-nemotron-super-49b-v1.5`." + "notes": "Requires `NEMOCLAW_EXPERIMENTAL=1` and a NIM-capable NVIDIA GPU. Host must have the NVIDIA Container Toolkit installed and a CDI spec present (`onboard` asserts CDI presence with `assertCdiNvidiaGpuSpecPresent`, `src/lib/onboard/fatal-runtime-preflight.ts`). NIM images pull from `nvcr.io` and require NGC registry login. NemoClaw gates this path behind the experimental flag because it does not auto-select a NIM image for the host today. You must explicitly pick from the validated image list. Managed vLLM has host-specific default models and is not gated on the same boxes. Validated images referenced in `src/lib/inference/config.ts` and `nemoclaw/src/index.ts`: `nvidia/nemotron-3-super-120b-a12b` (default cloud model), `nvidia/nemotron-3-nano-30b-a3b`, `nvidia/llama-3.3-nemotron-super-49b-v1.5`." }, { "name": "Local vLLM (already running)", @@ -218,7 +218,7 @@ { "name": "Podman / other container runtimes", "status": "unsupported", - "notes": "Onboard surfaces an explicit unsupported-runtime error for Podman (`src/lib/onboard/fatal-runtime-preflight.ts:50` prints the rejection; `src/lib/onboard/preflight.ts:677` flags the unsupported runtime upstream). Only Docker Engine, Docker Desktop, and Colima are supported. See issue #420 (closed)." + "notes": "Onboard surfaces an explicit unsupported-runtime error for Podman (`src/lib/onboard/fatal-runtime-preflight.ts` prints the rejection; `src/lib/onboard/preflight.ts` flags the unsupported runtime upstream). Only Docker Engine, Docker Desktop, and Colima are supported. See issue #420 (closed)." }, { "name": "Intel Mac (macOS x86_64)", @@ -248,7 +248,7 @@ { "name": "Non-NVIDIA GPUs (AMD/ROCm, Intel Arc, Apple Metal)", "status": "unsupported", - "notes": "Local vLLM and NIM paths assert NVIDIA CDI presence with `assertCdiNvidiaGpuSpecPresent` (`src/lib/onboard/fatal-runtime-preflight.ts:78`). NemoClaw does not install non-NVIDIA accelerator drivers." + "notes": "Local vLLM and NIM paths assert NVIDIA CDI presence with `assertCdiNvidiaGpuSpecPresent` (`src/lib/onboard/fatal-runtime-preflight.ts`). NemoClaw does not install non-NVIDIA accelerator drivers." }, { "name": "Other LangChain, AutoGen, CrewAI, or non-listed agent harnesses", diff --git a/docs/inference/inference-options.mdx b/docs/inference/inference-options.mdx index 421c1556c4d..4970e5f9a94 100644 --- a/docs/inference/inference-options.mdx +++ b/docs/inference/inference-options.mdx @@ -49,7 +49,7 @@ NemoClaw uses provider-specific local tokens for those routes, and rebuilds of l | Google Gemini | Tested | OpenAI-compatible | Uses Google's OpenAI-compatible endpoint | | Hermes Provider | Hermes only | OpenAI-compatible route | Available when onboarding Hermes Agent through `nemohermes` | | Local Ollama | Tested with limitations | Local Ollama API | Available when Ollama is installed or running on the host. Validated default models: `qwen3.6:35b` (high VRAM), `nemotron-3-nano:30b` (medium VRAM), `qwen3.5:9b` (low VRAM fallback). | -| Local NVIDIA NIM | Experimental | Local OpenAI-compatible | Requires `NEMOCLAW_EXPERIMENTAL=1` and a NIM-capable NVIDIA GPU. Host must have the NVIDIA Container Toolkit installed and a CDI spec present (`onboard` asserts CDI presence with `assertCdiNvidiaGpuSpecPresent`, `src/lib/onboard/fatal-runtime-preflight.ts:78`). NIM images pull from `nvcr.io` and require NGC registry login. NemoClaw gates this path behind the experimental flag because it does not auto-select a NIM image for the host today. You must explicitly pick from the validated image list. Managed vLLM has host-specific default models and is not gated on the same boxes. Validated images referenced in `src/lib/inference/config.ts` and `nemoclaw/src/index.ts`: `nvidia/nemotron-3-super-120b-a12b` (default cloud model), `nvidia/nemotron-3-nano-30b-a3b`, `nvidia/llama-3.3-nemotron-super-49b-v1.5`. | +| Local NVIDIA NIM | Experimental | Local OpenAI-compatible | Requires `NEMOCLAW_EXPERIMENTAL=1` and a NIM-capable NVIDIA GPU. Host must have the NVIDIA Container Toolkit installed and a CDI spec present (`onboard` asserts CDI presence with `assertCdiNvidiaGpuSpecPresent`, `src/lib/onboard/fatal-runtime-preflight.ts`). NIM images pull from `nvcr.io` and require NGC registry login. NemoClaw gates this path behind the experimental flag because it does not auto-select a NIM image for the host today. You must explicitly pick from the validated image list. Managed vLLM has host-specific default models and is not gated on the same boxes. Validated images referenced in `src/lib/inference/config.ts` and `nemoclaw/src/index.ts`: `nvidia/nemotron-3-super-120b-a12b` (default cloud model), `nvidia/nemotron-3-nano-30b-a3b`, `nvidia/llama-3.3-nemotron-super-49b-v1.5`. | | Local vLLM (already running) | Tested with limitations | Local OpenAI-compatible | Appears in the onboarding menu when NemoClaw detects a server already on `localhost:8000`. No flag required. Model is whatever the existing server serves. | | Local vLLM (managed install/start) | Tested with limitations | Local OpenAI-compatible | Appears by default on DGX Spark and DGX Station. Generic Linux NVIDIA GPU hosts require `NEMOCLAW_EXPERIMENTAL=1` or `NEMOCLAW_PROVIDER=install-vllm`. Host must have the NVIDIA Container Toolkit installed and a CDI spec present (`onboard` asserts CDI presence). NemoClaw pulls or starts the stable NGC vLLM container for each host profile. See `src/lib/inference/vllm.ts:55,177` for the pins. DGX Spark and DGX Station use `nvcr.io/nvidia/vllm:26.05.post1-py3`; generic Linux NVIDIA GPU hosts use `nvcr.io/nvidia/vllm:26.03.post1-py3`. Validated defaults are listed in `src/lib/inference/vllm-models.ts`: DGX Spark uses `nvidia/Qwen3.6-35B-A3B-NVFP4`, DGX Station uses `deepseek-ai/DeepSeek-V4-Flash`, and Linux NVIDIA GPU uses `nvidia/NVIDIA-Nemotron-3-Nano-4B-FP8`. Image pulls require NGC registry login (`docker login nvcr.io`); onboard prompts for the NGC API key when authentication is missing. | {/* provider-status:end */} diff --git a/docs/reference/platform-support.mdx b/docs/reference/platform-support.mdx index d9053575804..341fce65745 100644 --- a/docs/reference/platform-support.mdx +++ b/docs/reference/platform-support.mdx @@ -101,7 +101,7 @@ NemoClaw routes inference through the OpenShell gateway. Each row below is a pro | Google Gemini | Tested | OpenAI-compatible | Uses Google's OpenAI-compatible endpoint | | Hermes Provider | Hermes only | OpenAI-compatible route | Available when onboarding Hermes Agent through `nemohermes` | | Local Ollama | Tested with limitations | Local Ollama API | Available when Ollama is installed or running on the host. Validated default models: `qwen3.6:35b` (high VRAM), `nemotron-3-nano:30b` (medium VRAM), `qwen3.5:9b` (low VRAM fallback). | -| Local NVIDIA NIM | Experimental | Local OpenAI-compatible | Requires `NEMOCLAW_EXPERIMENTAL=1` and a NIM-capable NVIDIA GPU. Host must have the NVIDIA Container Toolkit installed and a CDI spec present (`onboard` asserts CDI presence with `assertCdiNvidiaGpuSpecPresent`, `src/lib/onboard/fatal-runtime-preflight.ts:78`). NIM images pull from `nvcr.io` and require NGC registry login. NemoClaw gates this path behind the experimental flag because it does not auto-select a NIM image for the host today. You must explicitly pick from the validated image list. Managed vLLM has host-specific default models and is not gated on the same boxes. Validated images referenced in `src/lib/inference/config.ts` and `nemoclaw/src/index.ts`: `nvidia/nemotron-3-super-120b-a12b` (default cloud model), `nvidia/nemotron-3-nano-30b-a3b`, `nvidia/llama-3.3-nemotron-super-49b-v1.5`. | +| Local NVIDIA NIM | Experimental | Local OpenAI-compatible | Requires `NEMOCLAW_EXPERIMENTAL=1` and a NIM-capable NVIDIA GPU. Host must have the NVIDIA Container Toolkit installed and a CDI spec present (`onboard` asserts CDI presence with `assertCdiNvidiaGpuSpecPresent`, `src/lib/onboard/fatal-runtime-preflight.ts`). NIM images pull from `nvcr.io` and require NGC registry login. NemoClaw gates this path behind the experimental flag because it does not auto-select a NIM image for the host today. You must explicitly pick from the validated image list. Managed vLLM has host-specific default models and is not gated on the same boxes. Validated images referenced in `src/lib/inference/config.ts` and `nemoclaw/src/index.ts`: `nvidia/nemotron-3-super-120b-a12b` (default cloud model), `nvidia/nemotron-3-nano-30b-a3b`, `nvidia/llama-3.3-nemotron-super-49b-v1.5`. | | Local vLLM (already running) | Tested with limitations | Local OpenAI-compatible | Appears in the onboarding menu when NemoClaw detects a server already on `localhost:8000`. No flag required. Model is whatever the existing server serves. | | Local vLLM (managed install/start) | Tested with limitations | Local OpenAI-compatible | Appears by default on DGX Spark and DGX Station. Generic Linux NVIDIA GPU hosts require `NEMOCLAW_EXPERIMENTAL=1` or `NEMOCLAW_PROVIDER=install-vllm`. Host must have the NVIDIA Container Toolkit installed and a CDI spec present (`onboard` asserts CDI presence). NemoClaw pulls or starts the stable NGC vLLM container for each host profile. See `src/lib/inference/vllm.ts:55,177` for the pins. DGX Spark and DGX Station use `nvcr.io/nvidia/vllm:26.05.post1-py3`; generic Linux NVIDIA GPU hosts use `nvcr.io/nvidia/vllm:26.03.post1-py3`. Validated defaults are listed in `src/lib/inference/vllm-models.ts`: DGX Spark uses `nvidia/Qwen3.6-35B-A3B-NVFP4`, DGX Station uses `deepseek-ai/DeepSeek-V4-Flash`, and Linux NVIDIA GPU uses `nvidia/NVIDIA-Nemotron-3-Nano-4B-FP8`. Image pulls require NGC registry login (`docker login nvcr.io`); onboard prompts for the NGC API key when authentication is missing. | {/* provider-status-full:end */} @@ -160,13 +160,13 @@ They are listed here so launch material, sales conversations, and support triage {/* out-of-scope:begin */} | Item | Status | Why | |------|--------|-----| -| Podman / other container runtimes | Unsupported | Onboard surfaces an explicit unsupported-runtime error for Podman (`src/lib/onboard/fatal-runtime-preflight.ts:50` prints the rejection; `src/lib/onboard/preflight.ts:677` flags the unsupported runtime upstream). Only Docker Engine, Docker Desktop, and Colima are supported. See issue #420 (closed). | +| Podman / other container runtimes | Unsupported | Onboard surfaces an explicit unsupported-runtime error for Podman (`src/lib/onboard/fatal-runtime-preflight.ts` prints the rejection; `src/lib/onboard/preflight.ts` flags the unsupported runtime upstream). Only Docker Engine, Docker Desktop, and Colima are supported. See issue #420 (closed). | | Intel Mac (macOS x86_64) | Unsupported | OpenShell does not publish macOS x86_64 standalone gateway assets. Install hard-fails on x86_64 macOS (`scripts/install-openshell.sh:663`). See issue #954 (closed). | | Non-Ubuntu/Debian Linux distros | Unsupported | Installer assumes `apt-get`. Fedora/Rocky/Alma/Arch/NixOS are not validated and the installer's package-manager probes do not cover them. See open issue #899 (Fedora hang). | | Native Kubernetes or OpenShift deployments | Unsupported | NemoClaw runs the sandbox as a Docker container, not a Kubernetes pod. The default Docker-driver topology does not embed k3s. Operator-managed K8s/OpenShift deployments are out of scope; see issue #407 (community OpenShift through agent-sandbox CRD). | | Air-gapped / offline installs | Unsupported | Onboard assumes network reachability for package fetches, container pulls, and provider validation. See open issues #4872 and #2218 (production-deployment epic covering air-gapped support, China network guidance, multi-host topology). | | Windows-on-ARM GPU passthrough | Unsupported | Windows-on-ARM CPU paths run under WSL2 'tested with limitations', but GPU passthrough on WOA is denylisted (`src/lib/onboard/wsl-docker-desktop-gpu.ts:188`, `src/lib/inference/gpu-trust.test.ts:70`). See closed issue #4565. | -| Non-NVIDIA GPUs (AMD/ROCm, Intel Arc, Apple Metal) | Unsupported | Local vLLM and NIM paths assert NVIDIA CDI presence with `assertCdiNvidiaGpuSpecPresent` (`src/lib/onboard/fatal-runtime-preflight.ts:78`). NemoClaw does not install non-NVIDIA accelerator drivers. | +| Non-NVIDIA GPUs (AMD/ROCm, Intel Arc, Apple Metal) | Unsupported | Local vLLM and NIM paths assert NVIDIA CDI presence with `assertCdiNvidiaGpuSpecPresent` (`src/lib/onboard/fatal-runtime-preflight.ts`). NemoClaw does not install non-NVIDIA accelerator drivers. | | Other LangChain, AutoGen, CrewAI, or non-listed agent harnesses | Unsupported | LangChain Deep Agents Code is the only integrated LangChain-family harness (see the Agents section above; status `Experimental`). Other LangChain harnesses, AutoGen, CrewAI, and any agent runtime not listed in the Agents table are not integrated. Bringing more harnesses is tracked as a research epic (see open issue #4861) but is not on the current roadmap. | | Multi-user host sharing | Unsupported | Sandboxes are scoped to a single host user. NemoClaw treats multi-user hosts as a risk and warns at onboard; see `docs/security/openclaw-controls.mdx` Multi-user detection. | | Hosted SaaS / managed NemoClaw | Unsupported | There is no managed offering. Supported deployment paths are Local CLI onboard, Remote GPU with Brev CLI, and Brev web UI. | diff --git a/src/lib/cli/terminal-style.test.ts b/src/lib/cli/terminal-style.test.ts index 21194202011..852b13a0be7 100644 --- a/src/lib/cli/terminal-style.test.ts +++ b/src/lib/cli/terminal-style.test.ts @@ -1,9 +1,9 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; -import { B, D, G, R, RD, YW } from "./terminal-style"; +import { B, D, failLine, G, R, RD, warnLine, YW } from "./terminal-style"; describe("terminal-style", () => { it("exports terminal style strings", () => { @@ -12,3 +12,100 @@ describe("terminal-style", () => { } }); }); + +const ORIGINAL_STDOUT = { + isTTY: process.stdout.isTTY, + getColorDepth: process.stdout.getColorDepth, +}; +const ORIGINAL_STDERR = { + isTTY: process.stderr.isTTY, + getColorDepth: process.stderr.getColorDepth, +}; + +// styleText decides color from the target stream's reported color depth +// (`getColorDepth()`), which is where a real terminal folds in isTTY, NO_COLOR, +// NODE_DISABLE_COLORS and FORCE_COLOR. Depth 1 = no color (what NO_COLOR / a +// redirected pipe / CI report); depth 24 = truecolor. Model both directly so +// each case is deterministic regardless of the worker's own TTY/env. +function stubStream(stream: NodeJS.WriteStream, isTTY: boolean, colorDepth: number): void { + Object.defineProperty(stream, "isTTY", { value: isTTY, configurable: true }); + Object.defineProperty(stream, "getColorDepth", { value: () => colorDepth, configurable: true }); +} + +function restoreStream( + stream: NodeJS.WriteStream, + original: { isTTY: boolean | undefined; getColorDepth: unknown }, +): void { + Object.defineProperty(stream, "isTTY", { value: original.isTTY, configurable: true }); + Object.defineProperty(stream, "getColorDepth", { + value: original.getColorDepth, + configurable: true, + }); +} + +async function withRestoredStreams(callback: () => T | Promise): Promise { + try { + return await callback(); + } finally { + restoreStream(process.stdout, ORIGINAL_STDOUT); + restoreStream(process.stderr, ORIGINAL_STDERR); + } +} + +// styleText's `yellow`/`red`/`green` formats (as of Node 22.16) wrap text in +// SGR color codes with a `39` (default-foreground) reset. +const YELLOW = (s: string) => `\x1b[33m${s}\x1b[39m`; +const RED = (s: string) => `\x1b[31m${s}\x1b[39m`; +describe("preflight severity lines (#6004)", () => { + afterEach(() => { + vi.unstubAllEnvs(); + vi.resetModules(); + }); + + it("colors warn/error from stderr — their real stream — not stdout (#6004)", async () => { + await withRestoredStreams(() => { + // stdout redirected to a file, terminal still on stderr: warn/error must + // stay colored because they land on the color-capable stderr. + vi.stubEnv("NO_COLOR", ""); + stubStream(process.stderr, true, 24); + stubStream(process.stdout, false, 1); + expect(warnLine("disk low")).toBe(` ${YELLOW("⚠ disk low")}`); + expect(failLine("docker down")).toBe(` ${RED("✗ docker down")}`); + }); + }); + + it("drops warn/error color when stderr is redirected but stdout is a TTY (#6004)", async () => { + await withRestoredStreams(() => { + // The inverse leak: stderr redirected to a log, stdout still a terminal. + // warn/error must go plain so no raw ANSI lands in the log. + vi.stubEnv("NO_COLOR", ""); + stubStream(process.stdout, true, 24); + stubStream(process.stderr, false, 1); + expect(warnLine("disk low")).toBe(" ⚠ disk low"); + expect(failLine("docker down")).toBe(" ✗ docker down"); + }); + }); + + it("keeps NO_COLOR authoritative when FORCE_COLOR is also set", async () => { + await withRestoredStreams(() => { + vi.stubEnv("NO_COLOR", "1"); + vi.stubEnv("FORCE_COLOR", "1"); + stubStream(process.stdout, true, 24); + stubStream(process.stderr, true, 24); + expect(warnLine("a")).toBe(" ⚠ a"); + expect(failLine("b")).toBe(" ✗ b"); + }); + }); + + it("selects the legacy true-color green when configured before import", async () => { + await withRestoredStreams(async () => { + stubStream(process.stdout, true, 24); + vi.stubEnv("NO_COLOR", ""); + vi.stubEnv("COLORTERM", "truecolor"); + vi.resetModules(); + + const freshStyles = await import("./terminal-style"); + expect(freshStyles.G).toBe("\x1b[38;2;118;185;0m"); + }); + }); +}); diff --git a/src/lib/cli/terminal-style.ts b/src/lib/cli/terminal-style.ts index 82a1e1824a7..d93b11cea59 100644 --- a/src/lib/cli/terminal-style.ts +++ b/src/lib/cli/terminal-style.ts @@ -1,6 +1,15 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { styleText } from "node:util"; + +/** + * Legacy color constants (`G`, `B`, `D`, `R`, `RD`, `YW`) are frozen at module + * import time; import after `NO_COLOR` and TTY state are configured. Prefer the + * call-time severity helpers below for new output. The constants intentionally + * retain their historical raw ANSI values, while new output uses `styleText` + * so color capability is evaluated for the destination stream at call time. + */ const useColor = !process.env.NO_COLOR && !!process.stdout.isTTY; const trueColor = useColor && (process.env.COLORTERM === "truecolor" || process.env.COLORTERM === "24bit"); @@ -11,3 +20,19 @@ export const D = useColor ? "\x1b[2m" : ""; export const R = useColor ? "\x1b[0m" : ""; export const RD = useColor ? "\x1b[1;31m" : ""; export const YW = useColor ? "\x1b[1;33m" : ""; + +// WARN and ERROR lines are emitted on stderr. `styleText({ stream })` therefore +// keys color off stderr's capability and honors NO_COLOR / NODE_DISABLE_COLORS / +// FORCE_COLOR (#6004). The old output keyed color off stdout, which dropped +// color on `onboard >log` and leaked ANSI into `onboard 2>log`. +function stderrSeverityLine( + marker: "⚠ " | "✗ ", + format: "yellow" | "red", + message: string, +): string { + const line = `${marker}${message}`; + return ` ${process.env.NO_COLOR ? line : styleText(format, line, { stream: process.stderr })}`; +} + +export const warnLine = (message: string): string => stderrSeverityLine("⚠ ", "yellow", message); +export const failLine = (message: string): string => stderrSeverityLine("✗ ", "red", message); diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index afb39d2be8a..0038f375064 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -595,6 +595,11 @@ import { setupPoliciesWithSelection as setupPoliciesWithSelectionImpl, } from "./onboard/policy-selection"; import { createPolicySelectionPromptHelpers } from "./onboard/policy-selection-prompts"; +import { + printLowMemoryWarning, + printMessagingProviderMissing, + printSwapCreationFailed, +} from "./onboard/preflight-messages"; import { backupSandboxBeforeRecreate, shouldSkipPreRecreateBackup, @@ -1653,6 +1658,7 @@ async function preflight( cliDisplayName: cliDisplayName(), dashboardPort: getOnboardDashboardPort(), log: console.log, + warn: console.warn, runOpenshell, destroyGateway, destroyGatewayForReuse, @@ -1833,9 +1839,7 @@ async function preflight( const mem = getMemoryInfo(); if (mem) { if (mem.totalMB < 12000) { - console.log( - ` ⚠ Low memory detected (${mem.totalRamMB} MB RAM + ${mem.totalSwapMB} MB swap = ${mem.totalMB} MB total)`, - ); + printLowMemoryWarning(mem); let proceedWithSwap: boolean = false; if (!isNonInteractive()) { @@ -1861,8 +1865,7 @@ async function preflight( console.log(` ✓ Memory OK: ${mem.totalRamMB} MB RAM + ${mem.totalSwapMB} MB swap`); } } else { - console.log(` ⚠ Could not create swap: ${swapResult.reason}`); - console.log(" Sandbox creation may fail with OOM on low-memory systems."); + printSwapCreationFailed(swapResult.reason); } } } else { @@ -3156,11 +3159,7 @@ async function createSandbox( // cannot be verified via CLI yet — only gateway-level existence is checked). for (const p of messagingProviders) { if (!providerExistsInGateway(p)) { - console.error(` ⚠ Messaging provider '${p}' was not found in the gateway.`); - console.error(` The credential may not be available inside the sandbox.`); - console.error( - ` To fix: openshell provider create --name ${p} --type generic --credential `, - ); + printMessagingProviderMissing(p); } } diff --git a/src/lib/onboard/bridge-dns-preflight.ts b/src/lib/onboard/bridge-dns-preflight.ts index f5b8f47d5d9..a5e31810f7c 100644 --- a/src/lib/onboard/bridge-dns-preflight.ts +++ b/src/lib/onboard/bridge-dns-preflight.ts @@ -2,16 +2,12 @@ // SPDX-License-Identifier: Apache-2.0 /** - * Bridge + DNS preflight gate, extracted from `onboard.ts` so it can be - * reused as a `--resume` backstop without growing the top-level file - * past the `onboard-entrypoint-budget` CI ceiling. - * - * - `assertDockerBridgeAndContainerDnsHealthy(host)` runs the bridge - * container start probe (#3508 Jetson veth) and the DNS-from-inside- - * container probe (#3630), and exits with platform-aware remediation - * on the fatal reasons described in `[[isFatalContainerDnsProbeFailure]]`. + * Bridge + DNS preflight gate extracted from `onboard.ts` for reuse as a + * `--resume` backstop. It validates bridge container start (#3508 Jetson veth) + * and container DNS (#3630), with platform-aware remediation on fatal results. */ +import { failLine, warnLine } from "../cli/terminal-style"; import { cliDisplayName, cliName } from "./branding"; interface DaemonJsonDnsPatchOpts { @@ -30,19 +26,14 @@ interface DaemonJsonDnsPatchOpts { } /** - * Print a copy-pastable shell snippet that adds a `dns` key to the - * given daemon.json safely. The snippet: - * - creates the containing directory, - * - backs up the existing daemon.json, - * - requires `jq` (prints an install hint and aborts if missing — no - * bare-echo fallback that would clobber an existing daemon.json), - * - merges into an existing JSON object via `jq '. + {...}'`, - * - creates a new JSON object via `jq -n {...}` when daemon.json is - * absent, - * - refuses to write if the existing file is not parseable, asking - * the user to fix it manually first. + * Print a copy-pastable shell snippet that creates the config directory, backs + * up daemon.json, requires `jq`, merges or creates the `dns` key, and refuses + * to write invalid JSON. * - * The snippet is printed verbatim; nothing here executes it. + * Source boundary: this is privileged, platform-owned Docker configuration. + * Unprivileged onboarding cannot safely mutate it or restart Docker without + * explicit user consent, so the commands stay plain and nothing executes them. + * Remove this only when Docker/OpenShell exposes a managed daemon-DNS API. */ function printDaemonJsonDnsPatch(opts: DaemonJsonDnsPatchOpts): void { const { daemonJsonPath, configDir, dnsValue, sudo, installJqHint, indent } = opts; @@ -96,7 +87,7 @@ export function printDockerBridgeContainerStartFailure( result: DockerBridgeContainerStartProbeResult, host?: Pick, ): void { - console.error(" ✗ Docker could not start a bridge-network test container."); + console.error(failLine("Docker could not start a bridge-network test container.")); if (result.details) { for (const line of String(result.details).split("\n").slice(-4)) { if (line.trim()) console.error(` ${line.trim()}`); @@ -174,7 +165,9 @@ export function assertDockerBridgeAndContainerDnsHealthy( exitProcess(1); } else { console.warn( - ` ⚠ Bridge container start probe inconclusive (reason: ${bridgeStart.reason ?? "unknown"}).`, + warnLine( + `Bridge container start probe inconclusive (reason: ${bridgeStart.reason ?? "unknown"}).`, + ), ); if (bridgeStart.details) { for (const line of String(bridgeStart.details).split("\n").slice(-3)) { @@ -207,14 +200,16 @@ export function assertDockerBridgeAndContainerDnsHealthy( if (!dnsIsFatal) { if (dns.reason === "image_pull_failed") { console.warn( - " ⚠ Container DNS probe inconclusive: docker couldn't pull the busybox test image.", + warnLine("Container DNS probe inconclusive: docker couldn't pull the busybox test image."), ); console.warn(" This usually means the docker daemon itself can't reach Docker Hub,"); console.warn( " but doesn't prove container DNS is broken — the sandbox build may still succeed.", ); } else { - console.warn(` ⚠ Container DNS probe inconclusive (reason: ${dns.reason ?? "unknown"}).`); + console.warn( + warnLine(`Container DNS probe inconclusive (reason: ${dns.reason ?? "unknown"}).`), + ); } if (dns.details) { for (const line of String(dns.details).split("\n").slice(-3)) { @@ -254,15 +249,15 @@ export function assertDockerBridgeAndContainerDnsHealthy( exitProcess(1); } if (dns.reason === "timeout" || dns.reason === "killed") { - console.error(" ✗ Container DNS probe did not complete."); + console.error(failLine("Container DNS probe did not complete.")); } else if (dns.reason === "image_pull_failed") { - console.error(" ✗ Docker could not resolve or pull the DNS probe image."); + console.error(failLine("Docker could not resolve or pull the DNS probe image.")); } else if (dns.reason === "resolution_failed") { console.error( - " ✗ Container DNS server is reachable but rejected the query (NXDOMAIN/REFUSED).", + failLine("Container DNS server is reachable but rejected the query (NXDOMAIN/REFUSED)."), ); } else { - console.error(" ✗ DNS resolution from inside a docker container failed."); + console.error(failLine("DNS resolution from inside a docker container failed.")); } if (dns.details) { for (const line of String(dns.details).split("\n").slice(-4)) { @@ -401,7 +396,7 @@ export function assertHostDnsHealthy(host: Host, opts: AssertHostDnsHealthyOpts return; } if (!isFatalHostDnsProbeFailure(result)) { - console.warn(` ⚠ Host DNS probe inconclusive (reason: ${result.reason ?? "unknown"}).`); + console.warn(warnLine(`Host DNS probe inconclusive (reason: ${result.reason ?? "unknown"}).`)); if (result.details) { console.warn(` ${String(result.details).trim()}`); } @@ -412,11 +407,15 @@ export function assertHostDnsHealthy(host: Host, opts: AssertHostDnsHealthyOpts } if (result.reason === "timeout" || result.reason === "killed") { - console.error(` ✗ Host DNS probe did not complete (could not resolve ${result.hostname}).`); + console.error( + failLine(`Host DNS probe did not complete (could not resolve ${result.hostname}).`), + ); } else if (result.reason === "resolution_failed") { - console.error(` ✗ Host could not resolve ${result.hostname} (resolver answered, no record).`); + console.error( + failLine(`Host could not resolve ${result.hostname} (resolver answered, no record).`), + ); } else { - console.error(` ✗ Host DNS resolution failed (could not resolve ${result.hostname}).`); + console.error(failLine(`Host DNS resolution failed (could not resolve ${result.hostname}).`)); } if (result.details) { console.error(` ${String(result.details).trim()}`); diff --git a/src/lib/onboard/fatal-runtime-preflight.ts b/src/lib/onboard/fatal-runtime-preflight.ts index e5d8b34ddce..72cda1cb19a 100644 --- a/src/lib/onboard/fatal-runtime-preflight.ts +++ b/src/lib/onboard/fatal-runtime-preflight.ts @@ -2,7 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 import { detectGpu, type GpuDetection } from "../inference/nim"; -import { cliDisplayName } from "./branding"; import { assertDockerBridgeAndContainerDnsHealthy } from "./bridge-dns-preflight"; import { isLinuxDockerDriverGatewayEnabled } from "./docker-driver-platform"; import { warnIfHostProxyMissesLoopback } from "./http-proxy-preflight"; @@ -12,6 +11,7 @@ import { type HostAssessment, planHostRemediation, } from "./preflight"; +import { printDockerNotReachableError, printUnsupportedRuntimeError } from "./preflight-messages"; import { printRemediationActions } from "./remediation"; import { resolveSandboxGpuConfig, type SandboxGpuConfig } from "./sandbox-gpu-mode"; import { @@ -46,9 +46,7 @@ export function rejectUnsupportedContainerRuntime( exitProcess: (code: number) => never = exitProcessByDefault, ): void { if (isLinuxDockerDriverGatewayEnabled() && host.runtime === "podman") { - console.error(` ✗ ${cliDisplayName()} onboarding now uses OpenShell's Docker driver.`); - console.error(` Podman is not supported for this ${cliDisplayName()} integration path.`); - console.error(" Switch to Docker Engine and rerun onboarding."); + printUnsupportedRuntimeError(); exitProcess(1); } } @@ -61,7 +59,7 @@ export function runFatalOnboardRuntimePreflight( const exitProcess = context.exitProcess ?? exitProcessByDefault; const host = assessHost(); if (!host.dockerReachable) { - console.error(" Docker is not reachable. Please fix Docker and try again."); + printDockerNotReachableError(); printRemediationActions(planHostRemediation(host)); exitProcess(1); } diff --git a/src/lib/onboard/gateway-sandbox-reachability-severity.test.ts b/src/lib/onboard/gateway-sandbox-reachability-severity.test.ts new file mode 100644 index 00000000000..9c12dd4178a --- /dev/null +++ b/src/lib/onboard/gateway-sandbox-reachability-severity.test.ts @@ -0,0 +1,87 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { + formatSandboxBridgeUnreachableMessage, + verifySandboxBridgeGatewayReachableOrExit, +} from "./gateway-sandbox-reachability"; + +async function withColoredStderr(callback: () => T | Promise): Promise { + const originalIsTTY = process.stderr.isTTY; + const originalGetColorDepth = process.stderr.getColorDepth; + Object.defineProperty(process.stderr, "isTTY", { value: true, configurable: true }); + Object.defineProperty(process.stderr, "getColorDepth", { + value: () => 24, + configurable: true, + }); + vi.stubEnv("NO_COLOR", ""); + try { + return await callback(); + } finally { + Object.defineProperty(process.stderr, "isTTY", { + value: originalIsTTY, + configurable: true, + }); + Object.defineProperty(process.stderr, "getColorDepth", { + value: originalGetColorDepth, + configurable: true, + }); + vi.unstubAllEnvs(); + } +} + +describe("sandbox bridge reachability severity (#6004)", () => { + it("routes warning and fatal first lines through the stderr severity renderer", async () => { + await withColoredStderr(() => { + const warning = formatSandboxBridgeUnreachableMessage({ + ok: false, + reason: "probe_unavailable", + }); + const fatal = formatSandboxBridgeUnreachableMessage({ + ok: false, + reason: "veth_unsupported", + }); + + expect(warning.split("\n")[0]).toBe( + " \x1b[33m⚠ Could not verify sandbox bridge reachability.\x1b[39m", + ); + expect(fatal.split("\n")[0]).toBe( + " \x1b[31m✗ Docker could not create the sandbox bridge veth pair.\x1b[39m", + ); + }); + }); + + it("colors the UFW auto-apply fallback warning", async () => { + await withColoredStderr(async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + const error = vi.spyOn(console, "error").mockImplementation(() => undefined); + try { + await expect( + verifySandboxBridgeGatewayReachableOrExit(false, { + autoApplyImpl: () => ({ + applied: false, + reason: "sudo_unavailable", + detail: "passwordless sudo is unavailable", + }), + autoApplyOptedInImpl: () => true, + reachabilityImpl: () => ({ + ok: false, + reason: "tcp_failed", + routeKind: "bridge_gateway", + subnet: "172.18.0.0/16", + gatewayIp: "172.18.0.1", + }), + }), + ).rejects.toThrow("sandbox-bridge unreachable"); + expect(warn.mock.calls[0]?.[0]).toMatch( + /^ \x1b\[33m⚠ NEMOCLAW_AUTO_FIX_FIREWALL=1 set but could not auto-apply UFW rule/, + ); + } finally { + warn.mockRestore(); + error.mockRestore(); + } + }); + }); +}); diff --git a/src/lib/onboard/gateway-sandbox-reachability.ts b/src/lib/onboard/gateway-sandbox-reachability.ts index 0afbfdb4bfd..81689206f2f 100644 --- a/src/lib/onboard/gateway-sandbox-reachability.ts +++ b/src/lib/onboard/gateway-sandbox-reachability.ts @@ -13,6 +13,7 @@ import os from "node:os"; import { dockerCapture, dockerRun } from "../adapters/docker/run"; +import { failLine, warnLine } from "../cli/terminal-style"; import { GATEWAY_PORT } from "../core/ports"; import { cliDisplayName, cliName } from "./branding"; import { @@ -406,7 +407,7 @@ export function formatSandboxBridgeUnreachableMessage( const includeWslIntegrationHint = opts.isWsl ?? isRunningInWsl(); if (result.reason === "probe_unavailable") { return [ - " ⚠ Could not verify sandbox bridge reachability.", + warnLine("Could not verify sandbox bridge reachability."), " This does not prove the gateway is unreachable; continuing.", result.detail ? ` ${result.detail}` : undefined, ] @@ -416,7 +417,7 @@ export function formatSandboxBridgeUnreachableMessage( if (result.reason === "veth_unsupported") { return [ - " ✗ Docker could not create the sandbox bridge veth pair.", + failLine("Docker could not create the sandbox bridge veth pair."), result.detail ? ` ${result.detail}` : undefined, " This matches Jetson kernel/Docker bridge environments where veth creation returns `operation not supported`.", ` Update the host kernel/Docker bridge networking support, or run ${cliDisplayName()} on a host whose Docker bridge networking can create veth interfaces.`, @@ -427,7 +428,7 @@ export function formatSandboxBridgeUnreachableMessage( if (result.reason === "probe_timeout") { return [ - " ✗ Docker-driver sandbox bridge reachability probe timed out.", + failLine("Docker-driver sandbox bridge reachability probe timed out."), result.detail ? ` ${result.detail}` : undefined, ` Restart Docker and check for stuck container/network operations before retrying \`${cliName()} onboard\`.`, ] @@ -437,7 +438,7 @@ export function formatSandboxBridgeUnreachableMessage( if (result.reason === "docker_daemon_unreachable") { return [ - " ✗ Docker daemon is not reachable for the sandbox bridge probe.", + failLine("Docker daemon is not reachable for the sandbox bridge probe."), result.detail ? ` ${result.detail}` : undefined, includeWslIntegrationHint ? ` ${DOCKER_DESKTOP_WSL_INTEGRATION_HINT}` : undefined, " Restart the Docker daemon (e.g. `sudo systemctl restart docker`, or restart Docker Desktop/Colima)", @@ -449,7 +450,7 @@ export function formatSandboxBridgeUnreachableMessage( if (result.routeKind === "host_gateway") { return [ - ` ✗ Sandbox containers cannot reach the gateway at ${HOST_INTERNAL_NAME}:${port}.`, + failLine(`Sandbox containers cannot reach the gateway at ${HOST_INTERNAL_NAME}:${port}.`), " The probe used Docker's host-gateway route, matching Docker Desktop/VM-backed Docker.", ` Restart Docker and the OpenShell gateway, then re-run \`${cliName()} onboard\`.`, ].join("\n"); @@ -468,7 +469,7 @@ export function formatSandboxBridgeUnreachableMessage( ? `${HOST_INTERNAL_NAME}:${port} (${result.gatewayIp}:${port})` : `${HOST_INTERNAL_NAME}:${port}`; return [ - ` ✗ Sandbox containers cannot reach the gateway at ${target}.`, + failLine(`Sandbox containers cannot reach the gateway at ${target}.`), " A host firewall may be blocking traffic from the OpenShell Docker bridge.", " To allow it:", allowCmd, @@ -562,7 +563,9 @@ export async function verifySandboxBridgeGatewayReachableOrExit( if (reach.ok) return; } else if (!SILENT_UFW_AUTO_APPLY_REASONS.has(autoApplyResult.reason)) { console.warn( - ` ⚠ NEMOCLAW_AUTO_FIX_FIREWALL=1 set but could not auto-apply UFW rule (${autoApplyResult.reason}${autoApplyResult.detail ? `: ${autoApplyResult.detail}` : ""}); falling back to manual instructions.`, + warnLine( + `NEMOCLAW_AUTO_FIX_FIREWALL=1 set but could not auto-apply UFW rule (${autoApplyResult.reason}${autoApplyResult.detail ? `: ${autoApplyResult.detail}` : ""}); falling back to manual instructions.`, + ), ); } } diff --git a/src/lib/onboard/http-proxy-preflight.test.ts b/src/lib/onboard/http-proxy-preflight.test.ts index db7fb98e3ee..7496494547c 100644 --- a/src/lib/onboard/http-proxy-preflight.test.ts +++ b/src/lib/onboard/http-proxy-preflight.test.ts @@ -1,10 +1,25 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { redactProxyCredentials, warnIfHostProxyMissesLoopback } from "./http-proxy-preflight"; +function withStderrColorDepth(colorDepth: number, callback: () => T): T { + const stderr = Object.assign(Object.create(process.stderr), { + getColorDepth: () => colorDepth, + isTTY: true, + }) as typeof process.stderr; + const getStderr = vi.spyOn(process, "stderr", "get").mockReturnValue(stderr); + vi.stubEnv("NO_COLOR", ""); + try { + return callback(); + } finally { + getStderr.mockRestore(); + vi.unstubAllEnvs(); + } +} + describe("redactProxyCredentials", () => { it("returns plain proxy URLs unchanged", () => { expect(redactProxyCredentials("http://127.0.0.1:8118")).toBe("http://127.0.0.1:8118"); @@ -110,6 +125,25 @@ describe("warnIfHostProxyMissesLoopback", () => { expect(joined).toContain("proxy.example.com:3128"); }); + it("colors only the warning line on color-capable stderr and keeps proxy credentials redacted", () => { + withStderrColorDepth(24, () => { + const lines: string[] = []; + warnIfHostProxyMissesLoopback( + { http_proxy: "http://alice:s3cret@proxy.example.com:3128" }, + (line) => lines.push(line), + ); + + expect(lines[0]).toBe( + " \x1b[33m⚠ HTTP_PROXY/http_proxy is set without " + + "NO_PROXY=localhost,127.0.0.1,inference.local.\x1b[39m", + ); + expect(lines.slice(1).join("\n")).not.toContain("\x1b["); + expect(lines.join("\n")).not.toContain("alice"); + expect(lines.join("\n")).not.toContain("s3cret"); + expect(lines.join("\n")).toContain("****@proxy.example.com:3128"); + }); + }); + it("respects uppercase HTTP_PROXY too", () => { const lines: string[] = []; const fired = warnIfHostProxyMissesLoopback({ HTTP_PROXY: "http://corp-proxy:3128" }, (line) => diff --git a/src/lib/onboard/http-proxy-preflight.ts b/src/lib/onboard/http-proxy-preflight.ts index 68990faa821..02a53ccb8a6 100644 --- a/src/lib/onboard/http-proxy-preflight.ts +++ b/src/lib/onboard/http-proxy-preflight.ts @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { warnLine } from "../cli/terminal-style"; + /** * Preflight warning when the user's shell has HTTP_PROXY set without a * NO_PROXY bypass for loopback and the managed inference hostname. @@ -27,7 +29,9 @@ export function warnIfHostProxyMissesLoopback( const hasLoopback = /(^|,)\s*127\.0\.0\.1\s*(,|$)/.test(noProxyEnv); const hasInference = /(^|,)\s*inference\.local\s*(,|$)/.test(noProxyEnv); if (hasLocalhost && hasLoopback && hasInference) return false; - warn(" ⚠ HTTP_PROXY/http_proxy is set without NO_PROXY=localhost,127.0.0.1,inference.local."); + warn( + warnLine("HTTP_PROXY/http_proxy is set without NO_PROXY=localhost,127.0.0.1,inference.local."), + ); warn(` Detected proxy: ${redactProxyCredentials(proxyEnv)}`); warn(" NemoClaw injects NO_PROXY for its own subprocess spawns (loopback hosts,"); warn(" container-host aliases, and the managed inference hostname inference.local),"); diff --git a/src/lib/onboard/preflight-cdi.test.ts b/src/lib/onboard/preflight-cdi.test.ts index 5930a94b24b..6001dfaebbe 100644 --- a/src/lib/onboard/preflight-cdi.test.ts +++ b/src/lib/onboard/preflight-cdi.test.ts @@ -1,9 +1,14 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; // Import source directly so tests cannot pass against a stale build. -import { assessHost, planHostRemediation, shouldEnforceCdiNvidiaGpuSpec } from "./preflight"; +import { + assertCdiNvidiaGpuSpecPresent, + assessHost, + planHostRemediation, + shouldEnforceCdiNvidiaGpuSpec, +} from "./preflight"; type HostAssessment = Parameters[0]; @@ -37,6 +42,21 @@ function baseAssessment(overrides: Partial = {}): HostAssessment }; } +function withStderrColorDepth(colorDepth: number, noColor: string, callback: () => T): T { + const stderr = Object.assign(Object.create(process.stderr), { + getColorDepth: () => colorDepth, + isTTY: true, + }) as typeof process.stderr; + const getStderr = vi.spyOn(process, "stderr", "get").mockReturnValue(stderr); + vi.stubEnv("NO_COLOR", noColor); + try { + return callback(); + } finally { + getStderr.mockRestore(); + vi.unstubAllEnvs(); + } +} + function runCaptureWithLspci(lspciOutput: string): (command: readonly string[]) => string { const resultByCmd: Record = { "nvidia-smi": "", lspci: lspciOutput }; return (command) => { @@ -474,3 +494,48 @@ describe("shouldEnforceCdiNvidiaGpuSpec enforcement gate (#5489)", () => { ).toBe(false); }); }); + +describe("assertCdiNvidiaGpuSpecPresent severity (#6004)", () => { + it("colors the fatal missing-CDI line red before exiting", () => { + withStderrColorDepth(24, "", () => { + const error = vi.spyOn(console, "error").mockImplementation(() => undefined); + const exitProcess = vi.fn((code: number): never => { + throw new Error(`exit ${code}`); + }); + + expect(() => + assertCdiNvidiaGpuSpecPresent( + baseAssessment({ cdiNvidiaGpuSpecMissing: true }), + false, + null, + exitProcess, + ), + ).toThrow("exit 1"); + expect(error.mock.calls[0]?.[0]).toBe( + " \x1b[31m✗ Docker is configured for CDI device injection (CDISpecDirs is set), but the NVIDIA GPU CDI spec is missing or stale. OpenShell GPU startup can fail until the CDI spec is refreshed.\x1b[39m", + ); + expect(exitProcess).toHaveBeenCalledWith(1); + error.mockRestore(); + }); + }); + + it("keeps the fatal missing-CDI line plain under NO_COLOR", () => { + withStderrColorDepth(24, "1", () => { + const error = vi.spyOn(console, "error").mockImplementation(() => undefined); + + expect(() => + assertCdiNvidiaGpuSpecPresent( + baseAssessment({ cdiNvidiaGpuSpecNeedsRepair: true }), + false, + null, + (code): never => { + throw new Error(`exit ${code}`); + }, + ), + ).toThrow("exit 1"); + expect(String(error.mock.calls[0]?.[0])).toContain(" ✗ Docker is configured for CDI"); + expect(String(error.mock.calls[0]?.[0])).not.toContain("\x1b["); + error.mockRestore(); + }); + }); +}); diff --git a/src/lib/onboard/preflight-gateway-cleanup-decision.test.ts b/src/lib/onboard/preflight-gateway-cleanup-decision.test.ts index 2c2b90b2a73..6e8021f12bb 100644 --- a/src/lib/onboard/preflight-gateway-cleanup-decision.test.ts +++ b/src/lib/onboard/preflight-gateway-cleanup-decision.test.ts @@ -1,16 +1,30 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it, vi } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import type { GatewayReuseState } from "../state/gateway"; import { - PREFLIGHT_DEFERRED_RECREATE_MESSAGE, applyPreflightGatewayCleanup, + PREFLIGHT_DEFERRED_RECREATE_MESSAGE, preflightGatewayCleanupDecision, } from "./preflight-gateway-cleanup-decision"; +function stubStderrColorDepth(colorDepth: number): void { + const stderr = Object.assign(Object.create(process.stderr), { + getColorDepth: () => colorDepth, + isTTY: true, + }) as typeof process.stderr; + vi.spyOn(process, "stderr", "get").mockReturnValue(stderr); + vi.stubEnv("NO_COLOR", ""); +} + +afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); +}); + describe("preflightGatewayCleanupDecision", () => { it("defers when state is stale and Docker-driver gateway is enabled", () => { expect( @@ -69,6 +83,7 @@ describe("applyPreflightGatewayCleanup", () => { isDockerDriverGatewayEnabled: boolean; }) { const log = vi.fn(); + const warn = vi.fn(); const runOpenshell = vi.fn(() => ({ status: 0 })); const destroyGateway = vi.fn(() => true); const destroyGatewayForReuse = vi.fn< @@ -84,27 +99,44 @@ describe("applyPreflightGatewayCleanup", () => { cliDisplayName: "NemoClaw", dashboardPort: 8081, log, + warn, runOpenshell, destroyGateway, destroyGatewayForReuse, }, log, + warn, runOpenshell, destroyGateway, destroyGatewayForReuse, }; } - it("logs the deferral notice without invoking destroy on the Docker-driver path", () => { + it("warns in yellow without invoking destroy on the Docker-driver path", () => { + stubStderrColorDepth(24); const ctx = makeDeps({ gatewayReuseState: "stale", isDockerDriverGatewayEnabled: true }); const next = applyPreflightGatewayCleanup(ctx.deps); expect(next).toBe("stale"); - expect(ctx.log).toHaveBeenCalledWith(PREFLIGHT_DEFERRED_RECREATE_MESSAGE); + expect(ctx.warn).toHaveBeenCalledWith( + ` \x1b[33m⚠ ${PREFLIGHT_DEFERRED_RECREATE_MESSAGE}\x1b[39m`, + ); + expect(ctx.log).not.toHaveBeenCalled(); expect(ctx.destroyGateway).not.toHaveBeenCalled(); expect(ctx.destroyGatewayForReuse).not.toHaveBeenCalled(); expect(ctx.runOpenshell).not.toHaveBeenCalled(); }); + it("prints the deferral warning without ANSI when NO_COLOR is set", () => { + stubStderrColorDepth(24); + vi.stubEnv("NO_COLOR", "1"); + const ctx = makeDeps({ gatewayReuseState: "stale", isDockerDriverGatewayEnabled: true }); + + applyPreflightGatewayCleanup(ctx.deps); + + expect(ctx.warn).toHaveBeenCalledWith(` ⚠ ${PREFLIGHT_DEFERRED_RECREATE_MESSAGE}`); + expect(String(ctx.warn.mock.calls[0]?.[0])).not.toContain("\x1b["); + }); + it("destroys the legacy gateway and stops the dashboard forward on the non-Docker-driver path", () => { const ctx = makeDeps({ gatewayReuseState: "stale", isDockerDriverGatewayEnabled: false }); const next = applyPreflightGatewayCleanup(ctx.deps); @@ -123,6 +155,7 @@ describe("applyPreflightGatewayCleanup", () => { const next = applyPreflightGatewayCleanup(ctx.deps); expect(next).toBe(state); expect(ctx.log).not.toHaveBeenCalled(); + expect(ctx.warn).not.toHaveBeenCalled(); expect(ctx.destroyGateway).not.toHaveBeenCalled(); expect(ctx.destroyGatewayForReuse).not.toHaveBeenCalled(); expect(ctx.runOpenshell).not.toHaveBeenCalled(); diff --git a/src/lib/onboard/preflight-gateway-cleanup-decision.ts b/src/lib/onboard/preflight-gateway-cleanup-decision.ts index 1b4312e93d9..1b4e47da93a 100644 --- a/src/lib/onboard/preflight-gateway-cleanup-decision.ts +++ b/src/lib/onboard/preflight-gateway-cleanup-decision.ts @@ -1,12 +1,13 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { warnLine } from "../cli/terminal-style"; import type { GatewayReuseState } from "../state/gateway"; export type PreflightGatewayCleanupAction = "defer" | "destroy-legacy" | "noop"; export const PREFLIGHT_DEFERRED_RECREATE_MESSAGE = - " ⚠ Gateway will be recreated when sandbox creation starts — this will affect running sandboxes."; + "Gateway will be recreated when sandbox creation starts — this will affect running sandboxes."; export function preflightGatewayCleanupDecision(opts: { gatewayReuseState: GatewayReuseState; @@ -24,6 +25,7 @@ export interface PreflightGatewayCleanupDeps { cliDisplayName: string; dashboardPort: number; log: (line: string) => void; + warn: (line: string) => void; runOpenshell: (args: string[], options: { ignoreError: true }) => unknown; destroyGateway: () => boolean; destroyGatewayForReuse: ( @@ -39,7 +41,7 @@ export function applyPreflightGatewayCleanup(deps: PreflightGatewayCleanupDeps): isDockerDriverGatewayEnabled: deps.isDockerDriverGatewayEnabled, }); if (action === "defer") { - deps.log(PREFLIGHT_DEFERRED_RECREATE_MESSAGE); + deps.warn(warnLine(PREFLIGHT_DEFERRED_RECREATE_MESSAGE)); return deps.gatewayReuseState; } if (action === "destroy-legacy") { diff --git a/src/lib/onboard/preflight-messages.test.ts b/src/lib/onboard/preflight-messages.test.ts new file mode 100644 index 00000000000..f416401ceec --- /dev/null +++ b/src/lib/onboard/preflight-messages.test.ts @@ -0,0 +1,114 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + printDockerNotReachableError, + printLowMemoryWarning, + printMessagingProviderMissing, + printSwapCreationFailed, + printUnderProvisionedRuntimeWarning, + printUnsupportedRuntimeError, +} from "./preflight-messages"; + +function lines(spy: ReturnType): string[] { + return spy.mock.calls.map((call: unknown[]) => String(call[0])); +} + +function withStderrColorDepth(colorDepth: number, callback: () => T): T { + const stderr = Object.assign(Object.create(process.stderr), { + getColorDepth: () => colorDepth, + isTTY: true, + }) as typeof process.stderr; + const getStderr = vi.spyOn(process, "stderr", "get").mockReturnValue(stderr); + vi.stubEnv("NO_COLOR", ""); + try { + return callback(); + } finally { + getStderr.mockRestore(); + vi.unstubAllEnvs(); + } +} + +describe("onboard preflight severity messages (#6004)", () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("colors representative failure and warning messages when stderr supports color", () => { + withStderrColorDepth(24, () => { + const err = vi.spyOn(console, "error").mockImplementation(() => undefined); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + printDockerNotReachableError(); + printLowMemoryWarning({ totalRamMB: 4000, totalSwapMB: 0, totalMB: 4000 }); + expect(lines(err)[0]).toBe( + " \x1b[31m✗ Docker is not reachable. Please fix Docker and try again.\x1b[39m", + ); + expect(lines(warn)[0]).toBe( + " \x1b[33m⚠ Low memory detected (4000 MB RAM + 0 MB swap = 4000 MB total)\x1b[39m", + ); + }); + }); + + it("prints representative failure and warning messages without ANSI on plain stderr", () => { + withStderrColorDepth(1, () => { + const err = vi.spyOn(console, "error").mockImplementation(() => undefined); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + printDockerNotReachableError(); + printLowMemoryWarning({ totalRamMB: 4000, totalSwapMB: 0, totalMB: 4000 }); + expect(lines(err)[0]).toBe(" ✗ Docker is not reachable. Please fix Docker and try again."); + expect(lines(warn)[0]).toBe( + " ⚠ Low memory detected (4000 MB RAM + 0 MB swap = 4000 MB total)", + ); + expect([...lines(err), ...lines(warn)].join("\n")).not.toContain("\x1b["); + }); + }); + + it("prints the unsupported-runtime failure to stderr with a ✗ marker", () => { + const err = vi.spyOn(console, "error").mockImplementation(() => undefined); + printUnsupportedRuntimeError(); + expect(err).toHaveBeenCalledTimes(3); + expect(lines(err)[0]).toContain("✗"); + expect(lines(err)[0]).toContain("Docker driver"); + expect(lines(err).join("\n")).toContain("Switch to Docker Engine"); + }); + + it("prints the under-provisioned warning to stderr with a ⚠ marker and colima resize", () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + printUnderProvisionedRuntimeWarning({ + detectedStr: "2 vCPU / 2.0 GiB", + runtime: "colima", + recommendedCpus: 4, + recommendedMemGib: 12, + }); + expect(lines(warn)[0]).toContain("⚠"); + expect(lines(warn)[0]).toContain("under-provisioned: 2 vCPU / 2.0 GiB"); + expect(lines(warn).join("\n")).toContain("colima start --cpu 4 --memory 12"); + }); + + it("prints the Docker Desktop resize hint for the docker-desktop runtime", () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + printUnderProvisionedRuntimeWarning({ + detectedStr: "x", + runtime: "docker-desktop", + recommendedCpus: 4, + recommendedMemGib: 12, + }); + expect(lines(warn).join("\n")).toContain("Docker Desktop → Settings → Resources"); + }); + + it("prints the swap-creation failure to stderr with a ⚠ marker", () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + printSwapCreationFailed("mkswap failed"); + expect(lines(warn)[0]).toContain("⚠ Could not create swap: mkswap failed"); + expect(lines(warn).join("\n")).toContain("may fail with OOM"); + }); + + it("prints a missing messaging provider to stderr with a ⚠ marker and fix hint", () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + printMessagingProviderMissing("slack"); + expect(lines(warn)[0]).toContain("⚠ Messaging provider 'slack' was not found in the gateway."); + expect(lines(warn).join("\n")).toContain("openshell provider create --name slack"); + }); +}); diff --git a/src/lib/onboard/preflight-messages.ts b/src/lib/onboard/preflight-messages.ts new file mode 100644 index 00000000000..a8dff7330b8 --- /dev/null +++ b/src/lib/onboard/preflight-messages.ts @@ -0,0 +1,88 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Onboard preflight severity messages, extracted from `onboard.ts` so they can + * adopt the shared `warnLine`/`failLine` renderer (#6004) without growing the + * top-level entrypoint past the `onboard-entrypoint-budget` / codebase-growth + * CI ceiling (same extraction pattern as `bridge-dns-preflight.ts`). + * + * Every WARN line here is emitted through `console.warn` and every ERROR line + * through `console.error`, so the renderer's stderr-keyed color decision + * matches the stream the line lands on. + */ + +import { failLine, warnLine } from "../cli/terminal-style"; +import { cliDisplayName } from "./branding"; + +/** Docker cannot be reached, so onboarding cannot continue. */ +export function printDockerNotReachableError(): void { + console.error(failLine("Docker is not reachable. Please fix Docker and try again.")); +} + +/** Podman under the Linux Docker-driver path is unsupported. */ +export function printUnsupportedRuntimeError(): void { + console.error(failLine(`${cliDisplayName()} onboarding now uses OpenShell's Docker driver.`)); + console.error(` Podman is not supported for this ${cliDisplayName()} integration path.`); + console.error(" Switch to Docker Engine and rerun onboarding."); +} + +export interface UnderProvisionedRuntimeWarning { + /** Human-readable detected resources, e.g. "2 vCPU / 2.0 GiB". */ + detectedStr: string; + /** Container runtime kind (drives the resize suggestion). */ + runtime: string; + recommendedCpus: number; + recommendedMemGib: number; +} + +/** Container runtime detected below the recommended CPU/memory floor. */ +export function printUnderProvisionedRuntimeWarning( + opts: UnderProvisionedRuntimeWarning, + warn: (message: string) => void = console.warn, +): void { + const { detectedStr, runtime, recommendedCpus, recommendedMemGib } = opts; + warn( + warnLine( + `Container runtime under-provisioned: ${detectedStr} detected ` + + `(recommended: ${recommendedCpus} vCPU / ${recommendedMemGib} GiB).`, + ), + ); + warn(" The sandbox build will be slow and may stall on default Colima settings."); + if (runtime === "colima") { + warn( + ` Suggested: colima stop && colima start --cpu ${recommendedCpus} --memory ${recommendedMemGib}`, + ); + } else if (runtime === "docker-desktop") { + warn(" Suggested: Docker Desktop → Settings → Resources, raise CPU/memory."); + } + warn(" Set NEMOCLAW_IGNORE_RUNTIME_RESOURCES=1 to silence this check."); +} + +/** Total system memory is below the sandbox-build comfort threshold. */ +export function printLowMemoryWarning(mem: { + totalRamMB: number; + totalSwapMB: number; + totalMB: number; +}): void { + console.warn( + warnLine( + `Low memory detected (${mem.totalRamMB} MB RAM + ${mem.totalSwapMB} MB swap = ${mem.totalMB} MB total)`, + ), + ); +} + +/** Swap-file creation failed on a low-memory host. */ +export function printSwapCreationFailed(reason: string | undefined): void { + console.warn(warnLine(`Could not create swap: ${reason}`)); + console.warn(" Sandbox creation may fail with OOM on low-memory systems."); +} + +/** A configured messaging provider was not present in the gateway. */ +export function printMessagingProviderMissing(providerName: string): void { + console.warn(warnLine(`Messaging provider '${providerName}' was not found in the gateway.`)); + console.warn(" The credential may not be available inside the sandbox."); + console.warn( + ` To fix: openshell provider create --name ${providerName} --type generic --credential `, + ); +} diff --git a/src/lib/onboard/preflight-runtime-resources.test.ts b/src/lib/onboard/preflight-runtime-resources.test.ts index 59a4b4e4239..508e8f4da8d 100644 --- a/src/lib/onboard/preflight-runtime-resources.test.ts +++ b/src/lib/onboard/preflight-runtime-resources.test.ts @@ -40,6 +40,7 @@ describe("checkContainerRuntimeResources", () => { ).rejects.toThrow("exit:1"); expect(confirm).toHaveBeenCalledOnce(); + expect(warn.mock.calls[0]?.[0]).toContain("⚠ Container runtime under-provisioned"); expect(warn.mock.calls.flat().join("\n")).toContain("2 vCPU / 2.0 GiB"); expect(error).toHaveBeenCalledWith(expect.stringContaining("Aborted by user")); expect(exit).toHaveBeenCalledWith(1); diff --git a/src/lib/onboard/preflight.ts b/src/lib/onboard/preflight.ts index b3b7f299b3e..fe94b3145de 100644 --- a/src/lib/onboard/preflight.ts +++ b/src/lib/onboard/preflight.ts @@ -15,6 +15,7 @@ import net from "node:net"; import os from "node:os"; import path from "node:path"; +import { failLine } from "../cli/terminal-style"; import { DASHBOARD_PORT } from "../core/ports"; import { assessNvidiaCdiHost, @@ -27,6 +28,7 @@ import { extractCdiMismatchFilePath, getNvidiaCdiSpecPath, } from "./docker-cdi"; +import { printUnderProvisionedRuntimeWarning } from "./preflight-messages"; import { printRemediationActions } from "./remediation"; import { isWslDockerDesktopRuntime, @@ -395,19 +397,15 @@ export async function checkContainerRuntimeResources( return; } - warn( - ` ⚠ Container runtime under-provisioned: ${detected.join(" / ") || "unknown"} detected ` + - `(recommended: ${MIN_RECOMMENDED_DOCKER_CPUS} vCPU / ${MIN_RECOMMENDED_DOCKER_MEM_GIB} GiB).`, + printUnderProvisionedRuntimeWarning( + { + detectedStr: detected.join(" / ") || "unknown", + runtime: host.runtime, + recommendedCpus: MIN_RECOMMENDED_DOCKER_CPUS, + recommendedMemGib: MIN_RECOMMENDED_DOCKER_MEM_GIB, + }, + warn, ); - warn(" The sandbox build will be slow and may stall on default Colima settings."); - if (host.runtime === "colima") { - warn( - ` Suggested: colima stop && colima start --cpu ${MIN_RECOMMENDED_DOCKER_CPUS} --memory ${MIN_RECOMMENDED_DOCKER_MEM_GIB}`, - ); - } else if (host.runtime === "docker-desktop") { - warn(" Suggested: Docker Desktop → Settings → Resources, raise CPU/memory."); - } - warn(" Set NEMOCLAW_IGNORE_RUNTIME_RESOURCES=1 to silence this check."); if (options.nonInteractive) { warn(" WARNING: Non-interactive mode is continuing despite under-provisioned runtime."); return; @@ -731,7 +729,9 @@ export function assertCdiNvidiaGpuSpecPresent( ) return; console.error( - " Docker is configured for CDI device injection (CDISpecDirs is set), but the NVIDIA GPU CDI spec is missing or stale. OpenShell GPU startup can fail until the CDI spec is refreshed.", + failLine( + "Docker is configured for CDI device injection (CDISpecDirs is set), but the NVIDIA GPU CDI spec is missing or stale. OpenShell GPU startup can fail until the CDI spec is refreshed.", + ), ); printRemediationActions(planHostRemediation(host)); exitProcess(1); diff --git a/src/lib/onboard/sandbox-gpu-preflight.ts b/src/lib/onboard/sandbox-gpu-preflight.ts index ee3a695f199..eb7be13eb58 100644 --- a/src/lib/onboard/sandbox-gpu-preflight.ts +++ b/src/lib/onboard/sandbox-gpu-preflight.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { dockerInfoFormat } from "../adapters/docker"; +import { failLine, warnLine } from "../cli/terminal-style"; import type { GpuDetection } from "../inference/nim"; import type { SandboxGpuProofResult } from "../state/registry"; import { findReadableNvidiaCdiSpecFiles, getDockerCdiSpecDirs } from "./docker-cdi"; @@ -86,7 +87,7 @@ export function exitOnSandboxGpuConfigErrors( ): void { if (config.errors.length > 0) { console.error(""); - for (const error of config.errors) console.error(` ✗ ${error}`); + for (const error of config.errors) console.error(failLine(error)); exitProcess(1); } } @@ -132,7 +133,7 @@ function validateJetsonSandboxGpuPreflight( ): void { if (!dockerNvidiaRuntimeAvailable(deps)) { console.error(""); - console.error(" ✗ Docker NVIDIA runtime was not detected for Jetson/Tegra sandbox GPU."); + console.error(failLine("Docker NVIDIA runtime was not detected for Jetson/Tegra sandbox GPU.")); console.error(" Jetson sandbox GPU uses NVIDIA Container Runtime semantics, not CDI."); console.error( " Install/configure NVIDIA Container Toolkit for Docker, then restart Docker:", @@ -234,7 +235,7 @@ export function createDirectSandboxGpuVerifier( if (proof.optional !== true) { // Required proof (e.g. the sandbox-exec wrapper itself): keep the // historical hard-fail so onboarding aborts and rolls back. - console.error(` ✗ GPU proof failed: ${proof.label}`); + console.error(failLine(`GPU proof failed: ${proof.label}`)); if (diagnostic) console.error(` ${diagnostic}`); for (const line of sandboxGpuRemediationLines({ wslDockerDesktopStatus: detectWslDockerDesktopStatus(deps), @@ -254,7 +255,7 @@ export function createDirectSandboxGpuVerifier( if (proof.id === CUDA_USABILITY_PROOF_ID && cudaInitRan) { cudaFailure = { label: proof.label, detail: diagnostic }; } - console.warn(` ⚠ GPU proof inconclusive: ${proof.label}`); + console.warn(warnLine(`GPU proof inconclusive: ${proof.label}`)); if (diagnostic) console.warn(` ${diagnostic}`); } const status: SandboxGpuProofResult["status"] = cudaVerified @@ -265,7 +266,7 @@ export function createDirectSandboxGpuVerifier( if (status === "verified") { console.log(" ✓ Sandbox CUDA usability proven (cuInit succeeded)."); } else if (status === "failed") { - console.warn(` ⚠ Sandbox CUDA proof failed: ${cudaFailure?.label}`); + console.warn(warnLine(`Sandbox CUDA proof failed: ${cudaFailure?.label}`)); const lines = resolvedPlatform === "jetson" ? jetsonGpuProofRemediationLines() @@ -274,7 +275,9 @@ export function createDirectSandboxGpuVerifier( }); for (const line of lines) console.warn(` ${line}`); } else { - console.warn(" ⚠ Sandbox GPU enabled but CUDA usability is unverified (no CUDA proof ran)."); + console.warn( + warnLine("Sandbox GPU enabled but CUDA usability is unverified (no CUDA proof ran)."), + ); } return { status, @@ -315,7 +318,7 @@ export function validateSandboxGpuPreflight( ); if (cdiSpecFiles.length === 0) { console.error(""); - console.error(" ✗ Docker CDI GPU support was not detected."); + console.error(failLine("Docker CDI GPU support was not detected.")); for (const line of sandboxGpuRemediationLines({ wslDockerDesktopStatus, })) {