From f8a9a58fc3b51c8c7d182fa3db26e36f1bf8d9ad Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sun, 14 Jun 2026 11:24:05 -0700 Subject: [PATCH 01/23] refactor(onboard): lower cognitive complexity ratchet to 245 Signed-off-by: Carlos Villela (cherry picked from commit 41fcd6acb853b81a25b5a33e520bf9ebeffe0a1c) --- biome.json | 2 +- src/lib/onboard.ts | 1453 +++++++++++++++++++++++--------------------- 2 files changed, 774 insertions(+), 681 deletions(-) diff --git a/biome.json b/biome.json index e990b87412c..99455513804 100644 --- a/biome.json +++ b/biome.json @@ -86,7 +86,7 @@ "noExcessiveCognitiveComplexity": { "level": "error", "options": { - "maxAllowedComplexity": 255 + "maxAllowedComplexity": 245 } } }, diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 3ad2da4e3a1..f77f55ecf01 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -3343,6 +3343,703 @@ async function selectAndValidateOllamaModel( } } +type SetupNimSelectionState = { + model: string | typeof BACK_TO_SELECTION | null; + provider: string; + endpointUrl: string | null; + credentialEnv: string | null; + hermesAuthMethod: HermesAuthMethod | null; + hermesToolGateways: string[]; + preferredInferenceApi: string | null; + nimContainer: string | null; +}; + +type SetupNimSelectionResult = "selected" | "retry-selection"; + +type RemoteProviderSelectionArgs = { + selected: ProviderChoice; + requestedModel: string | null; + recoveredFromSandbox: boolean; + recoveredModel: string | null; + sandboxName: string | null; +}; + +type RemoteProviderConfig = (typeof REMOTE_PROVIDER_CONFIG)[keyof typeof REMOTE_PROVIDER_CONFIG]; + +type RemoteModelValidationResult = "selected" | "retry-model" | "retry-selection"; + +async function validateSelectedRemoteModel( + selected: ProviderChoice, + remoteConfig: RemoteProviderConfig, + state: SetupNimSelectionState, + selectedCredentialEnv: string, +): Promise { + const selectedModel = requireValue( + isBackToSelection(state.model) ? null : state.model, + `Missing model for ${remoteConfig.label}`, + ); + if (selected.key === "custom") { + const validation = await validateCustomOpenAiLikeSelection( + remoteConfig.label, + state.endpointUrl || OPENAI_ENDPOINT_URL, + selectedModel, + selectedCredentialEnv, + remoteConfig.helpUrl, + ); + if (validation.ok) { + const explicitApi = (process.env.NEMOCLAW_PREFERRED_API || "").trim().toLowerCase(); + if ( + explicitApi && + explicitApi !== "openai-completions" && + explicitApi !== "chat-completions" + ) { + state.preferredInferenceApi = validation.api; + } else { + if (validation.api !== "openai-completions") { + console.log( + " ℹ Using chat completions API (compatible endpoints may not support the Responses API developer role)", + ); + } + state.preferredInferenceApi = "openai-completions"; + } + return "selected"; + } + if ( + validation.retry === "credential" || + validation.retry === "retry" || + validation.retry === "model" + ) { + return "retry-model"; + } + return validation.retry === "selection" ? "retry-selection" : "retry-model"; + } + + if (selected.key === "anthropicCompatible") { + const validation = await validateCustomAnthropicSelection( + remoteConfig.label, + state.endpointUrl || ANTHROPIC_ENDPOINT_URL, + selectedModel, + selectedCredentialEnv, + remoteConfig.helpUrl, + ); + if (validation.ok) { + state.preferredInferenceApi = validation.api; + return "selected"; + } + if ( + validation.retry === "credential" || + validation.retry === "retry" || + validation.retry === "model" + ) { + return "retry-model"; + } + return validation.retry === "selection" ? "retry-selection" : "retry-model"; + } + + const retryMessage = "Please choose a provider/model again."; + if (selected.key === "anthropic") { + const validation = await validateAnthropicSelectionWithRetryMessage( + remoteConfig.label, + state.endpointUrl || ANTHROPIC_ENDPOINT_URL, + selectedModel, + selectedCredentialEnv, + retryMessage, + remoteConfig.helpUrl, + ); + if (validation.ok) { + state.preferredInferenceApi = validation.api; + return "selected"; + } + if ( + validation.retry === "credential" || + validation.retry === "retry" || + validation.retry === "model" + ) { + return "retry-model"; + } + return "retry-selection"; + } + + const validation = await validateOpenAiLikeSelection( + remoteConfig.label, + requireValue(state.endpointUrl, `Missing endpoint URL for ${remoteConfig.label}`), + selectedModel, + selectedCredentialEnv, + retryMessage, + remoteConfig.helpUrl, + { + requireResponsesToolCalling: shouldRequireResponsesToolCalling(state.provider), + skipResponsesProbe: shouldSkipResponsesProbe(state.provider), + authMode: getProbeAuthMode(state.provider), + }, + ); + if (validation.ok) { + state.preferredInferenceApi = validation.api; + return "selected"; + } + if ( + validation.retry === "credential" || + validation.retry === "retry" || + validation.retry === "model" + ) { + return "retry-model"; + } + return "retry-selection"; +} + +async function handleVllmSelection( + state: SetupNimSelectionState, +): Promise { + console.log(` ✓ Using existing vLLM on localhost:${VLLM_PORT}`); + state.provider = "vllm-local"; + // Local vLLM uses an internal credential env, no user API key. + state.credentialEnv = null; + state.endpointUrl = getLocalProviderBaseUrl(state.provider); + if (!state.endpointUrl) { + console.error(" Local vLLM base URL could not be determined."); + process.exit(1); + } + + const vllmModelsRaw = runCapture(["curl", "-sf", `http://127.0.0.1:${VLLM_PORT}/v1/models`], { + ignoreError: true, + }); + let vllmModels: { data?: Array<{ id?: unknown }> } = {}; + try { + vllmModels = JSON.parse(vllmModelsRaw); + if (vllmModels.data && vllmModels.data.length > 0) { + const detectedModel = + typeof vllmModels.data[0]?.id === "string" ? vllmModels.data[0].id : null; + state.model = detectedModel; + if (!detectedModel || !isSafeModelId(detectedModel)) { + console.error(` Detected model ID contains invalid characters: ${state.model}`); + process.exit(1); + } + console.log(` Detected model: ${state.model}`); + } else { + console.error(" Could not detect model from vLLM. Please specify manually."); + process.exit(1); + } + } catch { + console.error( + ` Could not query vLLM models endpoint. Is vLLM running on localhost:${VLLM_PORT}?`, + ); + process.exit(1); + } + + const validationBaseUrl = getLocalProviderValidationBaseUrl(state.provider); + if (!validationBaseUrl) { + console.error(" Local vLLM validation URL could not be determined."); + process.exit(1); + } + const validation = await validateOpenAiLikeSelection( + "Local vLLM", + validationBaseUrl, + requireValue(state.model as string | null | undefined, "Expected a detected vLLM model"), + null, + ); + if (validation.retry === "selection" || validation.retry === "model") { + return "retry-selection"; + } + if (!validation.ok) return "retry-selection"; + + localInference.applyVllmRuntimeContextWindow(vllmModels, state.model as string); + state.preferredInferenceApi = validation.api; + // Force chat completions — vLLM's /v1/responses endpoint does not run the + // --tool-call-parser, so tool calls arrive as raw text (#976). + if (state.preferredInferenceApi !== "openai-completions") { + console.log(" ℹ Using chat completions API (tool-call-parser requires /v1/chat/completions)"); + } + state.preferredInferenceApi = "openai-completions"; + return "selected"; +} + +async function handleRoutedSelection( + state: SetupNimSelectionState, +): Promise { + const bp = loadBlueprintProfile("routed"); + if (!bp || bp.router?.enabled !== true) { + console.error(" Router is not enabled in nemoclaw-blueprint/blueprint.yaml."); + if (isNonInteractive()) process.exit(1); + return "retry-selection"; + } + + const routerCredentialEnv = + bp.router?.credential_env || bp.credential_env || DEFAULT_MODEL_ROUTER_CREDENTIAL_ENV; + state.credentialEnv = routerCredentialEnv; + const routedCredential = + hydrateCredentialEnv(routerCredentialEnv) || + normalizeCredentialValue(bp.credential_default || ""); + if (routedCredential) { + saveCredential(routerCredentialEnv, routedCredential); + } + + const _providerKeyHint = (process.env.NEMOCLAW_PROVIDER_KEY || "").trim(); + if (_providerKeyHint && !resolveProviderCredential(routerCredentialEnv)) { + saveCredential(routerCredentialEnv, _providerKeyHint); + } + if (isNonInteractive()) { + if (!resolveProviderCredential(routerCredentialEnv)) { + console.error( + ` ${routerCredentialEnv} (or NEMOCLAW_PROVIDER_KEY) is required for Model Router in non-interactive mode.`, + ); + process.exit(1); + } + } else if (!resolveProviderCredential(routerCredentialEnv)) { + console.log(""); + console.log(" Model Router accepts NVIDIA API keys (nvapi-...)."); + console.log(" Get one at https://build.nvidia.com"); + console.log(""); + const routerCredentialResult = await ensureNamedCredential( + routerCredentialEnv, + "Model Router API key", + null, + ); + if (credentialPrompt.returningToProviderSelection(routerCredentialResult)) { + return "retry-selection"; + } + } + + state.provider = bp.provider_name || "nvidia-router"; + state.model = bp.model; + const { HOST_GATEWAY_URL } = require("./inference/local"); + const routerEndpointUrl = bp.endpoint || ""; + state.endpointUrl = routerEndpointUrl; + if (routerEndpointUrl.match(/localhost|127\.0\.0\.1/)) { + const u = new URL(routerEndpointUrl); + state.endpointUrl = `${HOST_GATEWAY_URL}:${u.port}${u.pathname}`; + } + state.preferredInferenceApi = "openai-completions"; + console.log(` ✓ Using Model Router: ${state.provider} / ${state.model}`); + return "selected"; +} + +async function handleNimLocalSelection( + gpu: ReturnType, + args: Pick< + RemoteProviderSelectionArgs, + "requestedModel" | "recoveredFromSandbox" | "recoveredModel" + >, + state: SetupNimSelectionState, +): Promise { + const localGpu = requireValue(gpu, "GPU details are required for local NIM model selection"); + const models = nim.listModels().filter((m) => m.minGpuMemoryMB <= localGpu.totalMemoryMB); + if (models.length === 0) { + console.log(" No NIM models fit your GPU VRAM. Falling back to cloud API."); + return "selected"; + } + + let sel; + if (isNonInteractive()) { + const targetModel = + args.requestedModel || (args.recoveredFromSandbox ? args.recoveredModel : null); + if (targetModel) { + sel = models.find((m) => m.name === targetModel); + if (!sel) { + const label = args.requestedModel ? "NEMOCLAW_MODEL for NIM" : "Recorded NIM model"; + console.error(` Unsupported ${label}: ${targetModel}`); + process.exit(1); + } + } else { + sel = models[0]; + } + note(` [non-interactive] NIM model: ${sel.name}`); + } else { + console.log(""); + console.log(" Models that fit your GPU:"); + models.forEach((m, i) => { + console.log(` ${i + 1}) ${m.name} (min ${m.minGpuMemoryMB} MB)`); + }); + console.log(""); + + const modelChoice = await prompt(` Choose model [1]: `); + sel = selectFromNumberedMenuOrExit(modelChoice, 1, models); + } + state.model = sel.name; + + let ngcApiKey: string | null = null; + if (!nim.isNgcLoggedIn()) { + if (isNonInteractive()) { + console.error( + " Docker is not logged in to nvcr.io. In non-interactive mode, run `docker login nvcr.io` first and retry.", + ); + process.exit(1); + } + console.log(""); + console.log(" NGC API Key required to pull NIM images."); + console.log(" Get one from: https://org.ngc.nvidia.com/setup/api-key"); + console.log(""); + let ngcKey = await credentialPrompt.readValue(" NGC API Key: "); + if (credentialPrompt.returningToProviderSelection(ngcKey)) return "retry-selection"; + if (!ngcKey) { + console.error(" NGC API Key is required for Local NIM."); + process.exit(1); + } + if (!nim.dockerLoginNgc(ngcKey)) { + console.error(" Failed to login to NGC registry. Check your API key and try again."); + console.log(""); + ngcKey = await credentialPrompt.readValue(" NGC API Key: "); + if (credentialPrompt.returningToProviderSelection(ngcKey)) return "retry-selection"; + if (!ngcKey || !nim.dockerLoginNgc(ngcKey)) { + console.error(" NGC login failed. Cannot pull NIM images."); + process.exit(1); + } + } + ngcApiKey = ngcKey; + } else { + ngcApiKey = + hydrateCredentialEnv("NGC_API_KEY") || hydrateCredentialEnv("NVIDIA_INFERENCE_API_KEY"); + if (!ngcApiKey && !isNonInteractive()) { + console.log(""); + console.log(" NGC API Key required to download NIM model weights at runtime."); + console.log(" (Docker is logged in to nvcr.io, but the key was not saved.)"); + const ngcKey = await credentialPrompt.readValue(" NGC API Key: "); + if (credentialPrompt.returningToProviderSelection(ngcKey)) return "retry-selection"; + ngcApiKey = ngcKey || null; + } + } + + console.log(` Pulling NIM image for ${state.model}...`); + nim.pullNimImage(state.model); + + console.log(" Starting NIM container..."); + const nimContainerNameLocal = nim.containerName(GATEWAY_NAME); + state.nimContainer = nim.startNimContainerByName(nimContainerNameLocal, state.model, undefined, { + ngcApiKey: ngcApiKey ?? undefined, + }); + + console.log(" Waiting for NIM to become healthy..."); + if (!nim.waitForNimHealth(undefined, undefined, { container: nimContainerNameLocal })) { + console.error(" NIM failed to start. Falling back to cloud API."); + state.model = null; + state.nimContainer = null; + return "selected"; + } + + state.provider = "vllm-local"; + state.credentialEnv = null; + state.endpointUrl = getLocalProviderBaseUrl(state.provider); + if (!state.endpointUrl) { + console.error(" Local NVIDIA NIM base URL could not be determined."); + process.exit(1); + } + state.model = nim.adoptServedModelId(state.model); + const nimValidationUrl = getLocalProviderValidationBaseUrl(state.provider) || state.endpointUrl; + const validation = await validateOpenAiLikeSelection( + "Local NVIDIA NIM", + nimValidationUrl, + requireValue(state.model, "Expected a Local NVIDIA NIM model after startup"), + null, + ); + if (validation.retry === "selection" || validation.retry === "model") return "retry-selection"; + if (!validation.ok) return "retry-selection"; + if (validation.api !== "openai-completions") { + console.log(" ℹ Using chat completions API (tool-call-parser requires /v1/chat/completions)"); + } + state.preferredInferenceApi = "openai-completions"; + return "selected"; +} + +async function handleRemoteProviderSelection( + args: RemoteProviderSelectionArgs, + state: SetupNimSelectionState, +): Promise { + const { selected, requestedModel, recoveredFromSandbox, recoveredModel, sandboxName } = args; + const remoteConfig = REMOTE_PROVIDER_CONFIG[selected.key]; + state.provider = remoteConfig.providerName; + state.credentialEnv = remoteConfig.credentialEnv; + state.endpointUrl = remoteConfig.endpointUrl; + state.preferredInferenceApi = null; + + if (selected.key === "custom" || selected.key === "anthropicCompatible") { + const kind = selected.key === "custom" ? "openai" : "anthropic"; + const _envUrl = (process.env.NEMOCLAW_ENDPOINT_URL || "").trim(); + const endpointInput = isNonInteractive() + ? _envUrl + : (await prompt( + _envUrl + ? ` ${kind === "openai" ? "OpenAI" : "Anthropic"}-compatible base URL [${_envUrl}]: ` + : kind === "openai" + ? " OpenAI-compatible base URL (e.g., https://openrouter.ai): " + : " Anthropic-compatible base URL (e.g., https://proxy.example.com): ", + )) || _envUrl; + const navigation = getNavigationChoice(endpointInput); + if (navigation === "back") { + console.log(" Returning to provider selection."); + console.log(""); + return "retry-selection"; + } + if (navigation === "exit") { + exitOnboardFromPrompt(); + } + state.endpointUrl = normalizeProviderBaseUrl(endpointInput, kind); + if (!state.endpointUrl) { + console.error( + selected.key === "custom" + ? " Endpoint URL is required for Other OpenAI-compatible endpoint." + : " Endpoint URL is required for Other Anthropic-compatible endpoint.", + ); + if (isNonInteractive()) { + process.exit(1); + } + console.log(""); + return "retry-selection"; + } + if (selected.key === "anthropicCompatible") { + state.endpointUrl = bedrockRuntimeOnboard.normalizeCustomAnthropicEndpointUrl( + state.endpointUrl, + ); + } + } + + if (selected.key === "hermesProvider") { + const selectedHermesAuthMethod = await promptHermesAuthMethod(); + if (isBackToSelection(selectedHermesAuthMethod)) { + state.hermesAuthMethod = null; + console.log(" Returning to provider selection."); + console.log(""); + return "retry-selection"; + } + state.hermesAuthMethod = normalizeHermesAuthMethod( + selectedHermesAuthMethod as string | null | undefined, + ); + if (state.hermesAuthMethod === HERMES_AUTH_METHOD_API_KEY) { + state.credentialEnv = HERMES_NOUS_API_KEY_CREDENTIAL_ENV; + stageNousApiKeyProviderEnv(); + if (isNonInteractive()) { + if (!resolveHermesNousApiKey()) { + console.error(" Hermes Provider Nous API Key is required in non-interactive mode."); + process.exit(1); + } + } else { + const hermesKeyResult = await ensureHermesNousApiKeyEnv(); + if (credentialPrompt.returningToProviderSelection(hermesKeyResult)) { + return "retry-selection"; + } + } + } else { + state.credentialEnv = remoteConfig.credentialEnv; + } + const recordedHermesToolGateways = sandboxName + ? normalizeHermesToolGatewaySelections(registry.getSandbox(sandboxName)?.hermesToolGateways) + : null; + state.hermesToolGateways = await setupHermesToolGateways( + state.provider, + state.hermesAuthMethod, + recordedHermesToolGateways, + { prompt, note, isNonInteractive }, + ); + + const defaultModel = + requestedModel || (recoveredFromSandbox && recoveredModel) || remoteConfig.defaultModel; + if (isNonInteractive()) { + state.model = defaultModel; + } else { + let hermesProviderModels: string[] = []; + try { + hermesProviderModels = await nousModels.getHermesProviderModelOptions(); + } catch (err) { + const detail = err instanceof Error ? err.message : String(err); + console.warn( + ` Warning: failed to load Nous model recommendations; falling back to the current/default model (${detail}).`, + ); + } + state.model = await promptRemoteModel(remoteConfig.label, selected.key, defaultModel, null, { + otherShowsFullList: true, + remoteModelOptions: { [selected.key]: hermesProviderModels }, + topLevelModelLimit: 10, + }); + } + if (isBackToSelection(state.model)) { + console.log(" Returning to provider selection."); + console.log(""); + return "retry-selection"; + } + state.preferredInferenceApi = "openai-completions"; + console.log(` Using ${remoteConfig.label} with model: ${state.model}`); + return "selected"; + } + + hydrateCredentialEnv(state.credentialEnv); + + if (selected.key === "build") { + const _nvProviderKey = (process.env.NEMOCLAW_PROVIDER_KEY || "").trim(); + const existingNvidiaKey = ["NVIDIA_INFERENCE_API_KEY", "NVIDIA_API_KEY"] + .map((envName) => normalizeCredentialValue(process.env[envName] ?? "")) + .find(Boolean); + if (_nvProviderKey && !existingNvidiaKey) { + process.env.NVIDIA_INFERENCE_API_KEY = _nvProviderKey; + } + if (isNonInteractive()) { + const resolvedNvidiaKey = resolveProviderCredential("NVIDIA_INFERENCE_API_KEY"); + if (resolvedNvidiaKey) { + const keyError = validateNvidiaApiKeyValue(resolvedNvidiaKey); + if (keyError) { + console.error(keyError); + console.error(` Get a key from ${REMOTE_PROVIDER_CONFIG.build.helpUrl}`); + process.exit(1); + } + } else if (!providerExistsInGateway(state.provider)) { + logMissingNvidiaApiKeyHelp(REMOTE_PROVIDER_CONFIG.build.helpUrl); + process.exit(1); + } + } else { + await ensureApiKey(); + } + const _envModel = (process.env.NEMOCLAW_MODEL || "").trim(); + state.model = + requestedModel || + (recoveredFromSandbox && recoveredModel) || + (isNonInteractive() + ? DEFAULT_CLOUD_MODEL + : await promptCloudModel({ defaultModelId: _envModel || undefined })) || + DEFAULT_CLOUD_MODEL; + if (isBackToSelection(state.model)) { + console.log(" Returning to provider selection."); + console.log(""); + return "retry-selection"; + } + } else { + const _providerKeyHint = (process.env.NEMOCLAW_PROVIDER_KEY || "").trim(); + if (_providerKeyHint && state.credentialEnv) { + const existingCredentialKey = normalizeCredentialValue( + process.env[state.credentialEnv] ?? "", + ); + if (!existingCredentialKey) { + process.env[state.credentialEnv] = _providerKeyHint; + } + } + + const _envModelRemote = (process.env.NEMOCLAW_MODEL || "").trim(); + const defaultModel = + requestedModel || + _envModelRemote || + (recoveredFromSandbox && recoveredModel) || + remoteConfig.defaultModel; + const selectedCredentialEnv = requireValue( + state.credentialEnv, + `Missing credential env for ${remoteConfig.label}`, + ); + const bedrockSelection = await bedrockRuntimeOnboard.selectBedrockRuntimeCustomAnthropic({ + selectedKey: selected.key, + endpointUrl: state.endpointUrl, + credentialEnv: selectedCredentialEnv, + label: remoteConfig.label, + helpUrl: remoteConfig.helpUrl, + defaultModel, + backToSelection: BACK_TO_SELECTION, + isNonInteractive, + promptInputModel, + replaceNamedCredential, + }); + if (bedrockSelection.action === "retry-selection") { + console.log(" Returning to provider selection."); + console.log(""); + return "retry-selection"; + } + if (bedrockSelection.action === "selected") { + state.model = bedrockSelection.model; + state.preferredInferenceApi = bedrockSelection.preferredInferenceApi; + return "selected"; + } + if (isNonInteractive()) { + if ( + !resolveProviderCredential(selectedCredentialEnv) && + !providerExistsInGateway(state.provider) + ) { + console.error( + ` ${selectedCredentialEnv} (or NEMOCLAW_PROVIDER_KEY) is required for ${remoteConfig.label} in non-interactive mode.`, + ); + process.exit(1); + } + } else { + const credentialResult = await ensureNamedCredential( + selectedCredentialEnv, + `${remoteConfig.label} API key`, + remoteConfig.helpUrl, + ); + if (credentialPrompt.returningToProviderSelection(credentialResult)) { + return "retry-selection"; + } + } + let modelValidator: ((candidate: string) => ModelValidationResult) | null = null; + if (selected.key === "openai" || selected.key === "gemini") { + const modelAuthMode = getProbeAuthMode(state.provider); + modelValidator = (candidate) => + validateOpenAiLikeModel( + remoteConfig.label, + state.endpointUrl || remoteConfig.endpointUrl, + candidate, + getCredential(selectedCredentialEnv) || "", + ...(modelAuthMode ? [{ authMode: modelAuthMode }] : []), + ); + } else if (selected.key === "anthropic") { + modelValidator = (candidate) => + validateAnthropicModel( + state.endpointUrl || ANTHROPIC_ENDPOINT_URL, + candidate, + getCredential(selectedCredentialEnv) || "", + ); + } + while (true) { + if (isNonInteractive()) { + state.model = defaultModel; + } else if (remoteConfig.modelMode === "curated") { + state.model = await promptRemoteModel( + remoteConfig.label, + selected.key, + defaultModel, + modelValidator, + ); + } else { + state.model = await promptInputModel(remoteConfig.label, defaultModel, modelValidator); + } + if (isBackToSelection(state.model)) { + console.log(" Returning to provider selection."); + console.log(""); + return "retry-selection"; + } + + const validationResult = await validateSelectedRemoteModel( + selected, + remoteConfig, + state, + selectedCredentialEnv, + ); + if (validationResult === "selected") break; + if (validationResult === "retry-selection") return "retry-selection"; + } + } + + if (selected.key === "build") { + while (true) { + const validation = await validateOpenAiLikeSelection( + remoteConfig.label, + requireValue(state.endpointUrl, `Missing endpoint URL for ${remoteConfig.label}`), + state.model, + state.credentialEnv, + "Please choose a provider/model again.", + remoteConfig.helpUrl, + { + requireResponsesToolCalling: shouldRequireResponsesToolCalling(state.provider), + skipResponsesProbe: shouldSkipResponsesProbe(state.provider), + authMode: getProbeAuthMode(state.provider), + }, + ); + if (validation.ok) { + state.preferredInferenceApi = validation.api; + break; + } + if (validation.retry === "credential" || validation.retry === "retry") { + continue; + } + return "retry-selection"; + } + } + + console.log(` Using ${remoteConfig.label} with model: ${state.model}`); + return "selected"; +} + async function setupNim( gpu: ReturnType, sandboxName: string | null = null, @@ -3495,575 +4192,58 @@ async function setupNim( } if (REMOTE_PROVIDER_CONFIG[selected.key]) { - const remoteConfig = REMOTE_PROVIDER_CONFIG[selected.key]; - provider = remoteConfig.providerName; - credentialEnv = remoteConfig.credentialEnv; - endpointUrl = remoteConfig.endpointUrl; - preferredInferenceApi = null; - - if (selected.key === "custom") { - const _envUrl = (process.env.NEMOCLAW_ENDPOINT_URL || "").trim(); - const endpointInput = isNonInteractive() - ? _envUrl - : (await prompt( - _envUrl - ? ` OpenAI-compatible base URL [${_envUrl}]: ` - : " OpenAI-compatible base URL (e.g., https://openrouter.ai): ", - )) || _envUrl; - const navigation = getNavigationChoice(endpointInput); - if (navigation === "back") { - console.log(" Returning to provider selection."); - console.log(""); - continue selectionLoop; - } - if (navigation === "exit") { - exitOnboardFromPrompt(); - } - endpointUrl = normalizeProviderBaseUrl(endpointInput, "openai"); - if (!endpointUrl) { - console.error(" Endpoint URL is required for Other OpenAI-compatible endpoint."); - if (isNonInteractive()) { - process.exit(1); - } - console.log(""); - continue selectionLoop; - } - } else if (selected.key === "anthropicCompatible") { - const _envUrl = (process.env.NEMOCLAW_ENDPOINT_URL || "").trim(); - const endpointInput = isNonInteractive() - ? _envUrl - : (await prompt( - _envUrl - ? ` Anthropic-compatible base URL [${_envUrl}]: ` - : " Anthropic-compatible base URL (e.g., https://proxy.example.com): ", - )) || _envUrl; - const navigation = getNavigationChoice(endpointInput); - if (navigation === "back") { - console.log(" Returning to provider selection."); - console.log(""); - continue selectionLoop; - } - if (navigation === "exit") { - exitOnboardFromPrompt(); - } - endpointUrl = normalizeProviderBaseUrl(endpointInput, "anthropic"); - if (!endpointUrl) { - console.error(" Endpoint URL is required for Other Anthropic-compatible endpoint."); - if (isNonInteractive()) { - process.exit(1); - } - console.log(""); - continue selectionLoop; - } - endpointUrl = bedrockRuntimeOnboard.normalizeCustomAnthropicEndpointUrl(endpointUrl); - } - - if (selected.key === "hermesProvider") { - const selectedHermesAuthMethod = await promptHermesAuthMethod(); - if (isBackToSelection(selectedHermesAuthMethod)) { - hermesAuthMethod = null; - console.log(" Returning to provider selection."); - console.log(""); - continue selectionLoop; - } - hermesAuthMethod = normalizeHermesAuthMethod( - selectedHermesAuthMethod as string | null | undefined, - ); - if (hermesAuthMethod === HERMES_AUTH_METHOD_API_KEY) { - credentialEnv = HERMES_NOUS_API_KEY_CREDENTIAL_ENV; - stageNousApiKeyProviderEnv(); - if (isNonInteractive()) { - if (!resolveHermesNousApiKey()) { - console.error( - " Hermes Provider Nous API Key is required in non-interactive mode.", - ); - process.exit(1); - } - } else { - const hermesKeyResult = await ensureHermesNousApiKeyEnv(); - if (credentialPrompt.returningToProviderSelection(hermesKeyResult)) - continue selectionLoop; - } - } else { - credentialEnv = remoteConfig.credentialEnv; - } - const recordedHermesToolGateways = sandboxName - ? normalizeHermesToolGatewaySelections( - registry.getSandbox(sandboxName)?.hermesToolGateways, - ) - : null; - hermesToolGateways = await setupHermesToolGateways( - provider, - hermesAuthMethod, - recordedHermesToolGateways, - { prompt, note, isNonInteractive }, - ); - - const defaultModel = - requestedModel || (recoveredFromSandbox && recoveredModel) || remoteConfig.defaultModel; - if (isNonInteractive()) { - model = defaultModel; - } else { - let hermesProviderModels: string[] = []; - try { - hermesProviderModels = await nousModels.getHermesProviderModelOptions(); - } catch (err) { - const detail = err instanceof Error ? err.message : String(err); - console.warn( - ` Warning: failed to load Nous model recommendations; falling back to the current/default model (${detail}).`, - ); - } - model = await promptRemoteModel(remoteConfig.label, selected.key, defaultModel, null, { - otherShowsFullList: true, - remoteModelOptions: { [selected.key]: hermesProviderModels }, - topLevelModelLimit: 10, - }); - } - if (isBackToSelection(model)) { - console.log(" Returning to provider selection."); - console.log(""); - continue selectionLoop; - } - preferredInferenceApi = "openai-completions"; - console.log(` Using ${remoteConfig.label} with model: ${model}`); - break; - } - - // Hydrate from credential env vars set earlier in this process - // before checking env, so rebuild and other non-interactive callers - // can resolve keys stored during the original interactive onboard. - // See #2273. - hydrateCredentialEnv(credentialEnv); - - if (selected.key === "build") { - // Let NEMOCLAW_PROVIDER_KEY fill the NVIDIA key without overriding explicit env. - const _nvProviderKey = (process.env.NEMOCLAW_PROVIDER_KEY || "").trim(); - const existingNvidiaKey = ["NVIDIA_INFERENCE_API_KEY", "NVIDIA_API_KEY"] - .map((envName) => normalizeCredentialValue(process.env[envName] ?? "")) - .find(Boolean); - if (_nvProviderKey && !existingNvidiaKey) { - process.env.NVIDIA_INFERENCE_API_KEY = _nvProviderKey; - } - if (isNonInteractive()) { - const resolvedNvidiaKey = resolveProviderCredential("NVIDIA_INFERENCE_API_KEY"); - if (resolvedNvidiaKey) { - const keyError = validateNvidiaApiKeyValue(resolvedNvidiaKey); - if (keyError) { - console.error(keyError); - console.error(` Get a key from ${REMOTE_PROVIDER_CONFIG.build.helpUrl}`); - process.exit(1); - } - } else if (!providerExistsInGateway(provider)) { - logMissingNvidiaApiKeyHelp(REMOTE_PROVIDER_CONFIG.build.helpUrl); - process.exit(1); - } - } else { - await ensureApiKey(); - } - const _envModel = (process.env.NEMOCLAW_MODEL || "").trim(); - model = - requestedModel || - (recoveredFromSandbox && recoveredModel) || - (isNonInteractive() - ? DEFAULT_CLOUD_MODEL - : await promptCloudModel({ defaultModelId: _envModel || undefined })) || - DEFAULT_CLOUD_MODEL; - if (isBackToSelection(model)) { - console.log(" Returning to provider selection."); - console.log(""); - continue selectionLoop; - } - } else { - // NEMOCLAW_PROVIDER_KEY is a universal alias: if the specific credential env - // isn't already set, use NEMOCLAW_PROVIDER_KEY as the API key for this provider. - // Check raw process.env — the override must apply before resolving from credentials.json. - const _providerKeyHint = (process.env.NEMOCLAW_PROVIDER_KEY || "").trim(); - if (_providerKeyHint && credentialEnv) { - const existingCredentialKey = normalizeCredentialValue( - // check-direct-credential-env-ignore -- intentional: checking if env is already set before applying NEMOCLAW_PROVIDER_KEY override - process.env[credentialEnv] ?? "", - ); - if (!existingCredentialKey) { - process.env[credentialEnv] = _providerKeyHint; - } - } - - const _envModelRemote = (process.env.NEMOCLAW_MODEL || "").trim(); - const defaultModel = - requestedModel || - _envModelRemote || - (recoveredFromSandbox && recoveredModel) || - remoteConfig.defaultModel; - const selectedCredentialEnv = requireValue( - credentialEnv, - `Missing credential env for ${remoteConfig.label}`, - ); - const bedrockSelection = await bedrockRuntimeOnboard.selectBedrockRuntimeCustomAnthropic({ - selectedKey: selected.key, - endpointUrl, - credentialEnv: selectedCredentialEnv, - label: remoteConfig.label, - helpUrl: remoteConfig.helpUrl, - defaultModel, - backToSelection: BACK_TO_SELECTION, - isNonInteractive, - promptInputModel, - replaceNamedCredential, - }); - if (bedrockSelection.action === "retry-selection") { - console.log(" Returning to provider selection."); - console.log(""); - continue selectionLoop; - } - if (bedrockSelection.action === "selected") { - model = bedrockSelection.model; - preferredInferenceApi = bedrockSelection.preferredInferenceApi; - break; - } - if (isNonInteractive()) { - if ( - !resolveProviderCredential(selectedCredentialEnv) && - !providerExistsInGateway(provider) - ) { - console.error( - ` ${selectedCredentialEnv} (or NEMOCLAW_PROVIDER_KEY) is required for ${remoteConfig.label} in non-interactive mode.`, - ); - process.exit(1); - } - } else { - const credentialResult = await ensureNamedCredential( - selectedCredentialEnv, - remoteConfig.label + " API key", - remoteConfig.helpUrl, - ); - if (credentialPrompt.returningToProviderSelection(credentialResult)) - continue selectionLoop; - } - let modelValidator: ((candidate: string) => ModelValidationResult) | null = null; - if (selected.key === "openai" || selected.key === "gemini") { - const modelAuthMode = getProbeAuthMode(provider); - modelValidator = (candidate) => - validateOpenAiLikeModel( - remoteConfig.label, - endpointUrl || remoteConfig.endpointUrl, - candidate, - getCredential(selectedCredentialEnv) || "", - ...(modelAuthMode ? [{ authMode: modelAuthMode }] : []), - ); - } else if (selected.key === "anthropic") { - modelValidator = (candidate) => - validateAnthropicModel( - endpointUrl || ANTHROPIC_ENDPOINT_URL, - candidate, - getCredential(selectedCredentialEnv) || "", - ); - } - while (true) { - if (isNonInteractive()) { - model = defaultModel; - } else if (remoteConfig.modelMode === "curated") { - model = await promptRemoteModel( - remoteConfig.label, - selected.key, - defaultModel, - modelValidator, - ); - } else { - model = await promptInputModel(remoteConfig.label, defaultModel, modelValidator); - } - if (isBackToSelection(model)) { - console.log(" Returning to provider selection."); - console.log(""); - continue selectionLoop; - } - - if (selected.key === "custom") { - const validation = await validateCustomOpenAiLikeSelection( - remoteConfig.label, - endpointUrl || OPENAI_ENDPOINT_URL, - model, - selectedCredentialEnv, - remoteConfig.helpUrl, - ); - if (validation.ok) { - // Force chat completions for all OpenAI-compatible endpoints - // unless the user explicitly opted in to responses via env var. - // Many backends (Ollama, vLLM, LiteLLM) expose /v1/responses - // but do not correctly handle the `developer` role used by the - // Responses API — messages with that role are silently dropped, - // causing the model to receive no system prompt or tool - // definitions. Chat completions uses the `system` role which - // is universally supported. - // See: https://github.com/NVIDIA/NemoClaw/issues/1932 - const explicitApi = (process.env.NEMOCLAW_PREFERRED_API || "").trim().toLowerCase(); - if ( - explicitApi && - explicitApi !== "openai-completions" && - explicitApi !== "chat-completions" - ) { - preferredInferenceApi = validation.api; - } else { - if (validation.api !== "openai-completions") { - console.log( - " ℹ Using chat completions API (compatible endpoints may not support the Responses API developer role)", - ); - } - preferredInferenceApi = "openai-completions"; - } - break; - } - if ( - validation.retry === "credential" || - validation.retry === "retry" || - validation.retry === "model" - ) { - continue; - } - if (validation.retry === "selection") { - continue selectionLoop; - } - } else if (selected.key === "anthropicCompatible") { - const validation = await validateCustomAnthropicSelection( - remoteConfig.label, - endpointUrl || ANTHROPIC_ENDPOINT_URL, - model, - selectedCredentialEnv, - remoteConfig.helpUrl, - ); - if (validation.ok) { - preferredInferenceApi = validation.api; - break; - } - if ( - validation.retry === "credential" || - validation.retry === "retry" || - validation.retry === "model" - ) { - continue; - } - if (validation.retry === "selection") { - continue selectionLoop; - } - } else { - const retryMessage = "Please choose a provider/model again."; - if (selected.key === "anthropic") { - const validation = await validateAnthropicSelectionWithRetryMessage( - remoteConfig.label, - endpointUrl || ANTHROPIC_ENDPOINT_URL, - model, - selectedCredentialEnv, - retryMessage, - remoteConfig.helpUrl, - ); - if (validation.ok) { - preferredInferenceApi = validation.api; - break; - } - if ( - validation.retry === "credential" || - validation.retry === "retry" || - validation.retry === "model" - ) { - continue; - } - } else { - const validation = await validateOpenAiLikeSelection( - remoteConfig.label, - requireValue(endpointUrl, `Missing endpoint URL for ${remoteConfig.label}`), - model, - selectedCredentialEnv, - retryMessage, - remoteConfig.helpUrl, - { - requireResponsesToolCalling: shouldRequireResponsesToolCalling(provider), - skipResponsesProbe: shouldSkipResponsesProbe(provider), - authMode: getProbeAuthMode(provider), - }, - ); - if (validation.ok) { - preferredInferenceApi = validation.api; - break; - } - if ( - validation.retry === "credential" || - validation.retry === "retry" || - validation.retry === "model" - ) { - continue; - } - } - continue selectionLoop; - } - } - } - - if (selected.key === "build") { - while (true) { - const validation = await validateOpenAiLikeSelection( - remoteConfig.label, - requireValue(endpointUrl, `Missing endpoint URL for ${remoteConfig.label}`), - model, - credentialEnv, - "Please choose a provider/model again.", - remoteConfig.helpUrl, - { - requireResponsesToolCalling: shouldRequireResponsesToolCalling(provider), - skipResponsesProbe: shouldSkipResponsesProbe(provider), - authMode: getProbeAuthMode(provider), - }, - ); - if (validation.ok) { - preferredInferenceApi = validation.api; - break; - } - if (validation.retry === "credential" || validation.retry === "retry") { - continue; - } - continue selectionLoop; - } - } - - console.log(` Using ${remoteConfig.label} with model: ${model}`); + const state: SetupNimSelectionState = { + model, + provider, + endpointUrl, + credentialEnv, + hermesAuthMethod, + hermesToolGateways, + preferredInferenceApi, + nimContainer, + }; + const result = await handleRemoteProviderSelection( + { selected, requestedModel, recoveredFromSandbox, recoveredModel, sandboxName }, + state, + ); + ({ + model, + provider, + endpointUrl, + credentialEnv, + hermesAuthMethod, + hermesToolGateways, + preferredInferenceApi, + } = state); + if (result === "retry-selection") continue selectionLoop; break; } else if (selected.key === "nim-local") { - const localGpu = requireValue( + const state: SetupNimSelectionState = { + model, + provider, + endpointUrl, + credentialEnv, + hermesAuthMethod, + hermesToolGateways, + preferredInferenceApi, + nimContainer, + }; + const result = await handleNimLocalSelection( gpu, - "GPU details are required for local NIM model selection", + { requestedModel, recoveredFromSandbox, recoveredModel }, + state, ); - // List models that fit GPU VRAM - const models = nim.listModels().filter((m) => m.minGpuMemoryMB <= localGpu.totalMemoryMB); - if (models.length === 0) { - console.log(" No NIM models fit your GPU VRAM. Falling back to cloud API."); - } else { - let sel; - if (isNonInteractive()) { - const targetModel = requestedModel || (recoveredFromSandbox ? recoveredModel : null); - if (targetModel) { - sel = models.find((m) => m.name === targetModel); - if (!sel) { - const label = requestedModel ? "NEMOCLAW_MODEL for NIM" : "Recorded NIM model"; - console.error(` Unsupported ${label}: ${targetModel}`); - process.exit(1); - } - } else { - sel = models[0]; - } - note(` [non-interactive] NIM model: ${sel.name}`); - } else { - console.log(""); - console.log(" Models that fit your GPU:"); - models.forEach((m, i) => { - console.log(` ${i + 1}) ${m.name} (min ${m.minGpuMemoryMB} MB)`); - }); - console.log(""); - - const modelChoice = await prompt(` Choose model [1]: `); - sel = selectFromNumberedMenuOrExit(modelChoice, 1, models); - } - model = sel.name; - - // Ensure Docker is logged in to NGC registry before pulling NIM images. - // The key is also forwarded into the NIM container at runtime (#3333), - // so we hoist it out of the not-logged-in branch. - let ngcApiKey: string | null = null; - if (!nim.isNgcLoggedIn()) { - if (isNonInteractive()) { - console.error( - " Docker is not logged in to nvcr.io. In non-interactive mode, run `docker login nvcr.io` first and retry.", - ); - process.exit(1); - } - console.log(""); - console.log(" NGC API Key required to pull NIM images."); - console.log(" Get one from: https://org.ngc.nvidia.com/setup/api-key"); - console.log(""); - let ngcKey = await credentialPrompt.readValue(" NGC API Key: "); - if (credentialPrompt.returningToProviderSelection(ngcKey)) continue selectionLoop; - if (!ngcKey) { - console.error(" NGC API Key is required for Local NIM."); - process.exit(1); - } - if (!nim.dockerLoginNgc(ngcKey)) { - console.error(" Failed to login to NGC registry. Check your API key and try again."); - console.log(""); - ngcKey = await credentialPrompt.readValue(" NGC API Key: "); - if (credentialPrompt.returningToProviderSelection(ngcKey)) continue selectionLoop; - if (!ngcKey || !nim.dockerLoginNgc(ngcKey)) { - console.error(" NGC login failed. Cannot pull NIM images."); - process.exit(1); - } - } - ngcApiKey = ngcKey; - } else { - // Docker is already logged in, but NIM still needs the key in its - // container env to download model manifests. Users hit by the - // original #3333 bug typically have a cached docker login from - // the earlier broken attempt while the NGC key was never saved - // anywhere, so a passive lookup would silently reproduce the - // failure. Try env first, then prompt interactively; an empty - // answer falls through to startNimContainerByName's warning so - // we don't double-fail in non-interactive callers. - ngcApiKey = - hydrateCredentialEnv("NGC_API_KEY") || - hydrateCredentialEnv("NVIDIA_INFERENCE_API_KEY"); - if (!ngcApiKey && !isNonInteractive()) { - console.log(""); - console.log(" NGC API Key required to download NIM model weights at runtime."); - console.log(" (Docker is logged in to nvcr.io, but the key was not saved.)"); - const ngcKey = await credentialPrompt.readValue(" NGC API Key: "); - if (credentialPrompt.returningToProviderSelection(ngcKey)) continue selectionLoop; - ngcApiKey = ngcKey || null; - } - } - - console.log(` Pulling NIM image for ${model}...`); - nim.pullNimImage(model); - - console.log(" Starting NIM container..."); - const nimContainerNameLocal = nim.containerName(GATEWAY_NAME); - nimContainer = nim.startNimContainerByName(nimContainerNameLocal, model, undefined, { - ngcApiKey: ngcApiKey ?? undefined, - }); - - console.log(" Waiting for NIM to become healthy..."); - if (!nim.waitForNimHealth(undefined, undefined, { container: nimContainerNameLocal })) { - console.error(" NIM failed to start. Falling back to cloud API."); - model = null; - nimContainer = null; - } else { - provider = "vllm-local"; - // Local NIM (vLLM under the hood) does not require a host API key — - // setupInference registers the gateway provider with an internal - // credential env (NEMOCLAW_VLLM_LOCAL_TOKEN). See GH #2519. - credentialEnv = null; - endpointUrl = getLocalProviderBaseUrl(provider); - if (!endpointUrl) { - console.error(" Local NVIDIA NIM base URL could not be determined."); - process.exit(1); - } - model = nim.adoptServedModelId(model); - const nimValidationUrl = getLocalProviderValidationBaseUrl(provider) || endpointUrl; - const validation = await validateOpenAiLikeSelection( - "Local NVIDIA NIM", - nimValidationUrl, - requireValue(model, "Expected a Local NVIDIA NIM model after startup"), - null, - ); - if (validation.retry === "selection" || validation.retry === "model") { - continue selectionLoop; - } - if (!validation.ok) { - continue selectionLoop; - } - // NIM (vLLM) mishandles the /v1/responses developer role; force chat completions. - if (validation.api !== "openai-completions") { - console.log( - " ℹ Using chat completions API (tool-call-parser requires /v1/chat/completions)", - ); - } - preferredInferenceApi = "openai-completions"; - } - } + ({ + model, + provider, + endpointUrl, + credentialEnv, + hermesAuthMethod, + hermesToolGateways, + preferredInferenceApi, + nimContainer, + } = state); + if (result === "retry-selection") continue selectionLoop; break; } else if (selected.key === "ollama") { if (rejectWindowsHostOllama(selected.key, isWindowsHostOllama)) { @@ -4272,123 +4452,36 @@ async function setupNim( // intentional fall-through to the next branch } if (selected.key === "vllm") { - console.log(` ✓ Using existing vLLM on localhost:${VLLM_PORT}`); - provider = "vllm-local"; - // See NIM branch above — internal credential env, no user API key. - credentialEnv = null; - endpointUrl = getLocalProviderBaseUrl(provider); - if (!endpointUrl) { - console.error(" Local vLLM base URL could not be determined."); - process.exit(1); - } - // Query vLLM for the actual model ID - const vllmModelsRaw = runCapture( - ["curl", "-sf", `http://127.0.0.1:${VLLM_PORT}/v1/models`], - { - ignoreError: true, - }, - ); - let vllmModels: { data?: Array<{ id?: unknown }> } = {}; - try { - vllmModels = JSON.parse(vllmModelsRaw); - if (vllmModels.data && vllmModels.data.length > 0) { - const detectedModel = - typeof vllmModels.data[0]?.id === "string" ? vllmModels.data[0].id : null; - model = detectedModel; - if (!detectedModel || !isSafeModelId(detectedModel)) { - console.error(` Detected model ID contains invalid characters: ${model}`); - process.exit(1); - } - console.log(` Detected model: ${model}`); - } else { - console.error(" Could not detect model from vLLM. Please specify manually."); - process.exit(1); - } - } catch { - console.error( - ` Could not query vLLM models endpoint. Is vLLM running on localhost:${VLLM_PORT}?`, - ); - process.exit(1); - } - const validationBaseUrl = getLocalProviderValidationBaseUrl(provider); - if (!validationBaseUrl) { - console.error(" Local vLLM validation URL could not be determined."); - process.exit(1); - } - const validation = await validateOpenAiLikeSelection( - "Local vLLM", - validationBaseUrl, - requireValue(model as string | null | undefined, "Expected a detected vLLM model"), - null, - ); - if (validation.retry === "selection" || validation.retry === "model") { - continue selectionLoop; - } - if (!validation.ok) continue selectionLoop; - localInference.applyVllmRuntimeContextWindow(vllmModels, model as string); - preferredInferenceApi = validation.api; - // Force chat completions — vLLM's /v1/responses endpoint does not - // run the --tool-call-parser, so tool calls arrive as raw text (#976). - if (preferredInferenceApi !== "openai-completions") { - console.log( - " ℹ Using chat completions API (tool-call-parser requires /v1/chat/completions)", - ); - } - preferredInferenceApi = "openai-completions"; + const state: SetupNimSelectionState = { + model, + provider, + endpointUrl, + credentialEnv, + hermesAuthMethod, + hermesToolGateways, + preferredInferenceApi, + nimContainer, + }; + const result = await handleVllmSelection(state); + ({ model, provider, endpointUrl, credentialEnv, preferredInferenceApi, nimContainer } = + state); + if (result === "retry-selection") continue selectionLoop; break; } else if (selected.key === "routed") { - const bp = loadBlueprintProfile("routed"); - if (!bp || bp.router?.enabled !== true) { - console.error(" Router is not enabled in nemoclaw-blueprint/blueprint.yaml."); - if (isNonInteractive()) process.exit(1); - continue selectionLoop; - } - const routerCredentialEnv = - bp.router?.credential_env || bp.credential_env || DEFAULT_MODEL_ROUTER_CREDENTIAL_ENV; - credentialEnv = routerCredentialEnv; - const routedCredential = - hydrateCredentialEnv(routerCredentialEnv) || - normalizeCredentialValue(bp.credential_default || ""); - if (routedCredential) { - saveCredential(routerCredentialEnv, routedCredential); - } - const _providerKeyHint = (process.env.NEMOCLAW_PROVIDER_KEY || "").trim(); - if (_providerKeyHint && !resolveProviderCredential(routerCredentialEnv)) { - saveCredential(routerCredentialEnv, _providerKeyHint); - } - if (isNonInteractive()) { - if (!resolveProviderCredential(routerCredentialEnv)) { - console.error( - ` ${routerCredentialEnv} (or NEMOCLAW_PROVIDER_KEY) is required for Model Router in non-interactive mode.`, - ); - process.exit(1); - } - } else { - if (!resolveProviderCredential(routerCredentialEnv)) { - console.log(""); - console.log(" Model Router accepts NVIDIA API keys (nvapi-...)."); - console.log(" Get one at https://build.nvidia.com"); - console.log(""); - const routerCredentialResult = await ensureNamedCredential( - routerCredentialEnv, - "Model Router API key", - null, - ); - if (credentialPrompt.returningToProviderSelection(routerCredentialResult)) - continue selectionLoop; - } - } - provider = bp.provider_name || "nvidia-router"; - model = bp.model; - const { HOST_GATEWAY_URL } = require("./inference/local"); - const routerEndpointUrl = bp.endpoint || ""; - endpointUrl = routerEndpointUrl; - if (routerEndpointUrl.match(/localhost|127\.0\.0\.1/)) { - const u = new URL(routerEndpointUrl); - endpointUrl = `${HOST_GATEWAY_URL}:${u.port}${u.pathname}`; - } - preferredInferenceApi = "openai-completions"; - console.log(` ✓ Using Model Router: ${provider} / ${model}`); + const state: SetupNimSelectionState = { + model, + provider, + endpointUrl, + credentialEnv, + hermesAuthMethod, + hermesToolGateways, + preferredInferenceApi, + nimContainer, + }; + const result = await handleRoutedSelection(state); + ({ model, provider, endpointUrl, credentialEnv, preferredInferenceApi, nimContainer } = + state); + if (result === "retry-selection") continue selectionLoop; break; } } From 65d757893e4597207336fc0fb87657e0aca0f4be Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sun, 14 Jun 2026 11:38:05 -0700 Subject: [PATCH 02/23] refactor(rebuild): lower cognitive complexity ratchet to 244 Signed-off-by: Carlos Villela --- biome.json | 2 +- src/lib/actions/sandbox/rebuild.ts | 25 +++++++++++++------------ 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/biome.json b/biome.json index 99455513804..2be5a2ab7bc 100644 --- a/biome.json +++ b/biome.json @@ -86,7 +86,7 @@ "noExcessiveCognitiveComplexity": { "level": "error", "options": { - "maxAllowedComplexity": 245 + "maxAllowedComplexity": 244 } } }, diff --git a/src/lib/actions/sandbox/rebuild.ts b/src/lib/actions/sandbox/rebuild.ts index 27ef48a8ceb..f7de0e975e4 100644 --- a/src/lib/actions/sandbox/rebuild.ts +++ b/src/lib/actions/sandbox/rebuild.ts @@ -291,6 +291,18 @@ function hookOutputsFromBuildSteps( return { outputs }; } +function countActiveSandboxSessionsForRebuild(sandboxName: string): number { + const opsBinRebuild = resolveOpenshell(); + if (!opsBinRebuild) return 0; + + try { + const sessionResult = getActiveSandboxSessions(sandboxName, createSessionDeps(opsBinRebuild)); + return sessionResult.detected ? sessionResult.sessions.length : 0; + } catch { + return 0; + } +} + async function reapplyMessagingManifestAfterOpenClawDoctor( sandboxName: string, plan: SandboxMessagingPlan | null, @@ -345,18 +357,7 @@ export async function rebuildSandbox( : (_msg: string, code = 1) => process.exit(code); // Active session detection — enrich the confirmation prompt if sessions are active - let rebuildActiveSessionCount = 0; - const opsBinRebuild = resolveOpenshell(); - if (opsBinRebuild) { - try { - const sessionResult = getActiveSandboxSessions(sandboxName, createSessionDeps(opsBinRebuild)); - if (sessionResult.detected) { - rebuildActiveSessionCount = sessionResult.sessions.length; - } - } catch { - /* non-fatal */ - } - } + const rebuildActiveSessionCount = countActiveSandboxSessionsForRebuild(sandboxName); const sb = registry.getSandbox(sandboxName) as any; if (!sb) { From 433ba8a15dff02640b12c9730e355f09b989c801 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sun, 14 Jun 2026 11:42:38 -0700 Subject: [PATCH 03/23] refactor(rebuild): lower cognitive complexity ratchet to 243 Signed-off-by: Carlos Villela --- biome.json | 2 +- src/lib/actions/sandbox/rebuild.ts | 58 +++++++++++++++++------------- 2 files changed, 35 insertions(+), 25 deletions(-) diff --git a/biome.json b/biome.json index 2be5a2ab7bc..0339890e330 100644 --- a/biome.json +++ b/biome.json @@ -86,7 +86,7 @@ "noExcessiveCognitiveComplexity": { "level": "error", "options": { - "maxAllowedComplexity": 244 + "maxAllowedComplexity": 243 } } }, diff --git a/src/lib/actions/sandbox/rebuild.ts b/src/lib/actions/sandbox/rebuild.ts index f7de0e975e4..1610e494c0b 100644 --- a/src/lib/actions/sandbox/rebuild.ts +++ b/src/lib/actions/sandbox/rebuild.ts @@ -303,6 +303,35 @@ function countActiveSandboxSessionsForRebuild(sandboxName: string): number { } } +async function confirmSandboxRebuildIfNeeded( + skipConfirm: boolean, + rebuildActiveSessionCount: number, +): Promise { + if (skipConfirm) return true; + + if (rebuildActiveSessionCount > 0) { + const plural = rebuildActiveSessionCount > 1 ? "sessions" : "session"; + console.log( + ` ${YW}⚠ Active SSH ${plural} detected (${rebuildActiveSessionCount} connection${rebuildActiveSessionCount > 1 ? "s" : ""})${R}`, + ); + console.log( + ` Rebuilding will terminate ${rebuildActiveSessionCount === 1 ? "the" : "all"} active ${plural} with a Broken pipe error.`, + ); + console.log(""); + } + console.log(" This will:"); + console.log(" 1. Back up workspace state"); + console.log(" 2. Destroy and recreate the sandbox with the current image"); + console.log(" 3. Restore workspace state into the new sandbox"); + console.log(""); + const answer = await askPrompt(" Proceed? [y/N]: "); + if (answer.trim().toLowerCase() !== "y" && answer.trim().toLowerCase() !== "yes") { + console.log(" Cancelled."); + return false; + } + return true; +} + async function reapplyMessagingManifestAfterOpenClawDoctor( sandboxName: string, plan: SandboxMessagingPlan | null, @@ -428,30 +457,11 @@ export async function rebuildSandbox( } console.log(""); - let rebuildConfirmed = false; - if (!skipConfirm) { - if (rebuildActiveSessionCount > 0) { - const plural = rebuildActiveSessionCount > 1 ? "sessions" : "session"; - console.log( - ` ${YW}⚠ Active SSH ${plural} detected (${rebuildActiveSessionCount} connection${rebuildActiveSessionCount > 1 ? "s" : ""})${R}`, - ); - console.log( - ` Rebuilding will terminate ${rebuildActiveSessionCount === 1 ? "the" : "all"} active ${plural} with a Broken pipe error.`, - ); - console.log(""); - } - console.log(" This will:"); - console.log(" 1. Back up workspace state"); - console.log(" 2. Destroy and recreate the sandbox with the current image"); - console.log(" 3. Restore workspace state into the new sandbox"); - console.log(""); - const answer = await askPrompt(" Proceed? [y/N]: "); - if (answer.trim().toLowerCase() !== "y" && answer.trim().toLowerCase() !== "yes") { - console.log(" Cancelled."); - return; - } - rebuildConfirmed = true; - } + const rebuildConfirmed = await confirmSandboxRebuildIfNeeded( + skipConfirm, + rebuildActiveSessionCount, + ); + if (!rebuildConfirmed) return; // Step 0: Preflight — verify recreate preconditions BEFORE destroying // anything. The most common rebuild failure is a missing provider From 30e2b108928aa05a0d04fadefdf0d17b50a2266c Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sun, 14 Jun 2026 16:45:13 -0700 Subject: [PATCH 04/23] refactor(onboard): move setupNim validation helpers Signed-off-by: Carlos Villela --- src/lib/onboard.ts | 145 +++-------------- src/lib/onboard/setup-nim-selection.ts | 207 +++++++++++++++++++++++++ 2 files changed, 226 insertions(+), 126 deletions(-) create mode 100644 src/lib/onboard/setup-nim-selection.ts diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index f77f55ecf01..51d17650da2 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -21,6 +21,9 @@ const { const { createInferenceSelectionValidationHelpers, }: typeof import("./onboard/inference-selection-validation") = require("./onboard/inference-selection-validation"); +const { + createRemoteModelValidator, +}: typeof import("./onboard/setup-nim-selection") = require("./onboard/setup-nim-selection"); const inferenceInputCapability = require("./onboard/inference-input-capability"); const { cleanupTempDir }: typeof import("./onboard/temp-files") = require("./onboard/temp-files"); const { @@ -1033,6 +1036,19 @@ const { agentProductName, promptValidationRecovery, }); +const { validateSelectedRemoteModel } = createRemoteModelValidator({ + OPENAI_ENDPOINT_URL, + ANTHROPIC_ENDPOINT_URL, + requireValue, + isBackToSelection, + validateCustomOpenAiLikeSelection, + validateCustomAnthropicSelection, + validateAnthropicSelectionWithRetryMessage, + validateOpenAiLikeSelection, + shouldRequireResponsesToolCalling, + shouldSkipResponsesProbe, + getProbeAuthMode, +}); const { promptCloudModel, promptRemoteModel, promptInputModel } = modelPrompts; const { validateAnthropicModel, validateOpenAiLikeModel } = providerModels; @@ -3364,129 +3380,6 @@ type RemoteProviderSelectionArgs = { sandboxName: string | null; }; -type RemoteProviderConfig = (typeof REMOTE_PROVIDER_CONFIG)[keyof typeof REMOTE_PROVIDER_CONFIG]; - -type RemoteModelValidationResult = "selected" | "retry-model" | "retry-selection"; - -async function validateSelectedRemoteModel( - selected: ProviderChoice, - remoteConfig: RemoteProviderConfig, - state: SetupNimSelectionState, - selectedCredentialEnv: string, -): Promise { - const selectedModel = requireValue( - isBackToSelection(state.model) ? null : state.model, - `Missing model for ${remoteConfig.label}`, - ); - if (selected.key === "custom") { - const validation = await validateCustomOpenAiLikeSelection( - remoteConfig.label, - state.endpointUrl || OPENAI_ENDPOINT_URL, - selectedModel, - selectedCredentialEnv, - remoteConfig.helpUrl, - ); - if (validation.ok) { - const explicitApi = (process.env.NEMOCLAW_PREFERRED_API || "").trim().toLowerCase(); - if ( - explicitApi && - explicitApi !== "openai-completions" && - explicitApi !== "chat-completions" - ) { - state.preferredInferenceApi = validation.api; - } else { - if (validation.api !== "openai-completions") { - console.log( - " ℹ Using chat completions API (compatible endpoints may not support the Responses API developer role)", - ); - } - state.preferredInferenceApi = "openai-completions"; - } - return "selected"; - } - if ( - validation.retry === "credential" || - validation.retry === "retry" || - validation.retry === "model" - ) { - return "retry-model"; - } - return validation.retry === "selection" ? "retry-selection" : "retry-model"; - } - - if (selected.key === "anthropicCompatible") { - const validation = await validateCustomAnthropicSelection( - remoteConfig.label, - state.endpointUrl || ANTHROPIC_ENDPOINT_URL, - selectedModel, - selectedCredentialEnv, - remoteConfig.helpUrl, - ); - if (validation.ok) { - state.preferredInferenceApi = validation.api; - return "selected"; - } - if ( - validation.retry === "credential" || - validation.retry === "retry" || - validation.retry === "model" - ) { - return "retry-model"; - } - return validation.retry === "selection" ? "retry-selection" : "retry-model"; - } - - const retryMessage = "Please choose a provider/model again."; - if (selected.key === "anthropic") { - const validation = await validateAnthropicSelectionWithRetryMessage( - remoteConfig.label, - state.endpointUrl || ANTHROPIC_ENDPOINT_URL, - selectedModel, - selectedCredentialEnv, - retryMessage, - remoteConfig.helpUrl, - ); - if (validation.ok) { - state.preferredInferenceApi = validation.api; - return "selected"; - } - if ( - validation.retry === "credential" || - validation.retry === "retry" || - validation.retry === "model" - ) { - return "retry-model"; - } - return "retry-selection"; - } - - const validation = await validateOpenAiLikeSelection( - remoteConfig.label, - requireValue(state.endpointUrl, `Missing endpoint URL for ${remoteConfig.label}`), - selectedModel, - selectedCredentialEnv, - retryMessage, - remoteConfig.helpUrl, - { - requireResponsesToolCalling: shouldRequireResponsesToolCalling(state.provider), - skipResponsesProbe: shouldSkipResponsesProbe(state.provider), - authMode: getProbeAuthMode(state.provider), - }, - ); - if (validation.ok) { - state.preferredInferenceApi = validation.api; - return "selected"; - } - if ( - validation.retry === "credential" || - validation.retry === "retry" || - validation.retry === "model" - ) { - return "retry-model"; - } - return "retry-selection"; -} - async function handleVllmSelection( state: SetupNimSelectionState, ): Promise { @@ -3947,7 +3840,7 @@ async function handleRemoteProviderSelection( !providerExistsInGateway(state.provider) ) { console.error( - ` ${selectedCredentialEnv} (or NEMOCLAW_PROVIDER_KEY) is required for ${remoteConfig.label} in non-interactive mode.`, + ` Provider credential (or NEMOCLAW_PROVIDER_KEY) is required for ${remoteConfig.label} in non-interactive mode.`, ); process.exit(1); } @@ -3999,12 +3892,12 @@ async function handleRemoteProviderSelection( return "retry-selection"; } - const validationResult = await validateSelectedRemoteModel( + const validationResult = await validateSelectedRemoteModel({ selected, remoteConfig, state, selectedCredentialEnv, - ); + }); if (validationResult === "selected") break; if (validationResult === "retry-selection") return "retry-selection"; } diff --git a/src/lib/onboard/setup-nim-selection.ts b/src/lib/onboard/setup-nim-selection.ts new file mode 100644 index 00000000000..6042649066a --- /dev/null +++ b/src/lib/onboard/setup-nim-selection.ts @@ -0,0 +1,207 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export type SetupNimSelectionBackNavigation = Readonly<{ kind: "NEMOCLAW_BACK_TO_SELECTION" }>; + +export type SetupNimSelectionState = { + model: string | SetupNimSelectionBackNavigation | null; + provider: string; + endpointUrl: string | null; + credentialEnv: string | null; + hermesAuthMethod: unknown | null; + hermesToolGateways: string[]; + preferredInferenceApi: string | null; + nimContainer: string | null; +}; + +type ProviderChoice = { + key: string; +}; + +type RemoteProviderConfig = { + label: string; + endpointUrl: string; + helpUrl: string | null; +}; + +type ProbeAuthMode = "bearer" | "query-param" | undefined; + +type ProbeOptions = { + requireResponsesToolCalling?: boolean; + skipResponsesProbe?: boolean; + authMode?: ProbeAuthMode; +}; + +type ValidationResult = + | { ok: true; api: string | null; retry?: never } + | { ok: false; api?: string; retry?: "credential" | "retry" | "model" | "selection" | string }; + +type RemoteModelValidationResult = "selected" | "retry-model" | "retry-selection"; + +type RemoteModelValidatorDeps = { + OPENAI_ENDPOINT_URL: string; + ANTHROPIC_ENDPOINT_URL: string; + requireValue: (value: T | null | undefined, message: string) => T; + isBackToSelection: (value: unknown) => value is SetupNimSelectionBackNavigation; + validateCustomOpenAiLikeSelection: ( + label: string, + endpointUrl: string, + model: string, + credentialEnv: string, + helpUrl: string | null, + ) => Promise; + validateCustomAnthropicSelection: ( + label: string, + endpointUrl: string, + model: string, + credentialEnv: string, + helpUrl: string | null, + ) => Promise; + validateAnthropicSelectionWithRetryMessage: ( + label: string, + endpointUrl: string, + model: string, + credentialEnv: string, + retryMessage: string, + helpUrl: string | null, + ) => Promise; + validateOpenAiLikeSelection: ( + label: string, + endpointUrl: string, + model: string, + credentialEnv: string | null, + retryMessage?: string, + helpUrl?: string | null, + options?: ProbeOptions, + ) => Promise; + shouldRequireResponsesToolCalling: (provider: string) => boolean; + shouldSkipResponsesProbe: (provider: string) => boolean; + getProbeAuthMode: (provider: string) => ProbeAuthMode; +}; + +type ValidateSelectedRemoteModelArgs = { + selected: ProviderChoice; + remoteConfig: RemoteProviderConfig; + state: SetupNimSelectionState; + selectedCredentialEnv: string; +}; + +function shouldRetryModel(validation: ValidationResult): boolean { + return ( + !validation.ok && + (validation.retry === "credential" || + validation.retry === "retry" || + validation.retry === "model") + ); +} + +export function createRemoteModelValidator(deps: RemoteModelValidatorDeps): { + validateSelectedRemoteModel: ( + args: ValidateSelectedRemoteModelArgs, + ) => Promise; +} { + return { + validateSelectedRemoteModel: async ({ + selected, + remoteConfig, + state, + selectedCredentialEnv, + }) => { + const selectedModel = deps.requireValue( + deps.isBackToSelection(state.model) ? null : state.model, + `Missing model for ${remoteConfig.label}`, + ); + if (selected.key === "custom") { + const validation = await deps.validateCustomOpenAiLikeSelection( + remoteConfig.label, + state.endpointUrl || deps.OPENAI_ENDPOINT_URL, + selectedModel, + selectedCredentialEnv, + remoteConfig.helpUrl, + ); + if (validation.ok) { + const explicitApi = (process.env.NEMOCLAW_PREFERRED_API || "").trim().toLowerCase(); + if ( + explicitApi && + explicitApi !== "openai-completions" && + explicitApi !== "chat-completions" + ) { + state.preferredInferenceApi = validation.api; + } else { + if (validation.api !== "openai-completions") { + console.log( + " ℹ Using chat completions API (compatible endpoints may not support the Responses API developer role)", + ); + } + state.preferredInferenceApi = "openai-completions"; + } + return "selected"; + } + if (shouldRetryModel(validation)) { + return "retry-model"; + } + return validation.retry === "selection" ? "retry-selection" : "retry-model"; + } + + if (selected.key === "anthropicCompatible") { + const validation = await deps.validateCustomAnthropicSelection( + remoteConfig.label, + state.endpointUrl || deps.ANTHROPIC_ENDPOINT_URL, + selectedModel, + selectedCredentialEnv, + remoteConfig.helpUrl, + ); + if (validation.ok) { + state.preferredInferenceApi = validation.api; + return "selected"; + } + if (shouldRetryModel(validation)) { + return "retry-model"; + } + return validation.retry === "selection" ? "retry-selection" : "retry-model"; + } + + const retryMessage = "Please choose a provider/model again."; + if (selected.key === "anthropic") { + const validation = await deps.validateAnthropicSelectionWithRetryMessage( + remoteConfig.label, + state.endpointUrl || deps.ANTHROPIC_ENDPOINT_URL, + selectedModel, + selectedCredentialEnv, + retryMessage, + remoteConfig.helpUrl, + ); + if (validation.ok) { + state.preferredInferenceApi = validation.api; + return "selected"; + } + if (shouldRetryModel(validation)) { + return "retry-model"; + } + return "retry-selection"; + } + + const validation = await deps.validateOpenAiLikeSelection( + remoteConfig.label, + deps.requireValue(state.endpointUrl, `Missing endpoint URL for ${remoteConfig.label}`), + selectedModel, + selectedCredentialEnv, + retryMessage, + remoteConfig.helpUrl, + { + requireResponsesToolCalling: deps.shouldRequireResponsesToolCalling(state.provider), + skipResponsesProbe: deps.shouldSkipResponsesProbe(state.provider), + authMode: deps.getProbeAuthMode(state.provider), + }, + ); + if (validation.ok) { + state.preferredInferenceApi = validation.api; + return "selected"; + } + if (shouldRetryModel(validation)) { + return "retry-model"; + } + return "retry-selection"; + }, + }; +} From cdc58d7295a1d0601fa711258bebb2addd598492 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sun, 14 Jun 2026 16:51:38 -0700 Subject: [PATCH 05/23] test(rebuild): cover interactive confirmation Signed-off-by: Carlos Villela --- test/rebuild-credential-preflight.test.ts | 71 ++++++++++++++++++----- 1 file changed, 55 insertions(+), 16 deletions(-) diff --git a/test/rebuild-credential-preflight.test.ts b/test/rebuild-credential-preflight.test.ts index e5f97dbfecb..2b03e60045d 100644 --- a/test/rebuild-credential-preflight.test.ts +++ b/test/rebuild-credential-preflight.test.ts @@ -301,24 +301,24 @@ process.exit(0); function runRebuild( fixture: ReturnType, extraEnv: Record = {}, + options: { yes?: boolean; input?: string } = {}, ) { - return spawnSync( - process.execPath, - [path.join(REPO_ROOT, "bin", "nemoclaw.js"), fixture.sandboxName, "rebuild", "--yes"], - { - cwd: REPO_ROOT, - encoding: "utf-8", - env: { - HOME: fixture.tmpDir, - PATH: fixture.tmpDir + ":" + NODE_BIN + ":/usr/bin:/bin", - NEMOCLAW_NON_INTERACTIVE: "1", - NEMOCLAW_NO_CONNECT_HINT: "1", - NO_COLOR: "1", - ...extraEnv, - }, - timeout: 30_000, + const argv = [path.join(REPO_ROOT, "bin", "nemoclaw.js"), fixture.sandboxName, "rebuild"]; + if (options.yes !== false) argv.push("--yes"); + return spawnSync(process.execPath, argv, { + cwd: REPO_ROOT, + encoding: "utf-8", + input: options.input, + env: { + HOME: fixture.tmpDir, + PATH: fixture.tmpDir + ":" + NODE_BIN + ":/usr/bin:/bin", + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_NO_CONNECT_HINT: "1", + NO_COLOR: "1", + ...extraEnv, }, - ); + timeout: 30_000, + }); } function registryHasSandbox(fixture: ReturnType): boolean { @@ -334,6 +334,45 @@ function registryHasSandbox(fixture: ReturnType): boolean describe("Issue #2273: atomic rebuild", () => { describe("Layer 2: preflight credential check", () => { + it("cancels interactive rebuild before credential preflight or backup on non-affirmative input", { + timeout: 60_000, + }, () => { + const f = createFixture({ + credentialEnv: "NVIDIA_INFERENCE_API_KEY", + providerRegistered: false, + }); + + const result = runRebuild(f, {}, { yes: false, input: "n\n" }); + const output = (result.stderr || "") + (result.stdout || ""); + + expect(result.status).toBe(0); + expect(output).toContain("Proceed? [y/N]:"); + expect(output).toContain("Cancelled."); + expect(output).not.toContain("preflight failed"); + expect(output).not.toContain("Backing up sandbox state"); + expect(registryHasSandbox(f)).toBe(true); + }); + + it("accepts trimmed case-insensitive yes input before continuing rebuild", { + timeout: 60_000, + }, () => { + const f = createFixture({ + credentialEnv: "NVIDIA_INFERENCE_API_KEY", + savedCredential: { + key: "NVIDIA_INFERENCE_API_KEY", + value: "nvapi-test-key-for-rebuild", + }, + }); + + const result = runRebuild(f, {}, { yes: false, input: " YES \n" }); + const output = (result.stderr || "") + (result.stdout || ""); + + expect(output).toContain("Proceed? [y/N]:"); + expect(output).not.toContain("Cancelled."); + expect(output).not.toContain("preflight failed"); + expect(output).toContain("Backing up sandbox state"); + }); + it("aborts rebuild BEFORE destroying sandbox when credential is missing", { timeout: 60_000, }, () => { From 2879093941888197a1a7bc0f673fc21646087e9a Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sun, 14 Jun 2026 17:06:35 -0700 Subject: [PATCH 06/23] refactor(rebuild): lower cognitive complexity ratchet to 225 Signed-off-by: Carlos Villela --- biome.json | 2 +- src/lib/actions/sandbox/rebuild.ts | 20 ++++++++++++++------ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/biome.json b/biome.json index 0339890e330..fd410f5d3ce 100644 --- a/biome.json +++ b/biome.json @@ -86,7 +86,7 @@ "noExcessiveCognitiveComplexity": { "level": "error", "options": { - "maxAllowedComplexity": 243 + "maxAllowedComplexity": 225 } } }, diff --git a/src/lib/actions/sandbox/rebuild.ts b/src/lib/actions/sandbox/rebuild.ts index 1610e494c0b..901ba913fb2 100644 --- a/src/lib/actions/sandbox/rebuild.ts +++ b/src/lib/actions/sandbox/rebuild.ts @@ -332,6 +332,19 @@ async function confirmSandboxRebuildIfNeeded( return true; } +function isSingleAgentRebuildSupported( + sb: registry.SandboxEntry & { agents?: unknown[] }, + bail: (msg: string, code?: number) => never, +): boolean { + if (sb.agents && sb.agents.length > 1) { + console.error(" Multi-agent sandbox rebuild is not yet supported."); + console.error(` Back up state manually and recreate with \`${CLI_NAME} onboard\`.`); + bail("Multi-agent sandbox rebuild is not yet supported."); + return false; + } + return true; +} + async function reapplyMessagingManifestAfterOpenClawDoctor( sandboxName: string, plan: SandboxMessagingPlan | null, @@ -396,12 +409,7 @@ export async function rebuildSandbox( } // Multi-agent guard (temporary — until swarm lands) - if (sb.agents && sb.agents.length > 1) { - console.error(" Multi-agent sandbox rebuild is not yet supported."); - console.error(` Back up state manually and recreate with \`${CLI_NAME} onboard\`.`); - bail("Multi-agent sandbox rebuild is not yet supported."); - return; - } + if (!isSingleAgentRebuildSupported(sb, bail)) return; const rebuildAgent = sb.agent || null; const agent = agentRuntime.getSessionAgent(sandboxName); From df63d1a3ed60eb6ad9f836ccd9dc5616e327f644 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sun, 14 Jun 2026 17:15:58 -0700 Subject: [PATCH 07/23] refactor(rebuild): lower cognitive complexity ratchet to 224 Signed-off-by: Carlos Villela --- biome.json | 2 +- src/lib/actions/sandbox/rebuild.ts | 97 +++++++++++++++++------------- 2 files changed, 57 insertions(+), 42 deletions(-) diff --git a/biome.json b/biome.json index fd410f5d3ce..ef6f488be91 100644 --- a/biome.json +++ b/biome.json @@ -86,7 +86,7 @@ "noExcessiveCognitiveComplexity": { "level": "error", "options": { - "maxAllowedComplexity": 225 + "maxAllowedComplexity": 224 } } }, diff --git a/src/lib/actions/sandbox/rebuild.ts b/src/lib/actions/sandbox/rebuild.ts index 901ba913fb2..65081d23e28 100644 --- a/src/lib/actions/sandbox/rebuild.ts +++ b/src/lib/actions/sandbox/rebuild.ts @@ -262,6 +262,8 @@ async function stageMessagingManifestPlanForRebuild( return plan; } +type RebuildSandboxEntry = registry.SandboxEntry & { agents?: unknown[] }; + const runMessagingOpenshell: MessagingOpenShellRunner = (args, options = {}) => runOpenshell([...args], { env: options.env as NodeJS.ProcessEnv | undefined, @@ -332,6 +334,35 @@ async function confirmSandboxRebuildIfNeeded( return true; } +function checkRebuildGatewaySchemaPreflight( + sandboxName: string, + bail: (msg: string, code?: number) => never, +): boolean { + const gatewayPreflightIssue = detectOpenShellStateRpcPreflightIssue(); + if (gatewayPreflightIssue) { + printOpenShellStateRpcIssue(gatewayPreflightIssue, { + action: `rebuilding sandbox '${sandboxName}'`, + command: `${CLI_NAME} ${sandboxName} rebuild`, + }); + bail("OpenShell gateway schema mismatch."); + return false; + } + return true; +} + +function getRebuildSandboxEntryOrBail( + sandboxName: string, + bail: (msg: string, code?: number) => never, +): RebuildSandboxEntry | null { + const sb = registry.getSandbox(sandboxName) as RebuildSandboxEntry | null; + if (!sb) { + console.error(` Sandbox '${sandboxName}' not found in registry.`); + bail(`Sandbox '${sandboxName}' not found in registry.`); + return null; + } + return sb; +} + function isSingleAgentRebuildSupported( sb: registry.SandboxEntry & { agents?: unknown[] }, bail: (msg: string, code?: number) => never, @@ -345,6 +376,24 @@ function isSingleAgentRebuildSupported( return true; } +function stashWechatMetadataForRebuild(sandboxName: string, log: (msg: string) => void): void { + const rebuildSession = onboardSession.loadSession(); + const wc = + rebuildSession?.sandboxName === sandboxName ? (rebuildSession.wechatConfig ?? null) : null; + if (wc?.accountId && !process.env.WECHAT_ACCOUNT_ID) { + process.env.WECHAT_ACCOUNT_ID = wc.accountId; + } + if (wc?.baseUrl && !process.env.WECHAT_BASE_URL) { + process.env.WECHAT_BASE_URL = wc.baseUrl; + } + if (wc?.userId && !process.env.WECHAT_USER_ID) { + process.env.WECHAT_USER_ID = wc.userId; + } + if (wc?.accountId) { + log(`Stashed WeChat account metadata for rebuild: accountId=${wc.accountId}`); + } +} + async function reapplyMessagingManifestAfterOpenClawDoctor( sandboxName: string, plan: SandboxMessagingPlan | null, @@ -401,12 +450,8 @@ export async function rebuildSandbox( // Active session detection — enrich the confirmation prompt if sessions are active const rebuildActiveSessionCount = countActiveSandboxSessionsForRebuild(sandboxName); - const sb = registry.getSandbox(sandboxName) as any; - if (!sb) { - console.error(` Sandbox '${sandboxName}' not found in registry.`); - bail(`Sandbox '${sandboxName}' not found in registry.`); - return; - } + const sb = getRebuildSandboxEntryOrBail(sandboxName, bail); + if (!sb) return; // Multi-agent guard (temporary — until swarm lands) if (!isSingleAgentRebuildSupported(sb, bail)) return; @@ -415,43 +460,13 @@ export async function rebuildSandbox( const agent = agentRuntime.getSessionAgent(sandboxName); const agentName = agentRuntime.getAgentDisplayName(agent); - const gatewayPreflightIssue = detectOpenShellStateRpcPreflightIssue(); - if (gatewayPreflightIssue) { - printOpenShellStateRpcIssue(gatewayPreflightIssue, { - action: `rebuilding sandbox '${sandboxName}'`, - command: `${CLI_NAME} ${sandboxName} rebuild`, - }); - bail("OpenShell gateway schema mismatch."); - return; - } + if (!checkRebuildGatewaySchemaPreflight(sandboxName, bail)) return; // Stash WeChat per-account metadata into process.env before the rebuild - // touches anything destructive. The metadata lives in session.wechatConfig - // (captured during the original onboard's host-side QR login) — the only - // durable source today. Surfacing it as WECHAT_ACCOUNT_ID / WECHAT_BASE_URL - // / WECHAT_USER_ID lets the in-process onboard --resume that fires later - // see it directly via the wechatConfig builder's process.env path. - // `openclaw-weixin/` runtime state is intentionally NOT in state_dirs — - // the manifest post-agent-install hook rebuilds account files from these - // env-backed config inputs every image build, so keeping the envs here is - // what the next image needs to put the right accountId/baseUrl/userId back - // into openclaw.json + the accounts state file. - { - // Only hydrate from the session when it belongs to THIS sandbox. The - // global session file holds the most recent onboard, which may be for a - // different sandbox — pulling its wechatConfig would leak that - // sandbox's accountId / baseUrl / userId into this image build. - const rebuildSession = onboardSession.loadSession(); - const wc = - rebuildSession?.sandboxName === sandboxName ? (rebuildSession.wechatConfig ?? null) : null; - if (wc?.accountId && !process.env.WECHAT_ACCOUNT_ID) - process.env.WECHAT_ACCOUNT_ID = wc.accountId; - if (wc?.baseUrl && !process.env.WECHAT_BASE_URL) process.env.WECHAT_BASE_URL = wc.baseUrl; - if (wc?.userId && !process.env.WECHAT_USER_ID) process.env.WECHAT_USER_ID = wc.userId; - if (wc?.accountId) { - log(`Stashed WeChat account metadata for rebuild: accountId=${wc.accountId}`); - } - } + // touches anything destructive. Only hydrate from the session when it belongs + // to this sandbox so metadata from a different recent onboard cannot leak into + // this image build. + stashWechatMetadataForRebuild(sandboxName, log); // Version check — show what's changing const versionCheck = sandboxVersion.checkAgentVersion(sandboxName); From 494c64118113c63b2cdb09740237ac71ae70e949 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sun, 14 Jun 2026 17:19:38 -0700 Subject: [PATCH 08/23] refactor(rebuild): lower cognitive complexity ratchet to 215 Signed-off-by: Carlos Villela --- biome.json | 2 +- src/lib/actions/sandbox/rebuild.ts | 26 +++++++++++++++++--------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/biome.json b/biome.json index ef6f488be91..24b13ecf941 100644 --- a/biome.json +++ b/biome.json @@ -86,7 +86,7 @@ "noExcessiveCognitiveComplexity": { "level": "error", "options": { - "maxAllowedComplexity": 224 + "maxAllowedComplexity": 215 } } }, diff --git a/src/lib/actions/sandbox/rebuild.ts b/src/lib/actions/sandbox/rebuild.ts index 65081d23e28..f1a26b89a0f 100644 --- a/src/lib/actions/sandbox/rebuild.ts +++ b/src/lib/actions/sandbox/rebuild.ts @@ -394,6 +394,22 @@ function stashWechatMetadataForRebuild(sandboxName: string, log: (msg: string) = } } +function printRebuildVersionSummary( + sandboxName: string, + agentName: string, + versionCheck: ReturnType, +): void { + console.log(""); + console.log(` ${B}Rebuild sandbox '${sandboxName}'${R}`); + if (versionCheck.sandboxVersion) { + console.log(` Current: ${agentName} v${versionCheck.sandboxVersion}`); + } + if (versionCheck.expectedVersion) { + console.log(` Target: ${agentName} v${versionCheck.expectedVersion}`); + } + console.log(""); +} + async function reapplyMessagingManifestAfterOpenClawDoctor( sandboxName: string, plan: SandboxMessagingPlan | null, @@ -470,15 +486,7 @@ export async function rebuildSandbox( // Version check — show what's changing const versionCheck = sandboxVersion.checkAgentVersion(sandboxName); - console.log(""); - console.log(` ${B}Rebuild sandbox '${sandboxName}'${R}`); - if (versionCheck.sandboxVersion) { - console.log(` Current: ${agentName} v${versionCheck.sandboxVersion}`); - } - if (versionCheck.expectedVersion) { - console.log(` Target: ${agentName} v${versionCheck.expectedVersion}`); - } - console.log(""); + printRebuildVersionSummary(sandboxName, agentName, versionCheck); const rebuildConfirmed = await confirmSandboxRebuildIfNeeded( skipConfirm, From 2f250de8cc85c274682452c0864c866379662f87 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sun, 14 Jun 2026 17:42:43 -0700 Subject: [PATCH 09/23] refactor(rebuild): lower cognitive complexity ratchet to 186 Signed-off-by: Carlos Villela --- biome.json | 2 +- src/lib/actions/sandbox/rebuild.ts | 243 ++++++++++++++--------------- 2 files changed, 118 insertions(+), 127 deletions(-) diff --git a/biome.json b/biome.json index 24b13ecf941..e148bc70899 100644 --- a/biome.json +++ b/biome.json @@ -86,7 +86,7 @@ "noExcessiveCognitiveComplexity": { "level": "error", "options": { - "maxAllowedComplexity": 215 + "maxAllowedComplexity": 186 } } }, diff --git a/src/lib/actions/sandbox/rebuild.ts b/src/lib/actions/sandbox/rebuild.ts index f1a26b89a0f..f85081d6e16 100644 --- a/src/lib/actions/sandbox/rebuild.ts +++ b/src/lib/actions/sandbox/rebuild.ts @@ -394,6 +394,112 @@ function stashWechatMetadataForRebuild(sandboxName: string, log: (msg: string) = } } +async function stageRebuildMessagingPlanOrBail( + sandboxName: string, + sb: RebuildSandboxEntry, + rebuildAgent: string | null, + log: (msg: string) => void, + bail: (msg: string, code?: number) => never, +): Promise { + try { + return await stageMessagingManifestPlanForRebuild(sandboxName, sb, rebuildAgent, log); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + console.error(""); + console.error( + ` ${_RD}Rebuild preflight failed:${R} messaging manifest plan could not be staged.`, + ); + console.error(` ${message}`); + console.error(""); + console.error(" Sandbox is untouched — no data was lost."); + bail(message); + return null; + } +} + +function preflightRebuildCredentials( + sandboxName: string, + sb: RebuildSandboxEntry, + log: (msg: string) => void, + bail: (msg: string, code?: number) => never, +): boolean { + const session = onboardSession.loadSession(); + const sessionMatchesTarget = session?.sandboxName === sandboxName; + let rebuildCredentialEnv = sessionMatchesTarget + ? session?.credentialEnv || null + : getRebuildCredentialEnvFromRegistry(sb.provider); + if (!sessionMatchesTarget && session?.sandboxName) { + log( + `Preflight warning: session belongs to '${session.sandboxName}', not '${sandboxName}' — using registry credential env ${rebuildCredentialEnv || "(none)"}`, + ); + console.log( + ` ${D}Note: onboard session belongs to '${session.sandboxName}', not '${sandboxName}'. ` + + `Using the '${sandboxName}' registry entry for credential preflight.${R}`, + ); + } + + const rebuildProvider = sessionMatchesTarget ? session?.provider || sb.provider : sb.provider; + if ( + (session?.provider === "ollama-local" || session?.provider === "vllm-local") && + rebuildCredentialEnv === "OPENAI_API_KEY" + ) { + console.log( + ` ${D}Note: migrating ${session.provider} sandbox off OPENAI_API_KEY (GH #2519). ` + + `Local inference does not require a host API key.${R}`, + ); + log( + `Preflight: legacy ${session.provider} sandbox detected (credentialEnv=OPENAI_API_KEY) — clearing for rebuild`, + ); + rebuildCredentialEnv = null; + } + + if (rebuildProvider === hermesProviderAuth.HERMES_PROVIDER_NAME) { + if ( + !preflightHermesProviderCredentials( + sessionMatchesTarget ? session : null, + rebuildCredentialEnv, + log, + ) + ) { + bail("Missing Hermes Provider credentials"); + return false; + } + rebuildCredentialEnv = null; + } + + if (!rebuildCredentialEnv) { + log( + "Preflight credential check: no credentialEnv in session (local inference or missing session)", + ); + return true; + } + + const credentialValue = hydrateCredentialEnv(rebuildCredentialEnv); + log( + `Preflight credential check: ${rebuildCredentialEnv} → ${credentialValue ? "present" : "MISSING"}`, + ); + if (credentialValue) return true; + if (rebuildProvider && providerExistsInGateway(rebuildProvider, runOpenshell)) { + log( + `Preflight credential check: provider '${rebuildProvider}' registered in gateway — skipping env check for ${rebuildCredentialEnv}`, + ); + return true; + } + + console.error(""); + console.error(` ${_RD}Rebuild preflight failed:${R} provider credential not found.`); + console.error(` The non-interactive recreate step requires ${rebuildCredentialEnv},`); + console.error(" but it is not set in the environment."); + console.error(""); + console.error(" To fix, do one of:"); + console.error(` export ${rebuildCredentialEnv}=`); + console.error(` ${CLI_NAME} onboard # re-enter the key interactively`); + console.error(""); + console.error(" Sandbox is untouched — no data was lost."); + bail(`Missing credential: ${rebuildCredentialEnv}`); + return false; +} + function printRebuildVersionSummary( sandboxName: string, agentName: string, @@ -495,133 +601,18 @@ export async function rebuildSandbox( if (!rebuildConfirmed) return; // Step 0: Preflight — verify recreate preconditions BEFORE destroying - // anything. The most common rebuild failure is a missing provider - // credential when onboard runs in non-interactive mode. Checking now - // lets us abort with the sandbox still intact. See #2273. - const session = onboardSession.loadSession(); - const sessionMatchesTarget = session?.sandboxName === sandboxName; - let rebuildCredentialEnv: string | null = null; - if (!sessionMatchesTarget) { - // Session belongs to a different sandbox — its credentialEnv may be - // wrong (e.g. hermes session while rebuilding openclaw). Resolve the - // target sandbox provider from the registry instead so destructive - // operations still get a credential preflight for the sandbox being rebuilt. - rebuildCredentialEnv = getRebuildCredentialEnvFromRegistry(sb.provider); - if (session?.sandboxName) { - log( - `Preflight warning: session belongs to '${session.sandboxName}', not '${sandboxName}' — using registry credential env ${rebuildCredentialEnv || "(none)"}`, - ); - console.log( - ` ${D}Note: onboard session belongs to '${session.sandboxName}', not '${sandboxName}'. ` + - `Using the '${sandboxName}' registry entry for credential preflight.${R}`, - ); - } - } else { - rebuildCredentialEnv = session?.credentialEnv || null; - } - const rebuildProvider = sessionMatchesTarget ? session?.provider || sb.provider : sb.provider; - // Legacy migration: pre-fix local-inference sandboxes (GH #2519, GH #2625) - // recorded credentialEnv="OPENAI_API_KEY" in onboard-session.json even - // though the sandbox does not actually need a host OpenAI key (ollama-local - // uses an auth proxy with an internal token; vllm-local accepts a static - // dummy bearer). Treat the legacy value as null so rebuild does not demand - // a credential that was never actually used. - // - // Post-#2625 the write path persists credentialEnv=null directly when the - // wizard selects a local provider, so fresh sessions no longer need this - // migration. We retain it for users whose session.json on disk predates - // the fix. - if ( - (session?.provider === "ollama-local" || session?.provider === "vllm-local") && - rebuildCredentialEnv === "OPENAI_API_KEY" - ) { - console.log( - ` ${D}Note: migrating ${session.provider} sandbox off OPENAI_API_KEY (GH #2519). ` + - `Local inference does not require a host API key.${R}`, - ); - log( - `Preflight: legacy ${session.provider} sandbox detected (credentialEnv=OPENAI_API_KEY) — clearing for rebuild`, - ); - rebuildCredentialEnv = null; - } - if (rebuildProvider === hermesProviderAuth.HERMES_PROVIDER_NAME) { - if ( - !preflightHermesProviderCredentials( - sessionMatchesTarget ? session : null, - rebuildCredentialEnv, - log, - ) - ) { - bail("Missing Hermes Provider credentials"); - return; - } - // Hermes Provider credentials belong to OpenShell provider storage. Do not - // fall through to the generic env-var preflight, which would incorrectly - // demand OPENAI_API_KEY/NOUS_API_KEY after the provider is registered. - rebuildCredentialEnv = null; - } - if (rebuildCredentialEnv) { - // hydrateCredentialEnv migrates any pre-fix legacy credentials.json - // into process.env once, so users upgrading from a release that wrote - // the plaintext file can still rebuild without re-entering keys. - const credentialValue = hydrateCredentialEnv(rebuildCredentialEnv); - log( - `Preflight credential check: ${rebuildCredentialEnv} → ${credentialValue ? "present" : "MISSING"}`, - ); - if (!credentialValue) { - // When the inference provider is already registered in the OpenShell - // gateway, the recreate step does not need a host env value — the - // gateway is the source of truth for the secret. Skip the env-only - // preflight in that case so flows like `channels add` + rebuild keep - // working when the user has logged out of the original shell. - if (rebuildProvider && providerExistsInGateway(rebuildProvider, runOpenshell)) { - log( - `Preflight credential check: provider '${rebuildProvider}' registered in gateway — skipping env check for ${rebuildCredentialEnv}`, - ); - } else { - console.error(""); - console.error(` ${_RD}Rebuild preflight failed:${R} provider credential not found.`); - console.error(` The non-interactive recreate step requires ${rebuildCredentialEnv},`); - console.error(" but it is not set in the environment."); - console.error(""); - console.error(" To fix, do one of:"); - console.error(` export ${rebuildCredentialEnv}=`); - console.error(` ${CLI_NAME} onboard # re-enter the key interactively`); - console.error(""); - console.error(" Sandbox is untouched — no data was lost."); - bail(`Missing credential: ${rebuildCredentialEnv}`); - return; - } - } - } else { - // No credentialEnv in session — local inference (Ollama/vLLM) or - // session was lost. Either way, skip the credential preflight; - // onboard will handle it. - log( - "Preflight credential check: no credentialEnv in session (local inference or missing session)", - ); - } + // anything. The most common rebuild failure is a missing provider credential + // when onboard runs in non-interactive mode. Checking now lets us abort with + // the sandbox still intact. See #2273. + if (!preflightRebuildCredentials(sandboxName, sb, log, bail)) return; - let rebuildMessagingPlan: SandboxMessagingPlan | null = null; - try { - rebuildMessagingPlan = await stageMessagingManifestPlanForRebuild( - sandboxName, - sb, - rebuildAgent, - log, - ); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - console.error(""); - console.error( - ` ${_RD}Rebuild preflight failed:${R} messaging manifest plan could not be staged.`, - ); - console.error(` ${message}`); - console.error(""); - console.error(" Sandbox is untouched — no data was lost."); - bail(message); - return; - } + const rebuildMessagingPlan = await stageRebuildMessagingPlanOrBail( + sandboxName, + sb, + rebuildAgent, + log, + bail, + ); // Step 1: Ensure sandbox is live for backup const recordedGateway = resolveSandboxGatewayName(sb); From 843e3a47cc27faef1be95b294ab4e230ef9d2293 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sun, 14 Jun 2026 17:46:11 -0700 Subject: [PATCH 10/23] refactor(snapshot): lower cognitive complexity ratchet to 185 Signed-off-by: Carlos Villela --- biome.json | 2 +- src/lib/actions/sandbox/snapshot.ts | 90 ++++++++++++++--------------- src/lib/onboard.ts | 13 +++-- 3 files changed, 55 insertions(+), 50 deletions(-) diff --git a/biome.json b/biome.json index e148bc70899..a4e33371856 100644 --- a/biome.json +++ b/biome.json @@ -86,7 +86,7 @@ "noExcessiveCognitiveComplexity": { "level": "error", "options": { - "maxAllowedComplexity": 186 + "maxAllowedComplexity": 185 } } }, diff --git a/src/lib/actions/sandbox/snapshot.ts b/src/lib/actions/sandbox/snapshot.ts index 6a1c3931215..8a27a7ccbf8 100644 --- a/src/lib/actions/sandbox/snapshot.ts +++ b/src/lib/actions/sandbox/snapshot.ts @@ -344,57 +344,57 @@ function isSnapshotCreationAllowedByShields(sandboxName: string): boolean { return isShieldsDown(sandboxName); } +function runSnapshotCreate( + sandboxName: string, + request: Extract, +): void { + const liveNames = requireLiveSandboxesOnSandboxGateway( + sandboxName, + " Failed to query live sandbox state from OpenShell.", + ); + if (!liveNames.has(sandboxName)) { + console.error(` Sandbox '${sandboxName}' is not running. Cannot create snapshot.`); + snapshotExit(1); + } + if (!isSnapshotCreationAllowedByShields(sandboxName)) { + console.error(" Cannot create snapshot while shields are up."); + console.error(` Run \`${CLI_NAME} ${sandboxName} shields down\` first, then retry.`); + snapshotExit(1); + } + const label = request.name ? ` (--name ${request.name})` : ""; + console.log(` Creating snapshot of '${sandboxName}'${label}...`); + const result = sandboxState.backupSandboxState(sandboxName, { name: request.name ?? null }); + if (result.success) { + const manifest = result.manifest!; + const entry = sandboxState.findBackup(sandboxName, manifest.timestamp).match ?? manifest; + const v = formatSnapshotVersion(entry); + const nameSuffix = entry.name ? ` name=${entry.name}` : ""; + const itemSummary = `${result.backedUpDirs.length} directories, ${result.backedUpFiles.length} files`; + console.log(` ${G}✓${R} Snapshot ${v}${nameSuffix} created (${itemSummary})`); + console.log(` ${manifest.backupPath}`); + return; + } + if (result.error) { + console.error(` ${result.error}`); + } else { + console.error(" Snapshot failed."); + if (result.failedDirs.length > 0) { + console.error(` Failed directories: ${result.failedDirs.join(", ")}`); + } + if (result.failedFiles.length > 0) { + console.error(` Failed files: ${result.failedFiles.join(", ")}`); + } + } + snapshotExit(1); +} + export async function runSandboxSnapshot( sandboxName: string, request: SnapshotRequest = { kind: "help" }, ) { switch (request.kind) { case "create": { - // Select the sandbox's gateway before the health probe and `sandbox list` - // — Docker/VM-driver health and the sandbox listing both require the - // sandbox's gateway to be the active one, otherwise a sandbox registered - // on a non-default `NEMOCLAW_GATEWAY_PORT` fails health-check before the - // subsequent list ever runs. - const liveNames = requireLiveSandboxesOnSandboxGateway( - sandboxName, - " Failed to query live sandbox state from OpenShell.", - ); - if (!liveNames.has(sandboxName)) { - console.error(` Sandbox '${sandboxName}' is not running. Cannot create snapshot.`); - snapshotExit(1); - } - if (!isSnapshotCreationAllowedByShields(sandboxName)) { - console.error(" Cannot create snapshot while shields are up."); - console.error(` Run \`${CLI_NAME} ${sandboxName} shields down\` first, then retry.`); - snapshotExit(1); - } - const label = request.name ? ` (--name ${request.name})` : ""; - console.log(` Creating snapshot of '${sandboxName}'${label}...`); - const result = sandboxState.backupSandboxState(sandboxName, { name: request.name ?? null }); - if (result.success) { - // Virtual snapshotVersion is only assigned by listBackups, so re-resolve - // the just-created snapshot by its timestamp to get a valid v. - const manifest = result.manifest!; - const entry = sandboxState.findBackup(sandboxName, manifest.timestamp).match ?? manifest; - const v = formatSnapshotVersion(entry); - const nameSuffix = entry.name ? ` name=${entry.name}` : ""; - const itemSummary = `${result.backedUpDirs.length} directories, ${result.backedUpFiles.length} files`; - console.log(` ${G}\u2713${R} Snapshot ${v}${nameSuffix} created (${itemSummary})`); - console.log(` ${manifest.backupPath}`); - } else { - if (result.error) { - console.error(` ${result.error}`); - } else { - console.error(" Snapshot failed."); - if (result.failedDirs.length > 0) { - console.error(` Failed directories: ${result.failedDirs.join(", ")}`); - } - if (result.failedFiles.length > 0) { - console.error(` Failed files: ${result.failedFiles.join(", ")}`); - } - } - snapshotExit(1); - } + runSnapshotCreate(sandboxName, request); break; } case "list": { diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 51d17650da2..01967a27d45 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -3380,6 +3380,14 @@ type RemoteProviderSelectionArgs = { sandboxName: string | null; }; +function requireProviderChoice(selected: ProviderChoice | undefined): ProviderChoice { + if (!selected) { + console.error(" No provider was selected."); + process.exit(1); + } + return selected; +} + async function handleVllmSelection( state: SetupNimSelectionState, ): Promise { @@ -4075,10 +4083,7 @@ async function setupNim( }); } - if (!selected) { - console.error(" No provider was selected."); - process.exit(1); - } + selected = requireProviderChoice(selected); if (selected.key !== "hermesProvider") { hermesAuthMethod = null; hermesToolGateways = []; From 8b83cbfcc49cf0ab777187517ce3854153151935 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sun, 14 Jun 2026 17:53:08 -0700 Subject: [PATCH 11/23] chore(lint): lower cognitive complexity ratchet to 184 (#5431) ## Summary Continue the stacked cognitive-complexity ratchet by lowering the threshold from 185 to 184. The previous stack step brought all remaining offenders to 184 or below, so this PR only tightens the Biome configuration. ## Changes - Lowered `complexity/noExcessiveCognitiveComplexity` from `185` to `184` in `biome.json`. ## Type of Change - [x] Code change (feature, bug fix, or refactor) - [ ] Code change with doc updates - [ ] Doc only (prose changes, no code sample modifications) - [ ] Doc only (includes code sample changes) ## Verification - `npx @biomejs/biome lint --only=complexity/noExcessiveCognitiveComplexity --max-diagnostics=none .` - [x] Git hooks passed during commit and push, or `npx prek run --from-ref main --to-ref HEAD` passes - [x] Targeted tests pass for changed behavior - [ ] Tests added or updated for new or changed behavior - [ ] Full `npm test` passes (broad runtime changes only) - [x] No secrets, API keys, or credentials committed - [ ] Docs updated for user-facing behavior changes - [ ] `npm run docs` builds without warnings (doc changes only) - [ ] Doc pages follow the [style guide](https://github.com/NVIDIA/NemoClaw/blob/main/docs/CONTRIBUTING.md) (doc changes only) - [ ] New doc pages include SPDX header and frontmatter (new pages only) --- Signed-off-by: Carlos Villela Signed-off-by: Carlos Villela --- biome.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/biome.json b/biome.json index a4e33371856..86aa98a855d 100644 --- a/biome.json +++ b/biome.json @@ -86,7 +86,7 @@ "noExcessiveCognitiveComplexity": { "level": "error", "options": { - "maxAllowedComplexity": 185 + "maxAllowedComplexity": 184 } } }, From 557f501775d3142be88e350b0ad6b2a816d61fba Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sun, 14 Jun 2026 18:19:00 -0700 Subject: [PATCH 12/23] refactor(onboard): keep snapshot ratchet step net-neutral Signed-off-by: Carlos Villela --- src/lib/onboard.ts | 9 +-------- src/lib/onboard/setup-nim-selection.ts | 8 ++++++++ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 01967a27d45..26d4718251c 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -23,6 +23,7 @@ const { }: typeof import("./onboard/inference-selection-validation") = require("./onboard/inference-selection-validation"); const { createRemoteModelValidator, + requireProviderChoice, }: typeof import("./onboard/setup-nim-selection") = require("./onboard/setup-nim-selection"); const inferenceInputCapability = require("./onboard/inference-input-capability"); const { cleanupTempDir }: typeof import("./onboard/temp-files") = require("./onboard/temp-files"); @@ -3380,14 +3381,6 @@ type RemoteProviderSelectionArgs = { sandboxName: string | null; }; -function requireProviderChoice(selected: ProviderChoice | undefined): ProviderChoice { - if (!selected) { - console.error(" No provider was selected."); - process.exit(1); - } - return selected; -} - async function handleVllmSelection( state: SetupNimSelectionState, ): Promise { diff --git a/src/lib/onboard/setup-nim-selection.ts b/src/lib/onboard/setup-nim-selection.ts index 6042649066a..78d6dad48ea 100644 --- a/src/lib/onboard/setup-nim-selection.ts +++ b/src/lib/onboard/setup-nim-selection.ts @@ -18,6 +18,14 @@ type ProviderChoice = { key: string; }; +export function requireProviderChoice(selected: T | undefined): T { + if (!selected) { + console.error(" No provider was selected."); + process.exit(1); + } + return selected; +} + type RemoteProviderConfig = { label: string; endpointUrl: string; From b451a5bb199d1e6419bdfa6335ab7a8408ea0e85 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 15 Jun 2026 08:39:58 -0700 Subject: [PATCH 13/23] fix(onboard): preserve local NIM fallback state Signed-off-by: Carlos Villela --- src/lib/onboard.ts | 28 +++++----- src/lib/onboard/setup-nim-selection.test.ts | 59 +++++++++++++++++++++ src/lib/onboard/setup-nim-selection.ts | 27 +++++++++- 3 files changed, 98 insertions(+), 16 deletions(-) create mode 100644 src/lib/onboard/setup-nim-selection.test.ts diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index ac72cf507a9..5d85a21caf2 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -22,6 +22,8 @@ const { createInferenceSelectionValidationHelpers, }: typeof import("./onboard/inference-selection-validation") = require("./onboard/inference-selection-validation"); const { + applyCloudFallbackSelection, + clearNimContainerBeforeRetry, createRemoteModelValidator, }: typeof import("./onboard/setup-nim-selection") = require("./onboard/setup-nim-selection"); const inferenceInputCapability = require("./onboard/inference-input-capability"); @@ -3353,16 +3355,8 @@ async function selectAndValidateOllamaModel( } } -type SetupNimSelectionState = { - model: string | typeof BACK_TO_SELECTION | null; - provider: string; - endpointUrl: string | null; - credentialEnv: string | null; - hermesAuthMethod: HermesAuthMethod | null; - hermesToolGateways: string[]; - preferredInferenceApi: string | null; - nimContainer: string | null; -}; +type SetupNimSelectionState = + import("./onboard/setup-nim-selection").SetupNimSelectionState; type SetupNimSelectionResult = "selected" | "retry-selection"; @@ -3512,6 +3506,7 @@ async function handleNimLocalSelection( const models = nim.listModels().filter((m) => m.minGpuMemoryMB <= localGpu.totalMemoryMB); if (models.length === 0) { console.log(" No NIM models fit your GPU VRAM. Falling back to cloud API."); + applyCloudFallbackSelection(state, REMOTE_PROVIDER_CONFIG.build); return "selected"; } @@ -3597,8 +3592,7 @@ async function handleNimLocalSelection( console.log(" Waiting for NIM to become healthy..."); if (!nim.waitForNimHealth(undefined, undefined, { container: nimContainerNameLocal })) { console.error(" NIM failed to start. Falling back to cloud API."); - state.model = null; - state.nimContainer = null; + applyCloudFallbackSelection(state, REMOTE_PROVIDER_CONFIG.build); return "selected"; } @@ -3617,8 +3611,14 @@ async function handleNimLocalSelection( requireValue(state.model, "Expected a Local NVIDIA NIM model after startup"), null, ); - if (validation.retry === "selection" || validation.retry === "model") return "retry-selection"; - if (!validation.ok) return "retry-selection"; + if (validation.retry === "selection" || validation.retry === "model") { + clearNimContainerBeforeRetry(state); + return "retry-selection"; + } + if (!validation.ok) { + clearNimContainerBeforeRetry(state); + return "retry-selection"; + } if (validation.api !== "openai-completions") { console.log(" ℹ Using chat completions API (tool-call-parser requires /v1/chat/completions)"); } diff --git a/src/lib/onboard/setup-nim-selection.test.ts b/src/lib/onboard/setup-nim-selection.test.ts new file mode 100644 index 00000000000..79f3ef24f49 --- /dev/null +++ b/src/lib/onboard/setup-nim-selection.test.ts @@ -0,0 +1,59 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; + +import { describe, it } from "vitest"; + +import { + applyCloudFallbackSelection, + clearNimContainerBeforeRetry, + type SetupNimSelectionState, +} from "./setup-nim-selection"; + +function makeState(): SetupNimSelectionState { + return { + model: "nvidia/local-nim", + provider: "vllm-local", + endpointUrl: "http://127.0.0.1:8000/v1", + credentialEnv: null, + hermesAuthMethod: null, + hermesToolGateways: [], + preferredInferenceApi: "openai-completions", + nimContainer: "nemoclaw-nim-test", + }; +} + +describe("setupNim selection state helpers", () => { + it("applies a complete cloud fallback and clears stale NIM state", () => { + const state = makeState(); + + applyCloudFallbackSelection(state, { + providerName: "nvidia-prod", + endpointUrl: "https://integrate.api.nvidia.com/v1", + credentialEnv: "NVIDIA_INFERENCE_API_KEY", + defaultModel: "meta/llama-3.3-70b-instruct", + }); + + assert.deepEqual(state, { + model: "meta/llama-3.3-70b-instruct", + provider: "nvidia-prod", + endpointUrl: "https://integrate.api.nvidia.com/v1", + credentialEnv: "NVIDIA_INFERENCE_API_KEY", + hermesAuthMethod: null, + hermesToolGateways: [], + preferredInferenceApi: null, + nimContainer: null, + }); + }); + + it("clears stale NIM containers before retrying provider selection", () => { + const state = makeState(); + + clearNimContainerBeforeRetry(state); + + assert.equal(state.nimContainer, null); + assert.equal(state.model, "nvidia/local-nim"); + assert.equal(state.provider, "vllm-local"); + }); +}); diff --git a/src/lib/onboard/setup-nim-selection.ts b/src/lib/onboard/setup-nim-selection.ts index 6042649066a..1a825e9251f 100644 --- a/src/lib/onboard/setup-nim-selection.ts +++ b/src/lib/onboard/setup-nim-selection.ts @@ -3,17 +3,40 @@ export type SetupNimSelectionBackNavigation = Readonly<{ kind: "NEMOCLAW_BACK_TO_SELECTION" }>; -export type SetupNimSelectionState = { +export type SetupNimSelectionState = { model: string | SetupNimSelectionBackNavigation | null; provider: string; endpointUrl: string | null; credentialEnv: string | null; - hermesAuthMethod: unknown | null; + hermesAuthMethod: THermesAuthMethod | null; hermesToolGateways: string[]; preferredInferenceApi: string | null; nimContainer: string | null; }; +export type CloudFallbackConfig = { + providerName: string; + endpointUrl: string | null; + credentialEnv: string | null; + defaultModel: string; +}; + +export function applyCloudFallbackSelection( + state: SetupNimSelectionState, + cloudConfig: CloudFallbackConfig, +): void { + state.provider = cloudConfig.providerName; + state.endpointUrl = cloudConfig.endpointUrl; + state.credentialEnv = cloudConfig.credentialEnv; + state.model = cloudConfig.defaultModel; + state.preferredInferenceApi = null; + state.nimContainer = null; +} + +export function clearNimContainerBeforeRetry(state: SetupNimSelectionState): void { + state.nimContainer = null; +} + type ProviderChoice = { key: string; }; From 74968a30ebb4190797e2b4342f745222f837f3d8 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 15 Jun 2026 09:04:39 -0700 Subject: [PATCH 14/23] refactor(rebuild): isolate messaging config hydration Signed-off-by: Carlos Villela --- src/lib/actions/sandbox/rebuild.ts | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/lib/actions/sandbox/rebuild.ts b/src/lib/actions/sandbox/rebuild.ts index 32ba42c8514..d1d9af0a7d9 100644 --- a/src/lib/actions/sandbox/rebuild.ts +++ b/src/lib/actions/sandbox/rebuild.ts @@ -484,6 +484,16 @@ function preflightRebuildCredentials( return false; } +function hydrateMessagingConfigForRebuild(sandboxName: string, log: (msg: string) => void): void { + const rebuildSession = onboardSession.loadSession(); + const hydratedMessagingConfig = hydrateMessagingChannelConfig( + getStoredMessagingChannelConfig(sandboxName, rebuildSession), + ); + if (hydratedMessagingConfig) { + log(`Stashed messaging config for rebuild: ${Object.keys(hydratedMessagingConfig).join(",")}`); + } +} + function printRebuildVersionSummary( sandboxName: string, agentName: string, @@ -572,17 +582,7 @@ export async function rebuildSandbox( // destructive. The manifest plan in registry is the durable source; legacy // session channel fields are read only as compatibility fallback by // getStoredMessagingChannelConfig(). - { - const rebuildSession = onboardSession.loadSession(); - const hydratedMessagingConfig = hydrateMessagingChannelConfig( - getStoredMessagingChannelConfig(sandboxName, rebuildSession), - ); - if (hydratedMessagingConfig) { - log( - `Stashed messaging config for rebuild: ${Object.keys(hydratedMessagingConfig).join(",")}`, - ); - } - } + hydrateMessagingConfigForRebuild(sandboxName, log); // Version check — show what's changing const versionCheck = sandboxVersion.checkAgentVersion(sandboxName); From ed3170a0fdce56fa51f1294f36c1100a9f841fa2 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 15 Jun 2026 10:41:20 -0700 Subject: [PATCH 15/23] test(onboard): cover setupNim selection validator state Signed-off-by: Carlos Villela --- src/lib/onboard.ts | 10 +++ src/lib/onboard/setup-nim-selection.test.ts | 80 +++++++++++++++++++++ 2 files changed, 90 insertions(+) diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 5d85a21caf2..afc356655f0 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -3381,6 +3381,11 @@ async function handleVllmSelection( process.exit(1); } + // Source boundary: local vLLM is an external process, so /v1/models can be + // unreachable, malformed, empty, or return an unsafe served id. setupNim is + // the last safe point before writing provider state, so fail closed here + // rather than returning a partially configured local provider. Remove this + // local guard only if the vLLM manager owns a typed, validated model probe. const vllmModelsRaw = runCapture(["curl", "-sf", `http://127.0.0.1:${VLLM_PORT}/v1/models`], { ignoreError: true, }); @@ -3725,6 +3730,11 @@ async function handleRemoteProviderSelection( try { hermesProviderModels = await nousModels.getHermesProviderModelOptions(); } catch (err) { + // Source boundary: Nous model recommendations are advisory network data, + // while the user's requested/default model remains the source of truth + // for onboarding. Keep Hermes auth/tool-gateway state and continue with + // fallback model prompting. Remove this fallback only when the provider + // registry can supply recommendations without network failure modes. const detail = err instanceof Error ? err.message : String(err); console.warn( ` Warning: failed to load Nous model recommendations; falling back to the current/default model (${detail}).`, diff --git a/src/lib/onboard/setup-nim-selection.test.ts b/src/lib/onboard/setup-nim-selection.test.ts index 79f3ef24f49..7d4cab8bef9 100644 --- a/src/lib/onboard/setup-nim-selection.test.ts +++ b/src/lib/onboard/setup-nim-selection.test.ts @@ -8,6 +8,7 @@ import { describe, it } from "vitest"; import { applyCloudFallbackSelection, clearNimContainerBeforeRetry, + createRemoteModelValidator, type SetupNimSelectionState, } from "./setup-nim-selection"; @@ -57,3 +58,82 @@ describe("setupNim selection state helpers", () => { assert.equal(state.provider, "vllm-local"); }); }); + +describe("createRemoteModelValidator", () => { + it("forces custom compatible endpoints to chat completions unless the API is explicit", async () => { + const state = makeState(); + state.provider = "openai-compatible"; + state.endpointUrl = "https://compatible.example/v1"; + state.model = "model-a"; + let calledEndpoint: string | null = null; + const { validateSelectedRemoteModel } = createRemoteModelValidator({ + OPENAI_ENDPOINT_URL: "https://default-openai.example/v1", + ANTHROPIC_ENDPOINT_URL: "https://default-anthropic.example/v1", + requireValue: (value, message) => { + if (value === null || value === undefined) throw new Error(message); + return value; + }, + isBackToSelection: (_value): _value is never => false, + validateCustomOpenAiLikeSelection: async (_label, endpointUrl) => { + calledEndpoint = endpointUrl; + return { ok: true, api: "responses" }; + }, + validateCustomAnthropicSelection: async () => ({ ok: false, retry: "selection" }), + validateAnthropicSelectionWithRetryMessage: async () => ({ ok: false, retry: "selection" }), + validateOpenAiLikeSelection: async () => ({ ok: false, retry: "selection" }), + shouldRequireResponsesToolCalling: () => false, + shouldSkipResponsesProbe: () => false, + getProbeAuthMode: () => undefined, + }); + + const result = await validateSelectedRemoteModel({ + selected: { key: "custom" }, + remoteConfig: { + label: "Other OpenAI-compatible endpoint", + endpointUrl: "https://remote-config.example/v1", + helpUrl: null, + }, + state, + selectedCredentialEnv: "OPENAI_API_KEY", + }); + + assert.equal(result, "selected"); + assert.equal(calledEndpoint, "https://compatible.example/v1"); + assert.equal(state.preferredInferenceApi, "openai-completions"); + }); + + it("maps provider validation model retries without mutating selected model state", async () => { + const state = makeState(); + const { validateSelectedRemoteModel } = createRemoteModelValidator({ + OPENAI_ENDPOINT_URL: "https://default-openai.example/v1", + ANTHROPIC_ENDPOINT_URL: "https://default-anthropic.example/v1", + requireValue: (value, message) => { + if (value === null || value === undefined) throw new Error(message); + return value; + }, + isBackToSelection: (_value): _value is never => false, + validateCustomOpenAiLikeSelection: async () => ({ ok: false, retry: "selection" }), + validateCustomAnthropicSelection: async () => ({ ok: false, retry: "model" }), + validateAnthropicSelectionWithRetryMessage: async () => ({ ok: false, retry: "selection" }), + validateOpenAiLikeSelection: async () => ({ ok: false, retry: "selection" }), + shouldRequireResponsesToolCalling: () => false, + shouldSkipResponsesProbe: () => false, + getProbeAuthMode: () => undefined, + }); + + const result = await validateSelectedRemoteModel({ + selected: { key: "anthropicCompatible" }, + remoteConfig: { + label: "Other Anthropic-compatible endpoint", + endpointUrl: "https://anthropic.example/v1", + helpUrl: null, + }, + state, + selectedCredentialEnv: "ANTHROPIC_API_KEY", + }); + + assert.equal(result, "retry-model"); + assert.equal(state.model, "nvidia/local-nim"); + assert.equal(state.nimContainer, "nemoclaw-nim-test"); + }); +}); From ebe72baa68ba4b25fd2abc34d3a74dab45622ce1 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 15 Jun 2026 11:18:30 -0700 Subject: [PATCH 16/23] test(onboard): align sandbox registration fixture Signed-off-by: Carlos Villela --- src/lib/onboard/sandbox-registration.test.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/lib/onboard/sandbox-registration.test.ts b/src/lib/onboard/sandbox-registration.test.ts index 643f7b27dd2..ac8cf79a0e9 100644 --- a/src/lib/onboard/sandbox-registration.test.ts +++ b/src/lib/onboard/sandbox-registration.test.ts @@ -43,7 +43,6 @@ describe("buildCreatedSandboxRegistryEntry", () => { dashboardPort: 18789, gatewayName: "nemoclaw-19080", gatewayPort: 19080, - providerCredentialHashes: {}, }); expect(entry).toMatchObject({ @@ -91,7 +90,6 @@ describe("buildCreatedSandboxRegistryEntry", () => { dashboardPort: 18789, gatewayName: "nemoclaw", gatewayPort: 8080, - providerCredentialHashes: {}, }); expect(entry.model).toBeNull(); @@ -128,7 +126,6 @@ describe("registerCreatedSandbox", () => { dashboardPort: 18789, gatewayName: "nemoclaw", gatewayPort: 8080, - providerCredentialHashes: {}, registerSandbox, }); From cb696071d251e2bd5d18833c3e2a9f252a41d918 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 15 Jun 2026 12:16:36 -0700 Subject: [PATCH 17/23] test(rebuild): cover active session warning Signed-off-by: Carlos Villela --- src/lib/actions/sandbox/rebuild.ts | 6 ++ test/rebuild-credential-preflight.test.ts | 92 +++++++++++++++++++---- 2 files changed, 82 insertions(+), 16 deletions(-) diff --git a/src/lib/actions/sandbox/rebuild.ts b/src/lib/actions/sandbox/rebuild.ts index 59cc64a475d..540efb40d1c 100644 --- a/src/lib/actions/sandbox/rebuild.ts +++ b/src/lib/actions/sandbox/rebuild.ts @@ -295,6 +295,12 @@ function hookOutputsFromBuildSteps( function countActiveSandboxSessionsForRebuild(sandboxName: string): number { const opsBinRebuild = resolveOpenshell(); + // Source boundary: active-session detection depends on host process listing + // and the OpenShell binary being installed. A failed/unavailable detector is + // not evidence of active sessions, and rebuild's safety preflights still run + // before destructive work. Keep the prior fail-open prompt behavior here; + // remove this fallback only if session detection becomes a required, typed + // OpenShell API that can distinguish "zero sessions" from "unavailable". if (!opsBinRebuild) return 0; try { diff --git a/test/rebuild-credential-preflight.test.ts b/test/rebuild-credential-preflight.test.ts index e5f97dbfecb..394dfadd8db 100644 --- a/test/rebuild-credential-preflight.test.ts +++ b/test/rebuild-credential-preflight.test.ts @@ -80,6 +80,7 @@ function createFixture(opts: { messagingPlanChannels?: string[] | null; dockerBuildExitCode?: number; providerRegistered?: boolean; + activeSessionCount?: number | null; }) { const { sandboxName = "my-assistant", @@ -92,6 +93,7 @@ function createFixture(opts: { messagingPlanChannels = null, dockerBuildExitCode = 0, providerRegistered = true, + activeSessionCount = 0, } = opts; const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-2273-")); tmpFixtures.push(tmpDir); @@ -248,6 +250,21 @@ process.exit(0); { mode: 0o755 }, ); + // ── Fake ps for active SSH session detection ────────────────── + const activeSessionLines = Array.from( + { length: activeSessionCount ?? 0 }, + (_, index) => `${9000 + index} ssh openshell-${sandboxName}`, + ).join("\n"); + fs.writeFileSync( + path.join(tmpDir, "ps"), + `#!/usr/bin/env node +if (${activeSessionCount === null ? "true" : "false"}) process.exit(1); +process.stdout.write(${JSON.stringify(activeSessionLines)} + (${JSON.stringify(activeSessionLines)} ? "\\n" : "")); +process.exit(0); +`, + { mode: 0o755 }, + ); + // ── Fake Docker ─────────────────────────────────────────────── // Hermes rebuild forces a base-image build before backup/delete. // This fixture only exercises rebuild session state, so Docker succeeds. @@ -301,24 +318,24 @@ process.exit(0); function runRebuild( fixture: ReturnType, extraEnv: Record = {}, + options: { yes?: boolean; input?: string } = {}, ) { - return spawnSync( - process.execPath, - [path.join(REPO_ROOT, "bin", "nemoclaw.js"), fixture.sandboxName, "rebuild", "--yes"], - { - cwd: REPO_ROOT, - encoding: "utf-8", - env: { - HOME: fixture.tmpDir, - PATH: fixture.tmpDir + ":" + NODE_BIN + ":/usr/bin:/bin", - NEMOCLAW_NON_INTERACTIVE: "1", - NEMOCLAW_NO_CONNECT_HINT: "1", - NO_COLOR: "1", - ...extraEnv, - }, - timeout: 30_000, + const argv = [path.join(REPO_ROOT, "bin", "nemoclaw.js"), fixture.sandboxName, "rebuild"]; + if (options.yes !== false) argv.push("--yes"); + return spawnSync(process.execPath, argv, { + cwd: REPO_ROOT, + encoding: "utf-8", + input: options.input, + env: { + HOME: fixture.tmpDir, + PATH: fixture.tmpDir + ":" + NODE_BIN + ":/usr/bin:/bin", + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_NO_CONNECT_HINT: "1", + NO_COLOR: "1", + ...extraEnv, }, - ); + timeout: 30_000, + }); } function registryHasSandbox(fixture: ReturnType): boolean { @@ -334,6 +351,49 @@ function registryHasSandbox(fixture: ReturnType): boolean describe("Issue #2273: atomic rebuild", () => { describe("Layer 2: preflight credential check", () => { + it("prints active SSH session warning before interactive confirmation", { + timeout: 60_000, + }, () => { + const f = createFixture({ + activeSessionCount: 2, + savedCredential: { + key: "NVIDIA_INFERENCE_API_KEY", + value: "nvapi-test-key-for-rebuild", + }, + }); + + const result = runRebuild(f, {}, { yes: false, input: "n\n" }); + const output = (result.stderr || "") + (result.stdout || ""); + + expect(result.status).toBe(0); + expect(output).toContain("Active SSH sessions detected (2 connections)"); + expect(output).toContain("terminate all active sessions with a Broken pipe error"); + expect(output).toContain("Proceed? [y/N]:"); + expect(output).toContain("Cancelled."); + expect(output).not.toContain("Backing up sandbox state"); + }); + + it("omits active SSH warning when detection is unavailable", { + timeout: 60_000, + }, () => { + const f = createFixture({ + activeSessionCount: null, + savedCredential: { + key: "NVIDIA_INFERENCE_API_KEY", + value: "nvapi-test-key-for-rebuild", + }, + }); + + const result = runRebuild(f, {}, { yes: false, input: "n\n" }); + const output = (result.stderr || "") + (result.stdout || ""); + + expect(result.status).toBe(0); + expect(output).not.toContain("Active SSH"); + expect(output).toContain("Proceed? [y/N]:"); + expect(output).toContain("Cancelled."); + expect(output).not.toContain("Backing up sandbox state"); + }); + it("aborts rebuild BEFORE destroying sandbox when credential is missing", { timeout: 60_000, }, () => { From 8ec4bd66d1de007182dcc445aeaa20a92751f742 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 15 Jun 2026 14:35:48 -0700 Subject: [PATCH 18/23] fix(rebuild): target legacy local credential bypass --- src/lib/actions/sandbox/rebuild.ts | 13 +++++++---- test/rebuild-credential-preflight.test.ts | 27 +++++++++++++++++++++++ 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/src/lib/actions/sandbox/rebuild.ts b/src/lib/actions/sandbox/rebuild.ts index 6b36f58e710..d0cdd8dc73f 100644 --- a/src/lib/actions/sandbox/rebuild.ts +++ b/src/lib/actions/sandbox/rebuild.ts @@ -139,8 +139,12 @@ function _rebuildLog(msg: string) { /** * Resolve the credential environment variable required to recreate a sandbox. */ +function isLocalInferenceProvider(provider: string | null | undefined): provider is string { + return Boolean(provider && LOCAL_INFERENCE_PROVIDERS.includes(provider)); +} + function getRebuildCredentialEnvFromRegistry(provider: string | null | undefined): string | null { - if (!provider || LOCAL_INFERENCE_PROVIDERS.includes(provider)) { + if (!provider || isLocalInferenceProvider(provider)) { return null; } const remoteConfig = @@ -430,15 +434,16 @@ function preflightRebuildCredentials( const rebuildProvider = sessionMatchesTarget ? session?.provider || sb.provider : sb.provider; if ( - (session?.provider === "ollama-local" || session?.provider === "vllm-local") && + sessionMatchesTarget && + isLocalInferenceProvider(sb.provider) && rebuildCredentialEnv === "OPENAI_API_KEY" ) { console.log( - ` ${D}Note: migrating ${session.provider} sandbox off OPENAI_API_KEY (GH #2519). ` + + ` ${D}Note: migrating ${sb.provider} sandbox off OPENAI_API_KEY (GH #2519). ` + `Local inference does not require a host API key.${R}`, ); log( - `Preflight: legacy ${session.provider} sandbox detected (credentialEnv=OPENAI_API_KEY) — clearing for rebuild`, + `Preflight: legacy ${sb.provider} sandbox detected (credentialEnv=OPENAI_API_KEY) — clearing for rebuild`, ); rebuildCredentialEnv = null; } diff --git a/test/rebuild-credential-preflight.test.ts b/test/rebuild-credential-preflight.test.ts index a1ae4b58581..5babf26b5cc 100644 --- a/test/rebuild-credential-preflight.test.ts +++ b/test/rebuild-credential-preflight.test.ts @@ -582,6 +582,33 @@ describe("Issue #2273: atomic rebuild", () => { expect(output).toContain("Backing up sandbox state"); }, 60_000); + it("does not let a mismatched stale local session bypass the target OPENAI_API_KEY preflight", { + timeout: 60_000, + }, () => { + const f = createFixture({ + provider: "openai-api", + credentialEnv: "OPENAI_API_KEY", + providerRegistered: false, + }); + const sessionPath = path.join(f.nemoclawDir, "onboard-session.json"); + const session = JSON.parse(fs.readFileSync(sessionPath, "utf-8")); + session.sandboxName = "other-local-sandbox"; + session.provider = "ollama-local"; + session.credentialEnv = "OPENAI_API_KEY"; + fs.writeFileSync(sessionPath, JSON.stringify(session), { mode: 0o600 }); + + const result = runRebuild(f); + const output = (result.stderr || "") + (result.stdout || ""); + + expect(result.status).not.toBe(0); + expect(output).toContain("preflight failed"); + expect(output).toContain("requires OPENAI_API_KEY"); + expect(output).not.toContain("GH #2519"); + expect(output).not.toContain("Backing up sandbox state"); + expect(output).not.toContain("Old sandbox deleted"); + expect(registryHasSandbox(f)).toBe(true); + }); + it("preflight works for non-NVIDIA providers (OpenAI, Anthropic, etc.)", { timeout: 60_000, }, () => { From dfe58bf9f6a5b72f0961285a258e0ae61ec8a970 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 15 Jun 2026 14:45:02 -0700 Subject: [PATCH 19/23] test(rebuild): cover messaging staging preflight --- src/lib/actions/sandbox/rebuild-flow.test.ts | 29 ++++++++++++++++++++ src/lib/actions/sandbox/rebuild.ts | 9 ++++++ 2 files changed, 38 insertions(+) diff --git a/src/lib/actions/sandbox/rebuild-flow.test.ts b/src/lib/actions/sandbox/rebuild-flow.test.ts index 27957ca4482..49de9fcd09c 100644 --- a/src/lib/actions/sandbox/rebuild-flow.test.ts +++ b/src/lib/actions/sandbox/rebuild-flow.test.ts @@ -24,6 +24,7 @@ type RebuildFlowOverrides = { failedDirs: string[]; failedFiles: string[]; }; + buildMessagingRebuildPlan?: () => Promise | unknown; }; type RebuildFlowHarness = { @@ -38,6 +39,7 @@ type RebuildFlowHarness = { relockSpy: MockInstance; restoreSandboxStateSpy: MockInstance; runOpenshellSpy: MockInstance; + messagingRebuildPlanSpy: MockInstance; }; const originalSandboxName = process.env.NEMOCLAW_SANDBOX_NAME; @@ -66,6 +68,7 @@ function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): Rebuild const nim = requireDist("../../../../dist/lib/inference/nim.js"); const policies = requireDist("../../../../dist/lib/policy/index.js"); const processRecovery = requireDist("../../../../dist/lib/actions/sandbox/process-recovery.js"); + const messaging = requireDist("../../../../dist/lib/messaging/index.js"); const shields = requireDist("../../../../dist/lib/shields/index.js"); const session = { @@ -167,6 +170,9 @@ function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): Rebuild vi.spyOn(shields, "repairMutableConfigPerms").mockImplementation( overrides.repairMutableConfigPerms ?? (() => ({ applied: true, verified: true, errors: [] })), ); + const messagingRebuildPlanSpy = vi + .spyOn(messaging.MessagingWorkflowPlanner.prototype, "buildRebuildPlanFromSandboxEntry") + .mockImplementation(overrides.buildMessagingRebuildPlan ?? (() => null)); errorSpy.mockClear(); logSpy.mockClear(); @@ -184,6 +190,7 @@ function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): Rebuild relockSpy, restoreSandboxStateSpy, runOpenshellSpy, + messagingRebuildPlanSpy, }; } @@ -240,6 +247,28 @@ describe("rebuildSandbox flow", () => { ); }); + it("aborts before backup/delete when messaging manifest staging fails", async () => { + const harness = createRebuildFlowHarness({ + buildMessagingRebuildPlan: () => { + throw new Error("manifest boom"); + }, + }); + + await expect(harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true })).rejects.toThrow( + "manifest boom", + ); + + const errors = harness.errorSpy.mock.calls.map((call) => String(call[0])).join("\n"); + expect(errors).toContain("messaging manifest plan could not be staged"); + expect(errors).toContain("Sandbox is untouched"); + expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled(); + expect(harness.runOpenshellSpy).not.toHaveBeenCalledWith( + ["sandbox", "delete", "alpha"], + expect.anything(), + ); + expect(harness.onboardSpy).not.toHaveBeenCalled(); + }); + it("finishes the rebuild while surfacing incomplete post-restore work", async () => { const harness = createRebuildFlowHarness({ executeSandboxCommand: () => ({ status: 1, stdout: "", stderr: "hash refresh failed" }), diff --git a/src/lib/actions/sandbox/rebuild.ts b/src/lib/actions/sandbox/rebuild.ts index d0cdd8dc73f..b2eceb8f36d 100644 --- a/src/lib/actions/sandbox/rebuild.ts +++ b/src/lib/actions/sandbox/rebuild.ts @@ -398,6 +398,10 @@ async function stageRebuildMessagingPlanOrBail( try { return await stageMessagingManifestPlanForRebuild(sandboxName, sb, rebuildAgent, log); } catch (err) { + // Source boundary: registry messaging plans and agent manifests are durable + // host-side inputs from prior onboarding. If they drift or become invalid, + // rebuild must fail here before backup/delete; remove this boundary only if + // manifest staging becomes total over all persisted registry states. const message = err instanceof Error ? err.message : String(err); console.error(""); console.error( @@ -433,6 +437,11 @@ function preflightRebuildCredentials( } const rebuildProvider = sessionMatchesTarget ? session?.provider || sb.provider : sb.provider; + // Compatibility boundary for GH #2519: pre-fix local-provider sessions could + // persist credentialEnv="OPENAI_API_KEY" even though current local-provider + // write paths persist null. Only a session for this sandbox plus a local + // target registry provider may bypass the key; keep until legacy sessions are + // no longer supported by rebuild migration tests. if ( sessionMatchesTarget && isLocalInferenceProvider(sb.provider) && From 8eca2e22b35c83c5e2d4f639f065dd9bc4e792be Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 15 Jun 2026 14:56:04 -0700 Subject: [PATCH 20/23] chore(rebuild): apply formatter --- src/lib/actions/sandbox/rebuild-flow.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib/actions/sandbox/rebuild-flow.test.ts b/src/lib/actions/sandbox/rebuild-flow.test.ts index 49de9fcd09c..f68c09b90b1 100644 --- a/src/lib/actions/sandbox/rebuild-flow.test.ts +++ b/src/lib/actions/sandbox/rebuild-flow.test.ts @@ -254,9 +254,9 @@ describe("rebuildSandbox flow", () => { }, }); - await expect(harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true })).rejects.toThrow( - "manifest boom", - ); + await expect( + harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).rejects.toThrow("manifest boom"); const errors = harness.errorSpy.mock.calls.map((call) => String(call[0])).join("\n"); expect(errors).toContain("messaging manifest plan could not be staged"); From 1d8239eac9432a6c2c393c4ad6681b24d0676baa Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 15 Jun 2026 15:12:16 -0700 Subject: [PATCH 21/23] fix(rebuild): fail closed on missing remote session credential --- src/lib/actions/sandbox/rebuild.ts | 5 ++++- test/rebuild-credential-preflight.test.ts | 24 +++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/src/lib/actions/sandbox/rebuild.ts b/src/lib/actions/sandbox/rebuild.ts index b2eceb8f36d..d7d2fe82dda 100644 --- a/src/lib/actions/sandbox/rebuild.ts +++ b/src/lib/actions/sandbox/rebuild.ts @@ -423,8 +423,11 @@ function preflightRebuildCredentials( ): boolean { const session = onboardSession.loadSession(); const sessionMatchesTarget = session?.sandboxName === sandboxName; + // The target registry entry is authoritative when a matching legacy session + // omitted credentialEnv; rebuild rewrites provider/model from this entry later, + // so remote registry providers must still fail closed before backup/delete. let rebuildCredentialEnv = sessionMatchesTarget - ? session?.credentialEnv || null + ? session?.credentialEnv || getRebuildCredentialEnvFromRegistry(sb.provider) : getRebuildCredentialEnvFromRegistry(sb.provider); if (!sessionMatchesTarget && session?.sandboxName) { log( diff --git a/test/rebuild-credential-preflight.test.ts b/test/rebuild-credential-preflight.test.ts index 5babf26b5cc..a36ff8bf2a0 100644 --- a/test/rebuild-credential-preflight.test.ts +++ b/test/rebuild-credential-preflight.test.ts @@ -582,6 +582,30 @@ describe("Issue #2273: atomic rebuild", () => { expect(output).toContain("Backing up sandbox state"); }, 60_000); + it("fails closed when a matching session omits the remote target provider credential", { + timeout: 60_000, + }, () => { + const f = createFixture({ + provider: "openai-api", + credentialEnv: "OPENAI_API_KEY", + providerRegistered: false, + }); + const sessionPath = path.join(f.nemoclawDir, "onboard-session.json"); + const session = JSON.parse(fs.readFileSync(sessionPath, "utf-8")); + session.credentialEnv = null; + fs.writeFileSync(sessionPath, JSON.stringify(session), { mode: 0o600 }); + + const result = runRebuild(f); + const output = (result.stderr || "") + (result.stdout || ""); + + expect(result.status).not.toBe(0); + expect(output).toContain("preflight failed"); + expect(output).toContain("requires OPENAI_API_KEY"); + expect(output).not.toContain("Backing up sandbox state"); + expect(output).not.toContain("Old sandbox deleted"); + expect(registryHasSandbox(f)).toBe(true); + }); + it("does not let a mismatched stale local session bypass the target OPENAI_API_KEY preflight", { timeout: 60_000, }, () => { From 804dc164110a262bc441ce8447ff421cde94eb62 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 15 Jun 2026 15:19:30 -0700 Subject: [PATCH 22/23] fix(rebuild): use registry provider for credential preflight --- src/lib/actions/sandbox/rebuild.ts | 2 +- test/rebuild-credential-preflight.test.ts | 34 ++++++++++++++++++++++- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/src/lib/actions/sandbox/rebuild.ts b/src/lib/actions/sandbox/rebuild.ts index d7d2fe82dda..bd341522b9c 100644 --- a/src/lib/actions/sandbox/rebuild.ts +++ b/src/lib/actions/sandbox/rebuild.ts @@ -439,7 +439,7 @@ function preflightRebuildCredentials( ); } - const rebuildProvider = sessionMatchesTarget ? session?.provider || sb.provider : sb.provider; + const rebuildProvider = sb.provider; // Compatibility boundary for GH #2519: pre-fix local-provider sessions could // persist credentialEnv="OPENAI_API_KEY" even though current local-provider // write paths persist null. Only a session for this sandbox plus a local diff --git a/test/rebuild-credential-preflight.test.ts b/test/rebuild-credential-preflight.test.ts index a36ff8bf2a0..427b6538c31 100644 --- a/test/rebuild-credential-preflight.test.ts +++ b/test/rebuild-credential-preflight.test.ts @@ -80,6 +80,7 @@ function createFixture(opts: { messagingPlanChannels?: string[] | null; dockerBuildExitCode?: number; providerRegistered?: boolean; + registeredProviders?: string[]; activeSessionCount?: number | null; }) { const { @@ -93,6 +94,7 @@ function createFixture(opts: { messagingPlanChannels = null, dockerBuildExitCode = 0, providerRegistered = true, + registeredProviders, activeSessionCount = 0, } = opts; const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-2273-")); @@ -230,10 +232,12 @@ function createFixture(opts: { " UserKnownHostsFile /dev/null", ].join("\\n"); + const registeredProvidersLiteral = JSON.stringify(registeredProviders ?? null); fs.writeFileSync( path.join(tmpDir, "openshell"), `#!/usr/bin/env node const a = process.argv.slice(2); +const registeredProviders = ${registeredProvidersLiteral}; if (a[0]==="sandbox" && a[1]==="list") { process.stdout.write("${sandboxName}\\n"); process.exit(0); } if (a[0]==="sandbox" && a[1]==="ssh-config") { process.stdout.write("${sshConfig}\\n"); process.exit(0); } if (a[0]==="sandbox" && a[1]==="delete") { process.exit(0); } @@ -242,7 +246,10 @@ if (a[0]==="gateway" && a[1]==="info") { process.stdout.write("nemoclaw\\n if (a[0]==="gateway" && a[1]==="select") { process.exit(0); } if (a[0]==="inference" && a[1]==="get") { process.stdout.write('{"provider":"${provider}","model":"meta/llama-3.3-70b-instruct"}\\n'); process.exit(0); } if (a[0]==="inference" && a[1]==="set") { process.exit(0); } -if (a[0]==="provider" && a[1]==="get") { process.exit(${providerRegistered ? 0 : 1}); } +if (a[0]==="provider" && a[1]==="get") { + if (Array.isArray(registeredProviders)) process.exit(registeredProviders.includes(a[2]) ? 0 : 1); + process.exit(${providerRegistered ? 0 : 1}); +} if (a[0]==="provider") { process.exit(0); } if (a[0]==="forward") { process.exit(0); } process.exit(0); @@ -606,6 +613,31 @@ describe("Issue #2273: atomic rebuild", () => { expect(registryHasSandbox(f)).toBe(true); }); + it("uses the target registry provider when a matching session has a stale registered provider", { + timeout: 60_000, + }, () => { + const f = createFixture({ + provider: "openai-api", + credentialEnv: "OPENAI_API_KEY", + registeredProviders: ["nvidia-prod"], + }); + const sessionPath = path.join(f.nemoclawDir, "onboard-session.json"); + const session = JSON.parse(fs.readFileSync(sessionPath, "utf-8")); + session.provider = "nvidia-prod"; + session.credentialEnv = null; + fs.writeFileSync(sessionPath, JSON.stringify(session), { mode: 0o600 }); + + const result = runRebuild(f); + const output = (result.stderr || "") + (result.stdout || ""); + + expect(result.status).not.toBe(0); + expect(output).toContain("preflight failed"); + expect(output).toContain("requires OPENAI_API_KEY"); + expect(output).not.toContain("Backing up sandbox state"); + expect(output).not.toContain("Old sandbox deleted"); + expect(registryHasSandbox(f)).toBe(true); + }); + it("does not let a mismatched stale local session bypass the target OPENAI_API_KEY preflight", { timeout: 60_000, }, () => { From af3257b0c98c2028776c3a93be46439bdc0e3d6f Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 15 Jun 2026 16:44:47 -0700 Subject: [PATCH 23/23] test(snapshot): cover create failure branches --- src/lib/actions/sandbox/snapshot.test.ts | 77 +++++++++++++++++++++++- src/lib/actions/sandbox/snapshot.ts | 8 ++- 2 files changed, 80 insertions(+), 5 deletions(-) diff --git a/src/lib/actions/sandbox/snapshot.test.ts b/src/lib/actions/sandbox/snapshot.test.ts index 2dd22b61e16..5313739ecde 100644 --- a/src/lib/actions/sandbox/snapshot.test.ts +++ b/src/lib/actions/sandbox/snapshot.test.ts @@ -3,9 +3,22 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +const shieldsMock = vi.hoisted(() => { + const isShieldsDownMock = vi.fn(() => true); + let isShieldsDownExport: unknown = isShieldsDownMock; + return { + isShieldsDownMock, + getIsShieldsDownExport: () => isShieldsDownExport, + setIsShieldsDownExport: (value: unknown) => { + isShieldsDownExport = value; + }, + }; +}); + const backupSandboxStateMock = vi.fn(); const captureOpenshellMock = vi.fn(() => ({ status: 0, output: "alpha Ready\n" })); const dockerInspectMock = vi.fn(() => ({ status: 0, stdout: "true\n" })); +const findBackupMock = vi.fn(); const getSandboxMock = vi.fn(() => null); const isGatewayHealthyMock = vi.fn(() => true); const parseLiveSandboxNamesMock = vi.fn(() => new Set(["alpha"])); @@ -43,7 +56,9 @@ vi.mock("../../runtime-recovery", () => ({ })); vi.mock("../../shields", () => ({ - isShieldsDown: undefined, + get isShieldsDown() { + return shieldsMock.getIsShieldsDownExport(); + }, })); vi.mock("../../state/gateway", () => ({ @@ -58,7 +73,7 @@ vi.mock("../../state/registry", () => ({ vi.mock("../../state/sandbox", () => ({ backupSandboxState: backupSandboxStateMock, - findBackup: vi.fn(), + findBackup: findBackupMock, listBackups: vi.fn(() => []), })); @@ -70,8 +85,11 @@ vi.mock("./destroy", () => ({ describe("runSandboxSnapshot", () => { beforeEach(() => { vi.clearAllMocks(); + shieldsMock.setIsShieldsDownExport(shieldsMock.isShieldsDownMock); + shieldsMock.isShieldsDownMock.mockReturnValue(true); captureOpenshellMock.mockReturnValue({ status: 0, output: "alpha Ready\n" }); dockerInspectMock.mockReturnValue({ status: 0, stdout: "true\n" }); + findBackupMock.mockReturnValue({ match: null }); getSandboxMock.mockReturnValue(null); isGatewayHealthyMock.mockReturnValue(true); parseLiveSandboxNamesMock.mockReturnValue(new Set(["alpha"])); @@ -82,6 +100,7 @@ describe("runSandboxSnapshot", () => { }); it("refuses snapshot creation before backup when the shields gate helper is unavailable", async () => { + shieldsMock.setIsShieldsDownExport(undefined); const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}); const { runSandboxSnapshot } = await import("./snapshot"); @@ -94,4 +113,58 @@ describe("runSandboxSnapshot", () => { "Cannot verify shields state. Refusing to create snapshot.", ); }); + + it("refuses snapshot creation before backup when the sandbox is not live", async () => { + parseLiveSandboxNamesMock.mockReturnValue(new Set(["beta"])); + const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}); + const { runSandboxSnapshot } = await import("./snapshot"); + + await expect(runSandboxSnapshot("alpha", { kind: "create" })).rejects.toMatchObject({ + exitCode: 1, + }); + + expect(backupSandboxStateMock).not.toHaveBeenCalled(); + expect(consoleError.mock.calls.flat().join("\n")).toContain( + "Sandbox 'alpha' is not running. Cannot create snapshot.", + ); + }); + + it("prints backup error details when snapshot creation fails with an error", async () => { + backupSandboxStateMock.mockReturnValue({ + success: false, + error: "tar exploded", + failedDirs: [], + failedFiles: [], + }); + const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}); + vi.spyOn(console, "log").mockImplementation(() => {}); + const { runSandboxSnapshot } = await import("./snapshot"); + + await expect(runSandboxSnapshot("alpha", { kind: "create" })).rejects.toMatchObject({ + exitCode: 1, + }); + + expect(backupSandboxStateMock).toHaveBeenCalledWith("alpha", { name: null }); + expect(consoleError.mock.calls.flat().join("\n")).toContain("tar exploded"); + }); + + it("prints failed dirs and files when snapshot creation fails without an error", async () => { + backupSandboxStateMock.mockReturnValue({ + success: false, + failedDirs: ["workspace", "skills"], + failedFiles: ["openclaw.json"], + }); + const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}); + vi.spyOn(console, "log").mockImplementation(() => {}); + const { runSandboxSnapshot } = await import("./snapshot"); + + await expect(runSandboxSnapshot("alpha", { kind: "create" })).rejects.toMatchObject({ + exitCode: 1, + }); + + const errors = consoleError.mock.calls.flat().join("\n"); + expect(errors).toContain("Snapshot failed."); + expect(errors).toContain("Failed directories: workspace, skills"); + expect(errors).toContain("Failed files: openclaw.json"); + }); }); diff --git a/src/lib/actions/sandbox/snapshot.ts b/src/lib/actions/sandbox/snapshot.ts index 0f4dcadd217..62bc1e1d5a1 100644 --- a/src/lib/actions/sandbox/snapshot.ts +++ b/src/lib/actions/sandbox/snapshot.ts @@ -328,9 +328,11 @@ function verifyRestoreDestinationOnOwnGateway(targetSandbox: string): void { } function isSnapshotCreationAllowedByShields(sandboxName: string): boolean { - // Snapshot creation is a shields/policy boundary. If a packaged or mocked - // CommonJS interop surface ever omits the helper, fail closed before any - // backup side effect instead of throwing an ambiguous TypeError. + // Snapshot creation is a shields/policy boundary. Production builds should + // always export this helper, but stale compiled artifacts, package-boundary + // skew, or test doubles can present a missing CommonJS interop surface. There + // is no safe runtime source fix once snapshot creation has started, so keep + // this as permanent defense-in-depth and fail closed before backup side effects. const isShieldsDown = shields.isShieldsDown; if (typeof isShieldsDown !== "function") { console.error(" Cannot verify shields state. Refusing to create snapshot.");