diff --git a/biome.json b/biome.json index 86aa98a855d..2e36b633f70 100644 --- a/biome.json +++ b/biome.json @@ -86,7 +86,7 @@ "noExcessiveCognitiveComplexity": { "level": "error", "options": { - "maxAllowedComplexity": 184 + "maxAllowedComplexity": 149 } } }, diff --git a/src/lib/actions/sandbox/rebuild-flow-helpers.ts b/src/lib/actions/sandbox/rebuild-flow-helpers.ts new file mode 100644 index 00000000000..0b2f92b2795 --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-flow-helpers.ts @@ -0,0 +1,224 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + detectOpenShellStateRpcResultIssue, + printOpenShellStateRpcIssue, +} from "../../adapters/openshell/gateway-drift"; +import { ensureAgentBaseImage } from "../../agent/onboard"; +import { RD as _RD, G, R, YW } from "../../cli/terminal-style"; +import { getNamedGatewayLifecycleState } from "../../gateway-runtime-action"; +import { + captureSandboxListWithGatewayRecovery, + printSandboxListFailureWithRecoveryContext, +} from "../../openshell-sandbox-list"; +import { parseLiveSandboxNames } from "../../runtime-recovery"; +import * as shields from "../../shields"; +import * as registry from "../../state/registry"; +import * as sandboxState from "../../state/sandbox"; +import { loadAgent } from "../../agent/defs"; +import { CLI_NAME } from "../../cli/branding"; +import { resolveSandboxGatewayName } from "../../onboard/gateway-binding"; +import { + getReconciledSandboxGatewayState, + printGatewayLifecycleHint, + printWrongGatewayActiveGuidance, +} from "./gateway-state"; +import { openRebuildShieldsWindow, type RebuildShieldsWindow } from "./rebuild-shields"; + +export type RebuildSandboxEntry = registry.SandboxEntry & { agents?: unknown[] }; + +export type RebuildLiveState = { + staleRecovery: boolean; + staleRegistrySnapshot: ReturnType | null; +}; + +export async function resolveRebuildLiveState( + sandboxName: string, + sb: RebuildSandboxEntry, + log: (msg: string) => void, + bail: (msg: string, code?: number) => never, +): Promise { + const recordedGateway = resolveSandboxGatewayName(sb); + log(`Checking sandbox liveness on ${recordedGateway}: openshell sandbox list`); + const liveRecovery = await captureSandboxListWithGatewayRecovery({ + gatewayName: recordedGateway, + }); + const isLive = liveRecovery.result; + log( + `openshell sandbox list exit=${isLive.status}, output=${(isLive.output || "").substring(0, 200)}`, + ); + const liveListIssue = detectOpenShellStateRpcResultIssue(isLive); + if (liveListIssue) { + printOpenShellStateRpcIssue(liveListIssue, { + action: `rebuilding sandbox '${sandboxName}'`, + command: `${CLI_NAME} ${sandboxName} rebuild`, + }); + bail("OpenShell gateway schema mismatch."); + return null; + } + if (isLive.status !== 0) { + printSandboxListFailureWithRecoveryContext(liveRecovery); + bail("Failed to query running sandboxes from OpenShell.", isLive.status || 1); + return null; + } + + const liveNames = parseLiveSandboxNames(isLive.output || ""); + log(`Live sandboxes: ${Array.from(liveNames).join(", ") || "(none)"}`); + if (liveNames.has(sandboxName)) return { staleRecovery: false, staleRegistrySnapshot: null }; + + const reconciled = await getReconciledSandboxGatewayState(sandboxName); + if (reconciled.state === "present") { + const lifecycle = getNamedGatewayLifecycleState(recordedGateway); + if (lifecycle.state !== "healthy_named") { + printWrongGatewayActiveGuidance( + sandboxName, + lifecycle.activeGateway, + console.error, + "rebuild --yes", + ); + bail( + `Could not confirm '${sandboxName}' against gateway '${recordedGateway}' (gateway '${lifecycle.activeGateway ?? "unknown"}' is active).`, + ); + return null; + } + log("Sandbox live on the healthy named gateway; using normal rebuild path"); + return { staleRecovery: false, staleRegistrySnapshot: null }; + } + + if (reconciled.state === "missing") { + // Source boundary: the local registry is the durable NemoClaw intent record, + // while OpenShell owns live sandbox presence. A missing live sandbox on a + // healthy named gateway can come from external deletion or failed prior + // provisioning, so rebuild recovers from registry metadata instead of + // treating the preserved local entry as corrupt. Keep until OpenShell exposes + // an atomic recreate-from-registry recovery API. + console.log(""); + console.log( + ` ${YW}⚠${R} Sandbox '${sandboxName}' is registered locally but absent from the live OpenShell gateway.`, + ); + console.log( + " No live workspace state to back up — recreating from the preserved registry metadata.", + ); + log( + "Stale-sandbox recovery: live sandbox missing on healthy named gateway; skipping backup/restore and recreating from registry metadata", + ); + return { + staleRecovery: true, + staleRegistrySnapshot: JSON.parse(JSON.stringify(registry.load())), + }; + } + + if (reconciled.state === "gateway_schema_mismatch") { + console.error(reconciled.output); + bail("OpenShell gateway schema mismatch."); + return null; + } + + if (reconciled.state === "wrong_gateway_active") { + printWrongGatewayActiveGuidance( + sandboxName, + reconciled.activeGateway, + console.error, + "rebuild --yes", + ); + } else { + console.error( + ` Sandbox '${sandboxName}' is not visible on gateway '${recordedGateway}' and its live state could not be confirmed.`, + ); + console.error(" Your local registry entry has been preserved — nothing was removed."); + printGatewayLifecycleHint(reconciled.output || "", sandboxName, console.error); + } + bail(`Could not confirm live state of '${sandboxName}' (gateway not in a known-good state).`); + return null; +} + +export function openRebuildShieldsWindowForState( + sandboxName: string, + staleRecovery: boolean, +): { rebuildShieldsWindow: RebuildShieldsWindow | null; staleSandboxWasLocked: boolean } { + if (staleRecovery) { + return { + staleSandboxWasLocked: !shields.isShieldsDown(sandboxName), + rebuildShieldsWindow: { relocked: false, wasLocked: false }, + }; + } + return { + staleSandboxWasLocked: false, + rebuildShieldsWindow: openRebuildShieldsWindow(sandboxName, CLI_NAME), + }; +} + +export function ensureRebuildAgentBaseImage( + rebuildAgent: string | null, + bail: (msg: string, code?: number) => never, +): boolean { + if (!rebuildAgent) return true; + const agentDef = loadAgent(rebuildAgent); + try { + ensureAgentBaseImage(agentDef, { forceBaseImageRebuild: true }); + return true; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + console.error(""); + console.error(` ${_RD}Rebuild preflight failed:${R} agent base image could not be built.`); + console.error(` ${message}`); + console.error(""); + console.error(" Sandbox is untouched — no data was lost."); + bail(message); + return false; + } +} + +export function backupSandboxStateForRebuild( + sandboxName: string, + sb: RebuildSandboxEntry, + staleRecovery: boolean, + log: (msg: string) => void, + relockShieldsIfNeeded: (sandboxStillExists: boolean) => boolean, + bail: (msg: string, code?: number) => never, +): sandboxState.RebuildManifest | null | undefined { + if (staleRecovery) return null; + + console.log(" Backing up sandbox state..."); + log(`Agent type: ${sb.agent || "openclaw"}, stateDirs from manifest`); + const backup = sandboxState.backupSandboxState(sandboxName); + log( + `Backup result: success=${backup.success}, backed=${backup.backedUpDirs.join(",")}; files=${backup.backedUpFiles.join(",")}, failed=${backup.failedDirs.join(",")}; failedFiles=${backup.failedFiles.join(",")}`, + ); + const hasAnyBackup = backup.backedUpDirs.length > 0 || backup.backedUpFiles.length > 0; + if (!backup.success && !hasAnyBackup) { + console.error(" Failed to back up sandbox state."); + if (backup.failedDirs.length > 0) console.error(` Failed: ${backup.failedDirs.join(", ")}`); + if (backup.failedFiles.length > 0) + console.error(` Failed files: ${backup.failedFiles.join(", ")}`); + console.error(" Aborting rebuild to prevent data loss."); + relockShieldsIfNeeded(true); + bail("Failed to back up sandbox state."); + return undefined; + } + const backupManifest = backup.manifest ?? null; + if (!backupManifest) { + console.error(" Failed to record backup metadata."); + console.error(" Aborting rebuild to prevent data loss."); + relockShieldsIfNeeded(true); + bail("Failed to record backup metadata."); + return undefined; + } + if (!backup.success) { + console.warn( + ` ${YW}⚠${R} Partial backup: ${backup.backedUpDirs.length} dirs and ${backup.backedUpFiles.length} files OK; ${backup.failedDirs.length} dirs and ${backup.failedFiles.length} files failed`, + ); + if (backup.failedDirs.length > 0) + console.warn(` Failed dirs: ${backup.failedDirs.join(", ")}`); + if (backup.failedFiles.length > 0) + console.warn(` Failed files: ${backup.failedFiles.join(", ")}`); + console.warn(" Rebuild will continue — failed state could not be preserved."); + } else { + console.log( + ` ${G}✓${R} State backed up (${backup.backedUpDirs.length} directories, ${backup.backedUpFiles.length} files)`, + ); + } + console.log(` Backup: ${backupManifest.backupPath}`); + return backupManifest; +} diff --git a/src/lib/actions/sandbox/rebuild.ts b/src/lib/actions/sandbox/rebuild.ts index 239c8b56a69..dbd6f023ac3 100644 --- a/src/lib/actions/sandbox/rebuild.ts +++ b/src/lib/actions/sandbox/rebuild.ts @@ -3,7 +3,6 @@ import { CLI_NAME } from "../../cli/branding"; import { prompt as askPrompt } from "../../credentials/store"; -import { resolveSandboxGatewayName } from "../../onboard/gateway-binding"; import { normalizeRebuildSandboxOptions, type RebuildSandboxOptions, @@ -32,17 +31,14 @@ const { LOCAL_INFERENCE_PROVIDERS, REMOTE_PROVIDER_CONFIG, providerExistsInGatew import { detectOpenShellStateRpcPreflightIssue, - detectOpenShellStateRpcResultIssue, printOpenShellStateRpcIssue, } from "../../adapters/openshell/gateway-drift"; import { resolveOpenshell } from "../../adapters/openshell/resolve"; import { runOpenshell } from "../../adapters/openshell/runtime"; import { loadAgent } from "../../agent/defs"; -import { ensureAgentBaseImage } from "../../agent/onboard"; import * as agentRuntime from "../../agent/runtime"; import { RD as _RD, B, D, G, R, YW } from "../../cli/terminal-style"; import { getSandboxDeleteOutcome } from "../../domain/sandbox/destroy"; -import { getNamedGatewayLifecycleState } from "../../gateway-runtime-action"; import * as nim from "../../inference/nim"; import type { MessagingHookApplyRequest, @@ -59,13 +55,8 @@ import { import { hydrateMessagingChannelConfig } from "../../messaging-channel-config"; import { getStoredMessagingChannelConfig } from "../../onboard/messaging-config"; import { pruneDisabledMessagingPolicyPresets } from "../../onboard/messaging-policy-presets"; -import { - captureSandboxListWithGatewayRecovery, - printSandboxListFailureWithRecoveryContext, -} from "../../openshell-sandbox-list"; import * as policies from "../../policy"; import { shellQuote } from "../../runner"; -import { parseLiveSandboxNames } from "../../runtime-recovery"; import * as sandboxVersion from "../../sandbox/version"; import { redact } from "../../security/redact"; import * as shields from "../../shields"; @@ -78,19 +69,16 @@ import { getActiveSandboxSessions, } from "../../state/sandbox-session"; import { removeSandboxRegistryEntry } from "./destroy"; -import { - getReconciledSandboxGatewayState, - printGatewayLifecycleHint, - printWrongGatewayActiveGuidance, -} from "./gateway-state"; import { executeSandboxCommand } from "./process-recovery"; import { buildRebuildRecreateOnboardOpts } from "./rebuild-gpu-opt-out"; import { - openRebuildShieldsWindow, - printRebuildShieldsRecovery, - type RebuildShieldsWindow, - relockRebuildShieldsWindow, -} from "./rebuild-shields"; + backupSandboxStateForRebuild, + ensureRebuildAgentBaseImage, + openRebuildShieldsWindowForState, + resolveRebuildLiveState, + type RebuildSandboxEntry, +} from "./rebuild-flow-helpers"; +import { printRebuildShieldsRecovery, relockRebuildShieldsWindow } from "./rebuild-shields"; export function buildRefreshMutableOpenClawConfigHashCommand( configDir = "/sandbox/.openclaw", @@ -268,8 +256,6 @@ async function stageMessagingManifestPlanForRebuild( return plan; } -type RebuildSandboxEntry = registry.SandboxEntry & { agents?: unknown[] }; - const runMessagingOpenshell: MessagingOpenShellRunner = (args, options = {}) => runOpenshell([...args], { env: options.env as NodeJS.ProcessEnv | undefined, @@ -631,143 +617,22 @@ export async function rebuildSandbox( bail, ); - // Step 1: Ensure sandbox is live for backup - const recordedGateway = resolveSandboxGatewayName(sb); - log(`Checking sandbox liveness on ${recordedGateway}: openshell sandbox list`); - const liveRecovery = await captureSandboxListWithGatewayRecovery({ - gatewayName: recordedGateway, - }); - const isLive = liveRecovery.result; - log( - `openshell sandbox list exit=${isLive.status}, output=${(isLive.output || "").substring(0, 200)}`, - ); - const liveListIssue = detectOpenShellStateRpcResultIssue(isLive); - if (liveListIssue) { - printOpenShellStateRpcIssue(liveListIssue, { - action: `rebuilding sandbox '${sandboxName}'`, - command: `${CLI_NAME} ${sandboxName} rebuild`, - }); - bail("OpenShell gateway schema mismatch."); - return; - } - if (isLive.status !== 0) { - printSandboxListFailureWithRecoveryContext(liveRecovery); - bail("Failed to query running sandboxes from OpenShell.", isLive.status || 1); - return; - } - const liveNames = parseLiveSandboxNames(isLive.output || ""); - log(`Live sandboxes: ${Array.from(liveNames).join(", ") || "(none)"}`); - // Stale-sandbox recovery (#4497): the local registry still holds this entry, - // but the live OpenShell gateway no longer lists the sandbox — a stuck or - // diverged provision, or the live container was reaped. This is exactly the - // state `status` flags with a `rebuild --yes` hint, and the non-destructive - // `connect` recovery now preserves the registry entry so this rebuild can act - // on it. There is no live workspace state to back up, so rather than aborting - // with "Cannot back up state" — which dead-ended the very recovery path the - // CLI recommended — skip the backup/restore steps and recreate the sandbox - // from the preserved registry + onboard-session metadata. - let staleRecovery = false; - // Full registry snapshot captured before stale recovery touches anything - // destructive. If the recovery recreate fails after the entry was removed, - // the snapshot is restored verbatim so no local metadata (defaultSandbox, - // customPolicies, every field) is silently dropped or mutated (#4497). - let staleRegistrySnapshot: ReturnType | null = null; - if (!liveNames.has(sandboxName)) { - const reconciled = await getReconciledSandboxGatewayState(sandboxName); - if (reconciled.state === "present") { - const lifecycle = getNamedGatewayLifecycleState(recordedGateway); - if (lifecycle.state !== "healthy_named") { - printWrongGatewayActiveGuidance( - sandboxName, - lifecycle.activeGateway, - console.error, - "rebuild --yes", - ); - bail( - `Could not confirm '${sandboxName}' against gateway '${recordedGateway}' (gateway '${lifecycle.activeGateway ?? "unknown"}' is active).`, - ); - return; - } - // Live on the healthy named gateway (the list omitted it). Fall through to - // the normal backup/rebuild path so workspace state is preserved. - log("Sandbox live on the healthy named gateway; using normal rebuild path"); - } else if (reconciled.state === "missing") { - // Genuinely absent on a healthy named gateway — stale-sandbox recovery. - staleRecovery = true; - staleRegistrySnapshot = JSON.parse(JSON.stringify(registry.load())); - console.log(""); - console.log( - ` ${YW}⚠${R} Sandbox '${sandboxName}' is registered locally but absent from the live OpenShell gateway.`, - ); - console.log( - " No live workspace state to back up — recreating from the preserved registry metadata.", - ); - log( - "Stale-sandbox recovery: live sandbox missing on healthy named gateway; skipping backup/restore and recreating from registry metadata", - ); - } else if (reconciled.state === "gateway_schema_mismatch") { - console.error(reconciled.output); - bail("OpenShell gateway schema mismatch."); - return; - } else { - // Ambiguous gateway state (wrong gateway active, or the named gateway is - // missing/unreachable/drifted). Do NOT destroy — a transient gateway - // problem must never be mistaken for a gone sandbox. Surface guidance and - // abort with the registry entry intact. - if (reconciled.state === "wrong_gateway_active") { - printWrongGatewayActiveGuidance( - sandboxName, - reconciled.activeGateway, - console.error, - "rebuild --yes", - ); - } else { - console.error( - ` Sandbox '${sandboxName}' is not visible on gateway '${recordedGateway}' and its live state could not be confirmed.`, - ); - console.error(" Your local registry entry has been preserved — nothing was removed."); - printGatewayLifecycleHint(reconciled.output || "", sandboxName, console.error); - } - bail(`Could not confirm live state of '${sandboxName}' (gateway not in a known-good state).`); - return; - } - } + // Step 1: Ensure sandbox is live for backup, or identify stale-sandbox recovery. + const liveState = await resolveRebuildLiveState(sandboxName, sb, log, bail); + if (!liveState) return; + const { staleRecovery, staleRegistrySnapshot } = liveState; // Build agent base layers before backup/delete so Dockerfile.base errors leave // the existing sandbox intact. This is what applies local Hermes version edits. - if (rebuildAgent) { - const agentDef = loadAgent(rebuildAgent); - try { - ensureAgentBaseImage(agentDef, { forceBaseImageRebuild: true }); - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - console.error(""); - console.error(` ${_RD}Rebuild preflight failed:${R} agent base image could not be built.`); - console.error(` ${message}`); - console.error(""); - console.error(" Sandbox is untouched — no data was lost."); - bail(message); - return; - } - } + if (!ensureRebuildAgentBaseImage(rebuildAgent, bail)) return; // On stale-sandbox recovery the live sandbox is gone, so the normal - // unlock→recreate→relock cycle cannot run: openRebuildShieldsWindow would call - // `shieldsDown`, which captures policy from the absent sandbox and fails, and - // a later `shieldsUp` would try to re-seal the fresh image against the gone - // sandbox's stale file-hash seal and may refuse (#4497). Instead, just note - // whether the sandbox was locked (read-only). The stale shields state is reset - // only AFTER the recreate succeeds (below) so a failed recreate leaves the - // lockdown record intact for a retry; the operator re-applies `shields up` - // post-recovery (printed on success) since the workspace state was lost anyway. - let staleSandboxWasLocked = false; - let rebuildShieldsWindow: RebuildShieldsWindow | null; - if (staleRecovery) { - staleSandboxWasLocked = !shields.isShieldsDown(sandboxName); - rebuildShieldsWindow = { relocked: false, wasLocked: false }; - } else { - rebuildShieldsWindow = openRebuildShieldsWindow(sandboxName, CLI_NAME); - } + // unlock→recreate→relock cycle cannot run. Track stale lock state and defer + // clearing old shields state until recreate succeeds (#4497). + const { rebuildShieldsWindow, staleSandboxWasLocked } = openRebuildShieldsWindowForState( + sandboxName, + staleRecovery, + ); if (!rebuildShieldsWindow) return bail("Failed to auto-unlock shields."); const relockShieldsIfNeeded = (sandboxStillExists: boolean): boolean => @@ -775,67 +640,17 @@ export async function rebuildSandbox( let sandboxStillExists = true; - // backupManifest stays null on stale-sandbox recovery (#4497): the live - // sandbox is gone, so there is nothing to back up and the downstream - // restore/preset steps are skipped in favor of recreating from registry - // metadata. - let backupManifest: sandboxState.RebuildManifest | null = null; - try { // Step 2: Backup (skipped on stale-sandbox recovery -- no live state exists) - if (!staleRecovery) { - console.log(" Backing up sandbox state..."); - log(`Agent type: ${sb.agent || "openclaw"}, stateDirs from manifest`); - const backup = sandboxState.backupSandboxState(sandboxName); - log( - `Backup result: success=${backup.success}, backed=${backup.backedUpDirs.join(",")}; files=${backup.backedUpFiles.join(",")}, failed=${backup.failedDirs.join(",")}; failedFiles=${backup.failedFiles.join(",")}`, - ); - const hasAnyBackup = backup.backedUpDirs.length > 0 || backup.backedUpFiles.length > 0; - if (!backup.success && !hasAnyBackup) { - // Total failure — nothing was backed up at all. - console.error(" Failed to back up sandbox state."); - if (backup.failedDirs.length > 0) { - console.error(` Failed: ${backup.failedDirs.join(", ")}`); - } - if (backup.failedFiles.length > 0) { - console.error(` Failed files: ${backup.failedFiles.join(", ")}`); - } - console.error(" Aborting rebuild to prevent data loss."); - relockShieldsIfNeeded(true); - bail("Failed to back up sandbox state."); - return; - } - backupManifest = backup.manifest ?? null; - if (!backupManifest) { - console.error(" Failed to record backup metadata."); - console.error(" Aborting rebuild to prevent data loss."); - relockShieldsIfNeeded(true); - bail("Failed to record backup metadata."); - return; - } - if (!backup.success) { - // Partial backup — some state succeeded, some failed (e.g. root-owned - // files caused tar permission errors). Proceed with a warning so the - // rebuild isn't blocked by a handful of inaccessible files (#2727). - console.warn( - ` ${YW}⚠${R} Partial backup: ${backup.backedUpDirs.length} dirs and ` + - `${backup.backedUpFiles.length} files OK; ${backup.failedDirs.length} dirs and ` + - `${backup.failedFiles.length} files failed`, - ); - if (backup.failedDirs.length > 0) { - console.warn(` Failed dirs: ${backup.failedDirs.join(", ")}`); - } - if (backup.failedFiles.length > 0) { - console.warn(` Failed files: ${backup.failedFiles.join(", ")}`); - } - console.warn(" Rebuild will continue — failed state could not be preserved."); - } else { - console.log( - ` ${G}\u2713${R} State backed up (${backup.backedUpDirs.length} directories, ${backup.backedUpFiles.length} files)`, - ); - } - console.log(` Backup: ${backupManifest.backupPath}`); - } + const backupManifest = backupSandboxStateForRebuild( + sandboxName, + sb, + staleRecovery, + log, + relockShieldsIfNeeded, + bail, + ); + if (backupManifest === undefined) return; // Step 3: Delete sandbox without tearing down gateway or session. // sandboxDestroy() cleans up the gateway when it's the last sandbox and diff --git a/src/lib/actions/sandbox/snapshot.test.ts b/src/lib/actions/sandbox/snapshot.test.ts index 760f49214d9..f8414d53045 100644 --- a/src/lib/actions/sandbox/snapshot.test.ts +++ b/src/lib/actions/sandbox/snapshot.test.ts @@ -16,14 +16,26 @@ const shieldsMock = vi.hoisted(() => { }); const backupSandboxStateMock = vi.fn(); -const captureOpenshellMock = vi.fn(() => ({ status: 0, output: "alpha Ready\n" })); +const captureOpenshellMock = vi.fn(() => ({ + status: 0, + output: "alpha Ready\n", +})); const dockerInspectMock = vi.fn(() => ({ status: 0, stdout: "true\n" })); const findBackupMock = vi.fn(); +const getAppliedPresetsMock = vi.fn(() => [] as string[]); +const getCustomPoliciesMock = vi.fn( + () => [] as Array<{ name: string; content: string; sourcePath?: string }>, +); +const getLatestBackupMock = vi.fn(() => null as Record | null); +const applyPresetMock = vi.fn((_sandbox: string, _preset: string) => true); +const applyPresetContentMock = vi.fn( + (_sandbox: string, _name: string, _content: string, _options?: unknown) => true, +); +const removePresetMock = vi.fn((_sandbox: string, _preset: string) => true); const getSandboxMock = vi.fn(() => null); const isGatewayHealthyMock = vi.fn(() => true); const listBackupsMock = vi.fn<() => Array>>(() => []); const parseLiveSandboxNamesMock = vi.fn(() => new Set(["alpha"])); -const getLatestBackupMock = vi.fn(); const registerSandboxMock = vi.fn(); const restoreSandboxStateMock = vi.fn(); @@ -46,7 +58,12 @@ vi.mock("../../domain/sandbox/destroy", () => ({ getSandboxDeleteOutcome: vi.fn(() => ({ alreadyGone: false })), })); -vi.mock("../../policy", () => ({})); +vi.mock("../../policy", () => ({ + applyPreset: applyPresetMock, + applyPresetContent: applyPresetContentMock, + getAppliedPresets: getAppliedPresetsMock, + removePreset: removePresetMock, +})); vi.mock("../../runner", () => ({ ROOT: "/repo", @@ -63,7 +80,11 @@ vi.mock("../../shields", () => ({ get isShieldsDown() { return shieldsMock.getIsShieldsDownExport(); }, - repairMutableConfigPerms: vi.fn(() => ({ applied: true, verified: true, errors: [] })), + repairMutableConfigPerms: vi.fn(() => ({ + applied: true, + verified: true, + errors: [], + })), })); vi.mock("../../state/gateway", () => ({ @@ -74,6 +95,7 @@ vi.mock("../../state/gateway", () => ({ })); vi.mock("../../state/registry", () => ({ + getCustomPolicies: getCustomPoliciesMock, getSandbox: getSandboxMock, registerSandbox: registerSandboxMock, removeSandbox: vi.fn(), @@ -97,13 +119,21 @@ describe("runSandboxSnapshot", () => { vi.clearAllMocks(); shieldsMock.setIsShieldsDownExport(shieldsMock.isShieldsDownMock); shieldsMock.isShieldsDownMock.mockReturnValue(true); - captureOpenshellMock.mockReturnValue({ status: 0, output: "alpha Ready\n" }); + captureOpenshellMock.mockReturnValue({ + status: 0, + output: "alpha Ready\n", + }); dockerInspectMock.mockReturnValue({ status: 0, stdout: "true\n" }); findBackupMock.mockReturnValue({ match: null }); + getAppliedPresetsMock.mockReturnValue([]); + getCustomPoliciesMock.mockReturnValue([]); + getLatestBackupMock.mockReturnValue(null); + applyPresetMock.mockReturnValue(true); + applyPresetContentMock.mockReturnValue(true); + removePresetMock.mockReturnValue(true); getSandboxMock.mockReturnValue(null); isGatewayHealthyMock.mockReturnValue(true); listBackupsMock.mockReturnValue([]); - getLatestBackupMock.mockReturnValue(null); registerSandboxMock.mockReset(); restoreSandboxStateMock.mockReturnValue({ success: true, @@ -154,9 +184,14 @@ describe("runSandboxSnapshot", () => { }); const { runSandboxSnapshot } = await import("./snapshot"); - await runSandboxSnapshot("alpha", { kind: "create", name: "before-upgrade" }); + await runSandboxSnapshot("alpha", { + kind: "create", + name: "before-upgrade", + }); - expect(backupSandboxStateMock).toHaveBeenCalledWith("alpha", { name: "before-upgrade" }); + expect(backupSandboxStateMock).toHaveBeenCalledWith("alpha", { + name: "before-upgrade", + }); expect(findBackupMock).toHaveBeenCalledWith("alpha", manifest.timestamp); const output = consoleLog.mock.calls.flat().join("\n"); expect(output).toContain("Creating snapshot of 'alpha' (--name before-upgrade)"); @@ -248,10 +283,65 @@ describe("runSandboxSnapshot", () => { exitCode: 1, }); - expect(backupSandboxStateMock).toHaveBeenCalledWith("alpha", { name: null }); + expect(backupSandboxStateMock).toHaveBeenCalledWith("alpha", { + name: null, + }); expect(consoleError.mock.calls.flat().join("\n")).toContain("tar exploded"); }); + it("reconciles snapshot policies after restore and warns without failing on repair misses", async () => { + const consoleLog = vi.spyOn(console, "log").mockImplementation(() => {}); + const consoleWarn = vi.spyOn(console, "warn").mockImplementation(() => {}); + getLatestBackupMock.mockReturnValue({ + backupPath: "/tmp/alpha/v2", + timestamp: "2026-06-02T00:00:00.000Z", + policyPresets: ["npm", "github"], + customPolicies: [ + { + name: "team-egress", + content: "allow team.example", + sourcePath: "/policies/team.yaml", + }, + ], + }); + restoreSandboxStateMock.mockReturnValue({ + success: true, + restoredDirs: ["workspace"], + restoredFiles: ["openclaw.json"], + failedDirs: [], + failedFiles: [], + }); + getAppliedPresetsMock.mockReturnValue(["npm", "team-egress", "old-preset"]); + getCustomPoliciesMock.mockReturnValue([ + { + name: "team-egress", + content: "allow team.example", + sourcePath: "/policies/team.yaml", + }, + { name: "old-custom", content: "allow old.example", sourcePath: "/old.yaml" }, + ]); + removePresetMock.mockImplementation((_sandbox, preset) => preset !== "old-custom"); + const { runSandboxSnapshot } = await import("./snapshot"); + + await runSandboxSnapshot("alpha", { kind: "restore" }); + + expect(restoreSandboxStateMock).toHaveBeenCalledWith("alpha", "/tmp/alpha/v2"); + expect(removePresetMock).toHaveBeenCalledWith("alpha", "old-preset"); + expect(applyPresetMock).toHaveBeenCalledWith("alpha", "github"); + expect(removePresetMock).toHaveBeenCalledWith("alpha", "old-custom"); + expect(removePresetMock).not.toHaveBeenCalledWith("alpha", "team-egress"); + expect(applyPresetContentMock).not.toHaveBeenCalled(); + const output = consoleLog.mock.calls.flat().join("\n"); + expect(output).toContain("✓ Restored 1 directories, 1 files"); + expect(output).toContain( + "Reconciling policy presets on 'alpha': add github; remove old-preset", + ); + expect(output).toContain("Reconciling custom policies on 'alpha': remove old-custom"); + expect(consoleWarn.mock.calls.flat().join("\n")).toContain( + "Warning: could not reconcile custom policy(ies): old-custom (remove failed)", + ); + }); + it("prints failed dirs and files when snapshot creation fails without an error", async () => { backupSandboxStateMock.mockReturnValue({ success: false, diff --git a/src/lib/actions/sandbox/snapshot.ts b/src/lib/actions/sandbox/snapshot.ts index 62bc1e1d5a1..18cd17f21f2 100644 --- a/src/lib/actions/sandbox/snapshot.ts +++ b/src/lib/actions/sandbox/snapshot.ts @@ -154,19 +154,13 @@ async function autoCreateSandboxFromSource( srcName: string, dstName: string, srcEntry: SandboxEntry | { name: string }, + fromImage: string, ): Promise { const sandboxCreateStream = require("../../sandbox/create-stream"); const { isSandboxReady } = require("../../state/gateway"); const basePolicy = path.join(ROOT, "nemoclaw-blueprint", "policies", "openclaw-sandbox.yaml"); const openshellBin = getOpenshellBinary(); - const fromImage = resolveSrcPodImage(srcName, srcEntry); - if (!fromImage) { - console.error(` Cannot auto-create '${dstName}': could not resolve '${srcName}' pod image.`); - console.error(` Create '${dstName}' manually with '${CLI_NAME} onboard'.`); - snapshotExit(1); - } - const cmdParts = [ openshellBin, "sandbox", @@ -190,7 +184,9 @@ async function autoCreateSandboxFromSource( initialPhase: "create", // Wait until the sandbox actually reaches Ready state, not just appears in the list. readyCheck: () => { - const list = captureOpenshell(["sandbox", "list"], { ignoreError: true }); + const list = captureOpenshell(["sandbox", "list"], { + ignoreError: true, + }); if (list.status !== 0) return false; return isSandboxReady(list.output || "", dstName); }, @@ -281,7 +277,10 @@ function deleteSandboxForRestore(name: string): void { // manifests. // - shields-.json + shields timer: per-sandbox shields artifacts try { - fs.rmSync(`/tmp/nemoclaw-services-${name}`, { recursive: true, force: true }); + fs.rmSync(`/tmp/nemoclaw-services-${name}`, { + recursive: true, + force: true, + }); } catch { // PID dir may not exist \u2014 ignore. } @@ -360,7 +359,9 @@ function runSnapshotCreate( } const label = request.name ? ` (--name ${request.name})` : ""; console.log(` Creating snapshot of '${sandboxName}'${label}...`); - const result = sandboxState.backupSandboxState(sandboxName, { name: request.name ?? null }); + const result = sandboxState.backupSandboxState(sandboxName, { + name: request.name ?? null, + }); if (result.success) { const manifest = result.manifest!; const entry = sandboxState.findBackup(sandboxName, manifest.timestamp).match ?? manifest; @@ -385,6 +386,291 @@ function runSnapshotCreate( snapshotExit(1); } +function repairRestoredOpenClawConfigPerms( + targetSandbox: string, + result: ReturnType, +): void { + if (!result.restoredFiles.includes("openclaw.json")) return; + try { + const permRepair = shields.repairMutableConfigPerms(targetSandbox); + if (permRepair.applied && permRepair.verified) { + console.log(` ${G}✓${R} OpenClaw config permissions restored`); + } else if (!permRepair.applied && permRepair.skipReason === "unreadable") { + console.warn(` Warning: could not verify OpenClaw config permissions: ${permRepair.reason}`); + } else if (permRepair.applied && !permRepair.verified) { + console.warn( + ` Warning: OpenClaw config permission repair incomplete: ${permRepair.errors.join("; ")}`, + ); + } + } catch (err) { + console.warn( + ` Warning: OpenClaw config permission repair errored: ${err instanceof Error ? err.message : String(err)}`, + ); + } +} + +function reconcileSnapshotPolicyPresets( + targetSandbox: string, + resolvedSnapshot: ReturnType, +): void { + if (!resolvedSnapshot || !Array.isArray(resolvedSnapshot.policyPresets)) return; + const snapshotPresets = resolvedSnapshot.policyPresets; + // getAppliedPresets includes custom-policy names for display/CLI parity. + // Built-in preset reconciliation must not remove those; custom policy content + // is reconciled separately below from registry.getCustomPolicies(). + const customPolicyNames = new Set(registry.getCustomPolicies(targetSandbox).map((p) => p.name)); + const currentPresets = policies + .getAppliedPresets(targetSandbox) + .filter((preset: string) => !customPolicyNames.has(preset)); + const toRemove = currentPresets.filter((p: string) => !snapshotPresets.includes(p)); + const toAdd = snapshotPresets.filter((p: string) => !currentPresets.includes(p)); + if (toRemove.length === 0 && toAdd.length === 0) return; + + const summary: string[] = []; + if (toAdd.length > 0) summary.push(`add ${toAdd.join(", ")}`); + if (toRemove.length > 0) summary.push(`remove ${toRemove.join(", ")}`); + console.log(` Reconciling policy presets on '${targetSandbox}': ${summary.join("; ")}`); + + const failed: string[] = []; + for (const preset of toRemove) { + try { + if (!policies.removePreset(targetSandbox, preset)) failed.push(`${preset} (remove failed)`); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + failed.push(`${preset} (remove: ${message})`); + } + } + for (const preset of toAdd) { + try { + if (!policies.applyPreset(targetSandbox, preset)) failed.push(`${preset} (apply failed)`); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + failed.push(`${preset} (apply: ${message})`); + } + } + if (failed.length > 0) { + console.warn(` Warning: could not reconcile preset(s): ${failed.join("; ")}`); + } +} + +function reconcileSnapshotCustomPolicies( + targetSandbox: string, + resolvedSnapshot: ReturnType, +): void { + if (!resolvedSnapshot || !Array.isArray(resolvedSnapshot.customPolicies)) return; + const snapshotCustom = resolvedSnapshot.customPolicies; + const currentCustom = registry.getCustomPolicies(targetSandbox); + const snapshotByName = new Map(snapshotCustom.map((entry) => [entry.name, entry])); + const currentByName = new Map(currentCustom.map((entry) => [entry.name, entry])); + const toRemove = currentCustom.filter((c) => !snapshotByName.has(c.name)); + const toAdd = snapshotCustom.filter((sp) => { + const current = currentByName.get(sp.name); + return !current || current.content !== sp.content || current.sourcePath !== sp.sourcePath; + }); + if (toRemove.length === 0 && toAdd.length === 0) return; + + const summary: string[] = []; + if (toAdd.length > 0) summary.push(`add ${toAdd.map((c) => c.name).join(", ")}`); + if (toRemove.length > 0) summary.push(`remove ${toRemove.map((c) => c.name).join(", ")}`); + console.log(` Reconciling custom policies on '${targetSandbox}': ${summary.join("; ")}`); + + const failed: string[] = []; + for (const entry of toRemove) { + try { + if (!policies.removePreset(targetSandbox, entry.name)) { + failed.push(`${entry.name} (remove failed)`); + } + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + failed.push(`${entry.name} (remove: ${message})`); + } + } + for (const entry of toAdd) { + try { + if ( + !policies.applyPresetContent(targetSandbox, entry.name, entry.content, { + custom: { sourcePath: entry.sourcePath }, + }) + ) { + failed.push(`${entry.name} (apply failed)`); + } + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + failed.push(`${entry.name} (apply: ${message})`); + } + } + if (failed.length > 0) { + console.warn(` Warning: could not reconcile custom policy(ies): ${failed.join("; ")}`); + } +} + +async function runSnapshotRestore( + sandboxName: string, + request: Extract, +): Promise { + // `--to ` restores the snapshot from sandboxName into a different + // sandbox. If `dst` is not yet live, it is auto-created by cloning the + // source sandbox's baked image. Without `--to`, restore targets + // sandboxName itself + const target = request.to ?? sandboxName; + const targetSandbox = + target === sandboxName ? sandboxName : validateName(target, "target sandbox name"); + const sourceLiveNames = requireLiveSandboxesOnSandboxGateway( + sandboxName, + " Failed to query live sandbox state from OpenShell.", + ); + const isCrossSandboxRestore = targetSandbox !== sandboxName; + const targetEntry = isCrossSandboxRestore ? registry.getSandbox(targetSandbox) : null; + const targetExists = sourceLiveNames.has(targetSandbox) || Boolean(targetEntry); + + // #3756 P1 preflight: resolve the snapshot selector AND the source pod + // image before any destructive action. A bad selector, missing snapshot, + // or unresolvable source image must not be allowed to delete the + // destination first and only fail afterwards. + const selector = request.selector ?? null; + let backupPath: string; + let resolvedSnapshot: ReturnType; + if (selector) { + const { match } = sandboxState.findBackup(sandboxName, selector); + if (!match) { + console.error(` No snapshot matching '${selector}' found for '${sandboxName}'.`); + console.error(" Selector must be an exact version (v), name, or timestamp."); + console.error(` Run: ${CLI_NAME} ${sandboxName} snapshot list`); + snapshotExit(1); + } + backupPath = match.backupPath; + resolvedSnapshot = match; + const v = formatSnapshotVersion(match); + const nameSuffix = match.name ? ` name=${match.name}` : ""; + console.log(` Using snapshot ${v}${nameSuffix} (${match.timestamp})`); + } else { + const latest = sandboxState.getLatestBackup(sandboxName); + if (!latest) { + console.error(` No snapshots found for '${sandboxName}'.`); + snapshotExit(1); + } + backupPath = latest.backupPath; + resolvedSnapshot = latest; + const v = formatSnapshotVersion(latest); + const nameSuffix = latest.name ? ` name=${latest.name}` : ""; + console.log(` Using latest snapshot ${v}${nameSuffix} (${latest.timestamp})`); + } + + if (!isCrossSandboxRestore) { + // Self-restore: target is `sandboxName`. Cannot auto-create; the + // source pod is the target, so it must already be live. + if (!targetExists) { + console.error(` Sandbox '${targetSandbox}' is not running. Cannot restore snapshot.`); + snapshotExit(1); + } + } else { + // #3756: cross-sandbox restore into a destination that already exists + // used to overlay onto the live filesystem silently. Refuse by default + // *before* doing any source-side preflight, so the user sees the + // precise "destination exists" error instead of a misleading + // "source not found" or "cannot resolve image" message when both are + // also broken. + if (targetExists && !request.force) { + console.error(` Destination sandbox '${targetSandbox}' already exists.`); + console.error( + " Restoring into an existing sandbox is unsupported because it would silently mutate its filesystem.", + ); + console.error( + ` Re-run with --force to delete '${targetSandbox}' and recreate it from the snapshot, or pick a different name.`, + ); + snapshotExit(1); + } + // Cross-sandbox restore — whether dst exists (with --force) or not, + // we must be able to clone the source's running pod image. Resolve it + // upfront so a missing source / unresolvable image cannot delete the + // destination first (#3756 P1). + if (!sourceLiveNames.has(sandboxName)) { + if (targetExists) { + console.error( + ` Cannot recreate '${targetSandbox}' from snapshot: source '${sandboxName}' not found.`, + ); + } else { + console.error( + ` Cannot auto-create '${targetSandbox}': source '${sandboxName}' not found.`, + ); + console.error(` Create '${targetSandbox}' manually with '${CLI_NAME} onboard'.`); + } + snapshotExit(1); + } + const srcEntry = registry.getSandbox(sandboxName) || { name: sandboxName }; + const fromImage = resolveSrcPodImage(sandboxName, srcEntry); + if (!fromImage) { + console.error( + ` Cannot resolve image for source sandbox '${sandboxName}' — aborting before ` + + (targetExists ? `deleting '${targetSandbox}'.` : `creating '${targetSandbox}'.`), + ); + snapshotExit(1); + } + if (targetExists) { + // --force confirmed above. Prompt for the destination name (unless + // --yes or NEMOCLAW_NON_INTERACTIVE=1), then delete and recreate. + const nonInteractive = process.env.NEMOCLAW_NON_INTERACTIVE === "1"; + if (!request.yes && !nonInteractive) { + const answer = ( + await askPrompt( + ` This will DELETE sandbox '${targetSandbox}' and restore the snapshot into a fresh copy.\n` + + ` Type '${targetSandbox}' to confirm: `, + ) + ).trim(); + if (answer !== targetSandbox) { + console.error(" Confirmation did not match — aborting."); + snapshotExit(1); + } + } + if (targetEntry) { + verifyRestoreDestinationOnOwnGateway(targetSandbox); + } + deleteSandboxForRestore(targetSandbox); + requireLiveSandboxesOnSandboxGateway( + sandboxName, + " Failed to re-select source sandbox gateway after deleting destination.", + ); + } + await autoCreateSandboxFromSource(sandboxName, targetSandbox, srcEntry, fromImage); + } + if (targetSandbox !== sandboxName) { + console.log(` Restoring snapshot from '${sandboxName}' into '${targetSandbox}'...`); + } else { + console.log(` Restoring snapshot into '${sandboxName}'...`); + } + const result = sandboxState.restoreSandboxState(targetSandbox, backupPath); + if (result.success) { + console.log( + ` ${G}\u2713${R} Restored ${result.restoredDirs.length} directories, ${result.restoredFiles.length} files`, + ); + } else { + console.error(` Restore failed.`); + if (result.restoredDirs.length > 0) { + console.error(` Partial: ${result.restoredDirs.join(", ")}`); + } + if (result.failedDirs.length > 0) { + console.error(` Failed: ${result.failedDirs.join(", ")}`); + } + if (result.failedFiles.length > 0) { + console.error(` Failed files: ${result.failedFiles.join(", ")}`); + } + snapshotExit(1); + } + // Post-restore security-state reconciliation is best-effort by design: the + // filesystem restore succeeded and old snapshots may target hosts where policy + // providers or mutable-config repair are temporarily unavailable. Surface every + // failure as a warning, but keep the restore result tied to state restoration. + // #5027/#4538: openclaw.json restores via the generic copy strategy, which + // lands it at 0640. Repair the mutable config contract when needed. + repairRestoredOpenClawConfigPerms(targetSandbox, result); + // Reconcile the target's policy presets to match the snapshot manifest + // exactly. Skip legacy snapshots that predate the `policyPresets` field. + reconcileSnapshotPolicyPresets(targetSandbox, resolvedSnapshot); + // Reconcile custom policy presets (applied via --from-file/--from-dir). + // Skipped for legacy snapshots that predate the `customPolicies` field. + reconcileSnapshotCustomPolicies(targetSandbox, resolvedSnapshot); +} + export async function runSandboxSnapshot( sandboxName: string, request: SnapshotRequest = { kind: "help" }, @@ -409,280 +695,7 @@ export async function runSandboxSnapshot( break; } case "restore": { - // `--to ` restores the snapshot from sandboxName into a different - // sandbox. If `dst` is not yet live, it is auto-created by cloning the - // source sandbox's baked image. Without `--to`, restore targets - // sandboxName itself - const target = request.to ?? sandboxName; - const targetSandbox = - target === sandboxName ? sandboxName : validateName(target, "target sandbox name"); - const sourceLiveNames = requireLiveSandboxesOnSandboxGateway( - sandboxName, - " Failed to query live sandbox state from OpenShell.", - ); - const isCrossSandboxRestore = targetSandbox !== sandboxName; - const targetEntry = isCrossSandboxRestore ? registry.getSandbox(targetSandbox) : null; - const targetExists = sourceLiveNames.has(targetSandbox) || Boolean(targetEntry); - - // #3756 P1 preflight: resolve the snapshot selector AND the source pod - // image before any destructive action. A bad selector, missing snapshot, - // or unresolvable source image must not be allowed to delete the - // destination first and only fail afterwards. - const selector = request.selector ?? null; - let backupPath: string; - let resolvedSnapshot: ReturnType; - if (selector) { - const { match } = sandboxState.findBackup(sandboxName, selector); - if (!match) { - console.error(` No snapshot matching '${selector}' found for '${sandboxName}'.`); - console.error(" Selector must be an exact version (v), name, or timestamp."); - console.error(` Run: ${CLI_NAME} ${sandboxName} snapshot list`); - snapshotExit(1); - } - backupPath = match.backupPath; - resolvedSnapshot = match; - const v = formatSnapshotVersion(match); - const nameSuffix = match.name ? ` name=${match.name}` : ""; - console.log(` Using snapshot ${v}${nameSuffix} (${match.timestamp})`); - } else { - const latest = sandboxState.getLatestBackup(sandboxName); - if (!latest) { - console.error(` No snapshots found for '${sandboxName}'.`); - snapshotExit(1); - } - backupPath = latest.backupPath; - resolvedSnapshot = latest; - const v = formatSnapshotVersion(latest); - const nameSuffix = latest.name ? ` name=${latest.name}` : ""; - console.log(` Using latest snapshot ${v}${nameSuffix} (${latest.timestamp})`); - } - - if (!isCrossSandboxRestore) { - // Self-restore: target is `sandboxName`. Cannot auto-create; the - // source pod is the target, so it must already be live. - if (!targetExists) { - console.error(` Sandbox '${targetSandbox}' is not running. Cannot restore snapshot.`); - snapshotExit(1); - } - } else { - // #3756: cross-sandbox restore into a destination that already exists - // used to overlay onto the live filesystem silently. Refuse by default - // *before* doing any source-side preflight, so the user sees the - // precise "destination exists" error instead of a misleading - // "source not found" or "cannot resolve image" message when both are - // also broken. - if (targetExists && !request.force) { - console.error(` Destination sandbox '${targetSandbox}' already exists.`); - console.error( - " Restoring into an existing sandbox is unsupported because it would silently mutate its filesystem.", - ); - console.error( - ` Re-run with --force to delete '${targetSandbox}' and recreate it from the snapshot, or pick a different name.`, - ); - snapshotExit(1); - } - // Cross-sandbox restore — whether dst exists (with --force) or not, - // we must be able to clone the source's running pod image. Resolve it - // upfront so a missing source / unresolvable image cannot delete the - // destination first (#3756 P1). - if (!sourceLiveNames.has(sandboxName)) { - if (targetExists) { - console.error( - ` Cannot recreate '${targetSandbox}' from snapshot: source '${sandboxName}' not found.`, - ); - } else { - console.error( - ` Cannot auto-create '${targetSandbox}': source '${sandboxName}' not found.`, - ); - console.error(` Create '${targetSandbox}' manually with '${CLI_NAME} onboard'.`); - } - snapshotExit(1); - } - const srcEntry = registry.getSandbox(sandboxName) || { name: sandboxName }; - const fromImage = resolveSrcPodImage(sandboxName, srcEntry); - if (!fromImage) { - console.error( - ` Cannot resolve image for source sandbox '${sandboxName}' — aborting before ` + - (targetExists ? `deleting '${targetSandbox}'.` : `creating '${targetSandbox}'.`), - ); - snapshotExit(1); - } - if (targetExists) { - // --force confirmed above. Prompt for the destination name (unless - // --yes or NEMOCLAW_NON_INTERACTIVE=1), then delete and recreate. - const nonInteractive = process.env.NEMOCLAW_NON_INTERACTIVE === "1"; - if (!request.yes && !nonInteractive) { - const answer = ( - await askPrompt( - ` This will DELETE sandbox '${targetSandbox}' and restore the snapshot into a fresh copy.\n` + - ` Type '${targetSandbox}' to confirm: `, - ) - ).trim(); - if (answer !== targetSandbox) { - console.error(" Confirmation did not match — aborting."); - snapshotExit(1); - } - } - if (targetEntry) { - verifyRestoreDestinationOnOwnGateway(targetSandbox); - } - deleteSandboxForRestore(targetSandbox); - requireLiveSandboxesOnSandboxGateway( - sandboxName, - " Failed to re-select source sandbox gateway after deleting destination.", - ); - } - await autoCreateSandboxFromSource(sandboxName, targetSandbox, srcEntry); - } - if (targetSandbox !== sandboxName) { - console.log(` Restoring snapshot from '${sandboxName}' into '${targetSandbox}'...`); - } else { - console.log(` Restoring snapshot into '${sandboxName}'...`); - } - const result = sandboxState.restoreSandboxState(targetSandbox, backupPath); - if (result.success) { - console.log( - ` ${G}\u2713${R} Restored ${result.restoredDirs.length} directories, ${result.restoredFiles.length} files`, - ); - } else { - console.error(` Restore failed.`); - if (result.restoredDirs.length > 0) { - console.error(` Partial: ${result.restoredDirs.join(", ")}`); - } - if (result.failedDirs.length > 0) { - console.error(` Failed: ${result.failedDirs.join(", ")}`); - } - if (result.failedFiles.length > 0) { - console.error(` Failed files: ${result.failedFiles.join(", ")}`); - } - snapshotExit(1); - } - // #5027/#4538: openclaw.json restores via the generic copy strategy, - // which lands it at 0640. The always-on OpenClaw gateway needs the - // mutable config contract (setgid dir + group-writable openclaw.json) to - // keep writing config at runtime. Rebuild repairs this in its - // post-restore sequence; the standalone snapshot-restore path must do the - // same. Gated on openclaw.json having been restored (only OpenClaw - // declares it) and posture-aware (a no-op for shields-up/non-OpenClaw). - if (result.restoredFiles.includes("openclaw.json")) { - try { - const permRepair = shields.repairMutableConfigPerms(targetSandbox); - if (permRepair.applied && permRepair.verified) { - console.log(` ${G}✓${R} OpenClaw config permissions restored`); - } else if (!permRepair.applied && permRepair.skipReason === "unreadable") { - console.warn( - ` Warning: could not verify OpenClaw config permissions: ${permRepair.reason}`, - ); - } else if (permRepair.applied && !permRepair.verified) { - console.warn( - ` Warning: OpenClaw config permission repair incomplete: ${permRepair.errors.join("; ")}`, - ); - } - } catch (err) { - console.warn( - ` Warning: OpenClaw config permission repair errored: ${err instanceof Error ? err.message : String(err)}`, - ); - } - } - // Reconcile the target's policy presets to match the snapshot manifest - // exactly — add anything the snapshot recorded but the target is - // missing, and remove anything the target has that the snapshot did - // not. This mirrors how stateDirs are restored (full replacement, not - // additive) so the command's semantics are consistent. - // - // When the snapshot predates the `policyPresets` field (undefined), - // skip the reconcile entirely — we have no recorded state to match. - if (resolvedSnapshot && Array.isArray(resolvedSnapshot.policyPresets)) { - const snapshotPresets = resolvedSnapshot.policyPresets; - const currentPresets = policies.getAppliedPresets(targetSandbox); - const toRemove = currentPresets.filter((p: string) => !snapshotPresets.includes(p)); - const toAdd = snapshotPresets.filter((p: string) => !currentPresets.includes(p)); - - if (toRemove.length > 0 || toAdd.length > 0) { - const summary: string[] = []; - if (toAdd.length > 0) summary.push(`add ${toAdd.join(", ")}`); - if (toRemove.length > 0) summary.push(`remove ${toRemove.join(", ")}`); - console.log(` Reconciling policy presets on '${targetSandbox}': ${summary.join("; ")}`); - - const failed: string[] = []; - for (const preset of toRemove) { - try { - if (!policies.removePreset(targetSandbox, preset)) { - failed.push(`${preset} (remove failed)`); - } - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - failed.push(`${preset} (remove: ${message})`); - } - } - for (const preset of toAdd) { - try { - if (!policies.applyPreset(targetSandbox, preset)) { - failed.push(`${preset} (apply failed)`); - } - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - failed.push(`${preset} (apply: ${message})`); - } - } - if (failed.length > 0) { - console.warn(` Warning: could not reconcile preset(s): ${failed.join("; ")}`); - } - } - } - // Reconcile custom policy presets (applied via --from-file/--from-dir). - // Their full content travels in the manifest, so re-apply by content - // (which also re-records them in the registry). Diff by content + source, - // not just name: a same-name preset whose body changed must be re-applied. - // Full replacement, mirroring the built-in preset reconcile above; skipped - // for legacy snapshots that predate the `customPolicies` field. - if (resolvedSnapshot && Array.isArray(resolvedSnapshot.customPolicies)) { - const snapshotCustom = resolvedSnapshot.customPolicies; - const currentCustom = registry.getCustomPolicies(targetSandbox); - const snapshotByName = new Map(snapshotCustom.map((entry) => [entry.name, entry])); - const currentByName = new Map(currentCustom.map((entry) => [entry.name, entry])); - const toRemove = currentCustom.filter((c) => !snapshotByName.has(c.name)); - const toAdd = snapshotCustom.filter((sp) => { - const current = currentByName.get(sp.name); - return !current || current.content !== sp.content || current.sourcePath !== sp.sourcePath; - }); - - if (toRemove.length > 0 || toAdd.length > 0) { - const summary: string[] = []; - if (toAdd.length > 0) summary.push(`add ${toAdd.map((c) => c.name).join(", ")}`); - if (toRemove.length > 0) summary.push(`remove ${toRemove.map((c) => c.name).join(", ")}`); - console.log(` Reconciling custom policies on '${targetSandbox}': ${summary.join("; ")}`); - - const failed: string[] = []; - for (const entry of toRemove) { - try { - if (!policies.removePreset(targetSandbox, entry.name)) { - failed.push(`${entry.name} (remove failed)`); - } - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - failed.push(`${entry.name} (remove: ${message})`); - } - } - for (const entry of toAdd) { - try { - if ( - !policies.applyPresetContent(targetSandbox, entry.name, entry.content, { - custom: { sourcePath: entry.sourcePath }, - }) - ) { - failed.push(`${entry.name} (apply failed)`); - } - } catch (err) { - const message = err instanceof Error ? err.message : String(err); - failed.push(`${entry.name} (apply: ${message})`); - } - } - if (failed.length > 0) { - console.warn(` Warning: could not reconcile custom policy(ies): ${failed.join("; ")}`); - } - } - } + await runSnapshotRestore(sandboxName, request); break; } default: diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 90d6377f965..457bc1efb58 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -27,6 +27,9 @@ const { createRemoteModelValidator, requireProviderChoice, }: typeof import("./onboard/setup-nim-selection") = require("./onboard/setup-nim-selection"); +const { + createSetupNimOllamaHandlers, +}: typeof import("./onboard/setup-nim-ollama") = require("./onboard/setup-nim-ollama"); const inferenceInputCapability = require("./onboard/inference-input-capability"); const { cleanupTempDir }: typeof import("./onboard/temp-files") = require("./onboard/temp-files"); const { @@ -1065,6 +1068,36 @@ const { prepareOllamaModel, } = require("./inference/ollama/proxy"); +const { + handleWindowsHostOllamaSelection, + handleRunningOllamaSelection, + handleInstallOllamaSelection, +} = createSetupNimOllamaHandlers({ + OLLAMA_PORT, + OLLAMA_PROXY_PORT, + process, + isNonInteractive, + prompt, + checkOllamaPortsOrWarn, + ensureOllamaLoopbackSystemdOverride, + runOllamaStartupOrGate, + shouldFrontOllamaWithProxy, + startOllamaAuthProxy, + getLocalProviderBaseUrl, + selectAndValidateOllamaModel, + printOllamaExposureWarning, + switchToWindowsOllamaHost, + installOllamaOnWindowsHost, + awaitWindowsOllamaReady, + setupWindowsOllamaWith0000Binding, + printWindowsOllamaTimeoutDiagnostics, + resetOllamaHostCache, + installOllamaOnMacOS, + installOllamaOnLinux, + abortNonInteractive, + assertOllamaUpgradeApplied, +}); + const ollamaModelSize: typeof import("./inference/ollama/model-size") = require("./inference/ollama/model-size"); function isOpenshellInstalled(): boolean { @@ -4088,6 +4121,7 @@ async function setupNim( hermesToolGateways, preferredInferenceApi, nimContainer, + allowToolsIncompatible, }; const result = await handleRemoteProviderSelection( { selected, requestedModel, recoveredFromSandbox, recoveredModel, sandboxName }, @@ -4101,6 +4135,7 @@ async function setupNim( hermesAuthMethod, hermesToolGateways, preferredInferenceApi, + allowToolsIncompatible, } = state); if (result === "retry-selection") continue selectionLoop; break; @@ -4114,6 +4149,7 @@ async function setupNim( hermesToolGateways, preferredInferenceApi, nimContainer, + allowToolsIncompatible, }; const result = await handleNimLocalSelection( gpu, @@ -4136,187 +4172,96 @@ async function setupNim( if (rejectWindowsHostOllama(selected.key, isWindowsHostOllama)) { continue selectionLoop; } - if (!checkOllamaPortsOrWarn({ isNonInteractive })) continue selectionLoop; - let ollamaReady = ollamaRunning; - const overrideState = ensureOllamaLoopbackSystemdOverride({ isNonInteractive }); - if (overrideState === "ready") { - ollamaReady = true; - } else if (overrideState === "failed") { - console.error( - " Ollama systemd restart did not recover after applying the loopback override.", - ); - process.exit(1); - } - const ollamaStartup = runOllamaStartupOrGate({ - ollamaReady, - ollamaPort: OLLAMA_PORT, - getLocalProviderBaseUrl, - isNonInteractive, - }); - if (ollamaStartup.kind === "continue") continue selectionLoop; - if (ollamaStartup.kind === "fallback") { - ({ provider, credentialEnv, endpointUrl, model, preferredInferenceApi } = - ollamaStartup.result); - break; - } - if (shouldFrontOllamaWithProxy()) { - if (!startOllamaAuthProxy()) process.exit(1); - console.log( - ` ✓ Using Ollama on localhost:${OLLAMA_PORT} (proxy on :${OLLAMA_PROXY_PORT})`, - ); - } else { - console.log(` ✓ Using Ollama on localhost:${OLLAMA_PORT}`); - } - provider = "ollama-local"; - // Local Ollama needs no user-supplied API key — the auth proxy uses - // an internal token (NEMOCLAW_OLLAMA_PROXY_TOKEN, set in setupInference). - // Leaving this null prevents the wizard from prompting for / caching - // OPENAI_API_KEY and prevents the rebuild preflight from requiring it. - // See GH #2519. - credentialEnv = null; - endpointUrl = getLocalProviderBaseUrl(provider); - if (!endpointUrl) { - console.error(" Local Ollama base URL could not be determined."); - process.exit(1); - } - { - const result = await selectAndValidateOllamaModel(gpu, provider, { - requestedModel, - recoveredModel: recoveredFromSandbox ? recoveredModel : null, - }); - if (result.outcome === "back-to-selection") continue selectionLoop; - ({ model, allowToolsIncompatible } = result); - preferredInferenceApi = "openai-completions"; - } + const state: SetupNimSelectionState = { + model, + provider, + endpointUrl, + credentialEnv, + hermesAuthMethod, + hermesToolGateways, + preferredInferenceApi, + nimContainer, + allowToolsIncompatible, + }; + const result = await handleRunningOllamaSelection( + gpu, + requestedModel, + recoveredFromSandbox ? recoveredModel : null, + ollamaRunning, + state, + ); + ({ + model, + provider, + endpointUrl, + credentialEnv, + preferredInferenceApi, + allowToolsIncompatible, + } = state); + if (result === "retry-selection") continue selectionLoop; break; } else if (["start-windows-ollama", "install-windows-ollama"].includes(selected.key)) { if (rejectWindowsHostOllama(selected.key, true)) { continue selectionLoop; } - if (!checkOllamaPortsOrWarn({ isNonInteractive })) continue selectionLoop; - const isInstall = selected.key === "install-windows-ollama"; - const isSwitch = !isInstall && windowsOllamaReachable; - const isRestart = !isInstall && !isSwitch && winOllamaLoopbackOnly; - if (!isSwitch) { - printOllamaExposureWarning(); - } - const promptMsg = isInstall - ? " Install and launch Ollama on the Windows host with OLLAMA_HOST=0.0.0.0:11434? [Y/n]: " - : isSwitch - ? " Use Ollama on the Windows host (already running)? [Y/n]: " - : isRestart - ? " Stop the running Ollama and restart it with OLLAMA_HOST=0.0.0.0:11434? [Y/n]: " - : " Launch Ollama on the Windows host with OLLAMA_HOST=0.0.0.0:11434? [Y/n]: "; - const proceed = isNonInteractive() - ? true - : !(await prompt(promptMsg)).trim().toLowerCase().startsWith("n"); - if (!proceed) { - continue selectionLoop; - } - - if (isSwitch) { - switchToWindowsOllamaHost(); - } else if (isInstall) { - const installResult = await installOllamaOnWindowsHost(); - if (!installResult.ok) { - console.error( - " Install did not produce ollama.exe on PATH. Check the installer output above.", - ); - if (isNonInteractive()) process.exit(1); - continue selectionLoop; - } - if (!awaitWindowsOllamaReady()) { - console.log(" Installer did not leave a reachable Ollama daemon; restarting it..."); - if ( - !setupWindowsOllamaWith0000Binding({ - installedPath: installResult.path, - }) - ) { - printWindowsOllamaTimeoutDiagnostics(); - if (isNonInteractive()) process.exit(1); - continue selectionLoop; - } - } - console.log(` ✓ Using Ollama on host.docker.internal:${OLLAMA_PORT}`); - } else { - if ( - !setupWindowsOllamaWith0000Binding({ - announceStop: isRestart, - installedPath: winOllamaInstalledPath || undefined, - }) - ) { - printWindowsOllamaTimeoutDiagnostics(); - if (isNonInteractive()) process.exit(1); - continue selectionLoop; - } - console.log(` ✓ Using Ollama on host.docker.internal:${OLLAMA_PORT}`); - } - provider = "ollama-local"; - credentialEnv = null; - endpointUrl = getLocalProviderBaseUrl(provider); - if (!endpointUrl) { - console.error(" Local Ollama base URL could not be determined."); - process.exit(1); - } - - { - const result = await selectAndValidateOllamaModel(gpu, provider, { - requestedModel, - recoveredModel: null, - }); - if (result.outcome === "back-to-selection") { - // The Windows-host action pinned resolved host to - // host.docker.internal. Clear it so a subsequent provider pick - // (e.g. plain WSL Ollama) starts from a fresh probe. - resetOllamaHostCache(); - continue selectionLoop; - } - ({ model, allowToolsIncompatible } = result); - preferredInferenceApi = "openai-completions"; - } + const state: SetupNimSelectionState = { + model, + provider, + endpointUrl, + credentialEnv, + hermesAuthMethod, + hermesToolGateways, + preferredInferenceApi, + nimContainer, + allowToolsIncompatible, + }; + const result = await handleWindowsHostOllamaSelection( + gpu, + selected.key, + requestedModel, + windowsOllamaReachable, + winOllamaLoopbackOnly, + winOllamaInstalledPath, + state, + ); + ({ + model, + provider, + endpointUrl, + credentialEnv, + preferredInferenceApi, + allowToolsIncompatible, + } = state); + if (result === "retry-selection") continue selectionLoop; break; } else if (selected.key === "install-ollama") { - if (!checkOllamaPortsOrWarn({ isNonInteractive })) continue selectionLoop; - const isUpgrade = ollamaInstallMenu.hasUpgradableOllama; - const installResult = - process.platform === "darwin" - ? installOllamaOnMacOS({ isNonInteractive, isUpgrade }) - : installOllamaOnLinux({ isNonInteractive, isUpgrade }); - if (!installResult.ok) { - if (isNonInteractive()) abortNonInteractive("Ollama install failed. See errors above."); - continue selectionLoop; - } - const upgradeCheck = assertOllamaUpgradeApplied(ollamaInstallMenu); - if (!upgradeCheck.ok) { - console.error(` ${upgradeCheck.message}`); - if (isNonInteractive()) process.exit(1); - continue selectionLoop; - } - if (shouldFrontOllamaWithProxy()) { - if (!startOllamaAuthProxy()) process.exit(1); - console.log( - ` ✓ Using Ollama on localhost:${OLLAMA_PORT} (proxy on :${OLLAMA_PROXY_PORT})`, - ); - } else { - console.log(` ✓ Using Ollama on localhost:${OLLAMA_PORT}`); - } - provider = "ollama-local"; - // See above ollama branch — internal proxy token, no user API key. - credentialEnv = null; - endpointUrl = getLocalProviderBaseUrl(provider); - if (!endpointUrl) { - console.error(" Local Ollama base URL could not be determined."); - process.exit(1); - } - { - const result = await selectAndValidateOllamaModel(gpu, provider, { - requestedModel, - recoveredModel: recoveredFromSandbox ? recoveredModel : null, - }); - if (result.outcome === "back-to-selection") continue selectionLoop; - ({ model, allowToolsIncompatible } = result); - preferredInferenceApi = "openai-completions"; - } + const state: SetupNimSelectionState = { + model, + provider, + endpointUrl, + credentialEnv, + hermesAuthMethod, + hermesToolGateways, + preferredInferenceApi, + nimContainer, + allowToolsIncompatible, + }; + const result = await handleInstallOllamaSelection( + gpu, + requestedModel, + recoveredFromSandbox ? recoveredModel : null, + state, + ollamaInstallMenu, + ); + ({ + model, + provider, + endpointUrl, + credentialEnv, + preferredInferenceApi, + allowToolsIncompatible, + } = state); + if (result === "retry-selection") continue selectionLoop; break; } else if (selected.key === "install-vllm") { if (!vllmProfile) { @@ -4348,10 +4293,18 @@ async function setupNim( hermesToolGateways, preferredInferenceApi, nimContainer, + allowToolsIncompatible, }; const result = await handleVllmSelection(state); - ({ model, provider, endpointUrl, credentialEnv, preferredInferenceApi, nimContainer } = - state); + ({ + model, + provider, + endpointUrl, + credentialEnv, + preferredInferenceApi, + nimContainer, + allowToolsIncompatible, + } = state); if (result === "retry-selection") continue selectionLoop; break; } else if (selected.key === "routed") { @@ -4364,10 +4317,18 @@ async function setupNim( hermesToolGateways, preferredInferenceApi, nimContainer, + allowToolsIncompatible, }; const result = await handleRoutedSelection(state); - ({ model, provider, endpointUrl, credentialEnv, preferredInferenceApi, nimContainer } = - state); + ({ + model, + provider, + endpointUrl, + credentialEnv, + preferredInferenceApi, + nimContainer, + allowToolsIncompatible, + } = state); if (result === "retry-selection") continue selectionLoop; break; } diff --git a/src/lib/onboard/setup-nim-ollama.test.ts b/src/lib/onboard/setup-nim-ollama.test.ts new file mode 100644 index 00000000000..70940b0cf98 --- /dev/null +++ b/src/lib/onboard/setup-nim-ollama.test.ts @@ -0,0 +1,188 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; + +import { describe, it, vi } from "vitest"; + +import { createSetupNimOllamaHandlers } from "./setup-nim-ollama"; +import type { SetupNimSelectionState } from "./setup-nim-selection"; + +function makeState(): SetupNimSelectionState { + return { + model: null, + provider: "nvidia-prod", + endpointUrl: null, + credentialEnv: "NVIDIA_INFERENCE_API_KEY", + hermesAuthMethod: null, + hermesToolGateways: [], + preferredInferenceApi: null, + nimContainer: null, + allowToolsIncompatible: false, + }; +} + +type Deps = Parameters[0]; + +function makeDeps(overrides: Partial = {}): Deps { + return { + OLLAMA_PORT: 11434, + OLLAMA_PROXY_PORT: 11435, + process, + isNonInteractive: () => true, + prompt: async () => "y", + checkOllamaPortsOrWarn: () => true, + ensureOllamaLoopbackSystemdOverride: () => "unchanged", + runOllamaStartupOrGate: () => ({ kind: "ready" }), + shouldFrontOllamaWithProxy: () => false, + startOllamaAuthProxy: () => true, + getLocalProviderBaseUrl: () => "http://127.0.0.1:11434/v1", + selectAndValidateOllamaModel: async () => ({ + outcome: "selected", + model: "llama3.1:8b", + allowToolsIncompatible: true, + }), + printOllamaExposureWarning: () => {}, + switchToWindowsOllamaHost: () => {}, + installOllamaOnWindowsHost: async () => ({ ok: true, path: "C:/Ollama/ollama.exe" }), + awaitWindowsOllamaReady: () => true, + setupWindowsOllamaWith0000Binding: () => true, + printWindowsOllamaTimeoutDiagnostics: () => {}, + resetOllamaHostCache: () => {}, + installOllamaOnMacOS: () => ({ ok: true }), + installOllamaOnLinux: () => ({ ok: true }), + abortNonInteractive: (message: string): never => { + throw new Error(message); + }, + assertOllamaUpgradeApplied: () => ({ ok: true }), + ...overrides, + }; +} + +describe("createSetupNimOllamaHandlers", () => { + it("preserves accepted tools-incompatible state for running Ollama", async () => { + const state = makeState(); + const { handleRunningOllamaSelection } = createSetupNimOllamaHandlers(makeDeps()); + + const result = await handleRunningOllamaSelection(null, "requested", "recovered", true, state); + + assert.equal(result, "selected"); + assert.equal(state.model, "llama3.1:8b"); + assert.equal(state.provider, "ollama-local"); + assert.equal(state.allowToolsIncompatible, true); + }); + + it("preserves accepted tools-incompatible state for Windows-host Ollama", async () => { + const state = makeState(); + const { handleWindowsHostOllamaSelection } = createSetupNimOllamaHandlers(makeDeps()); + + const result = await handleWindowsHostOllamaSelection( + null, + "start-windows-ollama", + "requested", + true, + false, + null, + state, + ); + + assert.equal(result, "selected"); + assert.equal(state.provider, "ollama-local"); + assert.equal(state.allowToolsIncompatible, true); + }); + + it("preserves accepted tools-incompatible state for installed Ollama", async () => { + const state = makeState(); + const { handleInstallOllamaSelection } = createSetupNimOllamaHandlers(makeDeps()); + + const result = await handleInstallOllamaSelection(null, "requested", "recovered", state, { + hasUpgradableOllama: false, + }); + + assert.equal(result, "selected"); + assert.equal(state.provider, "ollama-local"); + assert.equal(state.allowToolsIncompatible, true); + }); + + it("fails closed on unknown Ollama startup outcomes without mutating state", async () => { + const state = makeState(); + const before = { ...state, hermesToolGateways: [...state.hermesToolGateways] }; + const exit = vi.fn((code?: number) => { + throw new Error(`exit ${code}`); + }); + const startProxy = vi.fn(() => true); + const selectModel = vi.fn(async () => ({ + outcome: "selected" as const, + model: "should-not-run", + allowToolsIncompatible: true, + })); + const { handleRunningOllamaSelection } = createSetupNimOllamaHandlers( + makeDeps({ + process: { ...process, exit: exit as never }, + runOllamaStartupOrGate: () => ({ kind: "mystery" }) as never, + startOllamaAuthProxy: startProxy, + selectAndValidateOllamaModel: selectModel, + }), + ); + + await assert.rejects( + handleRunningOllamaSelection(null, "requested", "recovered", true, state), + /exit 1/, + ); + + assert.deepEqual(state, before); + assert.equal(exit.mock.calls[0]?.[0], 1); + assert.equal(startProxy.mock.calls.length, 0); + assert.equal(selectModel.mock.calls.length, 0); + }); + + it("applies a complete safe fallback state from a dirty prior selection", async () => { + const state = makeState(); + state.provider = "openai-api"; + state.endpointUrl = "https://api.openai.example/v1"; + state.credentialEnv = "OPENAI_API_KEY"; + state.model = "gpt-stale"; + state.preferredInferenceApi = "responses"; + state.nimContainer = "stale-nim"; + state.allowToolsIncompatible = true; + const startProxy = vi.fn(() => true); + const selectModel = vi.fn(async () => ({ + outcome: "selected" as const, + model: "should-not-run", + allowToolsIncompatible: true, + })); + const { handleRunningOllamaSelection } = createSetupNimOllamaHandlers( + makeDeps({ + runOllamaStartupOrGate: () => ({ + kind: "fallback", + result: { + provider: "ollama-local", + credentialEnv: null, + endpointUrl: "http://127.0.0.1:11434/v1", + model: "qwen3:0.6b", + preferredInferenceApi: "openai-completions", + }, + }), + startOllamaAuthProxy: startProxy, + selectAndValidateOllamaModel: selectModel, + }), + ); + + const result = await handleRunningOllamaSelection(null, "requested", "recovered", false, state); + + assert.equal(result, "selected"); + assert.deepEqual(state, { + model: "qwen3:0.6b", + provider: "ollama-local", + endpointUrl: "http://127.0.0.1:11434/v1", + credentialEnv: null, + hermesAuthMethod: null, + hermesToolGateways: [], + preferredInferenceApi: "openai-completions", + nimContainer: null, + allowToolsIncompatible: false, + }); + assert.equal(startProxy.mock.calls.length, 0); + assert.equal(selectModel.mock.calls.length, 0); + }); +}); diff --git a/src/lib/onboard/setup-nim-ollama.ts b/src/lib/onboard/setup-nim-ollama.ts new file mode 100644 index 00000000000..6b89eb68f36 --- /dev/null +++ b/src/lib/onboard/setup-nim-ollama.ts @@ -0,0 +1,286 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { OllamaStartupOutcome } from "./ollama-startup"; +import type { SetupNimSelectionState } from "./setup-nim-selection"; + +type SetupNimSelectionResult = "selected" | "retry-selection"; + +type SetupNimOllamaDeps = { + OLLAMA_PORT: number; + OLLAMA_PROXY_PORT: number; + process: NodeJS.Process; + isNonInteractive: () => boolean; + prompt: (message: string) => Promise; + checkOllamaPortsOrWarn: (args: { isNonInteractive: () => boolean }) => boolean; + ensureOllamaLoopbackSystemdOverride: (args: { isNonInteractive: () => boolean }) => string; + runOllamaStartupOrGate: (args: { + ollamaReady: boolean; + ollamaPort: number; + getLocalProviderBaseUrl: (provider: string) => string | null; + isNonInteractive: () => boolean; + }) => OllamaStartupOutcome; + shouldFrontOllamaWithProxy: () => boolean; + startOllamaAuthProxy: () => boolean; + getLocalProviderBaseUrl: (provider: string) => string | null; + selectAndValidateOllamaModel: ( + gpu: any, + provider: string, + args: { requestedModel: string | null; recoveredModel: string | null }, + ) => Promise< + | { outcome: "back-to-selection" } + | { outcome: "selected"; model: string; allowToolsIncompatible: boolean } + >; + printOllamaExposureWarning: () => void; + switchToWindowsOllamaHost: () => void; + installOllamaOnWindowsHost: () => Promise<{ ok: boolean; path?: string | null }>; + awaitWindowsOllamaReady: () => boolean; + setupWindowsOllamaWith0000Binding: (args: { + announceStop?: boolean; + installedPath?: string | null; + }) => boolean; + printWindowsOllamaTimeoutDiagnostics: () => void; + resetOllamaHostCache: () => void; + installOllamaOnMacOS: (args: { isNonInteractive: () => boolean; isUpgrade: boolean }) => { + ok: boolean; + }; + installOllamaOnLinux: (args: { isNonInteractive: () => boolean; isUpgrade: boolean }) => { + ok: boolean; + }; + abortNonInteractive: (message: string) => never; + assertOllamaUpgradeApplied: (menu: { + hasUpgradableOllama: boolean; + }) => { ok: true } | { ok: false; message: string }; +}; + +export function createSetupNimOllamaHandlers(deps: SetupNimOllamaDeps): { + handleWindowsHostOllamaSelection: ( + gpu: any, + selectedKey: string, + requestedModel: string | null, + windowsOllamaReachable: boolean, + winOllamaLoopbackOnly: boolean, + winOllamaInstalledPath: string | null, + state: SetupNimSelectionState, + ) => Promise; + handleRunningOllamaSelection: ( + gpu: any, + requestedModel: string | null, + recoveredModel: string | null, + ollamaRunning: boolean, + state: SetupNimSelectionState, + ) => Promise; + handleInstallOllamaSelection: ( + gpu: any, + requestedModel: string | null, + recoveredModel: string | null, + state: SetupNimSelectionState, + ollamaInstallMenu: { hasUpgradableOllama: boolean }, + ) => Promise; +} { + async function selectModel( + gpu: any, + state: SetupNimSelectionState, + requestedModel: string | null, + recoveredModel: string | null, + ): Promise { + const result = await deps.selectAndValidateOllamaModel(gpu, state.provider, { + requestedModel, + recoveredModel, + }); + if (result.outcome === "back-to-selection") return "retry-selection"; + state.model = result.model; + state.allowToolsIncompatible = result.allowToolsIncompatible; + state.preferredInferenceApi = "openai-completions"; + return "selected"; + } + + function startProxyOrAnnounceDirect(): void { + if (deps.shouldFrontOllamaWithProxy()) { + if (!deps.startOllamaAuthProxy()) deps.process.exit(1); + console.log( + ` ✓ Using Ollama on localhost:${deps.OLLAMA_PORT} (proxy on :${deps.OLLAMA_PROXY_PORT})`, + ); + } else { + console.log(` ✓ Using Ollama on localhost:${deps.OLLAMA_PORT}`); + } + } + + function configureOllamaState(state: SetupNimSelectionState): void { + state.provider = "ollama-local"; + state.credentialEnv = null; + state.endpointUrl = deps.getLocalProviderBaseUrl(state.provider); + if (!state.endpointUrl) { + console.error(" Local Ollama base URL could not be determined."); + deps.process.exit(1); + } + } + + function applyOllamaFallbackState( + state: SetupNimSelectionState, + result: Extract["result"], + ): void { + state.provider = result.provider; + state.credentialEnv = result.credentialEnv; + state.endpointUrl = result.endpointUrl; + state.model = result.model; + state.preferredInferenceApi = result.preferredInferenceApi; + state.nimContainer = null; + state.allowToolsIncompatible = false; + } + + async function handleWindowsHostOllamaSelection( + gpu: any, + selectedKey: string, + requestedModel: string | null, + windowsOllamaReachable: boolean, + winOllamaLoopbackOnly: boolean, + winOllamaInstalledPath: string | null, + state: SetupNimSelectionState, + ): Promise { + if (!deps.checkOllamaPortsOrWarn({ isNonInteractive: deps.isNonInteractive })) { + return "retry-selection"; + } + const isInstall = selectedKey === "install-windows-ollama"; + const isSwitch = !isInstall && windowsOllamaReachable; + const isRestart = !isInstall && !isSwitch && winOllamaLoopbackOnly; + if (!isSwitch) deps.printOllamaExposureWarning(); + const promptMsg = isInstall + ? " Install and launch Ollama on the Windows host with OLLAMA_HOST=0.0.0.0:11434? [Y/n]: " + : isSwitch + ? " Use Ollama on the Windows host (already running)? [Y/n]: " + : isRestart + ? " Stop the running Ollama and restart it with OLLAMA_HOST=0.0.0.0:11434? [Y/n]: " + : " Launch Ollama on the Windows host with OLLAMA_HOST=0.0.0.0:11434? [Y/n]: "; + const proceed = deps.isNonInteractive() + ? true + : !(await deps.prompt(promptMsg)).trim().toLowerCase().startsWith("n"); + if (!proceed) return "retry-selection"; + + if (isSwitch) { + deps.switchToWindowsOllamaHost(); + } else if (isInstall) { + const installResult = await deps.installOllamaOnWindowsHost(); + if (!installResult.ok) { + console.error( + " Install did not produce ollama.exe on PATH. Check the installer output above.", + ); + if (deps.isNonInteractive()) deps.process.exit(1); + return "retry-selection"; + } + if (!deps.awaitWindowsOllamaReady()) { + console.log(" Installer did not leave a reachable Ollama daemon; restarting it..."); + if (!deps.setupWindowsOllamaWith0000Binding({ installedPath: installResult.path })) { + deps.printWindowsOllamaTimeoutDiagnostics(); + if (deps.isNonInteractive()) deps.process.exit(1); + return "retry-selection"; + } + } + console.log(` ✓ Using Ollama on host.docker.internal:${deps.OLLAMA_PORT}`); + } else { + if ( + !deps.setupWindowsOllamaWith0000Binding({ + announceStop: isRestart, + installedPath: winOllamaInstalledPath || undefined, + }) + ) { + deps.printWindowsOllamaTimeoutDiagnostics(); + if (deps.isNonInteractive()) deps.process.exit(1); + return "retry-selection"; + } + console.log(` ✓ Using Ollama on host.docker.internal:${deps.OLLAMA_PORT}`); + } + configureOllamaState(state); + const result = await selectModel(gpu, state, requestedModel, null); + if (result === "retry-selection") deps.resetOllamaHostCache(); + return result; + } + + async function handleRunningOllamaSelection( + gpu: any, + requestedModel: string | null, + recoveredModel: string | null, + ollamaRunning: boolean, + state: SetupNimSelectionState, + ): Promise { + if (!deps.checkOllamaPortsOrWarn({ isNonInteractive: deps.isNonInteractive })) { + return "retry-selection"; + } + let ollamaReady = ollamaRunning; + const overrideState = deps.ensureOllamaLoopbackSystemdOverride({ + isNonInteractive: deps.isNonInteractive, + }); + if (overrideState === "ready") { + ollamaReady = true; + } else if (overrideState === "failed") { + console.error( + " Ollama systemd restart did not recover after applying the loopback override.", + ); + deps.process.exit(1); + } + const startup = deps.runOllamaStartupOrGate({ + ollamaReady, + ollamaPort: deps.OLLAMA_PORT, + getLocalProviderBaseUrl: deps.getLocalProviderBaseUrl, + isNonInteractive: deps.isNonInteractive, + }); + // Source boundary: ollama-startup owns this closed outcome contract. If a + // stale package or test double presents an unknown kind, fail closed before + // mutating provider state or starting proxy/model validation work. + switch (startup.kind) { + case "continue": + return "retry-selection"; + case "fallback": + // Fallback crosses a provider boundary, so write a complete safe state + // rather than merging over stale cloud/NIM/Ollama selection fields. + applyOllamaFallbackState(state, startup.result); + return "selected"; + case "ready": + startProxyOrAnnounceDirect(); + configureOllamaState(state); + return selectModel(gpu, state, requestedModel, recoveredModel); + default: { + const kind = (startup as { kind?: unknown }).kind; + console.error(` Unknown Ollama startup outcome: ${String(kind)}`); + deps.process.exit(1); + } + } + } + + async function handleInstallOllamaSelection( + gpu: any, + requestedModel: string | null, + recoveredModel: string | null, + state: SetupNimSelectionState, + ollamaInstallMenu: { hasUpgradableOllama: boolean }, + ): Promise { + if (!deps.checkOllamaPortsOrWarn({ isNonInteractive: deps.isNonInteractive })) { + return "retry-selection"; + } + const isUpgrade = ollamaInstallMenu.hasUpgradableOllama; + const installResult = + deps.process.platform === "darwin" + ? deps.installOllamaOnMacOS({ isNonInteractive: deps.isNonInteractive, isUpgrade }) + : deps.installOllamaOnLinux({ isNonInteractive: deps.isNonInteractive, isUpgrade }); + if (!installResult.ok) { + if (deps.isNonInteractive()) + deps.abortNonInteractive("Ollama install failed. See errors above."); + return "retry-selection"; + } + const upgradeCheck = deps.assertOllamaUpgradeApplied(ollamaInstallMenu); + if (!upgradeCheck.ok) { + console.error(` ${upgradeCheck.message}`); + if (deps.isNonInteractive()) deps.process.exit(1); + return "retry-selection"; + } + startProxyOrAnnounceDirect(); + configureOllamaState(state); + return selectModel(gpu, state, requestedModel, recoveredModel); + } + + return { + handleWindowsHostOllamaSelection, + handleRunningOllamaSelection, + handleInstallOllamaSelection, + }; +} diff --git a/src/lib/onboard/setup-nim-selection.test.ts b/src/lib/onboard/setup-nim-selection.test.ts index 7d4cab8bef9..633a6d67037 100644 --- a/src/lib/onboard/setup-nim-selection.test.ts +++ b/src/lib/onboard/setup-nim-selection.test.ts @@ -22,12 +22,14 @@ function makeState(): SetupNimSelectionState { hermesToolGateways: [], preferredInferenceApi: "openai-completions", nimContainer: "nemoclaw-nim-test", + allowToolsIncompatible: false, }; } describe("setupNim selection state helpers", () => { - it("applies a complete cloud fallback and clears stale NIM state", () => { + it("applies a complete cloud fallback and clears stale local-provider state", () => { const state = makeState(); + state.allowToolsIncompatible = true; applyCloudFallbackSelection(state, { providerName: "nvidia-prod", @@ -45,6 +47,7 @@ describe("setupNim selection state helpers", () => { hermesToolGateways: [], preferredInferenceApi: null, nimContainer: null, + allowToolsIncompatible: false, }); }); diff --git a/src/lib/onboard/setup-nim-selection.ts b/src/lib/onboard/setup-nim-selection.ts index 88f7ea2a51f..204eddfafb8 100644 --- a/src/lib/onboard/setup-nim-selection.ts +++ b/src/lib/onboard/setup-nim-selection.ts @@ -12,6 +12,7 @@ export type SetupNimSelectionState = { hermesToolGateways: string[]; preferredInferenceApi: string | null; nimContainer: string | null; + allowToolsIncompatible: boolean; }; export type CloudFallbackConfig = { @@ -25,12 +26,16 @@ export function applyCloudFallbackSelection( state: SetupNimSelectionState, cloudConfig: CloudFallbackConfig, ): void { + // Source boundary: fallback may run after a local Ollama/NIM/vLLM branch + // accepted provider-specific tool constraints. Cloud fallback is a fresh + // provider selection, so clear local-only compatibility state here. state.provider = cloudConfig.providerName; state.endpointUrl = cloudConfig.endpointUrl; state.credentialEnv = cloudConfig.credentialEnv; state.model = cloudConfig.defaultModel; state.preferredInferenceApi = null; state.nimContainer = null; + state.allowToolsIncompatible = false; } export function clearNimContainerBeforeRetry(state: SetupNimSelectionState): void { diff --git a/test/rebuild-credential-preflight.test.ts b/test/rebuild-credential-preflight.test.ts index 427b6538c31..66664dfd2dc 100644 --- a/test/rebuild-credential-preflight.test.ts +++ b/test/rebuild-credential-preflight.test.ts @@ -76,6 +76,7 @@ function createFixture(opts: { /** If set, the onboard-session.json provider_selection step status */ providerSelectionStatus?: string; agent?: string | null; + agents?: unknown[] | null; hermesAuthMethod?: string | null; messagingPlanChannels?: string[] | null; dockerBuildExitCode?: number; @@ -90,6 +91,7 @@ function createFixture(opts: { savedCredential, providerSelectionStatus = "complete", agent = null, + agents = null, hermesAuthMethod = null, messagingPlanChannels = null, dockerBuildExitCode = 0, @@ -119,6 +121,7 @@ function createFixture(opts: { gpuEnabled: false, policies: [], agent, + ...(agents ? { agents } : {}), ...(messagingPlan ? { messaging: { schemaVersion: 1, plan: messagingPlan } } : {}), }, }, @@ -397,6 +400,27 @@ describe("Issue #2273: atomic rebuild", () => { expect(output).toContain("Backing up sandbox state"); }); + it("aborts multi-agent rebuild before prompting, preflight, or backup", { + timeout: 60_000, + }, () => { + const f = createFixture({ + agents: [{ name: "openclaw" }, { name: "hermes" }], + savedCredential: { + key: "NVIDIA_INFERENCE_API_KEY", + value: "nvapi-test-key-for-rebuild", + }, + }); + + const result = runRebuild(f, {}, { yes: false, input: "YES\n" }); + const output = (result.stderr || "") + (result.stdout || ""); + + expect(result.status).not.toBe(0); + expect(output).toContain("Multi-agent sandbox rebuild is not yet supported"); + expect(output).not.toContain("Proceed? [y/N]:"); + expect(output).not.toContain("Backing up sandbox state"); + expect(registryHasSandbox(f)).toBe(true); + }); + it("prints active SSH session warning before interactive confirmation", { timeout: 60_000, }, () => {