diff --git a/src/lib/gateway-runtime-action.ts b/src/lib/gateway-runtime-action.ts new file mode 100644 index 00000000000..ded22eb9bd5 --- /dev/null +++ b/src/lib/gateway-runtime-action.ts @@ -0,0 +1,112 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +const { startGatewayForRecovery } = require("./onboard") as { + startGatewayForRecovery: () => Promise; +}; +import { OPENSHELL_OPERATION_TIMEOUT_MS, OPENSHELL_PROBE_TIMEOUT_MS } from "./openshell-timeouts"; +import { stripAnsi } from "./openshell"; +import { captureOpenshell, runOpenshell } from "./openshell-runtime"; + +function hasNamedGateway(output = ""): boolean { + return stripAnsi(output).includes("Gateway: nemoclaw"); +} + +function getActiveGatewayName(output = ""): string | null { + const match = stripAnsi(output).match(/^\s*Gateway:\s+(.+?)\s*$/m); + return match ? match[1].trim() : null; +} + +export function getNamedGatewayLifecycleState() { + const status = captureOpenshell(["status"], { timeout: OPENSHELL_PROBE_TIMEOUT_MS }); + const gatewayInfo = captureOpenshell(["gateway", "info", "-g", "nemoclaw"], { + timeout: OPENSHELL_PROBE_TIMEOUT_MS, + }); + const cleanStatus = stripAnsi(status.output); + const activeGateway = getActiveGatewayName(status.output); + const connected = /^\s*Status:\s*Connected\b/im.test(cleanStatus); + const named = hasNamedGateway(gatewayInfo.output); + const refusing = /Connection refused|client error \(Connect\)|tcp connect error/i.test( + cleanStatus, + ); + if (connected && activeGateway === "nemoclaw" && named) { + return { + state: "healthy_named", + status: status.output, + gatewayInfo: gatewayInfo.output, + activeGateway, + }; + } + if (activeGateway === "nemoclaw" && named && refusing) { + return { + state: "named_unreachable", + status: status.output, + gatewayInfo: gatewayInfo.output, + activeGateway, + }; + } + if (activeGateway === "nemoclaw" && named) { + return { + state: "named_unhealthy", + status: status.output, + gatewayInfo: gatewayInfo.output, + activeGateway, + }; + } + if (connected) { + return { + state: "connected_other", + status: status.output, + gatewayInfo: gatewayInfo.output, + activeGateway, + }; + } + return { + state: "missing_named", + status: status.output, + gatewayInfo: gatewayInfo.output, + activeGateway, + }; +} + +/** Attempt to recover the named NemoClaw gateway after a restart or connectivity loss. */ +export async function recoverNamedGatewayRuntime() { + const before = getNamedGatewayLifecycleState(); + if (before.state === "healthy_named") { + return { recovered: true, before, after: before, attempted: false }; + } + + runOpenshell(["gateway", "select", "nemoclaw"], { + ignoreError: true, + timeout: OPENSHELL_OPERATION_TIMEOUT_MS, + }); + let after = getNamedGatewayLifecycleState(); + if (after.state === "healthy_named") { + process.env.OPENSHELL_GATEWAY = "nemoclaw"; + return { recovered: true, before, after, attempted: true, via: "select" }; + } + + const shouldStartGateway = [before.state, after.state].some((state) => + ["missing_named", "named_unhealthy", "named_unreachable", "connected_other"].includes(state), + ); + + if (shouldStartGateway) { + try { + await startGatewayForRecovery(); + } catch { + // Fall through to the lifecycle re-check below so we preserve the + // existing recovery result shape and emit the correct classification. + } + runOpenshell(["gateway", "select", "nemoclaw"], { + ignoreError: true, + timeout: OPENSHELL_OPERATION_TIMEOUT_MS, + }); + after = getNamedGatewayLifecycleState(); + if (after.state === "healthy_named") { + process.env.OPENSHELL_GATEWAY = "nemoclaw"; + return { recovered: true, before, after, attempted: true, via: "start" }; + } + } + + return { recovered: false, before, after, attempted: true }; +} diff --git a/src/lib/global-cli-actions.ts b/src/lib/global-cli-actions.ts index 45fbd4407d7..12b7211c53b 100644 --- a/src/lib/global-cli-actions.ts +++ b/src/lib/global-cli-actions.ts @@ -13,6 +13,7 @@ import { runSetupAction as executeSetupAction, runSetupSparkAction as executeSetupSparkAction, } from "./onboard-action"; +import { recoverNamedGatewayRuntime as recoverNamedGatewayRuntimeAction } from "./gateway-runtime-action"; import { getNemoClawRuntimeBridge } from "./nemoclaw-runtime-bridge"; import { help, version } from "./root-help-action"; @@ -53,7 +54,13 @@ export function showVersion(): void { } export async function recoverNamedGatewayRuntime(): Promise<{ recovered: boolean }> { - return getNemoClawRuntimeBridge().recoverNamedGatewayRuntime(); + const runtime = getNemoClawRuntimeBridge() as { + recoverNamedGatewayRuntime?: () => Promise<{ recovered: boolean }>; + }; + if (typeof runtime.recoverNamedGatewayRuntime === "function") { + return runtime.recoverNamedGatewayRuntime(); + } + return recoverNamedGatewayRuntimeAction(); } export function runOpenshellProviderCommand( diff --git a/src/lib/list-command-deps.ts b/src/lib/list-command-deps.ts index 21c6d3e978c..093cabd8135 100644 --- a/src/lib/list-command-deps.ts +++ b/src/lib/list-command-deps.ts @@ -9,13 +9,12 @@ import { parseGatewayInference } from "./inference-config"; import { OPENSHELL_PROBE_TIMEOUT_MS } from "./openshell-timeouts"; import { parseSshProcesses, createSystemDeps } from "./sandbox-session-state"; import { resolveOpenshell } from "./resolve-openshell"; - -import { getNemoClawRuntimeBridge } from "./nemoclaw-runtime-bridge"; +import { captureOpenshell } from "./openshell-runtime"; +import { recoverRegistryEntries } from "./registry-recovery-action"; export function buildListCommandDeps(): ListSandboxesCommandDeps { const opsBinList = resolveOpenshell(); const sessionDeps = opsBinList ? createSystemDeps(opsBinList) : null; - const runtime = getNemoClawRuntimeBridge(); // Cache the SSH process probe once for all sandboxes — avoids spawning ps // per sandbox row. The getSshProcesses() call is the expensive part (5s timeout). @@ -32,10 +31,10 @@ export function buildListCommandDeps(): ListSandboxesCommandDeps { }; return { - recoverRegistryEntries: () => runtime.recoverRegistryEntries(), + recoverRegistryEntries: () => recoverRegistryEntries(), getLiveInference: () => parseGatewayInference( - runtime.captureOpenshell(["inference", "get"], { + captureOpenshell(["inference", "get"], { ignoreError: true, timeout: OPENSHELL_PROBE_TIMEOUT_MS, }).output, diff --git a/src/lib/nemoclaw-runtime-bridge.ts b/src/lib/nemoclaw-runtime-bridge.ts index b0b439d01fb..15754d23733 100644 --- a/src/lib/nemoclaw-runtime-bridge.ts +++ b/src/lib/nemoclaw-runtime-bridge.ts @@ -3,31 +3,17 @@ /* v8 ignore start -- transitional bridge until command actions are extracted from src/nemoclaw.ts. */ -import type { RecoveryResult } from "./inventory-commands"; - export interface SpawnLikeResult { status: number | null; stdout?: string | Buffer; stderr?: string | Buffer; } -export interface GatewayRecoveryResult { - recovered: boolean; -} - export interface SandboxConnectOptions { probeOnly?: boolean; } export interface NemoClawRuntimeBridge { - captureOpenshell: ( - args: string[], - opts?: { ignoreError?: boolean; timeout?: number }, - ) => { status: number | null; output: string }; - recoverNamedGatewayRuntime: () => Promise; - recoverRegistryEntries: (options?: { - requestedSandboxName?: string | null; - }) => Promise; runOpenshell: ( args: string[], opts?: { diff --git a/src/lib/policy-channel-actions.ts b/src/lib/policy-channel-actions.ts index bc48a269459..a8ae48d0de4 100644 --- a/src/lib/policy-channel-actions.ts +++ b/src/lib/policy-channel-actions.ts @@ -8,7 +8,7 @@ import path from "node:path"; import { CLI_DISPLAY_NAME, CLI_NAME } from "./branding"; import { getCredential, prompt as askPrompt } from "./credentials"; -import { getNemoClawRuntimeBridge } from "./nemoclaw-runtime-bridge"; +import { recoverNamedGatewayRuntime } from "./gateway-runtime-action"; const { isNonInteractive } = require("./onboard") as { isNonInteractive: () => boolean }; const onboardProviders = require("./onboard-providers"); import * as policies from "./policies"; @@ -284,7 +284,7 @@ async function applyChannelAddToGatewayAndRegistry( channelName: string, acquired: Record, ): Promise { - const recovery = await getNemoClawRuntimeBridge().recoverNamedGatewayRuntime(); + const recovery = await recoverNamedGatewayRuntime(); if (!recovery.recovered) { console.error( ` Could not reach the ${CLI_DISPLAY_NAME} OpenShell gateway. Tokens were staged`, @@ -324,7 +324,7 @@ async function applyChannelRemoveToGatewayAndRegistry( channelName: string, channelTokenKeys: string[], ): Promise { - const recovery = await getNemoClawRuntimeBridge().recoverNamedGatewayRuntime(); + const recovery = await recoverNamedGatewayRuntime(); if (!recovery.recovered) { console.error( ` Could not reach the ${CLI_DISPLAY_NAME} OpenShell gateway to delete the bridge.`, diff --git a/src/lib/registry-recovery-action.ts b/src/lib/registry-recovery-action.ts new file mode 100644 index 00000000000..15fc64e612e --- /dev/null +++ b/src/lib/registry-recovery-action.ts @@ -0,0 +1,189 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { recoverNamedGatewayRuntime } from "./gateway-runtime-action"; +import type { RecoveryResult } from "./inventory-commands"; +import * as onboardSession from "./onboard-session"; +import { OPENSHELL_PROBE_TIMEOUT_MS } from "./openshell-timeouts"; +import { captureOpenshell } from "./openshell-runtime"; +import * as registry from "./registry"; +import type { SandboxEntry } from "./registry"; +import { resolveOpenshell } from "./resolve-openshell"; +import { parseLiveSandboxNames } from "./runtime-recovery"; +import { validateName } from "./runner"; + +type Session = ReturnType; + +type RecoveredSandboxMetadata = Partial< + Pick +> & { + policyPresets?: string[] | null; +}; + +function buildRecoveredSandboxEntry( + name: string, + metadata: RecoveredSandboxMetadata = {}, +): SandboxEntry { + return { + name, + model: metadata.model || null, + provider: metadata.provider || null, + gpuEnabled: metadata.gpuEnabled === true, + policies: Array.isArray(metadata.policies) + ? metadata.policies + : Array.isArray(metadata.policyPresets) + ? metadata.policyPresets + : [], + nimContainer: metadata.nimContainer || null, + agent: metadata.agent || null, + }; +} + +function upsertRecoveredSandbox(name: string, metadata: RecoveredSandboxMetadata = {}) { + let validName; + try { + validName = validateName(name, "sandbox name"); + } catch { + return false; + } + + const entry = buildRecoveredSandboxEntry(validName, metadata); + if (registry.getSandbox(validName)) { + registry.updateSandbox(validName, entry); + return false; + } + registry.registerSandbox(entry); + return true; +} + +function shouldRecoverRegistryEntries( + current: { sandboxes: Array<{ name: string }>; defaultSandbox?: string | null }, + session: Session | null, + requestedSandboxName: string | null, +) { + const sessionSandboxName = session?.sandboxName ?? null; + const hasSessionSandbox = Boolean(sessionSandboxName); + const missingSessionSandbox = + hasSessionSandbox && !current.sandboxes.some((sandbox) => sandbox.name === sessionSandboxName); + const missingRequestedSandbox = + Boolean(requestedSandboxName) && + !current.sandboxes.some((sandbox) => sandbox.name === requestedSandboxName); + const hasRecoverySeed = + current.sandboxes.length > 0 || hasSessionSandbox || Boolean(requestedSandboxName); + return { + missingRequestedSandbox, + shouldRecover: + hasRecoverySeed && + (current.sandboxes.length === 0 || missingRequestedSandbox || missingSessionSandbox), + }; +} + +function seedRecoveryMetadata( + current: { sandboxes: SandboxEntry[] }, + session: Session | null, + requestedSandboxName: string | null, +) { + const metadataByName = new Map( + current.sandboxes.map((sandbox: SandboxEntry) => [sandbox.name, sandbox]), + ); + let recoveredFromSession = false; + + if (!session?.sandboxName) { + return { metadataByName, recoveredFromSession }; + } + + metadataByName.set( + session.sandboxName, + buildRecoveredSandboxEntry(session.sandboxName, { + model: session.model || null, + provider: session.provider || null, + nimContainer: session.nimContainer || null, + policyPresets: session.policyPresets || null, + }), + ); + const sessionSandboxMissing = !current.sandboxes.some( + (sandbox: { name: string }) => sandbox.name === session.sandboxName, + ); + const shouldRecoverSessionSandbox = + current.sandboxes.length === 0 || + sessionSandboxMissing || + requestedSandboxName === session.sandboxName; + if (shouldRecoverSessionSandbox) { + recoveredFromSession = upsertRecoveredSandbox( + session.sandboxName, + metadataByName.get(session.sandboxName), + ); + } + return { metadataByName, recoveredFromSession }; +} + +async function recoverRegistryFromLiveGateway( + metadataByName: Map, +) { + if (!resolveOpenshell()) { + return 0; + } + const recovery = await recoverNamedGatewayRuntime(); + const canInspectLiveGateway = + recovery.recovered || + recovery.before?.state === "healthy_named" || + recovery.after?.state === "healthy_named"; + if (!canInspectLiveGateway) { + return 0; + } + + let recoveredFromGateway = 0; + const liveList = captureOpenshell(["sandbox", "list"], { + ignoreError: true, + timeout: OPENSHELL_PROBE_TIMEOUT_MS, + }); + const liveNames = Array.from(parseLiveSandboxNames(liveList.output)); + for (const name of liveNames) { + const metadata = metadataByName.get(name) || undefined; + if (upsertRecoveredSandbox(name, metadata)) { + recoveredFromGateway += 1; + } + } + return recoveredFromGateway; +} + +function applyRecoveredDefault( + currentDefaultSandbox: string | null, + requestedSandboxName: string | null, + session: Session | null, +) { + const recovered = registry.listSandboxes(); + const preferredDefault = + requestedSandboxName || (!currentDefaultSandbox ? session?.sandboxName || null : null); + if ( + preferredDefault && + recovered.sandboxes.some((sandbox: { name: string }) => sandbox.name === preferredDefault) + ) { + registry.setDefault(preferredDefault); + } + return registry.listSandboxes(); +} + +export async function recoverRegistryEntries({ + requestedSandboxName = null, +}: { requestedSandboxName?: string | null } = {}) { + const current = registry.listSandboxes(); + const session = onboardSession.loadSession(); + const recoveryCheck = shouldRecoverRegistryEntries(current, session, requestedSandboxName); + if (!recoveryCheck.shouldRecover) { + return { ...current, recoveredFromSession: false, recoveredFromGateway: 0 }; + } + + const seeded = seedRecoveryMetadata(current, session, requestedSandboxName); + const shouldProbeLiveGateway = + current.sandboxes.length > 0 || Boolean(session?.sandboxName) || Boolean(requestedSandboxName); + const recoveredFromGateway = shouldProbeLiveGateway + ? await recoverRegistryFromLiveGateway(seeded.metadataByName) + : 0; + const recovered = applyRecoveredDefault(current.defaultSandbox, requestedSandboxName, session); + return { + ...recovered, + recoveredFromSession: seeded.recoveredFromSession, + recoveredFromGateway, + }; +} diff --git a/src/nemoclaw.ts b/src/nemoclaw.ts index ec94d1731cf..a42f2382c56 100644 --- a/src/nemoclaw.ts +++ b/src/nemoclaw.ts @@ -36,12 +36,7 @@ const { dockerRmi, } = require("./lib/docker"); const { resolveOpenshell } = require("./lib/resolve-openshell"); -const { - startGatewayForRecovery, - pruneKnownHostsEntries, - hydrateCredentialEnv, - isNonInteractive, -} = require("./lib/onboard"); +const { pruneKnownHostsEntries, hydrateCredentialEnv, isNonInteractive } = require("./lib/onboard"); const { ensureOllamaAuthProxy } = require("./lib/onboard-ollama-proxy"); const { prompt: askPrompt } = require("./lib/credentials"); const registry = require("./lib/registry"); @@ -66,6 +61,11 @@ const { isCommandTimeout, runOpenshell, } = require("./lib/openshell-runtime"); +const { + getNamedGatewayLifecycleState, + recoverNamedGatewayRuntime, +} = require("./lib/gateway-runtime-action"); +const { recoverRegistryEntries } = require("./lib/registry-recovery-action"); const { runRegisteredOclifCommand } = require("./lib/oclif-runner"); const { isErrnoException }: typeof import("./lib/errno") = require("./lib/errno"); const agentRuntime = require("../bin/lib/agent-runtime"); @@ -450,178 +450,7 @@ function checkAndRecoverSandboxProcesses( return { checked: true, wasRunning: false, recovered }; } -function buildRecoveredSandboxEntry( - name: string, - metadata: RecoveredSandboxMetadata = {}, -): SandboxEntry { - return { - name, - model: metadata.model || null, - provider: metadata.provider || null, - gpuEnabled: metadata.gpuEnabled === true, - policies: Array.isArray(metadata.policies) - ? metadata.policies - : Array.isArray(metadata.policyPresets) - ? metadata.policyPresets - : [], - nimContainer: metadata.nimContainer || null, - agent: metadata.agent || null, - }; -} - -function upsertRecoveredSandbox(name: string, metadata: RecoveredSandboxMetadata = {}) { - let validName; - try { - validName = validateName(name, "sandbox name"); - } catch { - return false; - } - - const entry = buildRecoveredSandboxEntry(validName, metadata); - if (registry.getSandbox(validName)) { - registry.updateSandbox(validName, entry); - return false; - } - registry.registerSandbox(entry); - return true; -} - -function shouldRecoverRegistryEntries( - current: { sandboxes: Array<{ name: string }>; defaultSandbox?: string | null }, - session: Session | null, - requestedSandboxName: string | null, -) { - const sessionSandboxName = session?.sandboxName ?? null; - const hasSessionSandbox = Boolean(sessionSandboxName); - const missingSessionSandbox = - hasSessionSandbox && !current.sandboxes.some((sandbox) => sandbox.name === sessionSandboxName); - const missingRequestedSandbox = - Boolean(requestedSandboxName) && - !current.sandboxes.some((sandbox) => sandbox.name === requestedSandboxName); - const hasRecoverySeed = - current.sandboxes.length > 0 || hasSessionSandbox || Boolean(requestedSandboxName); - return { - missingRequestedSandbox, - shouldRecover: - hasRecoverySeed && - (current.sandboxes.length === 0 || missingRequestedSandbox || missingSessionSandbox), - }; -} - -function seedRecoveryMetadata( - current: { sandboxes: SandboxEntry[] }, - session: Session | null, - requestedSandboxName: string | null, -) { - const metadataByName = new Map( - current.sandboxes.map((sandbox: SandboxEntry) => [sandbox.name, sandbox]), - ); - let recoveredFromSession = false; - - if (!session?.sandboxName) { - return { metadataByName, recoveredFromSession }; - } - - metadataByName.set( - session.sandboxName, - buildRecoveredSandboxEntry(session.sandboxName, { - model: session.model || null, - provider: session.provider || null, - nimContainer: session.nimContainer || null, - policyPresets: session.policyPresets || null, - }), - ); - const sessionSandboxMissing = !current.sandboxes.some( - (sandbox: { name: string }) => sandbox.name === session.sandboxName, - ); - const shouldRecoverSessionSandbox = - current.sandboxes.length === 0 || - sessionSandboxMissing || - requestedSandboxName === session.sandboxName; - if (shouldRecoverSessionSandbox) { - recoveredFromSession = upsertRecoveredSandbox( - session.sandboxName, - metadataByName.get(session.sandboxName), - ); - } - return { metadataByName, recoveredFromSession }; -} - -async function recoverRegistryFromLiveGateway( - metadataByName: Map, -) { - if (!resolveOpenshell()) { - return 0; - } - const recovery = await recoverNamedGatewayRuntime(); - const canInspectLiveGateway = - recovery.recovered || - recovery.before?.state === "healthy_named" || - recovery.after?.state === "healthy_named"; - if (!canInspectLiveGateway) { - return 0; - } - - let recoveredFromGateway = 0; - const liveList = captureOpenshell(["sandbox", "list"], { - ignoreError: true, - timeout: OPENSHELL_PROBE_TIMEOUT_MS, - }); - const liveNames = Array.from(parseLiveSandboxNames(liveList.output)); - for (const name of liveNames) { - const metadata = metadataByName.get(name) || undefined; - if (upsertRecoveredSandbox(name, metadata)) { - recoveredFromGateway += 1; - } - } - return recoveredFromGateway; -} - -function applyRecoveredDefault( - currentDefaultSandbox: string | null, - requestedSandboxName: string | null, - session: Session | null, -) { - const recovered = registry.listSandboxes(); - const preferredDefault = - requestedSandboxName || (!currentDefaultSandbox ? session?.sandboxName || null : null); - if ( - preferredDefault && - recovered.sandboxes.some((sandbox: { name: string }) => sandbox.name === preferredDefault) - ) { - registry.setDefault(preferredDefault); - } - return registry.listSandboxes(); -} - -async function recoverRegistryEntries({ - requestedSandboxName = null, -}: { requestedSandboxName?: string | null } = {}) { - const current = registry.listSandboxes(); - const session = onboardSession.loadSession(); - const recoveryCheck = shouldRecoverRegistryEntries(current, session, requestedSandboxName); - if (!recoveryCheck.shouldRecover) { - return { ...current, recoveredFromSession: false, recoveredFromGateway: 0 }; - } - - const seeded = seedRecoveryMetadata(current, session, requestedSandboxName); - const shouldProbeLiveGateway = - current.sandboxes.length > 0 || Boolean(session?.sandboxName) || Boolean(requestedSandboxName); - const recoveredFromGateway = shouldProbeLiveGateway - ? await recoverRegistryFromLiveGateway(seeded.metadataByName) - : 0; - const recovered = applyRecoveredDefault(current.defaultSandbox, requestedSandboxName, session); - return { - ...recovered, - recoveredFromSession: seeded.recoveredFromSession, - recoveredFromGateway, - }; -} - exports.runtimeBridge = { - captureOpenshell, - recoverNamedGatewayRuntime, - recoverRegistryEntries, runOpenshell, sandboxConnect, sandboxDestroy, @@ -634,109 +463,6 @@ exports.ensureLiveSandboxOrExit = ensureLiveSandboxOrExit; exports.G = G; exports.R = R; -function hasNamedGateway(output = ""): boolean { - return stripAnsi(output).includes("Gateway: nemoclaw"); -} - -function getActiveGatewayName(output = ""): string | null { - const match = stripAnsi(output).match(/^\s*Gateway:\s+(.+?)\s*$/m); - return match ? match[1].trim() : null; -} - -function getNamedGatewayLifecycleState() { - const status = captureOpenshell(["status"], { timeout: OPENSHELL_PROBE_TIMEOUT_MS }); - const gatewayInfo = captureOpenshell(["gateway", "info", "-g", "nemoclaw"], { - timeout: OPENSHELL_PROBE_TIMEOUT_MS, - }); - const cleanStatus = stripAnsi(status.output); - const activeGateway = getActiveGatewayName(status.output); - const connected = /^\s*Status:\s*Connected\b/im.test(cleanStatus); - const named = hasNamedGateway(gatewayInfo.output); - const refusing = /Connection refused|client error \(Connect\)|tcp connect error/i.test( - cleanStatus, - ); - if (connected && activeGateway === "nemoclaw" && named) { - return { - state: "healthy_named", - status: status.output, - gatewayInfo: gatewayInfo.output, - activeGateway, - }; - } - if (activeGateway === "nemoclaw" && named && refusing) { - return { - state: "named_unreachable", - status: status.output, - gatewayInfo: gatewayInfo.output, - activeGateway, - }; - } - if (activeGateway === "nemoclaw" && named) { - return { - state: "named_unhealthy", - status: status.output, - gatewayInfo: gatewayInfo.output, - activeGateway, - }; - } - if (connected) { - return { - state: "connected_other", - status: status.output, - gatewayInfo: gatewayInfo.output, - activeGateway, - }; - } - return { - state: "missing_named", - status: status.output, - gatewayInfo: gatewayInfo.output, - activeGateway, - }; -} - -/** Attempt to recover the named NemoClaw gateway after a restart or connectivity loss. */ -async function recoverNamedGatewayRuntime() { - const before = getNamedGatewayLifecycleState(); - if (before.state === "healthy_named") { - return { recovered: true, before, after: before, attempted: false }; - } - - runOpenshell(["gateway", "select", "nemoclaw"], { - ignoreError: true, - timeout: OPENSHELL_OPERATION_TIMEOUT_MS, - }); - let after = getNamedGatewayLifecycleState(); - if (after.state === "healthy_named") { - process.env.OPENSHELL_GATEWAY = "nemoclaw"; - return { recovered: true, before, after, attempted: true, via: "select" }; - } - - const shouldStartGateway = [before.state, after.state].some((state) => - ["missing_named", "named_unhealthy", "named_unreachable", "connected_other"].includes(state), - ); - - if (shouldStartGateway) { - try { - await startGatewayForRecovery(); - } catch { - // Fall through to the lifecycle re-check below so we preserve the - // existing recovery result shape and emit the correct classification. - } - runOpenshell(["gateway", "select", "nemoclaw"], { - ignoreError: true, - timeout: OPENSHELL_OPERATION_TIMEOUT_MS, - }); - after = getNamedGatewayLifecycleState(); - if (after.state === "healthy_named") { - process.env.OPENSHELL_GATEWAY = "nemoclaw"; - return { recovered: true, before, after, attempted: true, via: "start" }; - } - } - - return { recovered: false, before, after, attempted: true }; -} - function mergeLivePolicyIntoSandboxOutput(output: string, livePolicyOutput: string): string { const rawLines = String(output).split("\n"); const cleanLines = stripAnsi(String(output)).split("\n");