diff --git a/ci/onboard-entry-composition-budget.json b/ci/onboard-entry-composition-budget.json new file mode 100644 index 00000000000..65c3c4f583e --- /dev/null +++ b/ci/onboard-entry-composition-budget.json @@ -0,0 +1,20 @@ +{ + "$comment": "SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\nSPDX-License-Identifier: Apache-2.0\nProvider decisions remain until #9169. Messaging decisions remain until #9170. Policy decisions remain until #9172. The budget permits no gateway decisions.", + "gateway": {}, + "messaging": { + "createSandboxWithBaseImageResolution": 9, + "runOnboard": 1 + }, + "policy": { + "createSandboxWithBaseImageResolution": 6, + "runOnboard": 5 + }, + "provider": { + "createSandboxWithBaseImageResolution": 15, + "handleNimLocalSelection": 32, + "handleRemoteProviderSelection": 80, + "handleRoutedSelection": 15, + "runOnboard": 8, + "selectAndValidateOllamaModel": 18 + } +} diff --git a/ci/source-architecture-budget.json b/ci/source-architecture-budget.json index 26d55a168d5..44c32252372 100644 --- a/ci/source-architecture-budget.json +++ b/ci/source-architecture-budget.json @@ -48,7 +48,7 @@ "src/lib/actions/uninstall/run-plan.ts": 26, "src/lib/inference/onboard-probes.ts": 21, "src/lib/inference/vllm.ts": 21, - "src/lib/onboard.ts": 210, + "src/lib/onboard.ts": 202, "src/lib/onboard/machine/handlers/sandbox.ts": 21, "src/lib/sandbox/config.ts": 22, "src/lib/shields/index.ts": 23 diff --git a/scripts/checks/onboard-entry-composition.mts b/scripts/checks/onboard-entry-composition.mts new file mode 100644 index 00000000000..197595c6ecb --- /dev/null +++ b/scripts/checks/onboard-entry-composition.mts @@ -0,0 +1,340 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import ts from "typescript"; + +export type OnboardDecisionCategory = "gateway" | "messaging" | "policy" | "provider"; +export type OnboardDecisionCounts = Readonly>; +export type OnboardEntryCompositionBudget = Readonly< + Record +>; +export type OnboardEntryCompositionViolation = { + readonly kind: "new-decision" | "decision-ratchet"; + readonly category: OnboardDecisionCategory; + readonly declaration: string; + readonly actualCount: number; + readonly budgetCount: number; +}; + +const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); +const ENTRY_PATH = path.join(REPO_ROOT, "src/lib/onboard.ts"); +const BUDGET_PATH = path.join(REPO_ROOT, "ci/onboard-entry-composition-budget.json"); +const CATEGORIES = ["gateway", "messaging", "policy", "provider"] as const; +const LOGICAL_OPERATORS = new Set([ + ts.SyntaxKind.AmpersandAmpersandToken, + ts.SyntaxKind.BarBarToken, + ts.SyntaxKind.QuestionQuestionToken, +]); +const RECOVERY_NAME = /recover|recovery|repair|restore|retry|fallback|rollback/i; + +function declarationBody(node: ts.Node): ts.ConciseBody | undefined { + if (ts.isFunctionDeclaration(node)) return node.body; + if (!ts.isVariableStatement(node)) return undefined; + for (const declaration of node.declarationList.declarations) { + const initializer = declaration.initializer; + if (initializer && (ts.isArrowFunction(initializer) || ts.isFunctionExpression(initializer))) { + return initializer.body; + } + } + return undefined; +} + +function declarationName(node: ts.Node): string | null { + if (ts.isFunctionDeclaration(node)) return node.name?.text ?? null; + if (!ts.isVariableStatement(node)) return null; + for (const declaration of node.declarationList.declarations) { + if ( + ts.isIdentifier(declaration.name) && + declaration.initializer && + (ts.isArrowFunction(declaration.initializer) || + ts.isFunctionExpression(declaration.initializer)) + ) { + return declaration.name.text; + } + } + return null; +} + +function isGatewayLifecycleIdentifier(identifier: string): boolean { + if (!/gateway/i.test(identifier)) return false; + if ( + /toolGateway|gatewayRoute|routeGateway|gatewayProvider|providerExistsInGateway|readGatewayProviderMetadata/i.test( + identifier, + ) + ) { + return false; + } + if ( + /gatewayCredential|gatewayEnvironment|gatewayName|gatewayPort|gatewayUrl|gatewayEndpoint/i.test( + identifier, + ) + ) { + return false; + } + return ( + /^(?:chooseGateway|gatewayState)$/i.test(identifier) || + /(?:start|stop|restart|launch|destroy|recover|repair|retire|terminate|kill|wait|ensure|attach|register|reuse).*gateway/i.test( + identifier, + ) || + /gateway.*(?:start|stop|restart|launch|destroy|recover|repair|retire|terminate|kill|wait|health|ready|readiness|running|stale|process|runtime|lifecycle)/i.test( + identifier, + ) + ); +} + +function identifierCategories(identifier: string): ReadonlySet { + const categories = new Set(); + if (isGatewayLifecycleIdentifier(identifier)) categories.add("gateway"); + if (/messaging|channel/i.test(identifier)) categories.add("messaging"); + if (/policy|preset/i.test(identifier)) categories.add("policy"); + if (/provider|inference|nim|ollama|routed|model/i.test(identifier)) categories.add("provider"); + return categories; +} + +function isLogicalDecision(node: ts.Node): node is ts.BinaryExpression { + return ts.isBinaryExpression(node) && LOGICAL_OPERATORS.has(node.operatorToken.kind); +} + +function isRecoveryCall(node: ts.Node): node is ts.CallExpression { + return ts.isCallExpression(node) && RECOVERY_NAME.test(node.expression.getText()); +} + +// Count branches, short-circuit operators, condition-controlled loops, try statements, and +// named recovery calls. Sequencing loops do not choose onboarding behavior. +function isDecisionNode(node: ts.Node): boolean { + return ( + ts.isIfStatement(node) || + ts.isSwitchStatement(node) || + ts.isConditionalExpression(node) || + isLogicalDecision(node) || + ts.isForStatement(node) || + ts.isWhileStatement(node) || + ts.isDoStatement(node) || + ts.isTryStatement(node) || + isRecoveryCall(node) + ); +} + +function decisionNodeCategories(node: ts.Node): ReadonlySet { + const categories = new Set(); + + function addIdentifiers(candidate: ts.Node): void { + if (ts.isIdentifier(candidate)) { + for (const category of identifierCategories(candidate.text)) categories.add(category); + } + ts.forEachChild(candidate, addIdentifiers); + } + + function scanCondition(candidate: ts.Node, root: boolean): void { + if (!root && isDecisionNode(candidate)) return; + if (ts.isIdentifier(candidate)) { + for (const category of identifierCategories(candidate.text)) categories.add(category); + } + ts.forEachChild(candidate, (child) => scanCondition(child, false)); + } + + function scanActions(candidate: ts.Node, root: boolean): void { + if (!root && isDecisionNode(candidate)) return; + if (ts.isCallExpression(candidate) || ts.isNewExpression(candidate)) { + addIdentifiers(candidate.expression); + return; + } + if ( + ts.isBinaryExpression(candidate) && + candidate.operatorToken.kind >= ts.SyntaxKind.FirstAssignment && + candidate.operatorToken.kind <= ts.SyntaxKind.LastAssignment + ) { + addIdentifiers(candidate.left); + scanActions(candidate.right, false); + return; + } + if (ts.isDeleteExpression(candidate)) { + addIdentifiers(candidate.expression); + return; + } + ts.forEachChild(candidate, (child) => scanActions(child, false)); + } + + if (ts.isIfStatement(node)) { + scanCondition(node.expression, false); + scanActions(node.thenStatement, true); + if (node.elseStatement) scanActions(node.elseStatement, true); + } else if (ts.isSwitchStatement(node)) { + scanCondition(node.expression, false); + scanActions(node.caseBlock, true); + } else if (ts.isConditionalExpression(node)) { + scanCondition(node.condition, false); + scanActions(node.whenTrue, true); + scanActions(node.whenFalse, true); + } else if (isLogicalDecision(node)) { + scanCondition(node, true); + } else if (ts.isForStatement(node)) { + if (node.condition) scanCondition(node.condition, false); + scanActions(node.statement, true); + } else if (ts.isWhileStatement(node) || ts.isDoStatement(node)) { + scanCondition(node.expression, false); + scanActions(node.statement, true); + } else if (ts.isTryStatement(node)) { + scanActions(node.tryBlock, true); + if (node.catchClause) scanActions(node.catchClause, true); + if (node.finallyBlock) scanActions(node.finallyBlock, true); + } else if (isRecoveryCall(node)) { + addIdentifiers(node.expression); + } + return categories; +} + +function decisionCounts( + name: string, + body: ts.ConciseBody, +): Record> { + const nameCategories = identifierCategories(name); + const counts: Record> = { + gateway: {}, + messaging: {}, + policy: {}, + provider: {}, + }; + + function visit(node: ts.Node): void { + if (isDecisionNode(node)) { + const categories = new Set([...nameCategories, ...decisionNodeCategories(node)]); + for (const category of categories) { + counts[category][name] = (counts[category][name] ?? 0) + 1; + } + } + ts.forEachChild(node, visit); + } + + visit(body); + return counts; +} + +function sortCounts(counts: Record): Record { + return Object.fromEntries( + Object.entries(counts).sort(([left], [right]) => left.localeCompare(right)), + ); +} + +export function collectOnboardEntryDecisions(sourceText: string): OnboardEntryCompositionBudget { + const sourceFile = ts.createSourceFile( + "src/lib/onboard.ts", + sourceText, + ts.ScriptTarget.Latest, + true, + ts.ScriptKind.TS, + ); + const decisions: Record> = { + gateway: {}, + messaging: {}, + policy: {}, + provider: {}, + }; + + for (const statement of sourceFile.statements) { + const name = declarationName(statement); + const body = declarationBody(statement); + if (!name || !body) continue; + const declarationCounts = decisionCounts(name, body); + for (const category of CATEGORIES) { + for (const [declaration, count] of Object.entries(declarationCounts[category])) { + decisions[category][declaration] = (decisions[category][declaration] ?? 0) + count; + } + } + } + + return Object.fromEntries( + CATEGORIES.map((category) => [category, sortCounts(decisions[category])]), + ) as Record>; +} + +function parseDecisionCounts( + value: unknown, + category: OnboardDecisionCategory, +): OnboardDecisionCounts { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error(`${BUDGET_PATH}.${category} must contain declaration occurrence counts`); + } + const entries = Object.entries(value as Record); + if ( + entries.some( + ([name, count]) => + !name.trim() || typeof count !== "number" || !Number.isSafeInteger(count) || count < 1, + ) + ) { + throw new Error(`${BUDGET_PATH}.${category} must contain positive integer occurrence counts`); + } + return sortCounts(Object.fromEntries(entries) as Record); +} + +export function parseOnboardEntryCompositionBudget( + sourceText: string, +): OnboardEntryCompositionBudget { + const parsed = JSON.parse(sourceText) as Record; + return Object.fromEntries( + CATEGORIES.map((category) => [category, parseDecisionCounts(parsed[category], category)]), + ) as Record; +} + +export function evaluateOnboardEntryComposition( + actual: OnboardEntryCompositionBudget, + budget: OnboardEntryCompositionBudget, +): OnboardEntryCompositionViolation[] { + const violations: OnboardEntryCompositionViolation[] = []; + for (const category of CATEGORIES) { + const declarations = new Set([ + ...Object.keys(actual[category]), + ...Object.keys(budget[category]), + ]); + for (const declaration of declarations) { + const actualCount = actual[category][declaration] ?? 0; + const budgetCount = budget[category][declaration] ?? 0; + if (actualCount === budgetCount) continue; + violations.push({ + kind: actualCount > budgetCount ? "new-decision" : "decision-ratchet", + category, + declaration, + actualCount, + budgetCount, + }); + } + } + return violations.sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b))); +} + +export function formatOnboardEntryCompositionViolations( + violations: readonly OnboardEntryCompositionViolation[], +): string { + return [ + "Onboarding entry composition boundary failed.", + "", + ...violations.map((violation) => + violation.kind === "new-decision" + ? `- ${violation.declaration}: ${violation.category} decisions increased from ${violation.budgetCount} to ${violation.actualCount} in src/lib/onboard.ts.` + : `- ${violation.declaration}: ${violation.category} decisions decreased from ${violation.budgetCount} to ${violation.actualCount}. Lower the budget.`, + ), + ].join("\n"); +} + +function totalDecisions(counts: OnboardDecisionCounts): number { + return Object.values(counts).reduce((total, count) => total + count, 0); +} + +function main(): void { + const actual = collectOnboardEntryDecisions(readFileSync(ENTRY_PATH, "utf8")); + const budget = parseOnboardEntryCompositionBudget(readFileSync(BUDGET_PATH, "utf8")); + const violations = evaluateOnboardEntryComposition(actual, budget); + if (violations.length > 0) { + console.error(formatOnboardEntryCompositionViolations(violations)); + process.exitCode = 1; + return; + } + console.log( + `Onboarding entry composition boundary passed. Decision counts: ${CATEGORIES.map((category) => `${category} ${totalDecisions(actual[category])}`).join(", ")}.`, + ); +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) main(); diff --git a/scripts/checks/run.mts b/scripts/checks/run.mts index 1baa69006e6..4cb03ecfbe1 100644 --- a/scripts/checks/run.mts +++ b/scripts/checks/run.mts @@ -78,6 +78,11 @@ export const CHECKS: readonly CheckCommand[] = [ command: TSX, args: ["scripts/checks/source-architecture.mts"], }, + { + name: "onboard-entry-composition", + command: TSX, + args: ["scripts/checks/onboard-entry-composition.mts"], + }, { name: "no-test-dist-imports", command: TSX, diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 1f100e0d9a7..728a744b3f2 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -74,11 +74,7 @@ const dockerGpuLocalInference: typeof import("./onboard/docker-gpu-local-inferen const dockerGpuSandboxCreate: typeof import("./onboard/docker-gpu-sandbox-create") = require("./onboard/docker-gpu-sandbox-create"); const dockerGpuRoute: typeof import("./onboard/docker-gpu-route") = require("./onboard/docker-gpu-route"); const sandboxGpuCreateFlow: typeof import("./onboard/sandbox-gpu-create-flow") = require("./onboard/sandbox-gpu-create-flow"); -const dockerDriverGatewayLaunch: typeof import("./onboard/docker-driver-gateway-launch") = require("./onboard/docker-driver-gateway-launch"); const dockerDriverGatewayRuntime: typeof import("./onboard/docker-driver-gateway-runtime") = require("./onboard/docker-driver-gateway-runtime"); -const dockerDriverGatewayCutover: typeof import("./onboard/docker-driver-gateway-cutover") = require("./onboard/docker-driver-gateway-cutover"); -const { reapHostGatewayBeforeLaunchOrFail, reapDuplicateHostGatewaysExceptOrFail } = - require("./onboard/docker-driver-gateway-prelaunch") as typeof import("./onboard/docker-driver-gateway-prelaunch"); const { findReadableNvidiaCdiSpecFiles, parseDockerCdiSpecDirs, @@ -317,14 +313,8 @@ const { rejectUnsupportedWindowsHostOllama, shouldFrontOllamaWithProxy, }: typeof import("./onboard/local-inference-topology") = require("./onboard/local-inference-topology"); -const { - formatGatewayHealthWaitLimit, - getGatewayHealthWaitConfig, - waitForGatewayHealth, -}: typeof import("./onboard/gateway-health-wait") = require("./onboard/gateway-health-wait"); -const { - waitForStandaloneDockerDriverGateway, -}: typeof import("./onboard/docker-driver-gateway-readiness") = require("./onboard/docker-driver-gateway-readiness"); +const { getGatewayHealthWaitConfig }: typeof import("./onboard/gateway-health-wait") = + require("./onboard/gateway-health-wait"); const { resolveOpenshell } = require("./adapters/openshell/resolve"); const credentials: typeof import("./credentials/store") = require("./credentials/store"); const { @@ -509,16 +499,19 @@ const { } = require("./onboard/gateway-http-readiness") as typeof import("./onboard/gateway-http-readiness"); const { isGatewayTcpReady: probeGatewayTcpReady } = require("./onboard/gateway-tcp-readiness") as typeof import("./onboard/gateway-tcp-readiness"); -const { trackChildExit } = - require("./onboard/child-exit-tracker") as typeof import("./onboard/child-exit-tracker"); -const { reportDockerDriverGatewayStartFailure: reportGatewayFailure } = - require("./onboard/docker-driver-gateway-failure") as typeof import("./onboard/docker-driver-gateway-failure"); -const { normalizeGatewayStartError } = - require("./onboard/gateway-start-failure") as typeof import("./onboard/gateway-start-failure"); const dockerDriverGatewayEnv: typeof import("./onboard/docker-driver-gateway-env") = require("./onboard/docker-driver-gateway-env"); -const dockerDriverGatewayRuntimeMarker: typeof import("./onboard/docker-driver-gateway-runtime-marker") = - require("./onboard/docker-driver-gateway-runtime-marker"); +const { + createDockerDriverGatewayStart, + createGatewayLifecycleApplication, + createGatewayRecoveryOrchestration, + createGatewayRegistration, + createGatewayStart, +} = require("./onboard/gateway/application") as typeof import("./onboard/gateway/application"); +const { createGatewayProcessLifecycle } = + require("./onboard/gateway/process-lifecycle") as typeof import("./onboard/gateway/process-lifecycle"); +const entryDecisions: typeof import("./onboard/gateway/entry-decisions") = + require("./onboard/gateway/entry-decisions"); const gatewayBinding: typeof import("./onboard/gateway-binding") = require("./onboard/gateway-binding"); const fatalRuntimePreflight: typeof import("./onboard/fatal-runtime-preflight") = require("./onboard/fatal-runtime-preflight"); @@ -554,7 +547,6 @@ const sandboxCreateFailureDiagnostics: typeof import("./onboard/sandbox-create-f import type { CurlProbeResult } from "./adapters/http/probe"; import type { AgentDefinition } from "./agent/defs"; -import { gatewayStartGuidance } from "./gateway-start-guidance"; import type { WebSearchConfig } from "./inference/web-search"; import { hydrateMessagingChannelConfig, @@ -1099,112 +1091,39 @@ function getOpenShellInstallDeps( }; } -function runQuietOpenshell(args: string[]) { - return runOpenshell(args, { - ignoreError: true, - stdio: ["ignore", "pipe", "pipe"], - suppressOutput: true, - }); -} - -function removeDockerDriverGatewayRegistration(): boolean { - const removeResult = runQuietOpenshell(["gateway", "remove", GATEWAY_NAME]); - if (removeResult.status === 0) return true; - - // OpenShell dev builds before NVIDIA/OpenShell#1221 used `gateway destroy` - // for local metadata cleanup. Post-#1221 builds removed lifecycle verbs and - // use `gateway remove` instead, so keep both forms quiet and best-effort. - const destroyResult = runQuietOpenshell(["gateway", "destroy", "-g", GATEWAY_NAME]); - return destroyResult.status === 0; -} - -function terminateDockerDriverGatewayProcess(pid: number): boolean { - if (!isPidAlive(pid)) { - return false; - } - - try { - process.kill(pid, "SIGTERM"); - for (let i = 0; i < 10; i += 1) { - if (!isPidAlive(pid)) break; - sleepSeconds(1); - } - if (isPidAlive(pid)) process.kill(pid, "SIGKILL"); - return true; - } catch { - return false; - } -} - -function stopDockerDriverGatewayProcess(): boolean { - const pid = getDockerDriverGatewayPid(); - if (pid === null || !isPidAlive(pid)) { - clearDockerDriverGatewayRuntimeFiles(); - return false; - } - if (!isDockerDriverGatewayProcess(pid, resolveOpenShellGatewayBinary())) { - clearDockerDriverGatewayRuntimeFiles(); - return false; - } - - const stopped = terminateDockerDriverGatewayProcess(pid); - clearDockerDriverGatewayRuntimeFiles(); - return stopped; -} - -function stopLegacyGatewayClusterContainer(): boolean { - const containerName = getGatewayClusterContainerName(GATEWAY_NAME); - const inspectResult = dockerInspect(["--type", "container", containerName], { - ignoreError: true, - suppressOutput: true, - }); - if (inspectResult.status !== 0) return false; - - dockerStop(containerName, { - ignoreError: true, - suppressOutput: true, - }); - dockerRm(containerName, { - ignoreError: true, - suppressOutput: true, - }); - - const postInspectResult = dockerInspect(["--type", "container", containerName], { - ignoreError: true, - suppressOutput: true, - }); - return postInspectResult.status !== 0; -} - -function retireLegacyGatewayForDockerDriverUpgrade(): void { - runOpenshell(["forward", "stop", String(getOnboardDashboardPort())], { ignoreError: true }); - stopDockerDriverGatewayProcess(); - const stoppedLegacyContainer = stopLegacyGatewayClusterContainer(); - removeDockerDriverGatewayRegistration(); - if (stoppedLegacyContainer) { - console.log(" ✓ Legacy OpenShell gateway container stopped for Docker-driver upgrade"); - } -} - function logDockerDriverGatewayRestart(reason: string): void { console.log(` Existing OpenShell Docker-driver gateway is stale (${reason}); restarting...`); } -function destroyGateway( - clearRegistry: () => void = registry.clearAll, - isDockerDriverGatewayEnabledForDestroy: () => boolean = isLinuxDockerDriverGatewayEnabled, -): boolean { - return destroyGatewayWithVolumeCleanup({ - clearRegistry, - dockerRemoveVolumesByPrefix, - gatewayName: GATEWAY_NAME, - hasLifecycleCommands: () => gatewayCliSupportsLifecycleCommands(runCaptureOpenshell), - isDockerDriverGatewayEnabled: isDockerDriverGatewayEnabledForDestroy, - removeDockerDriverGatewayRegistration, - runOpenshell, - stopDockerDriverGatewayProcess, - }); -} +const { + destroyGateway, + removeDockerDriverGatewayRegistration, + retireLegacyGatewayForDockerDriverUpgrade, + runQuietOpenshell, + stopDockerDriverGatewayProcess, +} = createGatewayProcessLifecycle({ + gatewayName: () => GATEWAY_NAME, + dashboardPort: getOnboardDashboardPort, + runOpenshell, + runCaptureOpenshell, + dockerInspect, + dockerStop, + dockerRm, + dockerRemoveVolumesByPrefix, + getGatewayClusterContainerName, + getDockerDriverGatewayPid, + isPidAlive, + isDockerDriverGatewayProcess, + resolveOpenShellGatewayBinary, + clearDockerDriverGatewayRuntimeFiles, + sleepSeconds, + isDockerDriverGatewayEnabled: isLinuxDockerDriverGatewayEnabled, + clearRegistry: registry.clearAll, + killProcess: process.kill.bind(process), + log: console.log, + gatewayCliSupportsLifecycleCommands, + destroyGatewayWithVolumeCleanup, +}); function getGatewayClusterContainerState(): string { const containerName = getGatewayClusterContainerName(GATEWAY_NAME); @@ -1239,79 +1158,6 @@ const { gatewayClusterHealthcheckPassed, repairGatewayBootstrapSecrets } = runCapture, }); -function registerDockerDriverGatewayEndpoint(): boolean { - const selectExisting = runQuietOpenshell(["gateway", "select", GATEWAY_NAME]); - if (selectExisting.status === 0) { - const status = runCaptureOpenshell(["status"], { ignoreError: true }); - const namedInfo = runCaptureOpenshell(["gateway", "info", "-g", GATEWAY_NAME], { - ignoreError: true, - }); - const currentInfo = runCaptureOpenshell(["gateway", "info"], { ignoreError: true }); - if (isGatewayHealthy(status, namedInfo, currentInfo)) { - process.env.OPENSHELL_GATEWAY = GATEWAY_NAME; - return true; - } - } - - let addResult = runOpenshell( - ["gateway", "add", getDockerDriverGatewayEndpointArg(), "--local", "--name", GATEWAY_NAME], - { ignoreError: true, suppressOutput: true }, - ); - if (addResult.status !== 0) { - removeDockerDriverGatewayRegistration(); - addResult = runOpenshell( - ["gateway", "add", getDockerDriverGatewayEndpointArg(), "--local", "--name", GATEWAY_NAME], - { ignoreError: true, suppressOutput: true }, - ); - } - const selectResult = runOpenshell(["gateway", "select", GATEWAY_NAME], { - ignoreError: true, - suppressOutput: true, - }); - const ok = - (addResult.status === 0 && selectResult.status === 0) || - (selectResult.status === 0 && - isGatewayHealthy( - runCaptureOpenshell(["status"], { ignoreError: true }), - runCaptureOpenshell(["gateway", "info", "-g", GATEWAY_NAME], { ignoreError: true }), - runCaptureOpenshell(["gateway", "info"], { ignoreError: true }), - )); - if (ok) { - process.env.OPENSHELL_GATEWAY = GATEWAY_NAME; - } else if (process.env.OPENSHELL_GATEWAY === GATEWAY_NAME) { - delete process.env.OPENSHELL_GATEWAY; - } - return ok; -} - -function attachGatewayMetadataIfNeeded({ - forceRefresh = false, -}: { - forceRefresh?: boolean; -} = {}): boolean { - const gwInfo = runCaptureOpenshell(["gateway", "info", "-g", GATEWAY_NAME], { - ignoreError: true, - }); - // runCaptureOpenshell may return stale-but-present gateway metadata. When - // hasStaleGateway(gwInfo) is truthy we skip runOpenshell unless a repair - // flow explicitly forces a refresh after recreating bootstrap secrets. - if (!forceRefresh && hasStaleGateway(gwInfo)) return true; - - if (isLinuxDockerDriverGatewayEnabled()) { - return registerDockerDriverGatewayEndpoint(); - } - - const addResult = runOpenshell( - ["gateway", "add", getGatewayLocalEndpoint(), "--local", "--name", GATEWAY_NAME], - { ignoreError: true, suppressOutput: true }, - ); - if (addResult.status === 0) { - console.log(" ✓ Gateway metadata reattached"); - return true; - } - return false; -} - // parsePolicyPresetEnv — see urlUtils import above // isSafeModelId — see validation import above @@ -1354,7 +1200,7 @@ async function preflight( externallySupervised: gatewayExternallySupervised, gatewayReuseState: initialGatewayReuseState, } = await onboardPreflightGatewayAuthority.prepareGatewayAuthority(); - let gatewayReuseState = initialGatewayReuseState; + let reuseState = initialGatewayReuseState; // Verify the legacy gateway container is actually running — openshell CLI // metadata can be stale after a manual `docker rm`. See #2020. Newer @@ -1362,8 +1208,8 @@ async function preflight( // Docker container, so the live CLI health check is the source of truth. // The reuse/cleanup/orphan stages run as one composed sequence so external // supervision is enforced across the whole path, not per stage (#6576). - gatewayReuseState = await runPreflightGatewaySequence({ - gatewayReuseState, + reuseState = await runPreflightGatewaySequence({ + gatewayReuseState: reuseState, externallySupervised: gatewayExternallySupervised, supportsLifecycleCommands: gatewayCliSupportsLifecycleCommands(runCaptureOpenshell), isDockerDriverGatewayEnabled: isLinuxDockerDriverGatewayEnabled(), @@ -1411,8 +1257,10 @@ async function preflight( dashboardLabel: `${cliDisplayName()} dashboard`, }); for (const { kind, port, label, envVar } of requiredPorts) { - const portCheckOptions = - kind === "gateway" ? dockerDriverGatewayEnv.getGatewayPortCheckOptions() : undefined; + const portCheckOptions = entryDecisions.selectGatewayPortCheckOptions( + kind, + dockerDriverGatewayEnv.getGatewayPortCheckOptions, + ); let portCheck = await checkPortAvailable(port, portCheckOptions); if (!portCheck.ok) { const reuse = await applyHealthyPortReuse({ @@ -1422,7 +1270,7 @@ async function preflight( label, runtimeDisplayName: cliDisplayName(), gatewayName: GATEWAY_NAME, - gatewayReuseState, + gatewayReuseState: reuseState, externallySupervised: gatewayExternallySupervised, portCheckOptions, supportsLifecycleCommands: gatewayCliSupportsLifecycleCommands(runCaptureOpenshell), @@ -1433,18 +1281,24 @@ async function preflight( }); if (reuse === "continue") continue; if (reuse) { - ({ gatewayReuseState, portCheck } = reuse); + reuseState = reuse.gatewayReuseState; + portCheck = reuse.portCheck; if (portCheck.ok) continue; } - if (kind === "gateway") { - const dockerGatewayPid = getDockerDriverGatewayPortListenerPid(portCheck); - if (dockerGatewayPid !== null) { - rememberDockerDriverGatewayPid(dockerGatewayPid); + const managedListenerPid = entryDecisions.selectManagedListenerPid(kind, () => + getDockerDriverGatewayPortListenerPid(portCheck), + ); + const managedListenerAccepted = entryDecisions.acceptManagedListener( + managedListenerPid, + (pid) => { + rememberDockerDriverGatewayPid(pid); console.log( ` ✓ Port ${port} already owned by NemoClaw OpenShell Docker gateway (${label})`, ); - continue; - } + }, + ); + if (managedListenerAccepted) { + continue; } // Auto-cleanup orphaned SSH port-forward from a previous NemoClaw session // (e.g. dashboard forward left behind after destroy). Only kill the process @@ -1527,287 +1381,6 @@ async function preflight( // ── Step 2: Gateway ────────────────────────────────────────────── -/** - * Start or reuse the OpenShell gateway for the current runtime provider. Only - * the Docker-driver provider starts a gateway process. Every other provider - * reuses a gateway that its own deployment started. - */ -async function startGatewayWithOptions( - _gpu: ReturnType, - { - exitOnFailure = true, - gpuPassthrough = false, - }: { exitOnFailure?: boolean; gpuPassthrough?: boolean } = {}, -) { - assertGatewayStartAllowed(exitOnFailure); - step(2, 8, "Starting OpenShell gateway"); - - if (isLinuxDockerDriverGatewayEnabled()) { - const selectedGpuRoute = dockerGpuRoute.initialDockerGpuRoute( - dockerGpuRoute.resolveDockerGpuRoutePlan( - { sandboxGpuEnabled: gpuPassthrough, hostGpuPlatform: _gpu?.platform }, - { - dockerDriverGateway: true, - dockerDesktopWsl: dockerGpuSandboxCreate.isDockerDesktopWslRuntime(), - }, - ), - ); - return startDockerDriverGateway({ - exitOnFailure, - skipSandboxBridgeReachability: dockerGpuLocalInference.shouldSkipGpuBridgeProbe( - gpuPassthrough, - _gpu?.platform, - selectedGpuRoute, - ), - }); - } - - const gatewaySnapshot = selectNamedGatewayForReuseIfNeeded(getGatewayReuseSnapshot()); - if ( - isGatewayHealthy( - gatewaySnapshot.gatewayStatus, - gatewaySnapshot.gwInfo, - gatewaySnapshot.activeGatewayInfo, - ) - ) { - // Final reuse gate — `isGatewayHealthy()` parses openshell CLI metadata, - // which can be stale when the gateway container was just restarted (e.g. - // after `colima stop && colima start`). Verify the gateway HTTP endpoint - // is actually serving before declaring reuse, so we don't skip startup - // and fail later in step 4 with "Connection refused". See #3258. - if (await isGatewayHttpReady()) { - console.log(" ✓ Reusing existing gateway"); - runOpenshell(["gateway", "select", GATEWAY_NAME], { ignoreError: true }); - process.env.OPENSHELL_GATEWAY = GATEWAY_NAME; - return; - } - console.log( - ` Gateway metadata reports healthy but ${getGatewayLocalEndpoint()}/ is not responding.`, - ); - } - - if (hasStaleGateway(gatewaySnapshot.gwInfo)) { - console.log(" Stale gateway detected."); - } - - // Reuse is the only startup this runtime provider has. The OpenShell CLI has - // no command that starts a gateway, so the gateway process belongs to - // whichever deployment created it. Only the `openshell` launcher reaches - // here, and its guidance already opens with this same sentence, so print the - // guidance alone and keep the sentence for the thrown error. - const unstartableGateway = `${cliDisplayName()} does not start the '${GATEWAY_NAME}' gateway on this host.`; - console.error(` ${gatewayStartGuidance(GATEWAY_NAME)}`); - if (exitOnFailure) process.exit(1); - throw normalizeGatewayStartError(new Error(unstartableGateway)); -} - -async function startDockerDriverGateway({ - exitOnFailure = true, - skipSandboxBridgeReachability = false, -}: { - exitOnFailure?: boolean; - skipSandboxBridgeReachability?: boolean; -} = {}): Promise { - const gatewayBin = resolveOpenShellGatewayBinary(); - const openshellVersionOutput = runCaptureOpenshell(["--version"], { ignoreError: true }); - const gatewayEnv = getDockerDriverGatewayEnv(openshellVersionOutput); - const stateDir = getDockerDriverGatewayStateDir(); - const runtimeIdentity = gatewayBin - ? dockerDriverGatewayLaunch.buildDockerDriverGatewayRuntimeIdentity({ - gatewayBin, - gatewayEnv, - stateDir, - sandboxBin: resolveOpenShellSandboxBinary(), - gatewayName: GATEWAY_NAME, - compatContainerName: gatewayBinding.resolveGatewayCompatContainerName(GATEWAY_PORT), - ensureLocalTlsBundle: true, - }) - : null; - const gatewayLaunch = runtimeIdentity?.launch ?? null; - const driftGatewayBin = dockerDriverGatewayLaunch.resolveDriftGatewayBin( - runtimeIdentity, - gatewayBin, - ); - const driftGatewayEnv = runtimeIdentity?.desiredEnv ?? gatewayEnv; - const identityGatewayBin = runtimeIdentity?.identityGatewayBin ?? gatewayBin; - const { verifySandboxBridgeGatewayReachableOrExit } = - require("./onboard/gateway-sandbox-reachability") as typeof import("./onboard/gateway-sandbox-reachability"); - const initialPortCheck = await checkGatewayPortAvailable(); - const servicePortOwnership = createGatewayServicePortOwnership(initialPortCheck, { - exitOnFailure, - gatewayBin: identityGatewayBin, - preparePort: (extraPids) => - reapHostGatewayBeforeLaunchOrFail({ - stateDir, - gatewayBin: identityGatewayBin, - extraPids, - exitOnFailure, - }), - }); - const cutover = await dockerDriverGatewayCutover.runDockerDriverGatewayManagedFallback( - () => - dockerDriverGatewayEnv.startPackageManagedDockerDriverGatewayWithEnvOverride({ - clearDockerDriverGatewayRuntimeFiles, - exitOnFailure, - gatewayEnv: driftGatewayEnv, - gatewayName: GATEWAY_NAME, - isDockerDriverGatewayReady: () => - isDockerDriverGatewayHttpReady(undefined, undefined, driftGatewayEnv), - registerDockerDriverGatewayEndpoint, - preparePortForOpenShellGatewayUserServiceStart: servicePortOwnership.preparePort, - runCaptureOpenshell, - skipSandboxBridgeReachability, - validatePortOwnerForOpenShellGatewayUserServiceStart: - servicePortOwnership.validatePortOwner, - verifySandboxBridgeGatewayReachableOrExit: (fail, options) => - verifySandboxBridgeGatewayReachableOrExit(fail, { - ...options, - port: GATEWAY_PORT, - }), - }), - async () => - dockerDriverGatewayCutover.runDockerDriverGatewayCutover( - { - gatewayBin, - identityGatewayBin, - driftGatewayBin, - driftGatewayEnv, - exitOnFailure, - skipSandboxBridgeReachability, - stateDir, - portListenerScan: getDockerDriverGatewayPortListenerScan( - await checkGatewayPortAvailable(), - { gatewayBin: identityGatewayBin }, - ), - pidFileGatewayPid: getDockerDriverGatewayPid(), - initialHealth: dockerDriverGatewayCutover.readDockerDriverGatewayHealth( - runCaptureOpenshell, - GATEWAY_NAME, - ), - }, - { - isDockerDriverGatewayProcessAlive, - isGatewayHealthy, - getDockerDriverGatewayRuntimeDrift, - logDockerDriverGatewayRestart, - registerDockerDriverGatewayEndpoint, - isDockerDriverGatewayHttpReady: () => - isDockerDriverGatewayHttpReady(undefined, undefined, driftGatewayEnv), - verifySandboxBridgeGatewayReachableOrExit: (fail, options) => - verifySandboxBridgeGatewayReachableOrExit(fail, { - ...options, - port: GATEWAY_PORT, - }), - readGatewayHealth: () => ({ - status: runCaptureOpenshell(["status"], { ignoreError: true }), - namedInfo: runCaptureOpenshell(["gateway", "info", "-g", GATEWAY_NAME], { - ignoreError: true, - }), - activeInfo: runCaptureOpenshell(["gateway", "info"], { ignoreError: true }), - }), - rememberDockerDriverGatewayPid, - reapDuplicateHostGatewaysExceptOrFail, - reapHostGatewayBeforeLaunchOrFail, - isGatewayPortAvailable: async () => { - const probe = await checkGatewayPortAvailable(); - return probe.ok && !probe.warning; - }, - reportUntrustedGatewayPort: servicePortOwnership.reportUntrustedGatewayPort, - reportMissingGatewayBinary: () => { - console.error(" OpenShell Docker-driver gateway binary not found."); - console.error( - ` Install OpenShell v${SUPPORTED_OPENSHELL_FALLBACK_VERSION}, or set NEMOCLAW_OPENSHELL_GATEWAY_BIN.`, - ); - if (exitOnFailure) process.exit(1); - throw new Error("OpenShell gateway binary not found"); - }, - log: (message) => console.log(message), - }, - ), - ); - if (cutover !== "launch") return; - if (!gatewayBin || !gatewayLaunch) { - throw new Error("OpenShell gateway launch missing after cutover"); - } - fs.mkdirSync(stateDir, { recursive: true, mode: 0o700 }); - const logPath = path.join(stateDir, "openshell-gateway.log"); - const log = dockerDriverGatewayLaunch.openDockerDriverGatewayLog(logPath, { exitOnFailure }); - console.log(" Starting OpenShell Docker-driver gateway..."); - console.log(` Gateway log: ${logPath}`); - dockerDriverGatewayLaunch.prepareAndLogDockerDriverGatewayLaunch(gatewayLaunch); - const child = dockerDriverGatewayLaunch.spawnDockerDriverGateway(gatewayLaunch, log.fd); - const childExit = trackChildExit(child); // #3111 zombie-safe liveness - child.unref(); - const childPid = child.pid ?? 0; - if (childPid <= 0) { - throw new Error("OpenShell gateway process did not return a pid"); - } - rememberDockerDriverGatewayPid(childPid); - dockerDriverGatewayRuntimeMarker.writeDockerDriverGatewayRuntimeMarkerForStateDir( - getDockerDriverGatewayStateDir(), - { - pid: childPid, - desiredEnv: driftGatewayEnv, - endpoint: getDockerDriverGatewayEndpoint(), - gatewayBin: driftGatewayBin, - openshellVersion: getInstalledOpenshellVersion(openshellVersionOutput), - dockerHost: process.env.DOCKER_HOST || null, - }, - ); - const pollCount = envInt("NEMOCLAW_HEALTH_POLL_COUNT", 30); - const pollInterval = envInt("NEMOCLAW_HEALTH_POLL_INTERVAL", 2); - const gatewayStartup = await waitForStandaloneDockerDriverGateway({ - childExited: () => childExit.exited, - childPid, - gatewayName: GATEWAY_NAME, - healthPollCount: pollCount, - healthPollIntervalSeconds: pollInterval, - isGatewayHealthy, - isGatewayTcpReady, - isPidAlive, - onHealthy: async () => { - await verifySandboxBridgeGatewayReachableOrExit(exitOnFailure, { - skip: skipSandboxBridgeReachability, - port: GATEWAY_PORT, - }); - }, - registerGatewayEndpoint: registerDockerDriverGatewayEndpoint, - runCaptureOpenshell, - sleepSeconds, - }); - if (gatewayStartup === "healthy") { - console.log(" ✓ Docker-driver gateway is healthy"); - return; - } - reportGatewayFailure(logPath, childExit, { - exitOnFailure, - isGatewayStateInUse: isDockerDriverGatewayStateInUse, - launchLogOffset: log.startOffset, - }); - if (gatewayStartup === "exited") { - throw new Error("Docker-driver gateway failed to start because the process exited"); - } - const waitLimit = formatGatewayHealthWaitLimit(pollCount, pollInterval); - throw new Error(`Docker-driver gateway failed to start within ${waitLimit}`); -} - -async function startGateway( - _gpu: ReturnType, - { gpuPassthrough = false }: { gpuPassthrough?: boolean } = {}, -): Promise { - return startGatewayWithOptions(_gpu, { exitOnFailure: true, gpuPassthrough }); -} - -async function startGatewayForRecovery(options = {}): Promise { - return require("./onboard/gateway-recovery").startGatewayForRecovery(options, { - assertGatewayStartAllowed, - runCaptureOpenshell, - runOpenshell, - startGatewayWithOptions, - isLinuxDockerDriverGatewayEnabled, - }); -} - const applyOverlayfsAutoFix = overlayfsAutoFix.createOverlayfsAutoFix({ assessHost: preflightUtils.assessHost, ensurePatchedClusterImage: clusterImagePatch.ensurePatchedClusterImage, @@ -1836,54 +1409,107 @@ const { waitForGatewayHttpReady, }); -async function recoverGatewayRuntime() { - assertGatewayStartAllowed(false); - if (isLinuxDockerDriverGatewayEnabled()) { - try { - await startDockerDriverGateway({ exitOnFailure: false }); - return true; - } catch { - return false; - } - } +const gatewayRegistration = createGatewayRegistration({ + gatewayName: () => GATEWAY_NAME, + getDockerDriverGatewayEndpointArg, + getGatewayLocalEndpoint, + hasStaleGateway, + isGatewayHealthy, + isLinuxDockerDriverGatewayEnabled, + removeDockerDriverGatewayRegistration, + runCaptureOpenshell, + runOpenshell, + runQuietOpenshell, +}); - runOpenshell(["gateway", "select", GATEWAY_NAME], { ignoreError: true }); - const status = runCaptureOpenshell(["status"], { ignoreError: true }); - if (status.includes("Connected") && isSelectedGateway(status) && (await isGatewayHttpReady())) { - process.env.OPENSHELL_GATEWAY = GATEWAY_NAME; - return true; - } +const dockerDriverGatewayStart = createDockerDriverGatewayStart({ + SUPPORTED_OPENSHELL_FALLBACK_VERSION, + checkGatewayPortAvailable, + clearDockerDriverGatewayRuntimeFiles, + createGatewayServicePortOwnership, + dockerDriverGatewayEnv, + envInt, + gatewayBinding, + gatewayName: () => GATEWAY_NAME, + gatewayPort: () => GATEWAY_PORT, + getDockerDriverGatewayEndpoint, + getDockerDriverGatewayEnv, + getDockerDriverGatewayPid, + getDockerDriverGatewayPortListenerScan, + getDockerDriverGatewayRuntimeDrift, + getDockerDriverGatewayStateDir, + getInstalledOpenshellVersion, + isDockerDriverGatewayHttpReady, + isDockerDriverGatewayProcessAlive, + isDockerDriverGatewayStateInUse, + isGatewayHealthy, + isGatewayTcpReady, + isPidAlive, + logDockerDriverGatewayRestart, + registerDockerDriverGatewayEndpoint: gatewayRegistration.registerDockerDriverGatewayEndpoint, + rememberDockerDriverGatewayPid, + resolveOpenShellGatewayBinary, + resolveOpenShellSandboxBinary, + runCaptureOpenshell, + sleepSeconds, +}); - const recoveryWait = getGatewayHealthWaitConfig(0, getGatewayClusterContainerState()); - const recoveryPollCount = recoveryWait.extended - ? recoveryWait.count - : envInt("NEMOCLAW_HEALTH_POLL_COUNT", 10); - const recoveryPollInterval = recoveryWait.extended - ? recoveryWait.interval - : envInt("NEMOCLAW_HEALTH_POLL_INTERVAL", 2); - const healthy = await waitForGatewayHealth({ - attachGatewayMetadataIfNeeded, - gatewayClusterHealthcheckPassed, - gatewayName: GATEWAY_NAME, - healthPollCount: recoveryPollCount, - healthPollIntervalSeconds: recoveryPollInterval, - isGatewayHealthy, - isGatewayHttpReady: (signal) => isGatewayHttpReady(undefined, undefined, undefined, signal), - repairGatewayBootstrapSecrets, - runCaptureOpenshell, - sleepSeconds, - }); - if (!healthy) { - console.error(` ${gatewayStartGuidance(GATEWAY_NAME)}`); - return false; - } +const gatewayStart = createGatewayStart({ + assertGatewayStartAllowed, + cliDisplayName, + dockerGpuLocalInference, + dockerGpuRoute, + dockerGpuSandboxCreate, + gatewayName: () => GATEWAY_NAME, + getGatewayLocalEndpoint, + getGatewayReuseSnapshot, + hasStaleGateway, + isGatewayHealthy, + isGatewayHttpReady, + isLinuxDockerDriverGatewayEnabled, + runOpenshell, + selectNamedGatewayForReuseIfNeeded, + startDockerDriverGateway: dockerDriverGatewayStart.startDockerDriverGateway, + step, +}); - process.env.OPENSHELL_GATEWAY = GATEWAY_NAME; - if (shouldPatchCoredns(getContainerRuntime())) { - run(["bash", path.join(SCRIPTS, "fix-coredns.sh"), GATEWAY_NAME], { ignoreError: true }); - } - return true; -} +const gatewayRecovery = createGatewayRecoveryOrchestration({ + SCRIPTS, + assertGatewayStartAllowed, + attachGatewayMetadataIfNeeded: gatewayRegistration.attachGatewayMetadataIfNeeded, + envInt, + gatewayClusterHealthcheckPassed, + gatewayName: () => GATEWAY_NAME, + getContainerRuntime, + getGatewayClusterContainerState, + isGatewayHealthy, + isGatewayHttpReady, + isLinuxDockerDriverGatewayEnabled, + isSelectedGateway, + repairGatewayBootstrapSecrets, + run, + runCaptureOpenshell, + runOpenshell, + shouldPatchCoredns, + sleepSeconds, + startDockerDriverGateway: dockerDriverGatewayStart.startDockerDriverGateway, + startGatewayWithOptions: gatewayStart.startGatewayWithOptions, +}); + +const { + attachGatewayMetadataIfNeeded, + recoverGatewayRuntime, + registerDockerDriverGatewayEndpoint, + startDockerDriverGateway, + startGateway, + startGatewayForRecovery, + startGatewayWithOptions, +} = createGatewayLifecycleApplication({ + dockerDriverStart: dockerDriverGatewayStart, + recovery: gatewayRecovery, + registration: gatewayRegistration, + start: gatewayStart, +}); const { getSandboxRuntimeRegistryFields, hasSandboxGpuDrift, updateReusedSandboxMetadata } = sandboxRegistryMetadata.createSandboxRegistryMetadataHelpers({ @@ -3391,9 +3017,9 @@ const recordRepairEvent = onboardRuntimeBoundary.recordRepairEvent.bind(onboardR async function preflightAuthoritativeRebuildTarget( opts: import("./onboard/authoritative-rebuild-target").AuthoritativeRebuildPreflightOptions, ): Promise { - const authoritativeGateway = - authoritativeRebuildTarget.resolveAuthoritativeOnboardGatewayBinding(opts); - if (!authoritativeGateway) throw new Error("Authoritative rebuild preflight has no gateway"); + const authoritativeGateway = entryDecisions.requireGatewayBinding( + authoritativeRebuildTarget.resolveAuthoritativeOnboardGatewayBinding(opts), + ); const previous = { dashboardPort: _preflightDashboardPort, gatewayName: GATEWAY_NAME, @@ -3523,7 +3149,7 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { try { const lockedRuntime = await resumeRuntime.prepare(opts, resume, isNonInteractive(), onboardSession.loadSession); portableEnvScope = lockedRuntime.environmentScope; - if (!authoritativeGateway) delete process.env.OPENSHELL_GATEWAY; + entryDecisions.clearGatewayEnvironmentWithoutBinding(authoritativeGateway, process.env); preparedDcodeRuntime.applyGatewayEnv(process.env); if (isNonInteractive() && !validatePolicyTierBeforeRuntime) validatePolicyTierEnvEarly(); // Validate provider/model hints only after the locked profile and runtime authority are active. @@ -3538,11 +3164,11 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { ), ); const onboardingComputePlan = dockerDriverPlatform.resolveCurrentOpenShellComputePlan(); - if (authoritativeGateway) { - GATEWAY_NAME = authoritativeGateway.name; - GATEWAY_PORT = authoritativeGateway.port; - process.env.OPENSHELL_GATEWAY = authoritativeGateway.name; - } + entryDecisions.applyGatewayBindingIfPresent(authoritativeGateway, (binding) => { + GATEWAY_NAME = binding.name; + GATEWAY_PORT = binding.port; + process.env.OPENSHELL_GATEWAY = binding.name; + }); onboardTrace = onboardTracing.startOnboardTrace(opts, process.env); let selectedMessagingChannels: string[] = []; let { session, fromDockerfile } = await onboardSessionBootstrap.prepareOnboardSessionValidated( @@ -3630,14 +3256,17 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { const recordedSandboxName = session?.steps?.sandbox?.status === "complete" ? session?.sandboxName || null : null; const checkpointedSandboxName = onboardSessionBootstrap.getCheckpointedSandboxName(resume, agent, session); - const gatewaySandboxName = resume - ? (recordedSandboxName ?? requestedSandboxName ?? checkpointedSandboxName) - : null; + const gatewaySandboxName = entryDecisions.selectResumeSandboxName( + resume, + recordedSandboxName, + requestedSandboxName, + checkpointedSandboxName, + ); const onboardGateway = gatewayBinding.resolveCoreOnboardGatewayBinding({ authoritativeGateway, currentGateway: { name: GATEWAY_NAME, port: GATEWAY_PORT }, resume, - sandbox: gatewaySandboxName ? registry.getSandbox(gatewaySandboxName) : null, + sandbox: entryDecisions.readSandboxForGatewayBinding(gatewaySandboxName, registry.getSandbox), }); ({ name: GATEWAY_NAME, port: GATEWAY_PORT } = onboardGateway); process.env.OPENSHELL_GATEWAY = GATEWAY_NAME; @@ -4119,8 +3748,7 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { onboardTracing.finishOnboardTrace(onboardTrace, completed); GATEWAY_NAME = previousGatewayBinding.name; GATEWAY_PORT = previousGatewayBinding.port; - if (previousOpenshellGateway === undefined) delete process.env.OPENSHELL_GATEWAY; - else process.env.OPENSHELL_GATEWAY = previousOpenshellGateway; + entryDecisions.restoreGatewayEnvironment(process.env, previousOpenshellGateway); if (previousOpenshellLocalTlsDir === undefined) delete process.env.OPENSHELL_LOCAL_TLS_DIR; else process.env.OPENSHELL_LOCAL_TLS_DIR = previousOpenshellLocalTlsDir; resetGatewayOwnerBinding(); diff --git a/src/lib/onboard/gateway/application.ts b/src/lib/onboard/gateway/application.ts new file mode 100644 index 00000000000..a94c80ccef3 --- /dev/null +++ b/src/lib/onboard/gateway/application.ts @@ -0,0 +1,33 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { DockerDriverGatewayStart } from "./docker-driver-start"; +import type { GatewayRecoveryOrchestration } from "./recovery"; +import type { GatewayRegistration } from "./registration"; +import type { GatewayStart } from "./start"; + +export { createDockerDriverGatewayStart } from "./docker-driver-start"; +export { createGatewayRecoveryOrchestration } from "./recovery"; +export { createGatewayRegistration } from "./registration"; +export { createGatewayStart } from "./start"; + +export type GatewayLifecycleApplication = GatewayRegistration & + DockerDriverGatewayStart & + GatewayStart & + GatewayRecoveryOrchestration; + +export interface GatewayLifecycleApplicationDeps { + dockerDriverStart: DockerDriverGatewayStart; + recovery: GatewayRecoveryOrchestration; + registration: GatewayRegistration; + start: GatewayStart; +} + +export function createGatewayLifecycleApplication({ + dockerDriverStart, + recovery, + registration, + start, +}: GatewayLifecycleApplicationDeps): GatewayLifecycleApplication { + return { ...registration, ...dockerDriverStart, ...start, ...recovery }; +} diff --git a/src/lib/onboard/gateway/docker-driver-start.ts b/src/lib/onboard/gateway/docker-driver-start.ts new file mode 100644 index 00000000000..8456934b85a --- /dev/null +++ b/src/lib/onboard/gateway/docker-driver-start.ts @@ -0,0 +1,262 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; +import { trackChildExit } from "../child-exit-tracker"; +import * as dockerDriverGatewayCutover from "../docker-driver-gateway-cutover"; +import { reportDockerDriverGatewayStartFailure } from "../docker-driver-gateway-failure"; +import * as dockerDriverGatewayLaunch from "../docker-driver-gateway-launch"; +import { + reapDuplicateHostGatewaysExceptOrFail, + reapHostGatewayBeforeLaunchOrFail, +} from "../docker-driver-gateway-prelaunch"; +import { waitForStandaloneDockerDriverGateway } from "../docker-driver-gateway-readiness"; +import * as dockerDriverGatewayRuntimeMarker from "../docker-driver-gateway-runtime-marker"; +import { formatGatewayHealthWaitLimit } from "../gateway-health-wait"; +import { verifySandboxBridgeGatewayReachableOrExit } from "../gateway-sandbox-reachability"; + +type GatewayRuntimeHelpers = ReturnType< + typeof import("../docker-driver-gateway-runtime").createDockerDriverGatewayRuntimeHelpers +>; +type DynamicGatewayHelpers = ReturnType< + typeof import("../gateway-binding").createDynamicGatewayRuntimeHelpers +>; + +export interface DockerDriverGatewayStartDeps { + SUPPORTED_OPENSHELL_FALLBACK_VERSION: string; + checkGatewayPortAvailable(): Promise; + clearDockerDriverGatewayRuntimeFiles: GatewayRuntimeHelpers["clearDockerDriverGatewayRuntimeFiles"]; + createGatewayServicePortOwnership: GatewayRuntimeHelpers["createGatewayServicePortOwnership"]; + dockerDriverGatewayEnv: typeof import("../docker-driver-gateway-env"); + envInt: typeof import("../env").envInt; + gatewayBinding: typeof import("../gateway-binding"); + gatewayName(): string; + gatewayPort(): number; + getDockerDriverGatewayEndpoint: DynamicGatewayHelpers["getDockerDriverGatewayEndpoint"]; + getDockerDriverGatewayEnv: GatewayRuntimeHelpers["getDockerDriverGatewayEnv"]; + getDockerDriverGatewayPid: GatewayRuntimeHelpers["getDockerDriverGatewayPid"]; + getDockerDriverGatewayPortListenerScan: GatewayRuntimeHelpers["getDockerDriverGatewayPortListenerScan"]; + getDockerDriverGatewayRuntimeDrift: GatewayRuntimeHelpers["getDockerDriverGatewayRuntimeDrift"]; + getDockerDriverGatewayStateDir: GatewayRuntimeHelpers["getDockerDriverGatewayStateDir"]; + getInstalledOpenshellVersion: typeof import("../openshell-version").getInstalledOpenshellVersion; + isDockerDriverGatewayHttpReady: DynamicGatewayHelpers["isDockerDriverGatewayHttpReady"]; + isDockerDriverGatewayProcessAlive: GatewayRuntimeHelpers["isDockerDriverGatewayProcessAlive"]; + isDockerDriverGatewayStateInUse: GatewayRuntimeHelpers["isDockerDriverGatewayStateInUse"]; + isGatewayHealthy(status: string, namedInfo: string, activeInfo: string): boolean; + isGatewayTcpReady: DynamicGatewayHelpers["isGatewayTcpReady"]; + isPidAlive: GatewayRuntimeHelpers["isPidAlive"]; + logDockerDriverGatewayRestart(reason: string): void; + registerDockerDriverGatewayEndpoint(): boolean; + rememberDockerDriverGatewayPid: GatewayRuntimeHelpers["rememberDockerDriverGatewayPid"]; + resolveOpenShellGatewayBinary: GatewayRuntimeHelpers["resolveOpenShellGatewayBinary"]; + resolveOpenShellSandboxBinary: GatewayRuntimeHelpers["resolveOpenShellSandboxBinary"]; + runCaptureOpenshell(args: string[], options?: { ignoreError?: boolean }): string; + sleepSeconds: typeof import("../../core/wait").sleepSeconds; + verifySandboxBridgeGatewayReachableOrExit?: typeof verifySandboxBridgeGatewayReachableOrExit; +} + +export interface DockerDriverGatewayStart { + startDockerDriverGateway(options?: { + exitOnFailure?: boolean; + skipSandboxBridgeReachability?: boolean; + }): Promise; +} + +export function createDockerDriverGatewayStart( + deps: DockerDriverGatewayStartDeps, +): DockerDriverGatewayStart { + async function startDockerDriverGateway({ + exitOnFailure = true, + skipSandboxBridgeReachability = false, + }: { + exitOnFailure?: boolean; + skipSandboxBridgeReachability?: boolean; + } = {}): Promise { + const verifyReachability = + deps.verifySandboxBridgeGatewayReachableOrExit ?? verifySandboxBridgeGatewayReachableOrExit; + const gatewayBin = deps.resolveOpenShellGatewayBinary(); + const openshellVersionOutput = deps.runCaptureOpenshell(["--version"], { ignoreError: true }); + const gatewayEnv = deps.getDockerDriverGatewayEnv(openshellVersionOutput); + const stateDir = deps.getDockerDriverGatewayStateDir(); + const runtimeIdentity = gatewayBin + ? dockerDriverGatewayLaunch.buildDockerDriverGatewayRuntimeIdentity({ + gatewayBin, + gatewayEnv, + stateDir, + sandboxBin: deps.resolveOpenShellSandboxBinary(), + gatewayName: deps.gatewayName(), + compatContainerName: deps.gatewayBinding.resolveGatewayCompatContainerName( + deps.gatewayPort(), + ), + ensureLocalTlsBundle: true, + }) + : null; + const gatewayLaunch = runtimeIdentity?.launch ?? null; + const driftGatewayBin = dockerDriverGatewayLaunch.resolveDriftGatewayBin( + runtimeIdentity, + gatewayBin, + ); + const driftGatewayEnv = runtimeIdentity?.desiredEnv ?? gatewayEnv; + const identityGatewayBin = runtimeIdentity?.identityGatewayBin ?? gatewayBin; + const initialPortCheck = await deps.checkGatewayPortAvailable(); + const servicePortOwnership = deps.createGatewayServicePortOwnership(initialPortCheck, { + exitOnFailure, + gatewayBin: identityGatewayBin, + preparePort: (extraPids: number[]) => + reapHostGatewayBeforeLaunchOrFail({ + stateDir, + gatewayBin: identityGatewayBin, + extraPids, + exitOnFailure, + }), + }); + const cutover = await dockerDriverGatewayCutover.runDockerDriverGatewayManagedFallback( + () => + deps.dockerDriverGatewayEnv.startPackageManagedDockerDriverGatewayWithEnvOverride({ + clearDockerDriverGatewayRuntimeFiles: deps.clearDockerDriverGatewayRuntimeFiles, + exitOnFailure, + gatewayEnv: driftGatewayEnv, + gatewayName: deps.gatewayName(), + isDockerDriverGatewayReady: () => + deps.isDockerDriverGatewayHttpReady(undefined, undefined, driftGatewayEnv), + registerDockerDriverGatewayEndpoint: deps.registerDockerDriverGatewayEndpoint, + preparePortForOpenShellGatewayUserServiceStart: servicePortOwnership.preparePort, + runCaptureOpenshell: deps.runCaptureOpenshell, + skipSandboxBridgeReachability, + validatePortOwnerForOpenShellGatewayUserServiceStart: + servicePortOwnership.validatePortOwner, + verifySandboxBridgeGatewayReachableOrExit: (fail, options) => + verifyReachability(fail, { + ...(options ?? {}), + port: deps.gatewayPort(), + }), + }), + async () => + dockerDriverGatewayCutover.runDockerDriverGatewayCutover( + { + gatewayBin, + identityGatewayBin, + driftGatewayBin, + driftGatewayEnv, + exitOnFailure, + skipSandboxBridgeReachability, + stateDir, + portListenerScan: deps.getDockerDriverGatewayPortListenerScan( + await deps.checkGatewayPortAvailable(), + { gatewayBin: identityGatewayBin }, + ), + pidFileGatewayPid: deps.getDockerDriverGatewayPid(), + initialHealth: dockerDriverGatewayCutover.readDockerDriverGatewayHealth( + deps.runCaptureOpenshell, + deps.gatewayName(), + ), + }, + { + isDockerDriverGatewayProcessAlive: deps.isDockerDriverGatewayProcessAlive, + isGatewayHealthy: deps.isGatewayHealthy, + getDockerDriverGatewayRuntimeDrift: deps.getDockerDriverGatewayRuntimeDrift, + logDockerDriverGatewayRestart: deps.logDockerDriverGatewayRestart, + registerDockerDriverGatewayEndpoint: deps.registerDockerDriverGatewayEndpoint, + isDockerDriverGatewayHttpReady: () => + deps.isDockerDriverGatewayHttpReady(undefined, undefined, driftGatewayEnv), + verifySandboxBridgeGatewayReachableOrExit: (fail, options) => + verifyReachability(fail, { + ...(options ?? {}), + port: deps.gatewayPort(), + }), + readGatewayHealth: () => ({ + status: deps.runCaptureOpenshell(["status"], { ignoreError: true }), + namedInfo: deps.runCaptureOpenshell(["gateway", "info", "-g", deps.gatewayName()], { + ignoreError: true, + }), + activeInfo: deps.runCaptureOpenshell(["gateway", "info"], { ignoreError: true }), + }), + rememberDockerDriverGatewayPid: deps.rememberDockerDriverGatewayPid, + reapDuplicateHostGatewaysExceptOrFail, + reapHostGatewayBeforeLaunchOrFail, + isGatewayPortAvailable: async () => { + const probe = await deps.checkGatewayPortAvailable(); + return probe.ok && !probe.warning; + }, + reportUntrustedGatewayPort: servicePortOwnership.reportUntrustedGatewayPort, + reportMissingGatewayBinary: () => { + console.error(" OpenShell Docker-driver gateway binary not found."); + console.error( + ` Install OpenShell v${deps.SUPPORTED_OPENSHELL_FALLBACK_VERSION}, or set NEMOCLAW_OPENSHELL_GATEWAY_BIN.`, + ); + if (exitOnFailure) process.exit(1); + throw new Error("OpenShell gateway binary not found"); + }, + log: console.log, + }, + ), + ); + if (cutover !== "launch") return; + if (!gatewayBin || !gatewayLaunch) { + throw new Error("OpenShell gateway launch missing after cutover"); + } + + fs.mkdirSync(stateDir, { recursive: true, mode: 0o700 }); + const logPath = path.join(stateDir, "openshell-gateway.log"); + const log = dockerDriverGatewayLaunch.openDockerDriverGatewayLog(logPath, { exitOnFailure }); + console.log(" Starting OpenShell Docker-driver gateway..."); + console.log(` Gateway log: ${logPath}`); + dockerDriverGatewayLaunch.prepareAndLogDockerDriverGatewayLaunch(gatewayLaunch); + const child = dockerDriverGatewayLaunch.spawnDockerDriverGateway(gatewayLaunch, log.fd); + const childExit = trackChildExit(child); + child.unref(); + const childPid = child.pid ?? 0; + if (childPid <= 0) throw new Error("OpenShell gateway process did not return a pid"); + deps.rememberDockerDriverGatewayPid(childPid); + dockerDriverGatewayRuntimeMarker.writeDockerDriverGatewayRuntimeMarkerForStateDir( + deps.getDockerDriverGatewayStateDir(), + { + pid: childPid, + desiredEnv: driftGatewayEnv, + endpoint: deps.getDockerDriverGatewayEndpoint(), + gatewayBin: driftGatewayBin, + openshellVersion: deps.getInstalledOpenshellVersion(openshellVersionOutput), + dockerHost: process.env.DOCKER_HOST || null, + }, + ); + const pollCount = deps.envInt("NEMOCLAW_HEALTH_POLL_COUNT", 30); + const pollInterval = deps.envInt("NEMOCLAW_HEALTH_POLL_INTERVAL", 2); + const startup = await waitForStandaloneDockerDriverGateway({ + childExited: () => childExit.exited, + childPid, + gatewayName: deps.gatewayName(), + healthPollCount: pollCount, + healthPollIntervalSeconds: pollInterval, + isGatewayHealthy: deps.isGatewayHealthy, + isGatewayTcpReady: deps.isGatewayTcpReady, + isPidAlive: deps.isPidAlive, + onHealthy: async () => { + await verifyReachability(exitOnFailure, { + skip: skipSandboxBridgeReachability, + port: deps.gatewayPort(), + }); + }, + registerGatewayEndpoint: deps.registerDockerDriverGatewayEndpoint, + runCaptureOpenshell: deps.runCaptureOpenshell, + sleepSeconds: deps.sleepSeconds, + }); + if (startup === "healthy") { + console.log(" ✓ Docker-driver gateway is healthy"); + return; + } + reportDockerDriverGatewayStartFailure(logPath, childExit, { + exitOnFailure, + isGatewayStateInUse: deps.isDockerDriverGatewayStateInUse, + launchLogOffset: log.startOffset, + }); + if (startup === "exited") { + throw new Error("Docker-driver gateway failed to start because the process exited"); + } + throw new Error( + `Docker-driver gateway failed to start within ${formatGatewayHealthWaitLimit(pollCount, pollInterval)}`, + ); + } + + return { startDockerDriverGateway }; +} diff --git a/src/lib/onboard/gateway/entry-decisions.ts b/src/lib/onboard/gateway/entry-decisions.ts new file mode 100644 index 00000000000..c324aa31567 --- /dev/null +++ b/src/lib/onboard/gateway/entry-decisions.ts @@ -0,0 +1,72 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export function selectManagedListenerPid( + portKind: string, + findListenerPid: () => number | null, +): number | null { + return portKind === "gateway" ? findListenerPid() : null; +} + +export function selectGatewayPortCheckOptions( + portKind: string, + getOptions: () => T, +): T | undefined { + return portKind === "gateway" ? getOptions() : undefined; +} + +export function acceptManagedListener( + listenerPid: number | null, + accept: (pid: number) => void, +): boolean { + if (listenerPid === null) return false; + accept(listenerPid); + return true; +} + +export function requireGatewayBinding(binding: T | null): T { + if (binding === null) throw new Error("Authoritative rebuild preflight has no gateway"); + return binding; +} + +export function hasGatewayBinding(binding: unknown): boolean { + return binding !== null && binding !== undefined; +} + +export function clearGatewayEnvironmentWithoutBinding( + binding: unknown, + env: NodeJS.ProcessEnv, +): void { + if (!hasGatewayBinding(binding)) delete env.OPENSHELL_GATEWAY; +} + +export function applyGatewayBindingIfPresent( + binding: T | null | undefined, + apply: (binding: T) => void, +): void { + if (binding !== null && binding !== undefined) apply(binding); +} + +export function selectResumeSandboxName( + resume: boolean, + recordedName: string | null, + requestedName: string | null, + checkpointedName: string | null, +): string | null { + return resume ? (recordedName ?? requestedName ?? checkpointedName) : null; +} + +export function readSandboxForGatewayBinding( + sandboxName: string | null, + readSandbox: (name: string) => T, +): T | null { + return sandboxName ? readSandbox(sandboxName) : null; +} + +export function restoreGatewayEnvironment( + env: NodeJS.ProcessEnv, + previousGateway: string | undefined, +): void { + if (previousGateway === undefined) delete env.OPENSHELL_GATEWAY; + else env.OPENSHELL_GATEWAY = previousGateway; +} diff --git a/src/lib/onboard/gateway/late-binding.test.ts b/src/lib/onboard/gateway/late-binding.test.ts new file mode 100644 index 00000000000..710e59f5750 --- /dev/null +++ b/src/lib/onboard/gateway/late-binding.test.ts @@ -0,0 +1,160 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; +import { createDockerDriverGatewayStart } from "./docker-driver-start"; +import { createGatewayRecoveryOrchestration } from "./recovery"; +import { createGatewayRegistration } from "./registration"; + +const runResult = (status = 0) => + ({ status, stdout: "", stderr: "" }) as ReturnType; + +describe("gateway lifecycle late binding", () => { + it("uses the current binding for select, add, and health commands", () => { + let name = "initial"; + const runCaptureOpenshell = vi.fn((args: string[]) => args.join(" ")); + const runOpenshell = vi.fn(() => runResult()); + const registration = createGatewayRegistration({ + gatewayName: () => name, + getDockerDriverGatewayEndpointArg: () => "https://127.0.0.1:9443", + getGatewayLocalEndpoint: () => "https://127.0.0.1:9443", + hasStaleGateway: () => false, + isGatewayHealthy: () => false, + isLinuxDockerDriverGatewayEnabled: () => true, + removeDockerDriverGatewayRegistration: () => true, + runCaptureOpenshell, + runOpenshell, + runQuietOpenshell: vi.fn(() => ({ status: 0 })), + }); + + name = "resumed"; + expect(registration.registerDockerDriverGatewayEndpoint()).toBe(true); + + expect(runCaptureOpenshell).toHaveBeenCalledWith( + ["gateway", "info", "-g", "resumed"], + expect.objectContaining({ ignoreError: true }), + ); + expect(runOpenshell).toHaveBeenCalledWith( + ["gateway", "add", "https://127.0.0.1:9443", "--local", "--name", "resumed"], + expect.objectContaining({ ignoreError: true }), + ); + expect(runOpenshell).toHaveBeenCalledWith( + ["gateway", "select", "resumed"], + expect.objectContaining({ ignoreError: true }), + ); + }); + + it("uses the current binding for recovery select and health commands", async () => { + let name = "initial"; + const runOpenshell = vi.fn(() => runResult()); + const runCaptureOpenshell = vi + .fn<(args: string[], options?: { ignoreError?: boolean }) => string>() + .mockReturnValueOnce("Disconnected") + .mockReturnValue("Connected"); + const recovery = createGatewayRecoveryOrchestration({ + SCRIPTS: "/tmp/scripts", + assertGatewayStartAllowed: vi.fn(), + attachGatewayMetadataIfNeeded: () => true, + envInt: (_name, fallback) => fallback, + gatewayClusterHealthcheckPassed: () => true, + gatewayName: () => name, + getContainerRuntime: () => "docker", + getGatewayClusterContainerState: () => "missing", + isGatewayHealthy: () => true, + isGatewayHttpReady: async () => true, + isLinuxDockerDriverGatewayEnabled: () => false, + isSelectedGateway: () => false, + repairGatewayBootstrapSecrets: () => ({ repaired: true, missingSecrets: [] }), + run: vi.fn(() => runResult()), + runCaptureOpenshell, + runOpenshell, + shouldPatchCoredns: () => false, + sleepSeconds: vi.fn(), + startDockerDriverGateway: vi.fn(), + startGatewayWithOptions: vi.fn(), + }); + + name = "resumed"; + await expect(recovery.recoverGatewayRuntime()).resolves.toBe(true); + + expect(runOpenshell).toHaveBeenCalledWith( + ["gateway", "select", "resumed"], + expect.objectContaining({ ignoreError: true }), + ); + expect(runCaptureOpenshell).toHaveBeenCalledWith( + ["gateway", "info", "-g", "resumed"], + expect.objectContaining({ ignoreError: true }), + ); + }); + + it("uses the current binding for Docker-driver reachability", async () => { + let name = "initial"; + let port = 9000; + const verifyReachability = vi.fn(async () => undefined); + const managedStart = vi.fn( + async ( + options: Parameters< + typeof import("../docker-driver-gateway-env").startPackageManagedDockerDriverGatewayWithEnvOverride + >[0], + ) => { + await options.verifySandboxBridgeGatewayReachableOrExit(false, {}); + return true; + }, + ); + const dockerDriverGatewayEnv = { + startPackageManagedDockerDriverGatewayWithEnvOverride: managedStart, + } as unknown as typeof import("../docker-driver-gateway-env"); + const start = createDockerDriverGatewayStart({ + SUPPORTED_OPENSHELL_FALLBACK_VERSION: "0.0.0", + checkGatewayPortAvailable: async () => ({ ok: true }), + clearDockerDriverGatewayRuntimeFiles: vi.fn(), + createGatewayServicePortOwnership: () => ({ + portListenerScan: { complete: true, pids: [], unverifiedPids: [] }, + preparePort: vi.fn(), + reportUntrustedGatewayPort: (message) => { + throw new Error(message); + }, + validatePortOwner: vi.fn(), + }), + dockerDriverGatewayEnv, + envInt: (_name, fallback) => fallback, + gatewayBinding: { + resolveGatewayCompatContainerName: (value: number) => `gateway-${value}`, + } as typeof import("../gateway-binding"), + gatewayName: () => name, + gatewayPort: () => port, + getDockerDriverGatewayEndpoint: () => "https://127.0.0.1", + getDockerDriverGatewayEnv: () => ({ OPENSHELL_SERVER_PORT: String(port) }), + getDockerDriverGatewayPid: () => null, + getDockerDriverGatewayPortListenerScan: () => ({ + complete: true, + pids: [], + unverifiedPids: [], + }), + getDockerDriverGatewayRuntimeDrift: () => null, + getDockerDriverGatewayStateDir: () => "/tmp/gateway", + getInstalledOpenshellVersion: () => "0.0.0", + isDockerDriverGatewayHttpReady: async () => true, + isDockerDriverGatewayProcessAlive: () => false, + isDockerDriverGatewayStateInUse: () => false, + isGatewayHealthy: () => true, + isGatewayTcpReady: async () => true, + isPidAlive: () => false, + logDockerDriverGatewayRestart: vi.fn(), + registerDockerDriverGatewayEndpoint: () => true, + rememberDockerDriverGatewayPid: vi.fn(), + resolveOpenShellGatewayBinary: () => null, + resolveOpenShellSandboxBinary: () => null, + runCaptureOpenshell: () => "", + sleepSeconds: vi.fn(), + verifySandboxBridgeGatewayReachableOrExit: verifyReachability, + }); + + name = "resumed"; + port = 9777; + await start.startDockerDriverGateway(); + + expect(managedStart).toHaveBeenCalledWith(expect.objectContaining({ gatewayName: "resumed" })); + expect(verifyReachability).toHaveBeenCalledWith(false, expect.objectContaining({ port: 9777 })); + }); +}); diff --git a/src/lib/onboard/gateway/process-lifecycle.test.ts b/src/lib/onboard/gateway/process-lifecycle.test.ts new file mode 100644 index 00000000000..e02747432ae --- /dev/null +++ b/src/lib/onboard/gateway/process-lifecycle.test.ts @@ -0,0 +1,72 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; +import { + createGatewayProcessLifecycle, + type GatewayProcessLifecycleDeps, +} from "./process-lifecycle"; + +function dependencies( + overrides: Partial = {}, +): GatewayProcessLifecycleDeps { + return { + gatewayName: () => "nemoclaw", + dashboardPort: () => 18789, + runOpenshell: () => ({ status: 0 }), + runCaptureOpenshell: () => "", + dockerInspect: () => ({ status: 1 }), + dockerStop: vi.fn(), + dockerRm: vi.fn(), + dockerRemoveVolumesByPrefix: vi.fn(), + getGatewayClusterContainerName: (name) => `openshell-cluster-${name}`, + getDockerDriverGatewayPid: () => null, + isPidAlive: () => false, + isDockerDriverGatewayProcess: () => false, + resolveOpenShellGatewayBinary: () => "/usr/bin/openshell-gateway", + clearDockerDriverGatewayRuntimeFiles: vi.fn(), + sleepSeconds: vi.fn(), + isDockerDriverGatewayEnabled: () => true, + clearRegistry: vi.fn(), + killProcess: vi.fn(), + log: vi.fn(), + gatewayCliSupportsLifecycleCommands: () => false, + destroyGatewayWithVolumeCleanup: () => true, + ...overrides, + }; +} + +describe("gateway process lifecycle", () => { + it("does not signal a process that does not match the gateway binary", () => { + const clearRuntimeFiles = vi.fn(); + const killProcess = vi.fn(); + const lifecycle = createGatewayProcessLifecycle( + dependencies({ + getDockerDriverGatewayPid: () => 42, + isPidAlive: () => true, + isDockerDriverGatewayProcess: () => false, + clearDockerDriverGatewayRuntimeFiles: clearRuntimeFiles, + killProcess, + }), + ); + + expect(lifecycle.stopDockerDriverGatewayProcess()).toBe(false); + expect(killProcess).not.toHaveBeenCalled(); + expect(clearRuntimeFiles).toHaveBeenCalledOnce(); + }); + + it("uses gateway destroy when gateway remove fails", () => { + const runOpenshell = vi + .fn() + .mockReturnValueOnce({ status: 1 }) + .mockReturnValueOnce({ status: 0 }); + const lifecycle = createGatewayProcessLifecycle(dependencies({ runOpenshell })); + + expect(lifecycle.removeDockerDriverGatewayRegistration()).toBe(true); + expect(runOpenshell).toHaveBeenNthCalledWith( + 2, + ["gateway", "destroy", "-g", "nemoclaw"], + expect.objectContaining({ ignoreError: true, suppressOutput: true }), + ); + }); +}); diff --git a/src/lib/onboard/gateway/process-lifecycle.ts b/src/lib/onboard/gateway/process-lifecycle.ts new file mode 100644 index 00000000000..670ae99e845 --- /dev/null +++ b/src/lib/onboard/gateway/process-lifecycle.ts @@ -0,0 +1,157 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { SpawnSyncReturns } from "node:child_process"; +import type { Buffer } from "node:buffer"; + +type CommandResult = Pick, "status">; +type CommandOptions = { + ignoreError?: boolean; + stdio?: ["ignore", "pipe", "pipe"]; + suppressOutput?: boolean; +}; + +export interface GatewayProcessLifecycleDeps { + gatewayName(): string; + dashboardPort(): number; + runOpenshell(args: string[], options?: CommandOptions): CommandResult; + runCaptureOpenshell(args: string[], options?: { ignoreError?: boolean }): string; + dockerInspect( + args: string[], + options?: { ignoreError?: boolean; suppressOutput?: boolean }, + ): CommandResult; + dockerStop(name: string, options?: { ignoreError?: boolean; suppressOutput?: boolean }): unknown; + dockerRm(name: string, options?: { ignoreError?: boolean; suppressOutput?: boolean }): unknown; + dockerRemoveVolumesByPrefix(prefix: string, options: { ignoreError: true }): unknown; + getGatewayClusterContainerName(gatewayName: string): string; + getDockerDriverGatewayPid(): number | null; + isPidAlive(pid: number): boolean; + isDockerDriverGatewayProcess(pid: number, gatewayBinary: string | null): boolean; + resolveOpenShellGatewayBinary(): string | null; + clearDockerDriverGatewayRuntimeFiles(): void; + sleepSeconds(seconds: number): void; + isDockerDriverGatewayEnabled(): boolean; + clearRegistry(): void; + killProcess(pid: number, signal: NodeJS.Signals): void; + log(message: string): void; + gatewayCliSupportsLifecycleCommands( + capture: GatewayProcessLifecycleDeps["runCaptureOpenshell"], + ): boolean; + destroyGatewayWithVolumeCleanup(input: { + clearRegistry(): void; + dockerRemoveVolumesByPrefix: GatewayProcessLifecycleDeps["dockerRemoveVolumesByPrefix"]; + gatewayName: string; + hasLifecycleCommands(): boolean; + isDockerDriverGatewayEnabled(): boolean; + removeDockerDriverGatewayRegistration(): boolean; + runOpenshell: GatewayProcessLifecycleDeps["runOpenshell"]; + stopDockerDriverGatewayProcess(): void; + }): boolean; +} + +export function createGatewayProcessLifecycle(deps: GatewayProcessLifecycleDeps) { + function runQuietOpenshell(args: string[]): CommandResult { + return deps.runOpenshell(args, { + ignoreError: true, + stdio: ["ignore", "pipe", "pipe"], + suppressOutput: true, + }); + } + + function removeDockerDriverGatewayRegistration(): boolean { + const removeResult = runQuietOpenshell(["gateway", "remove", deps.gatewayName()]); + if (removeResult.status === 0) return true; + + // OpenShell builds before NVIDIA/OpenShell#1221 used `gateway destroy` for metadata cleanup. + const destroyResult = runQuietOpenshell(["gateway", "destroy", "-g", deps.gatewayName()]); + return destroyResult.status === 0; + } + + function terminateDockerDriverGatewayProcess(pid: number): boolean { + if (!deps.isPidAlive(pid)) return false; + + try { + deps.killProcess(pid, "SIGTERM"); + for (let attempt = 0; attempt < 10; attempt += 1) { + if (!deps.isPidAlive(pid)) break; + deps.sleepSeconds(1); + } + if (deps.isPidAlive(pid)) deps.killProcess(pid, "SIGKILL"); + return true; + } catch { + return false; + } + } + + function stopDockerDriverGatewayProcess(): boolean { + const pid = deps.getDockerDriverGatewayPid(); + if (pid === null || !deps.isPidAlive(pid)) { + deps.clearDockerDriverGatewayRuntimeFiles(); + return false; + } + if (!deps.isDockerDriverGatewayProcess(pid, deps.resolveOpenShellGatewayBinary())) { + deps.clearDockerDriverGatewayRuntimeFiles(); + return false; + } + + const stopped = terminateDockerDriverGatewayProcess(pid); + deps.clearDockerDriverGatewayRuntimeFiles(); + return stopped; + } + + function stopLegacyGatewayClusterContainer(): boolean { + const containerName = deps.getGatewayClusterContainerName(deps.gatewayName()); + const inspectResult = deps.dockerInspect(["--type", "container", containerName], { + ignoreError: true, + suppressOutput: true, + }); + if (inspectResult.status !== 0) return false; + + deps.dockerStop(containerName, { ignoreError: true, suppressOutput: true }); + deps.dockerRm(containerName, { ignoreError: true, suppressOutput: true }); + + return ( + deps.dockerInspect(["--type", "container", containerName], { + ignoreError: true, + suppressOutput: true, + }).status !== 0 + ); + } + + function retireLegacyGatewayForDockerDriverUpgrade(): void { + deps.runOpenshell(["forward", "stop", String(deps.dashboardPort())], { + ignoreError: true, + }); + stopDockerDriverGatewayProcess(); + const stoppedLegacyContainer = stopLegacyGatewayClusterContainer(); + removeDockerDriverGatewayRegistration(); + if (stoppedLegacyContainer) { + deps.log(" ✓ Legacy OpenShell gateway container stopped for Docker-driver upgrade"); + } + } + + function destroyGateway( + clearRegistry: () => void = deps.clearRegistry, + isDockerDriverGatewayEnabled: () => boolean = deps.isDockerDriverGatewayEnabled, + ): boolean { + return deps.destroyGatewayWithVolumeCleanup({ + clearRegistry, + dockerRemoveVolumesByPrefix: deps.dockerRemoveVolumesByPrefix, + gatewayName: deps.gatewayName(), + hasLifecycleCommands: () => + deps.gatewayCliSupportsLifecycleCommands(deps.runCaptureOpenshell), + isDockerDriverGatewayEnabled, + removeDockerDriverGatewayRegistration, + runOpenshell: deps.runOpenshell, + stopDockerDriverGatewayProcess, + }); + } + + return { + destroyGateway, + removeDockerDriverGatewayRegistration, + retireLegacyGatewayForDockerDriverUpgrade, + runQuietOpenshell, + stopDockerDriverGatewayProcess, + }; +} diff --git a/src/lib/onboard/gateway/recovery.ts b/src/lib/onboard/gateway/recovery.ts new file mode 100644 index 00000000000..63d2da5f90f --- /dev/null +++ b/src/lib/onboard/gateway/recovery.ts @@ -0,0 +1,124 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import path from "node:path"; +import { gatewayStartGuidance } from "../../gateway-start-guidance"; +import { getGatewayHealthWaitConfig, waitForGatewayHealth } from "../gateway-health-wait"; +import { + startGatewayForRecovery as startGatewayForRecoveryFlow, + type StartGatewayForRecoveryOptions, +} from "../gateway-recovery"; + +type DynamicGatewayHelpers = ReturnType< + typeof import("../gateway-binding").createDynamicGatewayRuntimeHelpers +>; +type GatewayBootstrapRepairHelpers = ReturnType< + typeof import("../gateway-bootstrap").createGatewayBootstrapRepairHelpers +>; +type OnboardGpu = ReturnType; +type RunResult = ReturnType; + +export interface GatewayRecoveryOrchestrationDeps { + SCRIPTS: string; + assertGatewayStartAllowed( + exitOnFailure: boolean, + target?: { gatewayName: string; gatewayPort: number }, + ): void; + attachGatewayMetadataIfNeeded(options?: { forceRefresh?: boolean }): boolean; + envInt: typeof import("../env").envInt; + gatewayClusterHealthcheckPassed: GatewayBootstrapRepairHelpers["gatewayClusterHealthcheckPassed"]; + gatewayName(): string; + getContainerRuntime: typeof import("../local-inference-topology").getContainerRuntime; + getGatewayClusterContainerState(): string; + isGatewayHealthy(status: string, namedInfo: string, activeInfo: string): boolean; + isGatewayHttpReady: DynamicGatewayHelpers["isGatewayHttpReady"]; + isLinuxDockerDriverGatewayEnabled(): boolean; + isSelectedGateway(status: string): boolean; + repairGatewayBootstrapSecrets: GatewayBootstrapRepairHelpers["repairGatewayBootstrapSecrets"]; + run: typeof import("../../runner").run; + runCaptureOpenshell(args: string[], options?: { ignoreError?: boolean }): string; + runOpenshell(args: string[], options?: { ignoreError?: boolean }): RunResult; + shouldPatchCoredns: typeof import("../../platform").shouldPatchCoredns; + sleepSeconds: typeof import("../../core/wait").sleepSeconds; + startDockerDriverGateway(options?: { exitOnFailure?: boolean }): Promise; + startGatewayWithOptions(gpu: OnboardGpu, options: { exitOnFailure: false }): Promise; +} + +export interface GatewayRecoveryOrchestration { + recoverGatewayRuntime(): Promise; + startGatewayForRecovery(options?: StartGatewayForRecoveryOptions): Promise; +} + +export function createGatewayRecoveryOrchestration( + deps: GatewayRecoveryOrchestrationDeps, +): GatewayRecoveryOrchestration { + async function startGatewayForRecovery( + options: StartGatewayForRecoveryOptions = {}, + ): Promise { + return startGatewayForRecoveryFlow(options, { + assertGatewayStartAllowed: deps.assertGatewayStartAllowed, + runCaptureOpenshell: deps.runCaptureOpenshell, + runOpenshell: deps.runOpenshell, + startGatewayWithOptions: deps.startGatewayWithOptions, + isLinuxDockerDriverGatewayEnabled: deps.isLinuxDockerDriverGatewayEnabled, + }); + } + + async function recoverGatewayRuntime(): Promise { + deps.assertGatewayStartAllowed(false); + if (deps.isLinuxDockerDriverGatewayEnabled()) { + try { + await deps.startDockerDriverGateway({ exitOnFailure: false }); + return true; + } catch { + return false; + } + } + + deps.runOpenshell(["gateway", "select", deps.gatewayName()], { ignoreError: true }); + const status = deps.runCaptureOpenshell(["status"], { ignoreError: true }); + if ( + status.includes("Connected") && + deps.isSelectedGateway(status) && + (await deps.isGatewayHttpReady()) + ) { + process.env.OPENSHELL_GATEWAY = deps.gatewayName(); + return true; + } + + const recoveryWait = getGatewayHealthWaitConfig(0, deps.getGatewayClusterContainerState()); + const pollCount = recoveryWait.extended + ? recoveryWait.count + : deps.envInt("NEMOCLAW_HEALTH_POLL_COUNT", 10); + const pollInterval = recoveryWait.extended + ? recoveryWait.interval + : deps.envInt("NEMOCLAW_HEALTH_POLL_INTERVAL", 2); + const healthy = await waitForGatewayHealth({ + attachGatewayMetadataIfNeeded: deps.attachGatewayMetadataIfNeeded, + gatewayClusterHealthcheckPassed: deps.gatewayClusterHealthcheckPassed, + gatewayName: deps.gatewayName(), + healthPollCount: pollCount, + healthPollIntervalSeconds: pollInterval, + isGatewayHealthy: deps.isGatewayHealthy, + isGatewayHttpReady: (signal?: AbortSignal) => + deps.isGatewayHttpReady(undefined, undefined, undefined, signal), + repairGatewayBootstrapSecrets: deps.repairGatewayBootstrapSecrets, + runCaptureOpenshell: deps.runCaptureOpenshell, + sleepSeconds: deps.sleepSeconds, + }); + if (!healthy) { + console.error(` ${gatewayStartGuidance(deps.gatewayName())}`); + return false; + } + + process.env.OPENSHELL_GATEWAY = deps.gatewayName(); + if (deps.shouldPatchCoredns(deps.getContainerRuntime())) { + deps.run(["bash", path.join(deps.SCRIPTS, "fix-coredns.sh"), deps.gatewayName()], { + ignoreError: true, + }); + } + return true; + } + + return { recoverGatewayRuntime, startGatewayForRecovery }; +} diff --git a/src/lib/onboard/gateway/registration.ts b/src/lib/onboard/gateway/registration.ts new file mode 100644 index 00000000000..d8dbbfa0161 --- /dev/null +++ b/src/lib/onboard/gateway/registration.ts @@ -0,0 +1,113 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +type RunResult = ReturnType; + +export interface GatewayRegistrationDeps { + gatewayName(): string; + getDockerDriverGatewayEndpointArg(): string; + getGatewayLocalEndpoint(): string; + hasStaleGateway(gatewayInfo: string): boolean; + isGatewayHealthy(status: string, namedInfo: string, activeInfo: string): boolean; + isLinuxDockerDriverGatewayEnabled(): boolean; + removeDockerDriverGatewayRegistration(): boolean; + runCaptureOpenshell(args: string[], options?: { ignoreError?: boolean }): string; + runOpenshell( + args: string[], + options?: { ignoreError?: boolean; suppressOutput?: boolean }, + ): RunResult; + runQuietOpenshell(args: string[]): { status: number | null }; +} + +export interface GatewayRegistration { + attachGatewayMetadataIfNeeded(options?: { forceRefresh?: boolean }): boolean; + registerDockerDriverGatewayEndpoint(): boolean; +} + +export function createGatewayRegistration(deps: GatewayRegistrationDeps): GatewayRegistration { + function registerDockerDriverGatewayEndpoint(): boolean { + const selectExisting = deps.runQuietOpenshell(["gateway", "select", deps.gatewayName()]); + if (selectExisting.status === 0) { + const status = deps.runCaptureOpenshell(["status"], { ignoreError: true }); + const namedInfo = deps.runCaptureOpenshell(["gateway", "info", "-g", deps.gatewayName()], { + ignoreError: true, + }); + const currentInfo = deps.runCaptureOpenshell(["gateway", "info"], { ignoreError: true }); + if (deps.isGatewayHealthy(status, namedInfo, currentInfo)) { + process.env.OPENSHELL_GATEWAY = deps.gatewayName(); + return true; + } + } + + let addResult = deps.runOpenshell( + [ + "gateway", + "add", + deps.getDockerDriverGatewayEndpointArg(), + "--local", + "--name", + deps.gatewayName(), + ], + { ignoreError: true, suppressOutput: true }, + ); + if (addResult.status !== 0) { + deps.removeDockerDriverGatewayRegistration(); + addResult = deps.runOpenshell( + [ + "gateway", + "add", + deps.getDockerDriverGatewayEndpointArg(), + "--local", + "--name", + deps.gatewayName(), + ], + { ignoreError: true, suppressOutput: true }, + ); + } + const selectResult = deps.runOpenshell(["gateway", "select", deps.gatewayName()], { + ignoreError: true, + suppressOutput: true, + }); + const ok = + (addResult.status === 0 && selectResult.status === 0) || + (selectResult.status === 0 && + deps.isGatewayHealthy( + deps.runCaptureOpenshell(["status"], { ignoreError: true }), + deps.runCaptureOpenshell(["gateway", "info", "-g", deps.gatewayName()], { + ignoreError: true, + }), + deps.runCaptureOpenshell(["gateway", "info"], { ignoreError: true }), + )); + if (ok) { + process.env.OPENSHELL_GATEWAY = deps.gatewayName(); + } else if (process.env.OPENSHELL_GATEWAY === deps.gatewayName()) { + delete process.env.OPENSHELL_GATEWAY; + } + return ok; + } + + function attachGatewayMetadataIfNeeded({ + forceRefresh = false, + }: { + forceRefresh?: boolean; + } = {}): boolean { + const gatewayInfo = deps.runCaptureOpenshell(["gateway", "info", "-g", deps.gatewayName()], { + ignoreError: true, + }); + // The CLI can return stale but present metadata. Preserve the metadata unless + // the repair flow recreates the bootstrap secrets and forces a refresh. + if (!forceRefresh && deps.hasStaleGateway(gatewayInfo)) return true; + if (deps.isLinuxDockerDriverGatewayEnabled()) { + return registerDockerDriverGatewayEndpoint(); + } + const addResult = deps.runOpenshell( + ["gateway", "add", deps.getGatewayLocalEndpoint(), "--local", "--name", deps.gatewayName()], + { ignoreError: true, suppressOutput: true }, + ); + if (addResult.status !== 0) return false; + console.log(" ✓ Gateway metadata reattached"); + return true; + } + + return { attachGatewayMetadataIfNeeded, registerDockerDriverGatewayEndpoint }; +} diff --git a/src/lib/onboard/gateway/start.ts b/src/lib/onboard/gateway/start.ts new file mode 100644 index 00000000000..78e19469532 --- /dev/null +++ b/src/lib/onboard/gateway/start.ts @@ -0,0 +1,107 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { gatewayStartGuidance } from "../../gateway-start-guidance"; +import { normalizeGatewayStartError } from "../gateway-start-failure"; + +type OnboardGpu = ReturnType; +type DynamicGatewayHelpers = ReturnType< + typeof import("../gateway-binding").createDynamicGatewayRuntimeHelpers +>; +type GatewayReuseHelpers = ReturnType; + +export interface GatewayStartDeps { + assertGatewayStartAllowed(exitOnFailure: boolean): void; + cliDisplayName(): string; + dockerGpuLocalInference: typeof import("../docker-gpu-local-inference"); + dockerGpuRoute: typeof import("../docker-gpu-route"); + dockerGpuSandboxCreate: typeof import("../docker-gpu-sandbox-create"); + gatewayName(): string; + getGatewayLocalEndpoint(): string; + getGatewayReuseSnapshot: GatewayReuseHelpers["getGatewayReuseSnapshot"]; + hasStaleGateway(gatewayInfo: string): boolean; + isGatewayHealthy(status: string, namedInfo: string, activeInfo: string): boolean; + isGatewayHttpReady: DynamicGatewayHelpers["isGatewayHttpReady"]; + isLinuxDockerDriverGatewayEnabled(): boolean; + runOpenshell(args: string[], options?: { ignoreError?: boolean }): unknown; + selectNamedGatewayForReuseIfNeeded: GatewayReuseHelpers["selectNamedGatewayForReuseIfNeeded"]; + startDockerDriverGateway(options?: { + exitOnFailure?: boolean; + skipSandboxBridgeReachability?: boolean; + }): Promise; + step: typeof import("../prompt-helpers").step; +} + +export interface GatewayStart { + startGateway(gpu: OnboardGpu, options?: { gpuPassthrough?: boolean }): Promise; + startGatewayWithOptions( + gpu: OnboardGpu, + options?: { exitOnFailure?: boolean; gpuPassthrough?: boolean }, + ): Promise; +} + +export function createGatewayStart(deps: GatewayStartDeps): GatewayStart { + async function startGatewayWithOptions( + gpu: OnboardGpu, + { + exitOnFailure = true, + gpuPassthrough = false, + }: { exitOnFailure?: boolean; gpuPassthrough?: boolean } = {}, + ): Promise { + deps.assertGatewayStartAllowed(exitOnFailure); + deps.step(2, 8, "Starting OpenShell gateway"); + if (deps.isLinuxDockerDriverGatewayEnabled()) { + const selectedGpuRoute = deps.dockerGpuRoute.initialDockerGpuRoute( + deps.dockerGpuRoute.resolveDockerGpuRoutePlan( + { sandboxGpuEnabled: gpuPassthrough, hostGpuPlatform: gpu?.platform }, + { + dockerDriverGateway: true, + dockerDesktopWsl: deps.dockerGpuSandboxCreate.isDockerDesktopWslRuntime(), + }, + ), + ); + return deps.startDockerDriverGateway({ + exitOnFailure, + skipSandboxBridgeReachability: deps.dockerGpuLocalInference.shouldSkipGpuBridgeProbe( + gpuPassthrough, + gpu?.platform, + selectedGpuRoute, + ), + }); + } + + const snapshot = deps.selectNamedGatewayForReuseIfNeeded(deps.getGatewayReuseSnapshot()); + if ( + deps.isGatewayHealthy(snapshot.gatewayStatus, snapshot.gwInfo, snapshot.activeGatewayInfo) + ) { + // CLI metadata can remain healthy after a restart. Probe HTTP before reuse to + // prevent a later connection failure (#3258). + if (await deps.isGatewayHttpReady()) { + console.log(" ✓ Reusing existing gateway"); + deps.runOpenshell(["gateway", "select", deps.gatewayName()], { ignoreError: true }); + process.env.OPENSHELL_GATEWAY = deps.gatewayName(); + return; + } + console.log( + ` Gateway metadata reports healthy but ${deps.getGatewayLocalEndpoint()}/ is not responding.`, + ); + } + if (deps.hasStaleGateway(snapshot.gwInfo)) console.log(" Stale gateway detected."); + + // The deployment owns this gateway lifecycle. NemoClaw can reuse the gateway + // but cannot start it. + const message = `${deps.cliDisplayName()} does not start the '${deps.gatewayName()}' gateway on this host.`; + console.error(` ${gatewayStartGuidance(deps.gatewayName())}`); + if (exitOnFailure) process.exit(1); + throw normalizeGatewayStartError(new Error(message)); + } + + async function startGateway( + gpu: OnboardGpu, + { gpuPassthrough = false }: { gpuPassthrough?: boolean } = {}, + ): Promise { + return startGatewayWithOptions(gpu, { exitOnFailure: true, gpuPassthrough }); + } + + return { startGateway, startGatewayWithOptions }; +} diff --git a/test/checks-runner.test.ts b/test/checks-runner.test.ts index 4e0c5bb0f30..1cda675efd0 100644 --- a/test/checks-runner.test.ts +++ b/test/checks-runner.test.ts @@ -25,6 +25,14 @@ describe("checks runner", () => { }); }); + it("registers the onboarding entry composition check", () => { + expect(CHECKS).toContainEqual({ + name: "onboard-entry-composition", + command: process.platform === "win32" ? "tsx.cmd" : "tsx", + args: ["scripts/checks/onboard-entry-composition.mts"], + }); + }); + it("registers the test registration boundary check", () => { expect(CHECKS).toContainEqual({ name: "test-registration-boundary", diff --git a/test/onboard-entry-composition.test.ts b/test/onboard-entry-composition.test.ts new file mode 100644 index 00000000000..54a82c5dad8 --- /dev/null +++ b/test/onboard-entry-composition.test.ts @@ -0,0 +1,192 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { + collectOnboardEntryDecisions, + evaluateOnboardEntryComposition, + parseOnboardEntryCompositionBudget, + type OnboardEntryCompositionBudget, +} from "../scripts/checks/onboard-entry-composition.mts"; + +const REPO_ROOT = path.join(import.meta.dirname, ".."); +const EMPTY_BUDGET: OnboardEntryCompositionBudget = { + gateway: {}, + messaging: {}, + policy: {}, + provider: {}, +}; + +describe("onboarding entry composition boundary", () => { + it("accepts the recorded onboarding decision allowances", () => { + const actual = collectOnboardEntryDecisions( + fs.readFileSync(path.join(REPO_ROOT, "src/lib/onboard.ts"), "utf8"), + ); + const budget = parseOnboardEntryCompositionBudget( + fs.readFileSync(path.join(REPO_ROOT, "ci/onboard-entry-composition-budget.json"), "utf8"), + ); + + expect(evaluateOnboardEntryComposition(actual, budget)).toEqual([]); + expect(actual).toEqual({ + gateway: {}, + messaging: { createSandboxWithBaseImageResolution: 9, runOnboard: 1 }, + policy: { createSandboxWithBaseImageResolution: 6, runOnboard: 5 }, + provider: { + createSandboxWithBaseImageResolution: 15, + handleNimLocalSelection: 32, + handleRemoteProviderSelection: 80, + handleRoutedSelection: 15, + runOnboard: 8, + selectAndValidateOllamaModel: 18, + }, + }); + }); + + it("rejects a gateway action selected by a neutral condition", () => { + const actual = collectOnboardEntryDecisions( + "function choose(enabled: boolean) { if (enabled) startGateway(); }", + ); + + expect(actual.gateway).toEqual({ choose: 1 }); + }); + + it("rejects a gateway condition inside the onboarding entry function", () => { + const actual = collectOnboardEntryDecisions( + "function runOnboard() { if (gatewayState === 'stale') return; }", + ); + + expect(actual.gateway).toEqual({ runOnboard: 1 }); + }); + + it("rejects a messaging action selected by a neutral condition", () => { + const actual = collectOnboardEntryDecisions( + "function choose(enabled: boolean) { if (enabled) configureMessaging(); }", + ); + + expect(actual.messaging).toEqual({ choose: 1 }); + }); + + it.each([ + ["if", "if (enabled) startGateway();"], + ["switch", "switch (mode) { case 'start': startGateway(); }"], + ["conditional", "enabled ? startGateway() : stopGateway();"], + ["logical AND", "enabled && startGateway();"], + ["logical OR", "enabled || startGateway();"], + ["nullish coalescing", "enabled ?? startGateway();"], + ["for loop", "for (; gatewayRunning(); ) poll();"], + ["while loop", "while (gatewayRunning()) poll();"], + ["do loop", "do poll(); while (gatewayRunning());"], + ["try and catch", "try { startGateway(); } catch { reportFailure(); }"], + ["recovery call", "recoverGateway();"], + ])("counts a gateway decision expressed with %s", (_form, decision) => { + const actual = collectOnboardEntryDecisions( + `function choose(enabled: boolean, mode: string) { ${decision} }`, + ); + + expect(actual.gateway).toEqual({ choose: 1 }); + }); + + it("does not let a provider function name hide a gateway action", () => { + const actual = collectOnboardEntryDecisions( + "function chooseProvider(enabled: boolean) { if (enabled) startGateway(); }", + ); + + expect(actual.gateway).toEqual({ chooseProvider: 1 }); + expect(actual.provider).toEqual({ chooseProvider: 1 }); + }); + + it("does not let a provider condition hide a gateway action", () => { + const actual = collectOnboardEntryDecisions( + "function choose(providerEnabled: boolean) { if (providerEnabled) startGateway(); }", + ); + + expect(actual.gateway).toEqual({ choose: 1 }); + expect(actual.provider).toEqual({ choose: 1 }); + }); + + it("retains body categories when a function name has a gateway category", () => { + const actual = collectOnboardEntryDecisions( + "function chooseGateway(enabled: boolean) { if (enabled) configureMessaging(); }", + ); + + expect(actual.gateway).toEqual({ chooseGateway: 1 }); + expect(actual.messaging).toEqual({ chooseGateway: 1 }); + }); + + it("does not count a nested logical decision as part of its parent decision", () => { + const actual = collectOnboardEntryDecisions( + "function choose(enabled: boolean) { if (enabled && gatewayRunning()) poll(); }", + ); + + expect(actual.gateway).toEqual({ choose: 1 }); + }); + + it("does not classify sequencing loops as decisions", () => { + const actual = collectOnboardEntryDecisions( + "function runSteps(items: string[]) { for (const item of items) startGateway(item); }", + ); + + expect(actual.gateway).toEqual({}); + }); + + it("does not classify provider registry decisions as gateway lifecycle decisions", () => { + const actual = collectOnboardEntryDecisions( + "function choose(name: string) { if (providerExistsInGateway(name)) useProvider(name); }", + ); + + expect(actual.gateway).toEqual({}); + expect(actual.provider).toEqual({ choose: 1 }); + }); + + it("does not classify Hermes tool selection as a gateway lifecycle decision", () => { + const actual = collectOnboardEntryDecisions( + "function choose(enabled: boolean) { if (enabled) normalizeHermesToolGatewaySelections(); }", + ); + + expect(actual.gateway).toEqual({}); + }); + + it("rejects a decision added within an allowed declaration", () => { + const actual = collectOnboardEntryDecisions( + "function handleRemoteProviderSelection(enabled: boolean) { if (enabled) useProvider(); if (enabled) useProviderAgain(); }", + ); + + expect( + evaluateOnboardEntryComposition(actual, { + ...EMPTY_BUDGET, + provider: { handleRemoteProviderSelection: 1 }, + }), + ).toEqual([ + { + kind: "new-decision", + category: "provider", + declaration: "handleRemoteProviderSelection", + actualCount: 2, + budgetCount: 1, + }, + ]); + }); + + it("requires the budget to decrease when a decision leaves an allowed declaration", () => { + const actual = collectOnboardEntryDecisions( + "function handleRemoteProviderSelection(enabled: boolean) { if (enabled) useProvider(); }", + ); + + expect( + evaluateOnboardEntryComposition(actual, { + ...EMPTY_BUDGET, + provider: { handleRemoteProviderSelection: 2 }, + }), + ).toEqual([ + { + kind: "decision-ratchet", + category: "provider", + declaration: "handleRemoteProviderSelection", + actualCount: 1, + budgetCount: 2, + }, + ]); + }); +});