From b3317b572ac19a76c801d16ee340a7860f6d955e Mon Sep 17 00:00:00 2001 From: San Dang Date: Sun, 6 Sep 2026 23:54:51 +0700 Subject: [PATCH 01/20] fix(onboard): reuse existing OpenClaw dashboard forward Signed-off-by: San Dang --- .../openshell/forward-service.test.ts | 53 ++++++++- src/lib/adapters/openshell/forward-service.ts | 102 +++++++++++++++++- src/lib/onboard.ts | 8 +- src/lib/onboard/agent-dashboard-forward.ts | 4 + src/lib/onboard/dashboard-forward-control.ts | 3 + src/lib/onboard/dashboard-port.test.ts | 51 +++++++++ src/lib/onboard/dashboard-port.ts | 26 +++++ src/lib/onboard/dashboard.ts | 44 +++++++- .../onboard/sandbox-create/orchestration.ts | 11 ++ src/lib/onboard/sandbox-reuse.test.ts | 14 ++- src/lib/onboard/sandbox-reuse.ts | 21 ++-- ...ard-finalization-dashboard-forward.test.ts | 82 +++++++++++++- 12 files changed, 398 insertions(+), 21 deletions(-) diff --git a/src/lib/adapters/openshell/forward-service.test.ts b/src/lib/adapters/openshell/forward-service.test.ts index 2de4e61ca73..c340d7fffb0 100644 --- a/src/lib/adapters/openshell/forward-service.test.ts +++ b/src/lib/adapters/openshell/forward-service.test.ts @@ -6,11 +6,13 @@ import { describe, expect, it, vi } from "vitest"; import { buildForwardServiceArgs, launchForwardService, + matchesRunningForwardService, + type ForwardServiceProcess, type ForwardServiceTarget, } from "./forward-service"; const target: ForwardServiceTarget = { - executable: "/usr/local/bin/openshell", + executable: process.execPath, gatewayName: "nemoclaw", workspace: "default", sandboxName: "demo", @@ -19,6 +21,21 @@ const target: ForwardServiceTarget = { targetHost: "127.0.0.1", targetPort: 18_789, }; +const expectedCommand = [target.executable, ...buildForwardServiceArgs(target)].join(" "); +const testHome = process.env.HOME ?? ""; +const testUid = process.getuid?.() ?? 1_000; + +function matchingListenerOptions(overrides: Partial = {}) { + return { + inspectListener: () => ({ + commandLine: expectedCommand, + executable: target.executable, + home: testHome, + uid: testUid, + ...overrides, + }), + }; +} describe("OpenShell forward service", () => { it("builds the direct ForwardTcp command with explicit gateway authority", () => { @@ -74,6 +91,40 @@ describe("OpenShell forward service", () => { expect(spawnDetached).not.toHaveBeenCalled(); }); + it("matches the exact forward-service listener started by NemoClaw (#11074)", () => { + expect(matchesRunningForwardService(target, matchingListenerOptions())).toBe(true); + }); + + it.each([ + { + name: "different arguments", + overrides: { + commandLine: [ + target.executable, + ...buildForwardServiceArgs({ ...target, sandboxName: "other" }), + ].join(" "), + }, + }, + { + name: "different user", + overrides: { uid: testUid + 1 }, + }, + { + name: "different executable", + overrides: { executable: "/bin/sh" }, + }, + { + name: "different OpenShell home", + overrides: { home: `${testHome}-other` }, + }, + ])("rejects a listener with $name", ({ overrides }) => { + expect(matchesRunningForwardService(target, matchingListenerOptions(overrides))).toBe(false); + }); + + it("rejects an inconclusive listener inspection", () => { + expect(matchesRunningForwardService(target, { inspectListener: () => null })).toBe(false); + }); + it("fails when the detached service does not bind before the deadline", () => { expect(() => launchForwardService(target, { diff --git a/src/lib/adapters/openshell/forward-service.ts b/src/lib/adapters/openshell/forward-service.ts index 8f78fe81983..3fe774e4eae 100644 --- a/src/lib/adapters/openshell/forward-service.ts +++ b/src/lib/adapters/openshell/forward-service.ts @@ -1,7 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { spawn } from "node:child_process"; +import { spawn, spawnSync } from "node:child_process"; +import fs from "node:fs"; import path from "node:path"; import { isValidName } from "../../name-validation"; @@ -10,6 +11,8 @@ import { probeLocalForwardListener } from "./local-forward-listener"; const START_TIMEOUT_MS = 30_000; const POLL_INTERVAL_MS = 100; +const INSPECTION_TIMEOUT_MS = 1_000; +const INSPECTION_MAX_BUFFER_BYTES = 16 * 1024; const sleepBuffer = new Int32Array(new SharedArrayBuffer(4)); export interface ForwardServiceTarget { @@ -37,6 +40,17 @@ export interface ForwardServiceLaunchOptions { readonly timeoutMs?: number; } +export interface ForwardServiceProcess { + readonly commandLine: string; + readonly executable: string; + readonly home: string; + readonly uid: number; +} + +export interface ForwardServiceListenerOptions { + readonly inspectListener?: (port: number) => ForwardServiceProcess | null; +} + function isPort(value: unknown): value is number { return Number.isSafeInteger(value) && Number(value) >= 1 && Number(value) <= 65_535; } @@ -94,6 +108,92 @@ export function buildForwardServiceArgs(target: ForwardServiceTarget): string[] ]; } +function probeExecutable(candidates: readonly string[]): string | null { + return candidates.find((candidate) => fs.existsSync(candidate)) ?? null; +} + +function runInspection(executable: string, args: readonly string[]): string | null { + const result = spawnSync(executable, [...args], { + encoding: "utf8", + env: buildOpenShellSubprocessEnv(), + maxBuffer: INSPECTION_MAX_BUFFER_BYTES, + stdio: ["ignore", "pipe", "ignore"], + timeout: INSPECTION_TIMEOUT_MS, + }); + return result.error || result.status !== 0 ? null : result.stdout; +} + +function inspectListener(port: number): ForwardServiceProcess | null { + const lsof = probeExecutable(["/usr/sbin/lsof", "/usr/bin/lsof"]); + const ps = probeExecutable(["/bin/ps", "/usr/bin/ps"]); + if (!lsof || !ps) return null; + + const listenerOutput = runInspection(lsof, [ + "-nP", + `-iTCP:${String(port)}`, + "-sTCP:LISTEN", + "-Fp", + ]); + const pids = [ + ...new Set( + listenerOutput + ?.split(/\r?\n/u) + .flatMap((line) => /^p([1-9]\d*)$/u.exec(line.trim())?.[1] ?? []) + .map(Number) ?? [], + ), + ]; + const pid = pids.length === 1 ? pids[0] : undefined; + if (pid === undefined) return null; + + const uid = Number(runInspection(ps, ["-p", String(pid), "-o", "uid="])?.trim()); + const commandLine = runInspection(ps, ["-ww", "-p", String(pid), "-o", "command="])?.trim(); + const environment = runInspection(ps, ["eww", "-p", String(pid), "-o", "command="]); + const home = environment + ?.replace(/\s+/gu, "\0") + .split("\0") + .find((entry) => entry.startsWith("HOME=")) + ?.slice("HOME=".length); + const executable = runInspection(lsof, ["-a", "-p", String(pid), "-d", "txt", "-Fn"]) + ?.split(/\r?\n/u) + .find((line) => line.startsWith("n") && line.length > 1) + ?.slice(1); + return Number.isSafeInteger(uid) && uid >= 0 && commandLine && executable && home + ? { commandLine, executable, home, uid } + : null; +} + +function realpath(value: string): string | null { + try { + return fs.realpathSync.native(value); + } catch { + return null; + } +} + +/** Match one bound listener to the direct ForwardTcp command that NemoClaw starts. */ +export function matchesRunningForwardService( + target: ForwardServiceTarget, + options: ForwardServiceListenerOptions = {}, +): boolean { + validateForwardServiceTarget(target); + const currentUid = typeof process.getuid === "function" ? process.getuid() : null; + const observed = (options.inspectListener ?? inspectListener)(target.localPort); + if (!observed || currentUid === null || observed.uid !== currentUid) return false; + + const expectedEnvironment = buildOpenShellSubprocessEnv(); + if (!expectedEnvironment.HOME || observed.home !== expectedEnvironment.HOME) { + return false; + } + + const observedExecutable = observed.executable && realpath(observed.executable); + const expectedExecutable = realpath(target.executable); + if (!observedExecutable || !expectedExecutable || observedExecutable !== expectedExecutable) { + return false; + } + + return observed.commandLine === [target.executable, ...buildForwardServiceArgs(target)].join(" "); +} + /** Launch one foreground OpenShell service forward as a detached host child. */ export function launchForwardService( target: ForwardServiceTarget, diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index d7354275613..fcd0e1ee937 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -1515,6 +1515,9 @@ const sandboxCreateOrchestrationRuntime = { get getDashboardForwardPort() { return getDashboardForwardPort; }, + get matchesExistingDashboardForward() { + return matchesExistingDashboardForward; + }, readDcodeSelectionDrift: createDcodeSelectionDriftReader(runCaptureOpenshell, () => GATEWAY_NAME), getDefaultSandboxNameForAgent, getDockerDriverGatewayStateDir, @@ -2492,25 +2495,21 @@ const setupMessagingChannels = messagingChannelSetup.createSetupMessagingChannel isNonInteractive, prompt, }); - // ── Step 7: OpenClaw ───────────────────────────────────────────── const syncNemoClawConfigInSandbox = createNemoClawConfigSync({ getProviderSelectionConfig, run, openshellArgv, }); - const configureOpenclawSandbox = openclawSetup.createConfigureOpenclawSandbox({ syncNemoClawConfigInSandbox, reconcileWebSearch: openclawSetup.reconcileOpenClawWebSearchForReuse, }); - const setupOpenclaw = openclawSetup.createOpenclawSetup({ step, agentProductName, configureOpenclawSandbox, }); - const { buildChain, buildAgentVerifyChain, @@ -2521,6 +2520,7 @@ const { ensureAgentFixedForward, fetchGatewayAuthTokenFromSandbox, getDashboardForwardPort, + matchesExistingDashboardForward, printDashboard, stopAllDashboardForwards, } = onboardDashboard.createOnboardDashboardHelpers({ diff --git a/src/lib/onboard/agent-dashboard-forward.ts b/src/lib/onboard/agent-dashboard-forward.ts index e43b787318e..9ef697e67df 100644 --- a/src/lib/onboard/agent-dashboard-forward.ts +++ b/src/lib/onboard/agent-dashboard-forward.ts @@ -21,6 +21,7 @@ export type EnsureDashboardForward = ( chatUiUrl?: string, options?: { allowPortReallocation?: boolean; + reuseExistingOpenClawForward?: boolean; revalidateSandboxIdentity?: (operation: string) => void; }, ) => number; @@ -39,6 +40,7 @@ export async function ensureAgentDashboardForward(options: { /** Host port allocated to this sandbox's OpenAI-compatible API, when it has one. */ hermesApiPort?: number | null; beforeForwardPort?: (port: number) => Promise | void; + reuseExistingOpenClawForward?: boolean; revalidateSandboxIdentity?: (operation: string) => void; warn?: (message: string) => void; }): Promise { @@ -50,6 +52,7 @@ export async function ensureAgentDashboardForward(options: { controlUiPort, hermesApiPort, beforeForwardPort, + reuseExistingOpenClawForward = false, revalidateSandboxIdentity, warn = (message: string) => console.warn(message), } = options; @@ -108,6 +111,7 @@ export async function ensureAgentDashboardForward(options: { await beforeForwardPort?.(agentDashboardPort); const actualAgentDashboardPort = ensureDashboardForward(sandboxName, requestedDashboardUrl, { allowPortReallocation: false, + ...(reuseExistingOpenClawForward ? { reuseExistingOpenClawForward: true } : {}), ...(revalidateIdentity ? { revalidateSandboxIdentity: revalidateIdentity } : {}), }); if (!usesFixedApiPort) { diff --git a/src/lib/onboard/dashboard-forward-control.ts b/src/lib/onboard/dashboard-forward-control.ts index 7b58fd24393..57ff2f33acb 100644 --- a/src/lib/onboard/dashboard-forward-control.ts +++ b/src/lib/onboard/dashboard-forward-control.ts @@ -5,15 +5,18 @@ export interface DashboardForwardOptions { rollbackSandboxOnFailure?: boolean; gatewayName?: string; allowPortReallocation?: boolean; + reuseExistingOpenClawForward?: boolean; revalidateSandboxIdentity?: (operation: string) => void; } export function normalizeDashboardForwardOptions(options: DashboardForwardOptions = {}): { rollbackSandboxOnFailure: boolean; allowPortReallocation: boolean; + reuseExistingOpenClawForward: boolean; } { return { rollbackSandboxOnFailure: options.rollbackSandboxOnFailure === true, allowPortReallocation: options.allowPortReallocation !== false, + reuseExistingOpenClawForward: options.reuseExistingOpenClawForward === true, }; } diff --git a/src/lib/onboard/dashboard-port.test.ts b/src/lib/onboard/dashboard-port.test.ts index 2470261bdf1..8f149a991e2 100644 --- a/src/lib/onboard/dashboard-port.test.ts +++ b/src/lib/onboard/dashboard-port.test.ts @@ -442,6 +442,57 @@ describe("dashboard port reservation", () => { await result.reservation?.release(); assert.deepEqual(released, [18790]); }); + + it("keeps a matched direct service on the persisted port during reuse (#11074)", async () => { + const reservePort = vi.fn(); + const matchesPersistedForward = vi.fn(() => true); + + const result = await reserveCreateSandboxDashboardPort( + { + sandboxName: "cursor", + controlUiPort: null, + chatUiUrlEnv: null, + persistedPort: 18789, + agentForwardPort: 18789, + forwardListOutput: "", + registryOccupiedPorts: new Map(), + matchesPersistedForward, + }, + reservePort, + ); + + expect(result).toEqual({ + preferredPort: 18789, + effectivePort: 18789, + chatUiUrl: "http://127.0.0.1:18789", + reservation: null, + }); + expect(matchesPersistedForward).toHaveBeenCalledWith(18789, "http://127.0.0.1:18789"); + expect(reservePort).not.toHaveBeenCalled(); + }); + + it("does not reallocate an occupied persisted port when its service does not match", async () => { + const reservePort = vi.fn(); + + await expect( + reserveCreateSandboxDashboardPort( + { + sandboxName: "cursor", + controlUiPort: null, + chatUiUrlEnv: null, + persistedPort: 18789, + agentForwardPort: 18789, + forwardListOutput: "", + registryOccupiedPorts: new Map(), + findAvailablePort: (_sandboxName, preferredPort) => + preferredPort === 18789 ? 18790 : preferredPort, + matchesPersistedForward: () => false, + }, + reservePort, + ), + ).rejects.toThrow(/cannot be reallocated or adopted/u); + expect(reservePort).not.toHaveBeenCalled(); + }); }); describe("findAvailableDashboardPort multi-gateway registry occupancy", () => { diff --git a/src/lib/onboard/dashboard-port.ts b/src/lib/onboard/dashboard-port.ts index fe5ea779d56..7b041b5212e 100644 --- a/src/lib/onboard/dashboard-port.ts +++ b/src/lib/onboard/dashboard-port.ts @@ -413,6 +413,7 @@ export interface CreateSandboxDashboardPortInput { forwardListOutput: string | null; defaultPort?: number; findAvailablePort?: typeof findAvailableDashboardPort; + matchesPersistedForward?: (port: number, chatUiUrl: string) => boolean; warn?: (message: string) => void; // Cross-gateway occupancy view derived from the sandbox registry. Lets the // allocator avoid handing out a dashboard port that already belongs to a @@ -570,6 +571,26 @@ export async function reserveCreateSandboxDashboardPort( input: CreateSandboxDashboardPortInput, reservePort: (port: number) => Promise = reserveDashboardPort, ): Promise { + const matchesPersistedForward = input.matchesPersistedForward; + const persistedCandidate = + input.persistedPort !== null && matchesPersistedForward + ? resolveCreateSandboxDashboardPort({ + ...input, + findAvailablePort: (_sandboxName, preferredPort) => preferredPort, + registryOccupiedPorts: new Map(), + warn: undefined, + }) + : null; + const checksPersistedForward = + persistedCandidate !== null && input.persistedPort === persistedCandidate.preferredPort; + if ( + persistedCandidate && + checksPersistedForward && + matchesPersistedForward?.(persistedCandidate.preferredPort, persistedCandidate.chatUiUrl) === + true + ) { + return { ...persistedCandidate, reservation: null }; + } const occupied = new Map( input.registryOccupiedPorts ?? getRegistryOccupiedDashboardPorts(input.sandboxName), ); @@ -580,6 +601,11 @@ export async function reserveCreateSandboxDashboardPort( registryOccupiedPorts: occupied, warn: undefined, }); + if (checksPersistedForward && result.effectivePort !== result.preferredPort) { + throw new Error( + `Registered dashboard port ${String(result.preferredPort)} is already occupied; it cannot be reallocated or adopted.`, + ); + } if (forwardOwner.get(String(result.effectivePort)) === input.sandboxName) { if (result.effectivePort !== result.preferredPort) { input.warn?.( diff --git a/src/lib/onboard/dashboard.ts b/src/lib/onboard/dashboard.ts index 615e45e0ad7..a6978b1ab65 100644 --- a/src/lib/onboard/dashboard.ts +++ b/src/lib/onboard/dashboard.ts @@ -6,6 +6,7 @@ import os from "node:os"; import path from "node:path"; import { launchForwardService, + matchesRunningForwardService, type ForwardServiceTarget, } from "../adapters/openshell/forward-service"; import type { AgentDefinition } from "../agent/defs"; @@ -88,6 +89,7 @@ export interface OnboardDashboardDeps { forwardService?: { executable(): string; launch?(target: ForwardServiceTarget): void; + matchesListener?(target: ForwardServiceTarget): boolean; retireLegacy?(sandboxName: string, gatewayName: string, ports: readonly number[]): number; resolveGatewayName( sandbox: { gatewayName?: string | null; gatewayPort?: number | null } | null | undefined, @@ -166,6 +168,7 @@ export interface OnboardDashboardHelpers { chatUiUrl?: string, options?: Parameters[1], ): string; + matchesExistingDashboardForward(sandboxName: string, port: number, chatUiUrl: string): boolean; getWslHostAddress( options?: Parameters[0], ): string | null; @@ -224,6 +227,7 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa if (!executable) throw new Error("OpenShell is unavailable"); return executable; }, + matchesListener: matchesRunningForwardService, resolveGatewayName: productionForwardService.resolveGatewayName, retireLegacy: (sandboxName: string, gatewayName: string, ports: readonly number[]) => productionForwardService.retireLegacy(sandboxName, gatewayName, ports, { @@ -373,13 +377,29 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa process.exit(1); } + function matchesExistingDashboardForward( + sandboxName: string, + port: number, + chatUiUrl: string, + registryOccupiedPorts = getRegistryOccupiedDashboardPorts(sandboxName, listSandboxes), + ): boolean { + if (registryOccupiedPorts.has(String(port))) return false; + const gatewayName = resolveForwardServiceGateway(sandboxName); + return Boolean( + gatewayName && + forwardService?.matchesListener?.( + forwardTarget(sandboxName, gatewayName, port, getDashboardForwardTarget(chatUiUrl)), + ), + ); + } + function ensureDashboardForward( sandboxName: string, chatUiUrl = `http://127.0.0.1:${CONTROL_UI_PORT}`, options: DashboardForwardOptions = {}, ): number { chatUiUrl ||= `http://127.0.0.1:${CONTROL_UI_PORT}`; - const { rollbackSandboxOnFailure, allowPortReallocation } = + const { rollbackSandboxOnFailure, allowPortReallocation, reuseExistingOpenClawForward } = normalizeDashboardForwardOptions(options); const { revalidateSandboxIdentity } = options; const preferredPort = Number(getDashboardForwardPort(chatUiUrl)); @@ -394,7 +414,22 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa ); const isPortBound = deps.isPortBoundOnHost ?? isPortBoundOnHost; const persistedPort = getPersistedDashboardPort(sandboxName, listSandboxes); + const registryOccupiedPorts = getRegistryOccupiedDashboardPorts(sandboxName, listSandboxes); if (persistedPort === preferredPort && isPortBound(preferredPort)) { + if ( + reuseExistingOpenClawForward && + matchesExistingDashboardForward( + sandboxName, + preferredPort, + chatUiUrl, + registryOccupiedPorts, + ) + ) { + revalidateSandboxIdentity?.( + `retain dashboard forward ${String(preferredPort)} for sandbox '${sandboxName}'`, + ); + return preferredPort; + } throw new Error( `Registered dashboard port ${String(preferredPort)} is already occupied; it cannot be reallocated or adopted.`, ); @@ -406,7 +441,7 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa preferredPort, existingForwards, isPortBound, - getRegistryOccupiedDashboardPorts(sandboxName, listSandboxes), + registryOccupiedPorts, ); } catch (err) { if (!rollbackSandboxOnFailure) throw err; @@ -519,6 +554,7 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa envUrl || (persistedPort === null ? undefined : `http://127.0.0.1:${String(persistedPort)}`); const actualPort = ensureDashboardForward(sandboxName, requestedUrl, { allowPortReallocation: false, + reuseExistingOpenClawForward: true, ...(revalidateSandboxIdentity ? { revalidateSandboxIdentity } : {}), }); revalidateSandboxIdentity?.(`publish the dashboard URL for sandbox '${sandboxName}'`); @@ -534,6 +570,7 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa agent: { forwardPort?: number | null; forward_ports?: number[] | null }, options: { beforeForwardPort?: (port: number) => Promise | void; + reuseExistingOpenClawForward?: boolean; revalidateSandboxIdentity?: (operation: string) => void; } = {}, ): Promise { @@ -546,6 +583,7 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa controlUiPort: chatUiUrl ? Number(getDashboardForwardPort(chatUiUrl)) : undefined, hermesApiPort: getSandbox?.(sandboxName)?.hermesApiPort, beforeForwardPort: options.beforeForwardPort, + reuseExistingOpenClawForward: options.reuseExistingOpenClawForward, revalidateSandboxIdentity: options.revalidateSandboxIdentity, }); } @@ -561,6 +599,7 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa return agent ? ensureAgentDashboardForward(sandboxName, agent, { revalidateSandboxIdentity, + ...(agent.name === "openclaw" ? { reuseExistingOpenClawForward: true } : {}), beforeForwardPort: portReservation ? (port) => portReservation.releaseBeforeForward(agent.name, port) : undefined, @@ -787,6 +826,7 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa getDashboardForwardPort, getDashboardForwardTarget, getWslHostAddress, + matchesExistingDashboardForward, printDashboard, stopAllDashboardForwards, }; diff --git a/src/lib/onboard/sandbox-create/orchestration.ts b/src/lib/onboard/sandbox-create/orchestration.ts index 4fb8ba50361..3667a8844e3 100644 --- a/src/lib/onboard/sandbox-create/orchestration.ts +++ b/src/lib/onboard/sandbox-create/orchestration.ts @@ -1222,6 +1222,13 @@ function shouldInspectExistingSandbox(input: { return input.liveExists && !input.portableLifecycle && !input.resumingVerifiedCreate; } +function openClawPersistedForwardMatcher( + agentName: string | null, + matcher: (port: number, chatUiUrl: string) => boolean, +): typeof matcher | undefined { + return agentName === "openclaw" ? matcher : undefined; +} + type PortableAgentReceiptGenerationObservation = | { readonly kind: "absent" | "openclaw" } | { @@ -1331,6 +1338,7 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche formatSandboxAgentName, formatSandboxBuildEstimateNote, getDashboardForwardPort, + matchesExistingDashboardForward, readDcodeSelectionDrift, getDefaultSandboxNameForAgent, getDockerDriverGatewayStateDir, @@ -1489,6 +1497,9 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche agentForwardPort: dashboardRuntime.getAgentPrimaryForwardPort(agent, DASHBOARD_PORT), defaultPort: DASHBOARD_PORT, forwardListOutput: runCaptureOpenshell(["forward", "list"], { ignoreError: true }), + matchesPersistedForward: openClawPersistedForwardMatcher(requestedAgentName, (port, url) => + matchesExistingDashboardForward(sandboxName, port, url), + ), warn: (message: string) => console.warn(message), }); ({ effectivePort, chatUiUrl } = dashboardSelection); diff --git a/src/lib/onboard/sandbox-reuse.test.ts b/src/lib/onboard/sandbox-reuse.test.ts index c59aab6bc39..ed2f4480fd2 100644 --- a/src/lib/onboard/sandbox-reuse.test.ts +++ b/src/lib/onboard/sandbox-reuse.test.ts @@ -33,6 +33,7 @@ describe("applyReusedSandboxDashboardState", () => { it("clears Hermes dashboard registry fields when the reused sandbox has it disabled", () => { const updateSandbox = vi.fn(); + const ensureDashboardForward = vi.fn(() => 18789); const sandboxGpuConfig: SandboxGpuConfig = { hostGpuDetected: false, hostGpuPlatform: null, @@ -53,7 +54,7 @@ describe("applyReusedSandboxDashboardState", () => { sandboxGpuConfig, gatewayName: "nemoclaw", gatewayPort: 8080, - ensureDashboardForward: vi.fn(() => 18789), + ensureDashboardForward, hermesDashboardForwarding: { resolveStateForPort: vi.fn(() => hermesDashboardState), ensureForState: vi.fn(), @@ -71,6 +72,9 @@ describe("applyReusedSandboxDashboardState", () => { gatewayPort: 8080, }); expect(result.hermesDashboardState).toBe(hermesDashboardState); + expect(ensureDashboardForward).toHaveBeenCalledWith("reuse-me", "http://127.0.0.1:18789", { + reuseExistingOpenClawForward: true, + }); }); it("skips dashboard forwarding while preserving reuse metadata for terminal agents", () => { @@ -210,6 +214,7 @@ describe("applyReusedSandboxDashboardState", () => { throw new Error("Sandbox identity changed before the dashboard entry"); }); const ensureForState = vi.fn(); + const ensureDashboardForward = vi.fn(() => 18790); const updateReusedSandboxMetadata = vi.fn(); const updateSandbox = vi.fn(); @@ -218,7 +223,7 @@ describe("applyReusedSandboxDashboardState", () => { sandboxName: "reuse-me", chatUiUrl: "http://127.0.0.1:18789", env: {}, - agent: null, + agent: { name: "hermes" } as any, model: "test-model", provider: "openai-compatible", selectionVerified: true, @@ -232,7 +237,7 @@ describe("applyReusedSandboxDashboardState", () => { }, gatewayName: "nemoclaw", gatewayPort: 8080, - ensureDashboardForward: vi.fn(() => 18790), + ensureDashboardForward, hermesDashboardForwarding: { resolveStateForPort: vi.fn(() => ({ enabled: false, config: null })), ensureForState, @@ -244,6 +249,9 @@ describe("applyReusedSandboxDashboardState", () => { ).toThrow(/Sandbox identity changed before/u); expect(ensureForState).toHaveBeenCalledOnce(); + expect(ensureDashboardForward).toHaveBeenCalledWith("reuse-me", "http://127.0.0.1:18789", { + revalidateSandboxIdentity, + }); expect(updateReusedSandboxMetadata).not.toHaveBeenCalled(); expect(updateSandbox).not.toHaveBeenCalled(); }); diff --git a/src/lib/onboard/sandbox-reuse.ts b/src/lib/onboard/sandbox-reuse.ts index 4cfebedd42a..e791688681b 100644 --- a/src/lib/onboard/sandbox-reuse.ts +++ b/src/lib/onboard/sandbox-reuse.ts @@ -104,7 +104,10 @@ export interface ReusedSandboxDashboardStateInput { ensureDashboardForward( sandboxName: string, chatUiUrl: string, - options?: { revalidateSandboxIdentity?: (operation: string) => void }, + options?: { + reuseExistingOpenClawForward?: boolean; + revalidateSandboxIdentity?: (operation: string) => void; + }, ): number; hermesDashboardForwarding: ReusedSandboxDashboardForwarding; updateSandbox?(sandboxName: string, updates: Partial): unknown; @@ -141,15 +144,15 @@ export function applyReusedSandboxDashboardState( `Sandbox '${input.sandboxName}' was created without remote dashboard exposure. Re-run onboarding with NEMOCLAW_DASHBOARD_BIND=0.0.0.0 and --recreate-sandbox before opening a remote bind.`, ); } - input.revalidateSandboxIdentity?.( - `restore dashboard state for sandbox '${input.sandboxName}'`, - ); + input.revalidateSandboxIdentity?.(`restore dashboard state for sandbox '${input.sandboxName}'`); + const reuseExistingOpenClawForward = input.agent == null || input.agent.name === "openclaw"; const dashboardPort = manageDashboard - ? input.revalidateSandboxIdentity - ? input.ensureDashboardForward(input.sandboxName, input.chatUiUrl, { - revalidateSandboxIdentity: input.revalidateSandboxIdentity, - }) - : input.ensureDashboardForward(input.sandboxName, input.chatUiUrl) + ? input.ensureDashboardForward(input.sandboxName, input.chatUiUrl, { + ...(reuseExistingOpenClawForward ? { reuseExistingOpenClawForward: true } : {}), + ...(input.revalidateSandboxIdentity + ? { revalidateSandboxIdentity: input.revalidateSandboxIdentity } + : {}), + }) : 0; const chatUiUrl = manageDashboard ? `http://127.0.0.1:${dashboardPort}` : input.chatUiUrl; if (manageDashboard) { diff --git a/test/onboarding/onboard-finalization-dashboard-forward.test.ts b/test/onboarding/onboard-finalization-dashboard-forward.test.ts index 6a8eefd0f5d..e56380af6ef 100644 --- a/test/onboarding/onboard-finalization-dashboard-forward.test.ts +++ b/test/onboarding/onboard-finalization-dashboard-forward.test.ts @@ -3,14 +3,17 @@ import { describe, expect, it, vi } from "vitest"; +import type { ForwardServiceTarget } from "../../src/lib/adapters/openshell/forward-service"; import { createOnboardDashboardHelpers } from "../../src/lib/onboard/dashboard"; import type { ListSandboxesFn } from "../../src/lib/onboard/dashboard-port"; function harness(options: { listSandboxes: ListSandboxesFn; isPortBound?: (port: number) => boolean; + matchesListener?: (target: ForwardServiceTarget) => boolean; }) { const launch = vi.fn(); + const matchesListener = vi.fn(options.matchesListener ?? (() => false)); const helpers = createOnboardDashboardHelpers({ runOpenshell: vi.fn(() => ({ status: 0 })), runCaptureOpenshell: vi.fn(() => ""), @@ -28,11 +31,12 @@ function harness(options: { forwardService: { executable: () => "/usr/local/bin/openshell", launch, + matchesListener, resolveGatewayName: () => "nemoclaw", retireLegacy: vi.fn(() => 0), }, }); - return { helpers, launch }; + return { helpers, launch, matchesListener }; } describe("finalization dashboard ForwardTcp launch", () => { @@ -71,6 +75,82 @@ describe("finalization dashboard ForwardTcp launch", () => { expect(launch).not.toHaveBeenCalled(); }); + it("retains the exact OpenClaw forward service on repeated onboarding (#11074)", () => { + vi.stubEnv("CHAT_UI_URL", undefined); + const { helpers, launch, matchesListener } = harness({ + listSandboxes: () => ({ + sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], + }), + isPortBound: (port) => port === 18_790, + matchesListener: () => true, + }); + + expect(helpers.ensureFinalizationDashboardForward("reonboard-test")).toBe(18_790); + expect(matchesListener).toHaveBeenCalledWith( + expect.objectContaining({ + gatewayName: "nemoclaw", + sandboxName: "reonboard-test", + localPort: 18_790, + targetPort: 18_790, + }), + ); + expect(launch).not.toHaveBeenCalled(); + expect(process.env.CHAT_UI_URL).toBe("http://127.0.0.1:18790"); + }); + + it("does not retain a forward when another sandbox registers the same port", () => { + vi.stubEnv("CHAT_UI_URL", undefined); + const { helpers, matchesListener } = harness({ + listSandboxes: () => ({ + sandboxes: [ + { name: "reonboard-test", dashboardPort: 18_790 }, + { name: "other", dashboardPort: 18_790 }, + ], + }), + isPortBound: (port) => port === 18_790, + matchesListener: () => true, + }); + + expect(() => helpers.ensureFinalizationDashboardForward("reonboard-test")).toThrow( + /cannot be reallocated/u, + ); + expect(matchesListener).not.toHaveBeenCalled(); + }); + + it("enables retained-forward matching only for OpenClaw agents", async () => { + vi.stubEnv("CHAT_UI_URL", undefined); + const openClaw = harness({ + listSandboxes: () => ({ + sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], + }), + isPortBound: (port) => port === 18_790, + matchesListener: () => true, + }); + + await expect( + openClaw.helpers.ensureFinalizationAgentDashboardForward("reonboard-test", { + name: "openclaw", + forwardPort: 18_790, + }), + ).resolves.toBe(18_790); + + const hermes = harness({ + listSandboxes: () => ({ + sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], + }), + isPortBound: (port) => port === 18_790, + matchesListener: () => true, + }); + + await expect( + hermes.helpers.ensureFinalizationAgentDashboardForward("reonboard-test", { + name: "hermes", + forwardPort: 18_790, + }), + ).rejects.toThrow(/cannot be reallocated/u); + expect(hermes.matchesListener).not.toHaveBeenCalled(); + }); + it("honors an explicit dashboard URL", () => { vi.stubEnv("CHAT_UI_URL", "http://127.0.0.1:19001"); const { helpers, launch } = harness({ From d7fbc918589c204fe11283b0d76eaf819a4c141a Mon Sep 17 00:00:00 2001 From: San Dang Date: Mon, 7 Sep 2026 02:15:22 +0700 Subject: [PATCH 02/20] fix(onboard): reconcile reused dashboard forward lifecycle Signed-off-by: San Dang --- ci/e2e-assertion-budget.json | 14 +- src/lib/actions/onboard.test.ts | 11 +- src/lib/actions/onboard.ts | 13 +- .../openshell/forward-service.test.ts | 53 +---- src/lib/adapters/openshell/forward-service.ts | 102 +--------- src/lib/onboard.ts | 8 +- src/lib/onboard/agent-dashboard-forward.ts | 6 +- src/lib/onboard/dashboard-port.test.ts | 51 ----- src/lib/onboard/dashboard-port.ts | 26 --- src/lib/onboard/dashboard.ts | 154 ++++++++++---- .../onboard/dashboard/reuse-lifecycle.test.ts | 16 ++ src/lib/onboard/dashboard/reuse-lifecycle.ts | 26 +++ .../onboard/sandbox-create/orchestration.ts | 13 +- src/lib/onboard/sandbox-reuse.test.ts | 46 +++++ src/lib/onboard/sandbox-reuse.ts | 48 ++++- test/e2e/live/double-onboard.test.ts | 189 +++++++----------- ...ard-finalization-dashboard-forward.test.ts | 132 +++++++----- 17 files changed, 440 insertions(+), 468 deletions(-) create mode 100644 src/lib/onboard/dashboard/reuse-lifecycle.test.ts create mode 100644 src/lib/onboard/dashboard/reuse-lifecycle.ts diff --git a/ci/e2e-assertion-budget.json b/ci/e2e-assertion-budget.json index 7fca917cd9d..819c276091c 100644 --- a/ci/e2e-assertion-budget.json +++ b/ci/e2e-assertion-budget.json @@ -15,26 +15,26 @@ "testFileCount": 86, "liveFileCount": 222, "direct": { - "expectCalls": 1882, - "matcherAssertions": 1851, + "expectCalls": 1881, + "matcherAssertions": 1850, "nodeAssertions": 100, "namedAssertionHelpers": 627, "failCalls": 8, "throwGuards": 87, "objectFieldAssertions": 245, - "assertionPoints": 2918, + "assertionPoints": 2917, "generatedProbeBlocks": 136, "generatedProbeConditions": 354 }, "unique": { - "expectCalls": 2374, - "matcherAssertions": 2338, + "expectCalls": 2373, + "matcherAssertions": 2337, "nodeAssertions": 119, "namedAssertionHelpers": 930, "failCalls": 38, "throwGuards": 638, "objectFieldAssertions": 348, - "assertionPoints": 4411, + "assertionPoints": 4410, "generatedProbeBlocks": 290, "generatedProbeConditions": 975 }, @@ -60,7 +60,7 @@ "test/e2e/live/cron-preflight-inference-local.test.ts": [8,8,8,8,1], "test/e2e/live/dashboard-remote-bind.test.ts": [17,15,17,17,3], "test/e2e/live/device-auth-health.test.ts": [13,18,13,21,0], - "test/e2e/live/double-onboard.test.ts": [88,97,88,97,0], + "test/e2e/live/double-onboard.test.ts": [87,96,87,96,0], "test/e2e/live/external-gateway-health.test.ts": [4,5,4,11,0], "test/e2e/live/full-e2e.test.ts": [30,34,39,72,7], "test/e2e/live/gateway-guard-recovery.test.ts": [48,54,48,57,3], diff --git a/src/lib/actions/onboard.test.ts b/src/lib/actions/onboard.test.ts index 4085f1e87ff..e911867fbc8 100644 --- a/src/lib/actions/onboard.test.ts +++ b/src/lib/actions/onboard.test.ts @@ -25,13 +25,20 @@ describe("onboard action runtime composition", () => { ); }); - it("passes host-only Google Chat dependencies into legacy onboarding", async () => { + it("passes host-only runtime dependencies into legacy onboarding", async () => { const googlechatTunnelRuntime = { loadServices: vi.fn(), loadWebhookProxy: vi.fn(), }; + const dashboardReuseLifecycle = { + startSandbox: vi.fn(), + stopSandbox: vi.fn(), + }; - await runOnboardAction({ "non-interactive": true }, { googlechatTunnelRuntime }); + await runOnboardAction( + { "non-interactive": true }, + { googlechatTunnelRuntime, dashboardReuseLifecycle }, + ); expect(mocks.onboard).toHaveBeenCalledWith({ nonInteractive: true, diff --git a/src/lib/actions/onboard.ts b/src/lib/actions/onboard.ts index 9beb40d57d8..f0051bead30 100644 --- a/src/lib/actions/onboard.ts +++ b/src/lib/actions/onboard.ts @@ -5,12 +5,17 @@ import { loadServingCatalog } from "../inference/serving/catalog-loader"; import type { GooglechatTunnelRuntimeDeps } from "../messaging/channels/googlechat/hooks/tunnel-runtime"; import { type OnboardCommandOptions, runOnboardCommand } from "../onboard/command"; import { type OnboardFlags, readAgentRegistryNames } from "../onboard/command-support"; +import { + type DashboardReuseLifecycle, + withDashboardReuseLifecycle, +} from "../onboard/dashboard/reuse-lifecycle"; import { resolveOnboardResumeIntent } from "../onboard/session-bootstrap"; import { loadOnboardCommandResumeSession } from "../onboard/sandbox-registration"; import type { OnboardOptions } from "../onboard/types"; export interface OnboardActionRuntimeDeps { readonly googlechatTunnelRuntime?: Omit; + readonly dashboardReuseLifecycle?: DashboardReuseLifecycle; } async function runOnboard( @@ -22,7 +27,13 @@ async function runOnboard( const { onboard } = (await import("../onboard")) as unknown as { onboard: (onboardOptions?: OnboardOptions) => Promise; }; - await onboard({ ...options, googlechatTunnelRuntime: runtimeDeps.googlechatTunnelRuntime }); + const lifecycle = runtimeDeps.dashboardReuseLifecycle ?? { + startSandbox: (await import("./sandbox/start")).startSandbox, + stopSandbox: (await import("./sandbox/stop")).stopSandbox, + }; + await withDashboardReuseLifecycle(lifecycle, () => + onboard({ ...options, googlechatTunnelRuntime: runtimeDeps.googlechatTunnelRuntime }), + ); } function buildOnboardCommandDeps(flags: OnboardFlags, runtimeDeps: OnboardActionRuntimeDeps) { diff --git a/src/lib/adapters/openshell/forward-service.test.ts b/src/lib/adapters/openshell/forward-service.test.ts index c340d7fffb0..2de4e61ca73 100644 --- a/src/lib/adapters/openshell/forward-service.test.ts +++ b/src/lib/adapters/openshell/forward-service.test.ts @@ -6,13 +6,11 @@ import { describe, expect, it, vi } from "vitest"; import { buildForwardServiceArgs, launchForwardService, - matchesRunningForwardService, - type ForwardServiceProcess, type ForwardServiceTarget, } from "./forward-service"; const target: ForwardServiceTarget = { - executable: process.execPath, + executable: "/usr/local/bin/openshell", gatewayName: "nemoclaw", workspace: "default", sandboxName: "demo", @@ -21,21 +19,6 @@ const target: ForwardServiceTarget = { targetHost: "127.0.0.1", targetPort: 18_789, }; -const expectedCommand = [target.executable, ...buildForwardServiceArgs(target)].join(" "); -const testHome = process.env.HOME ?? ""; -const testUid = process.getuid?.() ?? 1_000; - -function matchingListenerOptions(overrides: Partial = {}) { - return { - inspectListener: () => ({ - commandLine: expectedCommand, - executable: target.executable, - home: testHome, - uid: testUid, - ...overrides, - }), - }; -} describe("OpenShell forward service", () => { it("builds the direct ForwardTcp command with explicit gateway authority", () => { @@ -91,40 +74,6 @@ describe("OpenShell forward service", () => { expect(spawnDetached).not.toHaveBeenCalled(); }); - it("matches the exact forward-service listener started by NemoClaw (#11074)", () => { - expect(matchesRunningForwardService(target, matchingListenerOptions())).toBe(true); - }); - - it.each([ - { - name: "different arguments", - overrides: { - commandLine: [ - target.executable, - ...buildForwardServiceArgs({ ...target, sandboxName: "other" }), - ].join(" "), - }, - }, - { - name: "different user", - overrides: { uid: testUid + 1 }, - }, - { - name: "different executable", - overrides: { executable: "/bin/sh" }, - }, - { - name: "different OpenShell home", - overrides: { home: `${testHome}-other` }, - }, - ])("rejects a listener with $name", ({ overrides }) => { - expect(matchesRunningForwardService(target, matchingListenerOptions(overrides))).toBe(false); - }); - - it("rejects an inconclusive listener inspection", () => { - expect(matchesRunningForwardService(target, { inspectListener: () => null })).toBe(false); - }); - it("fails when the detached service does not bind before the deadline", () => { expect(() => launchForwardService(target, { diff --git a/src/lib/adapters/openshell/forward-service.ts b/src/lib/adapters/openshell/forward-service.ts index 3fe774e4eae..8f78fe81983 100644 --- a/src/lib/adapters/openshell/forward-service.ts +++ b/src/lib/adapters/openshell/forward-service.ts @@ -1,8 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { spawn, spawnSync } from "node:child_process"; -import fs from "node:fs"; +import { spawn } from "node:child_process"; import path from "node:path"; import { isValidName } from "../../name-validation"; @@ -11,8 +10,6 @@ import { probeLocalForwardListener } from "./local-forward-listener"; const START_TIMEOUT_MS = 30_000; const POLL_INTERVAL_MS = 100; -const INSPECTION_TIMEOUT_MS = 1_000; -const INSPECTION_MAX_BUFFER_BYTES = 16 * 1024; const sleepBuffer = new Int32Array(new SharedArrayBuffer(4)); export interface ForwardServiceTarget { @@ -40,17 +37,6 @@ export interface ForwardServiceLaunchOptions { readonly timeoutMs?: number; } -export interface ForwardServiceProcess { - readonly commandLine: string; - readonly executable: string; - readonly home: string; - readonly uid: number; -} - -export interface ForwardServiceListenerOptions { - readonly inspectListener?: (port: number) => ForwardServiceProcess | null; -} - function isPort(value: unknown): value is number { return Number.isSafeInteger(value) && Number(value) >= 1 && Number(value) <= 65_535; } @@ -108,92 +94,6 @@ export function buildForwardServiceArgs(target: ForwardServiceTarget): string[] ]; } -function probeExecutable(candidates: readonly string[]): string | null { - return candidates.find((candidate) => fs.existsSync(candidate)) ?? null; -} - -function runInspection(executable: string, args: readonly string[]): string | null { - const result = spawnSync(executable, [...args], { - encoding: "utf8", - env: buildOpenShellSubprocessEnv(), - maxBuffer: INSPECTION_MAX_BUFFER_BYTES, - stdio: ["ignore", "pipe", "ignore"], - timeout: INSPECTION_TIMEOUT_MS, - }); - return result.error || result.status !== 0 ? null : result.stdout; -} - -function inspectListener(port: number): ForwardServiceProcess | null { - const lsof = probeExecutable(["/usr/sbin/lsof", "/usr/bin/lsof"]); - const ps = probeExecutable(["/bin/ps", "/usr/bin/ps"]); - if (!lsof || !ps) return null; - - const listenerOutput = runInspection(lsof, [ - "-nP", - `-iTCP:${String(port)}`, - "-sTCP:LISTEN", - "-Fp", - ]); - const pids = [ - ...new Set( - listenerOutput - ?.split(/\r?\n/u) - .flatMap((line) => /^p([1-9]\d*)$/u.exec(line.trim())?.[1] ?? []) - .map(Number) ?? [], - ), - ]; - const pid = pids.length === 1 ? pids[0] : undefined; - if (pid === undefined) return null; - - const uid = Number(runInspection(ps, ["-p", String(pid), "-o", "uid="])?.trim()); - const commandLine = runInspection(ps, ["-ww", "-p", String(pid), "-o", "command="])?.trim(); - const environment = runInspection(ps, ["eww", "-p", String(pid), "-o", "command="]); - const home = environment - ?.replace(/\s+/gu, "\0") - .split("\0") - .find((entry) => entry.startsWith("HOME=")) - ?.slice("HOME=".length); - const executable = runInspection(lsof, ["-a", "-p", String(pid), "-d", "txt", "-Fn"]) - ?.split(/\r?\n/u) - .find((line) => line.startsWith("n") && line.length > 1) - ?.slice(1); - return Number.isSafeInteger(uid) && uid >= 0 && commandLine && executable && home - ? { commandLine, executable, home, uid } - : null; -} - -function realpath(value: string): string | null { - try { - return fs.realpathSync.native(value); - } catch { - return null; - } -} - -/** Match one bound listener to the direct ForwardTcp command that NemoClaw starts. */ -export function matchesRunningForwardService( - target: ForwardServiceTarget, - options: ForwardServiceListenerOptions = {}, -): boolean { - validateForwardServiceTarget(target); - const currentUid = typeof process.getuid === "function" ? process.getuid() : null; - const observed = (options.inspectListener ?? inspectListener)(target.localPort); - if (!observed || currentUid === null || observed.uid !== currentUid) return false; - - const expectedEnvironment = buildOpenShellSubprocessEnv(); - if (!expectedEnvironment.HOME || observed.home !== expectedEnvironment.HOME) { - return false; - } - - const observedExecutable = observed.executable && realpath(observed.executable); - const expectedExecutable = realpath(target.executable); - if (!observedExecutable || !expectedExecutable || observedExecutable !== expectedExecutable) { - return false; - } - - return observed.commandLine === [target.executable, ...buildForwardServiceArgs(target)].join(" "); -} - /** Launch one foreground OpenShell service forward as a detached host child. */ export function launchForwardService( target: ForwardServiceTarget, diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index fcd0e1ee937..7ca1b51207d 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -1515,8 +1515,8 @@ const sandboxCreateOrchestrationRuntime = { get getDashboardForwardPort() { return getDashboardForwardPort; }, - get matchesExistingDashboardForward() { - return matchesExistingDashboardForward; + get reconcileOpenClawDashboardForwardReuse() { + return reconcileOpenClawDashboardForwardReuse; }, readDcodeSelectionDrift: createDcodeSelectionDriftReader(runCaptureOpenshell, () => GATEWAY_NAME), getDefaultSandboxNameForAgent, @@ -2520,8 +2520,8 @@ const { ensureAgentFixedForward, fetchGatewayAuthTokenFromSandbox, getDashboardForwardPort, - matchesExistingDashboardForward, printDashboard, + reconcileOpenClawDashboardForwardReuse, stopAllDashboardForwards, } = onboardDashboard.createOnboardDashboardHelpers({ runOpenshell, @@ -2632,7 +2632,6 @@ async function preflightAuthoritativeRebuildTarget( } } -// ── Main ───────────────────────────────────────────────────────── const wrappedOnboard = onboardEntryOptions.wrapOnboard(runOnboard, onboardSession); const onboard = onboardSessionBootstrap.wrapOnboardDeferredExit(wrappedOnboard); async function runOnboard(opts: OnboardOptions = {}): Promise { @@ -3247,6 +3246,7 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { selectedAgent, undefined, hermesApiPortReservationScope, + resume, ), persistDashboardPort: (name, port) => registry.updateSandbox(name, { dashboardPort: port }), diff --git a/src/lib/onboard/agent-dashboard-forward.ts b/src/lib/onboard/agent-dashboard-forward.ts index 9ef697e67df..d65042068c4 100644 --- a/src/lib/onboard/agent-dashboard-forward.ts +++ b/src/lib/onboard/agent-dashboard-forward.ts @@ -98,11 +98,7 @@ export async function ensureAgentDashboardForward(options: { .filter((port) => port !== declaredPrimaryPort || port === agentDashboardPort) .map(resolveDeclaredPort); const preservePorts = [ - ...new Set([ - agentDashboardPort, - ...declaredPorts, - optionalDashboardPort, - ]), + ...new Set([agentDashboardPort, ...declaredPorts, optionalDashboardPort]), ].filter(isValidForwardPort); const requestedDashboardUrl = !usesFixedApiPort && chatUiUrl diff --git a/src/lib/onboard/dashboard-port.test.ts b/src/lib/onboard/dashboard-port.test.ts index 8f149a991e2..2470261bdf1 100644 --- a/src/lib/onboard/dashboard-port.test.ts +++ b/src/lib/onboard/dashboard-port.test.ts @@ -442,57 +442,6 @@ describe("dashboard port reservation", () => { await result.reservation?.release(); assert.deepEqual(released, [18790]); }); - - it("keeps a matched direct service on the persisted port during reuse (#11074)", async () => { - const reservePort = vi.fn(); - const matchesPersistedForward = vi.fn(() => true); - - const result = await reserveCreateSandboxDashboardPort( - { - sandboxName: "cursor", - controlUiPort: null, - chatUiUrlEnv: null, - persistedPort: 18789, - agentForwardPort: 18789, - forwardListOutput: "", - registryOccupiedPorts: new Map(), - matchesPersistedForward, - }, - reservePort, - ); - - expect(result).toEqual({ - preferredPort: 18789, - effectivePort: 18789, - chatUiUrl: "http://127.0.0.1:18789", - reservation: null, - }); - expect(matchesPersistedForward).toHaveBeenCalledWith(18789, "http://127.0.0.1:18789"); - expect(reservePort).not.toHaveBeenCalled(); - }); - - it("does not reallocate an occupied persisted port when its service does not match", async () => { - const reservePort = vi.fn(); - - await expect( - reserveCreateSandboxDashboardPort( - { - sandboxName: "cursor", - controlUiPort: null, - chatUiUrlEnv: null, - persistedPort: 18789, - agentForwardPort: 18789, - forwardListOutput: "", - registryOccupiedPorts: new Map(), - findAvailablePort: (_sandboxName, preferredPort) => - preferredPort === 18789 ? 18790 : preferredPort, - matchesPersistedForward: () => false, - }, - reservePort, - ), - ).rejects.toThrow(/cannot be reallocated or adopted/u); - expect(reservePort).not.toHaveBeenCalled(); - }); }); describe("findAvailableDashboardPort multi-gateway registry occupancy", () => { diff --git a/src/lib/onboard/dashboard-port.ts b/src/lib/onboard/dashboard-port.ts index 7b041b5212e..fe5ea779d56 100644 --- a/src/lib/onboard/dashboard-port.ts +++ b/src/lib/onboard/dashboard-port.ts @@ -413,7 +413,6 @@ export interface CreateSandboxDashboardPortInput { forwardListOutput: string | null; defaultPort?: number; findAvailablePort?: typeof findAvailableDashboardPort; - matchesPersistedForward?: (port: number, chatUiUrl: string) => boolean; warn?: (message: string) => void; // Cross-gateway occupancy view derived from the sandbox registry. Lets the // allocator avoid handing out a dashboard port that already belongs to a @@ -571,26 +570,6 @@ export async function reserveCreateSandboxDashboardPort( input: CreateSandboxDashboardPortInput, reservePort: (port: number) => Promise = reserveDashboardPort, ): Promise { - const matchesPersistedForward = input.matchesPersistedForward; - const persistedCandidate = - input.persistedPort !== null && matchesPersistedForward - ? resolveCreateSandboxDashboardPort({ - ...input, - findAvailablePort: (_sandboxName, preferredPort) => preferredPort, - registryOccupiedPorts: new Map(), - warn: undefined, - }) - : null; - const checksPersistedForward = - persistedCandidate !== null && input.persistedPort === persistedCandidate.preferredPort; - if ( - persistedCandidate && - checksPersistedForward && - matchesPersistedForward?.(persistedCandidate.preferredPort, persistedCandidate.chatUiUrl) === - true - ) { - return { ...persistedCandidate, reservation: null }; - } const occupied = new Map( input.registryOccupiedPorts ?? getRegistryOccupiedDashboardPorts(input.sandboxName), ); @@ -601,11 +580,6 @@ export async function reserveCreateSandboxDashboardPort( registryOccupiedPorts: occupied, warn: undefined, }); - if (checksPersistedForward && result.effectivePort !== result.preferredPort) { - throw new Error( - `Registered dashboard port ${String(result.preferredPort)} is already occupied; it cannot be reallocated or adopted.`, - ); - } if (forwardOwner.get(String(result.effectivePort)) === input.sandboxName) { if (result.effectivePort !== result.preferredPort) { input.warn?.( diff --git a/src/lib/onboard/dashboard.ts b/src/lib/onboard/dashboard.ts index a6978b1ab65..333875239c7 100644 --- a/src/lib/onboard/dashboard.ts +++ b/src/lib/onboard/dashboard.ts @@ -6,7 +6,6 @@ import os from "node:os"; import path from "node:path"; import { launchForwardService, - matchesRunningForwardService, type ForwardServiceTarget, } from "../adapters/openshell/forward-service"; import type { AgentDefinition } from "../agent/defs"; @@ -22,6 +21,7 @@ import { } from "./agent-dashboard-forward"; import { fetchAgentWebAuthTokenFromSandbox as fetchAgentWebAuthToken } from "./agent-web-auth-token"; import * as dashboardAccess from "./dashboard-access"; +import { getDashboardReuseLifecycle } from "./dashboard/reuse-lifecycle"; import { type DashboardForwardOptions, normalizeDashboardForwardOptions, @@ -89,12 +89,16 @@ export interface OnboardDashboardDeps { forwardService?: { executable(): string; launch?(target: ForwardServiceTarget): void; - matchesListener?(target: ForwardServiceTarget): boolean; retireLegacy?(sandboxName: string, gatewayName: string, ports: readonly number[]): number; resolveGatewayName( sandbox: { gatewayName?: string | null; gatewayPort?: number | null } | null | undefined, ): string; }; + stopSandboxForDashboardReuse?(sandboxName: string): { exitCode: number; message?: string }; + startSandboxForDashboardReuse?(sandboxName: string): Promise<{ + exitCode: number; + message?: string; + }>; printAgentDashboardUi( sandboxName: string, token: string | null, @@ -143,7 +147,8 @@ export interface OnboardDashboardHelpers { ensureFinalizationDashboardForward( sandboxName: string, revalidateSandboxIdentity?: (operation: string) => void, - ): number; + reuseExistingOpenClawForward?: boolean, + ): Promise; ensureFinalizationAgentDashboardForward( sandboxName: string, agent: { name: string; forwardPort?: number | null; forward_ports?: number[] | null } | null, @@ -151,7 +156,13 @@ export interface OnboardDashboardHelpers { portReservation?: { releaseBeforeForward(agentName: string, port: number): Promise | void; }, - ): Promise | number; + reuseExistingOpenClawForward?: boolean, + ): Promise; + reconcileOpenClawDashboardForwardReuse( + sandboxName: string, + chatUiUrl: string, + revalidateSandboxIdentity?: (operation: string) => void, + ): Promise; ensureAgentFixedForward( sandboxName: string, port: number, @@ -168,7 +179,6 @@ export interface OnboardDashboardHelpers { chatUiUrl?: string, options?: Parameters[1], ): string; - matchesExistingDashboardForward(sandboxName: string, port: number, chatUiUrl: string): boolean; getWslHostAddress( options?: Parameters[0], ): string | null; @@ -227,7 +237,6 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa if (!executable) throw new Error("OpenShell is unavailable"); return executable; }, - matchesListener: matchesRunningForwardService, resolveGatewayName: productionForwardService.resolveGatewayName, retireLegacy: (sandboxName: string, gatewayName: string, ports: readonly number[]) => productionForwardService.retireLegacy(sandboxName, gatewayName, ports, { @@ -248,6 +257,7 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa }), } : undefined); + const reconciledOpenClawForwards = new Map(); function resolveForwardServiceGateway( sandboxName: string, @@ -377,20 +387,59 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa process.exit(1); } - function matchesExistingDashboardForward( + async function reconcileOpenClawDashboardForwardReuse( sandboxName: string, - port: number, chatUiUrl: string, - registryOccupiedPorts = getRegistryOccupiedDashboardPorts(sandboxName, listSandboxes), - ): boolean { - if (registryOccupiedPorts.has(String(port))) return false; - const gatewayName = resolveForwardServiceGateway(sandboxName); - return Boolean( - gatewayName && - forwardService?.matchesListener?.( - forwardTarget(sandboxName, gatewayName, port, getDashboardForwardTarget(chatUiUrl)), - ), + revalidateSandboxIdentity?: (operation: string) => void, + ): Promise { + const port = Number(getDashboardForwardPort(chatUiUrl)); + const isPortBound = deps.isPortBoundOnHost ?? isPortBoundOnHost; + if (reconciledOpenClawForwards.get(sandboxName) === port && isPortBound(port)) return; + if (!isPortBound(port)) return; + if (getRegistryOccupiedDashboardPorts(sandboxName, listSandboxes).has(String(port))) { + throw new Error( + `Registered dashboard port ${String(port)} is already occupied; it cannot be reallocated or adopted.`, + ); + } + + revalidateSandboxIdentity?.( + `restart sandbox '${sandboxName}' to reconcile dashboard forward ${String(port)}`, ); + const stopped = ( + deps.stopSandboxForDashboardReuse ?? getDashboardReuseLifecycle()?.stopSandbox + )?.(sandboxName) ?? { + exitCode: 1, + message: "sandbox lifecycle is unavailable", + }; + if (stopped.exitCode !== 0) { + throw new Error( + `Could not stop sandbox '${sandboxName}' to reconcile dashboard port ${String(port)}${ + stopped.message ? `: ${stopped.message}` : "." + }`, + ); + } + if (isPortBound(port)) { + throw new Error( + `Registered dashboard port ${String(port)} remained occupied after sandbox '${sandboxName}' stopped; it cannot be adopted.`, + ); + } + + const started = await ( + deps.startSandboxForDashboardReuse ?? getDashboardReuseLifecycle()?.startSandbox + )?.(sandboxName); + if (!started) { + throw new Error( + `Could not start sandbox '${sandboxName}' to reconcile dashboard port ${String(port)}: sandbox lifecycle is unavailable.`, + ); + } + if (started.exitCode !== 0 || !isPortBound(port)) { + throw new Error( + `Sandbox '${sandboxName}' did not restore dashboard port ${String(port)} after restart${ + started.message ? `: ${started.message}` : "." + }`, + ); + } + reconciledOpenClawForwards.set(sandboxName, port); } function ensureDashboardForward( @@ -418,16 +467,13 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa if (persistedPort === preferredPort && isPortBound(preferredPort)) { if ( reuseExistingOpenClawForward && - matchesExistingDashboardForward( - sandboxName, - preferredPort, - chatUiUrl, - registryOccupiedPorts, - ) + reconciledOpenClawForwards.get(sandboxName) === preferredPort && + !registryOccupiedPorts.has(String(preferredPort)) ) { revalidateSandboxIdentity?.( `retain dashboard forward ${String(preferredPort)} for sandbox '${sandboxName}'`, ); + reconciledOpenClawForwards.delete(sandboxName); return preferredPort; } throw new Error( @@ -544,17 +590,27 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa * `CHAT_UI_URL`, so after the forward starts this writes the bound port to * `CHAT_UI_URL`. (#8970) */ - function ensureFinalizationDashboardForward( + async function ensureFinalizationDashboardForward( sandboxName: string, revalidateSandboxIdentity?: (operation: string) => void, - ): number { + reuseExistingOpenClawForward = false, + ): Promise { const envUrl = process.env.CHAT_UI_URL; const persistedPort = envUrl ? null : getPersistedDashboardPort(sandboxName, listSandboxes); const requestedUrl = envUrl || (persistedPort === null ? undefined : `http://127.0.0.1:${String(persistedPort)}`); + const mayReuseForward = + reuseExistingOpenClawForward || reconciledOpenClawForwards.has(sandboxName); + if (mayReuseForward) { + await reconcileOpenClawDashboardForwardReuse( + sandboxName, + requestedUrl || `http://127.0.0.1:${CONTROL_UI_PORT}`, + revalidateSandboxIdentity, + ); + } const actualPort = ensureDashboardForward(sandboxName, requestedUrl, { allowPortReallocation: false, - reuseExistingOpenClawForward: true, + ...(mayReuseForward ? { reuseExistingOpenClawForward: true } : {}), ...(revalidateSandboxIdentity ? { revalidateSandboxIdentity } : {}), }); revalidateSandboxIdentity?.(`publish the dashboard URL for sandbox '${sandboxName}'`); @@ -588,23 +644,45 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa }); } - function ensureFinalizationAgentDashboardForward( + async function ensureFinalizationAgentDashboardForward( sandboxName: string, agent: { name: string; forwardPort?: number | null; forward_ports?: number[] | null } | null, revalidateSandboxIdentity?: (operation: string) => void, portReservation?: { releaseBeforeForward(agentName: string, port: number): Promise | void; }, - ): Promise | number { - return agent - ? ensureAgentDashboardForward(sandboxName, agent, { - revalidateSandboxIdentity, - ...(agent.name === "openclaw" ? { reuseExistingOpenClawForward: true } : {}), - beforeForwardPort: portReservation - ? (port) => portReservation.releaseBeforeForward(agent.name, port) - : undefined, - }) - : ensureFinalizationDashboardForward(sandboxName, revalidateSandboxIdentity); + reuseExistingOpenClawForward = false, + ): Promise { + if (!agent) { + return ensureFinalizationDashboardForward( + sandboxName, + revalidateSandboxIdentity, + reuseExistingOpenClawForward, + ); + } + const mayReuseOpenClawForward = + agent.name === "openclaw" && + (reuseExistingOpenClawForward || reconciledOpenClawForwards.has(sandboxName)); + if (mayReuseOpenClawForward) { + const registeredPort = getPersistedDashboardPort(sandboxName, listSandboxes); + const requestedUrl = + process.env.CHAT_UI_URL || + (registeredPort === null + ? `http://127.0.0.1:${CONTROL_UI_PORT}` + : `http://127.0.0.1:${String(registeredPort)}`); + await reconcileOpenClawDashboardForwardReuse( + sandboxName, + requestedUrl, + revalidateSandboxIdentity, + ); + } + return ensureAgentDashboardForward(sandboxName, agent, { + revalidateSandboxIdentity, + ...(mayReuseOpenClawForward ? { reuseExistingOpenClawForward: true } : {}), + beforeForwardPort: portReservation + ? (port) => portReservation.releaseBeforeForward(agent.name, port) + : undefined, + }); } function ensureAgentFixedForward( @@ -826,8 +904,8 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa getDashboardForwardPort, getDashboardForwardTarget, getWslHostAddress, - matchesExistingDashboardForward, printDashboard, + reconcileOpenClawDashboardForwardReuse, stopAllDashboardForwards, }; } diff --git a/src/lib/onboard/dashboard/reuse-lifecycle.test.ts b/src/lib/onboard/dashboard/reuse-lifecycle.test.ts new file mode 100644 index 00000000000..ebc7eab109d --- /dev/null +++ b/src/lib/onboard/dashboard/reuse-lifecycle.test.ts @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { expect, it, vi } from "vitest"; + +import { getDashboardReuseLifecycle, withDashboardReuseLifecycle } from "./reuse-lifecycle"; + +it("exposes dashboard reuse lifecycle only while onboarding runs", async () => { + const lifecycle = { startSandbox: vi.fn(), stopSandbox: vi.fn() }; + + await withDashboardReuseLifecycle(lifecycle, async () => { + expect(getDashboardReuseLifecycle()).toBe(lifecycle); + }); + + expect(getDashboardReuseLifecycle()).toBeUndefined(); +}); diff --git a/src/lib/onboard/dashboard/reuse-lifecycle.ts b/src/lib/onboard/dashboard/reuse-lifecycle.ts new file mode 100644 index 00000000000..18ec3c53842 --- /dev/null +++ b/src/lib/onboard/dashboard/reuse-lifecycle.ts @@ -0,0 +1,26 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export type DashboardReuseLifecycle = { + stopSandbox(sandboxName: string): { exitCode: number; message?: string }; + startSandbox(sandboxName: string): Promise<{ exitCode: number; message?: string }>; +}; + +let activeLifecycle: DashboardReuseLifecycle | undefined; + +export function getDashboardReuseLifecycle(): DashboardReuseLifecycle | undefined { + return activeLifecycle; +} + +export async function withDashboardReuseLifecycle( + lifecycle: DashboardReuseLifecycle | undefined, + operation: () => Promise, +): Promise { + const previous = activeLifecycle; + activeLifecycle = lifecycle; + try { + return await operation(); + } finally { + activeLifecycle = previous; + } +} diff --git a/src/lib/onboard/sandbox-create/orchestration.ts b/src/lib/onboard/sandbox-create/orchestration.ts index 3667a8844e3..1030b71d1e6 100644 --- a/src/lib/onboard/sandbox-create/orchestration.ts +++ b/src/lib/onboard/sandbox-create/orchestration.ts @@ -1222,13 +1222,6 @@ function shouldInspectExistingSandbox(input: { return input.liveExists && !input.portableLifecycle && !input.resumingVerifiedCreate; } -function openClawPersistedForwardMatcher( - agentName: string | null, - matcher: (port: number, chatUiUrl: string) => boolean, -): typeof matcher | undefined { - return agentName === "openclaw" ? matcher : undefined; -} - type PortableAgentReceiptGenerationObservation = | { readonly kind: "absent" | "openclaw" } | { @@ -1338,7 +1331,7 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche formatSandboxAgentName, formatSandboxBuildEstimateNote, getDashboardForwardPort, - matchesExistingDashboardForward, + reconcileOpenClawDashboardForwardReuse, readDcodeSelectionDrift, getDefaultSandboxNameForAgent, getDockerDriverGatewayStateDir, @@ -1497,9 +1490,6 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche agentForwardPort: dashboardRuntime.getAgentPrimaryForwardPort(agent, DASHBOARD_PORT), defaultPort: DASHBOARD_PORT, forwardListOutput: runCaptureOpenshell(["forward", "list"], { ignoreError: true }), - matchesPersistedForward: openClawPersistedForwardMatcher(requestedAgentName, (port, url) => - matchesExistingDashboardForward(sandboxName, port, url), - ), warn: (message: string) => console.warn(message), }); ({ effectivePort, chatUiUrl } = dashboardSelection); @@ -1724,6 +1714,7 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche gatewayPort: GATEWAY_PORT, manageDashboard, ensureDashboardForward, + reconcileOpenClawDashboardForwardReuse, hermesDashboardForwarding, updateReusedSandboxMetadata, releaseDashboardPort: dashboardPortReservationScope.release, diff --git a/src/lib/onboard/sandbox-reuse.test.ts b/src/lib/onboard/sandbox-reuse.test.ts index ed2f4480fd2..31b69a6e904 100644 --- a/src/lib/onboard/sandbox-reuse.test.ts +++ b/src/lib/onboard/sandbox-reuse.test.ts @@ -185,6 +185,7 @@ describe("applyReusedSandboxDashboardState", () => { }, gatewayName: "nemoclaw", gatewayPort: 8080, + getSandbox: () => null, releaseDashboardPort: vi.fn(async () => undefined), ensureDashboardForward, hermesDashboardForwarding: { @@ -204,6 +205,51 @@ describe("applyReusedSandboxDashboardState", () => { expect(updateSandbox).not.toHaveBeenCalled(); }); + it("restores the registered OpenClaw port after the pre-reuse allocator picked another port", async () => { + const releaseDashboardPort = vi.fn(async () => undefined); + const reconcileOpenClawDashboardForwardReuse = vi.fn(async () => undefined); + const ensureDashboardForward = vi.fn(() => 18_789); + + const result = await restoreReusedSandboxDashboardState({ + sandboxName: "reuse-me", + chatUiUrl: "http://127.0.0.1:18790", + env: {}, + agent: null, + model: "test-model", + provider: "openai-compatible", + selectionVerified: true, + sandboxGpuConfig: { + hostGpuDetected: false, + hostGpuPlatform: null, + sandboxGpuEnabled: false, + mode: "auto", + sandboxGpuDevice: null, + errors: [], + }, + gatewayName: "nemoclaw", + gatewayPort: 8080, + getSandbox: () => ({ dashboardPort: 18_789 }) as never, + releaseDashboardPort, + ensureDashboardForward, + reconcileOpenClawDashboardForwardReuse, + hermesDashboardForwarding: { + resolveStateForPort: vi.fn(() => ({ enabled: false, config: null })), + ensureForState: vi.fn(), + }, + updateSandbox: vi.fn(), + updateReusedSandboxMetadata: vi.fn(), + }); + + expect(releaseDashboardPort).toHaveBeenCalledOnce(); + expect(reconcileOpenClawDashboardForwardReuse).toHaveBeenCalledWith( + "reuse-me", + "http://127.0.0.1:18789", + undefined, + ); + expect(ensureDashboardForward).not.toHaveBeenCalled(); + expect(result.dashboardPort).toBe(18_789); + }); + it("rechecks after Hermes forwarding before reuse metadata (#9833)", () => { const revalidateSandboxIdentity = vi .fn<(operation: string) => void>() diff --git a/src/lib/onboard/sandbox-reuse.ts b/src/lib/onboard/sandbox-reuse.ts index e791688681b..1e5d31e21fd 100644 --- a/src/lib/onboard/sandbox-reuse.ts +++ b/src/lib/onboard/sandbox-reuse.ts @@ -100,6 +100,7 @@ export interface ReusedSandboxDashboardStateInput { gatewayName: string; gatewayPort: number; manageDashboard?: boolean; + preparedOpenClawDashboardPort?: number; getSandbox?(sandboxName: string): SandboxEntry | null; ensureDashboardForward( sandboxName: string, @@ -109,6 +110,11 @@ export interface ReusedSandboxDashboardStateInput { revalidateSandboxIdentity?: (operation: string) => void; }, ): number; + reconcileOpenClawDashboardForwardReuse?( + sandboxName: string, + chatUiUrl: string, + revalidateSandboxIdentity?: (operation: string) => void, + ): Promise; hermesDashboardForwarding: ReusedSandboxDashboardForwarding; updateSandbox?(sandboxName: string, updates: Partial): unknown; revalidateSandboxIdentity?(operation: string): void; @@ -147,12 +153,14 @@ export function applyReusedSandboxDashboardState( input.revalidateSandboxIdentity?.(`restore dashboard state for sandbox '${input.sandboxName}'`); const reuseExistingOpenClawForward = input.agent == null || input.agent.name === "openclaw"; const dashboardPort = manageDashboard - ? input.ensureDashboardForward(input.sandboxName, input.chatUiUrl, { - ...(reuseExistingOpenClawForward ? { reuseExistingOpenClawForward: true } : {}), - ...(input.revalidateSandboxIdentity - ? { revalidateSandboxIdentity: input.revalidateSandboxIdentity } - : {}), - }) + ? reuseExistingOpenClawForward && input.preparedOpenClawDashboardPort !== undefined + ? input.preparedOpenClawDashboardPort + : input.ensureDashboardForward(input.sandboxName, input.chatUiUrl, { + ...(reuseExistingOpenClawForward ? { reuseExistingOpenClawForward: true } : {}), + ...(input.revalidateSandboxIdentity + ? { revalidateSandboxIdentity: input.revalidateSandboxIdentity } + : {}), + }) : 0; const chatUiUrl = manageDashboard ? `http://127.0.0.1:${dashboardPort}` : input.chatUiUrl; if (manageDashboard) { @@ -199,7 +207,33 @@ export async function restoreReusedSandboxDashboardState( input: ReusedSandboxDashboardStateInput & { releaseDashboardPort(): Promise }, ): Promise { await input.releaseDashboardPort(); - return applyReusedSandboxDashboardState(input); + const reusesOpenClaw = input.agent == null || input.agent.name === "openclaw"; + const registeredPort = (input.getSandbox ?? registry.getSandbox)( + input.sandboxName, + )?.dashboardPort; + const preparedOpenClawDashboardPort = + reusesOpenClaw && + typeof registeredPort === "number" && + Number.isInteger(registeredPort) && + registeredPort > 0 && + registeredPort <= 65_535 + ? registeredPort + : undefined; + const chatUiUrl = preparedOpenClawDashboardPort + ? `http://127.0.0.1:${String(preparedOpenClawDashboardPort)}` + : input.chatUiUrl; + if ((input.manageDashboard ?? true) && reusesOpenClaw) { + await input.reconcileOpenClawDashboardForwardReuse?.( + input.sandboxName, + chatUiUrl, + input.revalidateSandboxIdentity, + ); + } + return applyReusedSandboxDashboardState({ + ...input, + chatUiUrl, + ...(preparedOpenClawDashboardPort ? { preparedOpenClawDashboardPort } : {}), + }); } export function createSandboxReuseHelpers(deps: SandboxReuseDeps): SandboxReuseHelpers { diff --git a/test/e2e/live/double-onboard.test.ts b/test/e2e/live/double-onboard.test.ts index e90538c514d..c7e8f55eb2f 100644 --- a/test/e2e/live/double-onboard.test.ts +++ b/test/e2e/live/double-onboard.test.ts @@ -32,7 +32,6 @@ const PHASE_TIMEOUT_MS = Number(process.env.NEMOCLAW_E2E_PHASE_TIMEOUT_MS ?? 1_2 const ONBOARD_TIMEOUT_MS = execTimeout(PHASE_TIMEOUT_MS); const PROBE_ATTEMPTS = Number(process.env.NEMOCLAW_E2E_PROBE_ATTEMPTS ?? 3); const PROBE_DELAY_MS = Number(process.env.NEMOCLAW_E2E_PROBE_DELAY_SECONDS ?? 3) * 1_000; -const PROBE_TIMEOUT_MS = Number(process.env.NEMOCLAW_E2E_PROBE_TIMEOUT_SECONDS ?? 180) * 1_000; const RECOVERY_PROBE_TIMEOUT_MS = Number(process.env.NEMOCLAW_E2E_RECOVERY_PROBE_TIMEOUT_SECONDS ?? 180) * 1_000; const TEST_TIMEOUT_MS = testTimeout(90 * 60_000); @@ -114,35 +113,38 @@ async function runOnboard( }); } -async function runProbeOnlyConnect( +async function waitForDashboardReachability( host: HostCliClient, - sandboxName: string, - artifactName: string, -): Promise { - return await host.command( - "bash", - [ - "-lc", + port: string, + expectedReachable: boolean, + artifactPrefix: string, +): Promise<{ reachable: boolean; output: string }> { + let reachable = false; + let output = ""; + for (let attempt = 1; attempt <= PROBE_ATTEMPTS; attempt += 1) { + const result = await host.command( + "curl", [ - "set +e", - 'log="$(mktemp)"', - '"$1" "$2" "$3" connect --probe-only >"$log" 2>&1', - "rc=$?", - 'cat "$log"', - 'rm -f "$log"', - 'exit "$rc"', - ].join("\n"), - "nemoclaw-probe-connect", - process.execPath, - CLI_ENTRYPOINT, - sandboxName, - ], - { - artifactName, - env: commandEnv(), - timeoutMs: PROBE_TIMEOUT_MS, - }, - ); + "--silent", + "--show-error", + "--output", + "/dev/null", + "--max-time", + "5", + `http://127.0.0.1:${port}/`, + ], + { + artifactName: `${artifactPrefix}-attempt-${attempt}`, + env: commandEnv(), + timeoutMs: 15_000, + }, + ); + output = resultText(result); + reachable = result.exitCode === 0 && !result.timedOut; + if (reachable === expectedReachable) break; + if (attempt < PROBE_ATTEMPTS) await sleep(PROBE_DELAY_MS); + } + return { reachable, output }; } async function cleanupDoubleOnboardState( @@ -234,44 +236,6 @@ function dashboardPortFromList(output: string, sandboxName: string): string | un return undefined; } -function forwardOwnerForPort(output: string, port: string): string | undefined { - for (const line of stripAnsi(output).split("\n")) { - const parts = line.trim().split(/\s+/); - if (parts.length < 5 || parts[0]?.toLowerCase() === "sandbox") continue; - const status = parts.slice(4).join(" ").toLowerCase(); - if (parts[2] === port && status.includes("running")) return parts[0]; - } - return undefined; -} - -async function waitForForwardOwner( - sandbox: SandboxClient, - port: string, - owner: string | undefined, - artifactPrefix: string, -): Promise<{ - owner: string | undefined; - output: string; - querySucceeded: boolean; -}> { - let observedOwner: string | undefined; - let lastOutput = ""; - let querySucceeded = false; - for (let attempt = 1; attempt <= PROBE_ATTEMPTS; attempt += 1) { - const result = await sandbox.openshell(["forward", "list"], { - artifactName: `${artifactPrefix}-attempt-${attempt}`, - env: commandEnv(), - timeoutMs: 30_000, - }); - lastOutput = resultText(result); - querySucceeded = result.exitCode === 0 && !result.timedOut; - observedOwner = querySucceeded ? forwardOwnerForPort(lastOutput, port) : undefined; - if (querySucceeded && observedOwner === owner) break; - if (attempt < PROBE_ATTEMPTS) await sleep(PROBE_DELAY_MS); - } - return { owner: observedOwner, output: lastOutput, querySucceeded }; -} - function hasOwn(object: object, key: string): boolean { return Object.prototype.hasOwnProperty.call(object, key); } @@ -588,6 +552,15 @@ test( }); expect(listAfterSecond.exitCode, resultText(listAfterSecond)).toBe(0); expect(stripAnsi(listAfterSecond.stdout)).toContain(SANDBOX_A); + const portAfterSecond = dashboardPortFromList(listAfterSecond.stdout, SANDBOX_A); + expect(portAfterSecond, resultText(listAfterSecond)).toBeTruthy(); + const dashboardAfterSecond = await waitForDashboardReachability( + host, + portAfterSecond ?? "", + true, + "phase-3-dashboard-after-second-onboard", + ); + expect(dashboardAfterSecond.reachable, dashboardAfterSecond.output).toBe(true); progress.phase("recreate same sandbox on existing gateway"); const gatewayBeforeRecreate = await gatewayRuntimeId(gateway); @@ -668,39 +641,20 @@ test( expect(portB, `nemoclaw list did not show ${SANDBOX_B} dashboard: ${list.stdout}`).toBeTruthy(); expect(portB).not.toBe(portA); - await sandbox.openshell(["forward", "stop", portB ?? ""], { - artifactName: "phase-4-stop-sandbox-b-dashboard-forward", - env: commandEnv(), - timeoutMs: 30_000, - }); - let probe: ShellProbeResult | undefined; - for (let attempt = 1; attempt <= PROBE_ATTEMPTS; attempt += 1) { - probe = await runProbeOnlyConnect( - host, - SANDBOX_B, - `phase-4-probe-connect-sandbox-b-attempt-${attempt}`, - ); - if (probe.exitCode === 0 && !probe.timedOut) break; - if (attempt < PROBE_ATTEMPTS) await sleep(PROBE_DELAY_MS); - } - expect(probe?.exitCode, probe ? resultText(probe) : "probe did not run").toBe(0); - expect(probe?.timedOut, probe ? resultText(probe) : "probe did not run").toBe(false); - - const restoredForwardB = await waitForForwardOwner( - sandbox, - portB ?? "", - SANDBOX_B, - "phase-4-openshell-forward-list-b", - ); - expect(restoredForwardB.owner, restoredForwardB.output).toBe(SANDBOX_B); - - const retainedForwardA = await waitForForwardOwner( - sandbox, + const dashboardABeforeStop = await waitForDashboardReachability( + host, portA ?? "", - SANDBOX_A, - "phase-4-openshell-forward-list-a", + true, + "phase-4-dashboard-a-before-stop", ); - expect(retainedForwardA.owner, retainedForwardA.output).toBe(SANDBOX_A); + expect(dashboardABeforeStop.reachable, dashboardABeforeStop.output).toBe(true); + const dashboardBBeforeStop = await waitForDashboardReachability( + host, + portB ?? "", + true, + "phase-4-dashboard-b-before-stop", + ); + expect(dashboardBBeforeStop.reachable, dashboardBBeforeStop.output).toBe(true); progress.phase("stop sibling sandbox without disturbing the first forward"); const stopB = await command(host, [SANDBOX_B, "stop"], { @@ -710,14 +664,13 @@ test( }); expect(stopB.exitCode, resultText(stopB)).toBe(0); - const releasedForwardB = await waitForForwardOwner( - sandbox, + const releasedForwardB = await waitForDashboardReachability( + host, portB ?? "", - undefined, - "phase-4-openshell-forward-list-b-after-stop", + false, + "phase-4-dashboard-b-after-stop", ); - expect(releasedForwardB.querySucceeded, releasedForwardB.output).toBe(true); - expect(releasedForwardB.owner, releasedForwardB.output).toBeUndefined(); + expect(releasedForwardB.reachable, releasedForwardB.output).toBe(false); const stoppedStatusB = await command(host, [SANDBOX_B, "status"], { artifactName: "phase-4-nemoclaw-status-sandbox-b-after-stop", @@ -729,13 +682,13 @@ test( expect(stoppedStatusTextB).toContain("sandbox_container_stopped"); expect(stoppedStatusTextB).not.toContain("sandbox_dashboard_port_conflict"); - const retainedForwardAAfterStop = await waitForForwardOwner( - sandbox, + const retainedForwardAAfterStop = await waitForDashboardReachability( + host, portA ?? "", - SANDBOX_A, - "phase-4-openshell-forward-list-a-after-b-stop", + true, + "phase-4-dashboard-a-after-b-stop", ); - expect(retainedForwardAAfterStop.owner, retainedForwardAAfterStop.output).toBe(SANDBOX_A); + expect(retainedForwardAAfterStop.reachable, retainedForwardAAfterStop.output).toBe(true); const startB = await command(host, [SANDBOX_B, "start"], { artifactName: "phase-4-nemoclaw-start-sandbox-b", @@ -743,13 +696,13 @@ test( timeoutMs: PHASE_TIMEOUT_MS, }); expect(startB.exitCode, resultText(startB)).toBe(0); - const restoredForwardBAfterStart = await waitForForwardOwner( - sandbox, + const restoredForwardBAfterStart = await waitForDashboardReachability( + host, portB ?? "", - SANDBOX_B, - "phase-4-openshell-forward-list-b-after-start", + true, + "phase-4-dashboard-b-after-start", ); - expect(restoredForwardBAfterStart.owner, restoredForwardBAfterStart.output).toBe(SANDBOX_B); + expect(restoredForwardBAfterStart.reachable, restoredForwardBAfterStart.output).toBe(true); progress.phase("replace sandbox after stale registry refusal"); // Phase 5: direct OpenShell deletion leaves a stale registry entry that @@ -888,19 +841,19 @@ test( gatewayStatusReportedServerEndpoint: Boolean(gatewayServerEndpoint), secondOnboardReusedGateway: gatewayAfterSecond === gatewayBeforeSecond && - secondText.includes("Reusing healthy NemoClaw gateway."), + secondText.includes("Reusing healthy NemoClaw gateway.") && + dashboardAfterSecond.reachable, thirdOnboardPreservedSibling: sandboxAAfterThird.exitCode === 0 && sandboxBAfterThird.exitCode === 0, distinctDashboardPorts: Boolean(portA && portB && portA !== portB), selectedStopReleasedOnlySelectedForward: stopB.exitCode === 0 && - releasedForwardB.querySucceeded && - releasedForwardB.owner === undefined && - retainedForwardAAfterStop.owner === SANDBOX_A && + !releasedForwardB.reachable && + retainedForwardAAfterStop.reachable && stoppedStatusTextB.includes("sandbox_container_stopped") && !stoppedStatusTextB.includes("sandbox_dashboard_port_conflict") && startB.exitCode === 0 && - restoredForwardBAfterStart.owner === SANDBOX_B, + restoredForwardBAfterStart.reachable, staleRegistryRecovered: rebuild.exitCode === 0, gatewayStopGuidance: /Recovered NemoClaw gateway runtime|gateway is no longer configured after restart\/rebuild|gateway is still refusing connections after restart|gateway trust material rotated after restart/.test( diff --git a/test/onboarding/onboard-finalization-dashboard-forward.test.ts b/test/onboarding/onboard-finalization-dashboard-forward.test.ts index e56380af6ef..5164e83cd44 100644 --- a/test/onboarding/onboard-finalization-dashboard-forward.test.ts +++ b/test/onboarding/onboard-finalization-dashboard-forward.test.ts @@ -3,17 +3,18 @@ import { describe, expect, it, vi } from "vitest"; -import type { ForwardServiceTarget } from "../../src/lib/adapters/openshell/forward-service"; import { createOnboardDashboardHelpers } from "../../src/lib/onboard/dashboard"; import type { ListSandboxesFn } from "../../src/lib/onboard/dashboard-port"; function harness(options: { listSandboxes: ListSandboxesFn; isPortBound?: (port: number) => boolean; - matchesListener?: (target: ForwardServiceTarget) => boolean; + stopSandbox?: (sandboxName: string) => { exitCode: number; message?: string }; + startSandbox?: (sandboxName: string) => Promise<{ exitCode: number; message?: string }>; }) { const launch = vi.fn(); - const matchesListener = vi.fn(options.matchesListener ?? (() => false)); + const stopSandbox = vi.fn(options.stopSandbox ?? (() => ({ exitCode: 0 }))); + const startSandbox = vi.fn(options.startSandbox ?? (async () => ({ exitCode: 0 }))); const helpers = createOnboardDashboardHelpers({ runOpenshell: vi.fn(() => ({ status: 0 })), runCaptureOpenshell: vi.fn(() => ""), @@ -28,19 +29,20 @@ function harness(options: { printAgentDashboardUi: vi.fn(), listSandboxes: options.listSandboxes, isPortBoundOnHost: options.isPortBound ?? (() => false), + stopSandboxForDashboardReuse: stopSandbox, + startSandboxForDashboardReuse: startSandbox, forwardService: { executable: () => "/usr/local/bin/openshell", launch, - matchesListener, resolveGatewayName: () => "nemoclaw", retireLegacy: vi.fn(() => 0), }, }); - return { helpers, launch, matchesListener }; + return { helpers, launch, startSandbox, stopSandbox }; } describe("finalization dashboard ForwardTcp launch", () => { - it("launches the persisted dashboard port and publishes its URL", () => { + it("launches the persisted dashboard port and publishes its URL", async () => { vi.stubEnv("CHAT_UI_URL", undefined); const { helpers, launch } = harness({ listSandboxes: () => ({ @@ -48,7 +50,9 @@ describe("finalization dashboard ForwardTcp launch", () => { }), }); - expect(helpers.ensureFinalizationDashboardForward("reonboard-test")).toBe(18_790); + await expect(helpers.ensureFinalizationDashboardForward("reonboard-test")).resolves.toBe( + 18_790, + ); expect(launch).toHaveBeenCalledWith( expect.objectContaining({ gatewayName: "nemoclaw", @@ -60,47 +64,70 @@ describe("finalization dashboard ForwardTcp launch", () => { expect(process.env.CHAT_UI_URL).toBe("http://127.0.0.1:18790"); }); - it("fails closed when a foreign listener occupies the persisted port", () => { + it("fails closed when a foreign listener occupies the persisted port", async () => { vi.stubEnv("CHAT_UI_URL", undefined); - const { helpers, launch } = harness({ + const { helpers, launch, startSandbox, stopSandbox } = harness({ listSandboxes: () => ({ sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], }), isPortBound: (port) => port === 18_790, }); - expect(() => helpers.ensureFinalizationDashboardForward("reonboard-test")).toThrow( + await expect(helpers.ensureFinalizationDashboardForward("reonboard-test")).rejects.toThrow( /cannot be reallocated/u, ); expect(launch).not.toHaveBeenCalled(); + expect(stopSandbox).not.toHaveBeenCalled(); + expect(startSandbox).not.toHaveBeenCalled(); }); - it("retains the exact OpenClaw forward service on repeated onboarding (#11074)", () => { + it("restarts the reused sandbox and retains its registered dashboard port (#11074)", async () => { vi.stubEnv("CHAT_UI_URL", undefined); - const { helpers, launch, matchesListener } = harness({ + let bound = true; + const { helpers, launch, startSandbox, stopSandbox } = harness({ listSandboxes: () => ({ sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], }), - isPortBound: (port) => port === 18_790, - matchesListener: () => true, + isPortBound: (port) => port === 18_790 && bound, + stopSandbox: () => { + bound = false; + return { exitCode: 0 }; + }, + startSandbox: async () => { + bound = true; + return { exitCode: 0 }; + }, }); - expect(helpers.ensureFinalizationDashboardForward("reonboard-test")).toBe(18_790); - expect(matchesListener).toHaveBeenCalledWith( - expect.objectContaining({ - gatewayName: "nemoclaw", - sandboxName: "reonboard-test", - localPort: 18_790, - targetPort: 18_790, - }), - ); + await expect( + helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true), + ).resolves.toBe(18_790); + expect(stopSandbox).toHaveBeenCalledWith("reonboard-test"); + expect(startSandbox).toHaveBeenCalledWith("reonboard-test"); expect(launch).not.toHaveBeenCalled(); expect(process.env.CHAT_UI_URL).toBe("http://127.0.0.1:18790"); }); - it("does not retain a forward when another sandbox registers the same port", () => { + it("rejects an ambiguous listener that remains after the reused sandbox stops", async () => { + vi.stubEnv("CHAT_UI_URL", undefined); + const { helpers, launch, startSandbox, stopSandbox } = harness({ + listSandboxes: () => ({ + sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], + }), + isPortBound: (port) => port === 18_790, + }); + + await expect( + helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true), + ).rejects.toThrow(/remained occupied.*cannot be adopted/u); + expect(stopSandbox).toHaveBeenCalledWith("reonboard-test"); + expect(startSandbox).not.toHaveBeenCalled(); + expect(launch).not.toHaveBeenCalled(); + }); + + it("does not reuse a forward when another sandbox registers the same port", async () => { vi.stubEnv("CHAT_UI_URL", undefined); - const { helpers, matchesListener } = harness({ + const { helpers, launch, stopSandbox } = harness({ listSandboxes: () => ({ sandboxes: [ { name: "reonboard-test", dashboardPort: 18_790 }, @@ -108,30 +135,41 @@ describe("finalization dashboard ForwardTcp launch", () => { ], }), isPortBound: (port) => port === 18_790, - matchesListener: () => true, }); - expect(() => helpers.ensureFinalizationDashboardForward("reonboard-test")).toThrow( - /cannot be reallocated/u, - ); - expect(matchesListener).not.toHaveBeenCalled(); + await expect( + helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true), + ).rejects.toThrow(/cannot be reallocated/u); + expect(stopSandbox).not.toHaveBeenCalled(); + expect(launch).not.toHaveBeenCalled(); }); - it("enables retained-forward matching only for OpenClaw agents", async () => { + it("enables lifecycle reconciliation only for OpenClaw agents", async () => { vi.stubEnv("CHAT_UI_URL", undefined); + let bound = true; const openClaw = harness({ listSandboxes: () => ({ sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], }), - isPortBound: (port) => port === 18_790, - matchesListener: () => true, + isPortBound: (port) => port === 18_790 && bound, + stopSandbox: () => { + bound = false; + return { exitCode: 0 }; + }, + startSandbox: async () => { + bound = true; + return { exitCode: 0 }; + }, }); await expect( - openClaw.helpers.ensureFinalizationAgentDashboardForward("reonboard-test", { - name: "openclaw", - forwardPort: 18_790, - }), + openClaw.helpers.ensureFinalizationAgentDashboardForward( + "reonboard-test", + { name: "openclaw", forwardPort: 18_790 }, + undefined, + undefined, + true, + ), ).resolves.toBe(18_790); const hermes = harness({ @@ -139,25 +177,29 @@ describe("finalization dashboard ForwardTcp launch", () => { sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], }), isPortBound: (port) => port === 18_790, - matchesListener: () => true, }); await expect( - hermes.helpers.ensureFinalizationAgentDashboardForward("reonboard-test", { - name: "hermes", - forwardPort: 18_790, - }), + hermes.helpers.ensureFinalizationAgentDashboardForward( + "reonboard-test", + { name: "hermes", forwardPort: 18_790 }, + undefined, + undefined, + true, + ), ).rejects.toThrow(/cannot be reallocated/u); - expect(hermes.matchesListener).not.toHaveBeenCalled(); + expect(hermes.stopSandbox).not.toHaveBeenCalled(); }); - it("honors an explicit dashboard URL", () => { + it("honors an explicit dashboard URL", async () => { vi.stubEnv("CHAT_UI_URL", "http://127.0.0.1:19001"); const { helpers, launch } = harness({ listSandboxes: () => ({ sandboxes: [] }), }); - expect(helpers.ensureFinalizationDashboardForward("reonboard-test")).toBe(19_001); + await expect(helpers.ensureFinalizationDashboardForward("reonboard-test")).resolves.toBe( + 19_001, + ); expect(launch).toHaveBeenCalledWith(expect.objectContaining({ localPort: 19_001 })); }); }); From 485e677f620e5ac8179a95d2ebda2ee6d9d1f967 Mon Sep 17 00:00:00 2001 From: San Dang Date: Mon, 7 Sep 2026 02:28:53 +0700 Subject: [PATCH 03/20] test(e2e): map double onboard lifecycle parity Signed-off-by: San Dang --- test/e2e/mock-parity.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/e2e/mock-parity.json b/test/e2e/mock-parity.json index d3bd307f025..9a7ffe511f5 100644 --- a/test/e2e/mock-parity.json +++ b/test/e2e/mock-parity.json @@ -474,7 +474,8 @@ "src/lib/onboard/machine/handlers/sandbox-route-publication.test.ts", "src/lib/onboard/sandbox-registration.test.ts", "test/e2e/support/e2e-cleanup-resources.test.ts", - "test/e2e/support/e2e-clients.test.ts" + "test/e2e/support/e2e-clients.test.ts", + "test/onboarding/onboard-finalization-dashboard-forward.test.ts" ] }, { From 677bb4062fe69a6cdad32ed10f9b8d3045631d9c Mon Sep 17 00:00:00 2001 From: San Dang Date: Mon, 7 Sep 2026 03:02:42 +0700 Subject: [PATCH 04/20] fix(onboard): harden dashboard reuse lifecycle --- src/lib/onboard/dashboard.ts | 103 +++++++++++++++--- .../onboard/dashboard/reuse-lifecycle.test.ts | 20 +++- src/lib/onboard/dashboard/reuse-lifecycle.ts | 18 ++- src/lib/onboard/sandbox-reuse.test.ts | 8 +- src/lib/onboard/sandbox-reuse.ts | 27 +++-- test/e2e/live/onboard-resume.test.ts | 73 +++++++++++-- ...ard-finalization-dashboard-forward.test.ts | 36 +++++- 7 files changed, 227 insertions(+), 58 deletions(-) diff --git a/src/lib/onboard/dashboard.ts b/src/lib/onboard/dashboard.ts index 333875239c7..be3acdd880e 100644 --- a/src/lib/onboard/dashboard.ts +++ b/src/lib/onboard/dashboard.ts @@ -33,6 +33,7 @@ import { isPortBoundOnHost, type ListSandboxesFn, } from "./dashboard-port"; +import { fingerprintSandboxLiveIdentity } from "./sandbox-recreate-transaction"; import { ensureMessagingHostForwardForSandbox, productionForwardServiceRegistryContext, @@ -80,6 +81,7 @@ export interface OnboardDashboardDeps { dashboardPort?: number | null; hermesApiPort?: number | null; hermesDashboardPort?: number | null; + lifecycleGeneration?: string; lifecycleLiveIdentityFingerprint?: string; pendingRouteReservation?: true; } @@ -162,7 +164,7 @@ export interface OnboardDashboardHelpers { sandboxName: string, chatUiUrl: string, revalidateSandboxIdentity?: (operation: string) => void, - ): Promise; + ): Promise; ensureAgentFixedForward( sandboxName: string, port: number, @@ -391,26 +393,69 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa sandboxName: string, chatUiUrl: string, revalidateSandboxIdentity?: (operation: string) => void, - ): Promise { + ): Promise { const port = Number(getDashboardForwardPort(chatUiUrl)); const isPortBound = deps.isPortBoundOnHost ?? isPortBoundOnHost; - if (reconciledOpenClawForwards.get(sandboxName) === port && isPortBound(port)) return; - if (!isPortBound(port)) return; + if (reconciledOpenClawForwards.get(sandboxName) === port && isPortBound(port)) return true; + if (!isPortBound(port)) return false; if (getRegistryOccupiedDashboardPorts(sandboxName, listSandboxes).has(String(port))) { throw new Error( `Registered dashboard port ${String(port)} is already occupied; it cannot be reallocated or adopted.`, ); } + const lifecycle = getDashboardReuseLifecycle(); + const stopSandbox = deps.stopSandboxForDashboardReuse ?? lifecycle?.stopSandbox; + const startSandbox = deps.startSandboxForDashboardReuse ?? lifecycle?.startSandbox; + if (!stopSandbox || !startSandbox) { + throw new Error( + `Could not restart sandbox '${sandboxName}' to reconcile dashboard port ${String(port)}: sandbox lifecycle is unavailable.`, + ); + } + + const readLiveIdentity = (gatewayName: string): string | null => + fingerprintSandboxLiveIdentity( + deps.runCaptureOpenshell(["sandbox", "get", "-g", gatewayName, sandboxName], { + ignoreError: true, + includeStderr: true, + }) ?? "", + ); + const registered = getSandbox?.(sandboxName); + const gatewayName = registered ? forwardService?.resolveGatewayName(registered) : null; + const observedIdentity = gatewayName ? readLiveIdentity(gatewayName) : null; + if ( + !registered || + !gatewayName || + !observedIdentity || + (registered.lifecycleLiveIdentityFingerprint && + registered.lifecycleLiveIdentityFingerprint !== observedIdentity) + ) { + throw new Error( + `Could not verify sandbox '${sandboxName}' before reconciling dashboard port ${String(port)}.`, + ); + } + const assertSameSandbox = (operation: string): void => { + const current = getSandbox?.(sandboxName); + const currentGateway = current ? forwardService?.resolveGatewayName(current) : null; + const currentIdentity = currentGateway ? readLiveIdentity(currentGateway) : null; + if ( + !current || + currentGateway !== gatewayName || + current.lifecycleGeneration !== registered.lifecycleGeneration || + current.lifecycleLiveIdentityFingerprint !== registered.lifecycleLiveIdentityFingerprint || + currentIdentity !== observedIdentity + ) { + throw new Error( + `Refusing to ${operation}: sandbox '${sandboxName}' identity changed during dashboard reconciliation.`, + ); + } + }; + revalidateSandboxIdentity?.( `restart sandbox '${sandboxName}' to reconcile dashboard forward ${String(port)}`, ); - const stopped = ( - deps.stopSandboxForDashboardReuse ?? getDashboardReuseLifecycle()?.stopSandbox - )?.(sandboxName) ?? { - exitCode: 1, - message: "sandbox lifecycle is unavailable", - }; + assertSameSandbox(`stop sandbox '${sandboxName}'`); + const stopped = stopSandbox(sandboxName); if (stopped.exitCode !== 0) { throw new Error( `Could not stop sandbox '${sandboxName}' to reconcile dashboard port ${String(port)}${ @@ -418,28 +463,47 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa }`, ); } - if (isPortBound(port)) { + try { + revalidateSandboxIdentity?.( + `start sandbox '${sandboxName}' to reconcile dashboard forward ${String(port)}`, + ); + assertSameSandbox(`start sandbox '${sandboxName}'`); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); throw new Error( - `Registered dashboard port ${String(port)} remained occupied after sandbox '${sandboxName}' stopped; it cannot be adopted.`, + `${detail} The selected sandbox was stopped; verify its identity, then run '${deps.cliName()} ${sandboxName} start'.`, ); } - const started = await ( - deps.startSandboxForDashboardReuse ?? getDashboardReuseLifecycle()?.startSandbox - )?.(sandboxName); - if (!started) { + const portRemainedBound = isPortBound(port); + let started: { exitCode: number; message?: string }; + try { + started = await startSandbox(sandboxName); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + throw new Error( + `Could not restart sandbox '${sandboxName}' after releasing dashboard port ${String(port)}: ${detail}. The sandbox may remain stopped; run '${deps.cliName()} ${sandboxName} start' before retrying onboarding.`, + ); + } + if (portRemainedBound) { + if (started.exitCode !== 0) { + throw new Error( + `Registered dashboard port ${String(port)} remained occupied after sandbox '${sandboxName}' stopped, and the sandbox could not restart${started.message ? `: ${started.message}` : "."} Run '${deps.cliName()} ${sandboxName} start' after resolving the listener conflict.`, + ); + } throw new Error( - `Could not start sandbox '${sandboxName}' to reconcile dashboard port ${String(port)}: sandbox lifecycle is unavailable.`, + `Registered dashboard port ${String(port)} remained occupied after sandbox '${sandboxName}' stopped; it cannot be adopted.`, ); } if (started.exitCode !== 0 || !isPortBound(port)) { throw new Error( `Sandbox '${sandboxName}' did not restore dashboard port ${String(port)} after restart${ started.message ? `: ${started.message}` : "." - }`, + } The sandbox may remain stopped; run '${deps.cliName()} ${sandboxName} start' before retrying onboarding.`, ); } reconciledOpenClawForwards.set(sandboxName, port); + return true; } function ensureDashboardForward( @@ -574,6 +638,9 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa }, }); } + if (fwdOk && reuseExistingOpenClawForward) { + reconciledOpenClawForwards.set(sandboxName, actualPort); + } return actualPort; } diff --git a/src/lib/onboard/dashboard/reuse-lifecycle.test.ts b/src/lib/onboard/dashboard/reuse-lifecycle.test.ts index ebc7eab109d..3e76a7fb5a3 100644 --- a/src/lib/onboard/dashboard/reuse-lifecycle.test.ts +++ b/src/lib/onboard/dashboard/reuse-lifecycle.test.ts @@ -5,12 +5,24 @@ import { expect, it, vi } from "vitest"; import { getDashboardReuseLifecycle, withDashboardReuseLifecycle } from "./reuse-lifecycle"; -it("exposes dashboard reuse lifecycle only while onboarding runs", async () => { - const lifecycle = { startSandbox: vi.fn(), stopSandbox: vi.fn() }; +it("keeps overlapping onboarding lifecycle scopes independent", async () => { + const first = { startSandbox: vi.fn(), stopSandbox: vi.fn() }; + const second = { startSandbox: vi.fn(), stopSandbox: vi.fn() }; + let releaseFirst!: () => void; + const firstPaused = new Promise((resolve) => { + releaseFirst = resolve; + }); - await withDashboardReuseLifecycle(lifecycle, async () => { - expect(getDashboardReuseLifecycle()).toBe(lifecycle); + const firstOperation = withDashboardReuseLifecycle(first, async () => { + expect(getDashboardReuseLifecycle()).toBe(first); + await firstPaused; + expect(getDashboardReuseLifecycle()).toBe(first); + }); + await withDashboardReuseLifecycle(second, async () => { + expect(getDashboardReuseLifecycle()).toBe(second); }); + releaseFirst(); + await firstOperation; expect(getDashboardReuseLifecycle()).toBeUndefined(); }); diff --git a/src/lib/onboard/dashboard/reuse-lifecycle.ts b/src/lib/onboard/dashboard/reuse-lifecycle.ts index 18ec3c53842..ae6bbbbd461 100644 --- a/src/lib/onboard/dashboard/reuse-lifecycle.ts +++ b/src/lib/onboard/dashboard/reuse-lifecycle.ts @@ -1,26 +1,22 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { AsyncLocalStorage } from "node:async_hooks"; + export type DashboardReuseLifecycle = { stopSandbox(sandboxName: string): { exitCode: number; message?: string }; startSandbox(sandboxName: string): Promise<{ exitCode: number; message?: string }>; }; -let activeLifecycle: DashboardReuseLifecycle | undefined; +const lifecycleStorage = new AsyncLocalStorage(); export function getDashboardReuseLifecycle(): DashboardReuseLifecycle | undefined { - return activeLifecycle; + return lifecycleStorage.getStore(); } -export async function withDashboardReuseLifecycle( - lifecycle: DashboardReuseLifecycle | undefined, +export function withDashboardReuseLifecycle( + lifecycle: DashboardReuseLifecycle, operation: () => Promise, ): Promise { - const previous = activeLifecycle; - activeLifecycle = lifecycle; - try { - return await operation(); - } finally { - activeLifecycle = previous; - } + return lifecycleStorage.run(lifecycle, operation); } diff --git a/src/lib/onboard/sandbox-reuse.test.ts b/src/lib/onboard/sandbox-reuse.test.ts index 31b69a6e904..743c6e4a37f 100644 --- a/src/lib/onboard/sandbox-reuse.test.ts +++ b/src/lib/onboard/sandbox-reuse.test.ts @@ -205,9 +205,9 @@ describe("applyReusedSandboxDashboardState", () => { expect(updateSandbox).not.toHaveBeenCalled(); }); - it("restores the registered OpenClaw port after the pre-reuse allocator picked another port", async () => { + it("launches the registered OpenClaw port when reuse finds no listener", async () => { const releaseDashboardPort = vi.fn(async () => undefined); - const reconcileOpenClawDashboardForwardReuse = vi.fn(async () => undefined); + const reconcileOpenClawDashboardForwardReuse = vi.fn(async () => false); const ensureDashboardForward = vi.fn(() => 18_789); const result = await restoreReusedSandboxDashboardState({ @@ -246,7 +246,9 @@ describe("applyReusedSandboxDashboardState", () => { "http://127.0.0.1:18789", undefined, ); - expect(ensureDashboardForward).not.toHaveBeenCalled(); + expect(ensureDashboardForward).toHaveBeenCalledWith("reuse-me", "http://127.0.0.1:18789", { + reuseExistingOpenClawForward: true, + }); expect(result.dashboardPort).toBe(18_789); }); diff --git a/src/lib/onboard/sandbox-reuse.ts b/src/lib/onboard/sandbox-reuse.ts index 1e5d31e21fd..9616b9d4d81 100644 --- a/src/lib/onboard/sandbox-reuse.ts +++ b/src/lib/onboard/sandbox-reuse.ts @@ -114,7 +114,7 @@ export interface ReusedSandboxDashboardStateInput { sandboxName: string, chatUiUrl: string, revalidateSandboxIdentity?: (operation: string) => void, - ): Promise; + ): Promise; hermesDashboardForwarding: ReusedSandboxDashboardForwarding; updateSandbox?(sandboxName: string, updates: Partial): unknown; revalidateSandboxIdentity?(operation: string): void; @@ -211,7 +211,7 @@ export async function restoreReusedSandboxDashboardState( const registeredPort = (input.getSandbox ?? registry.getSandbox)( input.sandboxName, )?.dashboardPort; - const preparedOpenClawDashboardPort = + const registeredOpenClawDashboardPort = reusesOpenClaw && typeof registeredPort === "number" && Number.isInteger(registeredPort) && @@ -219,20 +219,23 @@ export async function restoreReusedSandboxDashboardState( registeredPort <= 65_535 ? registeredPort : undefined; - const chatUiUrl = preparedOpenClawDashboardPort - ? `http://127.0.0.1:${String(preparedOpenClawDashboardPort)}` + const chatUiUrl = registeredOpenClawDashboardPort + ? `http://127.0.0.1:${String(registeredOpenClawDashboardPort)}` : input.chatUiUrl; - if ((input.manageDashboard ?? true) && reusesOpenClaw) { - await input.reconcileOpenClawDashboardForwardReuse?.( - input.sandboxName, - chatUiUrl, - input.revalidateSandboxIdentity, - ); - } + const reconciled = + (input.manageDashboard ?? true) && reusesOpenClaw + ? await input.reconcileOpenClawDashboardForwardReuse?.( + input.sandboxName, + chatUiUrl, + input.revalidateSandboxIdentity, + ) + : false; return applyReusedSandboxDashboardState({ ...input, chatUiUrl, - ...(preparedOpenClawDashboardPort ? { preparedOpenClawDashboardPort } : {}), + ...(reconciled && registeredOpenClawDashboardPort + ? { preparedOpenClawDashboardPort: registeredOpenClawDashboardPort } + : {}), }); } diff --git a/test/e2e/live/onboard-resume.test.ts b/test/e2e/live/onboard-resume.test.ts index fe94c8b644e..49143831fd4 100644 --- a/test/e2e/live/onboard-resume.test.ts +++ b/test/e2e/live/onboard-resume.test.ts @@ -10,6 +10,7 @@ import { ONBOARD_NO_RECREATE_COMMAND_TIMEOUT_MS, ONBOARD_RESUME_TEST_TIMEOUT_MS, } from "../../../tools/e2e/onboard-timeout-contract.mts"; +import { parseOpenShellSandboxId } from "../../../src/lib/adapters/openshell/sandbox-identity.ts"; import { parseSandboxPhase } from "../../../src/lib/state/gateway.ts"; import { execTimeout, testTimeout } from "../../helpers/timeouts.ts"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; @@ -107,6 +108,29 @@ function markSessionInProgress(file: string): void { fs.writeFileSync(file, JSON.stringify(session, null, 2), "utf8"); } +function registeredDashboardPort(): number { + const registry = JSON.parse(fs.readFileSync(REGISTRY_FILE, "utf8")) as { + sandboxes?: Record; + }; + return Number(registry.sandboxes?.[SANDBOX_NAME]?.dashboardPort); +} + +function requestDashboard(host: HostCliClient, port: number, artifactName: string) { + return host.command( + "curl", + [ + "--silent", + "--show-error", + "--output", + "/dev/null", + "--max-time", + "10", + `http://127.0.0.1:${String(port)}/`, + ], + { artifactName, env: buildAvailabilityProbeEnv(), timeoutMs: 20_000 }, + ); +} + function interruptedSessionSummary(session: SessionStateInterrupted): Record { return { status: session.status, @@ -192,6 +216,7 @@ test( "resume proves recreated sandbox provider attachments are selectively reconciled", "host trust-store anchor corporate CA source is baked and merged after resume", "an unreachable committed route pauses at final verification and completes after repair", + "resume reuses the same sandbox and registered dashboard port after restoring its forward", "implicit resume is detected and --fresh suppresses that auto-resume", ], }); @@ -597,7 +622,6 @@ test( expect(unavailableResumeText).not.toContain( `Deleting and recreating sandbox '${SANDBOX_NAME}'`, ); - expect(unavailableResumeText).not.toContain(`Sandbox '${SANDBOX_NAME}' created`); const paused = readSession(SESSION_FILE); await artifacts.writeJson("phase-3-5-session-route-unavailable.json", { @@ -619,7 +643,24 @@ test( requireAuth: true, requireAuthModels: true, }); - expect(fake.baseUrl).toBe(`http://${fakePublicHost}:${String(fakePort)}/v1`); + const sandboxBeforeRepairedResume = await sandbox.openshell(["sandbox", "get", SANDBOX_NAME], { + artifactName: "phase-3-5-sandbox-before-repaired-resume", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + const sandboxIdBeforeRepairedResume = parseOpenShellSandboxId( + resultText(sandboxBeforeRepairedResume), + ); + expect(sandboxIdBeforeRepairedResume).not.toBeNull(); + const dashboardPortBeforeRepairedResume = registeredDashboardPort(); + const dashboardBeforeRepairedResume = await requestDashboard( + host, + dashboardPortBeforeRepairedResume, + "phase-3-5-dashboard-before-repaired-resume", + ); + expect(dashboardBeforeRepairedResume.exitCode, resultText(dashboardBeforeRepairedResume)).toBe( + 0, + ); const repairedResumeRun = await host.command( "node", @@ -633,11 +674,23 @@ test( ); const repairedResumeText = `${repairedResumeRun.stdout}\n${repairedResumeRun.stderr}`; expect(repairedResumeRun.exitCode, repairedResumeText).toBe(0); - expect(repairedResumeText).toContain("is ready"); + expect(repairedResumeText).not.toContain("Registered dashboard port"); expect(repairedResumeText).not.toContain(`Deleting and recreating sandbox '${SANDBOX_NAME}'`); - expect(repairedResumeText).not.toContain(`Sandbox '${SANDBOX_NAME}' created`); - const repaired = readSession(SESSION_FILE); - expect(repaired.status).toBe("complete"); + const sandboxAfterRepairedResume = await sandbox.openshell(["sandbox", "get", SANDBOX_NAME], { + artifactName: "phase-3-5-sandbox-after-repaired-resume", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + expect(parseOpenShellSandboxId(resultText(sandboxAfterRepairedResume))).toBe( + sandboxIdBeforeRepairedResume, + ); + expect(registeredDashboardPort()).toBe(dashboardPortBeforeRepairedResume); + const dashboardAfterRepairedResume = await requestDashboard( + host, + dashboardPortBeforeRepairedResume, + "phase-3-5-dashboard-after-repaired-resume", + ); + expect(dashboardAfterRepairedResume.exitCode, resultText(dashboardAfterRepairedResume)).toBe(0); // ────────────────────────────────────────────────────────────────── // Phase 4: implicit resume — a plain `onboard` auto-detects an @@ -669,7 +722,6 @@ test( implicitResumeText, ).toBe(true); expect(implicitResumeText).not.toContain(`Deleting and recreating sandbox '${SANDBOX_NAME}'`); - expect(implicitResumeText).not.toContain(`Sandbox '${SANDBOX_NAME}' created`); markSessionInProgress(SESSION_FILE); const freshRun = await host.command( @@ -694,6 +746,11 @@ test( expect(freshText).toContain("[e2e] Forced onboarding failure at step 'preflight'."); expect(freshText).not.toContain("(resume mode)"); expect(freshText).not.toContain(`Sandbox '${SANDBOX_NAME}' created`); - await artifacts.target.complete({ id: "onboard-resume", status: "passed" }); + await artifacts.target.complete({ + id: "onboard-resume", + status: "passed", + resumeDashboardPort: dashboardPortBeforeRepairedResume, + resumeSandboxIdentityRetained: true, + }); }, ); diff --git a/test/onboarding/onboard-finalization-dashboard-forward.test.ts b/test/onboarding/onboard-finalization-dashboard-forward.test.ts index 5164e83cd44..20145e46bd3 100644 --- a/test/onboarding/onboard-finalization-dashboard-forward.test.ts +++ b/test/onboarding/onboard-finalization-dashboard-forward.test.ts @@ -9,6 +9,7 @@ import type { ListSandboxesFn } from "../../src/lib/onboard/dashboard-port"; function harness(options: { listSandboxes: ListSandboxesFn; isPortBound?: (port: number) => boolean; + sandboxIdentity?: () => string; stopSandbox?: (sandboxName: string) => { exitCode: number; message?: string }; startSandbox?: (sandboxName: string) => Promise<{ exitCode: number; message?: string }>; }) { @@ -17,7 +18,11 @@ function harness(options: { const startSandbox = vi.fn(options.startSandbox ?? (async () => ({ exitCode: 0 }))); const helpers = createOnboardDashboardHelpers({ runOpenshell: vi.fn(() => ({ status: 0 })), - runCaptureOpenshell: vi.fn(() => ""), + runCaptureOpenshell: vi.fn((args) => + args[0] === "sandbox" + ? `Name: reonboard-test\nId: ${options.sandboxIdentity?.() ?? "sandbox-id"}\nState: Ready\n` + : "", + ), openshellArgv: (args) => ["/usr/local/bin/openshell", ...args], cliName: () => "nemoclaw", agentProductName: () => "NemoClaw", @@ -28,6 +33,7 @@ function harness(options: { sleep: vi.fn(), printAgentDashboardUi: vi.fn(), listSandboxes: options.listSandboxes, + getSandbox: () => ({ gatewayName: "nemoclaw", dashboardPort: 18_790 }), isPortBoundOnHost: options.isPortBound ?? (() => false), stopSandboxForDashboardReuse: stopSandbox, startSandboxForDashboardReuse: startSandbox, @@ -115,11 +121,37 @@ describe("finalization dashboard ForwardTcp launch", () => { sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], }), isPortBound: (port) => port === 18_790, + startSandbox: async () => ({ exitCode: 1, message: "listener conflict" }), }); await expect( helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true), - ).rejects.toThrow(/remained occupied.*cannot be adopted/u); + ).rejects.toThrow(/remained occupied.*Run 'nemoclaw reonboard-test start'/u); + expect(stopSandbox).toHaveBeenCalledWith("reonboard-test"); + expect(startSandbox).toHaveBeenCalledWith("reonboard-test"); + expect(launch).not.toHaveBeenCalled(); + }); + + it("does not start a same-name replacement after the reused sandbox stops", async () => { + vi.stubEnv("CHAT_UI_URL", undefined); + let bound = true; + let identity = "original-id"; + const { helpers, launch, startSandbox, stopSandbox } = harness({ + listSandboxes: () => ({ + sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], + }), + isPortBound: (port) => port === 18_790 && bound, + sandboxIdentity: () => identity, + stopSandbox: () => { + bound = false; + identity = "replacement-id"; + return { exitCode: 0 }; + }, + }); + + await expect( + helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true), + ).rejects.toThrow(/identity changed.*selected sandbox was stopped/u); expect(stopSandbox).toHaveBeenCalledWith("reonboard-test"); expect(startSandbox).not.toHaveBeenCalled(); expect(launch).not.toHaveBeenCalled(); From be631a2de53a6ce01c81fdce8c2e1504b0886828 Mon Sep 17 00:00:00 2001 From: San Dang Date: Mon, 7 Sep 2026 03:23:09 +0700 Subject: [PATCH 05/20] fix(onboard): fence dashboard reuse lifecycle --- docs/manage-sandboxes/run-sandboxes.mdx | 8 +- src/lib/actions/onboard.test.ts | 1 + src/lib/actions/onboard.ts | 4 +- src/lib/actions/sandbox/start.ts | 1 + src/lib/onboard/dashboard.ts | 185 +++++++++--------- .../onboard/dashboard/reuse-lifecycle.test.ts | 12 +- src/lib/onboard/dashboard/reuse-lifecycle.ts | 1 + test/e2e/mock-parity.json | 1 + ...ard-finalization-dashboard-forward.test.ts | 37 +++- 9 files changed, 152 insertions(+), 98 deletions(-) diff --git a/docs/manage-sandboxes/run-sandboxes.mdx b/docs/manage-sandboxes/run-sandboxes.mdx index 280397eb4ac..246ba0cf466 100644 --- a/docs/manage-sandboxes/run-sandboxes.mdx +++ b/docs/manage-sandboxes/run-sandboxes.mdx @@ -62,7 +62,13 @@ The default port keeps the shared `~/.nemoclaw/` location. When other ports remain, `$$nemoclaw uninstall` removes only the selected gateway and keeps the shared CLI, services, images, providers, configuration, models, and swap. Gateway and dashboard cleanup is scoped by sandbox name and port. -A later onboarding run that uses a different `NEMOCLAW_GATEWAY_PORT` or `--control-ui-port` does not tear down the first sandbox's gateway or dashboard forward. +A later onboarding run for another sandbox does not tear down the first sandbox's gateway or dashboard forward. + + + +Re-onboarding the same ready OpenClaw sandbox briefly stops and starts it to verify the registered dashboard forward. +If verification leaves the sandbox stopped, resolve the listener conflict. +Run `nemoclaw start`, then retry onboarding. diff --git a/src/lib/actions/onboard.test.ts b/src/lib/actions/onboard.test.ts index e911867fbc8..4175d55fe2a 100644 --- a/src/lib/actions/onboard.test.ts +++ b/src/lib/actions/onboard.test.ts @@ -33,6 +33,7 @@ describe("onboard action runtime composition", () => { const dashboardReuseLifecycle = { startSandbox: vi.fn(), stopSandbox: vi.fn(), + withSandboxLifecycleLock: vi.fn(), }; await runOnboardAction( diff --git a/src/lib/actions/onboard.ts b/src/lib/actions/onboard.ts index f0051bead30..1c02297d695 100644 --- a/src/lib/actions/onboard.ts +++ b/src/lib/actions/onboard.ts @@ -27,9 +27,11 @@ async function runOnboard( const { onboard } = (await import("../onboard")) as unknown as { onboard: (onboardOptions?: OnboardOptions) => Promise; }; + const startActions = await import("./sandbox/start"); const lifecycle = runtimeDeps.dashboardReuseLifecycle ?? { - startSandbox: (await import("./sandbox/start")).startSandbox, + startSandbox: startActions.startSandbox, stopSandbox: (await import("./sandbox/stop")).stopSandbox, + withSandboxLifecycleLock: startActions.withSandboxLifecycleLock, }; await withDashboardReuseLifecycle(lifecycle, () => onboard({ ...options, googlechatTunnelRuntime: runtimeDeps.googlechatTunnelRuntime }), diff --git a/src/lib/actions/sandbox/start.ts b/src/lib/actions/sandbox/start.ts index 3a35164be90..f548c21b0a0 100644 --- a/src/lib/actions/sandbox/start.ts +++ b/src/lib/actions/sandbox/start.ts @@ -15,6 +15,7 @@ import { type SandboxInferenceInvocationResult, } from "./inference-invocation-probe"; import { withSandboxLifecycleLock } from "./gateway-state"; +export { withSandboxLifecycleLock }; import { getPersistedSandboxTargetGatewayName } from "./gateway-target"; import { resolveSandboxLifecycleProvider, diff --git a/src/lib/onboard/dashboard.ts b/src/lib/onboard/dashboard.ts index be3acdd880e..bfba01136f7 100644 --- a/src/lib/onboard/dashboard.ts +++ b/src/lib/onboard/dashboard.ts @@ -47,6 +47,7 @@ function looksLikeForwardPortConflict(diagnostic: string): boolean { } type CommandResult = { status: number | null }; +type SandboxLifecycleLock = (sandboxName: string, operation: () => Promise | T) => Promise; export interface OnboardDashboardDeps { runOpenshell(args: string[], opts?: Record): CommandResult; @@ -101,6 +102,7 @@ export interface OnboardDashboardDeps { exitCode: number; message?: string; }>; + withSandboxLifecycleLock?: SandboxLifecycleLock; printAgentDashboardUi( sandboxName: string, token: string | null, @@ -396,114 +398,117 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa ): Promise { const port = Number(getDashboardForwardPort(chatUiUrl)); const isPortBound = deps.isPortBoundOnHost ?? isPortBoundOnHost; - if (reconciledOpenClawForwards.get(sandboxName) === port && isPortBound(port)) return true; if (!isPortBound(port)) return false; - if (getRegistryOccupiedDashboardPorts(sandboxName, listSandboxes).has(String(port))) { - throw new Error( - `Registered dashboard port ${String(port)} is already occupied; it cannot be reallocated or adopted.`, - ); - } - const lifecycle = getDashboardReuseLifecycle(); - const stopSandbox = deps.stopSandboxForDashboardReuse ?? lifecycle?.stopSandbox; - const startSandbox = deps.startSandboxForDashboardReuse ?? lifecycle?.startSandbox; - if (!stopSandbox || !startSandbox) { + const withLifecycleLock = deps.withSandboxLifecycleLock ?? lifecycle?.withSandboxLifecycleLock; + if (!withLifecycleLock) { throw new Error( - `Could not restart sandbox '${sandboxName}' to reconcile dashboard port ${String(port)}: sandbox lifecycle is unavailable.`, + `Could not restart sandbox '${sandboxName}' to reconcile dashboard port ${String(port)}: sandbox lifecycle lock is unavailable.`, ); } + return await withLifecycleLock(sandboxName, async () => { + if (reconciledOpenClawForwards.get(sandboxName) === port && isPortBound(port)) return true; + if (!isPortBound(port)) return false; + if (getRegistryOccupiedDashboardPorts(sandboxName, listSandboxes).has(String(port))) { + throw new Error( + `Registered dashboard port ${String(port)} is already occupied; it cannot be reallocated or adopted.`, + ); + } - const readLiveIdentity = (gatewayName: string): string | null => - fingerprintSandboxLiveIdentity( - deps.runCaptureOpenshell(["sandbox", "get", "-g", gatewayName, sandboxName], { - ignoreError: true, - includeStderr: true, - }) ?? "", - ); - const registered = getSandbox?.(sandboxName); - const gatewayName = registered ? forwardService?.resolveGatewayName(registered) : null; - const observedIdentity = gatewayName ? readLiveIdentity(gatewayName) : null; - if ( - !registered || - !gatewayName || - !observedIdentity || - (registered.lifecycleLiveIdentityFingerprint && - registered.lifecycleLiveIdentityFingerprint !== observedIdentity) - ) { - throw new Error( - `Could not verify sandbox '${sandboxName}' before reconciling dashboard port ${String(port)}.`, - ); - } - const assertSameSandbox = (operation: string): void => { - const current = getSandbox?.(sandboxName); - const currentGateway = current ? forwardService?.resolveGatewayName(current) : null; - const currentIdentity = currentGateway ? readLiveIdentity(currentGateway) : null; + const stopSandbox = deps.stopSandboxForDashboardReuse ?? lifecycle?.stopSandbox; + const startSandbox = deps.startSandboxForDashboardReuse ?? lifecycle?.startSandbox; + if (!stopSandbox || !startSandbox) { + throw new Error( + `Could not restart sandbox '${sandboxName}' to reconcile dashboard port ${String(port)}: sandbox lifecycle is unavailable.`, + ); + } + + const readLiveIdentity = (gatewayName: string): string | null => + fingerprintSandboxLiveIdentity( + deps.runCaptureOpenshell(["sandbox", "get", "-g", gatewayName, sandboxName], { + ignoreError: true, + includeStderr: true, + }) ?? "", + ); + const registered = getSandbox?.(sandboxName); + const gatewayName = registered ? forwardService?.resolveGatewayName(registered) : null; + const observedIdentity = gatewayName ? readLiveIdentity(gatewayName) : null; if ( - !current || - currentGateway !== gatewayName || - current.lifecycleGeneration !== registered.lifecycleGeneration || - current.lifecycleLiveIdentityFingerprint !== registered.lifecycleLiveIdentityFingerprint || - currentIdentity !== observedIdentity + !registered || + !gatewayName || + !registered.lifecycleLiveIdentityFingerprint || + registered.lifecycleLiveIdentityFingerprint !== observedIdentity ) { throw new Error( - `Refusing to ${operation}: sandbox '${sandboxName}' identity changed during dashboard reconciliation.`, + `Could not verify sandbox '${sandboxName}' before reconciling dashboard port ${String(port)}.`, ); } - }; + const assertSameSandbox = (operation: string): void => { + const current = getSandbox?.(sandboxName); + const currentGateway = current ? forwardService?.resolveGatewayName(current) : null; + const currentIdentity = currentGateway ? readLiveIdentity(currentGateway) : null; + if ( + !current || + currentGateway !== gatewayName || + current.lifecycleGeneration !== registered.lifecycleGeneration || + current.lifecycleLiveIdentityFingerprint !== + registered.lifecycleLiveIdentityFingerprint || + currentIdentity !== observedIdentity + ) { + throw new Error( + `Refusing to ${operation}: sandbox '${sandboxName}' identity changed during dashboard reconciliation.`, + ); + } + }; - revalidateSandboxIdentity?.( - `restart sandbox '${sandboxName}' to reconcile dashboard forward ${String(port)}`, - ); - assertSameSandbox(`stop sandbox '${sandboxName}'`); - const stopped = stopSandbox(sandboxName); - if (stopped.exitCode !== 0) { - throw new Error( - `Could not stop sandbox '${sandboxName}' to reconcile dashboard port ${String(port)}${ - stopped.message ? `: ${stopped.message}` : "." - }`, - ); - } - try { revalidateSandboxIdentity?.( - `start sandbox '${sandboxName}' to reconcile dashboard forward ${String(port)}`, - ); - assertSameSandbox(`start sandbox '${sandboxName}'`); - } catch (error) { - const detail = error instanceof Error ? error.message : String(error); - throw new Error( - `${detail} The selected sandbox was stopped; verify its identity, then run '${deps.cliName()} ${sandboxName} start'.`, + `restart sandbox '${sandboxName}' to reconcile dashboard forward ${String(port)}`, ); - } + assertSameSandbox(`stop sandbox '${sandboxName}'`); + const stopped = stopSandbox(sandboxName); + if (stopped.exitCode !== 0) { + throw new Error( + `Could not stop sandbox '${sandboxName}' to reconcile dashboard port ${String(port)}${ + stopped.message ? `: ${stopped.message}` : "." + }`, + ); + } + try { + revalidateSandboxIdentity?.( + `start sandbox '${sandboxName}' to reconcile dashboard forward ${String(port)}`, + ); + assertSameSandbox(`start sandbox '${sandboxName}'`); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + throw new Error( + `${detail} The selected sandbox was stopped; verify its identity, then run '${deps.cliName()} ${sandboxName} start'.`, + ); + } - const portRemainedBound = isPortBound(port); - let started: { exitCode: number; message?: string }; - try { - started = await startSandbox(sandboxName); - } catch (error) { - const detail = error instanceof Error ? error.message : String(error); - throw new Error( - `Could not restart sandbox '${sandboxName}' after releasing dashboard port ${String(port)}: ${detail}. The sandbox may remain stopped; run '${deps.cliName()} ${sandboxName} start' before retrying onboarding.`, - ); - } - if (portRemainedBound) { - if (started.exitCode !== 0) { + if (isPortBound(port)) { throw new Error( - `Registered dashboard port ${String(port)} remained occupied after sandbox '${sandboxName}' stopped, and the sandbox could not restart${started.message ? `: ${started.message}` : "."} Run '${deps.cliName()} ${sandboxName} start' after resolving the listener conflict.`, + `Registered dashboard port ${String(port)} remained occupied after sandbox '${sandboxName}' stopped; it cannot be adopted. Resolve the listener, run '${deps.cliName()} ${sandboxName} start', then retry onboarding.`, ); } - throw new Error( - `Registered dashboard port ${String(port)} remained occupied after sandbox '${sandboxName}' stopped; it cannot be adopted.`, - ); - } - if (started.exitCode !== 0 || !isPortBound(port)) { - throw new Error( - `Sandbox '${sandboxName}' did not restore dashboard port ${String(port)} after restart${ - started.message ? `: ${started.message}` : "." - } The sandbox may remain stopped; run '${deps.cliName()} ${sandboxName} start' before retrying onboarding.`, - ); - } - reconciledOpenClawForwards.set(sandboxName, port); - return true; + let started: { exitCode: number; message?: string }; + try { + started = await startSandbox(sandboxName); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + throw new Error( + `Could not restart sandbox '${sandboxName}' after releasing dashboard port ${String(port)}: ${detail}. The sandbox may remain stopped; run '${deps.cliName()} ${sandboxName} start' before retrying onboarding.`, + ); + } + if (started.exitCode !== 0 || !isPortBound(port)) { + throw new Error( + `Sandbox '${sandboxName}' did not restore dashboard port ${String(port)} after restart${ + started.message ? `: ${started.message}` : "." + } The sandbox may remain stopped; run '${deps.cliName()} ${sandboxName} start' before retrying onboarding.`, + ); + } + reconciledOpenClawForwards.set(sandboxName, port); + return true; + }); } function ensureDashboardForward( diff --git a/src/lib/onboard/dashboard/reuse-lifecycle.test.ts b/src/lib/onboard/dashboard/reuse-lifecycle.test.ts index 3e76a7fb5a3..4a396963ecf 100644 --- a/src/lib/onboard/dashboard/reuse-lifecycle.test.ts +++ b/src/lib/onboard/dashboard/reuse-lifecycle.test.ts @@ -6,8 +6,16 @@ import { expect, it, vi } from "vitest"; import { getDashboardReuseLifecycle, withDashboardReuseLifecycle } from "./reuse-lifecycle"; it("keeps overlapping onboarding lifecycle scopes independent", async () => { - const first = { startSandbox: vi.fn(), stopSandbox: vi.fn() }; - const second = { startSandbox: vi.fn(), stopSandbox: vi.fn() }; + const first = { + startSandbox: vi.fn(), + stopSandbox: vi.fn(), + withSandboxLifecycleLock: vi.fn(), + }; + const second = { + startSandbox: vi.fn(), + stopSandbox: vi.fn(), + withSandboxLifecycleLock: vi.fn(), + }; let releaseFirst!: () => void; const firstPaused = new Promise((resolve) => { releaseFirst = resolve; diff --git a/src/lib/onboard/dashboard/reuse-lifecycle.ts b/src/lib/onboard/dashboard/reuse-lifecycle.ts index ae6bbbbd461..6789f1aa6bc 100644 --- a/src/lib/onboard/dashboard/reuse-lifecycle.ts +++ b/src/lib/onboard/dashboard/reuse-lifecycle.ts @@ -6,6 +6,7 @@ import { AsyncLocalStorage } from "node:async_hooks"; export type DashboardReuseLifecycle = { stopSandbox(sandboxName: string): { exitCode: number; message?: string }; startSandbox(sandboxName: string): Promise<{ exitCode: number; message?: string }>; + withSandboxLifecycleLock(sandboxName: string, operation: () => Promise | T): Promise; }; const lifecycleStorage = new AsyncLocalStorage(); diff --git a/test/e2e/mock-parity.json b/test/e2e/mock-parity.json index 9a7ffe511f5..5406e376221 100644 --- a/test/e2e/mock-parity.json +++ b/test/e2e/mock-parity.json @@ -328,6 +328,7 @@ "src/lib/onboard/sandbox-gpu-create-flow.test.ts", "src/lib/onboard/sandbox-readiness-tracing.test.ts", "test/onboarding/onboard-extra-provider-reconciliation.test.ts", + "test/onboarding/onboard-finalization-dashboard-forward.test.ts", "test/e2e/support/corporate-ca-workload-kind.test.ts", "test/runtime/gateway/gateway-state.test.ts", "test/e2e/support/e2e-cleanup-resources.test.ts", diff --git a/test/onboarding/onboard-finalization-dashboard-forward.test.ts b/test/onboarding/onboard-finalization-dashboard-forward.test.ts index 20145e46bd3..ff801e0f9fb 100644 --- a/test/onboarding/onboard-finalization-dashboard-forward.test.ts +++ b/test/onboarding/onboard-finalization-dashboard-forward.test.ts @@ -5,10 +5,12 @@ import { describe, expect, it, vi } from "vitest"; import { createOnboardDashboardHelpers } from "../../src/lib/onboard/dashboard"; import type { ListSandboxesFn } from "../../src/lib/onboard/dashboard-port"; +import { fingerprintSandboxLiveIdentity } from "../../src/lib/onboard/sandbox-recreate-transaction"; function harness(options: { listSandboxes: ListSandboxesFn; isPortBound?: (port: number) => boolean; + registeredIdentity?: boolean; sandboxIdentity?: () => string; stopSandbox?: (sandboxName: string) => { exitCode: number; message?: string }; startSandbox?: (sandboxName: string) => Promise<{ exitCode: number; message?: string }>; @@ -16,6 +18,9 @@ function harness(options: { const launch = vi.fn(); const stopSandbox = vi.fn(options.stopSandbox ?? (() => ({ exitCode: 0 }))); const startSandbox = vi.fn(options.startSandbox ?? (async () => ({ exitCode: 0 }))); + const recordedIdentity = fingerprintSandboxLiveIdentity( + `Id: ${options.sandboxIdentity?.() ?? "sandbox-id"}`, + ); const helpers = createOnboardDashboardHelpers({ runOpenshell: vi.fn(() => ({ status: 0 })), runCaptureOpenshell: vi.fn((args) => @@ -33,10 +38,17 @@ function harness(options: { sleep: vi.fn(), printAgentDashboardUi: vi.fn(), listSandboxes: options.listSandboxes, - getSandbox: () => ({ gatewayName: "nemoclaw", dashboardPort: 18_790 }), + getSandbox: () => ({ + gatewayName: "nemoclaw", + dashboardPort: 18_790, + lifecycleGeneration: "generation-1", + lifecycleLiveIdentityFingerprint: + options.registeredIdentity === false ? undefined : (recordedIdentity ?? undefined), + }), isPortBoundOnHost: options.isPortBound ?? (() => false), stopSandboxForDashboardReuse: stopSandbox, startSandboxForDashboardReuse: startSandbox, + withSandboxLifecycleLock: async (_sandboxName, operation) => await operation(), forwardService: { executable: () => "/usr/local/bin/openshell", launch, @@ -121,14 +133,13 @@ describe("finalization dashboard ForwardTcp launch", () => { sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], }), isPortBound: (port) => port === 18_790, - startSandbox: async () => ({ exitCode: 1, message: "listener conflict" }), }); await expect( helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true), - ).rejects.toThrow(/remained occupied.*Run 'nemoclaw reonboard-test start'/u); + ).rejects.toThrow(/remained occupied.*run 'nemoclaw reonboard-test start'/u); expect(stopSandbox).toHaveBeenCalledWith("reonboard-test"); - expect(startSandbox).toHaveBeenCalledWith("reonboard-test"); + expect(startSandbox).not.toHaveBeenCalled(); expect(launch).not.toHaveBeenCalled(); }); @@ -157,6 +168,24 @@ describe("finalization dashboard ForwardTcp launch", () => { expect(launch).not.toHaveBeenCalled(); }); + it("does not restart a reused sandbox without a registered live identity", async () => { + vi.stubEnv("CHAT_UI_URL", undefined); + const { helpers, launch, startSandbox, stopSandbox } = harness({ + listSandboxes: () => ({ + sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], + }), + isPortBound: (port) => port === 18_790, + registeredIdentity: false, + }); + + await expect( + helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true), + ).rejects.toThrow(/Could not verify sandbox/u); + expect(stopSandbox).not.toHaveBeenCalled(); + expect(startSandbox).not.toHaveBeenCalled(); + expect(launch).not.toHaveBeenCalled(); + }); + it("does not reuse a forward when another sandbox registers the same port", async () => { vi.stubEnv("CHAT_UI_URL", undefined); const { helpers, launch, stopSandbox } = harness({ From d2a0dd9f4f2d6b8c10c1e197bf13ea7cab18ce6f Mon Sep 17 00:00:00 2001 From: San Dang Date: Mon, 7 Sep 2026 03:51:33 +0700 Subject: [PATCH 06/20] fix(onboard): bind dashboard reuse lifecycle --- docs/manage-sandboxes/run-sandboxes.mdx | 6 +-- src/lib/actions/onboard.test.ts | 15 +++++- src/lib/actions/onboard.ts | 8 ++-- .../sandbox/rebuild-onboard-dependencies.ts | 10 +++- src/lib/actions/sandbox/stop.test.ts | 12 +++++ src/lib/actions/sandbox/stop.ts | 19 ++++---- src/lib/onboard/dashboard.ts | 17 ++++--- src/lib/onboard/dashboard/reuse-lifecycle.ts | 5 +- ...ard-finalization-dashboard-forward.test.ts | 46 +++++++++++++++++-- 9 files changed, 110 insertions(+), 28 deletions(-) diff --git a/docs/manage-sandboxes/run-sandboxes.mdx b/docs/manage-sandboxes/run-sandboxes.mdx index 246ba0cf466..71d3be2f2f9 100644 --- a/docs/manage-sandboxes/run-sandboxes.mdx +++ b/docs/manage-sandboxes/run-sandboxes.mdx @@ -66,9 +66,9 @@ A later onboarding run for another sandbox does not tear down the first sandbox' -Re-onboarding the same ready OpenClaw sandbox briefly stops and starts it to verify the registered dashboard forward. -If verification leaves the sandbox stopped, resolve the listener conflict. -Run `nemoclaw start`, then retry onboarding. +If re-onboarding finds the registered dashboard port already bound, NemoClaw briefly stops and starts the same ready OpenClaw sandbox to verify its dashboard forward. +If the port remains bound after the sandbox stops, resolve the listener conflict, run `nemoclaw start`, then retry onboarding. +For other failures, follow the reported recovery steps. diff --git a/src/lib/actions/onboard.test.ts b/src/lib/actions/onboard.test.ts index 4175d55fe2a..c53810ff5f7 100644 --- a/src/lib/actions/onboard.test.ts +++ b/src/lib/actions/onboard.test.ts @@ -13,7 +13,8 @@ vi.mock("../agent/defs", () => ({ listAgents: mocks.listAgents })); vi.mock("../onboard", () => ({ onboard: mocks.onboard })); vi.mock("../onboard/command", () => ({ runOnboardCommand: mocks.runOnboardCommand })); -import { runOnboardAction } from "./onboard"; +import { getDashboardReuseLifecycle } from "../onboard/dashboard/reuse-lifecycle"; +import { runOnboard, runOnboardAction } from "./onboard"; describe("onboard action runtime composition", () => { beforeEach(() => { @@ -47,4 +48,16 @@ describe("onboard action runtime composition", () => { googlechatTunnelRuntime, }); }); + + it("provides the default dashboard reuse lifecycle to direct onboarding callers", async () => { + mocks.onboard.mockImplementationOnce(async () => { + expect(getDashboardReuseLifecycle()).toEqual({ + startSandbox: expect.any(Function), + stopSandbox: expect.any(Function), + withSandboxLifecycleLock: expect.any(Function), + }); + }); + + await runOnboard({} as never); + }); }); diff --git a/src/lib/actions/onboard.ts b/src/lib/actions/onboard.ts index 1c02297d695..a21fb479dc9 100644 --- a/src/lib/actions/onboard.ts +++ b/src/lib/actions/onboard.ts @@ -18,9 +18,9 @@ export interface OnboardActionRuntimeDeps { readonly dashboardReuseLifecycle?: DashboardReuseLifecycle; } -async function runOnboard( +export async function runOnboard( options: OnboardCommandOptions, - runtimeDeps: OnboardActionRuntimeDeps, + runtimeDeps: OnboardActionRuntimeDeps = {}, ): Promise { // Keep the monolithic legacy onboarding graph lazy so command metadata/help // imports do not execute it. Resolve it only when the user invokes onboard. @@ -28,9 +28,11 @@ async function runOnboard( onboard: (onboardOptions?: OnboardOptions) => Promise; }; const startActions = await import("./sandbox/start"); + const stopActions = await import("./sandbox/stop"); const lifecycle = runtimeDeps.dashboardReuseLifecycle ?? { startSandbox: startActions.startSandbox, - stopSandbox: (await import("./sandbox/stop")).stopSandbox, + stopSandbox: (sandboxName: string, revalidateAtMutationEdge: () => void) => + stopActions.stopSandbox(sandboxName, { revalidateAtMutationEdge }), withSandboxLifecycleLock: startActions.withSandboxLifecycleLock, }; await withDashboardReuseLifecycle(lifecycle, () => diff --git a/src/lib/actions/sandbox/rebuild-onboard-dependencies.ts b/src/lib/actions/sandbox/rebuild-onboard-dependencies.ts index f93d591bc14..33ac3042d29 100644 --- a/src/lib/actions/sandbox/rebuild-onboard-dependencies.ts +++ b/src/lib/actions/sandbox/rebuild-onboard-dependencies.ts @@ -24,10 +24,18 @@ type RebuildOnboardModule = { ) => Promise; }; +type OnboardActionModule = { + runOnboard(options: RebuildRecreateOnboardOpts): Promise; +}; + function loadOnboardModule(): RebuildOnboardModule { return require("../../onboard") as RebuildOnboardModule; } +function loadOnboardActionModule(): OnboardActionModule { + return require("../onboard") as OnboardActionModule; +} + /** * Late-bound onboarding boundary for rebuild orchestration. Rebuild imports no * longer initialize the full onboarding graph, and focused tests can replace @@ -45,7 +53,7 @@ export const rebuildOnboardDependencies = { return loadOnboardModule().hydrateCredentialEnv(name); }, onboard(options: RebuildRecreateOnboardOpts): Promise { - return loadOnboardModule().onboard(options); + return loadOnboardActionModule().runOnboard(options); }, preflightAuthoritativeRebuildTarget( options: RebuildAuthoritativePreflightOptions, diff --git a/src/lib/actions/sandbox/stop.test.ts b/src/lib/actions/sandbox/stop.test.ts index 5d04551f92a..2c9d08b3e0a 100644 --- a/src/lib/actions/sandbox/stop.test.ts +++ b/src/lib/actions/sandbox/stop.test.ts @@ -273,6 +273,18 @@ describe("discoverActiveOllamaSandboxNames", () => { }); describe("stopSandbox", () => { + it("revalidates the selected sandbox at the stop mutation boundary", () => { + const revalidateAtMutationEdge = vi.fn(() => { + throw new Error("sandbox identity changed"); + }); + const h = harness({ revalidateAtMutationEdge }); + + expect(() => stopSandbox("my-sandbox", h.deps)).toThrow(/sandbox identity changed/u); + expect(revalidateAtMutationEdge).toHaveBeenCalledOnce(); + expect(h.stopSandboxChannels).not.toHaveBeenCalled(); + expect(h.dockerStop).not.toHaveBeenCalled(); + }); + it("gracefully stops in-sandbox channels before stopping the container (#6026)", () => { const h = harness(); diff --git a/src/lib/actions/sandbox/stop.ts b/src/lib/actions/sandbox/stop.ts index 6b0a27d7f3d..69c10bc1623 100644 --- a/src/lib/actions/sandbox/stop.ts +++ b/src/lib/actions/sandbox/stop.ts @@ -117,9 +117,7 @@ export function discoverActiveOllamaSandboxNames( }`, }; } - const phases = new Map( - parseEntries(result.output).map((entry) => [entry.name, entry.phase]), - ); + const phases = new Map(parseEntries(result.output).map((entry) => [entry.name, entry.phase])); const activeSandboxes: string[] = []; for (const peerName of peerNames) { const phase = phases.get(peerName); @@ -148,19 +146,20 @@ function releaseStoppedSandboxOllamaModel( if (!isLocalOllamaRouteOwner(sandbox)) return { ok: true }; try { - const proxy = require("../../inference/ollama/proxy") as typeof import("../../inference/ollama/proxy"); + const proxy = + require("../../inference/ollama/proxy") as typeof import("../../inference/ollama/proxy"); const withOwnershipLock = deps.withOllamaModelOwnershipLock ?? proxy.withOllamaModelOwnershipLock; - const loadPersistedOllamaHost = - deps.loadPersistedOllamaHost ?? proxy.loadPersistedOllamaHost; + const loadPersistedOllamaHost = deps.loadPersistedOllamaHost ?? proxy.loadPersistedOllamaHost; return withOwnershipLock(() => { const selectedHost = loadPersistedOllamaHost(); if (!isLocalOllamaRouteOwner(sandbox, selectedHost)) return { ok: true }; const { sandboxes } = (deps.listSandboxes ?? registry.listSandboxes)(); const matchingPeers = matchingOllamaModelPeers(sandbox, sandboxes, selectedHost); - const discovery = ( - deps.discoverActiveOllamaSandboxNames ?? discoverActiveOllamaSandboxNames - )(matchingPeers, deps.environment ?? process.env); + const discovery = (deps.discoverActiveOllamaSandboxNames ?? discoverActiveOllamaSandboxNames)( + matchingPeers, + deps.environment ?? process.env, + ); if (!discovery.ok) { return { ok: false, @@ -251,6 +250,7 @@ export interface SandboxStopDeps { loadPersistedOllamaHost?: () => OllamaHostRoute | null; withOllamaModelOwnershipLock?: typeof import("../../inference/ollama/proxy").withOllamaModelOwnershipLock; withLifecycleLockSync?: typeof withSandboxLifecycleLockSync; + revalidateAtMutationEdge?: () => void; log?: (message: string) => void; warn?: (message: string) => void; } @@ -292,6 +292,7 @@ function stopSandboxWithinLifecycleFence( const preflight = resolved.bundle.preflightDoctor.preflightLifecycle("stop", input); if (preflight) return preflight; + deps.revalidateAtMutationEdge?.(); let channelsStopped = false; const outcome = resolved.lifecycle.stop(input, { beforeStop() { diff --git a/src/lib/onboard/dashboard.ts b/src/lib/onboard/dashboard.ts index bfba01136f7..fc1eb230596 100644 --- a/src/lib/onboard/dashboard.ts +++ b/src/lib/onboard/dashboard.ts @@ -97,7 +97,10 @@ export interface OnboardDashboardDeps { sandbox: { gatewayName?: string | null; gatewayPort?: number | null } | null | undefined, ): string; }; - stopSandboxForDashboardReuse?(sandboxName: string): { exitCode: number; message?: string }; + stopSandboxForDashboardReuse?( + sandboxName: string, + revalidateAtMutationEdge: () => void, + ): { exitCode: number; message?: string }; startSandboxForDashboardReuse?(sandboxName: string): Promise<{ exitCode: number; message?: string; @@ -461,11 +464,13 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa } }; - revalidateSandboxIdentity?.( - `restart sandbox '${sandboxName}' to reconcile dashboard forward ${String(port)}`, - ); - assertSameSandbox(`stop sandbox '${sandboxName}'`); - const stopped = stopSandbox(sandboxName); + const revalidateAtStopBoundary = (): void => { + revalidateSandboxIdentity?.( + `restart sandbox '${sandboxName}' to reconcile dashboard forward ${String(port)}`, + ); + assertSameSandbox(`stop sandbox '${sandboxName}'`); + }; + const stopped = stopSandbox(sandboxName, revalidateAtStopBoundary); if (stopped.exitCode !== 0) { throw new Error( `Could not stop sandbox '${sandboxName}' to reconcile dashboard port ${String(port)}${ diff --git a/src/lib/onboard/dashboard/reuse-lifecycle.ts b/src/lib/onboard/dashboard/reuse-lifecycle.ts index 6789f1aa6bc..d97b9a39c84 100644 --- a/src/lib/onboard/dashboard/reuse-lifecycle.ts +++ b/src/lib/onboard/dashboard/reuse-lifecycle.ts @@ -4,7 +4,10 @@ import { AsyncLocalStorage } from "node:async_hooks"; export type DashboardReuseLifecycle = { - stopSandbox(sandboxName: string): { exitCode: number; message?: string }; + stopSandbox( + sandboxName: string, + revalidateAtMutationEdge: () => void, + ): { exitCode: number; message?: string }; startSandbox(sandboxName: string): Promise<{ exitCode: number; message?: string }>; withSandboxLifecycleLock(sandboxName: string, operation: () => Promise | T): Promise; }; diff --git a/test/onboarding/onboard-finalization-dashboard-forward.test.ts b/test/onboarding/onboard-finalization-dashboard-forward.test.ts index ff801e0f9fb..94fd36a4e0a 100644 --- a/test/onboarding/onboard-finalization-dashboard-forward.test.ts +++ b/test/onboarding/onboard-finalization-dashboard-forward.test.ts @@ -16,7 +16,10 @@ function harness(options: { startSandbox?: (sandboxName: string) => Promise<{ exitCode: number; message?: string }>; }) { const launch = vi.fn(); - const stopSandbox = vi.fn(options.stopSandbox ?? (() => ({ exitCode: 0 }))); + const stopSandbox = vi.fn((sandboxName: string, revalidateAtMutationEdge: () => void) => { + revalidateAtMutationEdge(); + return (options.stopSandbox ?? (() => ({ exitCode: 0 })))(sandboxName); + }); const startSandbox = vi.fn(options.startSandbox ?? (async () => ({ exitCode: 0 }))); const recordedIdentity = fingerprintSandboxLiveIdentity( `Id: ${options.sandboxIdentity?.() ?? "sandbox-id"}`, @@ -120,7 +123,7 @@ describe("finalization dashboard ForwardTcp launch", () => { await expect( helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true), ).resolves.toBe(18_790); - expect(stopSandbox).toHaveBeenCalledWith("reonboard-test"); + expect(stopSandbox).toHaveBeenCalledWith("reonboard-test", expect.any(Function)); expect(startSandbox).toHaveBeenCalledWith("reonboard-test"); expect(launch).not.toHaveBeenCalled(); expect(process.env.CHAT_UI_URL).toBe("http://127.0.0.1:18790"); @@ -138,7 +141,7 @@ describe("finalization dashboard ForwardTcp launch", () => { await expect( helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true), ).rejects.toThrow(/remained occupied.*run 'nemoclaw reonboard-test start'/u); - expect(stopSandbox).toHaveBeenCalledWith("reonboard-test"); + expect(stopSandbox).toHaveBeenCalledWith("reonboard-test", expect.any(Function)); expect(startSandbox).not.toHaveBeenCalled(); expect(launch).not.toHaveBeenCalled(); }); @@ -163,7 +166,7 @@ describe("finalization dashboard ForwardTcp launch", () => { await expect( helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true), ).rejects.toThrow(/identity changed.*selected sandbox was stopped/u); - expect(stopSandbox).toHaveBeenCalledWith("reonboard-test"); + expect(stopSandbox).toHaveBeenCalledWith("reonboard-test", expect.any(Function)); expect(startSandbox).not.toHaveBeenCalled(); expect(launch).not.toHaveBeenCalled(); }); @@ -186,6 +189,41 @@ describe("finalization dashboard ForwardTcp launch", () => { expect(launch).not.toHaveBeenCalled(); }); + it("does not cache a failed sandbox restart as reconciled", async () => { + vi.stubEnv("CHAT_UI_URL", undefined); + let bound = true; + const startOutcomes = [ + async () => ({ exitCode: 1, message: "restart failed" }), + async () => { + bound = true; + return { exitCode: 0 }; + }, + ]; + const { helpers, launch, startSandbox, stopSandbox } = harness({ + listSandboxes: () => ({ + sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], + }), + isPortBound: (port) => port === 18_790 && bound, + stopSandbox: () => { + bound = false; + return { exitCode: 0 }; + }, + startSandbox: async () => await startOutcomes.shift()!(), + }); + + await expect( + helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true), + ).rejects.toThrow(/did not restore dashboard port.*restart failed/u); + + bound = true; + await expect( + helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true), + ).resolves.toBe(18_790); + expect(stopSandbox).toHaveBeenCalledTimes(2); + expect(startSandbox).toHaveBeenCalledTimes(2); + expect(launch).not.toHaveBeenCalled(); + }); + it("does not reuse a forward when another sandbox registers the same port", async () => { vi.stubEnv("CHAT_UI_URL", undefined); const { helpers, launch, stopSandbox } = harness({ From 48d4988d57df82a9677a9e533a4c0d4362506d8b Mon Sep 17 00:00:00 2001 From: San Dang Date: Mon, 7 Sep 2026 04:10:05 +0700 Subject: [PATCH 07/20] fix(onboard): restore dashboard reuse failures --- src/lib/actions/onboard.ts | 3 +- src/lib/actions/sandbox/start.test.ts | 12 +++ src/lib/actions/sandbox/start.ts | 2 + src/lib/actions/sandbox/stop.test.ts | 1 + src/lib/actions/sandbox/stop.ts | 12 +-- src/lib/onboard/dashboard.ts | 41 ++++++---- src/lib/onboard/dashboard/reuse-lifecycle.ts | 7 +- ...ard-finalization-dashboard-forward.test.ts | 77 +++++++++++++++++-- 8 files changed, 123 insertions(+), 32 deletions(-) diff --git a/src/lib/actions/onboard.ts b/src/lib/actions/onboard.ts index a21fb479dc9..27a01babcdf 100644 --- a/src/lib/actions/onboard.ts +++ b/src/lib/actions/onboard.ts @@ -30,7 +30,8 @@ export async function runOnboard( const startActions = await import("./sandbox/start"); const stopActions = await import("./sandbox/stop"); const lifecycle = runtimeDeps.dashboardReuseLifecycle ?? { - startSandbox: startActions.startSandbox, + startSandbox: (sandboxName: string, revalidateAtMutationEdge: () => void) => + startActions.startSandbox(sandboxName, { revalidateAtMutationEdge }), stopSandbox: (sandboxName: string, revalidateAtMutationEdge: () => void) => stopActions.stopSandbox(sandboxName, { revalidateAtMutationEdge }), withSandboxLifecycleLock: startActions.withSandboxLifecycleLock, diff --git a/src/lib/actions/sandbox/start.test.ts b/src/lib/actions/sandbox/start.test.ts index 8557ff91f38..94557c9ecb9 100644 --- a/src/lib/actions/sandbox/start.test.ts +++ b/src/lib/actions/sandbox/start.test.ts @@ -116,6 +116,18 @@ function harness(overrides: Partial = {}) { } describe("startSandbox", () => { + it("revalidates the selected sandbox at the start mutation boundary", async () => { + const revalidateAtMutationEdge = vi.fn(() => { + throw new Error("sandbox identity changed"); + }); + const h = harness({ revalidateAtMutationEdge }); + + await expect(startSandbox("my-sandbox", h.deps)).rejects.toThrow(/sandbox identity changed/u); + expect(revalidateAtMutationEdge).toHaveBeenCalledOnce(); + expect(h.recoverDockerDriverSandbox).not.toHaveBeenCalled(); + expect(h.recoverPortableSandbox).not.toHaveBeenCalled(); + }); + it("waits for OpenShell readiness before recovering sandbox processes (#8978)", async () => { const waitForSandboxReady = vi.fn(); const restoreProcesses = vi.fn(() => SUCCESSFUL_RECOVERY); diff --git a/src/lib/actions/sandbox/start.ts b/src/lib/actions/sandbox/start.ts index f548c21b0a0..ffcced6110e 100644 --- a/src/lib/actions/sandbox/start.ts +++ b/src/lib/actions/sandbox/start.ts @@ -81,6 +81,7 @@ export interface SandboxStartDeps { verifyGateway?: (sandboxName: string) => Promise; probeInferenceInvocation?: typeof probeSandboxInferenceInvocation; withLifecycleLock?: typeof withSandboxLifecycleLock; + revalidateAtMutationEdge?: () => void; log?: (message: string) => void; } @@ -193,6 +194,7 @@ async function startSandboxWithinLifecycleFence( }; const preflight = resolved.bundle.preflightDoctor.preflightLifecycle("start", input); if (preflight) return preflight; + deps.revalidateAtMutationEdge?.(); const result = resolved.lifecycle.start(input); if (result.exitCode !== 0) return result; if ("hermesPortableVerified" in result && result.hermesPortableVerified === true) { diff --git a/src/lib/actions/sandbox/stop.test.ts b/src/lib/actions/sandbox/stop.test.ts index 2c9d08b3e0a..3cdc16a8092 100644 --- a/src/lib/actions/sandbox/stop.test.ts +++ b/src/lib/actions/sandbox/stop.test.ts @@ -872,6 +872,7 @@ describe("stopSandbox Ollama GPU release", () => { const result = stopSandbox("my-sandbox", h.deps); expect(result.exitCode).toBe(1); + expect(result.stopped).toBe(true); expect(result.message).toContain("curl: command not found"); expect(result.message).toContain("retry 'nemoclaw my-sandbox stop'"); }); diff --git a/src/lib/actions/sandbox/stop.ts b/src/lib/actions/sandbox/stop.ts index 69c10bc1623..b92be1d5538 100644 --- a/src/lib/actions/sandbox/stop.ts +++ b/src/lib/actions/sandbox/stop.ts @@ -233,6 +233,7 @@ function releaseStoppedSandboxOllamaModel( } export type { SandboxLifecycleResult } from "./runtime/lifecycle-runtime"; +export type SandboxStopResult = SandboxLifecycleResult & { stopped?: true }; export interface SandboxStopDeps { environment?: NodeJS.ProcessEnv; @@ -259,10 +260,7 @@ export interface SandboxStopDeps { * Stop the selected provider workload while preserving registry, workspace, * credentials, and shared gateway state. */ -export function stopSandbox( - sandboxName: string, - deps: SandboxStopDeps = {}, -): SandboxLifecycleResult { +export function stopSandbox(sandboxName: string, deps: SandboxStopDeps = {}): SandboxStopResult { return (deps.withLifecycleLockSync ?? withSandboxLifecycleLockSync)(sandboxName, () => stopSandboxWithinLifecycleFence(sandboxName, deps), ); @@ -271,7 +269,7 @@ export function stopSandbox( function stopSandboxWithinLifecycleFence( sandboxName: string, deps: SandboxStopDeps, -): SandboxLifecycleResult { +): SandboxStopResult { const log = deps.log ?? console.log; const warn = deps.warn ?? console.warn; const sandbox = (deps.getSandbox ?? registry.getSandbox)(sandboxName); @@ -321,7 +319,9 @@ function stopSandboxWithinLifecycleFence( warn, ); } - if (!ollamaRelease.ok) return { exitCode: 1, message: ollamaRelease.message }; + if (!ollamaRelease.ok) { + return { exitCode: 1, message: ollamaRelease.message, stopped: true }; + } if (hermesPortableVerified) { log( outcome.state === "already-stopped" diff --git a/src/lib/onboard/dashboard.ts b/src/lib/onboard/dashboard.ts index fc1eb230596..4d2a7dceaf2 100644 --- a/src/lib/onboard/dashboard.ts +++ b/src/lib/onboard/dashboard.ts @@ -100,8 +100,11 @@ export interface OnboardDashboardDeps { stopSandboxForDashboardReuse?( sandboxName: string, revalidateAtMutationEdge: () => void, - ): { exitCode: number; message?: string }; - startSandboxForDashboardReuse?(sandboxName: string): Promise<{ + ): { exitCode: number; message?: string; stopped?: true }; + startSandboxForDashboardReuse?( + sandboxName: string, + revalidateAtMutationEdge: () => void, + ): Promise<{ exitCode: number; message?: string; }>; @@ -471,22 +474,17 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa assertSameSandbox(`stop sandbox '${sandboxName}'`); }; const stopped = stopSandbox(sandboxName, revalidateAtStopBoundary); - if (stopped.exitCode !== 0) { - throw new Error( - `Could not stop sandbox '${sandboxName}' to reconcile dashboard port ${String(port)}${ - stopped.message ? `: ${stopped.message}` : "." - }`, - ); - } - try { + const revalidateAtStartBoundary = (): void => { revalidateSandboxIdentity?.( `start sandbox '${sandboxName}' to reconcile dashboard forward ${String(port)}`, ); assertSameSandbox(`start sandbox '${sandboxName}'`); - } catch (error) { - const detail = error instanceof Error ? error.message : String(error); + }; + if (stopped.exitCode !== 0 && stopped.stopped !== true) { throw new Error( - `${detail} The selected sandbox was stopped; verify its identity, then run '${deps.cliName()} ${sandboxName} start'.`, + `Could not stop sandbox '${sandboxName}' to reconcile dashboard port ${String(port)}${ + stopped.message ? `: ${stopped.message}` : "." + }`, ); } @@ -497,7 +495,7 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa } let started: { exitCode: number; message?: string }; try { - started = await startSandbox(sandboxName); + started = await startSandbox(sandboxName, revalidateAtStartBoundary); } catch (error) { const detail = error instanceof Error ? error.message : String(error); throw new Error( @@ -511,6 +509,13 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa } The sandbox may remain stopped; run '${deps.cliName()} ${sandboxName} start' before retrying onboarding.`, ); } + if (stopped.exitCode !== 0) { + throw new Error( + `Sandbox '${sandboxName}' was restored, but its stop cleanup failed${ + stopped.message ? `: ${stopped.message}` : "." + }`, + ); + } reconciledOpenClawForwards.set(sandboxName, port); return true; }); @@ -678,7 +683,7 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa envUrl || (persistedPort === null ? undefined : `http://127.0.0.1:${String(persistedPort)}`); const mayReuseForward = reuseExistingOpenClawForward || reconciledOpenClawForwards.has(sandboxName); - if (mayReuseForward) { + if (reuseExistingOpenClawForward && !reconciledOpenClawForwards.has(sandboxName)) { await reconcileOpenClawDashboardForwardReuse( sandboxName, requestedUrl || `http://127.0.0.1:${CONTROL_UI_PORT}`, @@ -740,7 +745,11 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa const mayReuseOpenClawForward = agent.name === "openclaw" && (reuseExistingOpenClawForward || reconciledOpenClawForwards.has(sandboxName)); - if (mayReuseOpenClawForward) { + if ( + agent.name === "openclaw" && + reuseExistingOpenClawForward && + !reconciledOpenClawForwards.has(sandboxName) + ) { const registeredPort = getPersistedDashboardPort(sandboxName, listSandboxes); const requestedUrl = process.env.CHAT_UI_URL || diff --git a/src/lib/onboard/dashboard/reuse-lifecycle.ts b/src/lib/onboard/dashboard/reuse-lifecycle.ts index d97b9a39c84..83a66a84a63 100644 --- a/src/lib/onboard/dashboard/reuse-lifecycle.ts +++ b/src/lib/onboard/dashboard/reuse-lifecycle.ts @@ -7,8 +7,11 @@ export type DashboardReuseLifecycle = { stopSandbox( sandboxName: string, revalidateAtMutationEdge: () => void, - ): { exitCode: number; message?: string }; - startSandbox(sandboxName: string): Promise<{ exitCode: number; message?: string }>; + ): { exitCode: number; message?: string; stopped?: true }; + startSandbox( + sandboxName: string, + revalidateAtMutationEdge: () => void, + ): Promise<{ exitCode: number; message?: string }>; withSandboxLifecycleLock(sandboxName: string, operation: () => Promise | T): Promise; }; diff --git a/test/onboarding/onboard-finalization-dashboard-forward.test.ts b/test/onboarding/onboard-finalization-dashboard-forward.test.ts index 94fd36a4e0a..ff1caae399a 100644 --- a/test/onboarding/onboard-finalization-dashboard-forward.test.ts +++ b/test/onboarding/onboard-finalization-dashboard-forward.test.ts @@ -12,7 +12,11 @@ function harness(options: { isPortBound?: (port: number) => boolean; registeredIdentity?: boolean; sandboxIdentity?: () => string; - stopSandbox?: (sandboxName: string) => { exitCode: number; message?: string }; + stopSandbox?: (sandboxName: string) => { + exitCode: number; + message?: string; + stopped?: true; + }; startSandbox?: (sandboxName: string) => Promise<{ exitCode: number; message?: string }>; }) { const launch = vi.fn(); @@ -20,7 +24,11 @@ function harness(options: { revalidateAtMutationEdge(); return (options.stopSandbox ?? (() => ({ exitCode: 0 })))(sandboxName); }); - const startSandbox = vi.fn(options.startSandbox ?? (async () => ({ exitCode: 0 }))); + const startSandboxOperation = vi.fn(options.startSandbox ?? (async () => ({ exitCode: 0 }))); + const startSandbox = vi.fn(async (sandboxName: string, revalidateAtMutationEdge: () => void) => { + revalidateAtMutationEdge(); + return await startSandboxOperation(sandboxName); + }); const recordedIdentity = fingerprintSandboxLiveIdentity( `Id: ${options.sandboxIdentity?.() ?? "sandbox-id"}`, ); @@ -59,7 +67,7 @@ function harness(options: { retireLegacy: vi.fn(() => 0), }, }); - return { helpers, launch, startSandbox, stopSandbox }; + return { helpers, launch, startSandbox, startSandboxOperation, stopSandbox }; } describe("finalization dashboard ForwardTcp launch", () => { @@ -124,11 +132,39 @@ describe("finalization dashboard ForwardTcp launch", () => { helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true), ).resolves.toBe(18_790); expect(stopSandbox).toHaveBeenCalledWith("reonboard-test", expect.any(Function)); - expect(startSandbox).toHaveBeenCalledWith("reonboard-test"); + expect(startSandbox).toHaveBeenCalledWith("reonboard-test", expect.any(Function)); expect(launch).not.toHaveBeenCalled(); expect(process.env.CHAT_UI_URL).toBe("http://127.0.0.1:18790"); }); + it("does not reconcile a reused dashboard forward twice", async () => { + vi.stubEnv("CHAT_UI_URL", undefined); + let bound = true; + const { helpers, startSandbox, stopSandbox } = harness({ + listSandboxes: () => ({ + sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], + }), + isPortBound: (port) => port === 18_790 && bound, + stopSandbox: () => { + bound = false; + return { exitCode: 0 }; + }, + startSandbox: async () => { + bound = true; + return { exitCode: 0 }; + }, + }); + + await helpers.reconcileOpenClawDashboardForwardReuse( + "reonboard-test", + "http://127.0.0.1:18790", + ); + await helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true); + + expect(stopSandbox).toHaveBeenCalledOnce(); + expect(startSandbox).toHaveBeenCalledOnce(); + }); + it("rejects an ambiguous listener that remains after the reused sandbox stops", async () => { vi.stubEnv("CHAT_UI_URL", undefined); const { helpers, launch, startSandbox, stopSandbox } = harness({ @@ -150,7 +186,7 @@ describe("finalization dashboard ForwardTcp launch", () => { vi.stubEnv("CHAT_UI_URL", undefined); let bound = true; let identity = "original-id"; - const { helpers, launch, startSandbox, stopSandbox } = harness({ + const { helpers, launch, startSandbox, startSandboxOperation, stopSandbox } = harness({ listSandboxes: () => ({ sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], }), @@ -165,9 +201,10 @@ describe("finalization dashboard ForwardTcp launch", () => { await expect( helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true), - ).rejects.toThrow(/identity changed.*selected sandbox was stopped/u); + ).rejects.toThrow(/identity changed.*may remain stopped/u); expect(stopSandbox).toHaveBeenCalledWith("reonboard-test", expect.any(Function)); - expect(startSandbox).not.toHaveBeenCalled(); + expect(startSandbox).toHaveBeenCalledWith("reonboard-test", expect.any(Function)); + expect(startSandboxOperation).not.toHaveBeenCalled(); expect(launch).not.toHaveBeenCalled(); }); @@ -224,6 +261,32 @@ describe("finalization dashboard ForwardTcp launch", () => { expect(launch).not.toHaveBeenCalled(); }); + it("restores a sandbox after its stop cleanup fails", async () => { + vi.stubEnv("CHAT_UI_URL", undefined); + let bound = true; + const { helpers, launch, startSandbox, stopSandbox } = harness({ + listSandboxes: () => ({ + sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], + }), + isPortBound: (port) => port === 18_790 && bound, + stopSandbox: () => { + bound = false; + return { exitCode: 1, message: "Ollama cleanup failed", stopped: true }; + }, + startSandbox: async () => { + bound = true; + return { exitCode: 0 }; + }, + }); + + await expect( + helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true), + ).rejects.toThrow(/was restored.*Ollama cleanup failed/u); + expect(stopSandbox).toHaveBeenCalledOnce(); + expect(startSandbox).toHaveBeenCalledOnce(); + expect(launch).not.toHaveBeenCalled(); + }); + it("does not reuse a forward when another sandbox registers the same port", async () => { vi.stubEnv("CHAT_UI_URL", undefined); const { helpers, launch, stopSandbox } = harness({ From 0ee6c0779279d69f71b48789c58185c54d0d73fb Mon Sep 17 00:00:00 2001 From: San Dang Date: Mon, 7 Sep 2026 04:37:10 +0700 Subject: [PATCH 08/20] fix(onboard): scope dashboard reuse evidence --- src/lib/onboard/dashboard.ts | 27 +++--- .../onboard/dashboard/reuse-lifecycle.test.ts | 10 +- src/lib/onboard/dashboard/reuse-lifecycle.ts | 15 ++- test/e2e/live/double-onboard.test.ts | 9 +- ...ard-finalization-dashboard-forward.test.ts | 95 ++++++++++++++----- 5 files changed, 116 insertions(+), 40 deletions(-) diff --git a/src/lib/onboard/dashboard.ts b/src/lib/onboard/dashboard.ts index 4d2a7dceaf2..b8bd7918bdd 100644 --- a/src/lib/onboard/dashboard.ts +++ b/src/lib/onboard/dashboard.ts @@ -21,7 +21,10 @@ import { } from "./agent-dashboard-forward"; import { fetchAgentWebAuthTokenFromSandbox as fetchAgentWebAuthToken } from "./agent-web-auth-token"; import * as dashboardAccess from "./dashboard-access"; -import { getDashboardReuseLifecycle } from "./dashboard/reuse-lifecycle"; +import { + getDashboardReuseLifecycle, + getDashboardReuseReconciledForwards, +} from "./dashboard/reuse-lifecycle"; import { type DashboardForwardOptions, normalizeDashboardForwardOptions, @@ -267,8 +270,6 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa }), } : undefined); - const reconciledOpenClawForwards = new Map(); - function resolveForwardServiceGateway( sandboxName: string, options: DashboardForwardOptions = {}, @@ -406,6 +407,7 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa const isPortBound = deps.isPortBoundOnHost ?? isPortBoundOnHost; if (!isPortBound(port)) return false; const lifecycle = getDashboardReuseLifecycle(); + const reconciledOpenClawForwards = getDashboardReuseReconciledForwards(); const withLifecycleLock = deps.withSandboxLifecycleLock ?? lifecycle?.withSandboxLifecycleLock; if (!withLifecycleLock) { throw new Error( @@ -413,7 +415,7 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa ); } return await withLifecycleLock(sandboxName, async () => { - if (reconciledOpenClawForwards.get(sandboxName) === port && isPortBound(port)) return true; + if (reconciledOpenClawForwards?.get(sandboxName) === port && isPortBound(port)) return true; if (!isPortBound(port)) return false; if (getRegistryOccupiedDashboardPorts(sandboxName, listSandboxes).has(String(port))) { throw new Error( @@ -516,7 +518,7 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa }`, ); } - reconciledOpenClawForwards.set(sandboxName, port); + reconciledOpenClawForwards?.set(sandboxName, port); return true; }); } @@ -543,10 +545,11 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa const isPortBound = deps.isPortBoundOnHost ?? isPortBoundOnHost; const persistedPort = getPersistedDashboardPort(sandboxName, listSandboxes); const registryOccupiedPorts = getRegistryOccupiedDashboardPorts(sandboxName, listSandboxes); + const reconciledOpenClawForwards = getDashboardReuseReconciledForwards(); if (persistedPort === preferredPort && isPortBound(preferredPort)) { if ( reuseExistingOpenClawForward && - reconciledOpenClawForwards.get(sandboxName) === preferredPort && + reconciledOpenClawForwards?.get(sandboxName) === preferredPort && !registryOccupiedPorts.has(String(preferredPort)) ) { revalidateSandboxIdentity?.( @@ -654,7 +657,7 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa }); } if (fwdOk && reuseExistingOpenClawForward) { - reconciledOpenClawForwards.set(sandboxName, actualPort); + reconciledOpenClawForwards?.set(sandboxName, actualPort); } return actualPort; } @@ -681,9 +684,10 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa const persistedPort = envUrl ? null : getPersistedDashboardPort(sandboxName, listSandboxes); const requestedUrl = envUrl || (persistedPort === null ? undefined : `http://127.0.0.1:${String(persistedPort)}`); + const reconciledOpenClawForwards = getDashboardReuseReconciledForwards(); const mayReuseForward = - reuseExistingOpenClawForward || reconciledOpenClawForwards.has(sandboxName); - if (reuseExistingOpenClawForward && !reconciledOpenClawForwards.has(sandboxName)) { + reuseExistingOpenClawForward || reconciledOpenClawForwards?.has(sandboxName) === true; + if (reuseExistingOpenClawForward && !reconciledOpenClawForwards?.has(sandboxName)) { await reconcileOpenClawDashboardForwardReuse( sandboxName, requestedUrl || `http://127.0.0.1:${CONTROL_UI_PORT}`, @@ -742,13 +746,14 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa reuseExistingOpenClawForward, ); } + const reconciledOpenClawForwards = getDashboardReuseReconciledForwards(); const mayReuseOpenClawForward = agent.name === "openclaw" && - (reuseExistingOpenClawForward || reconciledOpenClawForwards.has(sandboxName)); + (reuseExistingOpenClawForward || reconciledOpenClawForwards?.has(sandboxName) === true); if ( agent.name === "openclaw" && reuseExistingOpenClawForward && - !reconciledOpenClawForwards.has(sandboxName) + !reconciledOpenClawForwards?.has(sandboxName) ) { const registeredPort = getPersistedDashboardPort(sandboxName, listSandboxes); const requestedUrl = diff --git a/src/lib/onboard/dashboard/reuse-lifecycle.test.ts b/src/lib/onboard/dashboard/reuse-lifecycle.test.ts index 4a396963ecf..c92fe8110be 100644 --- a/src/lib/onboard/dashboard/reuse-lifecycle.test.ts +++ b/src/lib/onboard/dashboard/reuse-lifecycle.test.ts @@ -3,7 +3,11 @@ import { expect, it, vi } from "vitest"; -import { getDashboardReuseLifecycle, withDashboardReuseLifecycle } from "./reuse-lifecycle"; +import { + getDashboardReuseLifecycle, + getDashboardReuseReconciledForwards, + withDashboardReuseLifecycle, +} from "./reuse-lifecycle"; it("keeps overlapping onboarding lifecycle scopes independent", async () => { const first = { @@ -23,14 +27,18 @@ it("keeps overlapping onboarding lifecycle scopes independent", async () => { const firstOperation = withDashboardReuseLifecycle(first, async () => { expect(getDashboardReuseLifecycle()).toBe(first); + getDashboardReuseReconciledForwards()?.set("alpha", 18_789); await firstPaused; expect(getDashboardReuseLifecycle()).toBe(first); + expect(getDashboardReuseReconciledForwards()?.get("alpha")).toBe(18_789); }); await withDashboardReuseLifecycle(second, async () => { expect(getDashboardReuseLifecycle()).toBe(second); + expect(getDashboardReuseReconciledForwards()?.has("alpha")).toBe(false); }); releaseFirst(); await firstOperation; expect(getDashboardReuseLifecycle()).toBeUndefined(); + expect(getDashboardReuseReconciledForwards()).toBeUndefined(); }); diff --git a/src/lib/onboard/dashboard/reuse-lifecycle.ts b/src/lib/onboard/dashboard/reuse-lifecycle.ts index 83a66a84a63..3a9803e82c1 100644 --- a/src/lib/onboard/dashboard/reuse-lifecycle.ts +++ b/src/lib/onboard/dashboard/reuse-lifecycle.ts @@ -15,15 +15,24 @@ export type DashboardReuseLifecycle = { withSandboxLifecycleLock(sandboxName: string, operation: () => Promise | T): Promise; }; -const lifecycleStorage = new AsyncLocalStorage(); +type DashboardReuseTransaction = { + lifecycle: DashboardReuseLifecycle; + reconciledForwards: Map; +}; + +const lifecycleStorage = new AsyncLocalStorage(); export function getDashboardReuseLifecycle(): DashboardReuseLifecycle | undefined { - return lifecycleStorage.getStore(); + return lifecycleStorage.getStore()?.lifecycle; +} + +export function getDashboardReuseReconciledForwards(): Map | undefined { + return lifecycleStorage.getStore()?.reconciledForwards; } export function withDashboardReuseLifecycle( lifecycle: DashboardReuseLifecycle, operation: () => Promise, ): Promise { - return lifecycleStorage.run(lifecycle, operation); + return lifecycleStorage.run({ lifecycle, reconciledForwards: new Map() }, operation); } diff --git a/test/e2e/live/double-onboard.test.ts b/test/e2e/live/double-onboard.test.ts index c7e8f55eb2f..e9e2f9244d9 100644 --- a/test/e2e/live/double-onboard.test.ts +++ b/test/e2e/live/double-onboard.test.ts @@ -518,6 +518,13 @@ test( expect(sandboxAIdAfterFirst, resultText(sandboxAAfterFirst)).not.toBeNull(); expect(registryHas(SANDBOX_A), `${REGISTRY_FILE} missing ${SANDBOX_A}`).toBe(true); assertRegistryInferenceMetadata(SANDBOX_A, fake.baseUrl); + const listAfterFirst = await command(host, ["list"], { + artifactName: "phase-2-nemoclaw-list", + env: commandEnv(), + timeoutMs: 60_000, + }); + const portAfterFirst = + dashboardPortFromList(listAfterFirst.stdout, SANDBOX_A) ?? ""; progress.phase("re-onboard same sandbox on existing gateway"); // Phase 3: second onboard with the same name must reuse the healthy gateway. @@ -553,7 +560,7 @@ test( expect(listAfterSecond.exitCode, resultText(listAfterSecond)).toBe(0); expect(stripAnsi(listAfterSecond.stdout)).toContain(SANDBOX_A); const portAfterSecond = dashboardPortFromList(listAfterSecond.stdout, SANDBOX_A); - expect(portAfterSecond, resultText(listAfterSecond)).toBeTruthy(); + expect(portAfterSecond, resultText(listAfterSecond)).toBe(portAfterFirst); const dashboardAfterSecond = await waitForDashboardReachability( host, portAfterSecond ?? "", diff --git a/test/onboarding/onboard-finalization-dashboard-forward.test.ts b/test/onboarding/onboard-finalization-dashboard-forward.test.ts index ff1caae399a..3fd739a47c7 100644 --- a/test/onboarding/onboard-finalization-dashboard-forward.test.ts +++ b/test/onboarding/onboard-finalization-dashboard-forward.test.ts @@ -4,6 +4,7 @@ import { describe, expect, it, vi } from "vitest"; import { createOnboardDashboardHelpers } from "../../src/lib/onboard/dashboard"; +import { withDashboardReuseLifecycle } from "../../src/lib/onboard/dashboard/reuse-lifecycle"; import type { ListSandboxesFn } from "../../src/lib/onboard/dashboard-port"; import { fingerprintSandboxLiveIdentity } from "../../src/lib/onboard/sandbox-recreate-transaction"; @@ -67,7 +68,16 @@ function harness(options: { retireLegacy: vi.fn(() => 0), }, }); - return { helpers, launch, startSandbox, startSandboxOperation, stopSandbox }; + const run = (operation: () => Promise): Promise => + withDashboardReuseLifecycle( + { + startSandbox, + stopSandbox, + withSandboxLifecycleLock: async (_sandboxName, lockedOperation) => await lockedOperation(), + }, + operation, + ); + return { helpers, launch, run, startSandbox, startSandboxOperation, stopSandbox }; } describe("finalization dashboard ForwardTcp launch", () => { @@ -113,7 +123,7 @@ describe("finalization dashboard ForwardTcp launch", () => { it("restarts the reused sandbox and retains its registered dashboard port (#11074)", async () => { vi.stubEnv("CHAT_UI_URL", undefined); let bound = true; - const { helpers, launch, startSandbox, stopSandbox } = harness({ + const { helpers, launch, run, startSandbox, stopSandbox } = harness({ listSandboxes: () => ({ sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], }), @@ -129,7 +139,7 @@ describe("finalization dashboard ForwardTcp launch", () => { }); await expect( - helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true), + run(() => helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true)), ).resolves.toBe(18_790); expect(stopSandbox).toHaveBeenCalledWith("reonboard-test", expect.any(Function)); expect(startSandbox).toHaveBeenCalledWith("reonboard-test", expect.any(Function)); @@ -140,7 +150,7 @@ describe("finalization dashboard ForwardTcp launch", () => { it("does not reconcile a reused dashboard forward twice", async () => { vi.stubEnv("CHAT_UI_URL", undefined); let bound = true; - const { helpers, startSandbox, stopSandbox } = harness({ + const { helpers, run, startSandbox, stopSandbox } = harness({ listSandboxes: () => ({ sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], }), @@ -155,19 +165,54 @@ describe("finalization dashboard ForwardTcp launch", () => { }, }); - await helpers.reconcileOpenClawDashboardForwardReuse( - "reonboard-test", - "http://127.0.0.1:18790", - ); - await helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true); + await run(async () => { + await helpers.reconcileOpenClawDashboardForwardReuse( + "reonboard-test", + "http://127.0.0.1:18790", + ); + await helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true); + }); expect(stopSandbox).toHaveBeenCalledOnce(); expect(startSandbox).toHaveBeenCalledOnce(); }); + it("does not trust reconciliation from an earlier onboarding transaction", async () => { + vi.stubEnv("CHAT_UI_URL", undefined); + let bound = true; + let stopEffect = (): void => { + bound = false; + }; + const { helpers, run, startSandbox, stopSandbox } = harness({ + listSandboxes: () => ({ + sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], + }), + isPortBound: (port) => port === 18_790 && bound, + stopSandbox: () => { + stopEffect(); + return { exitCode: 0 }; + }, + startSandbox: async () => { + bound = true; + return { exitCode: 0 }; + }, + }); + + await run(() => + helpers.reconcileOpenClawDashboardForwardReuse("reonboard-test", "http://127.0.0.1:18790"), + ); + stopEffect = () => undefined; + + await expect( + run(() => helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true)), + ).rejects.toThrow(/remained occupied/u); + expect(stopSandbox).toHaveBeenCalledTimes(2); + expect(startSandbox).toHaveBeenCalledOnce(); + }); + it("rejects an ambiguous listener that remains after the reused sandbox stops", async () => { vi.stubEnv("CHAT_UI_URL", undefined); - const { helpers, launch, startSandbox, stopSandbox } = harness({ + const { helpers, launch, run, startSandbox, stopSandbox } = harness({ listSandboxes: () => ({ sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], }), @@ -175,7 +220,7 @@ describe("finalization dashboard ForwardTcp launch", () => { }); await expect( - helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true), + run(() => helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true)), ).rejects.toThrow(/remained occupied.*run 'nemoclaw reonboard-test start'/u); expect(stopSandbox).toHaveBeenCalledWith("reonboard-test", expect.any(Function)); expect(startSandbox).not.toHaveBeenCalled(); @@ -210,7 +255,7 @@ describe("finalization dashboard ForwardTcp launch", () => { it("does not restart a reused sandbox without a registered live identity", async () => { vi.stubEnv("CHAT_UI_URL", undefined); - const { helpers, launch, startSandbox, stopSandbox } = harness({ + const { helpers, launch, run, startSandbox, stopSandbox } = harness({ listSandboxes: () => ({ sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], }), @@ -219,7 +264,7 @@ describe("finalization dashboard ForwardTcp launch", () => { }); await expect( - helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true), + run(() => helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true)), ).rejects.toThrow(/Could not verify sandbox/u); expect(stopSandbox).not.toHaveBeenCalled(); expect(startSandbox).not.toHaveBeenCalled(); @@ -236,7 +281,7 @@ describe("finalization dashboard ForwardTcp launch", () => { return { exitCode: 0 }; }, ]; - const { helpers, launch, startSandbox, stopSandbox } = harness({ + const { helpers, launch, run, startSandbox, stopSandbox } = harness({ listSandboxes: () => ({ sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], }), @@ -249,12 +294,12 @@ describe("finalization dashboard ForwardTcp launch", () => { }); await expect( - helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true), + run(() => helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true)), ).rejects.toThrow(/did not restore dashboard port.*restart failed/u); bound = true; await expect( - helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true), + run(() => helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true)), ).resolves.toBe(18_790); expect(stopSandbox).toHaveBeenCalledTimes(2); expect(startSandbox).toHaveBeenCalledTimes(2); @@ -264,7 +309,7 @@ describe("finalization dashboard ForwardTcp launch", () => { it("restores a sandbox after its stop cleanup fails", async () => { vi.stubEnv("CHAT_UI_URL", undefined); let bound = true; - const { helpers, launch, startSandbox, stopSandbox } = harness({ + const { helpers, launch, run, startSandbox, stopSandbox } = harness({ listSandboxes: () => ({ sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], }), @@ -280,7 +325,7 @@ describe("finalization dashboard ForwardTcp launch", () => { }); await expect( - helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true), + run(() => helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true)), ).rejects.toThrow(/was restored.*Ollama cleanup failed/u); expect(stopSandbox).toHaveBeenCalledOnce(); expect(startSandbox).toHaveBeenCalledOnce(); @@ -325,12 +370,14 @@ describe("finalization dashboard ForwardTcp launch", () => { }); await expect( - openClaw.helpers.ensureFinalizationAgentDashboardForward( - "reonboard-test", - { name: "openclaw", forwardPort: 18_790 }, - undefined, - undefined, - true, + openClaw.run(() => + openClaw.helpers.ensureFinalizationAgentDashboardForward( + "reonboard-test", + { name: "openclaw", forwardPort: 18_790 }, + undefined, + undefined, + true, + ), ), ).resolves.toBe(18_790); From 2157f010b16225999ab719aa99c4685a7455b3cc Mon Sep 17 00:00:00 2001 From: San Dang Date: Mon, 7 Sep 2026 05:54:14 +0700 Subject: [PATCH 09/20] fix(onboard): verify owned dashboard forward --- docs/manage-sandboxes/run-sandboxes.mdx | 3 +- .../openshell/forward-service.test.ts | 34 ++++ src/lib/adapters/openshell/forward-service.ts | 40 ++++- src/lib/onboard/dashboard.ts | 62 +++---- .../onboard/dashboard/reuse-lifecycle.test.ts | 10 +- src/lib/onboard/dashboard/reuse-lifecycle.ts | 15 +- ...ard-finalization-dashboard-forward.test.ts | 151 +++++++----------- 7 files changed, 169 insertions(+), 146 deletions(-) diff --git a/docs/manage-sandboxes/run-sandboxes.mdx b/docs/manage-sandboxes/run-sandboxes.mdx index 71d3be2f2f9..584711c485a 100644 --- a/docs/manage-sandboxes/run-sandboxes.mdx +++ b/docs/manage-sandboxes/run-sandboxes.mdx @@ -66,7 +66,8 @@ A later onboarding run for another sandbox does not tear down the first sandbox' -If re-onboarding finds the registered dashboard port already bound, NemoClaw briefly stops and starts the same ready OpenClaw sandbox to verify its dashboard forward. +If re-onboarding finds the registered dashboard port already bound, NemoClaw verifies that the listener is the exact OpenShell forward for that sandbox and reuses it without restarting the sandbox. +If ownership cannot be proved, NemoClaw briefly stops the same ready sandbox to distinguish its forward from a foreign listener. If the port remains bound after the sandbox stops, resolve the listener conflict, run `nemoclaw start`, then retry onboarding. For other failures, follow the reported recovery steps. diff --git a/src/lib/adapters/openshell/forward-service.test.ts b/src/lib/adapters/openshell/forward-service.test.ts index 2de4e61ca73..f1f5789f240 100644 --- a/src/lib/adapters/openshell/forward-service.test.ts +++ b/src/lib/adapters/openshell/forward-service.test.ts @@ -5,6 +5,7 @@ import { describe, expect, it, vi } from "vitest"; import { buildForwardServiceArgs, + isForwardServiceListenerOwner, launchForwardService, type ForwardServiceTarget, } from "./forward-service"; @@ -45,6 +46,39 @@ describe("OpenShell forward service", () => { ); }); + it("proves the exact direct ForwardTcp listener before reuse", () => { + const expected = [target.executable, ...buildForwardServiceArgs(target)].join(" "); + const probe = vi.fn((executable: string) => + executable === "lsof" + ? { status: 0, stdout: "4321\n" } + : { status: 0, stdout: `${expected}\n` }, + ); + + expect(isForwardServiceListenerOwner(target, { probe })).toBe(true); + expect(probe).toHaveBeenCalledTimes(3); + }); + + it("rejects a listener whose process does not match the direct ForwardTcp target", () => { + const probe = vi.fn((executable: string) => + executable === "lsof" + ? { status: 0, stdout: "4321\n" } + : { status: 0, stdout: "/usr/bin/node foreign-listener.js\n" }, + ); + + expect(isForwardServiceListenerOwner(target, { probe })).toBe(false); + }); + + it("rejects ambiguous or changing listener ownership", () => { + const expected = [target.executable, ...buildForwardServiceArgs(target)].join(" "); + const probe = vi + .fn() + .mockReturnValueOnce({ status: 0, stdout: "4321\n" }) + .mockReturnValueOnce({ status: 0, stdout: `${expected}\n` }) + .mockReturnValueOnce({ status: 0, stdout: "9876\n" }); + + expect(isForwardServiceListenerOwner(target, { probe })).toBe(false); + }); + it("detaches the OpenShell child and waits for its local port", () => { const unref = vi.fn(); const spawnDetached = vi.fn(() => ({ unref })); diff --git a/src/lib/adapters/openshell/forward-service.ts b/src/lib/adapters/openshell/forward-service.ts index 8f78fe81983..9a2a492122a 100644 --- a/src/lib/adapters/openshell/forward-service.ts +++ b/src/lib/adapters/openshell/forward-service.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { spawn } from "node:child_process"; +import { spawn, spawnSync } from "node:child_process"; import path from "node:path"; import { isValidName } from "../../name-validation"; @@ -37,6 +37,15 @@ export interface ForwardServiceLaunchOptions { readonly timeoutMs?: number; } +type ForwardServiceOwnerProbe = ( + executable: string, + args: readonly string[], +) => { status: number | null; stdout: string }; + +export interface ForwardServiceOwnerOptions { + readonly probe?: ForwardServiceOwnerProbe; +} + function isPort(value: unknown): value is number { return Number.isSafeInteger(value) && Number(value) >= 1 && Number(value) <= 65_535; } @@ -94,6 +103,35 @@ export function buildForwardServiceArgs(target: ForwardServiceTarget): string[] ]; } +function captureProcess(executable: string, args: readonly string[]) { + const result = spawnSync(executable, [...args], { encoding: "utf8" }); + return { status: result.status, stdout: result.stdout ?? "" }; +} + +function listenerPids(port: number, probe: ForwardServiceOwnerProbe): string[] { + const result = probe("lsof", ["-ti", `:${String(port)}`, "-sTCP:LISTEN"]); + if (result.status !== 0) return []; + return [...new Set(result.stdout.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean))]; +} + +/** Prove that the current listener is the exact direct ForwardTcp command. */ +export function isForwardServiceListenerOwner( + target: ForwardServiceTarget, + options: ForwardServiceOwnerOptions = {}, +): boolean { + validateForwardServiceTarget(target); + const probe = options.probe ?? captureProcess; + const before = listenerPids(target.localPort, probe); + if (before.length !== 1 || !/^[1-9]\d*$/u.test(before[0]!)) return false; + const pid = before[0]!; + const process = probe("ps", ["-ww", "-p", pid, "-o", "args="]); + if (process.status !== 0) return false; + const expected = [target.executable, ...buildForwardServiceArgs(target)].join(" "); + if (process.stdout.trim() !== expected) return false; + const after = listenerPids(target.localPort, probe); + return after.length === 1 && after[0] === pid; +} + /** Launch one foreground OpenShell service forward as a detached host child. */ export function launchForwardService( target: ForwardServiceTarget, diff --git a/src/lib/onboard/dashboard.ts b/src/lib/onboard/dashboard.ts index b8bd7918bdd..611fc0db63c 100644 --- a/src/lib/onboard/dashboard.ts +++ b/src/lib/onboard/dashboard.ts @@ -5,6 +5,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { + isForwardServiceListenerOwner, launchForwardService, type ForwardServiceTarget, } from "../adapters/openshell/forward-service"; @@ -21,10 +22,7 @@ import { } from "./agent-dashboard-forward"; import { fetchAgentWebAuthTokenFromSandbox as fetchAgentWebAuthToken } from "./agent-web-auth-token"; import * as dashboardAccess from "./dashboard-access"; -import { - getDashboardReuseLifecycle, - getDashboardReuseReconciledForwards, -} from "./dashboard/reuse-lifecycle"; +import { getDashboardReuseLifecycle } from "./dashboard/reuse-lifecycle"; import { type DashboardForwardOptions, normalizeDashboardForwardOptions, @@ -94,6 +92,7 @@ export interface OnboardDashboardDeps { /** Direct ForwardTcp launcher. */ forwardService?: { executable(): string; + owns?(target: ForwardServiceTarget): boolean; launch?(target: ForwardServiceTarget): void; retireLegacy?(sandboxName: string, gatewayName: string, ports: readonly number[]): number; resolveGatewayName( @@ -250,6 +249,7 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa if (!executable) throw new Error("OpenShell is unavailable"); return executable; }, + owns: isForwardServiceListenerOwner, resolveGatewayName: productionForwardService.resolveGatewayName, retireLegacy: (sandboxName: string, gatewayName: string, ports: readonly number[]) => productionForwardService.retireLegacy(sandboxName, gatewayName, ports, { @@ -298,6 +298,24 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa }; } + function ownsDashboardForward( + sandboxName: string, + gatewayName: string, + port: number, + chatUiUrl: string, + ): boolean { + return ( + forwardService?.owns?.( + forwardTarget( + sandboxName, + gatewayName, + port, + getDashboardForwardTarget(chatUiUrl), + ), + ) === true + ); + } + function getDashboardForwardPort( chatUiUrl = process.env.CHAT_UI_URL || `http://127.0.0.1:${CONTROL_UI_PORT}`, options: Parameters[1] = {}, @@ -407,7 +425,6 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa const isPortBound = deps.isPortBoundOnHost ?? isPortBoundOnHost; if (!isPortBound(port)) return false; const lifecycle = getDashboardReuseLifecycle(); - const reconciledOpenClawForwards = getDashboardReuseReconciledForwards(); const withLifecycleLock = deps.withSandboxLifecycleLock ?? lifecycle?.withSandboxLifecycleLock; if (!withLifecycleLock) { throw new Error( @@ -415,7 +432,6 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa ); } return await withLifecycleLock(sandboxName, async () => { - if (reconciledOpenClawForwards?.get(sandboxName) === port && isPortBound(port)) return true; if (!isPortBound(port)) return false; if (getRegistryOccupiedDashboardPorts(sandboxName, listSandboxes).has(String(port))) { throw new Error( @@ -451,6 +467,7 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa `Could not verify sandbox '${sandboxName}' before reconciling dashboard port ${String(port)}.`, ); } + if (ownsDashboardForward(sandboxName, gatewayName, port, chatUiUrl)) return true; const assertSameSandbox = (operation: string): void => { const current = getSandbox?.(sandboxName); const currentGateway = current ? forwardService?.resolveGatewayName(current) : null; @@ -504,7 +521,11 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa `Could not restart sandbox '${sandboxName}' after releasing dashboard port ${String(port)}: ${detail}. The sandbox may remain stopped; run '${deps.cliName()} ${sandboxName} start' before retrying onboarding.`, ); } - if (started.exitCode !== 0 || !isPortBound(port)) { + if ( + started.exitCode !== 0 || + !isPortBound(port) || + !ownsDashboardForward(sandboxName, gatewayName, port, chatUiUrl) + ) { throw new Error( `Sandbox '${sandboxName}' did not restore dashboard port ${String(port)} after restart${ started.message ? `: ${started.message}` : "." @@ -518,7 +539,6 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa }`, ); } - reconciledOpenClawForwards?.set(sandboxName, port); return true; }); } @@ -545,17 +565,15 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa const isPortBound = deps.isPortBoundOnHost ?? isPortBoundOnHost; const persistedPort = getPersistedDashboardPort(sandboxName, listSandboxes); const registryOccupiedPorts = getRegistryOccupiedDashboardPorts(sandboxName, listSandboxes); - const reconciledOpenClawForwards = getDashboardReuseReconciledForwards(); if (persistedPort === preferredPort && isPortBound(preferredPort)) { if ( reuseExistingOpenClawForward && - reconciledOpenClawForwards?.get(sandboxName) === preferredPort && - !registryOccupiedPorts.has(String(preferredPort)) + !registryOccupiedPorts.has(String(preferredPort)) && + ownsDashboardForward(sandboxName, forwardGateway, preferredPort, chatUiUrl) ) { revalidateSandboxIdentity?.( `retain dashboard forward ${String(preferredPort)} for sandbox '${sandboxName}'`, ); - reconciledOpenClawForwards.delete(sandboxName); return preferredPort; } throw new Error( @@ -656,9 +674,6 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa }, }); } - if (fwdOk && reuseExistingOpenClawForward) { - reconciledOpenClawForwards?.set(sandboxName, actualPort); - } return actualPort; } @@ -684,10 +699,7 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa const persistedPort = envUrl ? null : getPersistedDashboardPort(sandboxName, listSandboxes); const requestedUrl = envUrl || (persistedPort === null ? undefined : `http://127.0.0.1:${String(persistedPort)}`); - const reconciledOpenClawForwards = getDashboardReuseReconciledForwards(); - const mayReuseForward = - reuseExistingOpenClawForward || reconciledOpenClawForwards?.has(sandboxName) === true; - if (reuseExistingOpenClawForward && !reconciledOpenClawForwards?.has(sandboxName)) { + if (reuseExistingOpenClawForward) { await reconcileOpenClawDashboardForwardReuse( sandboxName, requestedUrl || `http://127.0.0.1:${CONTROL_UI_PORT}`, @@ -696,7 +708,7 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa } const actualPort = ensureDashboardForward(sandboxName, requestedUrl, { allowPortReallocation: false, - ...(mayReuseForward ? { reuseExistingOpenClawForward: true } : {}), + ...(reuseExistingOpenClawForward ? { reuseExistingOpenClawForward: true } : {}), ...(revalidateSandboxIdentity ? { revalidateSandboxIdentity } : {}), }); revalidateSandboxIdentity?.(`publish the dashboard URL for sandbox '${sandboxName}'`); @@ -746,15 +758,9 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa reuseExistingOpenClawForward, ); } - const reconciledOpenClawForwards = getDashboardReuseReconciledForwards(); const mayReuseOpenClawForward = - agent.name === "openclaw" && - (reuseExistingOpenClawForward || reconciledOpenClawForwards?.has(sandboxName) === true); - if ( - agent.name === "openclaw" && - reuseExistingOpenClawForward && - !reconciledOpenClawForwards?.has(sandboxName) - ) { + agent.name === "openclaw" && reuseExistingOpenClawForward; + if (mayReuseOpenClawForward) { const registeredPort = getPersistedDashboardPort(sandboxName, listSandboxes); const requestedUrl = process.env.CHAT_UI_URL || diff --git a/src/lib/onboard/dashboard/reuse-lifecycle.test.ts b/src/lib/onboard/dashboard/reuse-lifecycle.test.ts index c92fe8110be..4a396963ecf 100644 --- a/src/lib/onboard/dashboard/reuse-lifecycle.test.ts +++ b/src/lib/onboard/dashboard/reuse-lifecycle.test.ts @@ -3,11 +3,7 @@ import { expect, it, vi } from "vitest"; -import { - getDashboardReuseLifecycle, - getDashboardReuseReconciledForwards, - withDashboardReuseLifecycle, -} from "./reuse-lifecycle"; +import { getDashboardReuseLifecycle, withDashboardReuseLifecycle } from "./reuse-lifecycle"; it("keeps overlapping onboarding lifecycle scopes independent", async () => { const first = { @@ -27,18 +23,14 @@ it("keeps overlapping onboarding lifecycle scopes independent", async () => { const firstOperation = withDashboardReuseLifecycle(first, async () => { expect(getDashboardReuseLifecycle()).toBe(first); - getDashboardReuseReconciledForwards()?.set("alpha", 18_789); await firstPaused; expect(getDashboardReuseLifecycle()).toBe(first); - expect(getDashboardReuseReconciledForwards()?.get("alpha")).toBe(18_789); }); await withDashboardReuseLifecycle(second, async () => { expect(getDashboardReuseLifecycle()).toBe(second); - expect(getDashboardReuseReconciledForwards()?.has("alpha")).toBe(false); }); releaseFirst(); await firstOperation; expect(getDashboardReuseLifecycle()).toBeUndefined(); - expect(getDashboardReuseReconciledForwards()).toBeUndefined(); }); diff --git a/src/lib/onboard/dashboard/reuse-lifecycle.ts b/src/lib/onboard/dashboard/reuse-lifecycle.ts index 3a9803e82c1..83a66a84a63 100644 --- a/src/lib/onboard/dashboard/reuse-lifecycle.ts +++ b/src/lib/onboard/dashboard/reuse-lifecycle.ts @@ -15,24 +15,15 @@ export type DashboardReuseLifecycle = { withSandboxLifecycleLock(sandboxName: string, operation: () => Promise | T): Promise; }; -type DashboardReuseTransaction = { - lifecycle: DashboardReuseLifecycle; - reconciledForwards: Map; -}; - -const lifecycleStorage = new AsyncLocalStorage(); +const lifecycleStorage = new AsyncLocalStorage(); export function getDashboardReuseLifecycle(): DashboardReuseLifecycle | undefined { - return lifecycleStorage.getStore()?.lifecycle; -} - -export function getDashboardReuseReconciledForwards(): Map | undefined { - return lifecycleStorage.getStore()?.reconciledForwards; + return lifecycleStorage.getStore(); } export function withDashboardReuseLifecycle( lifecycle: DashboardReuseLifecycle, operation: () => Promise, ): Promise { - return lifecycleStorage.run({ lifecycle, reconciledForwards: new Map() }, operation); + return lifecycleStorage.run(lifecycle, operation); } diff --git a/test/onboarding/onboard-finalization-dashboard-forward.test.ts b/test/onboarding/onboard-finalization-dashboard-forward.test.ts index 3fd739a47c7..09e788a21aa 100644 --- a/test/onboarding/onboard-finalization-dashboard-forward.test.ts +++ b/test/onboarding/onboard-finalization-dashboard-forward.test.ts @@ -4,13 +4,13 @@ import { describe, expect, it, vi } from "vitest"; import { createOnboardDashboardHelpers } from "../../src/lib/onboard/dashboard"; -import { withDashboardReuseLifecycle } from "../../src/lib/onboard/dashboard/reuse-lifecycle"; import type { ListSandboxesFn } from "../../src/lib/onboard/dashboard-port"; import { fingerprintSandboxLiveIdentity } from "../../src/lib/onboard/sandbox-recreate-transaction"; function harness(options: { listSandboxes: ListSandboxesFn; isPortBound?: (port: number) => boolean; + ownsForward?: () => boolean; registeredIdentity?: boolean; sandboxIdentity?: () => string; stopSandbox?: (sandboxName: string) => { @@ -64,20 +64,12 @@ function harness(options: { forwardService: { executable: () => "/usr/local/bin/openshell", launch, + owns: vi.fn(options.ownsForward ?? (() => false)), resolveGatewayName: () => "nemoclaw", retireLegacy: vi.fn(() => 0), }, }); - const run = (operation: () => Promise): Promise => - withDashboardReuseLifecycle( - { - startSandbox, - stopSandbox, - withSandboxLifecycleLock: async (_sandboxName, lockedOperation) => await lockedOperation(), - }, - operation, - ); - return { helpers, launch, run, startSandbox, startSandboxOperation, stopSandbox }; + return { helpers, launch, startSandbox, startSandboxOperation, stopSandbox }; } describe("finalization dashboard ForwardTcp launch", () => { @@ -120,99 +112,72 @@ describe("finalization dashboard ForwardTcp launch", () => { expect(startSandbox).not.toHaveBeenCalled(); }); - it("restarts the reused sandbox and retains its registered dashboard port (#11074)", async () => { + it("reuses an owned dashboard forward without restarting the sandbox (#11074)", async () => { vi.stubEnv("CHAT_UI_URL", undefined); - let bound = true; - const { helpers, launch, run, startSandbox, stopSandbox } = harness({ + const { helpers, launch, startSandbox, stopSandbox } = harness({ listSandboxes: () => ({ sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], }), - isPortBound: (port) => port === 18_790 && bound, - stopSandbox: () => { - bound = false; - return { exitCode: 0 }; - }, - startSandbox: async () => { - bound = true; - return { exitCode: 0 }; - }, + isPortBound: (port) => port === 18_790, + ownsForward: () => true, }); await expect( - run(() => helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true)), + helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true), ).resolves.toBe(18_790); - expect(stopSandbox).toHaveBeenCalledWith("reonboard-test", expect.any(Function)); - expect(startSandbox).toHaveBeenCalledWith("reonboard-test", expect.any(Function)); + expect(stopSandbox).not.toHaveBeenCalled(); + expect(startSandbox).not.toHaveBeenCalled(); expect(launch).not.toHaveBeenCalled(); expect(process.env.CHAT_UI_URL).toBe("http://127.0.0.1:18790"); }); it("does not reconcile a reused dashboard forward twice", async () => { vi.stubEnv("CHAT_UI_URL", undefined); - let bound = true; - const { helpers, run, startSandbox, stopSandbox } = harness({ + const { helpers, startSandbox, stopSandbox } = harness({ listSandboxes: () => ({ sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], }), - isPortBound: (port) => port === 18_790 && bound, - stopSandbox: () => { - bound = false; - return { exitCode: 0 }; - }, - startSandbox: async () => { - bound = true; - return { exitCode: 0 }; - }, + isPortBound: (port) => port === 18_790, + ownsForward: () => true, }); - await run(async () => { - await helpers.reconcileOpenClawDashboardForwardReuse( - "reonboard-test", - "http://127.0.0.1:18790", - ); - await helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true); - }); + await helpers.reconcileOpenClawDashboardForwardReuse( + "reonboard-test", + "http://127.0.0.1:18790", + ); + await helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true); - expect(stopSandbox).toHaveBeenCalledOnce(); - expect(startSandbox).toHaveBeenCalledOnce(); + expect(stopSandbox).not.toHaveBeenCalled(); + expect(startSandbox).not.toHaveBeenCalled(); }); - it("does not trust reconciliation from an earlier onboarding transaction", async () => { + it("rechecks ownership instead of trusting an earlier reconciliation", async () => { vi.stubEnv("CHAT_UI_URL", undefined); - let bound = true; - let stopEffect = (): void => { - bound = false; - }; - const { helpers, run, startSandbox, stopSandbox } = harness({ + let owned = true; + const { helpers, startSandbox, stopSandbox } = harness({ listSandboxes: () => ({ sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], }), - isPortBound: (port) => port === 18_790 && bound, - stopSandbox: () => { - stopEffect(); - return { exitCode: 0 }; - }, - startSandbox: async () => { - bound = true; - return { exitCode: 0 }; - }, + isPortBound: (port) => port === 18_790, + ownsForward: () => owned, }); - await run(() => - helpers.reconcileOpenClawDashboardForwardReuse("reonboard-test", "http://127.0.0.1:18790"), + await helpers.reconcileOpenClawDashboardForwardReuse( + "reonboard-test", + "http://127.0.0.1:18790", ); - stopEffect = () => undefined; + owned = false; await expect( - run(() => helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true)), + helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true), ).rejects.toThrow(/remained occupied/u); - expect(stopSandbox).toHaveBeenCalledTimes(2); - expect(startSandbox).toHaveBeenCalledOnce(); + expect(stopSandbox).toHaveBeenCalledOnce(); + expect(startSandbox).not.toHaveBeenCalled(); }); it("rejects an ambiguous listener that remains after the reused sandbox stops", async () => { vi.stubEnv("CHAT_UI_URL", undefined); - const { helpers, launch, run, startSandbox, stopSandbox } = harness({ + const { helpers, launch, startSandbox, stopSandbox } = harness({ listSandboxes: () => ({ sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], }), @@ -220,7 +185,7 @@ describe("finalization dashboard ForwardTcp launch", () => { }); await expect( - run(() => helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true)), + helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true), ).rejects.toThrow(/remained occupied.*run 'nemoclaw reonboard-test start'/u); expect(stopSandbox).toHaveBeenCalledWith("reonboard-test", expect.any(Function)); expect(startSandbox).not.toHaveBeenCalled(); @@ -255,7 +220,7 @@ describe("finalization dashboard ForwardTcp launch", () => { it("does not restart a reused sandbox without a registered live identity", async () => { vi.stubEnv("CHAT_UI_URL", undefined); - const { helpers, launch, run, startSandbox, stopSandbox } = harness({ + const { helpers, launch, startSandbox, stopSandbox } = harness({ listSandboxes: () => ({ sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], }), @@ -264,7 +229,7 @@ describe("finalization dashboard ForwardTcp launch", () => { }); await expect( - run(() => helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true)), + helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true), ).rejects.toThrow(/Could not verify sandbox/u); expect(stopSandbox).not.toHaveBeenCalled(); expect(startSandbox).not.toHaveBeenCalled(); @@ -274,18 +239,21 @@ describe("finalization dashboard ForwardTcp launch", () => { it("does not cache a failed sandbox restart as reconciled", async () => { vi.stubEnv("CHAT_UI_URL", undefined); let bound = true; + let owned = false; const startOutcomes = [ async () => ({ exitCode: 1, message: "restart failed" }), async () => { bound = true; + owned = true; return { exitCode: 0 }; }, ]; - const { helpers, launch, run, startSandbox, stopSandbox } = harness({ + const { helpers, launch, startSandbox, stopSandbox } = harness({ listSandboxes: () => ({ sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], }), isPortBound: (port) => port === 18_790 && bound, + ownsForward: () => owned, stopSandbox: () => { bound = false; return { exitCode: 0 }; @@ -294,12 +262,12 @@ describe("finalization dashboard ForwardTcp launch", () => { }); await expect( - run(() => helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true)), + helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true), ).rejects.toThrow(/did not restore dashboard port.*restart failed/u); bound = true; await expect( - run(() => helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true)), + helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true), ).resolves.toBe(18_790); expect(stopSandbox).toHaveBeenCalledTimes(2); expect(startSandbox).toHaveBeenCalledTimes(2); @@ -309,23 +277,26 @@ describe("finalization dashboard ForwardTcp launch", () => { it("restores a sandbox after its stop cleanup fails", async () => { vi.stubEnv("CHAT_UI_URL", undefined); let bound = true; - const { helpers, launch, run, startSandbox, stopSandbox } = harness({ + let owned = false; + const { helpers, launch, startSandbox, stopSandbox } = harness({ listSandboxes: () => ({ sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], }), isPortBound: (port) => port === 18_790 && bound, + ownsForward: () => owned, stopSandbox: () => { bound = false; return { exitCode: 1, message: "Ollama cleanup failed", stopped: true }; }, startSandbox: async () => { bound = true; + owned = true; return { exitCode: 0 }; }, }); await expect( - run(() => helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true)), + helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true), ).rejects.toThrow(/was restored.*Ollama cleanup failed/u); expect(stopSandbox).toHaveBeenCalledOnce(); expect(startSandbox).toHaveBeenCalledOnce(); @@ -351,33 +322,23 @@ describe("finalization dashboard ForwardTcp launch", () => { expect(launch).not.toHaveBeenCalled(); }); - it("enables lifecycle reconciliation only for OpenClaw agents", async () => { + it("enables owned-forward reuse only for OpenClaw agents", async () => { vi.stubEnv("CHAT_UI_URL", undefined); - let bound = true; const openClaw = harness({ listSandboxes: () => ({ sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], }), - isPortBound: (port) => port === 18_790 && bound, - stopSandbox: () => { - bound = false; - return { exitCode: 0 }; - }, - startSandbox: async () => { - bound = true; - return { exitCode: 0 }; - }, + isPortBound: (port) => port === 18_790, + ownsForward: () => true, }); await expect( - openClaw.run(() => - openClaw.helpers.ensureFinalizationAgentDashboardForward( - "reonboard-test", - { name: "openclaw", forwardPort: 18_790 }, - undefined, - undefined, - true, - ), + openClaw.helpers.ensureFinalizationAgentDashboardForward( + "reonboard-test", + { name: "openclaw", forwardPort: 18_790 }, + undefined, + undefined, + true, ), ).resolves.toBe(18_790); From 81c4d74fa685abfd73e214a47c52c43af72bebb5 Mon Sep 17 00:00:00 2001 From: San Dang Date: Mon, 7 Sep 2026 10:26:50 +0700 Subject: [PATCH 10/20] fix(onboard): support Linux forward ownership probe --- .../openshell/forward-service.test.ts | 30 +++++++++++++++++++ src/lib/adapters/openshell/forward-service.ts | 17 +++++++++-- 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/src/lib/adapters/openshell/forward-service.test.ts b/src/lib/adapters/openshell/forward-service.test.ts index f1f5789f240..2708711f454 100644 --- a/src/lib/adapters/openshell/forward-service.test.ts +++ b/src/lib/adapters/openshell/forward-service.test.ts @@ -58,6 +58,22 @@ describe("OpenShell forward service", () => { expect(probe).toHaveBeenCalledTimes(3); }); + it("uses ss when lsof is unavailable", () => { + const expected = [target.executable, ...buildForwardServiceArgs(target)].join(" "); + const responses: Record = { + lsof: { status: null, stdout: "" }, + ss: { + status: 0, + stdout: 'LISTEN 0 1024 127.0.0.1:18789 0.0.0.0:* users:(("openshell",pid=4321,fd=9))\n', + }, + ps: { status: 0, stdout: `${expected}\n` }, + }; + const probe = vi.fn((executable: string) => responses[executable] ?? { status: 1, stdout: "" }); + + expect(isForwardServiceListenerOwner(target, { probe })).toBe(true); + expect(probe).toHaveBeenCalledTimes(5); + }); + it("rejects a listener whose process does not match the direct ForwardTcp target", () => { const probe = vi.fn((executable: string) => executable === "lsof" @@ -79,6 +95,20 @@ describe("OpenShell forward service", () => { expect(isForwardServiceListenerOwner(target, { probe })).toBe(false); }); + it("rejects ambiguous ss listener ownership when lsof is unavailable", () => { + const responses: Record = { + lsof: { status: null, stdout: "" }, + ss: { + status: 0, + stdout: + 'LISTEN 0 1024 127.0.0.1:18789 0.0.0.0:* users:(("openshell",pid=4321,fd=9),("foreign",pid=9876,fd=8))\n', + }, + }; + const probe = vi.fn((executable: string) => responses[executable] ?? { status: 1, stdout: "" }); + + expect(isForwardServiceListenerOwner(target, { probe })).toBe(false); + }); + it("detaches the OpenShell child and waits for its local port", () => { const unref = vi.fn(); const spawnDetached = vi.fn(() => ({ unref })); diff --git a/src/lib/adapters/openshell/forward-service.ts b/src/lib/adapters/openshell/forward-service.ts index 9a2a492122a..0c34926cb32 100644 --- a/src/lib/adapters/openshell/forward-service.ts +++ b/src/lib/adapters/openshell/forward-service.ts @@ -110,8 +110,21 @@ function captureProcess(executable: string, args: readonly string[]) { function listenerPids(port: number, probe: ForwardServiceOwnerProbe): string[] { const result = probe("lsof", ["-ti", `:${String(port)}`, "-sTCP:LISTEN"]); - if (result.status !== 0) return []; - return [...new Set(result.stdout.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean))]; + if (result.status === 0) { + return [ + ...new Set( + result.stdout + .split(/\r?\n/u) + .map((line) => line.trim()) + .filter(Boolean), + ), + ]; + } + if (result.status !== null) return []; + + const fallback = probe("ss", ["-H", "-ltnp", `sport = :${String(port)}`]); + if (fallback.status !== 0) return []; + return [...new Set([...fallback.stdout.matchAll(/\bpid=(\d+)\b/gu)].map((match) => match[1]!))]; } /** Prove that the current listener is the exact direct ForwardTcp command. */ From 1c792092e8ac789cbac59215e5ba4060a6c87d2f Mon Sep 17 00:00:00 2001 From: San Dang Date: Mon, 7 Sep 2026 11:27:43 +0700 Subject: [PATCH 11/20] fix(onboard): reuse owned dashboard forward during finalization --- .../openshell/forward-service.test.ts | 30 --------- src/lib/adapters/openshell/forward-service.ts | 17 +---- src/lib/onboard.ts | 1 - src/lib/onboard/dashboard.ts | 42 ++++-------- ...ard-finalization-dashboard-forward.test.ts | 64 +++++++++---------- 5 files changed, 46 insertions(+), 108 deletions(-) diff --git a/src/lib/adapters/openshell/forward-service.test.ts b/src/lib/adapters/openshell/forward-service.test.ts index 2708711f454..f1f5789f240 100644 --- a/src/lib/adapters/openshell/forward-service.test.ts +++ b/src/lib/adapters/openshell/forward-service.test.ts @@ -58,22 +58,6 @@ describe("OpenShell forward service", () => { expect(probe).toHaveBeenCalledTimes(3); }); - it("uses ss when lsof is unavailable", () => { - const expected = [target.executable, ...buildForwardServiceArgs(target)].join(" "); - const responses: Record = { - lsof: { status: null, stdout: "" }, - ss: { - status: 0, - stdout: 'LISTEN 0 1024 127.0.0.1:18789 0.0.0.0:* users:(("openshell",pid=4321,fd=9))\n', - }, - ps: { status: 0, stdout: `${expected}\n` }, - }; - const probe = vi.fn((executable: string) => responses[executable] ?? { status: 1, stdout: "" }); - - expect(isForwardServiceListenerOwner(target, { probe })).toBe(true); - expect(probe).toHaveBeenCalledTimes(5); - }); - it("rejects a listener whose process does not match the direct ForwardTcp target", () => { const probe = vi.fn((executable: string) => executable === "lsof" @@ -95,20 +79,6 @@ describe("OpenShell forward service", () => { expect(isForwardServiceListenerOwner(target, { probe })).toBe(false); }); - it("rejects ambiguous ss listener ownership when lsof is unavailable", () => { - const responses: Record = { - lsof: { status: null, stdout: "" }, - ss: { - status: 0, - stdout: - 'LISTEN 0 1024 127.0.0.1:18789 0.0.0.0:* users:(("openshell",pid=4321,fd=9),("foreign",pid=9876,fd=8))\n', - }, - }; - const probe = vi.fn((executable: string) => responses[executable] ?? { status: 1, stdout: "" }); - - expect(isForwardServiceListenerOwner(target, { probe })).toBe(false); - }); - it("detaches the OpenShell child and waits for its local port", () => { const unref = vi.fn(); const spawnDetached = vi.fn(() => ({ unref })); diff --git a/src/lib/adapters/openshell/forward-service.ts b/src/lib/adapters/openshell/forward-service.ts index 0c34926cb32..9a2a492122a 100644 --- a/src/lib/adapters/openshell/forward-service.ts +++ b/src/lib/adapters/openshell/forward-service.ts @@ -110,21 +110,8 @@ function captureProcess(executable: string, args: readonly string[]) { function listenerPids(port: number, probe: ForwardServiceOwnerProbe): string[] { const result = probe("lsof", ["-ti", `:${String(port)}`, "-sTCP:LISTEN"]); - if (result.status === 0) { - return [ - ...new Set( - result.stdout - .split(/\r?\n/u) - .map((line) => line.trim()) - .filter(Boolean), - ), - ]; - } - if (result.status !== null) return []; - - const fallback = probe("ss", ["-H", "-ltnp", `sport = :${String(port)}`]); - if (fallback.status !== 0) return []; - return [...new Set([...fallback.stdout.matchAll(/\bpid=(\d+)\b/gu)].map((match) => match[1]!))]; + if (result.status !== 0) return []; + return [...new Set(result.stdout.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean))]; } /** Prove that the current listener is the exact direct ForwardTcp command. */ diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 7ca1b51207d..c6f3a24f6f3 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -3246,7 +3246,6 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { selectedAgent, undefined, hermesApiPortReservationScope, - resume, ), persistDashboardPort: (name, port) => registry.updateSandbox(name, { dashboardPort: port }), diff --git a/src/lib/onboard/dashboard.ts b/src/lib/onboard/dashboard.ts index 611fc0db63c..34f0aa523fe 100644 --- a/src/lib/onboard/dashboard.ts +++ b/src/lib/onboard/dashboard.ts @@ -159,7 +159,6 @@ export interface OnboardDashboardHelpers { ensureFinalizationDashboardForward( sandboxName: string, revalidateSandboxIdentity?: (operation: string) => void, - reuseExistingOpenClawForward?: boolean, ): Promise; ensureFinalizationAgentDashboardForward( sandboxName: string, @@ -168,7 +167,6 @@ export interface OnboardDashboardHelpers { portReservation?: { releaseBeforeForward(agentName: string, port: number): Promise | void; }, - reuseExistingOpenClawForward?: boolean, ): Promise; reconcileOpenClawDashboardForwardReuse( sandboxName: string, @@ -306,12 +304,7 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa ): boolean { return ( forwardService?.owns?.( - forwardTarget( - sandboxName, - gatewayName, - port, - getDashboardForwardTarget(chatUiUrl), - ), + forwardTarget(sandboxName, gatewayName, port, getDashboardForwardTarget(chatUiUrl)), ) === true ); } @@ -679,10 +672,10 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa /** * Reconcile the dashboard forward for the agent-less OpenClaw finalization - * branch. The resume path skips sandbox creation, so `CHAT_UI_URL` does not - * carry the port the in-sandbox gateway listens on; the registry entry - * persisted by onboarding is the only record of that port. The forward and - * the in-sandbox gateway must share one port number (`openshell forward` + * branch. A resumed or repeated onboarding can skip sandbox creation, so + * `CHAT_UI_URL` may not carry the port the in-sandbox gateway listens on; + * the registry entry persisted by onboarding is the only record of that + * port. The forward and the in-sandbox gateway must share one port number (`openshell forward` * binds the same port on both sides), so when the persisted port cannot be * forwarded this throws instead of reallocating: the resumed gateway only * listens on the persisted port, and a forward on any other port serves @@ -693,22 +686,19 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa async function ensureFinalizationDashboardForward( sandboxName: string, revalidateSandboxIdentity?: (operation: string) => void, - reuseExistingOpenClawForward = false, ): Promise { const envUrl = process.env.CHAT_UI_URL; const persistedPort = envUrl ? null : getPersistedDashboardPort(sandboxName, listSandboxes); const requestedUrl = envUrl || (persistedPort === null ? undefined : `http://127.0.0.1:${String(persistedPort)}`); - if (reuseExistingOpenClawForward) { - await reconcileOpenClawDashboardForwardReuse( - sandboxName, - requestedUrl || `http://127.0.0.1:${CONTROL_UI_PORT}`, - revalidateSandboxIdentity, - ); - } + await reconcileOpenClawDashboardForwardReuse( + sandboxName, + requestedUrl || `http://127.0.0.1:${CONTROL_UI_PORT}`, + revalidateSandboxIdentity, + ); const actualPort = ensureDashboardForward(sandboxName, requestedUrl, { allowPortReallocation: false, - ...(reuseExistingOpenClawForward ? { reuseExistingOpenClawForward: true } : {}), + reuseExistingOpenClawForward: true, ...(revalidateSandboxIdentity ? { revalidateSandboxIdentity } : {}), }); revalidateSandboxIdentity?.(`publish the dashboard URL for sandbox '${sandboxName}'`); @@ -749,17 +739,11 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa portReservation?: { releaseBeforeForward(agentName: string, port: number): Promise | void; }, - reuseExistingOpenClawForward = false, ): Promise { if (!agent) { - return ensureFinalizationDashboardForward( - sandboxName, - revalidateSandboxIdentity, - reuseExistingOpenClawForward, - ); + return ensureFinalizationDashboardForward(sandboxName, revalidateSandboxIdentity); } - const mayReuseOpenClawForward = - agent.name === "openclaw" && reuseExistingOpenClawForward; + const mayReuseOpenClawForward = agent.name === "openclaw"; if (mayReuseOpenClawForward) { const registeredPort = getPersistedDashboardPort(sandboxName, listSandboxes); const requestedUrl = diff --git a/test/onboarding/onboard-finalization-dashboard-forward.test.ts b/test/onboarding/onboard-finalization-dashboard-forward.test.ts index 09e788a21aa..c7850c81dc1 100644 --- a/test/onboarding/onboard-finalization-dashboard-forward.test.ts +++ b/test/onboarding/onboard-finalization-dashboard-forward.test.ts @@ -105,10 +105,10 @@ describe("finalization dashboard ForwardTcp launch", () => { }); await expect(helpers.ensureFinalizationDashboardForward("reonboard-test")).rejects.toThrow( - /cannot be reallocated/u, + /cannot be adopted/u, ); expect(launch).not.toHaveBeenCalled(); - expect(stopSandbox).not.toHaveBeenCalled(); + expect(stopSandbox).toHaveBeenCalledOnce(); expect(startSandbox).not.toHaveBeenCalled(); }); @@ -122,9 +122,9 @@ describe("finalization dashboard ForwardTcp launch", () => { ownsForward: () => true, }); - await expect( - helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true), - ).resolves.toBe(18_790); + await expect(helpers.ensureFinalizationDashboardForward("reonboard-test")).resolves.toBe( + 18_790, + ); expect(stopSandbox).not.toHaveBeenCalled(); expect(startSandbox).not.toHaveBeenCalled(); expect(launch).not.toHaveBeenCalled(); @@ -145,7 +145,7 @@ describe("finalization dashboard ForwardTcp launch", () => { "reonboard-test", "http://127.0.0.1:18790", ); - await helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true); + await helpers.ensureFinalizationDashboardForward("reonboard-test"); expect(stopSandbox).not.toHaveBeenCalled(); expect(startSandbox).not.toHaveBeenCalled(); @@ -168,9 +168,9 @@ describe("finalization dashboard ForwardTcp launch", () => { ); owned = false; - await expect( - helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true), - ).rejects.toThrow(/remained occupied/u); + await expect(helpers.ensureFinalizationDashboardForward("reonboard-test")).rejects.toThrow( + /remained occupied/u, + ); expect(stopSandbox).toHaveBeenCalledOnce(); expect(startSandbox).not.toHaveBeenCalled(); }); @@ -184,9 +184,9 @@ describe("finalization dashboard ForwardTcp launch", () => { isPortBound: (port) => port === 18_790, }); - await expect( - helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true), - ).rejects.toThrow(/remained occupied.*run 'nemoclaw reonboard-test start'/u); + await expect(helpers.ensureFinalizationDashboardForward("reonboard-test")).rejects.toThrow( + /remained occupied.*run 'nemoclaw reonboard-test start'/u, + ); expect(stopSandbox).toHaveBeenCalledWith("reonboard-test", expect.any(Function)); expect(startSandbox).not.toHaveBeenCalled(); expect(launch).not.toHaveBeenCalled(); @@ -209,9 +209,9 @@ describe("finalization dashboard ForwardTcp launch", () => { }, }); - await expect( - helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true), - ).rejects.toThrow(/identity changed.*may remain stopped/u); + await expect(helpers.ensureFinalizationDashboardForward("reonboard-test")).rejects.toThrow( + /identity changed.*may remain stopped/u, + ); expect(stopSandbox).toHaveBeenCalledWith("reonboard-test", expect.any(Function)); expect(startSandbox).toHaveBeenCalledWith("reonboard-test", expect.any(Function)); expect(startSandboxOperation).not.toHaveBeenCalled(); @@ -228,9 +228,9 @@ describe("finalization dashboard ForwardTcp launch", () => { registeredIdentity: false, }); - await expect( - helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true), - ).rejects.toThrow(/Could not verify sandbox/u); + await expect(helpers.ensureFinalizationDashboardForward("reonboard-test")).rejects.toThrow( + /Could not verify sandbox/u, + ); expect(stopSandbox).not.toHaveBeenCalled(); expect(startSandbox).not.toHaveBeenCalled(); expect(launch).not.toHaveBeenCalled(); @@ -261,14 +261,14 @@ describe("finalization dashboard ForwardTcp launch", () => { startSandbox: async () => await startOutcomes.shift()!(), }); - await expect( - helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true), - ).rejects.toThrow(/did not restore dashboard port.*restart failed/u); + await expect(helpers.ensureFinalizationDashboardForward("reonboard-test")).rejects.toThrow( + /did not restore dashboard port.*restart failed/u, + ); bound = true; - await expect( - helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true), - ).resolves.toBe(18_790); + await expect(helpers.ensureFinalizationDashboardForward("reonboard-test")).resolves.toBe( + 18_790, + ); expect(stopSandbox).toHaveBeenCalledTimes(2); expect(startSandbox).toHaveBeenCalledTimes(2); expect(launch).not.toHaveBeenCalled(); @@ -295,9 +295,9 @@ describe("finalization dashboard ForwardTcp launch", () => { }, }); - await expect( - helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true), - ).rejects.toThrow(/was restored.*Ollama cleanup failed/u); + await expect(helpers.ensureFinalizationDashboardForward("reonboard-test")).rejects.toThrow( + /was restored.*Ollama cleanup failed/u, + ); expect(stopSandbox).toHaveBeenCalledOnce(); expect(startSandbox).toHaveBeenCalledOnce(); expect(launch).not.toHaveBeenCalled(); @@ -315,14 +315,14 @@ describe("finalization dashboard ForwardTcp launch", () => { isPortBound: (port) => port === 18_790, }); - await expect( - helpers.ensureFinalizationDashboardForward("reonboard-test", undefined, true), - ).rejects.toThrow(/cannot be reallocated/u); + await expect(helpers.ensureFinalizationDashboardForward("reonboard-test")).rejects.toThrow( + /cannot be reallocated/u, + ); expect(stopSandbox).not.toHaveBeenCalled(); expect(launch).not.toHaveBeenCalled(); }); - it("enables owned-forward reuse only for OpenClaw agents", async () => { + it("enables owned-forward reuse only for OpenClaw agents during ordinary finalization", async () => { vi.stubEnv("CHAT_UI_URL", undefined); const openClaw = harness({ listSandboxes: () => ({ @@ -338,7 +338,6 @@ describe("finalization dashboard ForwardTcp launch", () => { { name: "openclaw", forwardPort: 18_790 }, undefined, undefined, - true, ), ).resolves.toBe(18_790); @@ -355,7 +354,6 @@ describe("finalization dashboard ForwardTcp launch", () => { { name: "hermes", forwardPort: 18_790 }, undefined, undefined, - true, ), ).rejects.toThrow(/cannot be reallocated/u); expect(hermes.stopSandbox).not.toHaveBeenCalled(); From 940aecdc53fc382d37363d384689b93f8a286c0d Mon Sep 17 00:00:00 2001 From: San Dang Date: Mon, 7 Sep 2026 12:22:06 +0700 Subject: [PATCH 12/20] test(e2e): preserve double-onboard lifecycle setup --- test/e2e/live/double-onboard.test.ts | 217 ++++++++++++++++++--------- 1 file changed, 147 insertions(+), 70 deletions(-) diff --git a/test/e2e/live/double-onboard.test.ts b/test/e2e/live/double-onboard.test.ts index e9e2f9244d9..d0540e3d7f7 100644 --- a/test/e2e/live/double-onboard.test.ts +++ b/test/e2e/live/double-onboard.test.ts @@ -32,6 +32,7 @@ const PHASE_TIMEOUT_MS = Number(process.env.NEMOCLAW_E2E_PHASE_TIMEOUT_MS ?? 1_2 const ONBOARD_TIMEOUT_MS = execTimeout(PHASE_TIMEOUT_MS); const PROBE_ATTEMPTS = Number(process.env.NEMOCLAW_E2E_PROBE_ATTEMPTS ?? 3); const PROBE_DELAY_MS = Number(process.env.NEMOCLAW_E2E_PROBE_DELAY_SECONDS ?? 3) * 1_000; +const PROBE_TIMEOUT_MS = Number(process.env.NEMOCLAW_E2E_PROBE_TIMEOUT_SECONDS ?? 180) * 1_000; const RECOVERY_PROBE_TIMEOUT_MS = Number(process.env.NEMOCLAW_E2E_RECOVERY_PROBE_TIMEOUT_SECONDS ?? 180) * 1_000; const TEST_TIMEOUT_MS = testTimeout(90 * 60_000); @@ -113,38 +114,35 @@ async function runOnboard( }); } -async function waitForDashboardReachability( +async function runProbeOnlyConnect( host: HostCliClient, - port: string, - expectedReachable: boolean, - artifactPrefix: string, -): Promise<{ reachable: boolean; output: string }> { - let reachable = false; - let output = ""; - for (let attempt = 1; attempt <= PROBE_ATTEMPTS; attempt += 1) { - const result = await host.command( - "curl", + sandboxName: string, + artifactName: string, +): Promise { + return await host.command( + "bash", + [ + "-lc", [ - "--silent", - "--show-error", - "--output", - "/dev/null", - "--max-time", - "5", - `http://127.0.0.1:${port}/`, - ], - { - artifactName: `${artifactPrefix}-attempt-${attempt}`, - env: commandEnv(), - timeoutMs: 15_000, - }, - ); - output = resultText(result); - reachable = result.exitCode === 0 && !result.timedOut; - if (reachable === expectedReachable) break; - if (attempt < PROBE_ATTEMPTS) await sleep(PROBE_DELAY_MS); - } - return { reachable, output }; + "set +e", + 'log="$(mktemp)"', + '"$1" "$2" "$3" connect --probe-only >"$log" 2>&1', + "rc=$?", + 'cat "$log"', + 'rm -f "$log"', + 'exit "$rc"', + ].join("\n"), + "nemoclaw-probe-connect", + process.execPath, + CLI_ENTRYPOINT, + sandboxName, + ], + { + artifactName, + env: commandEnv(), + timeoutMs: PROBE_TIMEOUT_MS, + }, + ); } async function cleanupDoubleOnboardState( @@ -236,6 +234,44 @@ function dashboardPortFromList(output: string, sandboxName: string): string | un return undefined; } +function forwardOwnerForPort(output: string, port: string): string | undefined { + for (const line of stripAnsi(output).split("\n")) { + const parts = line.trim().split(/\s+/); + if (parts.length < 5 || parts[0]?.toLowerCase() === "sandbox") continue; + const status = parts.slice(4).join(" ").toLowerCase(); + if (parts[2] === port && status.includes("running")) return parts[0]; + } + return undefined; +} + +async function waitForForwardOwner( + sandbox: SandboxClient, + port: string, + owner: string | undefined, + artifactPrefix: string, +): Promise<{ + owner: string | undefined; + output: string; + querySucceeded: boolean; +}> { + let observedOwner: string | undefined; + let lastOutput = ""; + let querySucceeded = false; + for (let attempt = 1; attempt <= PROBE_ATTEMPTS; attempt += 1) { + const result = await sandbox.openshell(["forward", "list"], { + artifactName: `${artifactPrefix}-attempt-${attempt}`, + env: commandEnv(), + timeoutMs: 30_000, + }); + lastOutput = resultText(result); + querySucceeded = result.exitCode === 0 && !result.timedOut; + observedOwner = querySucceeded ? forwardOwnerForPort(lastOutput, port) : undefined; + if (querySucceeded && observedOwner === owner) break; + if (attempt < PROBE_ATTEMPTS) await sleep(PROBE_DELAY_MS); + } + return { owner: observedOwner, output: lastOutput, querySucceeded }; +} + function hasOwn(object: object, key: string): boolean { return Object.prototype.hasOwnProperty.call(object, key); } @@ -557,17 +593,36 @@ test( env: commandEnv(), timeoutMs: 60_000, }); - expect(listAfterSecond.exitCode, resultText(listAfterSecond)).toBe(0); - expect(stripAnsi(listAfterSecond.stdout)).toContain(SANDBOX_A); const portAfterSecond = dashboardPortFromList(listAfterSecond.stdout, SANDBOX_A); - expect(portAfterSecond, resultText(listAfterSecond)).toBe(portAfterFirst); - const dashboardAfterSecond = await waitForDashboardReachability( - host, - portAfterSecond ?? "", - true, - "phase-3-dashboard-after-second-onboard", + const dashboardAfterSecond = await host.command( + "curl", + [ + "--silent", + "--show-error", + "--fail", + "--output", + "/dev/null", + "--retry", + "5", + "--retry-connrefused", + "--retry-delay", + "2", + "--connect-timeout", + "5", + "--max-time", + "30", + `http://127.0.0.1:${portAfterSecond ?? "0"}/`, + ], + { + artifactName: "phase-3-dashboard-after-second-onboard", + env: commandEnv(), + timeoutMs: 45_000, + }, ); - expect(dashboardAfterSecond.reachable, dashboardAfterSecond.output).toBe(true); + expect( + `${listAfterSecond.exitCode}:${portAfterSecond}:${dashboardAfterSecond.exitCode}:${dashboardAfterSecond.timedOut}`, + `${resultText(listAfterSecond)}\n${resultText(dashboardAfterSecond)}`, + ).toBe(`0:${portAfterFirst}:0:false`); progress.phase("recreate same sandbox on existing gateway"); const gatewayBeforeRecreate = await gatewayRuntimeId(gateway); @@ -648,20 +703,39 @@ test( expect(portB, `nemoclaw list did not show ${SANDBOX_B} dashboard: ${list.stdout}`).toBeTruthy(); expect(portB).not.toBe(portA); - const dashboardABeforeStop = await waitForDashboardReachability( - host, - portA ?? "", - true, - "phase-4-dashboard-a-before-stop", - ); - expect(dashboardABeforeStop.reachable, dashboardABeforeStop.output).toBe(true); - const dashboardBBeforeStop = await waitForDashboardReachability( - host, + await sandbox.openshell(["forward", "stop", portB ?? ""], { + artifactName: "phase-4-stop-sandbox-b-dashboard-forward", + env: commandEnv(), + timeoutMs: 30_000, + }); + let probe: ShellProbeResult | undefined; + for (let attempt = 1; attempt <= PROBE_ATTEMPTS; attempt += 1) { + probe = await runProbeOnlyConnect( + host, + SANDBOX_B, + `phase-4-probe-connect-sandbox-b-attempt-${attempt}`, + ); + if (probe.exitCode === 0 && !probe.timedOut) break; + if (attempt < PROBE_ATTEMPTS) await sleep(PROBE_DELAY_MS); + } + expect(probe?.exitCode, probe ? resultText(probe) : "probe did not run").toBe(0); + expect(probe?.timedOut, probe ? resultText(probe) : "probe did not run").toBe(false); + + const restoredForwardB = await waitForForwardOwner( + sandbox, portB ?? "", - true, - "phase-4-dashboard-b-before-stop", + SANDBOX_B, + "phase-4-openshell-forward-list-b", + ); + expect(restoredForwardB.owner, restoredForwardB.output).toBe(SANDBOX_B); + + const retainedForwardA = await waitForForwardOwner( + sandbox, + portA ?? "", + SANDBOX_A, + "phase-4-openshell-forward-list-a", ); - expect(dashboardBBeforeStop.reachable, dashboardBBeforeStop.output).toBe(true); + expect(retainedForwardA.owner, retainedForwardA.output).toBe(SANDBOX_A); progress.phase("stop sibling sandbox without disturbing the first forward"); const stopB = await command(host, [SANDBOX_B, "stop"], { @@ -671,13 +745,14 @@ test( }); expect(stopB.exitCode, resultText(stopB)).toBe(0); - const releasedForwardB = await waitForDashboardReachability( - host, + const releasedForwardB = await waitForForwardOwner( + sandbox, portB ?? "", - false, - "phase-4-dashboard-b-after-stop", + undefined, + "phase-4-openshell-forward-list-b-after-stop", ); - expect(releasedForwardB.reachable, releasedForwardB.output).toBe(false); + expect(releasedForwardB.querySucceeded, releasedForwardB.output).toBe(true); + expect(releasedForwardB.owner, releasedForwardB.output).toBeUndefined(); const stoppedStatusB = await command(host, [SANDBOX_B, "status"], { artifactName: "phase-4-nemoclaw-status-sandbox-b-after-stop", @@ -689,13 +764,13 @@ test( expect(stoppedStatusTextB).toContain("sandbox_container_stopped"); expect(stoppedStatusTextB).not.toContain("sandbox_dashboard_port_conflict"); - const retainedForwardAAfterStop = await waitForDashboardReachability( - host, + const retainedForwardAAfterStop = await waitForForwardOwner( + sandbox, portA ?? "", - true, - "phase-4-dashboard-a-after-b-stop", + SANDBOX_A, + "phase-4-openshell-forward-list-a-after-b-stop", ); - expect(retainedForwardAAfterStop.reachable, retainedForwardAAfterStop.output).toBe(true); + expect(retainedForwardAAfterStop.owner, retainedForwardAAfterStop.output).toBe(SANDBOX_A); const startB = await command(host, [SANDBOX_B, "start"], { artifactName: "phase-4-nemoclaw-start-sandbox-b", @@ -703,13 +778,13 @@ test( timeoutMs: PHASE_TIMEOUT_MS, }); expect(startB.exitCode, resultText(startB)).toBe(0); - const restoredForwardBAfterStart = await waitForDashboardReachability( - host, + const restoredForwardBAfterStart = await waitForForwardOwner( + sandbox, portB ?? "", - true, - "phase-4-dashboard-b-after-start", + SANDBOX_B, + "phase-4-openshell-forward-list-b-after-start", ); - expect(restoredForwardBAfterStart.reachable, restoredForwardBAfterStart.output).toBe(true); + expect(restoredForwardBAfterStart.owner, restoredForwardBAfterStart.output).toBe(SANDBOX_B); progress.phase("replace sandbox after stale registry refusal"); // Phase 5: direct OpenShell deletion leaves a stale registry entry that @@ -849,18 +924,20 @@ test( secondOnboardReusedGateway: gatewayAfterSecond === gatewayBeforeSecond && secondText.includes("Reusing healthy NemoClaw gateway.") && - dashboardAfterSecond.reachable, + dashboardAfterSecond.exitCode === 0 && + !dashboardAfterSecond.timedOut, thirdOnboardPreservedSibling: sandboxAAfterThird.exitCode === 0 && sandboxBAfterThird.exitCode === 0, distinctDashboardPorts: Boolean(portA && portB && portA !== portB), selectedStopReleasedOnlySelectedForward: stopB.exitCode === 0 && - !releasedForwardB.reachable && - retainedForwardAAfterStop.reachable && + releasedForwardB.querySucceeded && + releasedForwardB.owner === undefined && + retainedForwardAAfterStop.owner === SANDBOX_A && stoppedStatusTextB.includes("sandbox_container_stopped") && !stoppedStatusTextB.includes("sandbox_dashboard_port_conflict") && startB.exitCode === 0 && - restoredForwardBAfterStart.reachable, + restoredForwardBAfterStart.owner === SANDBOX_B, staleRegistryRecovered: rebuild.exitCode === 0, gatewayStopGuidance: /Recovered NemoClaw gateway runtime|gateway is no longer configured after restart\/rebuild|gateway is still refusing connections after restart|gateway trust material rotated after restart/.test( From 09cd9314376ff3a5722f5b514c9fa58aa271de1c Mon Sep 17 00:00:00 2001 From: San Dang Date: Mon, 7 Sep 2026 13:39:28 +0700 Subject: [PATCH 13/20] fix(onboard): reuse dashboard forward without restart Signed-off-by: San Dang --- docs/manage-sandboxes/run-sandboxes.mdx | 6 +- src/lib/actions/onboard.test.ts | 27 +- src/lib/actions/onboard.ts | 22 +- .../sandbox/rebuild-onboard-dependencies.ts | 10 +- src/lib/actions/sandbox/start.ts | 1 - src/lib/actions/sandbox/stop.test.ts | 1 - src/lib/actions/sandbox/stop.ts | 29 +- .../openshell/forward-service.test.ts | 11 + src/lib/adapters/openshell/forward-service.ts | 7 +- src/lib/onboard.ts | 9 +- src/lib/onboard/agent-dashboard-forward.ts | 6 +- src/lib/onboard/dashboard.ts | 179 +----------- .../onboard/dashboard/reuse-lifecycle.test.ts | 36 --- src/lib/onboard/dashboard/reuse-lifecycle.ts | 29 -- .../onboard/sandbox-create/orchestration.ts | 2 - src/lib/onboard/sandbox-reuse.test.ts | 7 - src/lib/onboard/sandbox-reuse.ts | 35 +-- test/e2e/live/double-onboard.test.ts | 217 +++++---------- test/e2e/live/onboard-resume.test.ts | 73 +---- ...ard-finalization-dashboard-forward.test.ts | 256 ++---------------- 20 files changed, 166 insertions(+), 797 deletions(-) delete mode 100644 src/lib/onboard/dashboard/reuse-lifecycle.test.ts delete mode 100644 src/lib/onboard/dashboard/reuse-lifecycle.ts diff --git a/docs/manage-sandboxes/run-sandboxes.mdx b/docs/manage-sandboxes/run-sandboxes.mdx index 584711c485a..3277d0b460c 100644 --- a/docs/manage-sandboxes/run-sandboxes.mdx +++ b/docs/manage-sandboxes/run-sandboxes.mdx @@ -62,14 +62,12 @@ The default port keeps the shared `~/.nemoclaw/` location. When other ports remain, `$$nemoclaw uninstall` removes only the selected gateway and keeps the shared CLI, services, images, providers, configuration, models, and swap. Gateway and dashboard cleanup is scoped by sandbox name and port. -A later onboarding run for another sandbox does not tear down the first sandbox's gateway or dashboard forward. +A later onboarding run that uses a different `NEMOCLAW_GATEWAY_PORT` or `--control-ui-port` does not tear down the first sandbox's gateway or dashboard forward. If re-onboarding finds the registered dashboard port already bound, NemoClaw verifies that the listener is the exact OpenShell forward for that sandbox and reuses it without restarting the sandbox. -If ownership cannot be proved, NemoClaw briefly stops the same ready sandbox to distinguish its forward from a foreign listener. -If the port remains bound after the sandbox stops, resolve the listener conflict, run `nemoclaw start`, then retry onboarding. -For other failures, follow the reported recovery steps. +If ownership cannot be proved, onboarding fails closed and reports the listener conflict. diff --git a/src/lib/actions/onboard.test.ts b/src/lib/actions/onboard.test.ts index c53810ff5f7..4085f1e87ff 100644 --- a/src/lib/actions/onboard.test.ts +++ b/src/lib/actions/onboard.test.ts @@ -13,8 +13,7 @@ vi.mock("../agent/defs", () => ({ listAgents: mocks.listAgents })); vi.mock("../onboard", () => ({ onboard: mocks.onboard })); vi.mock("../onboard/command", () => ({ runOnboardCommand: mocks.runOnboardCommand })); -import { getDashboardReuseLifecycle } from "../onboard/dashboard/reuse-lifecycle"; -import { runOnboard, runOnboardAction } from "./onboard"; +import { runOnboardAction } from "./onboard"; describe("onboard action runtime composition", () => { beforeEach(() => { @@ -26,21 +25,13 @@ describe("onboard action runtime composition", () => { ); }); - it("passes host-only runtime dependencies into legacy onboarding", async () => { + it("passes host-only Google Chat dependencies into legacy onboarding", async () => { const googlechatTunnelRuntime = { loadServices: vi.fn(), loadWebhookProxy: vi.fn(), }; - const dashboardReuseLifecycle = { - startSandbox: vi.fn(), - stopSandbox: vi.fn(), - withSandboxLifecycleLock: vi.fn(), - }; - await runOnboardAction( - { "non-interactive": true }, - { googlechatTunnelRuntime, dashboardReuseLifecycle }, - ); + await runOnboardAction({ "non-interactive": true }, { googlechatTunnelRuntime }); expect(mocks.onboard).toHaveBeenCalledWith({ nonInteractive: true, @@ -48,16 +39,4 @@ describe("onboard action runtime composition", () => { googlechatTunnelRuntime, }); }); - - it("provides the default dashboard reuse lifecycle to direct onboarding callers", async () => { - mocks.onboard.mockImplementationOnce(async () => { - expect(getDashboardReuseLifecycle()).toEqual({ - startSandbox: expect.any(Function), - stopSandbox: expect.any(Function), - withSandboxLifecycleLock: expect.any(Function), - }); - }); - - await runOnboard({} as never); - }); }); diff --git a/src/lib/actions/onboard.ts b/src/lib/actions/onboard.ts index 27a01babcdf..9beb40d57d8 100644 --- a/src/lib/actions/onboard.ts +++ b/src/lib/actions/onboard.ts @@ -5,40 +5,24 @@ import { loadServingCatalog } from "../inference/serving/catalog-loader"; import type { GooglechatTunnelRuntimeDeps } from "../messaging/channels/googlechat/hooks/tunnel-runtime"; import { type OnboardCommandOptions, runOnboardCommand } from "../onboard/command"; import { type OnboardFlags, readAgentRegistryNames } from "../onboard/command-support"; -import { - type DashboardReuseLifecycle, - withDashboardReuseLifecycle, -} from "../onboard/dashboard/reuse-lifecycle"; import { resolveOnboardResumeIntent } from "../onboard/session-bootstrap"; import { loadOnboardCommandResumeSession } from "../onboard/sandbox-registration"; import type { OnboardOptions } from "../onboard/types"; export interface OnboardActionRuntimeDeps { readonly googlechatTunnelRuntime?: Omit; - readonly dashboardReuseLifecycle?: DashboardReuseLifecycle; } -export async function runOnboard( +async function runOnboard( options: OnboardCommandOptions, - runtimeDeps: OnboardActionRuntimeDeps = {}, + runtimeDeps: OnboardActionRuntimeDeps, ): Promise { // Keep the monolithic legacy onboarding graph lazy so command metadata/help // imports do not execute it. Resolve it only when the user invokes onboard. const { onboard } = (await import("../onboard")) as unknown as { onboard: (onboardOptions?: OnboardOptions) => Promise; }; - const startActions = await import("./sandbox/start"); - const stopActions = await import("./sandbox/stop"); - const lifecycle = runtimeDeps.dashboardReuseLifecycle ?? { - startSandbox: (sandboxName: string, revalidateAtMutationEdge: () => void) => - startActions.startSandbox(sandboxName, { revalidateAtMutationEdge }), - stopSandbox: (sandboxName: string, revalidateAtMutationEdge: () => void) => - stopActions.stopSandbox(sandboxName, { revalidateAtMutationEdge }), - withSandboxLifecycleLock: startActions.withSandboxLifecycleLock, - }; - await withDashboardReuseLifecycle(lifecycle, () => - onboard({ ...options, googlechatTunnelRuntime: runtimeDeps.googlechatTunnelRuntime }), - ); + await onboard({ ...options, googlechatTunnelRuntime: runtimeDeps.googlechatTunnelRuntime }); } function buildOnboardCommandDeps(flags: OnboardFlags, runtimeDeps: OnboardActionRuntimeDeps) { diff --git a/src/lib/actions/sandbox/rebuild-onboard-dependencies.ts b/src/lib/actions/sandbox/rebuild-onboard-dependencies.ts index 33ac3042d29..f93d591bc14 100644 --- a/src/lib/actions/sandbox/rebuild-onboard-dependencies.ts +++ b/src/lib/actions/sandbox/rebuild-onboard-dependencies.ts @@ -24,18 +24,10 @@ type RebuildOnboardModule = { ) => Promise; }; -type OnboardActionModule = { - runOnboard(options: RebuildRecreateOnboardOpts): Promise; -}; - function loadOnboardModule(): RebuildOnboardModule { return require("../../onboard") as RebuildOnboardModule; } -function loadOnboardActionModule(): OnboardActionModule { - return require("../onboard") as OnboardActionModule; -} - /** * Late-bound onboarding boundary for rebuild orchestration. Rebuild imports no * longer initialize the full onboarding graph, and focused tests can replace @@ -53,7 +45,7 @@ export const rebuildOnboardDependencies = { return loadOnboardModule().hydrateCredentialEnv(name); }, onboard(options: RebuildRecreateOnboardOpts): Promise { - return loadOnboardActionModule().runOnboard(options); + return loadOnboardModule().onboard(options); }, preflightAuthoritativeRebuildTarget( options: RebuildAuthoritativePreflightOptions, diff --git a/src/lib/actions/sandbox/start.ts b/src/lib/actions/sandbox/start.ts index ffcced6110e..31e875309cb 100644 --- a/src/lib/actions/sandbox/start.ts +++ b/src/lib/actions/sandbox/start.ts @@ -15,7 +15,6 @@ import { type SandboxInferenceInvocationResult, } from "./inference-invocation-probe"; import { withSandboxLifecycleLock } from "./gateway-state"; -export { withSandboxLifecycleLock }; import { getPersistedSandboxTargetGatewayName } from "./gateway-target"; import { resolveSandboxLifecycleProvider, diff --git a/src/lib/actions/sandbox/stop.test.ts b/src/lib/actions/sandbox/stop.test.ts index 3cdc16a8092..2c9d08b3e0a 100644 --- a/src/lib/actions/sandbox/stop.test.ts +++ b/src/lib/actions/sandbox/stop.test.ts @@ -872,7 +872,6 @@ describe("stopSandbox Ollama GPU release", () => { const result = stopSandbox("my-sandbox", h.deps); expect(result.exitCode).toBe(1); - expect(result.stopped).toBe(true); expect(result.message).toContain("curl: command not found"); expect(result.message).toContain("retry 'nemoclaw my-sandbox stop'"); }); diff --git a/src/lib/actions/sandbox/stop.ts b/src/lib/actions/sandbox/stop.ts index b92be1d5538..6e1e62b1508 100644 --- a/src/lib/actions/sandbox/stop.ts +++ b/src/lib/actions/sandbox/stop.ts @@ -117,7 +117,9 @@ export function discoverActiveOllamaSandboxNames( }`, }; } - const phases = new Map(parseEntries(result.output).map((entry) => [entry.name, entry.phase])); + const phases = new Map( + parseEntries(result.output).map((entry) => [entry.name, entry.phase]), + ); const activeSandboxes: string[] = []; for (const peerName of peerNames) { const phase = phases.get(peerName); @@ -146,20 +148,19 @@ function releaseStoppedSandboxOllamaModel( if (!isLocalOllamaRouteOwner(sandbox)) return { ok: true }; try { - const proxy = - require("../../inference/ollama/proxy") as typeof import("../../inference/ollama/proxy"); + const proxy = require("../../inference/ollama/proxy") as typeof import("../../inference/ollama/proxy"); const withOwnershipLock = deps.withOllamaModelOwnershipLock ?? proxy.withOllamaModelOwnershipLock; - const loadPersistedOllamaHost = deps.loadPersistedOllamaHost ?? proxy.loadPersistedOllamaHost; + const loadPersistedOllamaHost = + deps.loadPersistedOllamaHost ?? proxy.loadPersistedOllamaHost; return withOwnershipLock(() => { const selectedHost = loadPersistedOllamaHost(); if (!isLocalOllamaRouteOwner(sandbox, selectedHost)) return { ok: true }; const { sandboxes } = (deps.listSandboxes ?? registry.listSandboxes)(); const matchingPeers = matchingOllamaModelPeers(sandbox, sandboxes, selectedHost); - const discovery = (deps.discoverActiveOllamaSandboxNames ?? discoverActiveOllamaSandboxNames)( - matchingPeers, - deps.environment ?? process.env, - ); + const discovery = ( + deps.discoverActiveOllamaSandboxNames ?? discoverActiveOllamaSandboxNames + )(matchingPeers, deps.environment ?? process.env); if (!discovery.ok) { return { ok: false, @@ -233,7 +234,6 @@ function releaseStoppedSandboxOllamaModel( } export type { SandboxLifecycleResult } from "./runtime/lifecycle-runtime"; -export type SandboxStopResult = SandboxLifecycleResult & { stopped?: true }; export interface SandboxStopDeps { environment?: NodeJS.ProcessEnv; @@ -260,7 +260,10 @@ export interface SandboxStopDeps { * Stop the selected provider workload while preserving registry, workspace, * credentials, and shared gateway state. */ -export function stopSandbox(sandboxName: string, deps: SandboxStopDeps = {}): SandboxStopResult { +export function stopSandbox( + sandboxName: string, + deps: SandboxStopDeps = {}, +): SandboxLifecycleResult { return (deps.withLifecycleLockSync ?? withSandboxLifecycleLockSync)(sandboxName, () => stopSandboxWithinLifecycleFence(sandboxName, deps), ); @@ -269,7 +272,7 @@ export function stopSandbox(sandboxName: string, deps: SandboxStopDeps = {}): Sa function stopSandboxWithinLifecycleFence( sandboxName: string, deps: SandboxStopDeps, -): SandboxStopResult { +): SandboxLifecycleResult { const log = deps.log ?? console.log; const warn = deps.warn ?? console.warn; const sandbox = (deps.getSandbox ?? registry.getSandbox)(sandboxName); @@ -319,9 +322,7 @@ function stopSandboxWithinLifecycleFence( warn, ); } - if (!ollamaRelease.ok) { - return { exitCode: 1, message: ollamaRelease.message, stopped: true }; - } + if (!ollamaRelease.ok) return { exitCode: 1, message: ollamaRelease.message }; if (hermesPortableVerified) { log( outcome.state === "already-stopped" diff --git a/src/lib/adapters/openshell/forward-service.test.ts b/src/lib/adapters/openshell/forward-service.test.ts index f1f5789f240..0ac9c68b114 100644 --- a/src/lib/adapters/openshell/forward-service.test.ts +++ b/src/lib/adapters/openshell/forward-service.test.ts @@ -79,6 +79,17 @@ describe("OpenShell forward service", () => { expect(isForwardServiceListenerOwner(target, { probe })).toBe(false); }); + it("rejects ownership when a host probe times out", () => { + const lsofTimeout = vi.fn(() => ({ status: null, stdout: "" })); + expect(isForwardServiceListenerOwner(target, { probe: lsofTimeout })).toBe(false); + + const psTimeout = vi + .fn() + .mockReturnValueOnce({ status: 0, stdout: "4321\n" }) + .mockReturnValueOnce({ status: null, stdout: "" }); + expect(isForwardServiceListenerOwner(target, { probe: psTimeout })).toBe(false); + }); + it("detaches the OpenShell child and waits for its local port", () => { const unref = vi.fn(); const spawnDetached = vi.fn(() => ({ unref })); diff --git a/src/lib/adapters/openshell/forward-service.ts b/src/lib/adapters/openshell/forward-service.ts index 9a2a492122a..48959bd53c2 100644 --- a/src/lib/adapters/openshell/forward-service.ts +++ b/src/lib/adapters/openshell/forward-service.ts @@ -46,6 +46,8 @@ export interface ForwardServiceOwnerOptions { readonly probe?: ForwardServiceOwnerProbe; } +const FORWARD_OWNER_PROBE_TIMEOUT_MS = 5_000; + function isPort(value: unknown): value is number { return Number.isSafeInteger(value) && Number(value) >= 1 && Number(value) <= 65_535; } @@ -104,7 +106,10 @@ export function buildForwardServiceArgs(target: ForwardServiceTarget): string[] } function captureProcess(executable: string, args: readonly string[]) { - const result = spawnSync(executable, [...args], { encoding: "utf8" }); + const result = spawnSync(executable, [...args], { + encoding: "utf8", + timeout: FORWARD_OWNER_PROBE_TIMEOUT_MS, + }); return { status: result.status, stdout: result.stdout ?? "" }; } diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index c6f3a24f6f3..d7354275613 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -1515,9 +1515,6 @@ const sandboxCreateOrchestrationRuntime = { get getDashboardForwardPort() { return getDashboardForwardPort; }, - get reconcileOpenClawDashboardForwardReuse() { - return reconcileOpenClawDashboardForwardReuse; - }, readDcodeSelectionDrift: createDcodeSelectionDriftReader(runCaptureOpenshell, () => GATEWAY_NAME), getDefaultSandboxNameForAgent, getDockerDriverGatewayStateDir, @@ -2495,21 +2492,25 @@ const setupMessagingChannels = messagingChannelSetup.createSetupMessagingChannel isNonInteractive, prompt, }); + // ── Step 7: OpenClaw ───────────────────────────────────────────── const syncNemoClawConfigInSandbox = createNemoClawConfigSync({ getProviderSelectionConfig, run, openshellArgv, }); + const configureOpenclawSandbox = openclawSetup.createConfigureOpenclawSandbox({ syncNemoClawConfigInSandbox, reconcileWebSearch: openclawSetup.reconcileOpenClawWebSearchForReuse, }); + const setupOpenclaw = openclawSetup.createOpenclawSetup({ step, agentProductName, configureOpenclawSandbox, }); + const { buildChain, buildAgentVerifyChain, @@ -2521,7 +2522,6 @@ const { fetchGatewayAuthTokenFromSandbox, getDashboardForwardPort, printDashboard, - reconcileOpenClawDashboardForwardReuse, stopAllDashboardForwards, } = onboardDashboard.createOnboardDashboardHelpers({ runOpenshell, @@ -2632,6 +2632,7 @@ async function preflightAuthoritativeRebuildTarget( } } +// ── Main ───────────────────────────────────────────────────────── const wrappedOnboard = onboardEntryOptions.wrapOnboard(runOnboard, onboardSession); const onboard = onboardSessionBootstrap.wrapOnboardDeferredExit(wrappedOnboard); async function runOnboard(opts: OnboardOptions = {}): Promise { diff --git a/src/lib/onboard/agent-dashboard-forward.ts b/src/lib/onboard/agent-dashboard-forward.ts index d65042068c4..9ef697e67df 100644 --- a/src/lib/onboard/agent-dashboard-forward.ts +++ b/src/lib/onboard/agent-dashboard-forward.ts @@ -98,7 +98,11 @@ export async function ensureAgentDashboardForward(options: { .filter((port) => port !== declaredPrimaryPort || port === agentDashboardPort) .map(resolveDeclaredPort); const preservePorts = [ - ...new Set([agentDashboardPort, ...declaredPorts, optionalDashboardPort]), + ...new Set([ + agentDashboardPort, + ...declaredPorts, + optionalDashboardPort, + ]), ].filter(isValidForwardPort); const requestedDashboardUrl = !usesFixedApiPort && chatUiUrl diff --git a/src/lib/onboard/dashboard.ts b/src/lib/onboard/dashboard.ts index 34f0aa523fe..f19c177b4c9 100644 --- a/src/lib/onboard/dashboard.ts +++ b/src/lib/onboard/dashboard.ts @@ -22,7 +22,6 @@ import { } from "./agent-dashboard-forward"; import { fetchAgentWebAuthTokenFromSandbox as fetchAgentWebAuthToken } from "./agent-web-auth-token"; import * as dashboardAccess from "./dashboard-access"; -import { getDashboardReuseLifecycle } from "./dashboard/reuse-lifecycle"; import { type DashboardForwardOptions, normalizeDashboardForwardOptions, @@ -34,7 +33,6 @@ import { isPortBoundOnHost, type ListSandboxesFn, } from "./dashboard-port"; -import { fingerprintSandboxLiveIdentity } from "./sandbox-recreate-transaction"; import { ensureMessagingHostForwardForSandbox, productionForwardServiceRegistryContext, @@ -48,7 +46,6 @@ function looksLikeForwardPortConflict(diagnostic: string): boolean { } type CommandResult = { status: number | null }; -type SandboxLifecycleLock = (sandboxName: string, operation: () => Promise | T) => Promise; export interface OnboardDashboardDeps { runOpenshell(args: string[], opts?: Record): CommandResult; @@ -83,7 +80,6 @@ export interface OnboardDashboardDeps { dashboardPort?: number | null; hermesApiPort?: number | null; hermesDashboardPort?: number | null; - lifecycleGeneration?: string; lifecycleLiveIdentityFingerprint?: string; pendingRouteReservation?: true; } @@ -99,18 +95,6 @@ export interface OnboardDashboardDeps { sandbox: { gatewayName?: string | null; gatewayPort?: number | null } | null | undefined, ): string; }; - stopSandboxForDashboardReuse?( - sandboxName: string, - revalidateAtMutationEdge: () => void, - ): { exitCode: number; message?: string; stopped?: true }; - startSandboxForDashboardReuse?( - sandboxName: string, - revalidateAtMutationEdge: () => void, - ): Promise<{ - exitCode: number; - message?: string; - }>; - withSandboxLifecycleLock?: SandboxLifecycleLock; printAgentDashboardUi( sandboxName: string, token: string | null, @@ -159,7 +143,7 @@ export interface OnboardDashboardHelpers { ensureFinalizationDashboardForward( sandboxName: string, revalidateSandboxIdentity?: (operation: string) => void, - ): Promise; + ): number; ensureFinalizationAgentDashboardForward( sandboxName: string, agent: { name: string; forwardPort?: number | null; forward_ports?: number[] | null } | null, @@ -167,12 +151,7 @@ export interface OnboardDashboardHelpers { portReservation?: { releaseBeforeForward(agentName: string, port: number): Promise | void; }, - ): Promise; - reconcileOpenClawDashboardForwardReuse( - sandboxName: string, - chatUiUrl: string, - revalidateSandboxIdentity?: (operation: string) => void, - ): Promise; + ): Promise | number; ensureAgentFixedForward( sandboxName: string, port: number, @@ -409,133 +388,6 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa process.exit(1); } - async function reconcileOpenClawDashboardForwardReuse( - sandboxName: string, - chatUiUrl: string, - revalidateSandboxIdentity?: (operation: string) => void, - ): Promise { - const port = Number(getDashboardForwardPort(chatUiUrl)); - const isPortBound = deps.isPortBoundOnHost ?? isPortBoundOnHost; - if (!isPortBound(port)) return false; - const lifecycle = getDashboardReuseLifecycle(); - const withLifecycleLock = deps.withSandboxLifecycleLock ?? lifecycle?.withSandboxLifecycleLock; - if (!withLifecycleLock) { - throw new Error( - `Could not restart sandbox '${sandboxName}' to reconcile dashboard port ${String(port)}: sandbox lifecycle lock is unavailable.`, - ); - } - return await withLifecycleLock(sandboxName, async () => { - if (!isPortBound(port)) return false; - if (getRegistryOccupiedDashboardPorts(sandboxName, listSandboxes).has(String(port))) { - throw new Error( - `Registered dashboard port ${String(port)} is already occupied; it cannot be reallocated or adopted.`, - ); - } - - const stopSandbox = deps.stopSandboxForDashboardReuse ?? lifecycle?.stopSandbox; - const startSandbox = deps.startSandboxForDashboardReuse ?? lifecycle?.startSandbox; - if (!stopSandbox || !startSandbox) { - throw new Error( - `Could not restart sandbox '${sandboxName}' to reconcile dashboard port ${String(port)}: sandbox lifecycle is unavailable.`, - ); - } - - const readLiveIdentity = (gatewayName: string): string | null => - fingerprintSandboxLiveIdentity( - deps.runCaptureOpenshell(["sandbox", "get", "-g", gatewayName, sandboxName], { - ignoreError: true, - includeStderr: true, - }) ?? "", - ); - const registered = getSandbox?.(sandboxName); - const gatewayName = registered ? forwardService?.resolveGatewayName(registered) : null; - const observedIdentity = gatewayName ? readLiveIdentity(gatewayName) : null; - if ( - !registered || - !gatewayName || - !registered.lifecycleLiveIdentityFingerprint || - registered.lifecycleLiveIdentityFingerprint !== observedIdentity - ) { - throw new Error( - `Could not verify sandbox '${sandboxName}' before reconciling dashboard port ${String(port)}.`, - ); - } - if (ownsDashboardForward(sandboxName, gatewayName, port, chatUiUrl)) return true; - const assertSameSandbox = (operation: string): void => { - const current = getSandbox?.(sandboxName); - const currentGateway = current ? forwardService?.resolveGatewayName(current) : null; - const currentIdentity = currentGateway ? readLiveIdentity(currentGateway) : null; - if ( - !current || - currentGateway !== gatewayName || - current.lifecycleGeneration !== registered.lifecycleGeneration || - current.lifecycleLiveIdentityFingerprint !== - registered.lifecycleLiveIdentityFingerprint || - currentIdentity !== observedIdentity - ) { - throw new Error( - `Refusing to ${operation}: sandbox '${sandboxName}' identity changed during dashboard reconciliation.`, - ); - } - }; - - const revalidateAtStopBoundary = (): void => { - revalidateSandboxIdentity?.( - `restart sandbox '${sandboxName}' to reconcile dashboard forward ${String(port)}`, - ); - assertSameSandbox(`stop sandbox '${sandboxName}'`); - }; - const stopped = stopSandbox(sandboxName, revalidateAtStopBoundary); - const revalidateAtStartBoundary = (): void => { - revalidateSandboxIdentity?.( - `start sandbox '${sandboxName}' to reconcile dashboard forward ${String(port)}`, - ); - assertSameSandbox(`start sandbox '${sandboxName}'`); - }; - if (stopped.exitCode !== 0 && stopped.stopped !== true) { - throw new Error( - `Could not stop sandbox '${sandboxName}' to reconcile dashboard port ${String(port)}${ - stopped.message ? `: ${stopped.message}` : "." - }`, - ); - } - - if (isPortBound(port)) { - throw new Error( - `Registered dashboard port ${String(port)} remained occupied after sandbox '${sandboxName}' stopped; it cannot be adopted. Resolve the listener, run '${deps.cliName()} ${sandboxName} start', then retry onboarding.`, - ); - } - let started: { exitCode: number; message?: string }; - try { - started = await startSandbox(sandboxName, revalidateAtStartBoundary); - } catch (error) { - const detail = error instanceof Error ? error.message : String(error); - throw new Error( - `Could not restart sandbox '${sandboxName}' after releasing dashboard port ${String(port)}: ${detail}. The sandbox may remain stopped; run '${deps.cliName()} ${sandboxName} start' before retrying onboarding.`, - ); - } - if ( - started.exitCode !== 0 || - !isPortBound(port) || - !ownsDashboardForward(sandboxName, gatewayName, port, chatUiUrl) - ) { - throw new Error( - `Sandbox '${sandboxName}' did not restore dashboard port ${String(port)} after restart${ - started.message ? `: ${started.message}` : "." - } The sandbox may remain stopped; run '${deps.cliName()} ${sandboxName} start' before retrying onboarding.`, - ); - } - if (stopped.exitCode !== 0) { - throw new Error( - `Sandbox '${sandboxName}' was restored, but its stop cleanup failed${ - stopped.message ? `: ${stopped.message}` : "." - }`, - ); - } - return true; - }); - } - function ensureDashboardForward( sandboxName: string, chatUiUrl = `http://127.0.0.1:${CONTROL_UI_PORT}`, @@ -683,19 +535,14 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa * `CHAT_UI_URL`, so after the forward starts this writes the bound port to * `CHAT_UI_URL`. (#8970) */ - async function ensureFinalizationDashboardForward( + function ensureFinalizationDashboardForward( sandboxName: string, revalidateSandboxIdentity?: (operation: string) => void, - ): Promise { + ): number { const envUrl = process.env.CHAT_UI_URL; const persistedPort = envUrl ? null : getPersistedDashboardPort(sandboxName, listSandboxes); const requestedUrl = envUrl || (persistedPort === null ? undefined : `http://127.0.0.1:${String(persistedPort)}`); - await reconcileOpenClawDashboardForwardReuse( - sandboxName, - requestedUrl || `http://127.0.0.1:${CONTROL_UI_PORT}`, - revalidateSandboxIdentity, - ); const actualPort = ensureDashboardForward(sandboxName, requestedUrl, { allowPortReallocation: false, reuseExistingOpenClawForward: true, @@ -732,30 +579,23 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa }); } - async function ensureFinalizationAgentDashboardForward( + function ensureFinalizationAgentDashboardForward( sandboxName: string, agent: { name: string; forwardPort?: number | null; forward_ports?: number[] | null } | null, revalidateSandboxIdentity?: (operation: string) => void, portReservation?: { releaseBeforeForward(agentName: string, port: number): Promise | void; }, - ): Promise { + ): Promise | number { if (!agent) { return ensureFinalizationDashboardForward(sandboxName, revalidateSandboxIdentity); } const mayReuseOpenClawForward = agent.name === "openclaw"; if (mayReuseOpenClawForward) { const registeredPort = getPersistedDashboardPort(sandboxName, listSandboxes); - const requestedUrl = - process.env.CHAT_UI_URL || - (registeredPort === null - ? `http://127.0.0.1:${CONTROL_UI_PORT}` - : `http://127.0.0.1:${String(registeredPort)}`); - await reconcileOpenClawDashboardForwardReuse( - sandboxName, - requestedUrl, - revalidateSandboxIdentity, - ); + if (!process.env.CHAT_UI_URL && registeredPort !== null) { + process.env.CHAT_UI_URL = `http://127.0.0.1:${String(registeredPort)}`; + } } return ensureAgentDashboardForward(sandboxName, agent, { revalidateSandboxIdentity, @@ -986,7 +826,6 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa getDashboardForwardTarget, getWslHostAddress, printDashboard, - reconcileOpenClawDashboardForwardReuse, stopAllDashboardForwards, }; } diff --git a/src/lib/onboard/dashboard/reuse-lifecycle.test.ts b/src/lib/onboard/dashboard/reuse-lifecycle.test.ts deleted file mode 100644 index 4a396963ecf..00000000000 --- a/src/lib/onboard/dashboard/reuse-lifecycle.test.ts +++ /dev/null @@ -1,36 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { expect, it, vi } from "vitest"; - -import { getDashboardReuseLifecycle, withDashboardReuseLifecycle } from "./reuse-lifecycle"; - -it("keeps overlapping onboarding lifecycle scopes independent", async () => { - const first = { - startSandbox: vi.fn(), - stopSandbox: vi.fn(), - withSandboxLifecycleLock: vi.fn(), - }; - const second = { - startSandbox: vi.fn(), - stopSandbox: vi.fn(), - withSandboxLifecycleLock: vi.fn(), - }; - let releaseFirst!: () => void; - const firstPaused = new Promise((resolve) => { - releaseFirst = resolve; - }); - - const firstOperation = withDashboardReuseLifecycle(first, async () => { - expect(getDashboardReuseLifecycle()).toBe(first); - await firstPaused; - expect(getDashboardReuseLifecycle()).toBe(first); - }); - await withDashboardReuseLifecycle(second, async () => { - expect(getDashboardReuseLifecycle()).toBe(second); - }); - releaseFirst(); - await firstOperation; - - expect(getDashboardReuseLifecycle()).toBeUndefined(); -}); diff --git a/src/lib/onboard/dashboard/reuse-lifecycle.ts b/src/lib/onboard/dashboard/reuse-lifecycle.ts deleted file mode 100644 index 83a66a84a63..00000000000 --- a/src/lib/onboard/dashboard/reuse-lifecycle.ts +++ /dev/null @@ -1,29 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { AsyncLocalStorage } from "node:async_hooks"; - -export type DashboardReuseLifecycle = { - stopSandbox( - sandboxName: string, - revalidateAtMutationEdge: () => void, - ): { exitCode: number; message?: string; stopped?: true }; - startSandbox( - sandboxName: string, - revalidateAtMutationEdge: () => void, - ): Promise<{ exitCode: number; message?: string }>; - withSandboxLifecycleLock(sandboxName: string, operation: () => Promise | T): Promise; -}; - -const lifecycleStorage = new AsyncLocalStorage(); - -export function getDashboardReuseLifecycle(): DashboardReuseLifecycle | undefined { - return lifecycleStorage.getStore(); -} - -export function withDashboardReuseLifecycle( - lifecycle: DashboardReuseLifecycle, - operation: () => Promise, -): Promise { - return lifecycleStorage.run(lifecycle, operation); -} diff --git a/src/lib/onboard/sandbox-create/orchestration.ts b/src/lib/onboard/sandbox-create/orchestration.ts index 1030b71d1e6..4fb8ba50361 100644 --- a/src/lib/onboard/sandbox-create/orchestration.ts +++ b/src/lib/onboard/sandbox-create/orchestration.ts @@ -1331,7 +1331,6 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche formatSandboxAgentName, formatSandboxBuildEstimateNote, getDashboardForwardPort, - reconcileOpenClawDashboardForwardReuse, readDcodeSelectionDrift, getDefaultSandboxNameForAgent, getDockerDriverGatewayStateDir, @@ -1714,7 +1713,6 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche gatewayPort: GATEWAY_PORT, manageDashboard, ensureDashboardForward, - reconcileOpenClawDashboardForwardReuse, hermesDashboardForwarding, updateReusedSandboxMetadata, releaseDashboardPort: dashboardPortReservationScope.release, diff --git a/src/lib/onboard/sandbox-reuse.test.ts b/src/lib/onboard/sandbox-reuse.test.ts index 743c6e4a37f..1b320781d82 100644 --- a/src/lib/onboard/sandbox-reuse.test.ts +++ b/src/lib/onboard/sandbox-reuse.test.ts @@ -207,7 +207,6 @@ describe("applyReusedSandboxDashboardState", () => { it("launches the registered OpenClaw port when reuse finds no listener", async () => { const releaseDashboardPort = vi.fn(async () => undefined); - const reconcileOpenClawDashboardForwardReuse = vi.fn(async () => false); const ensureDashboardForward = vi.fn(() => 18_789); const result = await restoreReusedSandboxDashboardState({ @@ -231,7 +230,6 @@ describe("applyReusedSandboxDashboardState", () => { getSandbox: () => ({ dashboardPort: 18_789 }) as never, releaseDashboardPort, ensureDashboardForward, - reconcileOpenClawDashboardForwardReuse, hermesDashboardForwarding: { resolveStateForPort: vi.fn(() => ({ enabled: false, config: null })), ensureForState: vi.fn(), @@ -241,11 +239,6 @@ describe("applyReusedSandboxDashboardState", () => { }); expect(releaseDashboardPort).toHaveBeenCalledOnce(); - expect(reconcileOpenClawDashboardForwardReuse).toHaveBeenCalledWith( - "reuse-me", - "http://127.0.0.1:18789", - undefined, - ); expect(ensureDashboardForward).toHaveBeenCalledWith("reuse-me", "http://127.0.0.1:18789", { reuseExistingOpenClawForward: true, }); diff --git a/src/lib/onboard/sandbox-reuse.ts b/src/lib/onboard/sandbox-reuse.ts index 9616b9d4d81..d1a3e44af1f 100644 --- a/src/lib/onboard/sandbox-reuse.ts +++ b/src/lib/onboard/sandbox-reuse.ts @@ -100,7 +100,6 @@ export interface ReusedSandboxDashboardStateInput { gatewayName: string; gatewayPort: number; manageDashboard?: boolean; - preparedOpenClawDashboardPort?: number; getSandbox?(sandboxName: string): SandboxEntry | null; ensureDashboardForward( sandboxName: string, @@ -110,11 +109,6 @@ export interface ReusedSandboxDashboardStateInput { revalidateSandboxIdentity?: (operation: string) => void; }, ): number; - reconcileOpenClawDashboardForwardReuse?( - sandboxName: string, - chatUiUrl: string, - revalidateSandboxIdentity?: (operation: string) => void, - ): Promise; hermesDashboardForwarding: ReusedSandboxDashboardForwarding; updateSandbox?(sandboxName: string, updates: Partial): unknown; revalidateSandboxIdentity?(operation: string): void; @@ -150,17 +144,17 @@ export function applyReusedSandboxDashboardState( `Sandbox '${input.sandboxName}' was created without remote dashboard exposure. Re-run onboarding with NEMOCLAW_DASHBOARD_BIND=0.0.0.0 and --recreate-sandbox before opening a remote bind.`, ); } - input.revalidateSandboxIdentity?.(`restore dashboard state for sandbox '${input.sandboxName}'`); + input.revalidateSandboxIdentity?.( + `restore dashboard state for sandbox '${input.sandboxName}'`, + ); const reuseExistingOpenClawForward = input.agent == null || input.agent.name === "openclaw"; const dashboardPort = manageDashboard - ? reuseExistingOpenClawForward && input.preparedOpenClawDashboardPort !== undefined - ? input.preparedOpenClawDashboardPort - : input.ensureDashboardForward(input.sandboxName, input.chatUiUrl, { - ...(reuseExistingOpenClawForward ? { reuseExistingOpenClawForward: true } : {}), - ...(input.revalidateSandboxIdentity - ? { revalidateSandboxIdentity: input.revalidateSandboxIdentity } - : {}), - }) + ? input.ensureDashboardForward(input.sandboxName, input.chatUiUrl, { + ...(reuseExistingOpenClawForward ? { reuseExistingOpenClawForward: true } : {}), + ...(input.revalidateSandboxIdentity + ? { revalidateSandboxIdentity: input.revalidateSandboxIdentity } + : {}), + }) : 0; const chatUiUrl = manageDashboard ? `http://127.0.0.1:${dashboardPort}` : input.chatUiUrl; if (manageDashboard) { @@ -222,20 +216,9 @@ export async function restoreReusedSandboxDashboardState( const chatUiUrl = registeredOpenClawDashboardPort ? `http://127.0.0.1:${String(registeredOpenClawDashboardPort)}` : input.chatUiUrl; - const reconciled = - (input.manageDashboard ?? true) && reusesOpenClaw - ? await input.reconcileOpenClawDashboardForwardReuse?.( - input.sandboxName, - chatUiUrl, - input.revalidateSandboxIdentity, - ) - : false; return applyReusedSandboxDashboardState({ ...input, chatUiUrl, - ...(reconciled && registeredOpenClawDashboardPort - ? { preparedOpenClawDashboardPort: registeredOpenClawDashboardPort } - : {}), }); } diff --git a/test/e2e/live/double-onboard.test.ts b/test/e2e/live/double-onboard.test.ts index d0540e3d7f7..e9e2f9244d9 100644 --- a/test/e2e/live/double-onboard.test.ts +++ b/test/e2e/live/double-onboard.test.ts @@ -32,7 +32,6 @@ const PHASE_TIMEOUT_MS = Number(process.env.NEMOCLAW_E2E_PHASE_TIMEOUT_MS ?? 1_2 const ONBOARD_TIMEOUT_MS = execTimeout(PHASE_TIMEOUT_MS); const PROBE_ATTEMPTS = Number(process.env.NEMOCLAW_E2E_PROBE_ATTEMPTS ?? 3); const PROBE_DELAY_MS = Number(process.env.NEMOCLAW_E2E_PROBE_DELAY_SECONDS ?? 3) * 1_000; -const PROBE_TIMEOUT_MS = Number(process.env.NEMOCLAW_E2E_PROBE_TIMEOUT_SECONDS ?? 180) * 1_000; const RECOVERY_PROBE_TIMEOUT_MS = Number(process.env.NEMOCLAW_E2E_RECOVERY_PROBE_TIMEOUT_SECONDS ?? 180) * 1_000; const TEST_TIMEOUT_MS = testTimeout(90 * 60_000); @@ -114,35 +113,38 @@ async function runOnboard( }); } -async function runProbeOnlyConnect( +async function waitForDashboardReachability( host: HostCliClient, - sandboxName: string, - artifactName: string, -): Promise { - return await host.command( - "bash", - [ - "-lc", + port: string, + expectedReachable: boolean, + artifactPrefix: string, +): Promise<{ reachable: boolean; output: string }> { + let reachable = false; + let output = ""; + for (let attempt = 1; attempt <= PROBE_ATTEMPTS; attempt += 1) { + const result = await host.command( + "curl", [ - "set +e", - 'log="$(mktemp)"', - '"$1" "$2" "$3" connect --probe-only >"$log" 2>&1', - "rc=$?", - 'cat "$log"', - 'rm -f "$log"', - 'exit "$rc"', - ].join("\n"), - "nemoclaw-probe-connect", - process.execPath, - CLI_ENTRYPOINT, - sandboxName, - ], - { - artifactName, - env: commandEnv(), - timeoutMs: PROBE_TIMEOUT_MS, - }, - ); + "--silent", + "--show-error", + "--output", + "/dev/null", + "--max-time", + "5", + `http://127.0.0.1:${port}/`, + ], + { + artifactName: `${artifactPrefix}-attempt-${attempt}`, + env: commandEnv(), + timeoutMs: 15_000, + }, + ); + output = resultText(result); + reachable = result.exitCode === 0 && !result.timedOut; + if (reachable === expectedReachable) break; + if (attempt < PROBE_ATTEMPTS) await sleep(PROBE_DELAY_MS); + } + return { reachable, output }; } async function cleanupDoubleOnboardState( @@ -234,44 +236,6 @@ function dashboardPortFromList(output: string, sandboxName: string): string | un return undefined; } -function forwardOwnerForPort(output: string, port: string): string | undefined { - for (const line of stripAnsi(output).split("\n")) { - const parts = line.trim().split(/\s+/); - if (parts.length < 5 || parts[0]?.toLowerCase() === "sandbox") continue; - const status = parts.slice(4).join(" ").toLowerCase(); - if (parts[2] === port && status.includes("running")) return parts[0]; - } - return undefined; -} - -async function waitForForwardOwner( - sandbox: SandboxClient, - port: string, - owner: string | undefined, - artifactPrefix: string, -): Promise<{ - owner: string | undefined; - output: string; - querySucceeded: boolean; -}> { - let observedOwner: string | undefined; - let lastOutput = ""; - let querySucceeded = false; - for (let attempt = 1; attempt <= PROBE_ATTEMPTS; attempt += 1) { - const result = await sandbox.openshell(["forward", "list"], { - artifactName: `${artifactPrefix}-attempt-${attempt}`, - env: commandEnv(), - timeoutMs: 30_000, - }); - lastOutput = resultText(result); - querySucceeded = result.exitCode === 0 && !result.timedOut; - observedOwner = querySucceeded ? forwardOwnerForPort(lastOutput, port) : undefined; - if (querySucceeded && observedOwner === owner) break; - if (attempt < PROBE_ATTEMPTS) await sleep(PROBE_DELAY_MS); - } - return { owner: observedOwner, output: lastOutput, querySucceeded }; -} - function hasOwn(object: object, key: string): boolean { return Object.prototype.hasOwnProperty.call(object, key); } @@ -593,36 +557,17 @@ test( env: commandEnv(), timeoutMs: 60_000, }); + expect(listAfterSecond.exitCode, resultText(listAfterSecond)).toBe(0); + expect(stripAnsi(listAfterSecond.stdout)).toContain(SANDBOX_A); const portAfterSecond = dashboardPortFromList(listAfterSecond.stdout, SANDBOX_A); - const dashboardAfterSecond = await host.command( - "curl", - [ - "--silent", - "--show-error", - "--fail", - "--output", - "/dev/null", - "--retry", - "5", - "--retry-connrefused", - "--retry-delay", - "2", - "--connect-timeout", - "5", - "--max-time", - "30", - `http://127.0.0.1:${portAfterSecond ?? "0"}/`, - ], - { - artifactName: "phase-3-dashboard-after-second-onboard", - env: commandEnv(), - timeoutMs: 45_000, - }, + expect(portAfterSecond, resultText(listAfterSecond)).toBe(portAfterFirst); + const dashboardAfterSecond = await waitForDashboardReachability( + host, + portAfterSecond ?? "", + true, + "phase-3-dashboard-after-second-onboard", ); - expect( - `${listAfterSecond.exitCode}:${portAfterSecond}:${dashboardAfterSecond.exitCode}:${dashboardAfterSecond.timedOut}`, - `${resultText(listAfterSecond)}\n${resultText(dashboardAfterSecond)}`, - ).toBe(`0:${portAfterFirst}:0:false`); + expect(dashboardAfterSecond.reachable, dashboardAfterSecond.output).toBe(true); progress.phase("recreate same sandbox on existing gateway"); const gatewayBeforeRecreate = await gatewayRuntimeId(gateway); @@ -703,39 +648,20 @@ test( expect(portB, `nemoclaw list did not show ${SANDBOX_B} dashboard: ${list.stdout}`).toBeTruthy(); expect(portB).not.toBe(portA); - await sandbox.openshell(["forward", "stop", portB ?? ""], { - artifactName: "phase-4-stop-sandbox-b-dashboard-forward", - env: commandEnv(), - timeoutMs: 30_000, - }); - let probe: ShellProbeResult | undefined; - for (let attempt = 1; attempt <= PROBE_ATTEMPTS; attempt += 1) { - probe = await runProbeOnlyConnect( - host, - SANDBOX_B, - `phase-4-probe-connect-sandbox-b-attempt-${attempt}`, - ); - if (probe.exitCode === 0 && !probe.timedOut) break; - if (attempt < PROBE_ATTEMPTS) await sleep(PROBE_DELAY_MS); - } - expect(probe?.exitCode, probe ? resultText(probe) : "probe did not run").toBe(0); - expect(probe?.timedOut, probe ? resultText(probe) : "probe did not run").toBe(false); - - const restoredForwardB = await waitForForwardOwner( - sandbox, - portB ?? "", - SANDBOX_B, - "phase-4-openshell-forward-list-b", - ); - expect(restoredForwardB.owner, restoredForwardB.output).toBe(SANDBOX_B); - - const retainedForwardA = await waitForForwardOwner( - sandbox, + const dashboardABeforeStop = await waitForDashboardReachability( + host, portA ?? "", - SANDBOX_A, - "phase-4-openshell-forward-list-a", + true, + "phase-4-dashboard-a-before-stop", + ); + expect(dashboardABeforeStop.reachable, dashboardABeforeStop.output).toBe(true); + const dashboardBBeforeStop = await waitForDashboardReachability( + host, + portB ?? "", + true, + "phase-4-dashboard-b-before-stop", ); - expect(retainedForwardA.owner, retainedForwardA.output).toBe(SANDBOX_A); + expect(dashboardBBeforeStop.reachable, dashboardBBeforeStop.output).toBe(true); progress.phase("stop sibling sandbox without disturbing the first forward"); const stopB = await command(host, [SANDBOX_B, "stop"], { @@ -745,14 +671,13 @@ test( }); expect(stopB.exitCode, resultText(stopB)).toBe(0); - const releasedForwardB = await waitForForwardOwner( - sandbox, + const releasedForwardB = await waitForDashboardReachability( + host, portB ?? "", - undefined, - "phase-4-openshell-forward-list-b-after-stop", + false, + "phase-4-dashboard-b-after-stop", ); - expect(releasedForwardB.querySucceeded, releasedForwardB.output).toBe(true); - expect(releasedForwardB.owner, releasedForwardB.output).toBeUndefined(); + expect(releasedForwardB.reachable, releasedForwardB.output).toBe(false); const stoppedStatusB = await command(host, [SANDBOX_B, "status"], { artifactName: "phase-4-nemoclaw-status-sandbox-b-after-stop", @@ -764,13 +689,13 @@ test( expect(stoppedStatusTextB).toContain("sandbox_container_stopped"); expect(stoppedStatusTextB).not.toContain("sandbox_dashboard_port_conflict"); - const retainedForwardAAfterStop = await waitForForwardOwner( - sandbox, + const retainedForwardAAfterStop = await waitForDashboardReachability( + host, portA ?? "", - SANDBOX_A, - "phase-4-openshell-forward-list-a-after-b-stop", + true, + "phase-4-dashboard-a-after-b-stop", ); - expect(retainedForwardAAfterStop.owner, retainedForwardAAfterStop.output).toBe(SANDBOX_A); + expect(retainedForwardAAfterStop.reachable, retainedForwardAAfterStop.output).toBe(true); const startB = await command(host, [SANDBOX_B, "start"], { artifactName: "phase-4-nemoclaw-start-sandbox-b", @@ -778,13 +703,13 @@ test( timeoutMs: PHASE_TIMEOUT_MS, }); expect(startB.exitCode, resultText(startB)).toBe(0); - const restoredForwardBAfterStart = await waitForForwardOwner( - sandbox, + const restoredForwardBAfterStart = await waitForDashboardReachability( + host, portB ?? "", - SANDBOX_B, - "phase-4-openshell-forward-list-b-after-start", + true, + "phase-4-dashboard-b-after-start", ); - expect(restoredForwardBAfterStart.owner, restoredForwardBAfterStart.output).toBe(SANDBOX_B); + expect(restoredForwardBAfterStart.reachable, restoredForwardBAfterStart.output).toBe(true); progress.phase("replace sandbox after stale registry refusal"); // Phase 5: direct OpenShell deletion leaves a stale registry entry that @@ -924,20 +849,18 @@ test( secondOnboardReusedGateway: gatewayAfterSecond === gatewayBeforeSecond && secondText.includes("Reusing healthy NemoClaw gateway.") && - dashboardAfterSecond.exitCode === 0 && - !dashboardAfterSecond.timedOut, + dashboardAfterSecond.reachable, thirdOnboardPreservedSibling: sandboxAAfterThird.exitCode === 0 && sandboxBAfterThird.exitCode === 0, distinctDashboardPorts: Boolean(portA && portB && portA !== portB), selectedStopReleasedOnlySelectedForward: stopB.exitCode === 0 && - releasedForwardB.querySucceeded && - releasedForwardB.owner === undefined && - retainedForwardAAfterStop.owner === SANDBOX_A && + !releasedForwardB.reachable && + retainedForwardAAfterStop.reachable && stoppedStatusTextB.includes("sandbox_container_stopped") && !stoppedStatusTextB.includes("sandbox_dashboard_port_conflict") && startB.exitCode === 0 && - restoredForwardBAfterStart.owner === SANDBOX_B, + restoredForwardBAfterStart.reachable, staleRegistryRecovered: rebuild.exitCode === 0, gatewayStopGuidance: /Recovered NemoClaw gateway runtime|gateway is no longer configured after restart\/rebuild|gateway is still refusing connections after restart|gateway trust material rotated after restart/.test( diff --git a/test/e2e/live/onboard-resume.test.ts b/test/e2e/live/onboard-resume.test.ts index 49143831fd4..fe94c8b644e 100644 --- a/test/e2e/live/onboard-resume.test.ts +++ b/test/e2e/live/onboard-resume.test.ts @@ -10,7 +10,6 @@ import { ONBOARD_NO_RECREATE_COMMAND_TIMEOUT_MS, ONBOARD_RESUME_TEST_TIMEOUT_MS, } from "../../../tools/e2e/onboard-timeout-contract.mts"; -import { parseOpenShellSandboxId } from "../../../src/lib/adapters/openshell/sandbox-identity.ts"; import { parseSandboxPhase } from "../../../src/lib/state/gateway.ts"; import { execTimeout, testTimeout } from "../../helpers/timeouts.ts"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; @@ -108,29 +107,6 @@ function markSessionInProgress(file: string): void { fs.writeFileSync(file, JSON.stringify(session, null, 2), "utf8"); } -function registeredDashboardPort(): number { - const registry = JSON.parse(fs.readFileSync(REGISTRY_FILE, "utf8")) as { - sandboxes?: Record; - }; - return Number(registry.sandboxes?.[SANDBOX_NAME]?.dashboardPort); -} - -function requestDashboard(host: HostCliClient, port: number, artifactName: string) { - return host.command( - "curl", - [ - "--silent", - "--show-error", - "--output", - "/dev/null", - "--max-time", - "10", - `http://127.0.0.1:${String(port)}/`, - ], - { artifactName, env: buildAvailabilityProbeEnv(), timeoutMs: 20_000 }, - ); -} - function interruptedSessionSummary(session: SessionStateInterrupted): Record { return { status: session.status, @@ -216,7 +192,6 @@ test( "resume proves recreated sandbox provider attachments are selectively reconciled", "host trust-store anchor corporate CA source is baked and merged after resume", "an unreachable committed route pauses at final verification and completes after repair", - "resume reuses the same sandbox and registered dashboard port after restoring its forward", "implicit resume is detected and --fresh suppresses that auto-resume", ], }); @@ -622,6 +597,7 @@ test( expect(unavailableResumeText).not.toContain( `Deleting and recreating sandbox '${SANDBOX_NAME}'`, ); + expect(unavailableResumeText).not.toContain(`Sandbox '${SANDBOX_NAME}' created`); const paused = readSession(SESSION_FILE); await artifacts.writeJson("phase-3-5-session-route-unavailable.json", { @@ -643,24 +619,7 @@ test( requireAuth: true, requireAuthModels: true, }); - const sandboxBeforeRepairedResume = await sandbox.openshell(["sandbox", "get", SANDBOX_NAME], { - artifactName: "phase-3-5-sandbox-before-repaired-resume", - env: buildAvailabilityProbeEnv(), - timeoutMs: 30_000, - }); - const sandboxIdBeforeRepairedResume = parseOpenShellSandboxId( - resultText(sandboxBeforeRepairedResume), - ); - expect(sandboxIdBeforeRepairedResume).not.toBeNull(); - const dashboardPortBeforeRepairedResume = registeredDashboardPort(); - const dashboardBeforeRepairedResume = await requestDashboard( - host, - dashboardPortBeforeRepairedResume, - "phase-3-5-dashboard-before-repaired-resume", - ); - expect(dashboardBeforeRepairedResume.exitCode, resultText(dashboardBeforeRepairedResume)).toBe( - 0, - ); + expect(fake.baseUrl).toBe(`http://${fakePublicHost}:${String(fakePort)}/v1`); const repairedResumeRun = await host.command( "node", @@ -674,23 +633,11 @@ test( ); const repairedResumeText = `${repairedResumeRun.stdout}\n${repairedResumeRun.stderr}`; expect(repairedResumeRun.exitCode, repairedResumeText).toBe(0); - expect(repairedResumeText).not.toContain("Registered dashboard port"); + expect(repairedResumeText).toContain("is ready"); expect(repairedResumeText).not.toContain(`Deleting and recreating sandbox '${SANDBOX_NAME}'`); - const sandboxAfterRepairedResume = await sandbox.openshell(["sandbox", "get", SANDBOX_NAME], { - artifactName: "phase-3-5-sandbox-after-repaired-resume", - env: buildAvailabilityProbeEnv(), - timeoutMs: 30_000, - }); - expect(parseOpenShellSandboxId(resultText(sandboxAfterRepairedResume))).toBe( - sandboxIdBeforeRepairedResume, - ); - expect(registeredDashboardPort()).toBe(dashboardPortBeforeRepairedResume); - const dashboardAfterRepairedResume = await requestDashboard( - host, - dashboardPortBeforeRepairedResume, - "phase-3-5-dashboard-after-repaired-resume", - ); - expect(dashboardAfterRepairedResume.exitCode, resultText(dashboardAfterRepairedResume)).toBe(0); + expect(repairedResumeText).not.toContain(`Sandbox '${SANDBOX_NAME}' created`); + const repaired = readSession(SESSION_FILE); + expect(repaired.status).toBe("complete"); // ────────────────────────────────────────────────────────────────── // Phase 4: implicit resume — a plain `onboard` auto-detects an @@ -722,6 +669,7 @@ test( implicitResumeText, ).toBe(true); expect(implicitResumeText).not.toContain(`Deleting and recreating sandbox '${SANDBOX_NAME}'`); + expect(implicitResumeText).not.toContain(`Sandbox '${SANDBOX_NAME}' created`); markSessionInProgress(SESSION_FILE); const freshRun = await host.command( @@ -746,11 +694,6 @@ test( expect(freshText).toContain("[e2e] Forced onboarding failure at step 'preflight'."); expect(freshText).not.toContain("(resume mode)"); expect(freshText).not.toContain(`Sandbox '${SANDBOX_NAME}' created`); - await artifacts.target.complete({ - id: "onboard-resume", - status: "passed", - resumeDashboardPort: dashboardPortBeforeRepairedResume, - resumeSandboxIdentityRetained: true, - }); + await artifacts.target.complete({ id: "onboard-resume", status: "passed" }); }, ); diff --git a/test/onboarding/onboard-finalization-dashboard-forward.test.ts b/test/onboarding/onboard-finalization-dashboard-forward.test.ts index c7850c81dc1..3a2c1760085 100644 --- a/test/onboarding/onboard-finalization-dashboard-forward.test.ts +++ b/test/onboarding/onboard-finalization-dashboard-forward.test.ts @@ -5,41 +5,16 @@ import { describe, expect, it, vi } from "vitest"; import { createOnboardDashboardHelpers } from "../../src/lib/onboard/dashboard"; import type { ListSandboxesFn } from "../../src/lib/onboard/dashboard-port"; -import { fingerprintSandboxLiveIdentity } from "../../src/lib/onboard/sandbox-recreate-transaction"; function harness(options: { listSandboxes: ListSandboxesFn; isPortBound?: (port: number) => boolean; ownsForward?: () => boolean; - registeredIdentity?: boolean; - sandboxIdentity?: () => string; - stopSandbox?: (sandboxName: string) => { - exitCode: number; - message?: string; - stopped?: true; - }; - startSandbox?: (sandboxName: string) => Promise<{ exitCode: number; message?: string }>; }) { const launch = vi.fn(); - const stopSandbox = vi.fn((sandboxName: string, revalidateAtMutationEdge: () => void) => { - revalidateAtMutationEdge(); - return (options.stopSandbox ?? (() => ({ exitCode: 0 })))(sandboxName); - }); - const startSandboxOperation = vi.fn(options.startSandbox ?? (async () => ({ exitCode: 0 }))); - const startSandbox = vi.fn(async (sandboxName: string, revalidateAtMutationEdge: () => void) => { - revalidateAtMutationEdge(); - return await startSandboxOperation(sandboxName); - }); - const recordedIdentity = fingerprintSandboxLiveIdentity( - `Id: ${options.sandboxIdentity?.() ?? "sandbox-id"}`, - ); const helpers = createOnboardDashboardHelpers({ runOpenshell: vi.fn(() => ({ status: 0 })), - runCaptureOpenshell: vi.fn((args) => - args[0] === "sandbox" - ? `Name: reonboard-test\nId: ${options.sandboxIdentity?.() ?? "sandbox-id"}\nState: Ready\n` - : "", - ), + runCaptureOpenshell: vi.fn(() => ""), openshellArgv: (args) => ["/usr/local/bin/openshell", ...args], cliName: () => "nemoclaw", agentProductName: () => "NemoClaw", @@ -50,17 +25,7 @@ function harness(options: { sleep: vi.fn(), printAgentDashboardUi: vi.fn(), listSandboxes: options.listSandboxes, - getSandbox: () => ({ - gatewayName: "nemoclaw", - dashboardPort: 18_790, - lifecycleGeneration: "generation-1", - lifecycleLiveIdentityFingerprint: - options.registeredIdentity === false ? undefined : (recordedIdentity ?? undefined), - }), isPortBoundOnHost: options.isPortBound ?? (() => false), - stopSandboxForDashboardReuse: stopSandbox, - startSandboxForDashboardReuse: startSandbox, - withSandboxLifecycleLock: async (_sandboxName, operation) => await operation(), forwardService: { executable: () => "/usr/local/bin/openshell", launch, @@ -69,11 +34,11 @@ function harness(options: { retireLegacy: vi.fn(() => 0), }, }); - return { helpers, launch, startSandbox, startSandboxOperation, stopSandbox }; + return { helpers, launch }; } describe("finalization dashboard ForwardTcp launch", () => { - it("launches the persisted dashboard port and publishes its URL", async () => { + it("launches the persisted dashboard port and publishes its URL", () => { vi.stubEnv("CHAT_UI_URL", undefined); const { helpers, launch } = harness({ listSandboxes: () => ({ @@ -81,9 +46,7 @@ describe("finalization dashboard ForwardTcp launch", () => { }), }); - await expect(helpers.ensureFinalizationDashboardForward("reonboard-test")).resolves.toBe( - 18_790, - ); + expect(helpers.ensureFinalizationDashboardForward("reonboard-test")).toBe(18_790); expect(launch).toHaveBeenCalledWith( expect.objectContaining({ gatewayName: "nemoclaw", @@ -95,26 +58,24 @@ describe("finalization dashboard ForwardTcp launch", () => { expect(process.env.CHAT_UI_URL).toBe("http://127.0.0.1:18790"); }); - it("fails closed when a foreign listener occupies the persisted port", async () => { + it("fails closed when a foreign or ambiguous listener occupies the persisted port", () => { vi.stubEnv("CHAT_UI_URL", undefined); - const { helpers, launch, startSandbox, stopSandbox } = harness({ + const { helpers, launch } = harness({ listSandboxes: () => ({ sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], }), isPortBound: (port) => port === 18_790, }); - await expect(helpers.ensureFinalizationDashboardForward("reonboard-test")).rejects.toThrow( - /cannot be adopted/u, + expect(() => helpers.ensureFinalizationDashboardForward("reonboard-test")).toThrow( + /cannot be reallocated or adopted/u, ); expect(launch).not.toHaveBeenCalled(); - expect(stopSandbox).toHaveBeenCalledOnce(); - expect(startSandbox).not.toHaveBeenCalled(); }); - it("reuses an owned dashboard forward without restarting the sandbox (#11074)", async () => { + it("reuses an exactly owned dashboard forward (#11074)", () => { vi.stubEnv("CHAT_UI_URL", undefined); - const { helpers, launch, startSandbox, stopSandbox } = harness({ + const { helpers, launch } = harness({ listSandboxes: () => ({ sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], }), @@ -122,190 +83,14 @@ describe("finalization dashboard ForwardTcp launch", () => { ownsForward: () => true, }); - await expect(helpers.ensureFinalizationDashboardForward("reonboard-test")).resolves.toBe( - 18_790, - ); - expect(stopSandbox).not.toHaveBeenCalled(); - expect(startSandbox).not.toHaveBeenCalled(); + expect(helpers.ensureFinalizationDashboardForward("reonboard-test")).toBe(18_790); expect(launch).not.toHaveBeenCalled(); expect(process.env.CHAT_UI_URL).toBe("http://127.0.0.1:18790"); }); - it("does not reconcile a reused dashboard forward twice", async () => { - vi.stubEnv("CHAT_UI_URL", undefined); - const { helpers, startSandbox, stopSandbox } = harness({ - listSandboxes: () => ({ - sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], - }), - isPortBound: (port) => port === 18_790, - ownsForward: () => true, - }); - - await helpers.reconcileOpenClawDashboardForwardReuse( - "reonboard-test", - "http://127.0.0.1:18790", - ); - await helpers.ensureFinalizationDashboardForward("reonboard-test"); - - expect(stopSandbox).not.toHaveBeenCalled(); - expect(startSandbox).not.toHaveBeenCalled(); - }); - - it("rechecks ownership instead of trusting an earlier reconciliation", async () => { - vi.stubEnv("CHAT_UI_URL", undefined); - let owned = true; - const { helpers, startSandbox, stopSandbox } = harness({ - listSandboxes: () => ({ - sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], - }), - isPortBound: (port) => port === 18_790, - ownsForward: () => owned, - }); - - await helpers.reconcileOpenClawDashboardForwardReuse( - "reonboard-test", - "http://127.0.0.1:18790", - ); - owned = false; - - await expect(helpers.ensureFinalizationDashboardForward("reonboard-test")).rejects.toThrow( - /remained occupied/u, - ); - expect(stopSandbox).toHaveBeenCalledOnce(); - expect(startSandbox).not.toHaveBeenCalled(); - }); - - it("rejects an ambiguous listener that remains after the reused sandbox stops", async () => { - vi.stubEnv("CHAT_UI_URL", undefined); - const { helpers, launch, startSandbox, stopSandbox } = harness({ - listSandboxes: () => ({ - sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], - }), - isPortBound: (port) => port === 18_790, - }); - - await expect(helpers.ensureFinalizationDashboardForward("reonboard-test")).rejects.toThrow( - /remained occupied.*run 'nemoclaw reonboard-test start'/u, - ); - expect(stopSandbox).toHaveBeenCalledWith("reonboard-test", expect.any(Function)); - expect(startSandbox).not.toHaveBeenCalled(); - expect(launch).not.toHaveBeenCalled(); - }); - - it("does not start a same-name replacement after the reused sandbox stops", async () => { + it("does not reuse a forward when another sandbox registers the same port", () => { vi.stubEnv("CHAT_UI_URL", undefined); - let bound = true; - let identity = "original-id"; - const { helpers, launch, startSandbox, startSandboxOperation, stopSandbox } = harness({ - listSandboxes: () => ({ - sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], - }), - isPortBound: (port) => port === 18_790 && bound, - sandboxIdentity: () => identity, - stopSandbox: () => { - bound = false; - identity = "replacement-id"; - return { exitCode: 0 }; - }, - }); - - await expect(helpers.ensureFinalizationDashboardForward("reonboard-test")).rejects.toThrow( - /identity changed.*may remain stopped/u, - ); - expect(stopSandbox).toHaveBeenCalledWith("reonboard-test", expect.any(Function)); - expect(startSandbox).toHaveBeenCalledWith("reonboard-test", expect.any(Function)); - expect(startSandboxOperation).not.toHaveBeenCalled(); - expect(launch).not.toHaveBeenCalled(); - }); - - it("does not restart a reused sandbox without a registered live identity", async () => { - vi.stubEnv("CHAT_UI_URL", undefined); - const { helpers, launch, startSandbox, stopSandbox } = harness({ - listSandboxes: () => ({ - sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], - }), - isPortBound: (port) => port === 18_790, - registeredIdentity: false, - }); - - await expect(helpers.ensureFinalizationDashboardForward("reonboard-test")).rejects.toThrow( - /Could not verify sandbox/u, - ); - expect(stopSandbox).not.toHaveBeenCalled(); - expect(startSandbox).not.toHaveBeenCalled(); - expect(launch).not.toHaveBeenCalled(); - }); - - it("does not cache a failed sandbox restart as reconciled", async () => { - vi.stubEnv("CHAT_UI_URL", undefined); - let bound = true; - let owned = false; - const startOutcomes = [ - async () => ({ exitCode: 1, message: "restart failed" }), - async () => { - bound = true; - owned = true; - return { exitCode: 0 }; - }, - ]; - const { helpers, launch, startSandbox, stopSandbox } = harness({ - listSandboxes: () => ({ - sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], - }), - isPortBound: (port) => port === 18_790 && bound, - ownsForward: () => owned, - stopSandbox: () => { - bound = false; - return { exitCode: 0 }; - }, - startSandbox: async () => await startOutcomes.shift()!(), - }); - - await expect(helpers.ensureFinalizationDashboardForward("reonboard-test")).rejects.toThrow( - /did not restore dashboard port.*restart failed/u, - ); - - bound = true; - await expect(helpers.ensureFinalizationDashboardForward("reonboard-test")).resolves.toBe( - 18_790, - ); - expect(stopSandbox).toHaveBeenCalledTimes(2); - expect(startSandbox).toHaveBeenCalledTimes(2); - expect(launch).not.toHaveBeenCalled(); - }); - - it("restores a sandbox after its stop cleanup fails", async () => { - vi.stubEnv("CHAT_UI_URL", undefined); - let bound = true; - let owned = false; - const { helpers, launch, startSandbox, stopSandbox } = harness({ - listSandboxes: () => ({ - sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], - }), - isPortBound: (port) => port === 18_790 && bound, - ownsForward: () => owned, - stopSandbox: () => { - bound = false; - return { exitCode: 1, message: "Ollama cleanup failed", stopped: true }; - }, - startSandbox: async () => { - bound = true; - owned = true; - return { exitCode: 0 }; - }, - }); - - await expect(helpers.ensureFinalizationDashboardForward("reonboard-test")).rejects.toThrow( - /was restored.*Ollama cleanup failed/u, - ); - expect(stopSandbox).toHaveBeenCalledOnce(); - expect(startSandbox).toHaveBeenCalledOnce(); - expect(launch).not.toHaveBeenCalled(); - }); - - it("does not reuse a forward when another sandbox registers the same port", async () => { - vi.stubEnv("CHAT_UI_URL", undefined); - const { helpers, launch, stopSandbox } = harness({ + const { helpers, launch } = harness({ listSandboxes: () => ({ sandboxes: [ { name: "reonboard-test", dashboardPort: 18_790 }, @@ -315,14 +100,13 @@ describe("finalization dashboard ForwardTcp launch", () => { isPortBound: (port) => port === 18_790, }); - await expect(helpers.ensureFinalizationDashboardForward("reonboard-test")).rejects.toThrow( - /cannot be reallocated/u, + expect(() => helpers.ensureFinalizationDashboardForward("reonboard-test")).toThrow( + /cannot be reallocated or adopted/u, ); - expect(stopSandbox).not.toHaveBeenCalled(); expect(launch).not.toHaveBeenCalled(); }); - it("enables owned-forward reuse only for OpenClaw agents during ordinary finalization", async () => { + it("enables owned-forward reuse only for OpenClaw agents", async () => { vi.stubEnv("CHAT_UI_URL", undefined); const openClaw = harness({ listSandboxes: () => ({ @@ -346,6 +130,7 @@ describe("finalization dashboard ForwardTcp launch", () => { sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], }), isPortBound: (port) => port === 18_790, + ownsForward: () => true, }); await expect( @@ -356,18 +141,15 @@ describe("finalization dashboard ForwardTcp launch", () => { undefined, ), ).rejects.toThrow(/cannot be reallocated/u); - expect(hermes.stopSandbox).not.toHaveBeenCalled(); }); - it("honors an explicit dashboard URL", async () => { + it("honors an explicit dashboard URL", () => { vi.stubEnv("CHAT_UI_URL", "http://127.0.0.1:19001"); const { helpers, launch } = harness({ listSandboxes: () => ({ sandboxes: [] }), }); - await expect(helpers.ensureFinalizationDashboardForward("reonboard-test")).resolves.toBe( - 19_001, - ); + expect(helpers.ensureFinalizationDashboardForward("reonboard-test")).toBe(19_001); expect(launch).toHaveBeenCalledWith(expect.objectContaining({ localPort: 19_001 })); }); }); From 72c5d12e6e34489dc7b9525f989fba7c82409b54 Mon Sep 17 00:00:00 2001 From: San Dang Date: Mon, 7 Sep 2026 15:05:30 +0700 Subject: [PATCH 14/20] fix(onboard): harden dashboard forward ownership --- src/lib/actions/sandbox/start.test.ts | 12 -- src/lib/actions/sandbox/start.ts | 2 - src/lib/actions/sandbox/stop.test.ts | 12 -- src/lib/actions/sandbox/stop.ts | 2 - .../openshell/forward-service.test.ts | 119 ++++++++++++++---- src/lib/adapters/openshell/forward-service.ts | 111 ++++++++++++++-- 6 files changed, 199 insertions(+), 59 deletions(-) diff --git a/src/lib/actions/sandbox/start.test.ts b/src/lib/actions/sandbox/start.test.ts index 94557c9ecb9..8557ff91f38 100644 --- a/src/lib/actions/sandbox/start.test.ts +++ b/src/lib/actions/sandbox/start.test.ts @@ -116,18 +116,6 @@ function harness(overrides: Partial = {}) { } describe("startSandbox", () => { - it("revalidates the selected sandbox at the start mutation boundary", async () => { - const revalidateAtMutationEdge = vi.fn(() => { - throw new Error("sandbox identity changed"); - }); - const h = harness({ revalidateAtMutationEdge }); - - await expect(startSandbox("my-sandbox", h.deps)).rejects.toThrow(/sandbox identity changed/u); - expect(revalidateAtMutationEdge).toHaveBeenCalledOnce(); - expect(h.recoverDockerDriverSandbox).not.toHaveBeenCalled(); - expect(h.recoverPortableSandbox).not.toHaveBeenCalled(); - }); - it("waits for OpenShell readiness before recovering sandbox processes (#8978)", async () => { const waitForSandboxReady = vi.fn(); const restoreProcesses = vi.fn(() => SUCCESSFUL_RECOVERY); diff --git a/src/lib/actions/sandbox/start.ts b/src/lib/actions/sandbox/start.ts index 31e875309cb..3a35164be90 100644 --- a/src/lib/actions/sandbox/start.ts +++ b/src/lib/actions/sandbox/start.ts @@ -80,7 +80,6 @@ export interface SandboxStartDeps { verifyGateway?: (sandboxName: string) => Promise; probeInferenceInvocation?: typeof probeSandboxInferenceInvocation; withLifecycleLock?: typeof withSandboxLifecycleLock; - revalidateAtMutationEdge?: () => void; log?: (message: string) => void; } @@ -193,7 +192,6 @@ async function startSandboxWithinLifecycleFence( }; const preflight = resolved.bundle.preflightDoctor.preflightLifecycle("start", input); if (preflight) return preflight; - deps.revalidateAtMutationEdge?.(); const result = resolved.lifecycle.start(input); if (result.exitCode !== 0) return result; if ("hermesPortableVerified" in result && result.hermesPortableVerified === true) { diff --git a/src/lib/actions/sandbox/stop.test.ts b/src/lib/actions/sandbox/stop.test.ts index 2c9d08b3e0a..5d04551f92a 100644 --- a/src/lib/actions/sandbox/stop.test.ts +++ b/src/lib/actions/sandbox/stop.test.ts @@ -273,18 +273,6 @@ describe("discoverActiveOllamaSandboxNames", () => { }); describe("stopSandbox", () => { - it("revalidates the selected sandbox at the stop mutation boundary", () => { - const revalidateAtMutationEdge = vi.fn(() => { - throw new Error("sandbox identity changed"); - }); - const h = harness({ revalidateAtMutationEdge }); - - expect(() => stopSandbox("my-sandbox", h.deps)).toThrow(/sandbox identity changed/u); - expect(revalidateAtMutationEdge).toHaveBeenCalledOnce(); - expect(h.stopSandboxChannels).not.toHaveBeenCalled(); - expect(h.dockerStop).not.toHaveBeenCalled(); - }); - it("gracefully stops in-sandbox channels before stopping the container (#6026)", () => { const h = harness(); diff --git a/src/lib/actions/sandbox/stop.ts b/src/lib/actions/sandbox/stop.ts index 6e1e62b1508..6b0a27d7f3d 100644 --- a/src/lib/actions/sandbox/stop.ts +++ b/src/lib/actions/sandbox/stop.ts @@ -251,7 +251,6 @@ export interface SandboxStopDeps { loadPersistedOllamaHost?: () => OllamaHostRoute | null; withOllamaModelOwnershipLock?: typeof import("../../inference/ollama/proxy").withOllamaModelOwnershipLock; withLifecycleLockSync?: typeof withSandboxLifecycleLockSync; - revalidateAtMutationEdge?: () => void; log?: (message: string) => void; warn?: (message: string) => void; } @@ -293,7 +292,6 @@ function stopSandboxWithinLifecycleFence( const preflight = resolved.bundle.preflightDoctor.preflightLifecycle("stop", input); if (preflight) return preflight; - deps.revalidateAtMutationEdge?.(); let channelsStopped = false; const outcome = resolved.lifecycle.stop(input, { beforeStop() { diff --git a/src/lib/adapters/openshell/forward-service.test.ts b/src/lib/adapters/openshell/forward-service.test.ts index 0ac9c68b114..7bfa224621c 100644 --- a/src/lib/adapters/openshell/forward-service.test.ts +++ b/src/lib/adapters/openshell/forward-service.test.ts @@ -1,7 +1,11 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it, vi } from "vitest"; +import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it, vi } from "vitest"; import { buildForwardServiceArgs, @@ -21,6 +25,45 @@ const target: ForwardServiceTarget = { targetPort: 18_789, }; +const ownerTarget: ForwardServiceTarget = { ...target, executable: process.execPath }; +const temporaryDirectories: string[] = []; + +function createLinuxOwnerFixture(actualExecutable?: string) { + const root = mkdtempSync(path.join(os.tmpdir(), "nemoclaw-forward-owner-")); + temporaryDirectories.push(root); + const procRoot = path.join(root, "proc"); + const binRoot = path.join(root, "bin"); + mkdirSync(path.join(procRoot, "net"), { recursive: true }); + mkdirSync(path.join(procRoot, "4321", "fd"), { recursive: true }); + mkdirSync(binRoot); + const executable = path.join(binRoot, "openshell"); + const runtime = actualExecutable ? path.join(binRoot, actualExecutable) : executable; + writeFileSync(executable, ""); + writeFileSync(runtime, ""); + writeFileSync( + path.join(procRoot, "net", "tcp"), + " 0: 0100007F:4965 00000000:0000 0A 00000000:00000000 00:00000000 00000000 998 0 12345 1\n", + ); + symlinkSync("socket:[12345]", path.join(procRoot, "4321", "fd", "7")); + symlinkSync(runtime, path.join(procRoot, "4321", "exe")); + return { procRoot, target: { ...target, executable } }; +} + +function darwinOwnerProbe(commandLine: string, finalListener = "4321\n") { + return vi + .fn() + .mockReturnValueOnce({ status: 0, stdout: "4321\n" }) + .mockReturnValueOnce({ status: 0, stdout: `p4321\nftxt\nn${process.execPath}\n` }) + .mockReturnValueOnce({ status: 0, stdout: commandLine }) + .mockReturnValueOnce({ status: 0, stdout: finalListener }); +} + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }); + } +}); + describe("OpenShell forward service", () => { it("builds the direct ForwardTcp command with explicit gateway authority", () => { expect(buildForwardServiceArgs(target)).toEqual([ @@ -47,47 +90,75 @@ describe("OpenShell forward service", () => { }); it("proves the exact direct ForwardTcp listener before reuse", () => { - const expected = [target.executable, ...buildForwardServiceArgs(target)].join(" "); - const probe = vi.fn((executable: string) => - executable === "lsof" - ? { status: 0, stdout: "4321\n" } - : { status: 0, stdout: `${expected}\n` }, - ); + const expected = [ownerTarget.executable, ...buildForwardServiceArgs(ownerTarget)].join(" "); + const probe = darwinOwnerProbe(`${expected}\n`); - expect(isForwardServiceListenerOwner(target, { probe })).toBe(true); - expect(probe).toHaveBeenCalledTimes(3); + expect(isForwardServiceListenerOwner(ownerTarget, { platform: "darwin", probe })).toBe(true); + expect(probe).toHaveBeenCalledTimes(4); }); it("rejects a listener whose process does not match the direct ForwardTcp target", () => { - const probe = vi.fn((executable: string) => - executable === "lsof" - ? { status: 0, stdout: "4321\n" } - : { status: 0, stdout: "/usr/bin/node foreign-listener.js\n" }, - ); + const probe = darwinOwnerProbe("/usr/bin/node foreign-listener.js\n"); - expect(isForwardServiceListenerOwner(target, { probe })).toBe(false); + expect(isForwardServiceListenerOwner(ownerTarget, { platform: "darwin", probe })).toBe(false); }); it("rejects ambiguous or changing listener ownership", () => { - const expected = [target.executable, ...buildForwardServiceArgs(target)].join(" "); - const probe = vi - .fn() - .mockReturnValueOnce({ status: 0, stdout: "4321\n" }) - .mockReturnValueOnce({ status: 0, stdout: `${expected}\n` }) - .mockReturnValueOnce({ status: 0, stdout: "9876\n" }); + const expected = [ownerTarget.executable, ...buildForwardServiceArgs(ownerTarget)].join(" "); + const probe = darwinOwnerProbe(`${expected}\n`, "9876\n"); - expect(isForwardServiceListenerOwner(target, { probe })).toBe(false); + expect(isForwardServiceListenerOwner(ownerTarget, { platform: "darwin", probe })).toBe(false); }); it("rejects ownership when a host probe times out", () => { const lsofTimeout = vi.fn(() => ({ status: null, stdout: "" })); - expect(isForwardServiceListenerOwner(target, { probe: lsofTimeout })).toBe(false); + expect( + isForwardServiceListenerOwner(ownerTarget, { platform: "darwin", probe: lsofTimeout }), + ).toBe(false); const psTimeout = vi .fn() .mockReturnValueOnce({ status: 0, stdout: "4321\n" }) + .mockReturnValueOnce({ status: 0, stdout: `p4321\nftxt\nn${process.execPath}\n` }) .mockReturnValueOnce({ status: null, stdout: "" }); - expect(isForwardServiceListenerOwner(target, { probe: psTimeout })).toBe(false); + expect( + isForwardServiceListenerOwner(ownerTarget, { platform: "darwin", probe: psTimeout }), + ).toBe(false); + }); + + it("proves Linux listener ownership through /proc without lsof", () => { + const fixture = createLinuxOwnerFixture(); + const expected = [fixture.target.executable, ...buildForwardServiceArgs(fixture.target)].join( + " ", + ); + const probe = vi.fn(() => ({ status: 0, stdout: `${expected}\n` })); + + expect( + isForwardServiceListenerOwner(fixture.target, { + platform: "linux", + probe, + procRoot: fixture.procRoot, + }), + ).toBe(true); + expect(probe).toHaveBeenCalledOnce(); + expect(probe).toHaveBeenCalledWith("ps", ["-ww", "-p", "4321", "-o", "args="]); + }); + + it("rejects spoofed arguments when the Linux executable is different", () => { + const fixture = createLinuxOwnerFixture("python3"); + const expected = [fixture.target.executable, ...buildForwardServiceArgs(fixture.target)].join( + " ", + ); + const probe = vi.fn(() => ({ status: 0, stdout: `${expected}\n` })); + + expect( + isForwardServiceListenerOwner(fixture.target, { + platform: "linux", + probe, + procRoot: fixture.procRoot, + }), + ).toBe(false); + expect(probe).not.toHaveBeenCalled(); }); it("detaches the OpenShell child and waits for its local port", () => { diff --git a/src/lib/adapters/openshell/forward-service.ts b/src/lib/adapters/openshell/forward-service.ts index 48959bd53c2..919b96a481d 100644 --- a/src/lib/adapters/openshell/forward-service.ts +++ b/src/lib/adapters/openshell/forward-service.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { spawn, spawnSync } from "node:child_process"; +import { readFileSync, readdirSync, readlinkSync, realpathSync } from "node:fs"; import path from "node:path"; import { isValidName } from "../../name-validation"; @@ -43,7 +44,9 @@ type ForwardServiceOwnerProbe = ( ) => { status: number | null; stdout: string }; export interface ForwardServiceOwnerOptions { + readonly platform?: NodeJS.Platform; readonly probe?: ForwardServiceOwnerProbe; + readonly procRoot?: string; } const FORWARD_OWNER_PROBE_TIMEOUT_MS = 5_000; @@ -113,10 +116,101 @@ function captureProcess(executable: string, args: readonly string[]) { return { status: result.status, stdout: result.stdout ?? "" }; } -function listenerPids(port: number, probe: ForwardServiceOwnerProbe): string[] { +function lsofListenerPids(port: number, probe: ForwardServiceOwnerProbe): string[] { const result = probe("lsof", ["-ti", `:${String(port)}`, "-sTCP:LISTEN"]); if (result.status !== 0) return []; - return [...new Set(result.stdout.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean))]; + return [ + ...new Set( + result.stdout + .split(/\r?\n/u) + .map((line) => line.trim()) + .filter(Boolean), + ), + ]; +} + +function linuxListenerPids(port: number, procRoot: string): string[] { + const portSuffix = `:${port.toString(16).padStart(4, "0").toUpperCase()}`; + const socketInodes = new Set(); + for (const table of ["tcp", "tcp6"]) { + try { + for (const line of readFileSync(path.join(procRoot, "net", table), "utf8").split("\n")) { + const fields = line.trim().split(/\s+/u); + if ( + fields[3] === "0A" && + fields[1]?.toUpperCase().endsWith(portSuffix) && + /^\d+$/u.test(fields[9] ?? "") + ) { + socketInodes.add(fields[9]!); + } + } + } catch { + // A missing or unreadable table cannot prove ownership. + } + } + if (socketInodes.size === 0) return []; + + const pids = new Set(); + try { + for (const entry of readdirSync(procRoot, { withFileTypes: true })) { + if (!entry.isDirectory() || !/^[1-9]\d*$/u.test(entry.name)) continue; + try { + for (const descriptor of readdirSync(path.join(procRoot, entry.name, "fd"))) { + const link = readlinkSync(path.join(procRoot, entry.name, "fd", descriptor)); + const match = /^socket:\[(\d+)\]$/u.exec(link); + if (match && socketInodes.has(match[1]!)) { + pids.add(entry.name); + break; + } + } + } catch { + // Processes can exit or deny access while /proc is being inspected. + } + } + } catch { + return []; + } + return [...pids]; +} + +function listenerPids( + port: number, + platform: NodeJS.Platform, + procRoot: string, + probe: ForwardServiceOwnerProbe, +): string[] { + return platform === "linux" + ? linuxListenerPids(port, procRoot) + : platform === "darwin" + ? lsofListenerPids(port, probe) + : []; +} + +function executableMatches(actualExecutable: string, expectedExecutable: string): boolean { + try { + return realpathSync(actualExecutable) === realpathSync(expectedExecutable); + } catch { + return false; + } +} + +function processExecutableMatches( + pid: string, + target: ForwardServiceTarget, + platform: NodeJS.Platform, + procRoot: string, + probe: ForwardServiceOwnerProbe, +): boolean { + if (platform === "linux") { + return executableMatches(path.join(procRoot, pid, "exe"), target.executable); + } + if (platform !== "darwin") return false; + const result = probe("lsof", ["-a", "-p", pid, "-d", "txt", "-Fn"]); + if (result.status !== 0) return false; + return result.stdout + .split(/\r?\n/u) + .filter((line) => line.startsWith("n/")) + .some((line) => executableMatches(line.slice(1), target.executable)); } /** Prove that the current listener is the exact direct ForwardTcp command. */ @@ -125,15 +219,18 @@ export function isForwardServiceListenerOwner( options: ForwardServiceOwnerOptions = {}, ): boolean { validateForwardServiceTarget(target); + const platform = options.platform ?? process.platform; const probe = options.probe ?? captureProcess; - const before = listenerPids(target.localPort, probe); + const procRoot = options.procRoot ?? "/proc"; + const before = listenerPids(target.localPort, platform, procRoot, probe); if (before.length !== 1 || !/^[1-9]\d*$/u.test(before[0]!)) return false; const pid = before[0]!; - const process = probe("ps", ["-ww", "-p", pid, "-o", "args="]); - if (process.status !== 0) return false; + if (!processExecutableMatches(pid, target, platform, procRoot, probe)) return false; + const commandLine = probe("ps", ["-ww", "-p", pid, "-o", "args="]); + if (commandLine.status !== 0) return false; const expected = [target.executable, ...buildForwardServiceArgs(target)].join(" "); - if (process.stdout.trim() !== expected) return false; - const after = listenerPids(target.localPort, probe); + if (commandLine.stdout.trim() !== expected) return false; + const after = listenerPids(target.localPort, platform, procRoot, probe); return after.length === 1 && after[0] === pid; } From ca7a69f0c67ad0808233b724ad7925a6fa5e8d12 Mon Sep 17 00:00:00 2001 From: San Dang Date: Mon, 7 Sep 2026 15:22:02 +0700 Subject: [PATCH 15/20] fix(onboard): complete forward ownership checks --- .../forward-recovery-declared-ports.test.ts | 27 +++++ src/lib/actions/sandbox/forward-recovery.ts | 16 ++- .../openshell/forward-service.test.ts | 35 +++++- src/lib/adapters/openshell/forward-service.ts | 25 ++-- test/e2e/live/double-onboard.test.ts | 111 +++++++++++++++++- ...ard-finalization-dashboard-forward.test.ts | 21 +++- 6 files changed, 210 insertions(+), 25 deletions(-) diff --git a/src/lib/actions/sandbox/forward-recovery-declared-ports.test.ts b/src/lib/actions/sandbox/forward-recovery-declared-ports.test.ts index 112f2a384c0..0b3e551b6fe 100644 --- a/src/lib/actions/sandbox/forward-recovery-declared-ports.test.ts +++ b/src/lib/actions/sandbox/forward-recovery-declared-ports.test.ts @@ -10,11 +10,13 @@ const mocks = vi.hoisted(() => ({ getSandbox: vi.fn(), getHermesDashboardRecoveryConfig: vi.fn(() => null), isLocalForwardReachable: vi.fn(() => true), + isForwardServiceListenerOwner: vi.fn(() => true), launchForwardService: vi.fn(), })); vi.mock("../../adapters/openshell/forward-service", async (importOriginal) => ({ ...(await importOriginal()), + isForwardServiceListenerOwner: mocks.isForwardServiceListenerOwner, launchForwardService: mocks.launchForwardService, })); @@ -63,6 +65,7 @@ beforeEach(() => { vi.unstubAllEnvs(); mocks.runOpenshell.mockReturnValue({ status: 0 }); mocks.isLocalForwardReachable.mockReturnValue(true); + mocks.isForwardServiceListenerOwner.mockReturnValue(true); mocks.launchForwardService.mockImplementation(() => { mocks.isLocalForwardReachable.mockReturnValue(true); }); @@ -82,9 +85,33 @@ describe("ensureDeclaredAgentForwardPortsHealthy", { timeout: 30_000 }, () => { const { ensureSandboxPortForward } = await import("./forward-recovery"); expect(ensureSandboxPortForward("remote-box")).toBe(true); + expect(mocks.isForwardServiceListenerOwner).toHaveBeenCalledWith({ + executable: "/usr/local/bin/openshell", + gatewayName: "nemoclaw", + workspace: "default", + sandboxName: "remote-box", + localHost: "0.0.0.0", + localPort: 18_789, + targetHost: "127.0.0.1", + targetPort: 18_789, + }); expect(mocks.launchForwardService).not.toHaveBeenCalled(); }); + it("fails closed when reachable direct service ownership cannot be proved", async () => { + mocks.getSandbox.mockReturnValue({ agent: "openclaw", dashboardPort: 18_789 }); + mocks.captureOpenshell.mockReturnValue(forwardList([])); + mocks.isForwardServiceListenerOwner.mockReturnValue(false); + mocks.launchForwardService.mockImplementation(() => { + throw new Error("host port is occupied"); + }); + const { ensureSandboxPortForward } = await import("./forward-recovery"); + + expect(ensureSandboxPortForward("foreign-listener")).toBe(false); + expect(mocks.isForwardServiceListenerOwner).toHaveBeenCalledOnce(); + expect(mocks.launchForwardService).toHaveBeenCalledOnce(); + }); + it("does not demand the manifest dashboard port from a sandbox that owns a different dashboard port (#8543)", async () => { mocks.getSandbox.mockReturnValue({ agent: "hermes", diff --git a/src/lib/actions/sandbox/forward-recovery.ts b/src/lib/actions/sandbox/forward-recovery.ts index 72d078565b1..ebc041e3b72 100644 --- a/src/lib/actions/sandbox/forward-recovery.ts +++ b/src/lib/actions/sandbox/forward-recovery.ts @@ -6,6 +6,7 @@ import { withSelectedOpenShellCommandOptions, } from "../../adapters/openshell/command-argv"; import { + isForwardServiceListenerOwner, launchForwardService, type ForwardServiceTarget, } from "../../adapters/openshell/forward-service"; @@ -350,7 +351,7 @@ export function isSandboxForwardHealthy( export function isSandboxPortForwardHealthy( sandboxName: string, port: number, - _expectedBind?: string, + expectedBind?: string, runtimeSelection?: OpenShellRuntimeSelection, ): SandboxForwardHealth { const sandbox = registry.getSandbox(sandboxName); @@ -376,7 +377,18 @@ export function isSandboxPortForwardHealthy( ) { return false; } - return true; + const executable = resolveOpenshell(); + if (!executable) return false; + return isForwardServiceListenerOwner( + forwardServiceTarget( + executable, + gatewayName, + sandboxName, + port, + expectedBind ?? "127.0.0.1", + runtimeSelection?.workspace ?? "default", + ), + ); } export function ensureSandboxPortForwardForPort( diff --git a/src/lib/adapters/openshell/forward-service.test.ts b/src/lib/adapters/openshell/forward-service.test.ts index 7bfa224621c..c74e3a16924 100644 --- a/src/lib/adapters/openshell/forward-service.test.ts +++ b/src/lib/adapters/openshell/forward-service.test.ts @@ -131,7 +131,13 @@ describe("OpenShell forward service", () => { const expected = [fixture.target.executable, ...buildForwardServiceArgs(fixture.target)].join( " ", ); - const probe = vi.fn(() => ({ status: 0, stdout: `${expected}\n` })); + const responses = { + lsof: { status: null, stdout: "" }, + ps: { status: 0, stdout: `${expected}\n` }, + }; + const probe = vi.fn( + (executable: string) => responses[executable as keyof typeof responses] ?? responses.lsof, + ); expect( isForwardServiceListenerOwner(fixture.target, { @@ -140,7 +146,7 @@ describe("OpenShell forward service", () => { procRoot: fixture.procRoot, }), ).toBe(true); - expect(probe).toHaveBeenCalledOnce(); + expect(probe).toHaveBeenCalledTimes(3); expect(probe).toHaveBeenCalledWith("ps", ["-ww", "-p", "4321", "-o", "args="]); }); @@ -149,16 +155,37 @@ describe("OpenShell forward service", () => { const expected = [fixture.target.executable, ...buildForwardServiceArgs(fixture.target)].join( " ", ); - const probe = vi.fn(() => ({ status: 0, stdout: `${expected}\n` })); + const responses = { + lsof: { status: null, stdout: "" }, + ps: { status: 0, stdout: `${expected}\n` }, + }; + const probe = vi.fn( + (executable: string) => responses[executable as keyof typeof responses] ?? responses.lsof, + ); + + expect( + isForwardServiceListenerOwner(fixture.target, { + platform: "linux", + probe, + procRoot: fixture.procRoot, + }), + ).toBe(false); + expect(probe).toHaveBeenCalledOnce(); + }); + + it("denies Linux ownership when the /proc work limit is reached", () => { + const fixture = createLinuxOwnerFixture(); + const probe = vi.fn(() => ({ status: null, stdout: "" })); expect( isForwardServiceListenerOwner(fixture.target, { platform: "linux", probe, procRoot: fixture.procRoot, + procWorkLimit: 1, }), ).toBe(false); - expect(probe).not.toHaveBeenCalled(); + expect(probe).toHaveBeenCalledOnce(); }); it("detaches the OpenShell child and waits for its local port", () => { diff --git a/src/lib/adapters/openshell/forward-service.ts b/src/lib/adapters/openshell/forward-service.ts index 919b96a481d..4f5efa587ec 100644 --- a/src/lib/adapters/openshell/forward-service.ts +++ b/src/lib/adapters/openshell/forward-service.ts @@ -47,9 +47,11 @@ export interface ForwardServiceOwnerOptions { readonly platform?: NodeJS.Platform; readonly probe?: ForwardServiceOwnerProbe; readonly procRoot?: string; + readonly procWorkLimit?: number; } const FORWARD_OWNER_PROBE_TIMEOUT_MS = 5_000; +const LINUX_PROC_WORK_LIMIT = 50_000; function isPort(value: unknown): value is number { return Number.isSafeInteger(value) && Number(value) >= 1 && Number(value) <= 65_535; @@ -116,8 +118,9 @@ function captureProcess(executable: string, args: readonly string[]) { return { status: result.status, stdout: result.stdout ?? "" }; } -function lsofListenerPids(port: number, probe: ForwardServiceOwnerProbe): string[] { +function lsofListenerPids(port: number, probe: ForwardServiceOwnerProbe): string[] | null { const result = probe("lsof", ["-ti", `:${String(port)}`, "-sTCP:LISTEN"]); + if (result.status === null) return null; if (result.status !== 0) return []; return [ ...new Set( @@ -129,7 +132,8 @@ function lsofListenerPids(port: number, probe: ForwardServiceOwnerProbe): string ]; } -function linuxListenerPids(port: number, procRoot: string): string[] { +function linuxListenerPids(port: number, procRoot: string, workLimit: number): string[] { + if (!Number.isSafeInteger(workLimit) || workLimit < 1) return []; const portSuffix = `:${port.toString(16).padStart(4, "0").toUpperCase()}`; const socketInodes = new Set(); for (const table of ["tcp", "tcp6"]) { @@ -151,11 +155,14 @@ function linuxListenerPids(port: number, procRoot: string): string[] { if (socketInodes.size === 0) return []; const pids = new Set(); + let inspected = 0; try { for (const entry of readdirSync(procRoot, { withFileTypes: true })) { if (!entry.isDirectory() || !/^[1-9]\d*$/u.test(entry.name)) continue; + if (++inspected > workLimit) return []; try { for (const descriptor of readdirSync(path.join(procRoot, entry.name, "fd"))) { + if (++inspected > workLimit) return []; const link = readlinkSync(path.join(procRoot, entry.name, "fd", descriptor)); const match = /^socket:\[(\d+)\]$/u.exec(link); if (match && socketInodes.has(match[1]!)) { @@ -177,13 +184,12 @@ function listenerPids( port: number, platform: NodeJS.Platform, procRoot: string, + procWorkLimit: number, probe: ForwardServiceOwnerProbe, ): string[] { - return platform === "linux" - ? linuxListenerPids(port, procRoot) - : platform === "darwin" - ? lsofListenerPids(port, probe) - : []; + const lsof = lsofListenerPids(port, probe); + if (lsof !== null || platform !== "linux") return lsof ?? []; + return linuxListenerPids(port, procRoot, procWorkLimit); } function executableMatches(actualExecutable: string, expectedExecutable: string): boolean { @@ -222,7 +228,8 @@ export function isForwardServiceListenerOwner( const platform = options.platform ?? process.platform; const probe = options.probe ?? captureProcess; const procRoot = options.procRoot ?? "/proc"; - const before = listenerPids(target.localPort, platform, procRoot, probe); + const procWorkLimit = options.procWorkLimit ?? LINUX_PROC_WORK_LIMIT; + const before = listenerPids(target.localPort, platform, procRoot, procWorkLimit, probe); if (before.length !== 1 || !/^[1-9]\d*$/u.test(before[0]!)) return false; const pid = before[0]!; if (!processExecutableMatches(pid, target, platform, procRoot, probe)) return false; @@ -230,7 +237,7 @@ export function isForwardServiceListenerOwner( if (commandLine.status !== 0) return false; const expected = [target.executable, ...buildForwardServiceArgs(target)].join(" "); if (commandLine.stdout.trim() !== expected) return false; - const after = listenerPids(target.localPort, platform, procRoot, probe); + const after = listenerPids(target.localPort, platform, procRoot, procWorkLimit, probe); return after.length === 1 && after[0] === pid; } diff --git a/test/e2e/live/double-onboard.test.ts b/test/e2e/live/double-onboard.test.ts index e9e2f9244d9..42cb0b1cd83 100644 --- a/test/e2e/live/double-onboard.test.ts +++ b/test/e2e/live/double-onboard.test.ts @@ -127,6 +127,7 @@ async function waitForDashboardReachability( [ "--silent", "--show-error", + "--fail", "--output", "/dev/null", "--max-time", @@ -147,6 +148,45 @@ async function waitForDashboardReachability( return { reachable, output }; } +async function inspectForwardListener( + host: HostCliClient, + port: string, + sandboxName: string, + artifactName: string, +): Promise { + return await host.command( + "bash", + [ + "-lc", + [ + "set -euo pipefail", + 'pid="$(lsof -ti ":$1" -sTCP:LISTEN)"', + '[[ "$pid" =~ ^[1-9][0-9]*$ ]]', + 'args="$(ps -ww -p "$pid" -o args=)"', + 'expected="--gateway nemoclaw --workspace default forward service $2 --target-port $1 --target-host 127.0.0.1 --local 127.0.0.1:$1"', + '[[ "$args" == *"$expected" ]]', + 'printf "%s\\t%s\\n" "$pid" "$args"', + ].join("\n"), + "nemoclaw-forward-listener", + port, + sandboxName, + ], + { artifactName, env: commandEnv(), timeoutMs: 15_000 }, + ); +} + +async function inspectNoListener( + host: HostCliClient, + port: string, + artifactName: string, +): Promise { + return await host.command( + "lsof", + ["-ti", `:${port}`, "-sTCP:LISTEN"], + { artifactName, env: commandEnv(), timeoutMs: 15_000 }, + ); +} + async function cleanupDoubleOnboardState( host: HostCliClient, lifecycle: LifecyclePhaseFixture, @@ -525,6 +565,12 @@ test( }); const portAfterFirst = dashboardPortFromList(listAfterFirst.stdout, SANDBOX_A) ?? ""; + const listenerBeforeSecond = await inspectForwardListener( + host, + portAfterFirst, + SANDBOX_A, + "phase-2-dashboard-listener-before-second-onboard", + ); progress.phase("re-onboard same sandbox on existing gateway"); // Phase 3: second onboard with the same name must reuse the healthy gateway. @@ -567,7 +613,16 @@ test( true, "phase-3-dashboard-after-second-onboard", ); - expect(dashboardAfterSecond.reachable, dashboardAfterSecond.output).toBe(true); + const listenerAfterSecond = await inspectForwardListener( + host, + portAfterSecond ?? "", + SANDBOX_A, + "phase-3-dashboard-listener-after-second-onboard", + ); + expect( + `${dashboardAfterSecond.reachable}:${listenerBeforeSecond.exitCode}:${listenerBeforeSecond.timedOut}:${listenerAfterSecond.exitCode}:${listenerAfterSecond.timedOut}:${resultText(listenerBeforeSecond) === resultText(listenerAfterSecond)}`, + `${dashboardAfterSecond.output}\n${resultText(listenerBeforeSecond)}\n${resultText(listenerAfterSecond)}`, + ).toBe("true:0:false:0:false:true"); progress.phase("recreate same sandbox on existing gateway"); const gatewayBeforeRecreate = await gatewayRuntimeId(gateway); @@ -654,14 +709,32 @@ test( true, "phase-4-dashboard-a-before-stop", ); - expect(dashboardABeforeStop.reachable, dashboardABeforeStop.output).toBe(true); + const listenerABeforeStop = await inspectForwardListener( + host, + portA ?? "", + SANDBOX_A, + "phase-4-dashboard-listener-a-before-stop", + ); + expect( + `${dashboardABeforeStop.reachable}:${listenerABeforeStop.exitCode}:${listenerABeforeStop.timedOut}`, + `${dashboardABeforeStop.output}\n${resultText(listenerABeforeStop)}`, + ).toBe("true:0:false"); const dashboardBBeforeStop = await waitForDashboardReachability( host, portB ?? "", true, "phase-4-dashboard-b-before-stop", ); - expect(dashboardBBeforeStop.reachable, dashboardBBeforeStop.output).toBe(true); + const listenerBBeforeStop = await inspectForwardListener( + host, + portB ?? "", + SANDBOX_B, + "phase-4-dashboard-listener-b-before-stop", + ); + expect( + `${dashboardBBeforeStop.reachable}:${listenerBBeforeStop.exitCode}:${listenerBBeforeStop.timedOut}`, + `${dashboardBBeforeStop.output}\n${resultText(listenerBBeforeStop)}`, + ).toBe("true:0:false"); progress.phase("stop sibling sandbox without disturbing the first forward"); const stopB = await command(host, [SANDBOX_B, "stop"], { @@ -677,7 +750,15 @@ test( false, "phase-4-dashboard-b-after-stop", ); - expect(releasedForwardB.reachable, releasedForwardB.output).toBe(false); + const listenerBAfterStop = await inspectNoListener( + host, + portB ?? "", + "phase-4-dashboard-listener-b-after-stop", + ); + expect( + `${releasedForwardB.reachable}:${listenerBAfterStop.exitCode}:${listenerBAfterStop.timedOut}`, + `${releasedForwardB.output}\n${resultText(listenerBAfterStop)}`, + ).toBe("false:1:false"); const stoppedStatusB = await command(host, [SANDBOX_B, "status"], { artifactName: "phase-4-nemoclaw-status-sandbox-b-after-stop", @@ -695,7 +776,16 @@ test( true, "phase-4-dashboard-a-after-b-stop", ); - expect(retainedForwardAAfterStop.reachable, retainedForwardAAfterStop.output).toBe(true); + const listenerAAfterStop = await inspectForwardListener( + host, + portA ?? "", + SANDBOX_A, + "phase-4-dashboard-listener-a-after-b-stop", + ); + expect( + `${retainedForwardAAfterStop.reachable}:${listenerAAfterStop.exitCode}:${listenerAAfterStop.timedOut}:${resultText(listenerAAfterStop) === resultText(listenerABeforeStop)}`, + `${retainedForwardAAfterStop.output}\n${resultText(listenerABeforeStop)}\n${resultText(listenerAAfterStop)}`, + ).toBe("true:0:false:true"); const startB = await command(host, [SANDBOX_B, "start"], { artifactName: "phase-4-nemoclaw-start-sandbox-b", @@ -709,7 +799,16 @@ test( true, "phase-4-dashboard-b-after-start", ); - expect(restoredForwardBAfterStart.reachable, restoredForwardBAfterStart.output).toBe(true); + const listenerBAfterStart = await inspectForwardListener( + host, + portB ?? "", + SANDBOX_B, + "phase-4-dashboard-listener-b-after-start", + ); + expect( + `${restoredForwardBAfterStart.reachable}:${listenerBAfterStart.exitCode}:${listenerBAfterStart.timedOut}`, + `${restoredForwardBAfterStart.output}\n${resultText(listenerBBeforeStop)}\n${resultText(listenerBAfterStart)}`, + ).toBe("true:0:false"); progress.phase("replace sandbox after stale registry refusal"); // Phase 5: direct OpenShell deletion leaves a stale registry entry that diff --git a/test/onboarding/onboard-finalization-dashboard-forward.test.ts b/test/onboarding/onboard-finalization-dashboard-forward.test.ts index 3a2c1760085..6530b47867e 100644 --- a/test/onboarding/onboard-finalization-dashboard-forward.test.ts +++ b/test/onboarding/onboard-finalization-dashboard-forward.test.ts @@ -3,15 +3,17 @@ import { describe, expect, it, vi } from "vitest"; +import type { ForwardServiceTarget } from "../../src/lib/adapters/openshell/forward-service"; import { createOnboardDashboardHelpers } from "../../src/lib/onboard/dashboard"; import type { ListSandboxesFn } from "../../src/lib/onboard/dashboard-port"; function harness(options: { listSandboxes: ListSandboxesFn; isPortBound?: (port: number) => boolean; - ownsForward?: () => boolean; + ownsForward?: (target: ForwardServiceTarget) => boolean; }) { const launch = vi.fn(); + const owns = vi.fn(options.ownsForward ?? (() => false)); const helpers = createOnboardDashboardHelpers({ runOpenshell: vi.fn(() => ({ status: 0 })), runCaptureOpenshell: vi.fn(() => ""), @@ -29,12 +31,12 @@ function harness(options: { forwardService: { executable: () => "/usr/local/bin/openshell", launch, - owns: vi.fn(options.ownsForward ?? (() => false)), + owns, resolveGatewayName: () => "nemoclaw", retireLegacy: vi.fn(() => 0), }, }); - return { helpers, launch }; + return { helpers, launch, owns }; } describe("finalization dashboard ForwardTcp launch", () => { @@ -75,7 +77,7 @@ describe("finalization dashboard ForwardTcp launch", () => { it("reuses an exactly owned dashboard forward (#11074)", () => { vi.stubEnv("CHAT_UI_URL", undefined); - const { helpers, launch } = harness({ + const { helpers, launch, owns } = harness({ listSandboxes: () => ({ sandboxes: [{ name: "reonboard-test", dashboardPort: 18_790 }], }), @@ -84,6 +86,17 @@ describe("finalization dashboard ForwardTcp launch", () => { }); expect(helpers.ensureFinalizationDashboardForward("reonboard-test")).toBe(18_790); + expect(owns).toHaveBeenCalledOnce(); + expect(owns).toHaveBeenCalledWith({ + executable: "/usr/local/bin/openshell", + gatewayName: "nemoclaw", + workspace: "default", + sandboxName: "reonboard-test", + localHost: "127.0.0.1", + localPort: 18_790, + targetHost: "127.0.0.1", + targetPort: 18_790, + }); expect(launch).not.toHaveBeenCalled(); expect(process.env.CHAT_UI_URL).toBe("http://127.0.0.1:18790"); }); From 6c7a0a06c5ee5c292c0a6592e0b1adcfd57ba286 Mon Sep 17 00:00:00 2001 From: San Dang Date: Mon, 7 Sep 2026 16:25:28 +0700 Subject: [PATCH 16/20] test(e2e): prove forward reuse across resume --- src/lib/actions/sandbox/forward-recovery.ts | 21 +++-- src/lib/adapters/openshell/forward-service.ts | 15 +++ src/lib/onboard/dashboard.ts | 21 +++-- test/e2e/fixtures/clients/host.ts | 69 ++++++++++++++ test/e2e/live/double-onboard.test.ts | 93 +++++++------------ test/e2e/live/onboard-resume.test.ts | 71 +++++++++++++- test/e2e/support/e2e-clients.test.ts | 24 +++++ 7 files changed, 232 insertions(+), 82 deletions(-) diff --git a/src/lib/actions/sandbox/forward-recovery.ts b/src/lib/actions/sandbox/forward-recovery.ts index ebc041e3b72..49260f03ec8 100644 --- a/src/lib/actions/sandbox/forward-recovery.ts +++ b/src/lib/actions/sandbox/forward-recovery.ts @@ -6,6 +6,7 @@ import { withSelectedOpenShellCommandOptions, } from "../../adapters/openshell/command-argv"; import { + createForwardServiceTarget, isForwardServiceListenerOwner, launchForwardService, type ForwardServiceTarget, @@ -179,16 +180,16 @@ function forwardServiceTarget( expectedBind = "127.0.0.1", workspace = "default", ): ForwardServiceTarget { - return { - executable, - gatewayName, - workspace, - sandboxName, - localHost: expectedBind === "0.0.0.0" ? ("0.0.0.0" as const) : ("127.0.0.1" as const), - localPort: port, - targetHost: "127.0.0.1", - targetPort: port, - }; + return createForwardServiceTarget( + { + executable, + gatewayName, + workspace, + sandboxName, + localHost: expectedBind === "0.0.0.0" ? "0.0.0.0" : "127.0.0.1", + }, + port, + ); } function isValidPort(value: unknown): value is number { diff --git a/src/lib/adapters/openshell/forward-service.ts b/src/lib/adapters/openshell/forward-service.ts index 4f5efa587ec..ad184f4b9f1 100644 --- a/src/lib/adapters/openshell/forward-service.ts +++ b/src/lib/adapters/openshell/forward-service.ts @@ -90,6 +90,21 @@ export function validateForwardServiceTarget(target: ForwardServiceTarget): Forw return target; } +export function createForwardServiceTarget( + target: Pick< + ForwardServiceTarget, + "executable" | "gatewayName" | "workspace" | "sandboxName" | "localHost" + >, + port: number, +): ForwardServiceTarget { + return validateForwardServiceTarget({ + ...target, + localPort: port, + targetHost: "127.0.0.1", + targetPort: port, + }); +} + /** Build the direct ForwardTcp command introduced in OpenShell 0.0.106. */ export function buildForwardServiceArgs(target: ForwardServiceTarget): string[] { validateForwardServiceTarget(target); diff --git a/src/lib/onboard/dashboard.ts b/src/lib/onboard/dashboard.ts index f19c177b4c9..4062356bb4b 100644 --- a/src/lib/onboard/dashboard.ts +++ b/src/lib/onboard/dashboard.ts @@ -5,6 +5,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { + createForwardServiceTarget, isForwardServiceListenerOwner, launchForwardService, type ForwardServiceTarget, @@ -263,16 +264,16 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa port: number, target: string, ): ForwardServiceTarget { - return { - executable: forwardService!.executable(), - gatewayName, - workspace: "default", - sandboxName, - localHost: target.startsWith("0.0.0.0:") ? ("0.0.0.0" as const) : ("127.0.0.1" as const), - localPort: port, - targetHost: "127.0.0.1", - targetPort: port, - }; + return createForwardServiceTarget( + { + executable: forwardService!.executable(), + gatewayName, + workspace: "default", + sandboxName, + localHost: target.startsWith("0.0.0.0:") ? "0.0.0.0" : "127.0.0.1", + }, + port, + ); } function ownsDashboardForward( diff --git a/test/e2e/fixtures/clients/host.ts b/test/e2e/fixtures/clients/host.ts index 4991582e2c3..903e88ead18 100644 --- a/test/e2e/fixtures/clients/host.ts +++ b/test/e2e/fixtures/clients/host.ts @@ -24,6 +24,12 @@ export interface HostClientOptions { openshellPath?: string; } +export interface ForwardListenerEvidence { + valid: boolean; + identity: string; + output: string; +} + const GATEWAY_ALREADY_ABSENT = /gateway[^\n]*(?:does not exist|not found)|No (?:active )?gateway|No gateway metadata found/i; const GATEWAY_REMOVE_UNSUPPORTED = @@ -177,6 +183,69 @@ export class HostCliClient { return result; } + async inspectOpenShellForwardListener( + port: string, + sandboxName: string, + options: ShellProbeRunOptions = {}, + ): Promise { + const artifactName = options.artifactName ?? `forward-listener-${port}`; + const probeOptions = { ...options, timeoutMs: options.timeoutMs ?? 15_000 }; + const [before, command] = await Promise.all([ + this.command("lsof", ["-ti", `:${port}`, "-sTCP:LISTEN"], { + ...probeOptions, + artifactName: `${artifactName}-listener-before`, + }), + this.command("which", [this.openshellPath], { + ...probeOptions, + artifactName: `${artifactName}-command`, + }), + ]); + const pids = [ + ...new Set(before.stdout.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean)), + ]; + const pid = pids.length === 1 && /^[1-9]\d*$/u.test(pids[0]!) ? pids[0]! : ""; + const commandPath = command.stdout.trim(); + if (!pid || !commandPath) { + return { valid: false, identity: "", output: `${resultText(before)}\n${resultText(command)}` }; + } + + const [actualExecutable, expectedExecutable, commandLine, after] = await Promise.all([ + this.command("readlink", ["-f", `/proc/${pid}/exe`], { + ...probeOptions, + artifactName: `${artifactName}-actual-executable`, + }), + this.command("readlink", ["-f", commandPath], { + ...probeOptions, + artifactName: `${artifactName}-expected-executable`, + }), + this.command("ps", ["-ww", "-p", pid, "-o", "args="], { + ...probeOptions, + artifactName: `${artifactName}-command-line`, + }), + this.command("lsof", ["-ti", `:${port}`, "-sTCP:LISTEN"], { + ...probeOptions, + artifactName: `${artifactName}-listener-after`, + }), + ]); + const expectedCommandLine = `${commandPath} --gateway nemoclaw --workspace default forward service ${sandboxName} --target-port ${port} --target-host 127.0.0.1 --local 127.0.0.1:${port}`; + const afterPids = [ + ...new Set(after.stdout.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean)), + ]; + const probes = [before, command, actualExecutable, expectedExecutable, commandLine, after]; + const identity = `${pid}\t${actualExecutable.stdout.trim()}\t${commandLine.stdout.trim()}`; + const valid = + probes.every((probe) => probe.exitCode === 0 && !probe.timedOut) && + actualExecutable.stdout.trim() === expectedExecutable.stdout.trim() && + commandLine.stdout.trim() === expectedCommandLine && + afterPids.length === 1 && + afterPids[0] === pid; + return { + valid, + identity, + output: probes.map(resultText).filter(Boolean).join("\n"), + }; + } + async destroySandbox( sandboxName: string, options: ShellProbeRunOptions = {}, diff --git a/test/e2e/live/double-onboard.test.ts b/test/e2e/live/double-onboard.test.ts index 42cb0b1cd83..3bd4235b50d 100644 --- a/test/e2e/live/double-onboard.test.ts +++ b/test/e2e/live/double-onboard.test.ts @@ -148,33 +148,6 @@ async function waitForDashboardReachability( return { reachable, output }; } -async function inspectForwardListener( - host: HostCliClient, - port: string, - sandboxName: string, - artifactName: string, -): Promise { - return await host.command( - "bash", - [ - "-lc", - [ - "set -euo pipefail", - 'pid="$(lsof -ti ":$1" -sTCP:LISTEN)"', - '[[ "$pid" =~ ^[1-9][0-9]*$ ]]', - 'args="$(ps -ww -p "$pid" -o args=)"', - 'expected="--gateway nemoclaw --workspace default forward service $2 --target-port $1 --target-host 127.0.0.1 --local 127.0.0.1:$1"', - '[[ "$args" == *"$expected" ]]', - 'printf "%s\\t%s\\n" "$pid" "$args"', - ].join("\n"), - "nemoclaw-forward-listener", - port, - sandboxName, - ], - { artifactName, env: commandEnv(), timeoutMs: 15_000 }, - ); -} - async function inspectNoListener( host: HostCliClient, port: string, @@ -565,11 +538,13 @@ test( }); const portAfterFirst = dashboardPortFromList(listAfterFirst.stdout, SANDBOX_A) ?? ""; - const listenerBeforeSecond = await inspectForwardListener( - host, + const listenerBeforeSecond = await host.inspectOpenShellForwardListener( portAfterFirst, SANDBOX_A, - "phase-2-dashboard-listener-before-second-onboard", + { + artifactName: "phase-2-dashboard-listener-before-second-onboard", + env: commandEnv(), + }, ); progress.phase("re-onboard same sandbox on existing gateway"); @@ -613,16 +588,18 @@ test( true, "phase-3-dashboard-after-second-onboard", ); - const listenerAfterSecond = await inspectForwardListener( - host, + const listenerAfterSecond = await host.inspectOpenShellForwardListener( portAfterSecond ?? "", SANDBOX_A, - "phase-3-dashboard-listener-after-second-onboard", + { + artifactName: "phase-3-dashboard-listener-after-second-onboard", + env: commandEnv(), + }, ); expect( - `${dashboardAfterSecond.reachable}:${listenerBeforeSecond.exitCode}:${listenerBeforeSecond.timedOut}:${listenerAfterSecond.exitCode}:${listenerAfterSecond.timedOut}:${resultText(listenerBeforeSecond) === resultText(listenerAfterSecond)}`, - `${dashboardAfterSecond.output}\n${resultText(listenerBeforeSecond)}\n${resultText(listenerAfterSecond)}`, - ).toBe("true:0:false:0:false:true"); + `${dashboardAfterSecond.reachable}:${listenerBeforeSecond.valid}:${listenerAfterSecond.valid}:${listenerBeforeSecond.identity === listenerAfterSecond.identity}`, + `${dashboardAfterSecond.output}\n${listenerBeforeSecond.output}\n${listenerAfterSecond.output}`, + ).toBe("true:true:true:true"); progress.phase("recreate same sandbox on existing gateway"); const gatewayBeforeRecreate = await gatewayRuntimeId(gateway); @@ -709,32 +686,30 @@ test( true, "phase-4-dashboard-a-before-stop", ); - const listenerABeforeStop = await inspectForwardListener( - host, + const listenerABeforeStop = await host.inspectOpenShellForwardListener( portA ?? "", SANDBOX_A, - "phase-4-dashboard-listener-a-before-stop", + { artifactName: "phase-4-dashboard-listener-a-before-stop", env: commandEnv() }, ); expect( - `${dashboardABeforeStop.reachable}:${listenerABeforeStop.exitCode}:${listenerABeforeStop.timedOut}`, - `${dashboardABeforeStop.output}\n${resultText(listenerABeforeStop)}`, - ).toBe("true:0:false"); + `${dashboardABeforeStop.reachable}:${listenerABeforeStop.valid}`, + `${dashboardABeforeStop.output}\n${listenerABeforeStop.output}`, + ).toBe("true:true"); const dashboardBBeforeStop = await waitForDashboardReachability( host, portB ?? "", true, "phase-4-dashboard-b-before-stop", ); - const listenerBBeforeStop = await inspectForwardListener( - host, + const listenerBBeforeStop = await host.inspectOpenShellForwardListener( portB ?? "", SANDBOX_B, - "phase-4-dashboard-listener-b-before-stop", + { artifactName: "phase-4-dashboard-listener-b-before-stop", env: commandEnv() }, ); expect( - `${dashboardBBeforeStop.reachable}:${listenerBBeforeStop.exitCode}:${listenerBBeforeStop.timedOut}`, - `${dashboardBBeforeStop.output}\n${resultText(listenerBBeforeStop)}`, - ).toBe("true:0:false"); + `${dashboardBBeforeStop.reachable}:${listenerBBeforeStop.valid}`, + `${dashboardBBeforeStop.output}\n${listenerBBeforeStop.output}`, + ).toBe("true:true"); progress.phase("stop sibling sandbox without disturbing the first forward"); const stopB = await command(host, [SANDBOX_B, "stop"], { @@ -776,16 +751,15 @@ test( true, "phase-4-dashboard-a-after-b-stop", ); - const listenerAAfterStop = await inspectForwardListener( - host, + const listenerAAfterStop = await host.inspectOpenShellForwardListener( portA ?? "", SANDBOX_A, - "phase-4-dashboard-listener-a-after-b-stop", + { artifactName: "phase-4-dashboard-listener-a-after-b-stop", env: commandEnv() }, ); expect( - `${retainedForwardAAfterStop.reachable}:${listenerAAfterStop.exitCode}:${listenerAAfterStop.timedOut}:${resultText(listenerAAfterStop) === resultText(listenerABeforeStop)}`, - `${retainedForwardAAfterStop.output}\n${resultText(listenerABeforeStop)}\n${resultText(listenerAAfterStop)}`, - ).toBe("true:0:false:true"); + `${retainedForwardAAfterStop.reachable}:${listenerAAfterStop.valid}:${listenerAAfterStop.identity === listenerABeforeStop.identity}`, + `${retainedForwardAAfterStop.output}\n${listenerABeforeStop.output}\n${listenerAAfterStop.output}`, + ).toBe("true:true:true"); const startB = await command(host, [SANDBOX_B, "start"], { artifactName: "phase-4-nemoclaw-start-sandbox-b", @@ -799,16 +773,15 @@ test( true, "phase-4-dashboard-b-after-start", ); - const listenerBAfterStart = await inspectForwardListener( - host, + const listenerBAfterStart = await host.inspectOpenShellForwardListener( portB ?? "", SANDBOX_B, - "phase-4-dashboard-listener-b-after-start", + { artifactName: "phase-4-dashboard-listener-b-after-start", env: commandEnv() }, ); expect( - `${restoredForwardBAfterStart.reachable}:${listenerBAfterStart.exitCode}:${listenerBAfterStart.timedOut}`, - `${restoredForwardBAfterStart.output}\n${resultText(listenerBBeforeStop)}\n${resultText(listenerBAfterStart)}`, - ).toBe("true:0:false"); + `${restoredForwardBAfterStart.reachable}:${listenerBAfterStart.valid}`, + `${restoredForwardBAfterStart.output}\n${listenerBBeforeStop.output}\n${listenerBAfterStart.output}`, + ).toBe("true:true"); progress.phase("replace sandbox after stale registry refusal"); // Phase 5: direct OpenShell deletion leaves a stale registry entry that diff --git a/test/e2e/live/onboard-resume.test.ts b/test/e2e/live/onboard-resume.test.ts index fe94c8b644e..c5ad98f7f10 100644 --- a/test/e2e/live/onboard-resume.test.ts +++ b/test/e2e/live/onboard-resume.test.ts @@ -10,6 +10,7 @@ import { ONBOARD_NO_RECREATE_COMMAND_TIMEOUT_MS, ONBOARD_RESUME_TEST_TIMEOUT_MS, } from "../../../tools/e2e/onboard-timeout-contract.mts"; +import { parseOpenShellSandboxId } from "../../../src/lib/adapters/openshell/sandbox-identity.ts"; import { parseSandboxPhase } from "../../../src/lib/state/gateway.ts"; import { execTimeout, testTimeout } from "../../helpers/timeouts.ts"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; @@ -107,6 +108,14 @@ function markSessionInProgress(file: string): void { fs.writeFileSync(file, JSON.stringify(session, null, 2), "utf8"); } +function registeredDashboardPort(): string { + const registry = JSON.parse(fs.readFileSync(REGISTRY_FILE, "utf8")) as { + sandboxes?: Record; + }; + const port = registry.sandboxes?.[SANDBOX_NAME]?.dashboardPort; + return typeof port === "number" ? String(port) : ""; +} + function interruptedSessionSummary(session: SessionStateInterrupted): Record { return { status: session.status, @@ -192,6 +201,7 @@ test( "resume proves recreated sandbox provider attachments are selectively reconciled", "host trust-store anchor corporate CA source is baked and merged after resume", "an unreachable committed route pauses at final verification and completes after repair", + "non-recreate resume retains the Ready sandbox, dashboard port, and exact forward listener", "implicit resume is detected and --fresh suppresses that auto-resume", ], }); @@ -577,6 +587,23 @@ test( // re-probe and complete without recreating the sandbox. // ────────────────────────────────────────────────────────────────── progress.phase("retry final verification after route repair"); + const sandboxBeforeRouteFailure = await sandbox.openshell( + ["sandbox", "get", SANDBOX_NAME], + { + artifactName: "phase-3-5-sandbox-before-route-failure", + env: probeEnv, + timeoutMs: 30_000, + }, + ); + const sandboxIdBeforeRouteFailure = parseOpenShellSandboxId( + resultText(sandboxBeforeRouteFailure), + ); + const dashboardPortBeforeRouteFailure = registeredDashboardPort(); + const listenerBeforeRouteFailure = await host.inspectOpenShellForwardListener( + dashboardPortBeforeRouteFailure, + SANDBOX_NAME, + { artifactName: "phase-3-5-listener-before-route-failure", env: probeEnv }, + ); markSessionInProgress(SESSION_FILE); await fake.close(); @@ -591,7 +618,15 @@ test( }, ); const unavailableResumeText = `${unavailableResumeRun.stdout}\n${unavailableResumeRun.stderr}`; - expect(unavailableResumeRun.exitCode, unavailableResumeText).not.toBe(0); + const listenerAfterRouteFailure = await host.inspectOpenShellForwardListener( + dashboardPortBeforeRouteFailure, + SANDBOX_NAME, + { artifactName: "phase-3-5-listener-after-route-failure", env: probeEnv }, + ); + expect( + `${unavailableResumeRun.exitCode !== 0}:${listenerBeforeRouteFailure.valid}:${listenerAfterRouteFailure.valid}:${listenerBeforeRouteFailure.identity === listenerAfterRouteFailure.identity}`, + `${unavailableResumeText}\n${listenerBeforeRouteFailure.output}\n${listenerAfterRouteFailure.output}`, + ).toBe("true:true:true:true"); expect(unavailableResumeText).toContain("Compatible endpoint sandbox smoke check failed"); expect(unavailableResumeText).toContain("inference.local"); expect(unavailableResumeText).not.toContain( @@ -632,7 +667,39 @@ test( }, ); const repairedResumeText = `${repairedResumeRun.stdout}\n${repairedResumeRun.stderr}`; - expect(repairedResumeRun.exitCode, repairedResumeText).toBe(0); + const sandboxAfterRouteRepair = await sandbox.openshell(["sandbox", "get", SANDBOX_NAME], { + artifactName: "phase-3-5-sandbox-after-route-repair", + env: probeEnv, + timeoutMs: 30_000, + }); + const dashboardPortAfterRouteRepair = registeredDashboardPort(); + const listenerAfterRouteRepair = await host.inspectOpenShellForwardListener( + dashboardPortAfterRouteRepair, + SANDBOX_NAME, + { artifactName: "phase-3-5-listener-after-route-repair", env: probeEnv }, + ); + const dashboardAfterRouteRepair = await host.command( + "curl", + [ + "--silent", + "--show-error", + "--fail", + "--output", + "/dev/null", + "--max-time", + "5", + `http://127.0.0.1:${dashboardPortAfterRouteRepair}/`, + ], + { + artifactName: "phase-3-5-dashboard-after-route-repair", + env: probeEnv, + timeoutMs: 15_000, + }, + ); + expect( + `${repairedResumeRun.exitCode}:${parseOpenShellSandboxId(resultText(sandboxAfterRouteRepair)) === sandboxIdBeforeRouteFailure}:${dashboardPortAfterRouteRepair === dashboardPortBeforeRouteFailure}:${listenerAfterRouteRepair.valid}:${listenerAfterRouteRepair.identity === listenerBeforeRouteFailure.identity}:${dashboardAfterRouteRepair.exitCode}:${repairedResumeText.includes("cannot be reallocated or adopted")}`, + `${repairedResumeText}\n${resultText(sandboxBeforeRouteFailure)}\n${resultText(sandboxAfterRouteRepair)}\n${listenerBeforeRouteFailure.output}\n${listenerAfterRouteRepair.output}\n${resultText(dashboardAfterRouteRepair)}`, + ).toBe("0:true:true:true:true:0:false"); expect(repairedResumeText).toContain("is ready"); expect(repairedResumeText).not.toContain(`Deleting and recreating sandbox '${SANDBOX_NAME}'`); expect(repairedResumeText).not.toContain(`Sandbox '${SANDBOX_NAME}' created`); diff --git a/test/e2e/support/e2e-clients.test.ts b/test/e2e/support/e2e-clients.test.ts index 9bf04d2b2e0..4d500b25fef 100644 --- a/test/e2e/support/e2e-clients.test.ts +++ b/test/e2e/support/e2e-clients.test.ts @@ -199,6 +199,30 @@ describe("E2E fixture clients", () => { expect(host.openshellCommandPath).toBe("openshell"); }); + it.each([ + { actualExecutable: "/opt/openshell", expected: true }, + { actualExecutable: "/usr/bin/python3", expected: false }, + ])( + "host client verifies an exact ForwardTcp listener executable [expected=$expected]", + async ({ actualExecutable, expected }) => { + const runner = new FakeRunner(); + runner.enqueue({ stdout: "4321\n" }); + runner.enqueue({ stdout: "/usr/local/bin/openshell\n" }); + runner.enqueue({ stdout: `${actualExecutable}\n` }); + runner.enqueue({ stdout: "/opt/openshell\n" }); + runner.enqueue({ + stdout: + "/usr/local/bin/openshell --gateway nemoclaw --workspace default forward service alpha --target-port 18789 --target-host 127.0.0.1 --local 127.0.0.1:18789\n", + }); + runner.enqueue({ stdout: "4321\n" }); + const host = new HostCliClient(runner); + + await expect( + host.inspectOpenShellForwardListener("18789", "alpha"), + ).resolves.toMatchObject({ valid: expected }); + }, + ); + it("composes installation, OpenShell resolution, and launch in authority order", async () => { const runner = new FakeRunner(); runner.enqueue({ stdout: "installation complete\n" }); From 46d405bded32f53f6e4c4c18f46ba333a27dba00 Mon Sep 17 00:00:00 2001 From: San Dang Date: Mon, 7 Sep 2026 17:14:45 +0700 Subject: [PATCH 17/20] test(e2e): isolate double-onboard reuse evidence --- ci/e2e-assertion-budget.json | 14 +++++++------- test/e2e/live/double-onboard.test.ts | 26 +------------------------- 2 files changed, 8 insertions(+), 32 deletions(-) diff --git a/ci/e2e-assertion-budget.json b/ci/e2e-assertion-budget.json index 819c276091c..e96e4f1992a 100644 --- a/ci/e2e-assertion-budget.json +++ b/ci/e2e-assertion-budget.json @@ -15,26 +15,26 @@ "testFileCount": 86, "liveFileCount": 222, "direct": { - "expectCalls": 1881, - "matcherAssertions": 1850, + "expectCalls": 1879, + "matcherAssertions": 1848, "nodeAssertions": 100, "namedAssertionHelpers": 627, "failCalls": 8, "throwGuards": 87, "objectFieldAssertions": 245, - "assertionPoints": 2917, + "assertionPoints": 2915, "generatedProbeBlocks": 136, "generatedProbeConditions": 354 }, "unique": { - "expectCalls": 2373, - "matcherAssertions": 2337, + "expectCalls": 2371, + "matcherAssertions": 2335, "nodeAssertions": 119, "namedAssertionHelpers": 930, "failCalls": 38, "throwGuards": 638, "objectFieldAssertions": 348, - "assertionPoints": 4410, + "assertionPoints": 4408, "generatedProbeBlocks": 290, "generatedProbeConditions": 975 }, @@ -60,7 +60,7 @@ "test/e2e/live/cron-preflight-inference-local.test.ts": [8,8,8,8,1], "test/e2e/live/dashboard-remote-bind.test.ts": [17,15,17,17,3], "test/e2e/live/device-auth-health.test.ts": [13,18,13,21,0], - "test/e2e/live/double-onboard.test.ts": [87,96,87,96,0], + "test/e2e/live/double-onboard.test.ts": [85,94,85,94,0], "test/e2e/live/external-gateway-health.test.ts": [4,5,4,11,0], "test/e2e/live/full-e2e.test.ts": [30,34,39,72,7], "test/e2e/live/gateway-guard-recovery.test.ts": [48,54,48,57,3], diff --git a/test/e2e/live/double-onboard.test.ts b/test/e2e/live/double-onboard.test.ts index 3bd4235b50d..56eb36eb2dc 100644 --- a/test/e2e/live/double-onboard.test.ts +++ b/test/e2e/live/double-onboard.test.ts @@ -761,28 +761,6 @@ test( `${retainedForwardAAfterStop.output}\n${listenerABeforeStop.output}\n${listenerAAfterStop.output}`, ).toBe("true:true:true"); - const startB = await command(host, [SANDBOX_B, "start"], { - artifactName: "phase-4-nemoclaw-start-sandbox-b", - env: commandEnv(), - timeoutMs: PHASE_TIMEOUT_MS, - }); - expect(startB.exitCode, resultText(startB)).toBe(0); - const restoredForwardBAfterStart = await waitForDashboardReachability( - host, - portB ?? "", - true, - "phase-4-dashboard-b-after-start", - ); - const listenerBAfterStart = await host.inspectOpenShellForwardListener( - portB ?? "", - SANDBOX_B, - { artifactName: "phase-4-dashboard-listener-b-after-start", env: commandEnv() }, - ); - expect( - `${restoredForwardBAfterStart.reachable}:${listenerBAfterStart.valid}`, - `${restoredForwardBAfterStart.output}\n${listenerBBeforeStop.output}\n${listenerBAfterStart.output}`, - ).toBe("true:true"); - progress.phase("replace sandbox after stale registry refusal"); // Phase 5: direct OpenShell deletion leaves a stale registry entry that // status/connect preserve the stale record; rebuild refuses to invent its @@ -930,9 +908,7 @@ test( !releasedForwardB.reachable && retainedForwardAAfterStop.reachable && stoppedStatusTextB.includes("sandbox_container_stopped") && - !stoppedStatusTextB.includes("sandbox_dashboard_port_conflict") && - startB.exitCode === 0 && - restoredForwardBAfterStart.reachable, + !stoppedStatusTextB.includes("sandbox_dashboard_port_conflict"), staleRegistryRecovered: rebuild.exitCode === 0, gatewayStopGuidance: /Recovered NemoClaw gateway runtime|gateway is no longer configured after restart\/rebuild|gateway is still refusing connections after restart|gateway trust material rotated after restart/.test( From bd210d1326ebc8921938fb033784f0e52b5afc59 Mon Sep 17 00:00:00 2001 From: San Dang Date: Mon, 7 Sep 2026 19:52:56 +0700 Subject: [PATCH 18/20] test(onboard): model owned forward fixtures Signed-off-by: San Dang --- .../process-recovery-managed-startup.test.ts | 2 ++ test/cli/connect-recovery.test.ts | 8 ++++---- test/helpers/onboard-script-mocks.cjs | 6 ++++++ test/helpers/platform-override-node-options.ts | 13 +++++++++++-- .../process-recovery-custom-agent.test.ts | 5 +++++ .../process-recovery-managed-controller.test.ts | 4 ++++ test/process-recovery/process-recovery.test.ts | 8 ++++++++ test/runtime/gateway/recover-port-forward.test.ts | 4 ++-- test/sandbox-connect-inference/helpers.ts | 4 ++-- test/state/snapshot-gateway-guard.test.ts | 2 ++ 10 files changed, 46 insertions(+), 10 deletions(-) diff --git a/src/lib/actions/sandbox/process-recovery-managed-startup.test.ts b/src/lib/actions/sandbox/process-recovery-managed-startup.test.ts index 4bd01baf350..3b6cfea9fa0 100644 --- a/src/lib/actions/sandbox/process-recovery-managed-startup.test.ts +++ b/src/lib/actions/sandbox/process-recovery-managed-startup.test.ts @@ -3,6 +3,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; +import * as forwardService from "../../adapters/openshell/forward-service"; import * as openshellRuntime from "../../adapters/openshell/runtime"; import * as agentRuntime from "../../agent/runtime"; import * as registry from "../../state/registry"; @@ -36,6 +37,7 @@ function mockOpenClawSandbox(sandboxName: string): void { function mockRecoveredForward(_sandboxName: string): void { vi.spyOn(forwardHealth, "isLocalForwardReachable").mockReturnValue(true); + vi.spyOn(forwardService, "isForwardServiceListenerOwner").mockReturnValue(true); vi.spyOn(openshellRuntime, "captureOpenshell").mockReturnValue({ status: 0, output: "SANDBOX BIND PORT PID STATUS", diff --git a/test/cli/connect-recovery.test.ts b/test/cli/connect-recovery.test.ts index 47512a420d3..0a5e70db46d 100644 --- a/test/cli/connect-recovery.test.ts +++ b/test/cli/connect-recovery.test.ts @@ -12,7 +12,7 @@ import { LAUNCH_READINESS_PAIRING_QUALIFICATION_OUTPUT, launchReadinessRegistryFixture, } from "../helpers/launch-readiness-fixture"; -import { nonWslPlatformNodeOptions } from "../helpers/platform-override-node-options"; +import { syntheticForwardNodeOptions } from "../helpers/platform-override-node-options"; import { runWithEnv, testTimeoutOptions, @@ -278,7 +278,7 @@ describe("CLI connect recovery process contracts", () => { try { const result = runWithEnv("alpha connect --probe-only", { HOME: home, - NODE_OPTIONS: nonWslPlatformNodeOptions(home), + NODE_OPTIONS: syntheticForwardNodeOptions(home), PATH: `${localBin}:${process.env.PATH || ""}`, }); @@ -357,7 +357,7 @@ describe("CLI connect recovery process contracts", () => { try { const result = runWithEnv("alpha connect --probe-only", { HOME: home, - NODE_OPTIONS: nonWslPlatformNodeOptions(home), + NODE_OPTIONS: syntheticForwardNodeOptions(home), PATH: `${localBin}:${process.env.PATH || ""}`, }); @@ -479,7 +479,7 @@ describe("CLI connect recovery process contracts", () => { const result = runWithEnv("alpha connect", { HOME: home, - NODE_OPTIONS: nonWslPlatformNodeOptions(home), + NODE_OPTIONS: syntheticForwardNodeOptions(home), PATH: `${localBin}:${process.env.PATH || ""}`, }); diff --git a/test/helpers/onboard-script-mocks.cjs b/test/helpers/onboard-script-mocks.cjs index 03a90b63ae3..be837e8f958 100644 --- a/test/helpers/onboard-script-mocks.cjs +++ b/test/helpers/onboard-script-mocks.cjs @@ -44,6 +44,12 @@ if (process.env.NEMOCLAW_TEST_FORWARD_SERVICE_FIXTURE === "1") { return ready; }; } + if ( + resolved.includes(`${path.sep}adapters${path.sep}openshell${path.sep}forward-service.`) && + typeof loaded?.isForwardServiceListenerOwner === "function" + ) { + loaded.isForwardServiceListenerOwner = () => true; + } return loaded; }; } diff --git a/test/helpers/platform-override-node-options.ts b/test/helpers/platform-override-node-options.ts index 7a3caa83997..374c048fd9a 100644 --- a/test/helpers/platform-override-node-options.ts +++ b/test/helpers/platform-override-node-options.ts @@ -4,17 +4,26 @@ import fs from "node:fs"; import path from "node:path"; -export function nonWslPlatformNodeOptions( +export function syntheticForwardNodeOptions( directory: string, inheritedNodeOptions = process.env.NODE_OPTIONS, ): string { - const preload = path.join(directory, "force-non-wsl-platform.cjs"); + const preload = path.join(directory, "synthetic-forward-platform.cjs"); fs.writeFileSync( preload, [ "delete process.env.WSL_DISTRO_NAME;", "delete process.env.WSL_INTEROP;", 'require("node:os").release = () => "6.8.0-linux";', + 'const Module = require("node:module");', + "const originalLoad = Module._load;", + "Module._load = function loadSyntheticForward(request, parent, isMain) {", + " const loaded = originalLoad.call(this, request, parent, isMain);", + ' if (String(request).endsWith("/adapters/openshell/forward-service")) {', + " loaded.isForwardServiceListenerOwner = () => true;", + " }", + " return loaded;", + "};", "", ].join("\n"), { mode: 0o600 }, diff --git a/test/process-recovery/process-recovery-custom-agent.test.ts b/test/process-recovery/process-recovery-custom-agent.test.ts index 763471e4fa2..4dd675f8103 100644 --- a/test/process-recovery/process-recovery-custom-agent.test.ts +++ b/test/process-recovery/process-recovery-custom-agent.test.ts @@ -12,6 +12,9 @@ const requireSource = createRequire(import.meta.url); const { checkAndRecoverSandboxProcesses: checkAndRecoverSandboxProcessesImpl } = requireSource( "../../src/lib/actions/sandbox/process-recovery.ts", ) as typeof import("../../src/lib/actions/sandbox/process-recovery.js"); +const forwardService = requireSource( + "../../src/lib/adapters/openshell/forward-service.ts", +) as typeof import("../../src/lib/adapters/openshell/forward-service.js"); function checkAndRecoverSandboxProcesses( sandboxName: string, @@ -144,6 +147,7 @@ describe("checkAndRecoverSandboxProcesses custom agent recovery", () => { dashboardPort: 19000, }); vi.spyOn(forwardHealth, "isLocalForwardReachable").mockReturnValue(true); + vi.spyOn(forwardService, "isForwardServiceListenerOwner").mockReturnValue(true); vi.spyOn(openshellRuntime, "captureOpenshell").mockReturnValue({ status: 0, output: "SANDBOX BIND PORT PID STATUS", @@ -217,6 +221,7 @@ describe("checkAndRecoverSandboxProcesses custom agent recovery", () => { dashboardPort: 19000, }); vi.spyOn(forwardHealth, "isLocalForwardReachable").mockReturnValue(true); + vi.spyOn(forwardService, "isForwardServiceListenerOwner").mockReturnValue(true); vi.spyOn(openshellRuntime, "captureOpenshell").mockReturnValue({ status: 0, output: runningForward, diff --git a/test/process-recovery/process-recovery-managed-controller.test.ts b/test/process-recovery/process-recovery-managed-controller.test.ts index 67c4df954cc..ef4d2cb2215 100644 --- a/test/process-recovery/process-recovery-managed-controller.test.ts +++ b/test/process-recovery/process-recovery-managed-controller.test.ts @@ -13,6 +13,9 @@ const requireSource = createRequire(import.meta.url); const { checkAndRecoverSandboxProcesses: checkAndRecoverSandboxProcessesImpl } = requireSource( "../../src/lib/actions/sandbox/process-recovery.ts", ) as typeof import("../../src/lib/actions/sandbox/process-recovery.js"); +const forwardService = requireSource( + "../../src/lib/adapters/openshell/forward-service.ts", +) as typeof import("../../src/lib/adapters/openshell/forward-service.js"); function checkAndRecoverSandboxProcesses( sandboxName: string, @@ -411,6 +414,7 @@ describe("managed gateway recovery controller", () => { ); vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue(null); vi.spyOn(forwardHealth, "isLocalForwardReachable").mockReturnValue(true); + vi.spyOn(forwardService, "isForwardServiceListenerOwner").mockReturnValue(true); vi.spyOn(registry, "getSandbox").mockReturnValue({ name: "beta", agent: "openclaw", diff --git a/test/process-recovery/process-recovery.test.ts b/test/process-recovery/process-recovery.test.ts index cad24dd2916..ff644030e76 100644 --- a/test/process-recovery/process-recovery.test.ts +++ b/test/process-recovery/process-recovery.test.ts @@ -19,6 +19,9 @@ const { ensureSandboxPortForwardForPort } = requireSource( const { createProbeTimingRecorder } = requireSource( "../../src/lib/actions/sandbox/probe/timing.ts", ) as typeof import("../../src/lib/actions/sandbox/probe/timing.js"); +const forwardService = requireSource( + "../../src/lib/adapters/openshell/forward-service.ts", +) as typeof import("../../src/lib/adapters/openshell/forward-service.js"); function checkAndRecoverSandboxProcesses( sandboxName: string, @@ -172,6 +175,7 @@ hermes-box 127.0.0.1 18789 12345 running`; dashboardPort: 18789, }); vi.spyOn(forwardHealth, "isLocalForwardReachable").mockReturnValue(true); + vi.spyOn(forwardService, "isForwardServiceListenerOwner").mockReturnValue(true); vi.spyOn(openshellRuntime, "captureOpenshell").mockReturnValue({ status: 0, output: runningForward, @@ -334,6 +338,7 @@ hermes-box 127.0.0.1 18789 12345 running`; hermesDashboardInternalPort: 19119, }); vi.spyOn(forwardHealth, "isLocalForwardReachable").mockReturnValue(true); + vi.spyOn(forwardService, "isForwardServiceListenerOwner").mockReturnValue(true); vi.spyOn(openshellRuntime, "captureOpenshell").mockReturnValue({ status: 0, output: "SANDBOX BIND PORT PID STATUS", @@ -563,6 +568,7 @@ hermes-box 127.0.0.1 18789 12345 running`; dashboardPort: 18789, }); vi.spyOn(forwardHealth, "isLocalForwardReachable").mockReturnValue(true); + vi.spyOn(forwardService, "isForwardServiceListenerOwner").mockReturnValue(true); vi.spyOn(openshellRuntime, "captureOpenshell").mockReturnValue({ status: 0, output: "SANDBOX BIND PORT PID STATUS", @@ -622,6 +628,7 @@ hermes-box 127.0.0.1 18789 12345 running`; dashboardPort: 18789, }); vi.spyOn(forwardHealth, "isLocalForwardReachable").mockReturnValue(true); + vi.spyOn(forwardService, "isForwardServiceListenerOwner").mockReturnValue(true); vi.spyOn(openshellRuntime, "captureOpenshell").mockReturnValue({ status: 0, output: `SANDBOX BIND PORT PID STATUS\nhermes-box 127.0.0.1 18789 12345 running`, @@ -671,6 +678,7 @@ hermes-box 127.0.0.1 18789 12345 running`; dashboardPort: 18789, }); vi.spyOn(forwardHealth, "isLocalForwardReachable").mockReturnValue(true); + vi.spyOn(forwardService, "isForwardServiceListenerOwner").mockReturnValue(true); vi.spyOn(openshellRuntime, "captureOpenshell").mockReturnValue({ status: 0, output: `SANDBOX BIND PORT PID STATUS\nbeta 127.0.0.1 18789 12345 running`, diff --git a/test/runtime/gateway/recover-port-forward.test.ts b/test/runtime/gateway/recover-port-forward.test.ts index 27848ae8343..c026c2886ad 100644 --- a/test/runtime/gateway/recover-port-forward.test.ts +++ b/test/runtime/gateway/recover-port-forward.test.ts @@ -11,7 +11,7 @@ import { LAUNCH_READINESS_FIXTURE_POLICY, launchReadinessRegistryFixture, } from "../../helpers/launch-readiness-fixture"; -import { nonWslPlatformNodeOptions } from "../../helpers/platform-override-node-options"; +import { syntheticForwardNodeOptions } from "../../helpers/platform-override-node-options"; import { execTimeout, testTimeoutOptions } from "../../helpers/timeouts"; const tmpFixtures: string[] = []; @@ -332,7 +332,7 @@ function runRecover(fixture: Fixture) { env: { ...process.env, HOME: fixture.tmpDir, - NODE_OPTIONS: nonWslPlatformNodeOptions(fixture.tmpDir), + NODE_OPTIONS: syntheticForwardNodeOptions(fixture.tmpDir), PATH: "/usr/bin:/bin", NEMOCLAW_NO_CONNECT_HINT: "1", NEMOCLAW_FORWARD_RECOVERY_WAIT_MS: fixture.recoveryWaitMs, diff --git a/test/sandbox-connect-inference/helpers.ts b/test/sandbox-connect-inference/helpers.ts index e2fa8ee1e64..42f9c31c696 100644 --- a/test/sandbox-connect-inference/helpers.ts +++ b/test/sandbox-connect-inference/helpers.ts @@ -11,7 +11,7 @@ import { LAUNCH_READINESS_PAIRING_QUALIFICATION_OUTPUT, launchReadinessRegistryFixture, } from "../helpers/launch-readiness-fixture"; -import { nonWslPlatformNodeOptions } from "../helpers/platform-override-node-options"; +import { syntheticForwardNodeOptions } from "../helpers/platform-override-node-options"; import { execTimeout } from "../helpers/timeouts"; /** @@ -587,7 +587,7 @@ export function runConnect( encoding: "utf-8", env: { HOME: tmpDir, - NODE_OPTIONS: nonWslPlatformNodeOptions(tmpDir, ""), + NODE_OPTIONS: syntheticForwardNodeOptions(tmpDir, ""), PATH: `${path.join(tmpDir, ".local", "bin")}:/usr/bin:/bin`, NEMOCLAW_DISABLE_GATEWAY_DRIFT_PREFLIGHT: "1", NEMOCLAW_NO_CONNECT_HINT: "1", diff --git a/test/state/snapshot-gateway-guard.test.ts b/test/state/snapshot-gateway-guard.test.ts index 39debb670e2..8c64c0cb4e5 100644 --- a/test/state/snapshot-gateway-guard.test.ts +++ b/test/state/snapshot-gateway-guard.test.ts @@ -12,6 +12,7 @@ import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; +import { syntheticForwardNodeOptions } from "../helpers/platform-override-node-options"; import { execTimeout } from "../helpers/timeouts"; const CLI = path.join(import.meta.dirname, "../..", "bin", "nemoclaw.js"); @@ -334,6 +335,7 @@ function makeVmRestoreToEnv( return { HOME: home, + NODE_OPTIONS: syntheticForwardNodeOptions(home), NEMOCLAW_OPENSHELL_BIN: path.join(localBin, "openshell"), NEMOCLAW_GATEWAY_RECOVERY_SETTLE_SECONDS: "0", NEMOCLAW_TEST_SNAPSHOT_RESTORE_MARKER: snapshotRestoreMarker, From 2c265bbf714fed4667a12fb1604a49638fdc1257 Mon Sep 17 00:00:00 2001 From: San Dang Date: Mon, 7 Sep 2026 21:09:48 +0700 Subject: [PATCH 19/20] test(onboard): isolate CI host dependencies Signed-off-by: San Dang --- .../process-recovery-managed-startup.test.ts | 2 ++ .../onboard-fresh-create-identity.test.ts | 29 ++++++++++++++----- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/src/lib/actions/sandbox/process-recovery-managed-startup.test.ts b/src/lib/actions/sandbox/process-recovery-managed-startup.test.ts index 3b6cfea9fa0..c3ecbd7dad2 100644 --- a/src/lib/actions/sandbox/process-recovery-managed-startup.test.ts +++ b/src/lib/actions/sandbox/process-recovery-managed-startup.test.ts @@ -4,6 +4,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import * as forwardService from "../../adapters/openshell/forward-service"; +import * as openshellResolve from "../../adapters/openshell/resolve"; import * as openshellRuntime from "../../adapters/openshell/runtime"; import * as agentRuntime from "../../agent/runtime"; import * as registry from "../../state/registry"; @@ -38,6 +39,7 @@ function mockOpenClawSandbox(sandboxName: string): void { function mockRecoveredForward(_sandboxName: string): void { vi.spyOn(forwardHealth, "isLocalForwardReachable").mockReturnValue(true); vi.spyOn(forwardService, "isForwardServiceListenerOwner").mockReturnValue(true); + vi.spyOn(openshellResolve, "resolveOpenshell").mockReturnValue("/usr/bin/openshell"); vi.spyOn(openshellRuntime, "captureOpenshell").mockReturnValue({ status: 0, output: "SANDBOX BIND PORT PID STATUS", diff --git a/test/onboarding/onboard-fresh-create-identity.test.ts b/test/onboarding/onboard-fresh-create-identity.test.ts index a15c2435d11..739bb62e469 100644 --- a/test/onboarding/onboard-fresh-create-identity.test.ts +++ b/test/onboarding/onboard-fresh-create-identity.test.ts @@ -5,6 +5,7 @@ import assert from "node:assert/strict"; import { spawnSync } from "node:child_process"; import { createHash } from "node:crypto"; import fs from "node:fs"; +import net from "node:net"; import os from "node:os"; import path from "node:path"; @@ -19,6 +20,18 @@ beforeEach(() => { vi.stubEnv("NEMOCLAW_SANDBOX_PREBUILD", "1"); }); +function reserveFreePort(): Promise { + return new Promise((resolve, reject) => { + const probe = net.createServer(); + probe.once("error", reject); + probe.listen(0, "127.0.0.1", () => { + const address = probe.address(); + const port = typeof address === "object" && address ? address.port : 0; + probe.close(() => resolve(port)); + }); + }); +} + describe("fresh create identity", () => { it.each([ { @@ -151,6 +164,8 @@ describe("fresh create identity", () => { async ({ agent, apfInterceptorRequested, expectedOutcome, model, provider }) => { const repoRoot = path.join(import.meta.dirname, "../.."); const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-create-ready-")); + const gatewayPort = await reserveFreePort(); + const gatewayName = `nemoclaw-${String(gatewayPort)}`; const fakeBin = path.join(tmpDir, "bin"); const scriptPath = path.join(tmpDir, "create-sandbox-ready-check.js"); const payloadPath = path.join(tmpDir, "payload.json"); @@ -661,7 +676,7 @@ if (${JSON.stringify( console.error(error); process.exit(1); }); -`; +`.replaceAll("18080", String(gatewayPort)); fs.writeFileSync(scriptPath, script); const childEnv = { @@ -669,7 +684,7 @@ if (${JSON.stringify( HOME: tmpDir, PATH: `${fakeBin}:${process.env.PATH || ""}`, NEMOCLAW_NON_INTERACTIVE: expectedOutcome.startsWith("cancel-after-create-") ? "" : "1", - NEMOCLAW_GATEWAY_PORT: "18080", + NEMOCLAW_GATEWAY_PORT: String(gatewayPort), OPENSHELL_DRIVERS: "docker", NEMOCLAW_MESSAGING_PLAN_B64: expectedOutcome === "staged-messaging-refusal" @@ -701,8 +716,8 @@ if (${JSON.stringify( ); const identityFingerprint = createHash("sha256").update(payload.sandboxId).digest("hex"); const assertRecoveryTuple = (record: Record) => { - assert.equal(record.gatewayName, "nemoclaw-18080"); - assert.equal(record.gatewayPort, 18080); + assert.equal(record.gatewayName, gatewayName); + assert.equal(record.gatewayPort, gatewayPort); assert.equal(record.sandboxIdentityFingerprint, identityFingerprint); assert.equal(record.lifecycleGeneration, payload.recoveryRegistryEntry.lifecycleGeneration); }; @@ -768,7 +783,7 @@ if (${JSON.stringify( /--label ai\.nvidia\.nemoclaw\.create-attempt=[0-9a-f]{62}/u, ); const ownerScopedObservations = payload.lifecycleObservationCommands.filter( - (command: string) => command.includes("-g nemoclaw-18080"), + (command: string) => command.includes(`-g ${gatewayName}`), ); assert.ok( ownerScopedObservations.length >= 6, @@ -777,8 +792,8 @@ if (${JSON.stringify( assert.ok( ownerScopedObservations.every( (command: string) => - command.includes("sandbox get -g nemoclaw-18080 my-assistant") || - command.includes("sandbox list -g nemoclaw-18080"), + command.includes(`sandbox get -g ${gatewayName} my-assistant`) || + command.includes(`sandbox list -g ${gatewayName}`), ), `fresh identity observations must remain scoped to the owning gateway: ${JSON.stringify(ownerScopedObservations)}`, ); From 766105a7320f8fcac12e82bfa79545e9b2996d25 Mon Sep 17 00:00:00 2001 From: San Dang Date: Mon, 7 Sep 2026 23:50:01 +0700 Subject: [PATCH 20/20] fix(onboard): scope forward ownership to IPv4 --- .../openshell/forward-service.test.ts | 9 ++++++- src/lib/adapters/openshell/forward-service.ts | 26 +++++++++---------- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/src/lib/adapters/openshell/forward-service.test.ts b/src/lib/adapters/openshell/forward-service.test.ts index c74e3a16924..4ac111bc664 100644 --- a/src/lib/adapters/openshell/forward-service.test.ts +++ b/src/lib/adapters/openshell/forward-service.test.ts @@ -35,6 +35,7 @@ function createLinuxOwnerFixture(actualExecutable?: string) { const binRoot = path.join(root, "bin"); mkdirSync(path.join(procRoot, "net"), { recursive: true }); mkdirSync(path.join(procRoot, "4321", "fd"), { recursive: true }); + mkdirSync(path.join(procRoot, "9876", "fd"), { recursive: true }); mkdirSync(binRoot); const executable = path.join(binRoot, "openshell"); const runtime = actualExecutable ? path.join(binRoot, actualExecutable) : executable; @@ -44,7 +45,12 @@ function createLinuxOwnerFixture(actualExecutable?: string) { path.join(procRoot, "net", "tcp"), " 0: 0100007F:4965 00000000:0000 0A 00000000:00000000 00:00000000 00000000 998 0 12345 1\n", ); + writeFileSync( + path.join(procRoot, "net", "tcp6"), + " 1: 00000000000000000000000001000000:4965 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 998 0 67890 1\n", + ); symlinkSync("socket:[12345]", path.join(procRoot, "4321", "fd", "7")); + symlinkSync("socket:[67890]", path.join(procRoot, "9876", "fd", "8")); symlinkSync(runtime, path.join(procRoot, "4321", "exe")); return { procRoot, target: { ...target, executable } }; } @@ -126,7 +132,7 @@ describe("OpenShell forward service", () => { ).toBe(false); }); - it("proves Linux listener ownership through /proc without lsof", () => { + it("proves Linux IPv4 ownership while ignoring an IPv6-only listener", () => { const fixture = createLinuxOwnerFixture(); const expected = [fixture.target.executable, ...buildForwardServiceArgs(fixture.target)].join( " ", @@ -147,6 +153,7 @@ describe("OpenShell forward service", () => { }), ).toBe(true); expect(probe).toHaveBeenCalledTimes(3); + expect(probe).toHaveBeenCalledWith("lsof", ["-ti4TCP:18789", "-sTCP:LISTEN"]); expect(probe).toHaveBeenCalledWith("ps", ["-ww", "-p", "4321", "-o", "args="]); }); diff --git a/src/lib/adapters/openshell/forward-service.ts b/src/lib/adapters/openshell/forward-service.ts index ad184f4b9f1..472d8a9cba3 100644 --- a/src/lib/adapters/openshell/forward-service.ts +++ b/src/lib/adapters/openshell/forward-service.ts @@ -134,7 +134,7 @@ function captureProcess(executable: string, args: readonly string[]) { } function lsofListenerPids(port: number, probe: ForwardServiceOwnerProbe): string[] | null { - const result = probe("lsof", ["-ti", `:${String(port)}`, "-sTCP:LISTEN"]); + const result = probe("lsof", [`-ti4TCP:${String(port)}`, "-sTCP:LISTEN"]); if (result.status === null) return null; if (result.status !== 0) return []; return [ @@ -151,21 +151,19 @@ function linuxListenerPids(port: number, procRoot: string, workLimit: number): s if (!Number.isSafeInteger(workLimit) || workLimit < 1) return []; const portSuffix = `:${port.toString(16).padStart(4, "0").toUpperCase()}`; const socketInodes = new Set(); - for (const table of ["tcp", "tcp6"]) { - try { - for (const line of readFileSync(path.join(procRoot, "net", table), "utf8").split("\n")) { - const fields = line.trim().split(/\s+/u); - if ( - fields[3] === "0A" && - fields[1]?.toUpperCase().endsWith(portSuffix) && - /^\d+$/u.test(fields[9] ?? "") - ) { - socketInodes.add(fields[9]!); - } + try { + for (const line of readFileSync(path.join(procRoot, "net", "tcp"), "utf8").split("\n")) { + const fields = line.trim().split(/\s+/u); + if ( + fields[3] === "0A" && + fields[1]?.toUpperCase().endsWith(portSuffix) && + /^\d+$/u.test(fields[9] ?? "") + ) { + socketInodes.add(fields[9]!); } - } catch { - // A missing or unreadable table cannot prove ownership. } + } catch { + // A missing or unreadable IPv4 table cannot prove ownership. } if (socketInodes.size === 0) return [];