diff --git a/src/lib/actions/sandbox/connect.ts b/src/lib/actions/sandbox/connect.ts index 3bb4e38c548..df40efa0f8f 100644 --- a/src/lib/actions/sandbox/connect.ts +++ b/src/lib/actions/sandbox/connect.ts @@ -128,7 +128,7 @@ function runSandboxConnectProbe(sandboxName: string): void { process.exit(1); } if (processCheck.wasRunning) { - ensureSandboxInferenceRouteOrExit(sandboxName, { quiet: false }); + ensureSandboxInferenceRoute(sandboxName, { quiet: true }); if (processCheck.forwardRecovered) { console.log( ` Probe complete: ${agentName} gateway is running in '${sandboxName}'; restored dashboard port forward.`, @@ -139,11 +139,11 @@ function runSandboxConnectProbe(sandboxName: string): void { return; } if (processCheck.recovered) { - ensureSandboxInferenceRouteOrExit(sandboxName, { quiet: false }); + ensureSandboxInferenceRoute(sandboxName, { quiet: true }); console.log(` Probe complete: recovered ${agentName} gateway in '${sandboxName}'.`); return; } - ensureSandboxInferenceRouteOrExit(sandboxName, { quiet: false }); + ensureSandboxInferenceRoute(sandboxName, { quiet: true }); console.error( ` Probe failed: ${agentName} gateway is not running in '${sandboxName}' and automatic recovery failed.`, ); diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index f4584f1266a..86a51d3f138 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -17,9 +17,7 @@ const { }: typeof import("./onboard/branding") = require("./onboard/branding"); const { cleanupTempDir }: typeof import("./onboard/temp-files") = require("./onboard/temp-files"); const { stopStaleDashboardListenersForSandbox } = require("./onboard/stale-gateway-cleanup"); -const { - runBackgroundForwardStartWithDiagnostics, -}: typeof import("./onboard/forward-start") = require("./onboard/forward-start"); +const { looksLikeForwardPortConflict, runBackgroundForwardStartWithPortReleaseRetries }: typeof import("./onboard/forward-start") = require("./onboard/forward-start"); const { ensureOllamaLoopbackSystemdOverride, }: typeof import("./onboard/ollama-systemd") = require("./onboard/ollama-systemd"); @@ -9303,17 +9301,16 @@ function ensureDashboardForward( parsedUrl.port = String(actualPort); const actualTarget = getDashboardForwardTarget(parsedUrl.toString()); runOpenshell(["forward", "stop", String(actualPort)], { ignoreError: true }); - const { result: fwdResult, diagnostic: fwdDiagnostic } = - runBackgroundForwardStartWithDiagnostics((stdio, timeout) => + const { result: fwdResult, diagnostic: fwdDiagnostic } = runBackgroundForwardStartWithPortReleaseRetries( + (stdio, timeout) => runOpenshell( ["forward", "start", "--background", actualTarget, sandboxName], { ignoreError: true, suppressOutput: true, stdio, timeout }, ), - ); + () => { sleep(1); runOpenshell(["forward", "stop", String(actualPort)], { ignoreError: true }); }, + ); if (fwdResult && fwdResult.status !== 0) { - const looksLikePortConflict = - fwdDiagnostic === "" || - /eaddrinuse|address already in use|port .* in use|bind: .*in use/i.test(fwdDiagnostic); + const looksLikePortConflict = looksLikeForwardPortConflict(fwdDiagnostic); if (rollbackSandboxOnFailure) { // The sandbox was just created, committed to actualPort via its // baked-in CHAT_UI_URL and NEMOCLAW_DASHBOARD_PORT env. Silently diff --git a/src/lib/onboard/forward-start.ts b/src/lib/onboard/forward-start.ts index 2146203b6c0..05e45e958ad 100644 --- a/src/lib/onboard/forward-start.ts +++ b/src/lib/onboard/forward-start.ts @@ -37,6 +37,10 @@ function readDiagnosticFile(filePath: string): string { } } +export function looksLikeForwardPortConflict(diagnostic: string): boolean { + return /eaddrinuse|address already in use|port .* in use|bind: .*in use/i.test(diagnostic); +} + export function runBackgroundForwardStartWithDiagnostics( runForwardStart: BackgroundForwardStartRunner, timeoutMs = 30_000, @@ -82,3 +86,20 @@ export function runBackgroundForwardStartWithDiagnostics( cleanupTempDir(forwardDiagPath, "nemoclaw-forward-start"); } } + +export function runBackgroundForwardStartWithPortReleaseRetries( + runForwardStart: BackgroundForwardStartRunner, + beforeRetry: () => void, + maxRetries = 3, +): { result: BackgroundForwardStartResult; diagnostic: string } { + let attempt = runBackgroundForwardStartWithDiagnostics(runForwardStart); + for ( + let retries = 0; + attempt.result.status !== 0 && looksLikeForwardPortConflict(attempt.diagnostic) && retries < maxRetries; + retries++ + ) { + beforeRetry(); + attempt = runBackgroundForwardStartWithDiagnostics(runForwardStart); + } + return attempt; +} diff --git a/test/onboard.test.ts b/test/onboard.test.ts index f8ce6b019a0..a7be88adf86 100644 --- a/test/onboard.test.ts +++ b/test/onboard.test.ts @@ -10315,9 +10315,9 @@ const { createSandbox } = require(${onboardPath}); assert.match(source, /if \(fwdResult && fwdResult\.status !== 0\)/); assert.match(source, /if \(rollbackSandboxOnFailure\)/); assert.match(source, /const looksLikePortConflict =/); - assert.match(source, /eaddrinuse\|address already in use/i); + assert.match(source, /looksLikeForwardPortConflict/); assert.match(source, /suppressOutput: true/); - assert.match(source, /runBackgroundForwardStartWithDiagnostics/); + assert.match(source, /runBackgroundForwardStartWithPortReleaseRetries/); assert.doesNotMatch( source, /forward", "start", "--background"[\s\S]{0,260}stdio: \["ignore", "pipe", "pipe"\]/, @@ -10329,6 +10329,8 @@ const { createSandbox } = require(${onboardPath}); ); assert.match(helperSource, /secureTempFile\("nemoclaw-forward-start", "\.out"\)/); assert.match(helperSource, /runForwardStart\(\["ignore", outFd, errFd\], timeoutMs\)/); + assert.match(helperSource, /eaddrinuse\|address already in use/i); + assert.match(helperSource, /maxRetries = 3/); assert.match( source, /runOpenshell\(\["sandbox", "delete", sandboxName\], \{ ignoreError: true \}\)/,