Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 15 additions & 9 deletions src/lib/onboard.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ const {
const { cleanupTempDir }: typeof import("./onboard/temp-files") = require("./onboard/temp-files");
const { stopStaleDashboardListenersForSandbox } = require("./onboard/stale-gateway-cleanup");
const { bestEffortForwardStop } = require("./onboard/forward-cleanup");
const { looksLikeForwardPortConflict, runBackgroundForwardStartWithPortReleaseRetries }: typeof import("./onboard/forward-start") = require("./onboard/forward-start");
const { buildDetachedForwardStartSpawn, looksLikeForwardPortConflict, runDetachedForwardStartWithPortReleaseRetries }: typeof import("./onboard/forward-start") = require("./onboard/forward-start");
const {
ensureOllamaLoopbackSystemdOverride,
}: typeof import("./onboard/ollama-systemd") = require("./onboard/ollama-systemd");
Expand Down Expand Up @@ -8644,15 +8644,21 @@ function ensureDashboardForward(
parsedUrl.port = String(actualPort);
const actualTarget = getDashboardForwardTarget(parsedUrl.toString());
bestEffortForwardStop(runOpenshell, actualPort);
const { result: fwdResult, diagnostic: fwdDiagnostic } = runBackgroundForwardStartWithPortReleaseRetries(
(stdio, timeout) =>
runOpenshell(
["forward", "start", "--background", actualTarget, sandboxName],
{ ignoreError: true, suppressOutput: true, stdio, timeout },
),
() => { sleep(1); bestEffortForwardStop(runOpenshell, actualPort); },
// Detached spawn + forward-list poll (#4064) so the openshell CLI's
// inherited stdio cannot trip spawnSync's timeout on Docker-compat hosts.
const fwdOutcome = runDetachedForwardStartWithPortReleaseRetries(
buildDetachedForwardStartSpawn(
openshellArgv(["forward", "start", "--background", actualTarget, sandboxName]),
),
() => runCaptureOpenshell(["forward", "list"], { ignoreError: true }),
{ port: actualPort, sandboxName },
() => {
sleep(1);
bestEffortForwardStop(runOpenshell, actualPort);
},
);
if (fwdResult && fwdResult.status !== 0) {
const fwdDiagnostic = fwdOutcome.diagnostic;
if (!fwdOutcome.ok) {
const looksLikePortConflict = looksLikeForwardPortConflict(fwdDiagnostic);
if (rollbackSandboxOnFailure) {
// The sandbox was just created, committed to actualPort via its
Expand Down
230 changes: 230 additions & 0 deletions src/lib/onboard/forward-start.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,230 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { describe, expect, it, vi } from "vitest";

import {
looksLikeForwardPortConflict,
runDetachedForwardStartWithDiagnostics,
runDetachedForwardStartWithPortReleaseRetries,
} from "../../../dist/lib/onboard/forward-start";

// Build an `openshell forward list`-shaped output for the given live entries.
// Mirrors the column layout (SANDBOX BIND PORT PID STATUS) that
// `getOccupiedPorts` parses, so the helper recognises the forward as live.
function forwardListWith(entries: Array<{ sandbox: string; port: number; status?: string }>): string {
const header = "SANDBOX BIND PORT PID STATUS";
const rows = entries.map(
(e) => `${e.sandbox} 127.0.0.1 ${e.port} 1234 ${e.status ?? "running"}`,
);
return [header, ...rows].join("\n");
}

describe("runDetachedForwardStartWithDiagnostics", () => {
it("returns ok as soon as the forward appears in the list", () => {
const fetchList = vi
.fn()
.mockReturnValueOnce(forwardListWith([])) // first poll: nothing yet
.mockReturnValue(forwardListWith([{ sandbox: "my-sandbox", port: 18789 }]));
const spawn = vi.fn().mockReturnValue({ pid: 42 });
const sleep = vi.fn();

const result = runDetachedForwardStartWithDiagnostics(
spawn,
fetchList,
{ port: 18789, sandboxName: "my-sandbox" },
{ overallTimeoutMs: 10_000, pollIntervalMs: 10, sleepMs: sleep },
);

expect(result.ok).toBe(true);
expect(result.reason).toBe("ok");
expect(result.pid).toBe(42);
expect(spawn).toHaveBeenCalledTimes(1);
// First poll missed → one sleep before the second poll observed the entry.
expect(sleep).toHaveBeenCalledTimes(1);
});

it("ignores entries that belong to a different sandbox", () => {
const fetchList = vi
.fn()
.mockReturnValue(forwardListWith([{ sandbox: "other-sandbox", port: 18789 }]));
const spawn = vi.fn().mockReturnValue({ pid: 42 });
const sleep = vi.fn();

const result = runDetachedForwardStartWithDiagnostics(
spawn,
fetchList,
{ port: 18789, sandboxName: "my-sandbox" },
{ overallTimeoutMs: 50, pollIntervalMs: 10, sleepMs: sleep },
);

expect(result.ok).toBe(false);
expect(result.reason).toBe("timeout");
});

it("reports timeout when the forward never appears", () => {
const fetchList = vi.fn().mockReturnValue(forwardListWith([]));
const spawn = vi.fn().mockReturnValue({ pid: 42 });
const sleep = vi.fn();

const result = runDetachedForwardStartWithDiagnostics(
spawn,
fetchList,
{ port: 18789, sandboxName: "my-sandbox" },
{ overallTimeoutMs: 30, pollIntervalMs: 10, sleepMs: sleep },
);

expect(result.ok).toBe(false);
expect(result.reason).toBe("timeout");
expect(result.diagnostic).toMatch(/forward did not appear in list within 30ms/);
});

it("surfaces spawn errors immediately without polling", () => {
const fetchList = vi.fn();
const spawn = vi.fn().mockReturnValue({ error: new Error("ENOENT: openshell not found") });
const sleep = vi.fn();

const result = runDetachedForwardStartWithDiagnostics(
spawn,
fetchList,
{ port: 18789, sandboxName: "my-sandbox" },
{ overallTimeoutMs: 10_000, pollIntervalMs: 10, sleepMs: sleep },
);

expect(result.ok).toBe(false);
expect(result.reason).toBe("spawn-error");
expect(result.diagnostic).toMatch(/ENOENT/);
expect(fetchList).not.toHaveBeenCalled();
expect(sleep).not.toHaveBeenCalled();
});

it("surfaces persistent fetchForwardList failures in the timeout diagnostic", () => {
const fetchList = vi.fn().mockImplementation(() => {
throw new Error("gateway transport: connection refused");
});
const spawn = vi.fn().mockReturnValue({ pid: 42 });
const sleep = vi.fn();

const result = runDetachedForwardStartWithDiagnostics(
spawn,
fetchList,
{ port: 18789, sandboxName: "my-sandbox" },
{ overallTimeoutMs: 30, pollIntervalMs: 10, sleepMs: sleep },
);

expect(result.ok).toBe(false);
expect(result.reason).toBe("timeout");
expect(result.diagnostic).toMatch(/openshell forward list failed/);
expect(result.diagnostic).toMatch(/connection refused/);
});

it("treats fetchForwardList exceptions as transient and keeps polling", () => {
const fetchList = vi
.fn()
.mockImplementationOnce(() => {
throw new Error("gateway not reachable yet");
})
.mockReturnValue(forwardListWith([{ sandbox: "my-sandbox", port: 18789 }]));
const spawn = vi.fn().mockReturnValue({ pid: 42 });
const sleep = vi.fn();

const result = runDetachedForwardStartWithDiagnostics(
spawn,
fetchList,
{ port: 18789, sandboxName: "my-sandbox" },
{ overallTimeoutMs: 10_000, pollIntervalMs: 10, sleepMs: sleep },
);

expect(result.ok).toBe(true);
expect(fetchList).toHaveBeenCalledTimes(2);
});
});

describe("runDetachedForwardStartWithPortReleaseRetries", () => {
it("retries after a port-conflict diagnostic, then succeeds", () => {
const fetchList = vi
.fn()
.mockReturnValueOnce(forwardListWith([])) // first attempt: never appears
.mockReturnValueOnce(forwardListWith([])) // (timeout settles)
.mockReturnValue(forwardListWith([{ sandbox: "my-sandbox", port: 18789 }]));
const beforeRetry = vi.fn();
// First spawn surfaces a port-conflict in its diagnostic synthesised via
// an Error message; the second spawn succeeds and the forward appears.
const spawn = vi
.fn()
.mockReturnValueOnce({ error: new Error("EADDRINUSE: address already in use") })
.mockReturnValueOnce({ pid: 99 });
const sleep = vi.fn();

const result = runDetachedForwardStartWithPortReleaseRetries(
spawn,
fetchList,
{ port: 18789, sandboxName: "my-sandbox" },
beforeRetry,
3,
{ overallTimeoutMs: 30, pollIntervalMs: 10, sleepMs: sleep },
);

expect(result.ok).toBe(true);
expect(beforeRetry).toHaveBeenCalledTimes(1);
expect(spawn).toHaveBeenCalledTimes(2);
});

it("does not retry when the failure does not look like a port conflict", () => {
const fetchList = vi.fn().mockReturnValue(forwardListWith([]));
const beforeRetry = vi.fn();
const spawn = vi.fn().mockReturnValue({ pid: 42 });
const sleep = vi.fn();

const result = runDetachedForwardStartWithPortReleaseRetries(
spawn,
fetchList,
{ port: 18789, sandboxName: "my-sandbox" },
beforeRetry,
3,
{ overallTimeoutMs: 20, pollIntervalMs: 10, sleepMs: sleep },
);

expect(result.ok).toBe(false);
expect(result.reason).toBe("timeout");
expect(beforeRetry).not.toHaveBeenCalled();
expect(spawn).toHaveBeenCalledTimes(1);
});

it("stops retrying after maxRetries even if conflict diagnostics persist", () => {
const fetchList = vi.fn().mockReturnValue(forwardListWith([]));
const beforeRetry = vi.fn();
const spawn = vi
.fn()
.mockReturnValue({ error: new Error("EADDRINUSE: address already in use") });
const sleep = vi.fn();

const result = runDetachedForwardStartWithPortReleaseRetries(
spawn,
fetchList,
{ port: 18789, sandboxName: "my-sandbox" },
beforeRetry,
2,
{ overallTimeoutMs: 20, pollIntervalMs: 10, sleepMs: sleep },
);

expect(result.ok).toBe(false);
expect(beforeRetry).toHaveBeenCalledTimes(2);
expect(spawn).toHaveBeenCalledTimes(3); // initial + 2 retries
});
});

describe("looksLikeForwardPortConflict", () => {
it("matches the common port-in-use signals", () => {
expect(looksLikeForwardPortConflict("listen tcp 0.0.0.0:18789: bind: address already in use")).toBe(
true,
);
expect(looksLikeForwardPortConflict("EADDRINUSE")).toBe(true);
expect(looksLikeForwardPortConflict("port 18789 in use")).toBe(true);
});

it("returns false for unrelated errors", () => {
expect(looksLikeForwardPortConflict("transport: connection refused")).toBe(false);
expect(looksLikeForwardPortConflict("")).toBe(false);
});
});
Loading
Loading