diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 69791c6b744..6f994d59601 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -325,9 +325,8 @@ const { rejectUnsupportedWindowsHostOllama, shouldFrontOllamaWithProxy, }: typeof import("./onboard/local-inference-topology") = require("./onboard/local-inference-topology"); -const { - waitForGatewayHealth, -}: typeof import("./onboard/gateway-health-wait") = require("./onboard/gateway-health-wait"); +const { waitForGatewayHealth }: typeof import("./onboard/gateway-health-wait") = + require("./onboard/gateway-health-wait"); const { resolveOpenshell } = require("./adapters/openshell/resolve"); const credentials: typeof import("./credentials/store") = require("./credentials/store"); const { @@ -524,8 +523,11 @@ const { trackChildExit } = require("./onboard/child-exit-tracker") as typeof import("./onboard/child-exit-tracker"); const { reportDockerDriverGatewayStartFailure } = require("./onboard/docker-driver-gateway-failure") as typeof import("./onboard/docker-driver-gateway-failure"); -const { createFinalGatewayStartFailureHandler, reportLegacyGatewayStartResultFailure } = - require("./onboard/gateway-start-failure") as typeof import("./onboard/gateway-start-failure"); +const { + createFinalGatewayStartFailureHandler, + normalizeGatewayStartError, + reportLegacyGatewayStartResultFailure, +} = require("./onboard/gateway-start-failure") as typeof import("./onboard/gateway-start-failure"); const dockerDriverGatewayEnv: typeof import("./onboard/docker-driver-gateway-env") = require("./onboard/docker-driver-gateway-env"); const dockerDriverGatewayRuntimeMarker: typeof import("./onboard/docker-driver-gateway-runtime-marker") = @@ -1896,7 +1898,7 @@ async function startGatewayWithOptions( return; } - throw new Error("Gateway failed to start"); + throw new Error(`Gateway failed within ${healthWait.count * healthWait.interval}s.`); }, { retries, @@ -1912,11 +1914,9 @@ async function startGatewayWithOptions( }, }, ); - } catch { - if (exitOnFailure) { - handleFinalGatewayStartFailure({ retries, dockerUnreachable }); - } - throw new Error("Gateway failed to start"); + } catch (error) { + if (exitOnFailure) handleFinalGatewayStartFailure({ retries, dockerUnreachable }); + throw normalizeGatewayStartError(error); } console.log(" ✓ Gateway is healthy"); diff --git a/src/lib/onboard/__test-helpers__/virtual-clock.ts b/src/lib/onboard/__test-helpers__/virtual-clock.ts new file mode 100644 index 00000000000..fcb88c270a1 --- /dev/null +++ b/src/lib/onboard/__test-helpers__/virtual-clock.ts @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { vi } from "vitest"; + +export function createVirtualClock(startMs = 1_000_000_000_000) { + let currentMs = startMs; + const advance = (seconds: number) => { + currentMs += Math.max(0, seconds) * 1000; + }; + return { + advance, + now: () => currentMs, + sleeper: vi.fn(advance), + }; +} diff --git a/src/lib/onboard/docker-driver-gateway-service.test.ts b/src/lib/onboard/docker-driver-gateway-service.test.ts index 14dda05e370..f50515d0cfa 100644 --- a/src/lib/onboard/docker-driver-gateway-service.test.ts +++ b/src/lib/onboard/docker-driver-gateway-service.test.ts @@ -3,13 +3,14 @@ import { describe, expect, it, vi } from "vitest"; +import { createVirtualClock } from "./__test-helpers__/virtual-clock"; import { getOpenShellGatewayUserServiceBinaryPaths, getOpenShellGatewayUserServicePaths, hasOpenShellGatewayUserService, - startPackageManagedDockerDriverGateway, - startOpenShellGatewayUserService, type SpawnSyncLikeResult, + startOpenShellGatewayUserService, + startPackageManagedDockerDriverGateway, } from "./docker-driver-gateway-service"; const STATUS_CONNECTED = ` @@ -254,6 +255,7 @@ describe("docker-driver-gateway-service", () => { it("uses the package-managed service only after endpoint, metadata, and gRPC health are ready", async () => { const events: string[] = []; + const clock = createVirtualClock(); let registerCount = 0; const registerDockerDriverGatewayEndpoint = vi.fn(() => { events.push("register"); @@ -268,14 +270,18 @@ describe("docker-driver-gateway-service", () => { gatewayName: "nemoclaw", hasOpenShellGatewayUserService: () => true, healthPollCount: 3, - healthPollInterval: 0, + healthPollInterval: 1, isDockerDriverGatewayReady: async () => { events.push("ready"); return true; }, + now: clock.now, registerDockerDriverGatewayEndpoint, runCaptureOpenshell: (args) => (args[0] === "status" ? STATUS_CONNECTED : GATEWAY_INFO), - sleepSeconds: () => events.push("sleep"), + sleepSeconds: (seconds) => { + events.push("sleep"); + clock.advance(seconds); + }, skipSandboxBridgeReachability: false, startOpenShellGatewayUserService: () => ({ attempted: true, @@ -291,6 +297,42 @@ describe("docker-driver-gateway-service", () => { expect(events).toEqual(["register", "sleep", "register", "ready", "clear", "verify"]); }); + it("preserves bounded immediate package-service probes when the interval is zero", async () => { + const clock = createVirtualClock(); + let registerCount = 0; + + await expect( + startPackageManagedDockerDriverGateway({ + clearDockerDriverGatewayRuntimeFiles: vi.fn(), + exitOnFailure: false, + gatewayName: "nemoclaw", + hasOpenShellGatewayUserService: () => true, + healthPollCount: 3, + healthPollInterval: 0, + isDockerDriverGatewayReady: async () => true, + now: clock.now, + registerDockerDriverGatewayEndpoint: () => { + registerCount += 1; + return registerCount >= 3; + }, + runCaptureOpenshell: (args) => (args[0] === "status" ? STATUS_CONNECTED : GATEWAY_INFO), + sleepSeconds: clock.sleeper, + skipSandboxBridgeReachability: false, + startOpenShellGatewayUserService: () => ({ + attempted: true, + fallbackAllowed: false, + started: true, + }), + verifySandboxBridgeGatewayReachableOrExit: vi.fn(), + }), + ).resolves.toBe(true); + + expect(registerCount).toBe(3); + expect(clock.sleeper).toHaveBeenCalledTimes(2); + expect(clock.sleeper).toHaveBeenNthCalledWith(1, 0); + expect(clock.sleeper).toHaveBeenNthCalledWith(2, 0); + }); + it("falls back to standalone when package-managed service startup is unavailable", async () => { const registerDockerDriverGatewayEndpoint = vi.fn(() => true); @@ -318,6 +360,7 @@ describe("docker-driver-gateway-service", () => { it("keeps standalone runtime breadcrumbs when service health never becomes ready", async () => { const clearDockerDriverGatewayRuntimeFiles = vi.fn(); + const clock = createVirtualClock(); await expect( startPackageManagedDockerDriverGateway({ @@ -326,9 +369,12 @@ describe("docker-driver-gateway-service", () => { gatewayName: "nemoclaw", hasOpenShellGatewayUserService: () => true, healthPollCount: 1, + healthPollInterval: 1, isDockerDriverGatewayReady: async () => false, + now: clock.now, registerDockerDriverGatewayEndpoint: () => true, runCaptureOpenshell: (args) => (args[0] === "status" ? STATUS_CONNECTED : GATEWAY_INFO), + sleepSeconds: clock.advance, skipSandboxBridgeReachability: false, startOpenShellGatewayUserService: () => ({ attempted: true, @@ -337,7 +383,7 @@ describe("docker-driver-gateway-service", () => { }), verifySandboxBridgeGatewayReachableOrExit: vi.fn(), }), - ).rejects.toThrow("did not become healthy"); + ).rejects.toThrow("configured 1s health deadline"); expect(clearDockerDriverGatewayRuntimeFiles).not.toHaveBeenCalled(); }); diff --git a/src/lib/onboard/docker-driver-gateway-service.ts b/src/lib/onboard/docker-driver-gateway-service.ts index 2b3258d1688..dcdc0db52a0 100644 --- a/src/lib/onboard/docker-driver-gateway-service.ts +++ b/src/lib/onboard/docker-driver-gateway-service.ts @@ -8,6 +8,10 @@ import path from "node:path"; import { sleepSeconds, waitUntilAsync } from "../core/wait"; import { isGatewayHealthy } from "../state/gateway"; import { envInt } from "./env"; +import { + createGatewayHealthWaitOptions, + formatGatewayHealthWaitLimit, +} from "./gateway-health-wait"; import { isDockerDriverGatewayHttpReady } from "./gateway-http-readiness"; export const OPENSHELL_GATEWAY_USER_SERVICE = "openshell-gateway"; @@ -49,6 +53,7 @@ export interface PackageManagedDockerDriverGatewayOptions { healthPollCount?: number; healthPollInterval?: number; isDockerDriverGatewayReady?: () => Promise; + now?: () => number; registerDockerDriverGatewayEndpoint: () => boolean; runCaptureOpenshell: (args: string[], opts?: { ignoreError?: boolean }) => string; sleepSeconds?: (seconds: number) => void; @@ -291,6 +296,7 @@ export async function startPackageManagedDockerDriverGateway({ healthPollCount, healthPollInterval, isDockerDriverGatewayReady = isDockerDriverGatewayHttpReady, + now = Date.now, registerDockerDriverGatewayEndpoint, runCaptureOpenshell, sleepSeconds: sleepSecondsImpl = sleepSeconds, @@ -323,29 +329,22 @@ export async function startPackageManagedDockerDriverGateway({ const pollCount = healthPollCount ?? envInt("NEMOCLAW_HEALTH_POLL_COUNT", 30); const pollInterval = healthPollInterval ?? envInt("NEMOCLAW_HEALTH_POLL_INTERVAL", 2); - const pollIntervalMs = Math.max(0, pollInterval * 1000); + const waitOptions = createGatewayHealthWaitOptions(pollCount, pollInterval, now, (ms) => + sleepSecondsImpl(ms / 1000), + ); const healthy = - pollCount > 0 && - (await waitUntilAsync( - async () => { - if (!registerDockerDriverGatewayEndpoint()) return false; - const status = runCaptureOpenshell(["status"], { ignoreError: true }); - const namedInfo = runCaptureOpenshell(["gateway", "info", "-g", gatewayName], { - ignoreError: true, - }); - const currentInfo = runCaptureOpenshell(["gateway", "info"], { ignoreError: true }); - return ( - isGatewayHealthy(status, namedInfo, currentInfo) && (await isDockerDriverGatewayReady()) - ); - }, - { - initialIntervalMs: pollIntervalMs, - maxIntervalMs: pollIntervalMs, - backoffFactor: 1, - maxAttempts: pollCount, - sleep: (ms) => sleepSecondsImpl(ms / 1000), - }, - )); + waitOptions !== null && + (await waitUntilAsync(async () => { + if (!registerDockerDriverGatewayEndpoint()) return false; + const status = runCaptureOpenshell(["status"], { ignoreError: true }); + const namedInfo = runCaptureOpenshell(["gateway", "info", "-g", gatewayName], { + ignoreError: true, + }); + const currentInfo = runCaptureOpenshell(["gateway", "info"], { ignoreError: true }); + return ( + isGatewayHealthy(status, namedInfo, currentInfo) && (await isDockerDriverGatewayReady()) + ); + }, waitOptions)); if (healthy) { clearDockerDriverGatewayRuntimeFiles(); await verifySandboxBridgeGatewayReachableOrExit(exitOnFailure, { @@ -355,7 +354,10 @@ export async function startPackageManagedDockerDriverGateway({ return true; } - const message = "OpenShell gateway user service started but did not become healthy."; + const message = `OpenShell gateway user service started but did not become healthy within the configured ${formatGatewayHealthWaitLimit( + pollCount, + pollInterval, + )}.`; console.error(` ${message}`); console.error(" Check: systemctl --user status openshell-gateway"); if (exitOnFailure) process.exit(1); diff --git a/src/lib/onboard/gateway-health-wait.test.ts b/src/lib/onboard/gateway-health-wait.test.ts index 94e23699016..66ebb68316f 100644 --- a/src/lib/onboard/gateway-health-wait.test.ts +++ b/src/lib/onboard/gateway-health-wait.test.ts @@ -3,7 +3,13 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; -import { type GatewayHealthWaitOptions, waitForGatewayHealth } from "./gateway-health-wait"; +import { createVirtualClock } from "./__test-helpers__/virtual-clock"; +import { + formatGatewayHealthWaitLimit, + type GatewayHealthWaitOptions, + getGatewayHealthWaitBudgetMs, + waitForGatewayHealth, +} from "./gateway-health-wait"; function buildOptions(overrides: Partial = {}): GatewayHealthWaitOptions { return { @@ -49,16 +55,19 @@ describe("waitForGatewayHealth", () => { }); it("returns false when HTTP readiness never follows healthy metadata", async () => { + const clock = createVirtualClock(); const options = buildOptions({ healthPollCount: 2, isGatewayHttpReady: vi.fn(async () => false), + now: clock.now, + sleepSeconds: clock.sleeper, }); await expect(waitForGatewayHealth(options)).resolves.toBe(false); expect(options.isGatewayHealthy).toHaveBeenCalledTimes(2); expect(options.isGatewayHttpReady).toHaveBeenCalledTimes(2); - expect(options.sleepSeconds).toHaveBeenCalledTimes(1); + expect(options.sleepSeconds).toHaveBeenCalledTimes(2); }); it("force-refreshes metadata after bootstrap secret repair", async () => { @@ -85,19 +94,94 @@ describe("waitForGatewayHealth", () => { expect(options.attachGatewayMetadataIfNeeded).toHaveBeenCalledWith(); }); - it("stops after healthPollCount attempts without sleeping after the final failed probe", async () => { + it("polls until the configured health deadline instead of stopping at the count cap (#3768)", async () => { + const clock = createVirtualClock(); + const isGatewayHealthy = vi.fn(() => { + clock.advance(1); + return false; + }); const options = buildOptions({ - healthPollCount: 3, - isGatewayHealthy: vi.fn(() => false), + healthPollCount: 10, + healthPollIntervalSeconds: 1, + isGatewayHealthy, + now: clock.now, + sleepSeconds: clock.sleeper, }); await expect(waitForGatewayHealth(options)).resolves.toBe(false); - expect(options.isGatewayHealthy).toHaveBeenCalledTimes(3); + expect(isGatewayHealthy).toHaveBeenCalled(); + expect(isGatewayHealthy.mock.calls.length).toBeLessThan(10); + expect(options.isGatewayHttpReady).toHaveBeenCalledTimes(isGatewayHealthy.mock.calls.length); + expect(clock.sleeper).toHaveBeenCalled(); + expect(clock.sleeper.mock.calls.every(([seconds]) => seconds === 1)).toBe(true); + }); + + it("preserves the configured immediate probes when the interval is zero (#3768)", async () => { + const probeSignals: Array = []; + const isGatewayHealthy = vi + .fn<() => boolean>() + .mockReturnValueOnce(false) + .mockReturnValueOnce(false) + .mockReturnValueOnce(true); + const sleepSeconds = vi.fn(); + const options = buildOptions({ + healthPollCount: 3, + healthPollIntervalSeconds: 0, + isGatewayHealthy, + isGatewayHttpReady: vi.fn(async (signal?: AbortSignal) => { + probeSignals.push(signal); + return true; + }), + now: vi.fn(() => Number.MAX_SAFE_INTEGER), + sleepSeconds, + }); + + await expect(waitForGatewayHealth(options)).resolves.toBe(true); + + expect(isGatewayHealthy).toHaveBeenCalledTimes(3); expect(options.isGatewayHttpReady).toHaveBeenCalledTimes(3); - expect(options.sleepSeconds).toHaveBeenCalledTimes(2); - expect(options.sleepSeconds).toHaveBeenNthCalledWith(1, 2); - expect(options.sleepSeconds).toHaveBeenNthCalledWith(2, 2); + expect(sleepSeconds).toHaveBeenCalledTimes(2); + expect(sleepSeconds).toHaveBeenNthCalledWith(1, 0); + expect(sleepSeconds).toHaveBeenNthCalledWith(2, 0); + expect(probeSignals.map((signal) => signal?.aborted)).toEqual([true, true, false]); + expect(formatGatewayHealthWaitLimit(3, 0)).toBe("3 immediate health probes"); + }); + + it("does not probe after a positive health deadline expires before the first attempt", async () => { + const now = vi.fn().mockReturnValueOnce(0).mockReturnValue(1000); + const options = buildOptions({ + healthPollCount: 1, + healthPollIntervalSeconds: 1, + now, + }); + + await expect(waitForGatewayHealth(options)).resolves.toBe(false); + + expect(options.runCaptureOpenshell).not.toHaveBeenCalled(); + expect(options.isGatewayHealthy).not.toHaveBeenCalled(); + expect(options.isGatewayHttpReady).not.toHaveBeenCalled(); + }); + + it("preserves a rejected HTTP readiness probe error", async () => { + const probeError = new Error("readiness transport failed"); + const options = buildOptions({ + healthPollCount: 3, + healthPollIntervalSeconds: 0, + isGatewayHttpReady: vi.fn(async () => Promise.reject(probeError)), + }); + + await expect(waitForGatewayHealth(options)).rejects.toBe(probeError); + + expect(options.isGatewayHealthy).toHaveBeenCalledOnce(); + expect(options.isGatewayHttpReady).toHaveBeenCalledOnce(); + expect(options.sleepSeconds).not.toHaveBeenCalled(); + }); + + it("clamps an overflowing health deadline budget to a finite value", () => { + expect(getGatewayHealthWaitBudgetMs(Number.MAX_VALUE, Number.MAX_VALUE)).toBe( + Number.MAX_SAFE_INTEGER, + ); }); it("returns false without probing when healthPollCount is zero", async () => { @@ -169,6 +253,7 @@ describe("waitForGatewayHealth", () => { }); it("aborts the HTTP readiness probe when OpenShell metadata is unhealthy", async () => { + const clock = createVirtualClock(); let observedSignal: AbortSignal | undefined; let aborted = false; const options = buildOptions({ @@ -188,6 +273,8 @@ describe("waitForGatewayHealth", () => { ); }), ), + now: clock.now, + sleepSeconds: clock.sleeper, }); await expect(waitForGatewayHealth(options)).resolves.toBe(false); diff --git a/src/lib/onboard/gateway-health-wait.ts b/src/lib/onboard/gateway-health-wait.ts index 0ba51ebf536..2193baff3a6 100644 --- a/src/lib/onboard/gateway-health-wait.ts +++ b/src/lib/onboard/gateway-health-wait.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { waitUntilAsync } from "../core/wait"; +import { type WaitUntilOptions, waitUntilAsync } from "../core/wait"; type RunCaptureOpenshell = (args: string[], opts?: { ignoreError?: boolean }) => string; @@ -16,6 +16,84 @@ export interface GatewayHealthWaitOptions { repairGatewayBootstrapSecrets: () => { repaired: boolean }; runCaptureOpenshell: RunCaptureOpenshell; sleepSeconds: (seconds: number) => void; + now?: () => number; +} + +export function getGatewayHealthWaitBudgetMs( + healthPollCount: number, + healthPollIntervalSeconds: number, +): number { + const normalizedCount = Number.isFinite(healthPollCount) ? Math.max(0, healthPollCount) : 0; + const normalizedIntervalSeconds = Number.isFinite(healthPollIntervalSeconds) + ? Math.max(0, healthPollIntervalSeconds) + : 0; + if (normalizedCount <= 0 || normalizedIntervalSeconds <= 0) return 0; + const budgetMs = normalizedCount * normalizedIntervalSeconds * 1000; + return Number.isFinite(budgetMs) + ? Math.max(1, Math.min(Number.MAX_SAFE_INTEGER, budgetMs)) + : Number.MAX_SAFE_INTEGER; +} + +export function formatGatewayHealthWaitBudget( + healthPollCount: number, + healthPollIntervalSeconds: number, +): string { + const budgetMs = getGatewayHealthWaitBudgetMs(healthPollCount, healthPollIntervalSeconds); + if (budgetMs <= 0) return "0s"; + if (budgetMs < 1000) return `${Math.ceil(budgetMs)}ms`; + const seconds = budgetMs / 1000; + return Number.isInteger(seconds) ? `${seconds}s` : `${seconds.toFixed(1)}s`; +} + +export function formatGatewayHealthWaitLimit( + healthPollCount: number, + healthPollIntervalSeconds: number, +): string { + const normalizedIntervalSeconds = Number.isFinite(healthPollIntervalSeconds) + ? Math.max(0, healthPollIntervalSeconds) + : 0; + const immediateAttempts = + normalizedIntervalSeconds === 0 && Number.isFinite(healthPollCount) + ? Math.max(0, Math.floor(healthPollCount)) + : 0; + if (immediateAttempts > 0) { + return `${String(immediateAttempts)} immediate health ${immediateAttempts === 1 ? "probe" : "probes"}`; + } + return `${formatGatewayHealthWaitBudget(healthPollCount, healthPollIntervalSeconds)} health deadline`; +} + +export function createGatewayHealthWaitOptions( + healthPollCount: number, + healthPollIntervalSeconds: number, + now: () => number, + sleep: (ms: number) => void, +): WaitUntilOptions | null { + const normalizedCount = Number.isFinite(healthPollCount) ? Math.max(0, healthPollCount) : 0; + if (normalizedCount <= 0) return null; + + const normalizedIntervalSeconds = Number.isFinite(healthPollIntervalSeconds) + ? Math.max(0, healthPollIntervalSeconds) + : 0; + const intervalMs = normalizedIntervalSeconds * 1000; + const commonOptions = { + initialIntervalMs: intervalMs, + maxIntervalMs: intervalMs, + backoffFactor: 1, + now, + sleep, + } satisfies WaitUntilOptions; + + // A zero interval is an accepted fast-test and operator configuration. It + // has no meaningful time budget, so preserve the former bounded attempt + // semantics instead of turning scheduling overhead into a zero-probe wait. + if (intervalMs === 0) { + return { ...commonOptions, maxAttempts: normalizedCount }; + } + + return { + ...commonOptions, + deadlineMs: now() + getGatewayHealthWaitBudgetMs(normalizedCount, normalizedIntervalSeconds), + }; } function startAbortableGatewayHttpProbe( @@ -49,38 +127,35 @@ export async function waitForGatewayHealth({ repairGatewayBootstrapSecrets, runCaptureOpenshell, sleepSeconds, + now = Date.now, }: GatewayHealthWaitOptions): Promise { - const healthPollIntervalMs = Math.max(0, healthPollIntervalSeconds * 1000); + const waitOptions = createGatewayHealthWaitOptions( + healthPollCount, + healthPollIntervalSeconds, + now, + (ms) => sleepSeconds(ms / 1000), + ); return ( - healthPollCount > 0 && - (await waitUntilAsync( - async () => { - const repairResult = repairGatewayBootstrapSecrets(); - if (repairResult.repaired) { - attachGatewayMetadataIfNeeded({ forceRefresh: true }); - } else if (gatewayClusterHealthcheckPassed()) { - attachGatewayMetadataIfNeeded(); - } - const httpProbe = startAbortableGatewayHttpProbe(isGatewayHttpReady); - runCaptureOpenshell(["gateway", "select", gatewayName], { ignoreError: true }); - const status = runCaptureOpenshell(["status"], { ignoreError: true }); - const namedInfo = runCaptureOpenshell(["gateway", "info", "-g", gatewayName], { - ignoreError: true, - }); - const currentInfo = runCaptureOpenshell(["gateway", "info"], { ignoreError: true }); - if (!isGatewayHealthy(status, namedInfo, currentInfo)) { - httpProbe.abort(); - return false; - } - return await httpProbe.ready; - }, - { - initialIntervalMs: healthPollIntervalMs, - maxIntervalMs: healthPollIntervalMs, - backoffFactor: 1, - maxAttempts: healthPollCount, - sleep: (ms) => sleepSeconds(ms / 1000), - }, - )) + waitOptions !== null && + (await waitUntilAsync(async () => { + const repairResult = repairGatewayBootstrapSecrets(); + if (repairResult.repaired) { + attachGatewayMetadataIfNeeded({ forceRefresh: true }); + } else if (gatewayClusterHealthcheckPassed()) { + attachGatewayMetadataIfNeeded(); + } + const httpProbe = startAbortableGatewayHttpProbe(isGatewayHttpReady); + runCaptureOpenshell(["gateway", "select", gatewayName], { ignoreError: true }); + const status = runCaptureOpenshell(["status"], { ignoreError: true }); + const namedInfo = runCaptureOpenshell(["gateway", "info", "-g", gatewayName], { + ignoreError: true, + }); + const currentInfo = runCaptureOpenshell(["gateway", "info"], { ignoreError: true }); + if (!isGatewayHealthy(status, namedInfo, currentInfo)) { + httpProbe.abort(); + return false; + } + return await httpProbe.ready; + }, waitOptions)) ); } diff --git a/src/lib/onboard/gateway-start-failure.test.ts b/src/lib/onboard/gateway-start-failure.test.ts index 9f199ad2040..da5a991efb7 100644 --- a/src/lib/onboard/gateway-start-failure.test.ts +++ b/src/lib/onboard/gateway-start-failure.test.ts @@ -6,9 +6,28 @@ import { describe, expect, it, vi } from "vitest"; import { classifyGatewayStartFailure } from "../validation"; import { createFinalGatewayStartFailureHandler, + normalizeGatewayStartError, reportLegacyGatewayStartResultFailure, } from "./gateway-start-failure"; +describe("normalizeGatewayStartError", () => { + it("preserves the original deadline-aware Error instance (#3768)", () => { + const deadlineError = new Error("Gateway failed within 10s."); + + expect(normalizeGatewayStartError(deadlineError)).toBe(deadlineError); + }); + + it.each([ + ["string", "connection refused", "connection refused"], + ["number", 500, "500"], + ])("wraps a thrown %s without losing its message (#3768)", (_kind, thrown, message) => { + const normalized = normalizeGatewayStartError(thrown); + + expect(normalized).toBeInstanceOf(Error); + expect(normalized.message).toBe(message); + }); +}); + describe("classifyGatewayStartFailure", () => { // Regression: NemoClaw #2347. When Colima is stopped on macOS, the // openshell gateway-start stream prints "Failed to create Docker client. diff --git a/src/lib/onboard/gateway-start-failure.ts b/src/lib/onboard/gateway-start-failure.ts index e7d15ae9974..a2aa51c4741 100644 --- a/src/lib/onboard/gateway-start-failure.ts +++ b/src/lib/onboard/gateway-start-failure.ts @@ -22,6 +22,10 @@ export type FinalGatewayStartFailureDeps = { cleanupGateway(): void; }; +export function normalizeGatewayStartError(error: unknown): Error { + return error instanceof Error ? error : new Error(String(error)); +} + export function reportLegacyGatewayStartResultFailure( output: string, log: (message: string) => void,