From 258433cbef22bfae44aacae75fb7949787eb5218 Mon Sep 17 00:00:00 2001 From: Ho Lim Date: Wed, 8 Jul 2026 12:29:36 -0700 Subject: [PATCH 1/2] perf(onboard): use deadlines for gateway health waits Signed-off-by: Ho Lim --- src/lib/onboard.ts | 15 ++++------ .../onboard/__test-helpers__/virtual-clock.ts | 16 +++++++++++ .../docker-driver-gateway-service.test.ts | 20 +++++++++---- .../onboard/docker-driver-gateway-service.ts | 14 ++++++++-- src/lib/onboard/gateway-health-wait.test.ts | 28 +++++++++++++------ src/lib/onboard/gateway-health-wait.ts | 28 ++++++++++++++++++- 6 files changed, 95 insertions(+), 26 deletions(-) create mode 100644 src/lib/onboard/__test-helpers__/virtual-clock.ts diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 9c348d06862..c2e052fd1ed 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -325,9 +325,8 @@ const { rejectUnsupportedWindowsHostOllama, shouldFrontOllamaWithProxy, }: typeof import("./onboard/local-inference-topology") = require("./onboard/local-inference-topology"); -const { - waitForGatewayHealth, -}: typeof import("./onboard/gateway-health-wait") = require("./onboard/gateway-health-wait"); +const { waitForGatewayHealth }: typeof import("./onboard/gateway-health-wait") = + require("./onboard/gateway-health-wait"); const { resolveOpenshell } = require("./adapters/openshell/resolve"); const credentials: typeof import("./credentials/store") = require("./credentials/store"); const { @@ -1896,7 +1895,7 @@ async function startGatewayWithOptions( return; } - throw new Error("Gateway failed to start"); + throw new Error(`Gateway failed within ${healthWait.count * healthWait.interval}s.`); }, { retries, @@ -1912,11 +1911,9 @@ async function startGatewayWithOptions( }, }, ); - } catch { - if (exitOnFailure) { - handleFinalGatewayStartFailure({ retries, dockerUnreachable }); - } - throw new Error("Gateway failed to start"); + } catch (error) { + if (exitOnFailure) handleFinalGatewayStartFailure({ retries, dockerUnreachable }); + throw error instanceof Error ? error : new Error(String(error)); } console.log(" ✓ Gateway is healthy"); diff --git a/src/lib/onboard/__test-helpers__/virtual-clock.ts b/src/lib/onboard/__test-helpers__/virtual-clock.ts new file mode 100644 index 00000000000..fcb88c270a1 --- /dev/null +++ b/src/lib/onboard/__test-helpers__/virtual-clock.ts @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { vi } from "vitest"; + +export function createVirtualClock(startMs = 1_000_000_000_000) { + let currentMs = startMs; + const advance = (seconds: number) => { + currentMs += Math.max(0, seconds) * 1000; + }; + return { + advance, + now: () => currentMs, + sleeper: vi.fn(advance), + }; +} diff --git a/src/lib/onboard/docker-driver-gateway-service.test.ts b/src/lib/onboard/docker-driver-gateway-service.test.ts index 14dda05e370..c5f3ab0fe63 100644 --- a/src/lib/onboard/docker-driver-gateway-service.test.ts +++ b/src/lib/onboard/docker-driver-gateway-service.test.ts @@ -3,13 +3,14 @@ import { describe, expect, it, vi } from "vitest"; +import { createVirtualClock } from "./__test-helpers__/virtual-clock"; import { getOpenShellGatewayUserServiceBinaryPaths, getOpenShellGatewayUserServicePaths, hasOpenShellGatewayUserService, - startPackageManagedDockerDriverGateway, - startOpenShellGatewayUserService, type SpawnSyncLikeResult, + startOpenShellGatewayUserService, + startPackageManagedDockerDriverGateway, } from "./docker-driver-gateway-service"; const STATUS_CONNECTED = ` @@ -254,6 +255,7 @@ describe("docker-driver-gateway-service", () => { it("uses the package-managed service only after endpoint, metadata, and gRPC health are ready", async () => { const events: string[] = []; + const clock = createVirtualClock(); let registerCount = 0; const registerDockerDriverGatewayEndpoint = vi.fn(() => { events.push("register"); @@ -268,14 +270,18 @@ describe("docker-driver-gateway-service", () => { gatewayName: "nemoclaw", hasOpenShellGatewayUserService: () => true, healthPollCount: 3, - healthPollInterval: 0, + healthPollInterval: 1, isDockerDriverGatewayReady: async () => { events.push("ready"); return true; }, + now: clock.now, registerDockerDriverGatewayEndpoint, runCaptureOpenshell: (args) => (args[0] === "status" ? STATUS_CONNECTED : GATEWAY_INFO), - sleepSeconds: () => events.push("sleep"), + sleepSeconds: (seconds) => { + events.push("sleep"); + clock.advance(seconds); + }, skipSandboxBridgeReachability: false, startOpenShellGatewayUserService: () => ({ attempted: true, @@ -318,6 +324,7 @@ describe("docker-driver-gateway-service", () => { it("keeps standalone runtime breadcrumbs when service health never becomes ready", async () => { const clearDockerDriverGatewayRuntimeFiles = vi.fn(); + const clock = createVirtualClock(); await expect( startPackageManagedDockerDriverGateway({ @@ -326,9 +333,12 @@ describe("docker-driver-gateway-service", () => { gatewayName: "nemoclaw", hasOpenShellGatewayUserService: () => true, healthPollCount: 1, + healthPollInterval: 1, isDockerDriverGatewayReady: async () => false, + now: clock.now, registerDockerDriverGatewayEndpoint: () => true, runCaptureOpenshell: (args) => (args[0] === "status" ? STATUS_CONNECTED : GATEWAY_INFO), + sleepSeconds: clock.advance, skipSandboxBridgeReachability: false, startOpenShellGatewayUserService: () => ({ attempted: true, @@ -337,7 +347,7 @@ describe("docker-driver-gateway-service", () => { }), verifySandboxBridgeGatewayReachableOrExit: vi.fn(), }), - ).rejects.toThrow("did not become healthy"); + ).rejects.toThrow("configured 1s health deadline"); expect(clearDockerDriverGatewayRuntimeFiles).not.toHaveBeenCalled(); }); diff --git a/src/lib/onboard/docker-driver-gateway-service.ts b/src/lib/onboard/docker-driver-gateway-service.ts index 75e5ed20ee1..7eda450e839 100644 --- a/src/lib/onboard/docker-driver-gateway-service.ts +++ b/src/lib/onboard/docker-driver-gateway-service.ts @@ -1,13 +1,14 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { spawnSync, type SpawnSyncOptions } from "node:child_process"; +import { type SpawnSyncOptions, spawnSync } from "node:child_process"; import fs from "node:fs"; import path from "node:path"; import { sleepSeconds, waitUntilAsync } from "../core/wait"; import { isGatewayHealthy } from "../state/gateway"; import { envInt } from "./env"; +import { formatGatewayHealthWaitBudget, getGatewayHealthWaitBudgetMs } from "./gateway-health-wait"; import { isDockerDriverGatewayHttpReady } from "./gateway-http-readiness"; export const OPENSHELL_GATEWAY_USER_SERVICE = "openshell-gateway"; @@ -49,6 +50,7 @@ export interface PackageManagedDockerDriverGatewayOptions { healthPollCount?: number; healthPollInterval?: number; isDockerDriverGatewayReady?: () => Promise; + now?: () => number; registerDockerDriverGatewayEndpoint: () => boolean; runCaptureOpenshell: (args: string[], opts?: { ignoreError?: boolean }) => string; sleepSeconds?: (seconds: number) => void; @@ -291,6 +293,7 @@ export async function startPackageManagedDockerDriverGateway({ healthPollCount, healthPollInterval, isDockerDriverGatewayReady = isDockerDriverGatewayHttpReady, + now = Date.now, registerDockerDriverGatewayEndpoint, runCaptureOpenshell, sleepSeconds: sleepSecondsImpl = sleepSeconds, @@ -324,6 +327,7 @@ export async function startPackageManagedDockerDriverGateway({ const pollCount = healthPollCount ?? envInt("NEMOCLAW_HEALTH_POLL_COUNT", 30); const pollInterval = healthPollInterval ?? envInt("NEMOCLAW_HEALTH_POLL_INTERVAL", 2); const pollIntervalMs = Math.max(0, pollInterval * 1000); + const waitBudgetMs = getGatewayHealthWaitBudgetMs(pollCount, pollInterval); const healthy = pollCount > 0 && (await waitUntilAsync( @@ -339,10 +343,11 @@ export async function startPackageManagedDockerDriverGateway({ ); }, { + deadlineMs: now() + waitBudgetMs, initialIntervalMs: pollIntervalMs, maxIntervalMs: pollIntervalMs, backoffFactor: 1, - maxAttempts: pollCount, + now, sleep: (ms) => sleepSecondsImpl(ms / 1000), }, )); @@ -355,7 +360,10 @@ export async function startPackageManagedDockerDriverGateway({ return true; } - const message = "OpenShell gateway user service started but did not become healthy."; + const message = `OpenShell gateway user service started but did not become healthy within the configured ${formatGatewayHealthWaitBudget( + pollCount, + pollInterval, + )} health deadline.`; console.error(` ${message}`); console.error(" Check: systemctl --user status openshell-gateway"); if (exitOnFailure) process.exit(1); diff --git a/src/lib/onboard/gateway-health-wait.test.ts b/src/lib/onboard/gateway-health-wait.test.ts index 96b5f37b1c4..82523db4f9f 100644 --- a/src/lib/onboard/gateway-health-wait.test.ts +++ b/src/lib/onboard/gateway-health-wait.test.ts @@ -3,6 +3,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; +import { createVirtualClock } from "./__test-helpers__/virtual-clock"; import { type GatewayHealthWaitOptions, waitForGatewayHealth } from "./gateway-health-wait"; function buildOptions(overrides: Partial = {}): GatewayHealthWaitOptions { @@ -49,16 +50,19 @@ describe("waitForGatewayHealth", () => { }); it("returns false when HTTP readiness never follows healthy metadata", async () => { + const clock = createVirtualClock(); const options = buildOptions({ healthPollCount: 2, isGatewayHttpReady: vi.fn(async () => false), + now: clock.now, + sleepSeconds: clock.sleeper, }); await expect(waitForGatewayHealth(options)).resolves.toBe(false); expect(options.isGatewayHealthy).toHaveBeenCalledTimes(2); expect(options.isGatewayHttpReady).toHaveBeenCalledTimes(2); - expect(options.sleepSeconds).toHaveBeenCalledTimes(1); + expect(options.sleepSeconds).toHaveBeenCalledTimes(2); }); it("force-refreshes metadata after bootstrap secret repair", async () => { @@ -85,19 +89,27 @@ describe("waitForGatewayHealth", () => { expect(options.attachGatewayMetadataIfNeeded).toHaveBeenCalledWith(); }); - it("stops after healthPollCount attempts without sleeping after the final failed probe", async () => { + it("polls until the configured health deadline instead of stopping at the count cap (#3768)", async () => { + const clock = createVirtualClock(); + const isGatewayHealthy = vi.fn(() => { + clock.advance(1); + return false; + }); const options = buildOptions({ - healthPollCount: 3, - isGatewayHealthy: vi.fn(() => false), + healthPollCount: 10, + healthPollIntervalSeconds: 1, + isGatewayHealthy, + now: clock.now, + sleepSeconds: clock.sleeper, }); await expect(waitForGatewayHealth(options)).resolves.toBe(false); - expect(options.isGatewayHealthy).toHaveBeenCalledTimes(3); + expect(isGatewayHealthy).toHaveBeenCalled(); + expect(isGatewayHealthy.mock.calls.length).toBeLessThan(10); expect(options.isGatewayHttpReady).not.toHaveBeenCalled(); - expect(options.sleepSeconds).toHaveBeenCalledTimes(2); - expect(options.sleepSeconds).toHaveBeenNthCalledWith(1, 2); - expect(options.sleepSeconds).toHaveBeenNthCalledWith(2, 2); + expect(clock.sleeper).toHaveBeenCalled(); + expect(clock.sleeper.mock.calls.every(([seconds]) => seconds === 1)).toBe(true); }); it("returns false without probing when healthPollCount is zero", async () => { diff --git a/src/lib/onboard/gateway-health-wait.ts b/src/lib/onboard/gateway-health-wait.ts index bd019cc43f8..a3a0bac1c72 100644 --- a/src/lib/onboard/gateway-health-wait.ts +++ b/src/lib/onboard/gateway-health-wait.ts @@ -16,6 +16,29 @@ export interface GatewayHealthWaitOptions { repairGatewayBootstrapSecrets: () => { repaired: boolean }; runCaptureOpenshell: RunCaptureOpenshell; sleepSeconds: (seconds: number) => void; + now?: () => number; +} + +export function getGatewayHealthWaitBudgetMs( + healthPollCount: number, + healthPollIntervalSeconds: number, +): number { + const normalizedCount = Number.isFinite(healthPollCount) ? Math.max(0, healthPollCount) : 0; + const normalizedIntervalSeconds = Number.isFinite(healthPollIntervalSeconds) + ? Math.max(0, healthPollIntervalSeconds) + : 0; + return normalizedCount <= 0 ? 0 : Math.max(1, normalizedCount * normalizedIntervalSeconds * 1000); +} + +export function formatGatewayHealthWaitBudget( + healthPollCount: number, + healthPollIntervalSeconds: number, +): string { + const budgetMs = getGatewayHealthWaitBudgetMs(healthPollCount, healthPollIntervalSeconds); + if (budgetMs <= 0) return "0s"; + if (budgetMs < 1000) return `${Math.ceil(budgetMs)}ms`; + const seconds = budgetMs / 1000; + return Number.isInteger(seconds) ? `${seconds}s` : `${seconds.toFixed(1)}s`; } export async function waitForGatewayHealth({ @@ -29,8 +52,10 @@ export async function waitForGatewayHealth({ repairGatewayBootstrapSecrets, runCaptureOpenshell, sleepSeconds, + now = Date.now, }: GatewayHealthWaitOptions): Promise { const healthPollIntervalMs = Math.max(0, healthPollIntervalSeconds * 1000); + const waitBudgetMs = getGatewayHealthWaitBudgetMs(healthPollCount, healthPollIntervalSeconds); return ( healthPollCount > 0 && (await waitUntilAsync( @@ -50,10 +75,11 @@ export async function waitForGatewayHealth({ return isGatewayHealthy(status, namedInfo, currentInfo) && (await isGatewayHttpReady()); }, { + deadlineMs: now() + waitBudgetMs, initialIntervalMs: healthPollIntervalMs, maxIntervalMs: healthPollIntervalMs, backoffFactor: 1, - maxAttempts: healthPollCount, + now, sleep: (ms) => sleepSeconds(ms / 1000), }, )) From 7251c2c053be6fa47368a75320b8bd3a95bc0d27 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Wed, 8 Jul 2026 17:51:10 -0700 Subject: [PATCH 2/2] fix(onboard): preserve zero-interval health probes Use bounded immediate attempts when the interval is zero. Positive intervals remain deadline-driven. Co-authored-by: Ho Lim Signed-off-by: Apurv Kumaria --- .../docker-driver-gateway-service.test.ts | 36 ++++++ .../onboard/docker-driver-gateway-service.ts | 48 ++++---- src/lib/onboard/gateway-health-wait.test.ts | 68 ++++++++++- src/lib/onboard/gateway-health-wait.ts | 109 +++++++++++++----- 4 files changed, 203 insertions(+), 58 deletions(-) diff --git a/src/lib/onboard/docker-driver-gateway-service.test.ts b/src/lib/onboard/docker-driver-gateway-service.test.ts index c5f3ab0fe63..eedf753137c 100644 --- a/src/lib/onboard/docker-driver-gateway-service.test.ts +++ b/src/lib/onboard/docker-driver-gateway-service.test.ts @@ -297,6 +297,42 @@ describe("docker-driver-gateway-service", () => { expect(events).toEqual(["register", "sleep", "register", "ready", "clear", "verify"]); }); + it("preserves bounded immediate package-service probes when the interval is zero", async () => { + let registerCount = 0; + const sleepSeconds = vi.fn(); + + await expect( + startPackageManagedDockerDriverGateway({ + clearDockerDriverGatewayRuntimeFiles: vi.fn(), + exitOnFailure: false, + gatewayName: "nemoclaw", + hasOpenShellGatewayUserService: () => true, + healthPollCount: 3, + healthPollInterval: 0, + isDockerDriverGatewayReady: async () => true, + now: () => Number.MAX_SAFE_INTEGER, + registerDockerDriverGatewayEndpoint: () => { + registerCount += 1; + return registerCount >= 3; + }, + runCaptureOpenshell: (args) => (args[0] === "status" ? STATUS_CONNECTED : GATEWAY_INFO), + sleepSeconds, + skipSandboxBridgeReachability: false, + startOpenShellGatewayUserService: () => ({ + attempted: true, + fallbackAllowed: false, + started: true, + }), + verifySandboxBridgeGatewayReachableOrExit: vi.fn(), + }), + ).resolves.toBe(true); + + expect(registerCount).toBe(3); + expect(sleepSeconds).toHaveBeenCalledTimes(2); + expect(sleepSeconds).toHaveBeenNthCalledWith(1, 0); + expect(sleepSeconds).toHaveBeenNthCalledWith(2, 0); + }); + it("falls back to standalone when package-managed service startup is unavailable", async () => { const registerDockerDriverGatewayEndpoint = vi.fn(() => true); diff --git a/src/lib/onboard/docker-driver-gateway-service.ts b/src/lib/onboard/docker-driver-gateway-service.ts index 7eda450e839..dcdc0db52a0 100644 --- a/src/lib/onboard/docker-driver-gateway-service.ts +++ b/src/lib/onboard/docker-driver-gateway-service.ts @@ -8,7 +8,10 @@ import path from "node:path"; import { sleepSeconds, waitUntilAsync } from "../core/wait"; import { isGatewayHealthy } from "../state/gateway"; import { envInt } from "./env"; -import { formatGatewayHealthWaitBudget, getGatewayHealthWaitBudgetMs } from "./gateway-health-wait"; +import { + createGatewayHealthWaitOptions, + formatGatewayHealthWaitLimit, +} from "./gateway-health-wait"; import { isDockerDriverGatewayHttpReady } from "./gateway-http-readiness"; export const OPENSHELL_GATEWAY_USER_SERVICE = "openshell-gateway"; @@ -326,31 +329,22 @@ export async function startPackageManagedDockerDriverGateway({ const pollCount = healthPollCount ?? envInt("NEMOCLAW_HEALTH_POLL_COUNT", 30); const pollInterval = healthPollInterval ?? envInt("NEMOCLAW_HEALTH_POLL_INTERVAL", 2); - const pollIntervalMs = Math.max(0, pollInterval * 1000); - const waitBudgetMs = getGatewayHealthWaitBudgetMs(pollCount, pollInterval); + const waitOptions = createGatewayHealthWaitOptions(pollCount, pollInterval, now, (ms) => + sleepSecondsImpl(ms / 1000), + ); const healthy = - pollCount > 0 && - (await waitUntilAsync( - async () => { - if (!registerDockerDriverGatewayEndpoint()) return false; - const status = runCaptureOpenshell(["status"], { ignoreError: true }); - const namedInfo = runCaptureOpenshell(["gateway", "info", "-g", gatewayName], { - ignoreError: true, - }); - const currentInfo = runCaptureOpenshell(["gateway", "info"], { ignoreError: true }); - return ( - isGatewayHealthy(status, namedInfo, currentInfo) && (await isDockerDriverGatewayReady()) - ); - }, - { - deadlineMs: now() + waitBudgetMs, - initialIntervalMs: pollIntervalMs, - maxIntervalMs: pollIntervalMs, - backoffFactor: 1, - now, - sleep: (ms) => sleepSecondsImpl(ms / 1000), - }, - )); + waitOptions !== null && + (await waitUntilAsync(async () => { + if (!registerDockerDriverGatewayEndpoint()) return false; + const status = runCaptureOpenshell(["status"], { ignoreError: true }); + const namedInfo = runCaptureOpenshell(["gateway", "info", "-g", gatewayName], { + ignoreError: true, + }); + const currentInfo = runCaptureOpenshell(["gateway", "info"], { ignoreError: true }); + return ( + isGatewayHealthy(status, namedInfo, currentInfo) && (await isDockerDriverGatewayReady()) + ); + }, waitOptions)); if (healthy) { clearDockerDriverGatewayRuntimeFiles(); await verifySandboxBridgeGatewayReachableOrExit(exitOnFailure, { @@ -360,10 +354,10 @@ export async function startPackageManagedDockerDriverGateway({ return true; } - const message = `OpenShell gateway user service started but did not become healthy within the configured ${formatGatewayHealthWaitBudget( + const message = `OpenShell gateway user service started but did not become healthy within the configured ${formatGatewayHealthWaitLimit( pollCount, pollInterval, - )} health deadline.`; + )}.`; console.error(` ${message}`); console.error(" Check: systemctl --user status openshell-gateway"); if (exitOnFailure) process.exit(1); diff --git a/src/lib/onboard/gateway-health-wait.test.ts b/src/lib/onboard/gateway-health-wait.test.ts index 82523db4f9f..6b7784193df 100644 --- a/src/lib/onboard/gateway-health-wait.test.ts +++ b/src/lib/onboard/gateway-health-wait.test.ts @@ -4,7 +4,12 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import { createVirtualClock } from "./__test-helpers__/virtual-clock"; -import { type GatewayHealthWaitOptions, waitForGatewayHealth } from "./gateway-health-wait"; +import { + formatGatewayHealthWaitLimit, + type GatewayHealthWaitOptions, + getGatewayHealthWaitBudgetMs, + waitForGatewayHealth, +} from "./gateway-health-wait"; function buildOptions(overrides: Partial = {}): GatewayHealthWaitOptions { return { @@ -112,6 +117,67 @@ describe("waitForGatewayHealth", () => { expect(clock.sleeper.mock.calls.every(([seconds]) => seconds === 1)).toBe(true); }); + it("preserves the configured immediate probes when the interval is zero (#3768)", async () => { + const isGatewayHealthy = vi + .fn<() => boolean>() + .mockReturnValueOnce(false) + .mockReturnValueOnce(false) + .mockReturnValueOnce(true); + const sleepSeconds = vi.fn(); + const options = buildOptions({ + healthPollCount: 3, + healthPollIntervalSeconds: 0, + isGatewayHealthy, + now: vi.fn(() => Number.MAX_SAFE_INTEGER), + sleepSeconds, + }); + + await expect(waitForGatewayHealth(options)).resolves.toBe(true); + + expect(isGatewayHealthy).toHaveBeenCalledTimes(3); + expect(options.isGatewayHttpReady).toHaveBeenCalledOnce(); + expect(sleepSeconds).toHaveBeenCalledTimes(2); + expect(sleepSeconds).toHaveBeenNthCalledWith(1, 0); + expect(sleepSeconds).toHaveBeenNthCalledWith(2, 0); + expect(formatGatewayHealthWaitLimit(3, 0)).toBe("3 immediate health probes"); + }); + + it("does not probe after a positive health deadline expires before the first attempt", async () => { + const now = vi.fn().mockReturnValueOnce(0).mockReturnValue(1000); + const options = buildOptions({ + healthPollCount: 1, + healthPollIntervalSeconds: 1, + now, + }); + + await expect(waitForGatewayHealth(options)).resolves.toBe(false); + + expect(options.runCaptureOpenshell).not.toHaveBeenCalled(); + expect(options.isGatewayHealthy).not.toHaveBeenCalled(); + expect(options.isGatewayHttpReady).not.toHaveBeenCalled(); + }); + + it("preserves a rejected HTTP readiness probe error", async () => { + const probeError = new Error("readiness transport failed"); + const options = buildOptions({ + healthPollCount: 3, + healthPollIntervalSeconds: 0, + isGatewayHttpReady: vi.fn(async () => Promise.reject(probeError)), + }); + + await expect(waitForGatewayHealth(options)).rejects.toBe(probeError); + + expect(options.isGatewayHealthy).toHaveBeenCalledOnce(); + expect(options.isGatewayHttpReady).toHaveBeenCalledOnce(); + expect(options.sleepSeconds).not.toHaveBeenCalled(); + }); + + it("clamps an overflowing health deadline budget to a finite value", () => { + expect(getGatewayHealthWaitBudgetMs(Number.MAX_VALUE, Number.MAX_VALUE)).toBe( + Number.MAX_SAFE_INTEGER, + ); + }); + it("returns false without probing when healthPollCount is zero", async () => { const options = buildOptions({ healthPollCount: 0 }); diff --git a/src/lib/onboard/gateway-health-wait.ts b/src/lib/onboard/gateway-health-wait.ts index a3a0bac1c72..da1b60a9055 100644 --- a/src/lib/onboard/gateway-health-wait.ts +++ b/src/lib/onboard/gateway-health-wait.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { waitUntilAsync } from "../core/wait"; +import { type WaitUntilOptions, waitUntilAsync } from "../core/wait"; type RunCaptureOpenshell = (args: string[], opts?: { ignoreError?: boolean }) => string; @@ -27,7 +27,11 @@ export function getGatewayHealthWaitBudgetMs( const normalizedIntervalSeconds = Number.isFinite(healthPollIntervalSeconds) ? Math.max(0, healthPollIntervalSeconds) : 0; - return normalizedCount <= 0 ? 0 : Math.max(1, normalizedCount * normalizedIntervalSeconds * 1000); + if (normalizedCount <= 0 || normalizedIntervalSeconds <= 0) return 0; + const budgetMs = normalizedCount * normalizedIntervalSeconds * 1000; + return Number.isFinite(budgetMs) + ? Math.max(1, Math.min(Number.MAX_SAFE_INTEGER, budgetMs)) + : Number.MAX_SAFE_INTEGER; } export function formatGatewayHealthWaitBudget( @@ -41,6 +45,57 @@ export function formatGatewayHealthWaitBudget( return Number.isInteger(seconds) ? `${seconds}s` : `${seconds.toFixed(1)}s`; } +export function formatGatewayHealthWaitLimit( + healthPollCount: number, + healthPollIntervalSeconds: number, +): string { + const normalizedIntervalSeconds = Number.isFinite(healthPollIntervalSeconds) + ? Math.max(0, healthPollIntervalSeconds) + : 0; + const immediateAttempts = + normalizedIntervalSeconds === 0 && Number.isFinite(healthPollCount) + ? Math.max(0, Math.floor(healthPollCount)) + : 0; + if (immediateAttempts > 0) { + return `${String(immediateAttempts)} immediate health ${immediateAttempts === 1 ? "probe" : "probes"}`; + } + return `${formatGatewayHealthWaitBudget(healthPollCount, healthPollIntervalSeconds)} health deadline`; +} + +export function createGatewayHealthWaitOptions( + healthPollCount: number, + healthPollIntervalSeconds: number, + now: () => number, + sleep: (ms: number) => void, +): WaitUntilOptions | null { + const normalizedCount = Number.isFinite(healthPollCount) ? Math.max(0, healthPollCount) : 0; + if (normalizedCount <= 0) return null; + + const normalizedIntervalSeconds = Number.isFinite(healthPollIntervalSeconds) + ? Math.max(0, healthPollIntervalSeconds) + : 0; + const intervalMs = normalizedIntervalSeconds * 1000; + const commonOptions = { + initialIntervalMs: intervalMs, + maxIntervalMs: intervalMs, + backoffFactor: 1, + now, + sleep, + } satisfies WaitUntilOptions; + + // A zero interval is an accepted fast-test and operator configuration. It + // has no meaningful time budget, so preserve the former bounded attempt + // semantics instead of turning scheduling overhead into a zero-probe wait. + if (intervalMs === 0) { + return { ...commonOptions, maxAttempts: normalizedCount }; + } + + return { + ...commonOptions, + deadlineMs: now() + getGatewayHealthWaitBudgetMs(normalizedCount, normalizedIntervalSeconds), + }; +} + export async function waitForGatewayHealth({ attachGatewayMetadataIfNeeded, gatewayClusterHealthcheckPassed, @@ -54,34 +109,28 @@ export async function waitForGatewayHealth({ sleepSeconds, now = Date.now, }: GatewayHealthWaitOptions): Promise { - const healthPollIntervalMs = Math.max(0, healthPollIntervalSeconds * 1000); - const waitBudgetMs = getGatewayHealthWaitBudgetMs(healthPollCount, healthPollIntervalSeconds); + const waitOptions = createGatewayHealthWaitOptions( + healthPollCount, + healthPollIntervalSeconds, + now, + (ms) => sleepSeconds(ms / 1000), + ); return ( - healthPollCount > 0 && - (await waitUntilAsync( - async () => { - const repairResult = repairGatewayBootstrapSecrets(); - if (repairResult.repaired) { - attachGatewayMetadataIfNeeded({ forceRefresh: true }); - } else if (gatewayClusterHealthcheckPassed()) { - attachGatewayMetadataIfNeeded(); - } - runCaptureOpenshell(["gateway", "select", gatewayName], { ignoreError: true }); - const status = runCaptureOpenshell(["status"], { ignoreError: true }); - const namedInfo = runCaptureOpenshell(["gateway", "info", "-g", gatewayName], { - ignoreError: true, - }); - const currentInfo = runCaptureOpenshell(["gateway", "info"], { ignoreError: true }); - return isGatewayHealthy(status, namedInfo, currentInfo) && (await isGatewayHttpReady()); - }, - { - deadlineMs: now() + waitBudgetMs, - initialIntervalMs: healthPollIntervalMs, - maxIntervalMs: healthPollIntervalMs, - backoffFactor: 1, - now, - sleep: (ms) => sleepSeconds(ms / 1000), - }, - )) + waitOptions !== null && + (await waitUntilAsync(async () => { + const repairResult = repairGatewayBootstrapSecrets(); + if (repairResult.repaired) { + attachGatewayMetadataIfNeeded({ forceRefresh: true }); + } else if (gatewayClusterHealthcheckPassed()) { + attachGatewayMetadataIfNeeded(); + } + runCaptureOpenshell(["gateway", "select", gatewayName], { ignoreError: true }); + const status = runCaptureOpenshell(["status"], { ignoreError: true }); + const namedInfo = runCaptureOpenshell(["gateway", "info", "-g", gatewayName], { + ignoreError: true, + }); + const currentInfo = runCaptureOpenshell(["gateway", "info"], { ignoreError: true }); + return isGatewayHealthy(status, namedInfo, currentInfo) && (await isGatewayHttpReady()); + }, waitOptions)) ); }