Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 6 additions & 9 deletions src/lib/onboard.ts
Original file line number Diff line number Diff line change
Expand Up @@ -325,9 +325,8 @@ const {
rejectUnsupportedWindowsHostOllama,
shouldFrontOllamaWithProxy,
}: typeof import("./onboard/local-inference-topology") = require("./onboard/local-inference-topology");
const {
waitForGatewayHealth,
}: typeof import("./onboard/gateway-health-wait") = require("./onboard/gateway-health-wait");
const { waitForGatewayHealth }: typeof import("./onboard/gateway-health-wait") =
require("./onboard/gateway-health-wait");
const { resolveOpenshell } = require("./adapters/openshell/resolve");
const credentials: typeof import("./credentials/store") = require("./credentials/store");
const {
Expand Down Expand Up @@ -1896,7 +1895,7 @@ async function startGatewayWithOptions(
return;
}

throw new Error("Gateway failed to start");
throw new Error(`Gateway failed within ${healthWait.count * healthWait.interval}s.`);
Comment thread
coderabbitai[bot] marked this conversation as resolved.
},
{
retries,
Expand All @@ -1912,11 +1911,9 @@ async function startGatewayWithOptions(
},
},
);
} catch {
if (exitOnFailure) {
handleFinalGatewayStartFailure({ retries, dockerUnreachable });
}
throw new Error("Gateway failed to start");
} catch (error) {
if (exitOnFailure) handleFinalGatewayStartFailure({ retries, dockerUnreachable });
throw error instanceof Error ? error : new Error(String(error));
}

console.log(" ✓ Gateway is healthy");
Expand Down
16 changes: 16 additions & 0 deletions src/lib/onboard/__test-helpers__/virtual-clock.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { vi } from "vitest";

export function createVirtualClock(startMs = 1_000_000_000_000) {
let currentMs = startMs;
const advance = (seconds: number) => {
currentMs += Math.max(0, seconds) * 1000;
};
return {
advance,
now: () => currentMs,
sleeper: vi.fn(advance),
};
}
56 changes: 51 additions & 5 deletions src/lib/onboard/docker-driver-gateway-service.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,14 @@

import { describe, expect, it, vi } from "vitest";

import { createVirtualClock } from "./__test-helpers__/virtual-clock";
import {
getOpenShellGatewayUserServiceBinaryPaths,
getOpenShellGatewayUserServicePaths,
hasOpenShellGatewayUserService,
startPackageManagedDockerDriverGateway,
startOpenShellGatewayUserService,
type SpawnSyncLikeResult,
startOpenShellGatewayUserService,
startPackageManagedDockerDriverGateway,
} from "./docker-driver-gateway-service";

const STATUS_CONNECTED = `
Expand Down Expand Up @@ -254,6 +255,7 @@ describe("docker-driver-gateway-service", () => {

it("uses the package-managed service only after endpoint, metadata, and gRPC health are ready", async () => {
const events: string[] = [];
const clock = createVirtualClock();
let registerCount = 0;
const registerDockerDriverGatewayEndpoint = vi.fn(() => {
events.push("register");
Expand All @@ -268,14 +270,18 @@ describe("docker-driver-gateway-service", () => {
gatewayName: "nemoclaw",
hasOpenShellGatewayUserService: () => true,
healthPollCount: 3,
healthPollInterval: 0,
healthPollInterval: 1,
isDockerDriverGatewayReady: async () => {
events.push("ready");
return true;
},
now: clock.now,
registerDockerDriverGatewayEndpoint,
runCaptureOpenshell: (args) => (args[0] === "status" ? STATUS_CONNECTED : GATEWAY_INFO),
sleepSeconds: () => events.push("sleep"),
sleepSeconds: (seconds) => {
events.push("sleep");
clock.advance(seconds);
},
skipSandboxBridgeReachability: false,
startOpenShellGatewayUserService: () => ({
attempted: true,
Expand All @@ -291,6 +297,42 @@ describe("docker-driver-gateway-service", () => {
expect(events).toEqual(["register", "sleep", "register", "ready", "clear", "verify"]);
});

it("preserves bounded immediate package-service probes when the interval is zero", async () => {
let registerCount = 0;
const sleepSeconds = vi.fn();

await expect(
startPackageManagedDockerDriverGateway({
clearDockerDriverGatewayRuntimeFiles: vi.fn(),
exitOnFailure: false,
gatewayName: "nemoclaw",
hasOpenShellGatewayUserService: () => true,
healthPollCount: 3,
healthPollInterval: 0,
isDockerDriverGatewayReady: async () => true,
now: () => Number.MAX_SAFE_INTEGER,
registerDockerDriverGatewayEndpoint: () => {
registerCount += 1;
return registerCount >= 3;
},
runCaptureOpenshell: (args) => (args[0] === "status" ? STATUS_CONNECTED : GATEWAY_INFO),
sleepSeconds,
skipSandboxBridgeReachability: false,
startOpenShellGatewayUserService: () => ({
attempted: true,
fallbackAllowed: false,
started: true,
}),
verifySandboxBridgeGatewayReachableOrExit: vi.fn(),
}),
).resolves.toBe(true);

expect(registerCount).toBe(3);
expect(sleepSeconds).toHaveBeenCalledTimes(2);
expect(sleepSeconds).toHaveBeenNthCalledWith(1, 0);
expect(sleepSeconds).toHaveBeenNthCalledWith(2, 0);
});

it("falls back to standalone when package-managed service startup is unavailable", async () => {
const registerDockerDriverGatewayEndpoint = vi.fn(() => true);

Expand Down Expand Up @@ -318,6 +360,7 @@ describe("docker-driver-gateway-service", () => {

it("keeps standalone runtime breadcrumbs when service health never becomes ready", async () => {
const clearDockerDriverGatewayRuntimeFiles = vi.fn();
const clock = createVirtualClock();

await expect(
startPackageManagedDockerDriverGateway({
Expand All @@ -326,9 +369,12 @@ describe("docker-driver-gateway-service", () => {
gatewayName: "nemoclaw",
hasOpenShellGatewayUserService: () => true,
healthPollCount: 1,
healthPollInterval: 1,
isDockerDriverGatewayReady: async () => false,
now: clock.now,
registerDockerDriverGatewayEndpoint: () => true,
runCaptureOpenshell: (args) => (args[0] === "status" ? STATUS_CONNECTED : GATEWAY_INFO),
sleepSeconds: clock.advance,
skipSandboxBridgeReachability: false,
startOpenShellGatewayUserService: () => ({
attempted: true,
Expand All @@ -337,7 +383,7 @@ describe("docker-driver-gateway-service", () => {
}),
verifySandboxBridgeGatewayReachableOrExit: vi.fn(),
}),
).rejects.toThrow("did not become healthy");
).rejects.toThrow("configured 1s health deadline");

expect(clearDockerDriverGatewayRuntimeFiles).not.toHaveBeenCalled();
});
Expand Down
50 changes: 26 additions & 24 deletions src/lib/onboard/docker-driver-gateway-service.ts
Original file line number Diff line number Diff line change
@@ -1,13 +1,17 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { spawnSync, type SpawnSyncOptions } from "node:child_process";
import { type SpawnSyncOptions, spawnSync } from "node:child_process";
import fs from "node:fs";
import path from "node:path";

import { sleepSeconds, waitUntilAsync } from "../core/wait";
import { isGatewayHealthy } from "../state/gateway";
import { envInt } from "./env";
import {
createGatewayHealthWaitOptions,
formatGatewayHealthWaitLimit,
} from "./gateway-health-wait";
import { isDockerDriverGatewayHttpReady } from "./gateway-http-readiness";

export const OPENSHELL_GATEWAY_USER_SERVICE = "openshell-gateway";
Expand Down Expand Up @@ -49,6 +53,7 @@ export interface PackageManagedDockerDriverGatewayOptions {
healthPollCount?: number;
healthPollInterval?: number;
isDockerDriverGatewayReady?: () => Promise<boolean>;
now?: () => number;
registerDockerDriverGatewayEndpoint: () => boolean;
runCaptureOpenshell: (args: string[], opts?: { ignoreError?: boolean }) => string;
sleepSeconds?: (seconds: number) => void;
Expand Down Expand Up @@ -291,6 +296,7 @@ export async function startPackageManagedDockerDriverGateway({
healthPollCount,
healthPollInterval,
isDockerDriverGatewayReady = isDockerDriverGatewayHttpReady,
now = Date.now,
registerDockerDriverGatewayEndpoint,
runCaptureOpenshell,
sleepSeconds: sleepSecondsImpl = sleepSeconds,
Expand Down Expand Up @@ -323,29 +329,22 @@ export async function startPackageManagedDockerDriverGateway({

const pollCount = healthPollCount ?? envInt("NEMOCLAW_HEALTH_POLL_COUNT", 30);
const pollInterval = healthPollInterval ?? envInt("NEMOCLAW_HEALTH_POLL_INTERVAL", 2);
const pollIntervalMs = Math.max(0, pollInterval * 1000);
const waitOptions = createGatewayHealthWaitOptions(pollCount, pollInterval, now, (ms) =>
sleepSecondsImpl(ms / 1000),
);
const healthy =
pollCount > 0 &&
(await waitUntilAsync(
async () => {
if (!registerDockerDriverGatewayEndpoint()) return false;
const status = runCaptureOpenshell(["status"], { ignoreError: true });
const namedInfo = runCaptureOpenshell(["gateway", "info", "-g", gatewayName], {
ignoreError: true,
});
const currentInfo = runCaptureOpenshell(["gateway", "info"], { ignoreError: true });
return (
isGatewayHealthy(status, namedInfo, currentInfo) && (await isDockerDriverGatewayReady())
);
},
{
initialIntervalMs: pollIntervalMs,
maxIntervalMs: pollIntervalMs,
backoffFactor: 1,
maxAttempts: pollCount,
sleep: (ms) => sleepSecondsImpl(ms / 1000),
},
));
waitOptions !== null &&
(await waitUntilAsync(async () => {
if (!registerDockerDriverGatewayEndpoint()) return false;
const status = runCaptureOpenshell(["status"], { ignoreError: true });
const namedInfo = runCaptureOpenshell(["gateway", "info", "-g", gatewayName], {
ignoreError: true,
});
const currentInfo = runCaptureOpenshell(["gateway", "info"], { ignoreError: true });
return (
isGatewayHealthy(status, namedInfo, currentInfo) && (await isDockerDriverGatewayReady())
);
}, waitOptions));
if (healthy) {
clearDockerDriverGatewayRuntimeFiles();
await verifySandboxBridgeGatewayReachableOrExit(exitOnFailure, {
Expand All @@ -355,7 +354,10 @@ export async function startPackageManagedDockerDriverGateway({
return true;
}

const message = "OpenShell gateway user service started but did not become healthy.";
const message = `OpenShell gateway user service started but did not become healthy within the configured ${formatGatewayHealthWaitLimit(
pollCount,
pollInterval,
)}.`;
console.error(` ${message}`);
console.error(" Check: systemctl --user status openshell-gateway");
if (exitOnFailure) process.exit(1);
Expand Down
94 changes: 86 additions & 8 deletions src/lib/onboard/gateway-health-wait.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,13 @@

import { beforeEach, describe, expect, it, vi } from "vitest";

import { type GatewayHealthWaitOptions, waitForGatewayHealth } from "./gateway-health-wait";
import { createVirtualClock } from "./__test-helpers__/virtual-clock";
import {
formatGatewayHealthWaitLimit,
type GatewayHealthWaitOptions,
getGatewayHealthWaitBudgetMs,
waitForGatewayHealth,
} from "./gateway-health-wait";

function buildOptions(overrides: Partial<GatewayHealthWaitOptions> = {}): GatewayHealthWaitOptions {
return {
Expand Down Expand Up @@ -49,16 +55,19 @@ describe("waitForGatewayHealth", () => {
});

it("returns false when HTTP readiness never follows healthy metadata", async () => {
const clock = createVirtualClock();
const options = buildOptions({
healthPollCount: 2,
isGatewayHttpReady: vi.fn(async () => false),
now: clock.now,
sleepSeconds: clock.sleeper,
});

await expect(waitForGatewayHealth(options)).resolves.toBe(false);

expect(options.isGatewayHealthy).toHaveBeenCalledTimes(2);
expect(options.isGatewayHttpReady).toHaveBeenCalledTimes(2);
expect(options.sleepSeconds).toHaveBeenCalledTimes(1);
expect(options.sleepSeconds).toHaveBeenCalledTimes(2);
});

it("force-refreshes metadata after bootstrap secret repair", async () => {
Expand All @@ -85,19 +94,88 @@ describe("waitForGatewayHealth", () => {
expect(options.attachGatewayMetadataIfNeeded).toHaveBeenCalledWith();
});

it("stops after healthPollCount attempts without sleeping after the final failed probe", async () => {
it("polls until the configured health deadline instead of stopping at the count cap (#3768)", async () => {
const clock = createVirtualClock();
const isGatewayHealthy = vi.fn(() => {
clock.advance(1);
return false;
});
const options = buildOptions({
healthPollCount: 10,
healthPollIntervalSeconds: 1,
isGatewayHealthy,
now: clock.now,
sleepSeconds: clock.sleeper,
});

await expect(waitForGatewayHealth(options)).resolves.toBe(false);

expect(isGatewayHealthy).toHaveBeenCalled();
expect(isGatewayHealthy.mock.calls.length).toBeLessThan(10);
expect(options.isGatewayHttpReady).not.toHaveBeenCalled();
expect(clock.sleeper).toHaveBeenCalled();
expect(clock.sleeper.mock.calls.every(([seconds]) => seconds === 1)).toBe(true);
});

it("preserves the configured immediate probes when the interval is zero (#3768)", async () => {
const isGatewayHealthy = vi
.fn<() => boolean>()
.mockReturnValueOnce(false)
.mockReturnValueOnce(false)
.mockReturnValueOnce(true);
const sleepSeconds = vi.fn();
const options = buildOptions({
healthPollCount: 3,
isGatewayHealthy: vi.fn(() => false),
healthPollIntervalSeconds: 0,
isGatewayHealthy,
now: vi.fn(() => Number.MAX_SAFE_INTEGER),
sleepSeconds,
});

await expect(waitForGatewayHealth(options)).resolves.toBe(true);

expect(isGatewayHealthy).toHaveBeenCalledTimes(3);
expect(options.isGatewayHttpReady).toHaveBeenCalledOnce();
expect(sleepSeconds).toHaveBeenCalledTimes(2);
expect(sleepSeconds).toHaveBeenNthCalledWith(1, 0);
expect(sleepSeconds).toHaveBeenNthCalledWith(2, 0);
expect(formatGatewayHealthWaitLimit(3, 0)).toBe("3 immediate health probes");
});

it("does not probe after a positive health deadline expires before the first attempt", async () => {
const now = vi.fn().mockReturnValueOnce(0).mockReturnValue(1000);
const options = buildOptions({
healthPollCount: 1,
healthPollIntervalSeconds: 1,
now,
});

await expect(waitForGatewayHealth(options)).resolves.toBe(false);

expect(options.isGatewayHealthy).toHaveBeenCalledTimes(3);
expect(options.runCaptureOpenshell).not.toHaveBeenCalled();
expect(options.isGatewayHealthy).not.toHaveBeenCalled();
expect(options.isGatewayHttpReady).not.toHaveBeenCalled();
expect(options.sleepSeconds).toHaveBeenCalledTimes(2);
expect(options.sleepSeconds).toHaveBeenNthCalledWith(1, 2);
expect(options.sleepSeconds).toHaveBeenNthCalledWith(2, 2);
});

it("preserves a rejected HTTP readiness probe error", async () => {
const probeError = new Error("readiness transport failed");
const options = buildOptions({
healthPollCount: 3,
healthPollIntervalSeconds: 0,
isGatewayHttpReady: vi.fn(async () => Promise.reject(probeError)),
});

await expect(waitForGatewayHealth(options)).rejects.toBe(probeError);

expect(options.isGatewayHealthy).toHaveBeenCalledOnce();
expect(options.isGatewayHttpReady).toHaveBeenCalledOnce();
expect(options.sleepSeconds).not.toHaveBeenCalled();
});

it("clamps an overflowing health deadline budget to a finite value", () => {
expect(getGatewayHealthWaitBudgetMs(Number.MAX_VALUE, Number.MAX_VALUE)).toBe(
Number.MAX_SAFE_INTEGER,
);
});

it("returns false without probing when healthPollCount is zero", async () => {
Expand Down
Loading
Loading