Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 11 additions & 11 deletions src/lib/onboard.ts
Original file line number Diff line number Diff line change
Expand Up @@ -325,9 +325,8 @@ const {
rejectUnsupportedWindowsHostOllama,
shouldFrontOllamaWithProxy,
}: typeof import("./onboard/local-inference-topology") = require("./onboard/local-inference-topology");
const {
waitForGatewayHealth,
}: typeof import("./onboard/gateway-health-wait") = require("./onboard/gateway-health-wait");
const { waitForGatewayHealth }: typeof import("./onboard/gateway-health-wait") =
require("./onboard/gateway-health-wait");
const { resolveOpenshell } = require("./adapters/openshell/resolve");
const credentials: typeof import("./credentials/store") = require("./credentials/store");
const {
Expand Down Expand Up @@ -524,8 +523,11 @@ const { trackChildExit } =
require("./onboard/child-exit-tracker") as typeof import("./onboard/child-exit-tracker");
const { reportDockerDriverGatewayStartFailure } =
require("./onboard/docker-driver-gateway-failure") as typeof import("./onboard/docker-driver-gateway-failure");
const { createFinalGatewayStartFailureHandler, reportLegacyGatewayStartResultFailure } =
require("./onboard/gateway-start-failure") as typeof import("./onboard/gateway-start-failure");
const {
createFinalGatewayStartFailureHandler,
normalizeGatewayStartError,
reportLegacyGatewayStartResultFailure,
} = require("./onboard/gateway-start-failure") as typeof import("./onboard/gateway-start-failure");
const dockerDriverGatewayEnv: typeof import("./onboard/docker-driver-gateway-env") =
require("./onboard/docker-driver-gateway-env");
const dockerDriverGatewayRuntimeMarker: typeof import("./onboard/docker-driver-gateway-runtime-marker") =
Expand Down Expand Up @@ -1896,7 +1898,7 @@ async function startGatewayWithOptions(
return;
}

throw new Error("Gateway failed to start");
throw new Error(`Gateway failed within ${healthWait.count * healthWait.interval}s.`);
},
{
retries,
Expand All @@ -1912,11 +1914,9 @@ async function startGatewayWithOptions(
},
},
);
} catch {
if (exitOnFailure) {
handleFinalGatewayStartFailure({ retries, dockerUnreachable });
}
throw new Error("Gateway failed to start");
} catch (error) {
if (exitOnFailure) handleFinalGatewayStartFailure({ retries, dockerUnreachable });
throw normalizeGatewayStartError(error);
}

console.log(" ✓ Gateway is healthy");
Expand Down
16 changes: 16 additions & 0 deletions src/lib/onboard/__test-helpers__/virtual-clock.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { vi } from "vitest";

export function createVirtualClock(startMs = 1_000_000_000_000) {
let currentMs = startMs;
const advance = (seconds: number) => {
currentMs += Math.max(0, seconds) * 1000;
};
return {
advance,
now: () => currentMs,
sleeper: vi.fn(advance),
};
}
56 changes: 51 additions & 5 deletions src/lib/onboard/docker-driver-gateway-service.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,14 @@

import { describe, expect, it, vi } from "vitest";

import { createVirtualClock } from "./__test-helpers__/virtual-clock";
import {
getOpenShellGatewayUserServiceBinaryPaths,
getOpenShellGatewayUserServicePaths,
hasOpenShellGatewayUserService,
startPackageManagedDockerDriverGateway,
startOpenShellGatewayUserService,
type SpawnSyncLikeResult,
startOpenShellGatewayUserService,
startPackageManagedDockerDriverGateway,
} from "./docker-driver-gateway-service";

const STATUS_CONNECTED = `
Expand Down Expand Up @@ -254,6 +255,7 @@ describe("docker-driver-gateway-service", () => {

it("uses the package-managed service only after endpoint, metadata, and gRPC health are ready", async () => {
const events: string[] = [];
const clock = createVirtualClock();
let registerCount = 0;
const registerDockerDriverGatewayEndpoint = vi.fn(() => {
events.push("register");
Expand All @@ -268,14 +270,18 @@ describe("docker-driver-gateway-service", () => {
gatewayName: "nemoclaw",
hasOpenShellGatewayUserService: () => true,
healthPollCount: 3,
healthPollInterval: 0,
healthPollInterval: 1,
isDockerDriverGatewayReady: async () => {
events.push("ready");
return true;
},
now: clock.now,
registerDockerDriverGatewayEndpoint,
runCaptureOpenshell: (args) => (args[0] === "status" ? STATUS_CONNECTED : GATEWAY_INFO),
sleepSeconds: () => events.push("sleep"),
sleepSeconds: (seconds) => {
events.push("sleep");
clock.advance(seconds);
},
skipSandboxBridgeReachability: false,
startOpenShellGatewayUserService: () => ({
attempted: true,
Expand All @@ -291,6 +297,42 @@ describe("docker-driver-gateway-service", () => {
expect(events).toEqual(["register", "sleep", "register", "ready", "clear", "verify"]);
});

it("preserves bounded immediate package-service probes when the interval is zero", async () => {
const clock = createVirtualClock();
let registerCount = 0;

await expect(
startPackageManagedDockerDriverGateway({
clearDockerDriverGatewayRuntimeFiles: vi.fn(),
exitOnFailure: false,
gatewayName: "nemoclaw",
hasOpenShellGatewayUserService: () => true,
healthPollCount: 3,
healthPollInterval: 0,
isDockerDriverGatewayReady: async () => true,
now: clock.now,
registerDockerDriverGatewayEndpoint: () => {
registerCount += 1;
return registerCount >= 3;
},
runCaptureOpenshell: (args) => (args[0] === "status" ? STATUS_CONNECTED : GATEWAY_INFO),
sleepSeconds: clock.sleeper,
skipSandboxBridgeReachability: false,
startOpenShellGatewayUserService: () => ({
attempted: true,
fallbackAllowed: false,
started: true,
}),
verifySandboxBridgeGatewayReachableOrExit: vi.fn(),
}),
).resolves.toBe(true);

expect(registerCount).toBe(3);
expect(clock.sleeper).toHaveBeenCalledTimes(2);
expect(clock.sleeper).toHaveBeenNthCalledWith(1, 0);
expect(clock.sleeper).toHaveBeenNthCalledWith(2, 0);
});

it("falls back to standalone when package-managed service startup is unavailable", async () => {
const registerDockerDriverGatewayEndpoint = vi.fn(() => true);

Expand Down Expand Up @@ -318,6 +360,7 @@ describe("docker-driver-gateway-service", () => {

it("keeps standalone runtime breadcrumbs when service health never becomes ready", async () => {
const clearDockerDriverGatewayRuntimeFiles = vi.fn();
const clock = createVirtualClock();

await expect(
startPackageManagedDockerDriverGateway({
Expand All @@ -326,9 +369,12 @@ describe("docker-driver-gateway-service", () => {
gatewayName: "nemoclaw",
hasOpenShellGatewayUserService: () => true,
healthPollCount: 1,
healthPollInterval: 1,
isDockerDriverGatewayReady: async () => false,
now: clock.now,
registerDockerDriverGatewayEndpoint: () => true,
runCaptureOpenshell: (args) => (args[0] === "status" ? STATUS_CONNECTED : GATEWAY_INFO),
sleepSeconds: clock.advance,
skipSandboxBridgeReachability: false,
startOpenShellGatewayUserService: () => ({
attempted: true,
Expand All @@ -337,7 +383,7 @@ describe("docker-driver-gateway-service", () => {
}),
verifySandboxBridgeGatewayReachableOrExit: vi.fn(),
}),
).rejects.toThrow("did not become healthy");
).rejects.toThrow("configured 1s health deadline");

expect(clearDockerDriverGatewayRuntimeFiles).not.toHaveBeenCalled();
});
Expand Down
48 changes: 25 additions & 23 deletions src/lib/onboard/docker-driver-gateway-service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,10 @@ import path from "node:path";
import { sleepSeconds, waitUntilAsync } from "../core/wait";
import { isGatewayHealthy } from "../state/gateway";
import { envInt } from "./env";
import {
createGatewayHealthWaitOptions,
formatGatewayHealthWaitLimit,
} from "./gateway-health-wait";
import { isDockerDriverGatewayHttpReady } from "./gateway-http-readiness";

export const OPENSHELL_GATEWAY_USER_SERVICE = "openshell-gateway";
Expand Down Expand Up @@ -49,6 +53,7 @@ export interface PackageManagedDockerDriverGatewayOptions {
healthPollCount?: number;
healthPollInterval?: number;
isDockerDriverGatewayReady?: () => Promise<boolean>;
now?: () => number;
registerDockerDriverGatewayEndpoint: () => boolean;
runCaptureOpenshell: (args: string[], opts?: { ignoreError?: boolean }) => string;
sleepSeconds?: (seconds: number) => void;
Expand Down Expand Up @@ -291,6 +296,7 @@ export async function startPackageManagedDockerDriverGateway({
healthPollCount,
healthPollInterval,
isDockerDriverGatewayReady = isDockerDriverGatewayHttpReady,
now = Date.now,
registerDockerDriverGatewayEndpoint,
runCaptureOpenshell,
sleepSeconds: sleepSecondsImpl = sleepSeconds,
Expand Down Expand Up @@ -323,29 +329,22 @@ export async function startPackageManagedDockerDriverGateway({

const pollCount = healthPollCount ?? envInt("NEMOCLAW_HEALTH_POLL_COUNT", 30);
const pollInterval = healthPollInterval ?? envInt("NEMOCLAW_HEALTH_POLL_INTERVAL", 2);
const pollIntervalMs = Math.max(0, pollInterval * 1000);
const waitOptions = createGatewayHealthWaitOptions(pollCount, pollInterval, now, (ms) =>
sleepSecondsImpl(ms / 1000),
);
const healthy =
pollCount > 0 &&
(await waitUntilAsync(
async () => {
if (!registerDockerDriverGatewayEndpoint()) return false;
const status = runCaptureOpenshell(["status"], { ignoreError: true });
const namedInfo = runCaptureOpenshell(["gateway", "info", "-g", gatewayName], {
ignoreError: true,
});
const currentInfo = runCaptureOpenshell(["gateway", "info"], { ignoreError: true });
return (
isGatewayHealthy(status, namedInfo, currentInfo) && (await isDockerDriverGatewayReady())
);
},
{
initialIntervalMs: pollIntervalMs,
maxIntervalMs: pollIntervalMs,
backoffFactor: 1,
maxAttempts: pollCount,
sleep: (ms) => sleepSecondsImpl(ms / 1000),
},
));
waitOptions !== null &&
(await waitUntilAsync(async () => {
if (!registerDockerDriverGatewayEndpoint()) return false;
const status = runCaptureOpenshell(["status"], { ignoreError: true });
const namedInfo = runCaptureOpenshell(["gateway", "info", "-g", gatewayName], {
ignoreError: true,
});
const currentInfo = runCaptureOpenshell(["gateway", "info"], { ignoreError: true });
return (
isGatewayHealthy(status, namedInfo, currentInfo) && (await isDockerDriverGatewayReady())
);
}, waitOptions));
if (healthy) {
clearDockerDriverGatewayRuntimeFiles();
await verifySandboxBridgeGatewayReachableOrExit(exitOnFailure, {
Expand All @@ -355,7 +354,10 @@ export async function startPackageManagedDockerDriverGateway({
return true;
}

const message = "OpenShell gateway user service started but did not become healthy.";
const message = `OpenShell gateway user service started but did not become healthy within the configured ${formatGatewayHealthWaitLimit(
pollCount,
pollInterval,
)}.`;
console.error(` ${message}`);
console.error(" Check: systemctl --user status openshell-gateway");
if (exitOnFailure) process.exit(1);
Expand Down
Loading
Loading