Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions docs/reference/commands.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,9 @@ If release metadata is unavailable, the installer uses its bundled fallback pin

When NemoClaw finds an existing gateway to reuse, it probes the host gateway HTTP endpoint before declaring the gateway reusable.
If the container is running but the upstream is still warming up (for example, immediately after a Docker daemon restart), NemoClaw rebuilds the gateway instead of trusting stale metadata.
On the Docker-driver gateway path, preflight stays read-only when it detects a stale gateway (for example, a Docker-driver runtime env hash drift).
It prints a `⚠ Gateway will be recreated when sandbox creation starts` notice and defers the actual teardown to step `[2/8] Starting OpenShell gateway`.
This means pressing `Ctrl+C` between preflight and step `[2/8]` leaves the running gateway and existing sandbox containers untouched, so `nemoclaw onboard` is safe to run just to check preflight output.
For Linux Docker-driver gateways, onboarding also checks that a helper container on the OpenShell Docker network can reach `host.openshell.internal:<gateway-port>`.
If a host firewall blocks that sandbox path, onboarding exits with a `sudo ufw allow from <subnet> to any port <gateway-port> proto tcp` command before it reports the gateway healthy.
Tune the wait via `NEMOCLAW_REUSE_HEALTH_POLL_COUNT` (default `6`) and `NEMOCLAW_REUSE_HEALTH_POLL_INTERVAL` (default `5` seconds).
Expand Down
37 changes: 17 additions & 20 deletions src/lib/onboard.ts
Original file line number Diff line number Diff line change
Expand Up @@ -426,6 +426,8 @@ const {
preflightDashboardPortRangeAvailability,
} = require("./onboard/dashboard-port") as typeof import("./onboard/dashboard-port");
const { destroyGatewayForReuse } = require("./onboard/gateway-cleanup") as typeof import("./onboard/gateway-cleanup");
const { applyPreflightGatewayCleanup } =
require("./onboard/preflight-gateway-cleanup-decision") as typeof import("./onboard/preflight-gateway-cleanup-decision");
const { verifyGatewayContainerRunning } =
require("./onboard/gateway-container-running") as typeof import("./onboard/gateway-container-running");
const { destroyGatewayWithVolumeCleanup } =
Expand Down Expand Up @@ -2083,11 +2085,11 @@ async function preflight(

ensureOpenshellForOnboard();

// Clean up stale or unnamed NemoClaw gateway state before checking ports.
// A healthy named gateway can be reused later in onboarding, so avoid
// tearing it down here. If some other gateway is active but the named
// NemoClaw gateway exists, select it before the port checks so onboarding
// reuses the user's NemoClaw gateway instead of reporting a false conflict.
// Classify gateway state before port checks. Legacy non-Docker-driver
// path destroys stale/unnamed gateways here so the port frees up for
// checks below; Docker-driver path defers the destructive recreate to
// step [2/8] (see applyPreflightGatewayCleanup). If another gateway is
// active but the named one exists, select it to avoid false conflicts.
const gatewaySnapshot = selectNamedGatewayForReuseIfNeeded(getGatewayReuseSnapshot());
let gatewayReuseState = gatewaySnapshot.gatewayReuseState;
gatewayReuseState = await refreshDockerDriverGatewayReuseState(gatewayReuseState);
Expand Down Expand Up @@ -2156,21 +2158,16 @@ async function preflight(
}
}

if (gatewayReuseState === "stale" || gatewayReuseState === "active-unnamed") {
console.log(` Cleaning up previous ${cliDisplayName()} session...`);
if (isLinuxDockerDriverGatewayEnabled()) {
retireLegacyGatewayForDockerDriverUpgrade();
gatewayReuseState = "missing";
console.log(" ✓ Previous session cleaned up");
} else {
runOpenshell(["forward", "stop", String(DASHBOARD_PORT)], { ignoreError: true });
gatewayReuseState = destroyGatewayForReuse(
destroyGateway,
" ✓ Previous session cleaned up",
" ! Previous session cleanup failed; leaving registry state intact.",
);
}
}
gatewayReuseState = applyPreflightGatewayCleanup({
gatewayReuseState,
isDockerDriverGatewayEnabled: isLinuxDockerDriverGatewayEnabled(),
cliDisplayName: cliDisplayName(),
dashboardPort: DASHBOARD_PORT,
log: console.log,
runOpenshell,
destroyGateway,
destroyGatewayForReuse,
});

// Clean up orphaned Docker containers from interrupted onboard (e.g. Ctrl+C
// during gateway start). The container may still be running even though
Expand Down
24 changes: 24 additions & 0 deletions src/lib/onboard/machine/handlers/gateway.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -222,4 +222,28 @@ describe("handleGatewayState", () => {
expect(calls.startGateway).toHaveBeenCalledOnce();
expect(result.gatewayReuseState).toBe("missing");
});

it("emits the step [2/8] header before retiring the legacy Docker-driver gateway", async () => {
const order: string[] = [];
const { deps, calls } = createDeps({
isLinuxDockerDriverGatewayEnabled: vi.fn(() => true),
reconcileGatewayGpuReuseForGpuIntent: vi.fn(() => "stale" as GatewayReuseState),
startRecordedStep: vi.fn(async (step: string) => {
order.push(`startRecordedStep:${step}`);
}),
retireLegacyGatewayForDockerDriverUpgrade: vi.fn(() => {
order.push("retireLegacy");
}),
startGateway: vi.fn(async () => {
order.push("startGateway");
}),
});

await handleGatewayState(baseOptions(deps, "healthy"));

expect(order).toEqual(["startRecordedStep:gateway", "retireLegacy", "startGateway"]);
expect(calls.note).toHaveBeenCalledWith(
" Replacing legacy OpenShell gateway metadata with Docker-driver gateway.",
);
});
});
2 changes: 1 addition & 1 deletion src/lib/onboard/machine/handlers/gateway.ts
Original file line number Diff line number Diff line change
Expand Up @@ -171,12 +171,12 @@ export async function handleGatewayState<Gpu>({
deps.note(" [resume] Recorded gateway state is unavailable; recreating it.");
}
}
await deps.startRecordedStep("gateway");
if (deps.isLinuxDockerDriverGatewayEnabled() && gatewayReuseState !== "missing") {
deps.note(" Replacing legacy OpenShell gateway metadata with Docker-driver gateway.");
deps.retireLegacyGatewayForDockerDriverUpgrade();
gatewayReuseState = "missing";
}
await deps.startRecordedStep("gateway");
await withGatewayTrace(gatewayReuseState, gpuPassthrough, () =>
deps.startGateway(gpu, { gpuPassthrough }),
);
Expand Down
135 changes: 135 additions & 0 deletions src/lib/onboard/preflight-gateway-cleanup-decision.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { describe, expect, it, vi } from "vitest";

import type { GatewayReuseState } from "../state/gateway";

import {
PREFLIGHT_DEFERRED_RECREATE_MESSAGE,
applyPreflightGatewayCleanup,
preflightGatewayCleanupDecision,
} from "./preflight-gateway-cleanup-decision";

describe("preflightGatewayCleanupDecision", () => {
it("defers when state is stale and Docker-driver gateway is enabled", () => {
expect(
preflightGatewayCleanupDecision({
gatewayReuseState: "stale",
isDockerDriverGatewayEnabled: true,
}),
).toBe("defer");
});

it("defers when state is active-unnamed and Docker-driver gateway is enabled", () => {
expect(
preflightGatewayCleanupDecision({
gatewayReuseState: "active-unnamed",
isDockerDriverGatewayEnabled: true,
}),
).toBe("defer");
});

it("destroys legacy gateway in preflight when Docker-driver gateway is not enabled", () => {
expect(
preflightGatewayCleanupDecision({
gatewayReuseState: "stale",
isDockerDriverGatewayEnabled: false,
}),
).toBe("destroy-legacy");
expect(
preflightGatewayCleanupDecision({
gatewayReuseState: "active-unnamed",
isDockerDriverGatewayEnabled: false,
}),
).toBe("destroy-legacy");
});

it("returns noop for non-stale states regardless of driver", () => {
for (const state of ["healthy", "missing", "foreign-active"] as const) {
expect(
preflightGatewayCleanupDecision({
gatewayReuseState: state,
isDockerDriverGatewayEnabled: true,
}),
).toBe("noop");
expect(
preflightGatewayCleanupDecision({
gatewayReuseState: state,
isDockerDriverGatewayEnabled: false,
}),
).toBe("noop");
}
});
});

describe("applyPreflightGatewayCleanup", () => {
function makeDeps(overrides: {
gatewayReuseState: GatewayReuseState;
isDockerDriverGatewayEnabled: boolean;
}) {
const log = vi.fn();
const runOpenshell = vi.fn(() => ({ status: 0 }));
const destroyGateway = vi.fn(() => true);
const destroyGatewayForReuse = vi.fn<
(
destroy: () => boolean,
success: string,
failure: string,
) => GatewayReuseState
>((destroy) => {
destroy();
return "missing";
});
return {
deps: {
gatewayReuseState: overrides.gatewayReuseState,
isDockerDriverGatewayEnabled: overrides.isDockerDriverGatewayEnabled,
cliDisplayName: "NemoClaw",
dashboardPort: 8081,
log,
runOpenshell,
destroyGateway,
destroyGatewayForReuse,
},
log,
runOpenshell,
destroyGateway,
destroyGatewayForReuse,
};
}

it("logs the deferral notice without invoking destroy on the Docker-driver path", () => {
const ctx = makeDeps({ gatewayReuseState: "stale", isDockerDriverGatewayEnabled: true });
const next = applyPreflightGatewayCleanup(ctx.deps);
expect(next).toBe("stale");
expect(ctx.log).toHaveBeenCalledWith(PREFLIGHT_DEFERRED_RECREATE_MESSAGE);
expect(ctx.destroyGateway).not.toHaveBeenCalled();
expect(ctx.destroyGatewayForReuse).not.toHaveBeenCalled();
expect(ctx.runOpenshell).not.toHaveBeenCalled();
});

it("destroys the legacy gateway and stops the dashboard forward on the non-Docker-driver path", () => {
const ctx = makeDeps({ gatewayReuseState: "stale", isDockerDriverGatewayEnabled: false });
const next = applyPreflightGatewayCleanup(ctx.deps);
expect(next).toBe("missing");
expect(ctx.log).toHaveBeenCalledWith(" Cleaning up previous NemoClaw session...");
expect(ctx.runOpenshell).toHaveBeenCalledWith(["forward", "stop", "8081"], {
ignoreError: true,
});
expect(ctx.destroyGatewayForReuse).toHaveBeenCalledTimes(1);
expect(ctx.destroyGateway).toHaveBeenCalledTimes(1);
});

it("is a no-op for healthy / missing / foreign-active states", () => {
for (const state of ["healthy", "missing", "foreign-active"] as const) {
const ctx = makeDeps({ gatewayReuseState: state, isDockerDriverGatewayEnabled: true });
const next = applyPreflightGatewayCleanup(ctx.deps);
expect(next).toBe(state);
expect(ctx.log).not.toHaveBeenCalled();
expect(ctx.destroyGateway).not.toHaveBeenCalled();
expect(ctx.destroyGatewayForReuse).not.toHaveBeenCalled();
expect(ctx.runOpenshell).not.toHaveBeenCalled();
}
});
});
57 changes: 57 additions & 0 deletions src/lib/onboard/preflight-gateway-cleanup-decision.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import type { GatewayReuseState } from "../state/gateway";

export type PreflightGatewayCleanupAction = "defer" | "destroy-legacy" | "noop";

export const PREFLIGHT_DEFERRED_RECREATE_MESSAGE =
" ⚠ Gateway will be recreated when sandbox creation starts — this will affect running sandboxes.";

export function preflightGatewayCleanupDecision(opts: {
gatewayReuseState: GatewayReuseState;
isDockerDriverGatewayEnabled: boolean;
}): PreflightGatewayCleanupAction {
if (opts.gatewayReuseState !== "stale" && opts.gatewayReuseState !== "active-unnamed") {
return "noop";
}
return opts.isDockerDriverGatewayEnabled ? "defer" : "destroy-legacy";
}

export interface PreflightGatewayCleanupDeps {
gatewayReuseState: GatewayReuseState;
isDockerDriverGatewayEnabled: boolean;
cliDisplayName: string;
dashboardPort: number;
log: (line: string) => void;
runOpenshell: (args: string[], options: { ignoreError: true }) => unknown;
destroyGateway: () => boolean;
destroyGatewayForReuse: (
destroy: () => boolean,
successMessage: string,
failureMessage: string,
) => GatewayReuseState;
}

export function applyPreflightGatewayCleanup(
deps: PreflightGatewayCleanupDeps,
): GatewayReuseState {
const action = preflightGatewayCleanupDecision({
gatewayReuseState: deps.gatewayReuseState,
isDockerDriverGatewayEnabled: deps.isDockerDriverGatewayEnabled,
});
if (action === "defer") {
deps.log(PREFLIGHT_DEFERRED_RECREATE_MESSAGE);
return deps.gatewayReuseState;
}
if (action === "destroy-legacy") {
deps.log(` Cleaning up previous ${deps.cliDisplayName} session...`);
deps.runOpenshell(["forward", "stop", String(deps.dashboardPort)], { ignoreError: true });
return deps.destroyGatewayForReuse(
deps.destroyGateway,
" ✓ Previous session cleaned up",
" ! Previous session cleanup failed; leaving registry state intact.",
);
}
return deps.gatewayReuseState;
}
Loading