diff --git a/docs/inference/set-up-vllm-on-two-dgx-stations.mdx b/docs/inference/set-up-vllm-on-two-dgx-stations.mdx
index d6b8062b54a..f916d5faad2 100644
--- a/docs/inference/set-up-vllm-on-two-dgx-stations.mdx
+++ b/docs/inference/set-up-vllm-on-two-dgx-stations.mdx
@@ -142,8 +142,8 @@ Follow [Set Up vLLM](set-up-vllm) for that workflow.
## Preserve Cleanup Ownership
After readiness and container validation pass, NemoClaw writes an owner-only cleanup receipt.
-It also copies the SSH binding under the selected gateway state root.
-A later onboarding run that reuses the validated pair recreates this cleanup ownership before it accepts the endpoint.
+It copies the SSH binding beside the receipt under the host-global `~/.nemoclaw/` state root so every gateway port uses the same ownership state.
+A later onboarding run recovers and revalidates this cleanup ownership before it accepts the existing endpoint.
The receipt contains no serving API key.
It records the peer, cluster, and GPU identities needed to revalidate and remove both managed containers.
diff --git a/docs/reference/host-files-and-state.mdx b/docs/reference/host-files-and-state.mdx
index db37110da7e..f9e283211eb 100644
--- a/docs/reference/host-files-and-state.mdx
+++ b/docs/reference/host-files-and-state.mdx
@@ -18,14 +18,12 @@ Deep Agents-specific runtime state lives inside the sandbox under `/sandbox/.dee
-Do not paste a legacy `credentials.json`, provider tokens, bot tokens, proxy tokens, or debug archives containing them into chat or issue comments.
+Do not paste a legacy `credentials.json`, provider tokens, bot tokens, proxy tokens, `dual-station-vllm-api-key`, or debug archives containing them into chat or issue comments.
Share redacted diagnostics only.
## Files
-In the table, `` is `~/.nemoclaw/` for the default gateway or `~/.nemoclaw/gateways//` for a non-default gateway.
-
| Path | Purpose | Safe to delete |
|---|---|---|
| `~/.nemoclaw/config.json` | Host-level CLI configuration and defaults created by onboarding or config commands. | Only if you want NemoClaw to forget host defaults and rebuild them on the next setup. |
@@ -34,8 +32,14 @@ In the table, `` is `~/.nemoclaw/` for the default gateway
| `~/.nemoclaw/onboard-session.json` | Resume marker for an onboarding attempt that failed before completion. | Yes, when you intentionally want to discard the failed session and start over. Prefer `$$nemoclaw onboard --fresh` when available. |
| `~/.nemoclaw/usage-notice.json` | Records the third-party software notice version in `acceptedVersion` and the acceptance time in `acceptedAt`. Install, onboarding, and rebuild flows consult this file and prompt again when its recorded version differs from the current notice or the file is absent. | Yes; deleting it makes the next applicable install, onboarding, or rebuild flow prompt for acceptance again. |
| `~/.nemoclaw/ollama-proxy-token` | Local auth token used by the host-side Ollama auth proxy. | Yes, but re-run onboarding afterward so NemoClaw recreates and registers the proxy token. |
-| `/dual-station-vllm-runtime.json` | Owner-only managed dual-Station cleanup receipt. It contains no serving API key and binds the peer, cluster, and GPU identities used to revalidate and remove both managed vLLM containers during full uninstall. | No while the managed pair exists. A full `$$nemoclaw uninstall` removes the receipt after both exact containers are removed; gateway-scoped uninstall preserves it. |
-| `/dual-station-vllm-runtime.json.ssh-binding/` | Owner-only copied SSH host-key and Docker-command binding needed to reach the recorded worker during full uninstall. | No while the managed pair exists. Full uninstall removes it with the cleanup receipt after pair cleanup succeeds; gateway-scoped uninstall preserves it. |
+| `~/.nemoclaw/dual-station-vllm-api-key` | Host-global, owner-only bearer key shared by gateways that route to the managed dual-Station vLLM endpoint. The head requires this key for inference requests, and the worker does not receive it. | No while the managed pair exists. Full uninstall removes it after pair cleanup succeeds; gateway-scoped uninstall preserves it while sibling gateways remain. |
+| `~/.nemoclaw/dual-station-vllm-runtime.json` | Host-global, owner-only managed dual-Station cleanup receipt. It contains no serving API key and binds the peer, cluster, and GPU identities used to revalidate and remove both managed vLLM containers during full uninstall. | No while the managed pair exists. A full `$$nemoclaw uninstall` removes the receipt after both exact containers are removed; gateway-scoped uninstall preserves it. |
+| `~/.nemoclaw/dual-station-vllm-runtime.json.ssh-binding/` | Host-global, owner-only copied SSH host-key and Docker-command binding needed to reach the recorded worker during full uninstall. | No while the managed pair exists. Full uninstall removes it with the cleanup receipt after pair cleanup succeeds; gateway-scoped uninstall preserves it. |
+
+Earlier releases can store the runtime receipt and its SSH-binding directory under `~/.nemoclaw/gateways//` when the managed pair was created from a non-default gateway port.
+The API key remains host-global at `~/.nemoclaw/dual-station-vllm-api-key`.
+Current releases discover and validate the legacy receipt location without moving or duplicating its ownership state.
+Do not move these files or directories manually.
`sandboxes.json` is the current registry file name.
If you see `registry.json` in older tests, notes, or discussions, treat it as legacy wording for the sandbox registry unless a specific release note says otherwise.
diff --git a/src/lib/actions/uninstall/run-plan-dual-station.test.ts b/src/lib/actions/uninstall/run-plan-dual-station.test.ts
index 3570e727f3c..c8ed8cbf3c7 100644
--- a/src/lib/actions/uninstall/run-plan-dual-station.test.ts
+++ b/src/lib/actions/uninstall/run-plan-dual-station.test.ts
@@ -48,6 +48,12 @@ describe("dual-Station runtime uninstall", () => {
fs.writeFileSync(path.join(stateDir, "dual-station-vllm-runtime.json"), "{}\n", {
mode: 0o600,
});
+ fs.writeFileSync(path.join(stateDir, "dual-station-vllm-api-key"), "ab".repeat(32), {
+ mode: 0o600,
+ });
+ fs.mkdirSync(path.join(stateDir, "dual-station-vllm-runtime.json.ssh-binding"), {
+ mode: 0o700,
+ });
const runDualStationRuntimeCleanup = vi.fn(() => ok());
const rmSync = vi.fn();
const runDocker = vi.fn(() => ok());
@@ -79,6 +85,111 @@ describe("dual-Station runtime uninstall", () => {
}
});
+ it("finds host-global pair ownership when the final gateway uses a non-default port", async () => {
+ const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-dual-port-"));
+ const port = 9123;
+ const stateDir = path.join(home, ".nemoclaw");
+ const legacyStateDir = path.join(stateDir, "gateways", String(port));
+ fs.mkdirSync(legacyStateDir, { recursive: true, mode: 0o700 });
+ fs.writeFileSync(path.join(legacyStateDir, "dual-station-vllm-runtime.json"), "{}\n", {
+ mode: 0o600,
+ });
+ fs.writeFileSync(path.join(stateDir, "dual-station-vllm-api-key"), "ab".repeat(32), {
+ mode: 0o600,
+ });
+ fs.mkdirSync(path.join(legacyStateDir, "dual-station-vllm-runtime.json.ssh-binding"), {
+ mode: 0o700,
+ });
+ const runDualStationRuntimeCleanup = vi.fn(() => ok());
+
+ try {
+ vi.stubEnv("NEMOCLAW_GATEWAY_PORT", String(port));
+ vi.resetModules();
+ const { runUninstallPlan: runPortUninstallBase } = await import("./run-plan");
+ const result = runPortUninstallBase(
+ { assumeYes: true, deleteModels: false, keepOpenShell: true },
+ {
+ commandExists: () => true,
+ env: {
+ HOME: home,
+ NEMOCLAW_GATEWAY_PORT: String(port),
+ TMPDIR: home,
+ } as NodeJS.ProcessEnv,
+ existsSync: () => false,
+ isTty: false,
+ log: vi.fn(),
+ resolveGatewayTeardownAuthority: ({ gatewayName, gatewayPort }) => ({
+ gatewayName,
+ gatewayPort,
+ mode: "nemoclaw-managed",
+ source: "standalone",
+ endpoint: null,
+ stateDir: null,
+ supervisor: null,
+ requiredCapabilities: [],
+ }),
+ rmSync: vi.fn(),
+ run: (command, args) =>
+ command === "openshell" && args[0] === "gateway" && args[1] === "list"
+ ? ok(JSON.stringify([{ name: `nemoclaw-${String(port)}` }]))
+ : ok(),
+ runDocker: () => ok(),
+ runDualStationRuntimeCleanup,
+ },
+ );
+
+ expect(result.exitCode).toBe(0);
+ expect(runDualStationRuntimeCleanup).toHaveBeenCalledOnce();
+ } finally {
+ vi.unstubAllEnvs();
+ vi.resetModules();
+ fs.rmSync(home, { recursive: true, force: true });
+ }
+ });
+
+ it("preserves host-global pair ownership while sibling gateways remain", () => {
+ const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-dual-scoped-"));
+ const stateDir = path.join(home, ".nemoclaw");
+ const apiKeyPath = path.join(stateDir, "dual-station-vllm-api-key");
+ const receiptPath = path.join(stateDir, "dual-station-vllm-runtime.json");
+ const bindingPath = `${receiptPath}.ssh-binding`;
+ const selectedStatePath = path.join(stateDir, "selected-only");
+ fs.mkdirSync(bindingPath, { recursive: true, mode: 0o700 });
+ fs.writeFileSync(apiKeyPath, "ab".repeat(32), { mode: 0o600 });
+ fs.writeFileSync(receiptPath, "{}\n", { mode: 0o600 });
+ fs.writeFileSync(selectedStatePath, "remove me\n");
+ const runDualStationRuntimeCleanup = vi.fn(() => ok());
+
+ try {
+ const result = runUninstallPlan(
+ { assumeYes: true, deleteModels: false, destroyUserData: true, keepOpenShell: true },
+ {
+ commandExists: (command) => command === "openshell",
+ env: { HOME: home, TMPDIR: home } as NodeJS.ProcessEnv,
+ existsSync: fs.existsSync,
+ isTty: false,
+ log: vi.fn(),
+ rmSync: fs.rmSync,
+ run: (command, args) =>
+ command === "openshell" && args[0] === "gateway" && args[1] === "list"
+ ? ok(JSON.stringify([{ name: "nemoclaw" }, { name: "sibling" }]))
+ : ok(),
+ runDocker: () => ok(),
+ runDualStationRuntimeCleanup,
+ },
+ );
+
+ expect(result.exitCode).toBe(0);
+ expect(fs.existsSync(apiKeyPath)).toBe(true);
+ expect(fs.existsSync(receiptPath)).toBe(true);
+ expect(fs.existsSync(bindingPath)).toBe(true);
+ expect(fs.existsSync(selectedStatePath)).toBe(false);
+ expect(runDualStationRuntimeCleanup).not.toHaveBeenCalled();
+ } finally {
+ fs.rmSync(home, { recursive: true, force: true });
+ }
+ });
+
it("does not start the remaining uninstall steps when managed pair cleanup fails", () => {
const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-dual-fail-"));
const stateDir = path.join(home, ".nemoclaw");
diff --git a/src/lib/actions/uninstall/run-plan.ts b/src/lib/actions/uninstall/run-plan.ts
index f564e3c25aa..500c5a17ab8 100644
--- a/src/lib/actions/uninstall/run-plan.ts
+++ b/src/lib/actions/uninstall/run-plan.ts
@@ -27,7 +27,11 @@ import {
} from "../../domain/uninstall/paths";
import { buildUninstallPlan, type UninstallPlan } from "../../domain/uninstall/plan";
import { isOllamaAuthProxyCommandLine } from "../../inference/ollama/process";
-import { DUAL_STATION_VLLM_RUNTIME_RECEIPT_FILE } from "../../inference/vllm-station-runtime-receipt-path";
+import {
+ DUAL_STATION_VLLM_API_KEY_FILE,
+ DUAL_STATION_VLLM_RUNTIME_RECEIPT_FILE,
+ discoverDualStationVllmRuntimeReceiptStateDirs,
+} from "../../inference/vllm-station-runtime-receipt-path";
import { buildDockerGatewayDebEnvFile } from "../../onboard/docker-driver-gateway-env";
import {
getNemoclawOpenShellGatewayUserServicePath,
@@ -251,6 +255,9 @@ const SHARED_HOST_STATE_ENTRIES = new Set([
"source",
GATEWAYS_SUBDIR,
"managed_swap",
+ DUAL_STATION_VLLM_API_KEY_FILE,
+ DUAL_STATION_VLLM_RUNTIME_RECEIPT_FILE,
+ `${DUAL_STATION_VLLM_RUNTIME_RECEIPT_FILE}.ssh-binding`,
...HTTPS_PIN_RUNTIME_ADAPTER_STATE_ENTRIES,
]);
@@ -1209,14 +1216,32 @@ function removeManagedDualStationRuntime(
paths: UninstallPaths,
runtime: UninstallRuntime,
): boolean {
- const receiptPath = path.join(paths.nemoclawStateDir, DUAL_STATION_VLLM_RUNTIME_RECEIPT_FILE);
+ const sharedStateDir = path.dirname(paths.managedSwapMarkerPath);
+ let receiptPaths: string[];
try {
- fs.lstatSync(receiptPath);
+ receiptPaths = discoverDualStationVllmRuntimeReceiptStateDirs(
+ sharedStateDir,
+ GATEWAYS_SUBDIR,
+ ).map((stateDir) => path.join(stateDir, DUAL_STATION_VLLM_RUNTIME_RECEIPT_FILE));
} catch (error) {
- if ((error as NodeJS.ErrnoException).code === "ENOENT") return true;
runtime.error(`Could not inspect managed dual-Station rollback state: ${formatError(error)}`);
return false;
}
+ let receiptFound = false;
+ for (const receiptPath of receiptPaths) {
+ try {
+ fs.lstatSync(receiptPath);
+ receiptFound = true;
+ } catch (error) {
+ if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
+ runtime.error(
+ `Could not inspect managed dual-Station rollback state: ${formatError(error)}`,
+ );
+ return false;
+ }
+ }
+ }
+ if (!receiptFound) return true;
const result = runtime.runDualStationRuntimeCleanup({
env: runtime.env,
stdio: "inherit",
@@ -1753,6 +1778,7 @@ function executePlan(
...(scopedToSelectedGateway
? [
...HTTPS_PIN_RUNTIME_ADAPTER_STATE_ENTRIES,
+ DUAL_STATION_VLLM_API_KEY_FILE,
DUAL_STATION_VLLM_RUNTIME_RECEIPT_FILE,
`${DUAL_STATION_VLLM_RUNTIME_RECEIPT_FILE}.ssh-binding`,
]
diff --git a/src/lib/inference/vllm-api-key.ts b/src/lib/inference/vllm-api-key.ts
index 0eda887c52b..afdcec37ee0 100644
--- a/src/lib/inference/vllm-api-key.ts
+++ b/src/lib/inference/vllm-api-key.ts
@@ -9,8 +9,8 @@ import path from "node:path";
import { DEFAULT_GATEWAY_PORT } from "../core/ports";
import { nemoclawStateRoot } from "../state/state-root";
import { ensureLocalAdapterStateDir } from "./local-adapter-lifecycle";
+import { DUAL_STATION_VLLM_API_KEY_FILE } from "./vllm-station-runtime-receipt-path";
-export const DUAL_STATION_VLLM_API_KEY_FILE = "dual-station-vllm-api-key";
export const DUAL_STATION_VLLM_API_KEY_PATTERN = /^[a-f0-9]{64}$/;
export interface DualStationVllmApiKeyOptions {
diff --git a/src/lib/inference/vllm-dual-station.test.ts b/src/lib/inference/vllm-dual-station.test.ts
index 82f04bfff98..462b7b88ba1 100644
--- a/src/lib/inference/vllm-dual-station.test.ts
+++ b/src/lib/inference/vllm-dual-station.test.ts
@@ -29,6 +29,7 @@ const mocks = vi.hoisted(() => ({
preflightOwnership: vi.fn(),
persistRuntimeReceipt: vi.fn(),
probeCapability: vi.fn(),
+ recoverRuntime: vi.fn(),
probeDockerStorage: vi.fn(),
probeHostStorage: vi.fn(),
runCapture: vi.fn(),
@@ -93,6 +94,7 @@ vi.mock("./vllm-station-cluster-lifecycle", () => ({
vi.mock("./vllm-station-runtime-receipt", () => ({
persistDualStationVllmRuntimeReceipt: mocks.persistRuntimeReceipt,
+ recoverInstalledDualStationVllmRuntime: mocks.recoverRuntime,
}));
vi.mock("./vllm-api-key", () => ({
@@ -244,6 +246,7 @@ beforeEach(() => {
mocks.cleanup.mockReturnValue({ ok: true, removedContainerIds: [] });
mocks.commitLegacyMigration.mockResolvedValue({ ok: true, cleanupWarnings: [] });
mocks.persistRuntimeReceipt.mockImplementation(() => {});
+ mocks.recoverRuntime.mockReturnValue({ kind: "not-installed" });
mocks.rollbackLegacyMigration.mockResolvedValue({ ok: true });
mocks.measureDirectorySizeBytes.mockReturnValue(0n);
mocks.probeDockerStorage.mockReturnValue({
@@ -310,6 +313,7 @@ describe("dual DGX Station running-runtime receipt adoption", () => {
});
expect(mocks.probeCapability).toHaveBeenCalledOnce();
+ expect(mocks.recoverRuntime).not.toHaveBeenCalled();
expect(mocks.preflightOwnership).toHaveBeenCalledWith(plan());
expect(mocks.areContainersRunning).toHaveBeenCalledWith(plan());
expect(mocks.persistRuntimeReceipt).toHaveBeenCalledWith(plan());
@@ -335,9 +339,106 @@ describe("dual DGX Station running-runtime receipt adoption", () => {
ok: false,
reason: "the managed dual-Station peer configuration is missing",
});
+ expect(mocks.recoverRuntime).toHaveBeenCalledOnce();
expect(mocks.probeCapability).not.toHaveBeenCalled();
expect(mocks.persistRuntimeReceipt).not.toHaveBeenCalled();
});
+
+ it("recovers and revalidates persisted pair ownership for a later onboarding process", async () => {
+ delete process.env.NEMOCLAW_DGX_STATION_PEER;
+ const recoveredPlan = plan();
+ let lifecycleActive = false;
+ mocks.recoverRuntime.mockReturnValue({ kind: "ready", plan: recoveredPlan });
+ mocks.withLifecycle.mockImplementation(async (operation) => {
+ lifecycleActive = true;
+ try {
+ return await operation();
+ } finally {
+ lifecycleActive = false;
+ }
+ });
+ mocks.recoverRuntime.mockImplementation(() => {
+ expect(lifecycleActive).toBe(true);
+ return { kind: "ready", plan: recoveredPlan };
+ });
+ mocks.preflightOwnership.mockImplementation(() => {
+ expect(lifecycleActive).toBe(true);
+ return { ok: true };
+ });
+ mocks.areContainersRunning.mockImplementation(() => {
+ expect(lifecycleActive).toBe(true);
+ return true;
+ });
+
+ await expect(persistConfiguredDualStationVllmRuntimeReceipt()).resolves.toEqual({
+ ok: true,
+ persisted: true,
+ });
+
+ expect(mocks.recoverRuntime).toHaveBeenCalledOnce();
+ expect(mocks.probeCapability).not.toHaveBeenCalled();
+ expect(mocks.preflightOwnership).toHaveBeenCalledWith(recoveredPlan);
+ expect(mocks.areContainersRunning).toHaveBeenCalledWith(recoveredPlan);
+ expect(mocks.persistRuntimeReceipt).not.toHaveBeenCalled();
+ expect(lifecycleActive).toBe(false);
+ });
+
+ it("fails closed when the receipt disappears before locked recovery", async () => {
+ delete process.env.NEMOCLAW_DGX_STATION_PEER;
+ let lifecycleActive = false;
+ mocks.withLifecycle.mockImplementation(async (operation) => {
+ lifecycleActive = true;
+ try {
+ mocks.recoverRuntime.mockImplementation(() => {
+ expect(lifecycleActive).toBe(true);
+ return { kind: "not-installed" };
+ });
+ return await operation();
+ } finally {
+ lifecycleActive = false;
+ }
+ });
+
+ await expect(persistConfiguredDualStationVllmRuntimeReceipt()).resolves.toEqual({
+ ok: false,
+ reason: "the managed dual-Station peer configuration is missing",
+ });
+ expect(mocks.recoverRuntime).toHaveBeenCalledOnce();
+ expect(mocks.preflightOwnership).not.toHaveBeenCalled();
+ expect(mocks.persistRuntimeReceipt).not.toHaveBeenCalled();
+ expect(lifecycleActive).toBe(false);
+ });
+
+ it("fails closed when persisted pair ownership is unsafe", async () => {
+ delete process.env.NEMOCLAW_DGX_STATION_PEER;
+ mocks.recoverRuntime.mockReturnValue({
+ kind: "unsafe",
+ reason: "could not revalidate the managed pair: managed runtime identity changed",
+ });
+
+ await expect(persistConfiguredDualStationVllmRuntimeReceipt()).resolves.toEqual({
+ ok: false,
+ reason:
+ "the managed dual-Station cleanup receipt is unsafe: could not revalidate the managed pair: managed runtime identity changed",
+ });
+ expect(mocks.preflightOwnership).not.toHaveBeenCalled();
+ expect(mocks.areContainersRunning).not.toHaveBeenCalled();
+ expect(mocks.persistRuntimeReceipt).not.toHaveBeenCalled();
+ });
+
+ it("fails closed when a recovered pair changes before locked ownership validation", async () => {
+ delete process.env.NEMOCLAW_DGX_STATION_PEER;
+ const recoveredPlan = plan();
+ mocks.recoverRuntime.mockReturnValue({ kind: "ready", plan: recoveredPlan });
+ mocks.areContainersRunning.mockReturnValue(false);
+
+ await expect(persistConfiguredDualStationVllmRuntimeReceipt()).resolves.toEqual({
+ ok: false,
+ reason: "the managed dual-Station containers changed before cleanup ownership validation",
+ });
+ expect(mocks.preflightOwnership).toHaveBeenCalledWith(recoveredPlan);
+ expect(mocks.persistRuntimeReceipt).not.toHaveBeenCalled();
+ });
});
describe("dual DGX Station vLLM install orchestration", () => {
diff --git a/src/lib/inference/vllm-station-runtime-receipt-path.ts b/src/lib/inference/vllm-station-runtime-receipt-path.ts
index 42e96cb4745..9cb6fc0f3c5 100644
--- a/src/lib/inference/vllm-station-runtime-receipt-path.ts
+++ b/src/lib/inference/vllm-station-runtime-receipt-path.ts
@@ -1,4 +1,37 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
+import fs from "node:fs";
+import path from "node:path";
+
+export const DUAL_STATION_VLLM_API_KEY_FILE = "dual-station-vllm-api-key";
export const DUAL_STATION_VLLM_RUNTIME_RECEIPT_FILE = "dual-station-vllm-runtime.json";
+
+export function discoverDualStationVllmRuntimeReceiptStateDirs(
+ sharedStateDir: string,
+ gatewaysSubdir: string,
+): string[] {
+ const gatewaysDir = path.join(sharedStateDir, gatewaysSubdir);
+ let metadata: fs.Stats;
+ try {
+ metadata = fs.lstatSync(gatewaysDir);
+ } catch (error) {
+ if ((error as NodeJS.ErrnoException).code === "ENOENT") return [sharedStateDir];
+ throw error;
+ }
+ if (metadata.isSymbolicLink() || !metadata.isDirectory()) {
+ throw new Error(`Dual-Station vLLM gateway state directory is unsafe: ${gatewaysDir}`);
+ }
+ const stateDirs = [sharedStateDir];
+ for (const entry of fs.readdirSync(gatewaysDir, { withFileTypes: true })) {
+ if (!/^\d{1,5}$/.test(entry.name)) continue;
+ const port = Number(entry.name);
+ if (port < 1 || port > 65535) continue;
+ const stateDir = path.join(gatewaysDir, entry.name);
+ if (entry.isSymbolicLink() || !entry.isDirectory()) {
+ throw new Error(`Dual-Station vLLM legacy gateway state directory is unsafe: ${stateDir}`);
+ }
+ stateDirs.push(stateDir);
+ }
+ return stateDirs;
+}
diff --git a/src/lib/inference/vllm-station-runtime-receipt.test.ts b/src/lib/inference/vllm-station-runtime-receipt.test.ts
index 88d3834f2e3..23272e70d0e 100644
--- a/src/lib/inference/vllm-station-runtime-receipt.test.ts
+++ b/src/lib/inference/vllm-station-runtime-receipt.test.ts
@@ -6,11 +6,16 @@ import os from "node:os";
import path from "node:path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
-import { DUAL_STATION_VLLM_RUNTIME, type DualStationVllmPlan } from "./vllm-station-cluster";
+import {
+ DUAL_STATION_VLLM_RUNTIME,
+ type DualStationVllmPlan,
+ probeDualStationVllmCapability,
+} from "./vllm-station-cluster";
import {
cleanupInstalledDualStationVllmRuntime,
dualStationVllmRuntimeReceiptPath,
persistDualStationVllmRuntimeReceipt,
+ recoverInstalledDualStationVllmRuntime,
} from "./vllm-station-runtime-receipt";
import {
createDualStationSshBindingFixture,
@@ -29,6 +34,7 @@ beforeEach(() => {
afterEach(() => {
vi.unstubAllEnvs();
+ vi.restoreAllMocks();
vi.resetModules();
sshFixture.cleanup();
fs.rmSync(root, { recursive: true, force: true });
@@ -108,17 +114,77 @@ function plan(): DualStationVllmPlan {
}
describe("dual-Station vLLM runtime rollback receipt", () => {
- it("uses the selected gateway state root", async () => {
+ it("uses the host-global state root across gateway ports", async () => {
+ vi.spyOn(os, "homedir").mockReturnValue(root);
vi.stubEnv("NEMOCLAW_GATEWAY_PORT", "18080");
vi.resetModules();
- const { dualStationVllmRuntimeReceiptPath: selectedReceiptPath } = await import(
- "./vllm-station-runtime-receipt"
- );
+ const firstGateway = await import("./vllm-station-runtime-receipt");
+ const expectedPlan = plan();
+ firstGateway.persistDualStationVllmRuntimeReceipt(expectedPlan);
- expect(selectedReceiptPath()).toBe(
- path.join(os.homedir(), ".nemoclaw", "gateways", "18080", "dual-station-vllm-runtime.json"),
+ vi.stubEnv("NEMOCLAW_GATEWAY_PORT", "18081");
+ vi.resetModules();
+ const secondGateway = await import("./vllm-station-runtime-receipt");
+
+ expect(secondGateway.dualStationVllmRuntimeReceiptPath()).toBe(
+ path.join(root, ".nemoclaw", "dual-station-vllm-runtime.json"),
);
+ expect(
+ secondGateway.recoverInstalledDualStationVllmRuntime({
+ probeCapability: () => ({
+ kind: "ready",
+ plan: expectedPlan,
+ peerModelSnapshot: "ready",
+ }),
+ }),
+ ).toEqual({
+ kind: "ready",
+ plan: expectedPlan,
+ });
+ });
+
+ it("recovers a released non-default-gateway receipt without duplicating ownership", async () => {
+ vi.spyOn(os, "homedir").mockReturnValue(root);
+ const legacyStateDir = path.join(root, ".nemoclaw", "gateways", "18080");
+ const expectedPlan = plan();
+ persistDualStationVllmRuntimeReceipt(expectedPlan, { stateDir: legacyStateDir });
+
+ expect(
+ recoverInstalledDualStationVllmRuntime({
+ probeCapability: () => ({
+ kind: "ready",
+ plan: expectedPlan,
+ peerModelSnapshot: "ready",
+ }),
+ }),
+ ).toEqual({ kind: "ready", plan: expectedPlan });
+
+ persistDualStationVllmRuntimeReceipt(expectedPlan, {
+ probeCapability: () => ({
+ kind: "ready",
+ plan: expectedPlan,
+ peerModelSnapshot: "ready",
+ }),
+ });
+ expect(fs.existsSync(dualStationVllmRuntimeReceiptPath())).toBe(false);
+ expect(fs.existsSync(dualStationVllmRuntimeReceiptPath(legacyStateDir))).toBe(true);
+ });
+
+ it("rejects ambiguous receipts across shared and legacy gateway roots", () => {
+ vi.spyOn(os, "homedir").mockReturnValue(root);
+ const expectedPlan = plan();
+ const sharedStateDir = path.join(root, ".nemoclaw");
+ const legacyStateDir = path.join(sharedStateDir, "gateways", "18080");
+ persistDualStationVllmRuntimeReceipt(expectedPlan, { stateDir: sharedStateDir });
+ persistDualStationVllmRuntimeReceipt(expectedPlan, { stateDir: legacyStateDir });
+ const probeCapability = vi.fn();
+
+ expect(recoverInstalledDualStationVllmRuntime({ probeCapability })).toEqual({
+ kind: "unsafe",
+ reason: "Multiple dual-Station vLLM runtime receipts were found; ownership is ambiguous",
+ });
+ expect(probeCapability).not.toHaveBeenCalled();
});
it("writes a private cleanup receipt and removes both exact containers before retiring it", async () => {
@@ -205,6 +271,99 @@ describe("dual-Station vLLM runtime rollback receipt", () => {
expect(fs.readFileSync(receiptPath, "utf8")).toBe(original);
});
+ it("recovers the exact installed pair through its private peer binding", () => {
+ const expectedPlan = plan();
+ persistDualStationVllmRuntimeReceipt(expectedPlan, { stateDir });
+ let recoveredEnv: NodeJS.ProcessEnv | undefined;
+ const probeCapability = vi.fn(
+ (options: Parameters[0]) => {
+ recoveredEnv = options?.env;
+ return {
+ kind: "ready" as const,
+ plan: expectedPlan,
+ peerModelSnapshot: "ready" as const,
+ };
+ },
+ );
+
+ expect(recoverInstalledDualStationVllmRuntime({ stateDir, probeCapability })).toEqual({
+ kind: "ready",
+ plan: expectedPlan,
+ });
+ expect(probeCapability).toHaveBeenCalledOnce();
+ expect(recoveredEnv?.NEMOCLAW_DGX_STATION_PEER).toBe(expectedPlan.peerSshBinding.peerTarget);
+ expect(recoveredEnv?.NEMOCLAW_DGX_STATION_SSH_BINDING).toMatch(/^[A-Za-z0-9_-]+$/);
+ });
+
+ it("reports that no installed pair can be recovered without a receipt", () => {
+ const probeCapability = vi.fn();
+
+ expect(recoverInstalledDualStationVllmRuntime({ stateDir, probeCapability })).toEqual({
+ kind: "not-installed",
+ });
+ expect(probeCapability).not.toHaveBeenCalled();
+ });
+
+ it("rejects a malformed recovery receipt before probing the peer", () => {
+ fs.mkdirSync(stateDir, { mode: 0o700 });
+ fs.writeFileSync(dualStationVllmRuntimeReceiptPath(stateDir), "{\n", { mode: 0o600 });
+ const probeCapability = vi.fn();
+
+ expect(recoverInstalledDualStationVllmRuntime({ stateDir, probeCapability })).toMatchObject({
+ kind: "unsafe",
+ reason: expect.stringContaining("malformed"),
+ });
+ expect(probeCapability).not.toHaveBeenCalled();
+ });
+
+ it("rejects recovered peer evidence that can no longer be revalidated", () => {
+ persistDualStationVllmRuntimeReceipt(plan(), { stateDir });
+
+ expect(
+ recoverInstalledDualStationVllmRuntime({
+ stateDir,
+ probeCapability: () => ({
+ kind: "unavailable",
+ code: "peer-ssh-config-unsafe",
+ reason: "installer-qualified Station SSH binding is invalid or changed",
+ }),
+ }),
+ ).toEqual({
+ kind: "unsafe",
+ reason:
+ "could not revalidate the managed pair: installer-qualified Station SSH binding is invalid or changed",
+ });
+ });
+
+ it("rejects a recovered pair whose immutable runtime identity changed", () => {
+ const expectedPlan = plan();
+ persistDualStationVllmRuntimeReceipt(expectedPlan, { stateDir });
+ const changedPlan = {
+ ...expectedPlan,
+ peer: {
+ ...expectedPlan.peer,
+ gpu: {
+ ...expectedPlan.peer.gpu,
+ uuid: "GPU-bbbbbbbb-cccc-dddd-eeee-ffffffffffff",
+ },
+ },
+ };
+
+ expect(
+ recoverInstalledDualStationVllmRuntime({
+ stateDir,
+ probeCapability: () => ({
+ kind: "ready",
+ plan: changedPlan,
+ peerModelSnapshot: "ready",
+ }),
+ }),
+ ).toEqual({
+ kind: "unsafe",
+ reason: "could not revalidate the managed pair: managed runtime identity changed",
+ });
+ });
+
it("refuses a symbolic-link receipt before peer probing or cleanup", async () => {
const receiptPath = dualStationVllmRuntimeReceiptPath(stateDir);
fs.mkdirSync(stateDir, { mode: 0o700 });
diff --git a/src/lib/inference/vllm-station-runtime-receipt.ts b/src/lib/inference/vllm-station-runtime-receipt.ts
index 2d5674e8225..3093edb6f37 100644
--- a/src/lib/inference/vllm-station-runtime-receipt.ts
+++ b/src/lib/inference/vllm-station-runtime-receipt.ts
@@ -5,7 +5,7 @@ import fs from "node:fs";
import os from "node:os";
import path from "node:path";
-import { getNemoclawStateRoot } from "../state/state-root";
+import { GATEWAYS_SUBDIR, STATE_DIR_NAME } from "../state/state-root";
import { ensureLocalAdapterStateDir } from "./local-adapter-lifecycle";
import { buildLocalDualStationDockerEnv } from "./vllm-docker-env";
import {
@@ -17,7 +17,10 @@ import {
cleanupDualStationManagedVllm,
dualStationVllmClusterId,
} from "./vllm-station-cluster-lifecycle";
-import { DUAL_STATION_VLLM_RUNTIME_RECEIPT_FILE } from "./vllm-station-runtime-receipt-path";
+import {
+ DUAL_STATION_VLLM_RUNTIME_RECEIPT_FILE,
+ discoverDualStationVllmRuntimeReceiptStateDirs,
+} from "./vllm-station-runtime-receipt-path";
import {
clearDualStationSshBinding,
copyDualStationSshBinding,
@@ -58,8 +61,15 @@ export type DualStationVllmRuntimeCleanupResult =
| { kind: "not-installed" }
| { kind: "removed"; removedContainerIds: string[] };
+export type DualStationVllmRuntimeRecoveryResult =
+ | { kind: "not-installed" }
+ | { kind: "ready"; plan: DualStationVllmPlan }
+ | { kind: "unsafe"; reason: string };
+
function defaultStateDir(): string {
- return getNemoclawStateRoot(os.homedir());
+ // The managed pair and its API key are host-global rather than gateway-scoped.
+ // Every gateway must therefore recover the same cleanup ownership receipt.
+ return path.join(os.homedir(), STATE_DIR_NAME);
}
export function dualStationVllmRuntimeReceiptPath(stateDir = defaultStateDir()): string {
@@ -179,6 +189,24 @@ function loadReceipt(
}
}
+function loadInstalledReceipt(
+ options: Pick,
+): { receipt: DualStationVllmRuntimeReceipt; stateDir: string } | null {
+ const stateDirs = options.stateDir
+ ? [options.stateDir]
+ : discoverDualStationVllmRuntimeReceiptStateDirs(defaultStateDir(), GATEWAYS_SUBDIR);
+ const matches = stateDirs.flatMap((stateDir) => {
+ const receipt = loadReceipt({ stateDir });
+ return receipt ? [{ receipt, stateDir }] : [];
+ });
+ if (matches.length > 1) {
+ throw new Error(
+ "Multiple dual-Station vLLM runtime receipts were found; ownership is ambiguous",
+ );
+ }
+ return matches[0] ?? null;
+}
+
function fsyncDirectory(directory: string): void {
const fd = fs.openSync(directory, fs.constants.O_RDONLY);
try {
@@ -225,10 +253,11 @@ export function persistDualStationVllmRuntimeReceipt(
plan: DualStationVllmPlan,
options: DualStationVllmRuntimeReceiptOptions = {},
): void {
- const stateDir = options.stateDir ?? defaultStateDir();
+ const installed = loadInstalledReceipt(options);
+ const stateDir = installed?.stateDir ?? options.stateDir ?? defaultStateDir();
ensureLocalAdapterStateDir(stateDir);
const clusterId = dualStationVllmClusterId(plan);
- const existing = loadReceipt({ stateDir });
+ const existing = installed?.receipt ?? null;
if (
existing &&
(existing.peerTarget !== plan.peerSshBinding.peerTarget ||
@@ -238,6 +267,13 @@ export function persistDualStationVllmRuntimeReceipt(
) {
throw new Error("A different managed dual-Station runtime receipt already owns rollback state");
}
+ if (existing) {
+ const recovered = probeReceiptPlan(existing, options);
+ if (!recovered.ok) {
+ throw new Error(`Could not revalidate the managed dual-Station pair: ${recovered.reason}`);
+ }
+ return;
+ }
const receiptPath = dualStationVllmRuntimeReceiptPath(stateDir);
const runtimeBinding = copyDualStationSshBinding(receiptPath, plan.peerSshBinding);
writeReceipt(
@@ -260,16 +296,10 @@ function clearReceipt(stateDir: string): void {
fsyncDirectory(stateDir);
}
-/**
- * Remove both exact owned containers before ordinary uninstall can retire the
- * controller state required to reach the worker.
- */
-export async function cleanupInstalledDualStationVllmRuntime(
- options: DualStationVllmRuntimeReceiptOptions = {},
-): Promise {
- const stateDir = options.stateDir ?? defaultStateDir();
- const receipt = loadReceipt({ stateDir });
- if (!receipt) return { kind: "not-installed" };
+function probeReceiptPlan(
+ receipt: DualStationVllmRuntimeReceipt,
+ options: DualStationVllmRuntimeReceiptOptions,
+): { ok: true; plan: DualStationVllmPlan } | { ok: false; reason: string } {
const capability = (options.probeCapability ?? probeDualStationVllmCapability)({
env: buildLocalDualStationDockerEnv({
[NEMOCLAW_DGX_STATION_PEER_ENV]: receipt.peerTarget,
@@ -279,19 +309,55 @@ export async function cleanupInstalledDualStationVllmRuntime(
if (capability.kind !== "ready") {
const reason =
capability.kind === "unavailable" ? capability.reason : "runtime peer is not configured";
- throw new Error(`Could not revalidate the managed dual-Station pair: ${reason}`);
+ return { ok: false, reason };
}
if (
dualStationVllmClusterId(capability.plan) !== receipt.clusterId ||
capability.plan.local.gpu.uuid !== receipt.localGpuUuid ||
capability.plan.peer.gpu.uuid !== receipt.peerGpuUuid
) {
- throw new Error("Managed dual-Station runtime identity changed; refusing pair cleanup");
+ return { ok: false, reason: "managed runtime identity changed" };
+ }
+ return { ok: true, plan: capability.plan };
+}
+
+/** Recover and revalidate exact managed-pair ownership for a later onboarding process. */
+export function recoverInstalledDualStationVllmRuntime(
+ options: DualStationVllmRuntimeReceiptOptions = {},
+): DualStationVllmRuntimeRecoveryResult {
+ let installed: { receipt: DualStationVllmRuntimeReceipt; stateDir: string } | null;
+ try {
+ installed = loadInstalledReceipt(options);
+ } catch (error) {
+ return { kind: "unsafe", reason: (error as Error).message };
+ }
+ if (!installed) return { kind: "not-installed" };
+ const recovered = probeReceiptPlan(installed.receipt, options);
+ return recovered.ok
+ ? { kind: "ready", plan: recovered.plan }
+ : {
+ kind: "unsafe",
+ reason: `could not revalidate the managed pair: ${recovered.reason}`,
+ };
+}
+
+/**
+ * Remove both exact owned containers before ordinary uninstall can retire the
+ * controller state required to reach the worker.
+ */
+export async function cleanupInstalledDualStationVllmRuntime(
+ options: DualStationVllmRuntimeReceiptOptions = {},
+): Promise {
+ const installed = loadInstalledReceipt(options);
+ if (!installed) return { kind: "not-installed" };
+ const recovered = probeReceiptPlan(installed.receipt, options);
+ if (!recovered.ok) {
+ throw new Error(`Could not revalidate the managed dual-Station pair: ${recovered.reason}`);
}
const cleanup = await (options.cleanupManagedVllm ?? cleanupDualStationManagedVllm)(
- capability.plan,
+ recovered.plan,
);
if (!cleanup.ok) throw new Error(cleanup.reason);
- clearReceipt(stateDir);
+ clearReceipt(installed.stateDir);
return { kind: "removed", removedContainerIds: cleanup.removedContainerIds };
}
diff --git a/src/lib/inference/vllm.ts b/src/lib/inference/vllm.ts
index 4a6b74b8f24..75effadb70b 100644
--- a/src/lib/inference/vllm.ts
+++ b/src/lib/inference/vllm.ts
@@ -69,7 +69,10 @@ import {
withDualStationManagedVllmLifecycle,
} from "./vllm-station-cluster-lifecycle";
import { stageDualStationModelSnapshot } from "./vllm-station-model-staging";
-import { persistDualStationVllmRuntimeReceipt } from "./vllm-station-runtime-receipt";
+import {
+ persistDualStationVllmRuntimeReceipt,
+ recoverInstalledDualStationVllmRuntime,
+} from "./vllm-station-runtime-receipt";
import {
findUnwritableModelCachePath,
formatStorageBytes,
@@ -894,34 +897,53 @@ export type PersistConfiguredDualStationVllmRuntimeResult =
*/
export async function persistConfiguredDualStationVllmRuntimeReceipt(): Promise {
const configuredPeer = String(process.env[NEMOCLAW_DGX_STATION_PEER_ENV] ?? "").trim();
- if (!configuredPeer) {
- return {
- ok: false,
- reason: "the managed dual-Station peer configuration is missing",
- };
- }
-
- const capability = probeDualStationVllmCapability();
- if (capability.kind !== "ready") {
- const reason =
- capability.kind === "unavailable"
- ? capability.reason
- : "the configured dual-Station peer disappeared";
- return { ok: false, reason };
+ let configuredPlan: DualStationVllmPlan | null = null;
+ if (configuredPeer) {
+ const capability = probeDualStationVllmCapability();
+ if (capability.kind !== "ready") {
+ const reason =
+ capability.kind === "unavailable"
+ ? capability.reason
+ : "the configured dual-Station peer disappeared";
+ return { ok: false, reason };
+ }
+ configuredPlan = capability.plan;
}
try {
return await withDualStationManagedVllmLifecycle(async () => {
- const preflight = preflightDualStationManagedVllm(capability.plan);
+ let plan: DualStationVllmPlan;
+ let receiptAlreadyPersisted = false;
+ if (configuredPlan) {
+ plan = configuredPlan;
+ } else {
+ const recovered = recoverInstalledDualStationVllmRuntime();
+ if (recovered.kind === "not-installed") {
+ return {
+ ok: false,
+ reason: "the managed dual-Station peer configuration is missing",
+ };
+ }
+ if (recovered.kind === "unsafe") {
+ return {
+ ok: false,
+ reason: `the managed dual-Station cleanup receipt is unsafe: ${recovered.reason}`,
+ };
+ }
+ plan = recovered.plan;
+ receiptAlreadyPersisted = true;
+ }
+ const preflight = preflightDualStationManagedVllm(plan);
if (!preflight.ok) return { ok: false, reason: preflight.reason };
- if (!areDualStationManagedVllmContainersRunning(capability.plan)) {
+ if (!areDualStationManagedVllmContainersRunning(plan)) {
return {
ok: false,
- reason: "the managed dual-Station containers changed before receipt persistence",
+ reason: "the managed dual-Station containers changed before cleanup ownership validation",
};
}
+ if (receiptAlreadyPersisted) return { ok: true, persisted: true };
try {
- persistDualStationVllmRuntimeReceipt(capability.plan);
+ persistDualStationVllmRuntimeReceipt(plan);
} catch (error) {
return { ok: false, reason: (error as Error).message };
}