Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 9 additions & 4 deletions docs/manage-sandboxes/run-sandboxes.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ When the default port is already held by another sandbox, `$$nemoclaw onboard` s

<AgentOnly variant="hermes">
Each Hermes sandbox also needs its own OpenAI-compatible API port.
When another sandbox or a host listener already holds the default API port `8642`, `$$nemoclaw onboard` scans ports `8642` through `8652`, uses the next free port, and records it for the sandbox.
When creating a Hermes sandbox, if another sandbox or a host listener already holds the default API port `8642`, `$$nemoclaw onboard` scans ports `8642` through `8652`, uses the next free port, and records it for the sandbox.
</AgentOnly>

<AgentOnly variant="openclaw,hermes">
Expand All @@ -65,9 +65,14 @@ Gateway and dashboard cleanup is scoped by sandbox name and port.
A later onboarding run that uses a different `NEMOCLAW_GATEWAY_PORT` or `--control-ui-port` does not tear down the first sandbox's gateway or dashboard forward.
</AgentOnly>

<AgentOnly variant="openclaw">
If re-onboarding finds the registered dashboard port already bound, NemoClaw verifies that the listener is the exact OpenShell forward for that sandbox and reuses it without restarting the sandbox.
If ownership cannot be proved, onboarding fails closed and reports the listener conflict.
<AgentOnly variant="openclaw,hermes">
Repeated onboarding and resume keep the sandbox's registered dashboard port and restore a missing forward on that port.
If the port is already bound, NemoClaw verifies that the listener is the exact OpenShell forward for that sandbox and reuses it without restarting the sandbox.
Ports registered to another sandbox, foreign listeners, or unverified ownership stop onboarding with a port conflict error.
</AgentOnly>

<AgentOnly variant="hermes">
Hermes also retains or restores its registered OpenAI-compatible API forward using the same ownership checks.
</AgentOnly>

<AgentOnly variant="deepagents">
Expand Down
50 changes: 50 additions & 0 deletions src/lib/adapters/openshell/forward-service.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -252,6 +252,7 @@ describe("OpenShell forward service", () => {

it("detaches the OpenShell child and waits for its local port", () => {
const unref = vi.fn();
const verifyReady = vi.fn(() => expect(unref).not.toHaveBeenCalled());
const spawnDetached = vi.fn(() => ({ unref }));
const terminateProcessTree = vi.fn();
let probes = 0;
Expand All @@ -261,6 +262,7 @@ describe("OpenShell forward service", () => {
sleep: () => {},
spawnDetached,
terminateProcessTree,
verifyReady,
timeoutMs: 1_000,
});

Expand All @@ -270,6 +272,7 @@ describe("OpenShell forward service", () => {
expect.any(Object),
);
expect(unref).toHaveBeenCalledOnce();
expect(verifyReady).toHaveBeenCalledOnce();
expect(terminateProcessTree).not.toHaveBeenCalled();
});

Expand Down Expand Up @@ -304,6 +307,53 @@ describe("OpenShell forward service", () => {
expect(spawnDetached).not.toHaveBeenCalled();
});

it.each([
{
cleanup: "terminated",
terminate: () => {},
remains: false,
expected: /Forward ownership changed/u,
},
{
cleanup: "termination-failed",
terminate: () => {
throw new Error("Child remained alive");
},
remains: false,
expected: ForwardServiceStartupCleanupError,
},
{
cleanup: "listener-remained",
terminate: () => {},
remains: true,
expected: ForwardServiceStartupCleanupError,
},
])(
"rejects failed startup verification with cleanup $cleanup",
({ terminate, remains, expected }) => {
const child = { pid: detachedChildPid, unref: vi.fn() };
const verificationError = new Error("Forward ownership changed");
const terminateProcessTree = vi.fn(terminate);
const launch = () =>
launchForwardService(target, {
isReachable: vi
.fn()
.mockReturnValueOnce(false)
.mockReturnValueOnce(true)
.mockReturnValue(remains),
spawnDetached: () => child,
terminateProcessTree,
verifyReady: () => {
throw verificationError;
},
});

expect(launch).toThrow(expected);
expect(terminateProcessTree).toHaveBeenCalledExactlyOnceWith(child);
expect(child.unref).not.toHaveBeenCalled();
},
);

it("terminates a detached service that does not bind before the deadline", () => {
const child = { pid: detachedChildPid, unref: vi.fn() };
const terminateProcessTree = vi.fn();
Expand Down
14 changes: 11 additions & 3 deletions src/lib/adapters/openshell/forward-service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@ export interface ForwardServiceLaunchOptions {
environment: NodeJS.ProcessEnv,
) => ForwardServiceChild;
readonly terminateProcessTree?: (child: ForwardServiceChild) => void;
/** Verify the bound forward before releasing the child from startup cleanup. */
readonly verifyReady?: () => void;
readonly timeoutMs?: number;
}

Expand Down Expand Up @@ -427,16 +429,22 @@ export function launchForwardService(
const sleep =
options.sleep ?? ((milliseconds: number) => Atomics.wait(sleepBuffer, 0, 0, milliseconds));
const deadline = Date.now() + (options.timeoutMs ?? START_TIMEOUT_MS);
let startupError = new Error(
`OpenShell forward service did not bind ${target.localHost}:${String(target.localPort)}`,
);
while (Date.now() < deadline) {
if (isReachable(target.localPort)) {
try {
options.verifyReady?.();
} catch (error) {
startupError = error instanceof Error ? error : new Error(String(error));
break;
}
child.unref();
return;
}
sleep(POLL_INTERVAL_MS);
}
const startupError = new Error(
`OpenShell forward service did not bind ${target.localHost}:${String(target.localPort)}`,
);
try {
(options.terminateProcessTree ?? terminateForwardServiceProcessTree)(child);
if (isReachable(target.localPort)) {
Expand Down
42 changes: 41 additions & 1 deletion src/lib/onboard/agent-dashboard-forward.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,27 @@
import { afterEach, describe, expect, it, vi } from "vitest";

import { ensureAgentDashboardForward } from "./agent-dashboard-forward";
import { canReuseDashboardForwardForAgent } from "./dashboard-runtime";

describe("agent dashboard forward reuse eligibility", () => {
it.each([
{ agent: undefined, expected: true },
{ agent: null, expected: true },
{ agent: { name: "openclaw" }, expected: true },
{ agent: { name: "hermes" }, expected: true },
{ agent: { name: "pi" }, expected: false },
{ agent: { name: "deepagents" }, expected: false },
])("permits reuse=$expected for $agent", ({ agent, expected }) => {
expect(canReuseDashboardForwardForAgent(agent)).toBe(expected);
});
});

describe("ensureAgentDashboardForward", () => {
afterEach(() => {
delete process.env.CHAT_UI_URL;
});

it("launches each declared host-forward port", async () => {
it.each([false, true])("forwards Hermes ports with reuse=%s", async (reuseExistingForward) => {
const ensureDashboardForward = vi.fn((_sandboxName, chatUiUrl = "http://127.0.0.1:18789") => {
const parsed = new URL(chatUiUrl);
return Number(parsed.port);
Expand All @@ -25,14 +39,17 @@ describe("ensureAgentDashboardForward", () => {
},
ensureDashboardForward,
hermesApiPort: 8642,
reuseExistingForward,
}),
).toBe(18789);

expect(ensureDashboardForward).toHaveBeenNthCalledWith(1, "hm", "http://127.0.0.1:18789", {
allowPortReallocation: false,
...(reuseExistingForward ? { reuseExistingForward: true } : {}),
});
expect(ensureDashboardForward).toHaveBeenNthCalledWith(2, "hm", "http://127.0.0.1:8642", {
allowPortReallocation: false,
...(reuseExistingForward ? { reuseExistingForward: true } : {}),
});
});

Expand Down Expand Up @@ -221,4 +238,27 @@ describe("ensureAgentDashboardForward", () => {
);
expect(process.env.CHAT_UI_URL).toBe("https://hermes.example.test:9120/ui");
});

it("fails reuse and restores the dashboard URL when the Hermes API forward fails", async () => {
process.env.CHAT_UI_URL = "http://127.0.0.1:18790";
const warn = vi.fn();
const ensureDashboardForward = vi
.fn()
.mockReturnValueOnce(18790)
.mockImplementationOnce(() => {
throw new Error("API forward ownership is unknown");
});
await expect(
ensureAgentDashboardForward({
sandboxName: "hm",
agent: { forwardPort: 18789, forward_ports: [18789, 8642] },
ensureDashboardForward,
hermesApiPort: 8643,
reuseExistingForward: true,
warn,
}),
).rejects.toThrow("API forward ownership is unknown");
expect(process.env.CHAT_UI_URL).toBe("http://127.0.0.1:18790");
expect(warn).not.toHaveBeenCalled();
});
});
13 changes: 7 additions & 6 deletions src/lib/onboard/agent-dashboard-forward.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ export type EnsureDashboardForward = (
chatUiUrl?: string,
options?: {
allowPortReallocation?: boolean;
reuseExistingOpenClawForward?: boolean;
reuseExistingForward?: boolean;
revalidateSandboxIdentity?: (operation: string) => void;
},
) => number;
Expand All @@ -40,7 +40,7 @@ export async function ensureAgentDashboardForward(options: {
/** Host port allocated to this sandbox's OpenAI-compatible API, when it has one. */
hermesApiPort?: number | null;
beforeForwardPort?: (port: number) => Promise<void> | void;
reuseExistingOpenClawForward?: boolean;
reuseExistingForward?: boolean;
revalidateSandboxIdentity?: (operation: string) => void;
warn?: (message: string) => void;
}): Promise<number> {
Expand All @@ -52,7 +52,7 @@ export async function ensureAgentDashboardForward(options: {
controlUiPort,
hermesApiPort,
beforeForwardPort,
reuseExistingOpenClawForward = false,
reuseExistingForward = false,
revalidateSandboxIdentity,
warn = (message: string) => console.warn(message),
} = options;
Expand Down Expand Up @@ -107,7 +107,7 @@ export async function ensureAgentDashboardForward(options: {
await beforeForwardPort?.(agentDashboardPort);
const actualAgentDashboardPort = ensureDashboardForward(sandboxName, requestedDashboardUrl, {
allowPortReallocation: false,
...(reuseExistingOpenClawForward ? { reuseExistingOpenClawForward: true } : {}),
...(reuseExistingForward ? { reuseExistingForward: true } : {}),
...(revalidateIdentity ? { revalidateSandboxIdentity: revalidateIdentity } : {}),
});
if (!usesFixedApiPort) {
Expand All @@ -125,10 +125,11 @@ export async function ensureAgentDashboardForward(options: {
: `http://127.0.0.1:${port}`;
ensureDashboardForward(sandboxName, forwardUrl, {
allowPortReallocation: false,
...(reuseExistingForward ? { reuseExistingForward: true } : {}),
...(revalidateIdentity ? { revalidateSandboxIdentity: revalidateIdentity } : {}),
});
} catch (err) {
if (err === identityFailure) throw err;
if (reuseExistingForward || err === identityFailure) throw err;
warn(
` ! Could not start optional agent port forward ${port}: ${
err instanceof Error ? err.message : String(err)
Expand All @@ -140,7 +141,7 @@ export async function ensureAgentDashboardForward(options: {
revalidateIdentity?.(`report successful dashboard forwarding for sandbox '${sandboxName}'`);
return actualAgentDashboardPort;
} catch (error) {
if (error === identityFailure) {
if (reuseExistingForward || error === identityFailure) {
restoreChatUiUrl();
}
throw error;
Expand Down
30 changes: 30 additions & 0 deletions src/lib/onboard/config-sync.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -275,6 +275,36 @@ describe("sandbox config sync helpers", () => {
expect(fs.statSync(configDir).uid).toBe(process.getuid?.());
});

itUnix("syncs Hermes selection while preserving an existing OpenClaw directory", async () => {
const homeDir = createConfigSyncHome();
const configDir = path.join(homeDir, ".openclaw");
const configFile = path.join(configDir, "openclaw.json");
const hashFile = path.join(configDir, ".config-hash");
fs.mkdirSync(configDir, { mode: 0o750 });
fs.writeFileSync(configFile, "retained user state\n", { mode: 0o640 });
fs.writeFileSync(hashFile, "retained hash\n", { mode: 0o640 });
const modesBefore = [configDir, configFile, hashFile].map(modeBits);
const runConnectScript = vi.fn(async (_name: string, script: string) => {
const { nativeCalls } = runConfigSyncScript(script, homeDir, "1234", "1234", {
validationStatus: 127,
});
expect(nativeCalls).toEqual([]);
});

await runSandboxConfigSync("hermes-reuse", {
getSelectionConfig: () => ({ ...selection, agent: "hermes" }),
runConnectScript,
});

expect(runConnectScript).toHaveBeenCalledOnce();
expect(
JSON.parse(fs.readFileSync(path.join(homeDir, ".nemoclaw", "config.json"), "utf8")),
).toMatchObject({ ...selection, agent: "hermes", onboardedAt: expect.any(String) });
expect(fs.readFileSync(configFile, "utf8")).toBe("retained user state\n");
expect(fs.readFileSync(hashFile, "utf8")).toBe("retained hash\n");
expect([configDir, configFile, hashFile].map(modeBits)).toEqual(modesBefore);
});

itUnix("keeps credential values out of sandbox selection config", () => {
const homeDir = createConfigSyncHome();
const anthropicSelection = {
Expand Down
12 changes: 9 additions & 3 deletions src/lib/onboard/config-sync.ts
Original file line number Diff line number Diff line change
Expand Up @@ -59,9 +59,10 @@ export async function runSandboxConfigSync(
await deps.runConnectScript(sandboxName, script);
}

export function buildSandboxConfigSyncScript(selectionConfig: ProviderSelectionConfig): string {
// Native baseline setup preserves valid routing and creates its own state.
return `
export function buildSandboxConfigSyncScript(
selectionConfig: ProviderSelectionConfig & { agent?: string },
): string {
const writeSelection = `
set -euo pipefail
# OpenShell exec and the OpenClaw gateway can expose different HOME values.
# The managed gateway always reads its NemoClaw state from /sandbox.
Expand All @@ -77,6 +78,11 @@ cat > "$nemoclaw_config" <<'EOF_NEMOCLAW_CFG'
${JSON.stringify(selectionConfig, null, 2)}
EOF_NEMOCLAW_CFG
chmod 600 "$nemoclaw_config"
`.trim();
// Retained Hermes sandboxes can contain an unrelated .openclaw directory.
if (selectionConfig.agent === "hermes") return writeSelection;
// Native baseline setup preserves valid routing and creates its own state.
return `${writeSelection}
config_dir=/sandbox/.openclaw
if [ -d "$config_dir" ]; then
config_dir_owner="$(stat -c '%U' "$config_dir" 2>/dev/null || echo unknown)"
Expand Down
6 changes: 3 additions & 3 deletions src/lib/onboard/dashboard-forward-control.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,18 +5,18 @@ export interface DashboardForwardOptions {
rollbackSandboxOnFailure?: boolean;
gatewayName?: string;
allowPortReallocation?: boolean;
reuseExistingOpenClawForward?: boolean;
reuseExistingForward?: boolean;
revalidateSandboxIdentity?: (operation: string) => void;
}

export function normalizeDashboardForwardOptions(options: DashboardForwardOptions = {}): {
rollbackSandboxOnFailure: boolean;
allowPortReallocation: boolean;
reuseExistingOpenClawForward: boolean;
reuseExistingForward: boolean;
} {
return {
rollbackSandboxOnFailure: options.rollbackSandboxOnFailure === true,
allowPortReallocation: options.allowPortReallocation !== false,
reuseExistingOpenClawForward: options.reuseExistingOpenClawForward === true,
reuseExistingForward: options.reuseExistingForward === true,
};
}
6 changes: 6 additions & 0 deletions src/lib/onboard/dashboard-runtime.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,3 +31,9 @@ export function shouldManageDashboardForAgent(agent: DashboardRuntimeAgent): boo
if (!agent || !isTerminalAgent(agent)) return true;
return getAgentDeclaredForwardPorts(agent).length > 0;
}

export function canReuseDashboardForwardForAgent(
agent: { name: string } | null | undefined,
): boolean {
return agent == null || agent.name === "openclaw" || agent.name === "hermes";
}
Loading
Loading