Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
b3317b5
fix(onboard): reuse existing OpenClaw dashboard forward
sandl99 Sep 6, 2026
d7fbc91
fix(onboard): reconcile reused dashboard forward lifecycle
sandl99 Sep 6, 2026
e68f475
Merge remote-tracking branch 'origin/main' into fix/onboard-dashboard…
sandl99 Sep 6, 2026
485e677
test(e2e): map double onboard lifecycle parity
sandl99 Sep 6, 2026
677bb40
fix(onboard): harden dashboard reuse lifecycle
sandl99 Sep 6, 2026
be631a2
fix(onboard): fence dashboard reuse lifecycle
sandl99 Sep 6, 2026
d2a0dd9
fix(onboard): bind dashboard reuse lifecycle
sandl99 Sep 6, 2026
48d4988
fix(onboard): restore dashboard reuse failures
sandl99 Sep 6, 2026
0ee6c07
fix(onboard): scope dashboard reuse evidence
sandl99 Sep 6, 2026
2157f01
fix(onboard): verify owned dashboard forward
sandl99 Sep 6, 2026
81c4d74
fix(onboard): support Linux forward ownership probe
sandl99 Sep 7, 2026
1c79209
fix(onboard): reuse owned dashboard forward during finalization
sandl99 Sep 7, 2026
940aecd
test(e2e): preserve double-onboard lifecycle setup
sandl99 Sep 7, 2026
09cd931
fix(onboard): reuse dashboard forward without restart
sandl99 Sep 7, 2026
72c5d12
fix(onboard): harden dashboard forward ownership
sandl99 Sep 7, 2026
eaf34e5
Merge branch 'main' into fix/onboard-dashboard-forward-reuse-11074-si…
sandl99 Sep 7, 2026
ca7a69f
fix(onboard): complete forward ownership checks
sandl99 Sep 7, 2026
6c7a0a0
test(e2e): prove forward reuse across resume
sandl99 Sep 7, 2026
46d405b
test(e2e): isolate double-onboard reuse evidence
sandl99 Sep 7, 2026
8cad53e
Merge branch 'main' into fix/onboard-dashboard-forward-reuse-11074-si…
sandl99 Sep 7, 2026
bd210d1
test(onboard): model owned forward fixtures
sandl99 Sep 7, 2026
2c265bb
test(onboard): isolate CI host dependencies
sandl99 Sep 7, 2026
766105a
fix(onboard): scope forward ownership to IPv4
sandl99 Sep 7, 2026
1c1b1a1
Merge branch 'main' into fix/onboard-dashboard-forward-reuse-11074-si…
sandl99 Sep 7, 2026
7d4408d
merge(main): resolve PR conflicts
sandl99 Sep 8, 2026
d7d8f82
Merge branch 'main' into fix/onboard-dashboard-forward-reuse-11074-si…
sandl99 Sep 8, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions ci/e2e-assertion-budget.json
Original file line number Diff line number Diff line change
Expand Up @@ -15,26 +15,26 @@
"testFileCount": 86,
"liveFileCount": 222,
"direct": {
"expectCalls": 1881,
"matcherAssertions": 1850,
"expectCalls": 1878,
"matcherAssertions": 1847,
"nodeAssertions": 100,
"namedAssertionHelpers": 624,
"failCalls": 8,
"throwGuards": 87,
"objectFieldAssertions": 245,
"assertionPoints": 2914,
"assertionPoints": 2911,
"generatedProbeBlocks": 135,
"generatedProbeConditions": 353
},
"unique": {
"expectCalls": 2373,
"matcherAssertions": 2337,
"expectCalls": 2370,
"matcherAssertions": 2334,
"nodeAssertions": 119,
"namedAssertionHelpers": 927,
"failCalls": 38,
"throwGuards": 633,
"objectFieldAssertions": 348,
"assertionPoints": 4402,
"assertionPoints": 4399,
"generatedProbeBlocks": 290,
"generatedProbeConditions": 975
},
Expand All @@ -60,7 +60,7 @@
"test/e2e/live/cron-preflight-inference-local.test.ts": [8,8,8,8,1],
"test/e2e/live/dashboard-remote-bind.test.ts": [17,15,17,17,3],
"test/e2e/live/device-auth-health.test.ts": [13,18,13,21,0],
"test/e2e/live/double-onboard.test.ts": [88,97,88,97,0],
"test/e2e/live/double-onboard.test.ts": [85,94,85,94,0],
"test/e2e/live/external-gateway-health.test.ts": [4,5,4,11,0],
"test/e2e/live/full-e2e.test.ts": [30,34,39,72,7],
"test/e2e/live/gateway-guard-recovery.test.ts": [48,54,48,57,3],
Expand Down
5 changes: 5 additions & 0 deletions docs/manage-sandboxes/run-sandboxes.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,11 @@ Gateway and dashboard cleanup is scoped by sandbox name and port.
A later onboarding run that uses a different `NEMOCLAW_GATEWAY_PORT` or `--control-ui-port` does not tear down the first sandbox's gateway or dashboard forward.
</AgentOnly>

<AgentOnly variant="openclaw">
If re-onboarding finds the registered dashboard port already bound, NemoClaw verifies that the listener is the exact OpenShell forward for that sandbox and reuses it without restarting the sandbox.
If ownership cannot be proved, onboarding fails closed and reports the listener conflict.
</AgentOnly>

<AgentOnly variant="deepagents">
If you intentionally run separate OpenShell gateways on the same host, set a different `NEMOCLAW_GATEWAY_PORT` before each onboarding run.
NemoClaw isolates the gateway name and local state by port so one port-specific gateway does not replace another.
Expand Down
27 changes: 27 additions & 0 deletions src/lib/actions/sandbox/forward-recovery-declared-ports.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,13 @@ const mocks = vi.hoisted(() => ({
getSandbox: vi.fn(),
getHermesDashboardRecoveryConfig: vi.fn(() => null),
isLocalForwardReachable: vi.fn(() => true),
isForwardServiceListenerOwner: vi.fn(() => true),
launchForwardService: vi.fn(),
}));

vi.mock("../../adapters/openshell/forward-service", async (importOriginal) => ({
...(await importOriginal<typeof import("../../adapters/openshell/forward-service")>()),
isForwardServiceListenerOwner: mocks.isForwardServiceListenerOwner,
launchForwardService: mocks.launchForwardService,
}));

Expand Down Expand Up @@ -63,6 +65,7 @@ beforeEach(() => {
vi.unstubAllEnvs();
mocks.runOpenshell.mockReturnValue({ status: 0 });
mocks.isLocalForwardReachable.mockReturnValue(true);
mocks.isForwardServiceListenerOwner.mockReturnValue(true);
mocks.launchForwardService.mockImplementation(() => {
mocks.isLocalForwardReachable.mockReturnValue(true);
});
Expand All @@ -82,9 +85,33 @@ describe("ensureDeclaredAgentForwardPortsHealthy", { timeout: 30_000 }, () => {
const { ensureSandboxPortForward } = await import("./forward-recovery");

expect(ensureSandboxPortForward("remote-box")).toBe(true);
expect(mocks.isForwardServiceListenerOwner).toHaveBeenCalledWith({
executable: "/usr/local/bin/openshell",
gatewayName: "nemoclaw",
workspace: "default",
sandboxName: "remote-box",
localHost: "0.0.0.0",
localPort: 18_789,
targetHost: "127.0.0.1",
targetPort: 18_789,
});
expect(mocks.launchForwardService).not.toHaveBeenCalled();
});

it("fails closed when reachable direct service ownership cannot be proved", async () => {
mocks.getSandbox.mockReturnValue({ agent: "openclaw", dashboardPort: 18_789 });
mocks.captureOpenshell.mockReturnValue(forwardList([]));
mocks.isForwardServiceListenerOwner.mockReturnValue(false);
mocks.launchForwardService.mockImplementation(() => {
throw new Error("host port is occupied");
});
const { ensureSandboxPortForward } = await import("./forward-recovery");

expect(ensureSandboxPortForward("foreign-listener")).toBe(false);
expect(mocks.isForwardServiceListenerOwner).toHaveBeenCalledOnce();
expect(mocks.launchForwardService).toHaveBeenCalledOnce();
});
Comment thread
sandl99 marked this conversation as resolved.

it("does not demand the manifest dashboard port from a sandbox that owns a different dashboard port (#8543)", async () => {
mocks.getSandbox.mockReturnValue({
agent: "hermes",
Expand Down
37 changes: 25 additions & 12 deletions src/lib/actions/sandbox/forward-recovery.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import {
withSelectedOpenShellCommandOptions,
} from "../../adapters/openshell/command-argv";
import {
createForwardServiceTarget,
isForwardServiceListenerOwner,
launchForwardService,
type ForwardServiceTarget,
} from "../../adapters/openshell/forward-service";
Expand Down Expand Up @@ -178,16 +180,16 @@ function forwardServiceTarget(
expectedBind = "127.0.0.1",
workspace = "default",
): ForwardServiceTarget {
return {
executable,
gatewayName,
workspace,
sandboxName,
localHost: expectedBind === "0.0.0.0" ? ("0.0.0.0" as const) : ("127.0.0.1" as const),
localPort: port,
targetHost: "127.0.0.1",
targetPort: port,
};
return createForwardServiceTarget(
{
executable,
gatewayName,
workspace,
sandboxName,
localHost: expectedBind === "0.0.0.0" ? "0.0.0.0" : "127.0.0.1",
},
port,
);
}

function isValidPort(value: unknown): value is number {
Expand Down Expand Up @@ -350,7 +352,7 @@ export function isSandboxForwardHealthy(
export function isSandboxPortForwardHealthy(
sandboxName: string,
port: number,
_expectedBind?: string,
expectedBind?: string,
runtimeSelection?: OpenShellRuntimeSelection,
): SandboxForwardHealth {
const sandbox = registry.getSandbox(sandboxName);
Expand All @@ -376,7 +378,18 @@ export function isSandboxPortForwardHealthy(
) {
return false;
}
return true;
const executable = resolveOpenshell();
if (!executable) return false;
return isForwardServiceListenerOwner(
forwardServiceTarget(
executable,
gatewayName,
sandboxName,
port,
expectedBind ?? "127.0.0.1",
runtimeSelection?.workspace ?? "default",
),
);
}

export function ensureSandboxPortForwardForPort(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@

import { afterEach, describe, expect, it, vi } from "vitest";

import * as forwardService from "../../adapters/openshell/forward-service";
import * as openshellResolve from "../../adapters/openshell/resolve";
import * as openshellRuntime from "../../adapters/openshell/runtime";
import * as agentRuntime from "../../agent/runtime";
import * as registry from "../../state/registry";
Expand Down Expand Up @@ -36,6 +38,8 @@ function mockOpenClawSandbox(sandboxName: string): void {

function mockRecoveredForward(_sandboxName: string): void {
vi.spyOn(forwardHealth, "isLocalForwardReachable").mockReturnValue(true);
vi.spyOn(forwardService, "isForwardServiceListenerOwner").mockReturnValue(true);
vi.spyOn(openshellResolve, "resolveOpenshell").mockReturnValue("/usr/bin/openshell");
vi.spyOn(openshellRuntime, "captureOpenshell").mockReturnValue({
status: 0,
output: "SANDBOX BIND PORT PID STATUS",
Expand Down
152 changes: 151 additions & 1 deletion src/lib/adapters/openshell/forward-service.test.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,15 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { describe, expect, it, vi } from "vitest";
import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
import os from "node:os";
import path from "node:path";

import { afterEach, describe, expect, it, vi } from "vitest";

import {
buildForwardServiceArgs,
isForwardServiceListenerOwner,
launchForwardService,
type ForwardServiceTarget,
} from "./forward-service";
Expand All @@ -20,6 +25,51 @@ const target: ForwardServiceTarget = {
targetPort: 18_789,
};

const ownerTarget: ForwardServiceTarget = { ...target, executable: process.execPath };
const temporaryDirectories: string[] = [];

function createLinuxOwnerFixture(actualExecutable?: string) {
const root = mkdtempSync(path.join(os.tmpdir(), "nemoclaw-forward-owner-"));
temporaryDirectories.push(root);
const procRoot = path.join(root, "proc");
const binRoot = path.join(root, "bin");
mkdirSync(path.join(procRoot, "net"), { recursive: true });
mkdirSync(path.join(procRoot, "4321", "fd"), { recursive: true });
mkdirSync(path.join(procRoot, "9876", "fd"), { recursive: true });
mkdirSync(binRoot);
const executable = path.join(binRoot, "openshell");
const runtime = actualExecutable ? path.join(binRoot, actualExecutable) : executable;
writeFileSync(executable, "");
writeFileSync(runtime, "");
writeFileSync(
path.join(procRoot, "net", "tcp"),
" 0: 0100007F:4965 00000000:0000 0A 00000000:00000000 00:00000000 00000000 998 0 12345 1\n",
);
writeFileSync(
path.join(procRoot, "net", "tcp6"),
" 1: 00000000000000000000000001000000:4965 00000000000000000000000000000000:0000 0A 00000000:00000000 00:00000000 00000000 998 0 67890 1\n",
);
symlinkSync("socket:[12345]", path.join(procRoot, "4321", "fd", "7"));
symlinkSync("socket:[67890]", path.join(procRoot, "9876", "fd", "8"));
symlinkSync(runtime, path.join(procRoot, "4321", "exe"));
return { procRoot, target: { ...target, executable } };
}

function darwinOwnerProbe(commandLine: string, finalListener = "4321\n") {
return vi
.fn()
.mockReturnValueOnce({ status: 0, stdout: "4321\n" })
.mockReturnValueOnce({ status: 0, stdout: `p4321\nftxt\nn${process.execPath}\n` })
.mockReturnValueOnce({ status: 0, stdout: commandLine })
.mockReturnValueOnce({ status: 0, stdout: finalListener });
}

afterEach(() => {
for (const directory of temporaryDirectories.splice(0)) {
rmSync(directory, { recursive: true, force: true });
}
});

describe("OpenShell forward service", () => {
it("builds the direct ForwardTcp command with explicit gateway authority", () => {
expect(buildForwardServiceArgs(target)).toEqual([
Expand All @@ -45,6 +95,106 @@ describe("OpenShell forward service", () => {
);
});

it("proves the exact direct ForwardTcp listener before reuse", () => {
const expected = [ownerTarget.executable, ...buildForwardServiceArgs(ownerTarget)].join(" ");
const probe = darwinOwnerProbe(`${expected}\n`);

expect(isForwardServiceListenerOwner(ownerTarget, { platform: "darwin", probe })).toBe(true);
expect(probe).toHaveBeenCalledTimes(4);
});

it("rejects a listener whose process does not match the direct ForwardTcp target", () => {
const probe = darwinOwnerProbe("/usr/bin/node foreign-listener.js\n");

expect(isForwardServiceListenerOwner(ownerTarget, { platform: "darwin", probe })).toBe(false);
});

it("rejects ambiguous or changing listener ownership", () => {
const expected = [ownerTarget.executable, ...buildForwardServiceArgs(ownerTarget)].join(" ");
const probe = darwinOwnerProbe(`${expected}\n`, "9876\n");

expect(isForwardServiceListenerOwner(ownerTarget, { platform: "darwin", probe })).toBe(false);
});

it("rejects ownership when a host probe times out", () => {
const lsofTimeout = vi.fn(() => ({ status: null, stdout: "" }));
expect(
isForwardServiceListenerOwner(ownerTarget, { platform: "darwin", probe: lsofTimeout }),
).toBe(false);

const psTimeout = vi
.fn()
.mockReturnValueOnce({ status: 0, stdout: "4321\n" })
.mockReturnValueOnce({ status: 0, stdout: `p4321\nftxt\nn${process.execPath}\n` })
.mockReturnValueOnce({ status: null, stdout: "" });
expect(
isForwardServiceListenerOwner(ownerTarget, { platform: "darwin", probe: psTimeout }),
).toBe(false);
});

it("proves Linux IPv4 ownership while ignoring an IPv6-only listener", () => {
const fixture = createLinuxOwnerFixture();
const expected = [fixture.target.executable, ...buildForwardServiceArgs(fixture.target)].join(
" ",
);
const responses = {
lsof: { status: null, stdout: "" },
ps: { status: 0, stdout: `${expected}\n` },
};
const probe = vi.fn(
(executable: string) => responses[executable as keyof typeof responses] ?? responses.lsof,
);

expect(
isForwardServiceListenerOwner(fixture.target, {
platform: "linux",
probe,
procRoot: fixture.procRoot,
}),
).toBe(true);
expect(probe).toHaveBeenCalledTimes(3);
expect(probe).toHaveBeenCalledWith("lsof", ["-ti4TCP:18789", "-sTCP:LISTEN"]);
expect(probe).toHaveBeenCalledWith("ps", ["-ww", "-p", "4321", "-o", "args="]);
});

it("rejects spoofed arguments when the Linux executable is different", () => {
const fixture = createLinuxOwnerFixture("python3");
const expected = [fixture.target.executable, ...buildForwardServiceArgs(fixture.target)].join(
" ",
);
const responses = {
lsof: { status: null, stdout: "" },
ps: { status: 0, stdout: `${expected}\n` },
};
const probe = vi.fn(
(executable: string) => responses[executable as keyof typeof responses] ?? responses.lsof,
);

expect(
isForwardServiceListenerOwner(fixture.target, {
platform: "linux",
probe,
procRoot: fixture.procRoot,
}),
).toBe(false);
expect(probe).toHaveBeenCalledOnce();
});

it("denies Linux ownership when the /proc work limit is reached", () => {
const fixture = createLinuxOwnerFixture();
const probe = vi.fn(() => ({ status: null, stdout: "" }));

expect(
isForwardServiceListenerOwner(fixture.target, {
platform: "linux",
probe,
procRoot: fixture.procRoot,
procWorkLimit: 1,
}),
).toBe(false);
expect(probe).toHaveBeenCalledOnce();
});

it("detaches the OpenShell child and waits for its local port", () => {
const unref = vi.fn();
const spawnDetached = vi.fn(() => ({ unref }));
Expand Down
Loading
Loading