Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion src/lib/actions/sandbox/forward-recovery.ts
Original file line number Diff line number Diff line change
Expand Up @@ -242,7 +242,13 @@ export function ensureSandboxPortForwardForPort(
stdio: "ignore",
},
);
if (startResult.status !== 0) return false;
// OpenShell 0.0.85 returns an error when start preflight finds a validated
// live forward for the requested port. Recovery cannot change that upstream
// CLI contract, so a reachable listener settles against the authoritative
// forward list below; an absent listener still fails fast. Remove this
// tolerance once every supported OpenShell release makes `forward start`
// idempotent for an already-tracked live forward.
if (startResult.status !== 0 && !isLocalForwardReachable(port)) return false;

// `forward start --background` can return before its authoritative list
// entry becomes visible. Poll for the exact live sandbox+port owner instead
Expand Down
152 changes: 152 additions & 0 deletions test/process-recovery-forward-failure.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@ import os from "node:os";
import path from "node:path";

import { afterEach, describe, expect, it, vi } from "vitest";
import * as forwardHealth from "../src/lib/actions/sandbox/forward-health.js";
import { ensureSandboxPortForwardForPort } from "../src/lib/actions/sandbox/forward-recovery.js";
import * as openshellRuntime from "../src/lib/adapters/openshell/runtime.js";

const requireSource = createRequire(import.meta.url);
const { checkAndRecoverSandboxProcesses: checkAndRecoverSandboxProcessesImpl } = requireSource(
Expand Down Expand Up @@ -190,3 +193,152 @@ beta 127.0.0.1 18789 12345 dead`,
);
});
});

describe("ensureSandboxPortForwardForPort already-forwarded idempotency (#7085)", () => {
it("accepts an already-active target-owned forward when `forward start` exits non-zero", () => {
// Forward visibility is fixed by mocks, so the production settle window is unnecessary.
vi.stubEnv("NEMOCLAW_FORWARD_RECOVERY_WAIT_MS", "0");

// The port is listening throughout; OpenShell's forward list only shows the
// live owner after the (idempotent) start, modelling the stale-list drift
// that makes recovery attempt a stop -> start on an already-active forward.
let started = false;
vi.spyOn(forwardHealth, "isLocalForwardReachable").mockReturnValue(true);
vi.spyOn(openshellRuntime, "captureOpenshell").mockImplementation(() => ({
status: 0,
output: started
? `SANDBOX BIND PORT PID STATUS
beta 127.0.0.1 18791 12345 running`
: `SANDBOX BIND PORT PID STATUS`,
}));
const runOpenshell = vi
.spyOn(openshellRuntime, "runOpenshell")
.mockImplementation((rawArgs: unknown) => {
const args = Array.isArray(rawArgs) ? rawArgs.map(String) : [];
const isForwardStart = args[0] === "forward" && args[1] === "start";
started ||= isForwardStart;
// OpenShell exits non-zero because the port is already forwarded.
return { status: Number(isForwardStart) } as never;
});

expect(
withFakeOpenshellBinary(() =>
ensureSandboxPortForwardForPort("beta", 18791, { expectedBind: "127.0.0.1" }),
),
).toBe(true);
expect(runOpenshell).toHaveBeenCalledWith(
["forward", "start", "--background", "18791", "beta"],
expect.objectContaining({ ignoreError: true }),
);
});

it("still fails when `forward start` exits non-zero and no target-owned forward is active", () => {
vi.stubEnv("NEMOCLAW_FORWARD_RECOVERY_WAIT_MS", "0");

// No live owner row ever appears: a genuine start failure must not be
// masked by the idempotency re-probe.
vi.spyOn(forwardHealth, "isLocalForwardReachable").mockReturnValue(false);
vi.spyOn(openshellRuntime, "captureOpenshell").mockReturnValue({
status: 0,
output: "SANDBOX BIND PORT PID STATUS",
});
vi.spyOn(openshellRuntime, "runOpenshell").mockImplementation((rawArgs: unknown) => {
const args = Array.isArray(rawArgs) ? rawArgs.map(String) : [];
return { status: args[0] === "forward" && args[1] === "start" ? 1 : 0 } as never;
});

expect(
withFakeOpenshellBinary(() =>
ensureSandboxPortForwardForPort("beta", 18791, { expectedBind: "127.0.0.1" }),
),
).toBe(false);
});

it("rejects a reachable listener that never gains authoritative ownership", () => {
vi.stubEnv("NEMOCLAW_FORWARD_RECOVERY_WAIT_MS", "25");
let started = false;

vi.spyOn(forwardHealth, "isLocalForwardReachable").mockImplementation(() => started);
const captureOpenshell = vi.spyOn(openshellRuntime, "captureOpenshell").mockReturnValue({
status: 0,
output: "SANDBOX BIND PORT PID STATUS",
});
vi.spyOn(openshellRuntime, "runOpenshell").mockImplementation((rawArgs: unknown) => {
const args = Array.isArray(rawArgs) ? rawArgs.map(String) : [];
const isForwardStart = args[0] === "forward" && args[1] === "start";
started ||= isForwardStart;
return { status: Number(isForwardStart) } as never;
});

expect(
withFakeOpenshellBinary(() =>
ensureSandboxPortForwardForPort("beta", 18791, { expectedBind: "127.0.0.1" }),
),
).toBe(false);
expect(captureOpenshell.mock.calls.length).toBeGreaterThanOrEqual(2);
});

it("waits for delayed target ownership after a non-zero `forward start`", () => {
vi.stubEnv("NEMOCLAW_FORWARD_RECOVERY_WAIT_MS", "250");
let started = false;
let postStartProbes = 0;

vi.spyOn(forwardHealth, "isLocalForwardReachable").mockImplementation(() => started);
vi.spyOn(openshellRuntime, "captureOpenshell").mockImplementation(() => {
postStartProbes += Number(started);
return {
status: 0,
output:
postStartProbes >= 2
? `SANDBOX BIND PORT PID STATUS
beta 127.0.0.1 18791 12345 running`
: "SANDBOX BIND PORT PID STATUS",
};
});
vi.spyOn(openshellRuntime, "runOpenshell").mockImplementation((rawArgs: unknown) => {
const args = Array.isArray(rawArgs) ? rawArgs.map(String) : [];
const isForwardStart = args[0] === "forward" && args[1] === "start";
started ||= isForwardStart;
return { status: Number(isForwardStart) } as never;
});

expect(
withFakeOpenshellBinary(() =>
ensureSandboxPortForwardForPort("beta", 18791, { expectedBind: "127.0.0.1" }),
),
).toBe(true);
expect(postStartProbes).toBe(2);
});

it("rejects delayed ownership by another sandbox after a non-zero `forward start`", () => {
vi.stubEnv("NEMOCLAW_FORWARD_RECOVERY_WAIT_MS", "250");
let started = false;
let postStartProbes = 0;

vi.spyOn(forwardHealth, "isLocalForwardReachable").mockImplementation(() => started);
vi.spyOn(openshellRuntime, "captureOpenshell").mockImplementation(() => {
postStartProbes += Number(started);
return {
status: 0,
output:
postStartProbes >= 2
? `SANDBOX BIND PORT PID STATUS
gamma 127.0.0.1 18791 12345 running`
: "SANDBOX BIND PORT PID STATUS",
};
});
vi.spyOn(openshellRuntime, "runOpenshell").mockImplementation((rawArgs: unknown) => {
const args = Array.isArray(rawArgs) ? rawArgs.map(String) : [];
const isForwardStart = args[0] === "forward" && args[1] === "start";
started ||= isForwardStart;
return { status: Number(isForwardStart) } as never;
});

expect(
withFakeOpenshellBinary(() =>
ensureSandboxPortForwardForPort("beta", 18791, { expectedBind: "127.0.0.1" }),
),
).toBe(false);
expect(postStartProbes).toBe(2);
});
});
Loading