fix(e2e): retry transient OpenClaw provider turns

docs/get-started/quickstart-hermes.mdx

@@ -381,8 +381,11 @@ Use these details when your first-run path needs more control.
     The onboard flow starts both port forwards automatically.
     For a new sandbox, NemoClaw reserves the selected dashboard loopback port through sandbox preparation and creation.
     If another listener claims the port before NemoClaw binds the reservation, NemoClaw selects another port before changing sandbox resources.
-    If OpenShell returns the `sandbox is not ready` response, NemoClaw waits 5 seconds and retries the affected forward up to 12 times.
-    The readiness-specific delays total at most 1 minute and preserve the existing sandbox and selected host port.
+    NemoClaw waits up to 90 seconds for each forward to bind.
+    If the exact forward process exits before it binds, NemoClaw waits 1 second and retries once.
+    If the process remains unbound at the deadline, NemoClaw stops that exact process and confirms its exit before retrying.
+    The complete two-attempt sequence can take just over 3 minutes for each forward.
+    NemoClaw does not adopt an unrelated listener or start another forward while it cannot confirm that the prior process stopped.
     NemoClaw releases the reservation immediately before OpenShell starts the dashboard forward.
     If forwarding then fails, onboarding removes the new sandbox and tells you to resolve the reported error before retrying.
     The Hermes dashboard URL does not include an OpenClaw `#token=` fragment.

docs/get-started/quickstart.mdx

@@ -458,8 +458,11 @@ Use these details when your first-run path needs more control.
     When that port is occupied, NemoClaw uses the next free dashboard port, such as `18790`, and includes the port in the URL.
     For a new sandbox, NemoClaw reserves the selected loopback port through sandbox preparation and creation.
     If another listener claims the port before NemoClaw binds the reservation, NemoClaw selects another port before changing sandbox resources.
-    If OpenShell returns the `sandbox is not ready` response, NemoClaw waits 5 seconds and retries the dashboard forward up to 12 times.
-    The readiness-specific delays total at most 1 minute and preserve the existing sandbox and selected port.
+    NemoClaw waits up to 90 seconds for the dashboard forward to bind.
+    If the exact forward process exits before it binds, NemoClaw waits 1 second and retries once.
+    If the process remains unbound at the deadline, NemoClaw stops that exact process and confirms its exit before retrying.
+    The complete two-attempt sequence can take just over 3 minutes.
+    NemoClaw does not adopt an unrelated listener or start another forward while it cannot confirm that the prior process stopped.
     NemoClaw releases the reservation immediately before OpenShell starts the dashboard forward.
     If forwarding then fails, onboarding removes the new sandbox and tells you to resolve the reported error before retrying.
     The installation transcript does not print the gateway token.

src/lib/adapters/openshell/forward-service.test.ts

@@ -19,6 +19,7 @@ const target: ForwardServiceTarget = {
   targetHost: "127.0.0.1",
   targetPort: 18_789,
 };
+const stableProcessIdentity = (pid: number): string => `start-${String(pid)}`;
 
 describe("OpenShell forward service", () => {
   it("builds the direct ForwardTcp command with explicit gateway authority", () => {
@@ -47,11 +48,13 @@ describe("OpenShell forward service", () => {
 
   it("detaches the OpenShell child and waits for its local port", () => {
     const unref = vi.fn();
-    const spawnDetached = vi.fn(() => ({ unref }));
+    const spawnDetached = vi.fn(() => ({ pid: 41, unref }));
     let probes = 0;
 
     launchForwardService(target, {
+      getProcessIdentity: stableProcessIdentity,
       isReachable: () => ++probes >= 3,
+      isProcessRunning: () => true,
       sleep: () => {},
       spawnDetached,
       timeoutMs: 1_000,
@@ -74,14 +77,157 @@ describe("OpenShell forward service", () => {
     expect(spawnDetached).not.toHaveBeenCalled();
   });
 
-  it("fails when the detached service does not bind before the deadline", () => {
+  it("stops a running service that remains unbound at the deadline (#11084)", () => {
+    const unboundTarget = { ...target, localPort: 18_791, targetPort: 18_791 };
+    let running = true;
+    const stopProcess = vi.fn(() => {
+      running = false;
+    });
+    expect(() =>
+      launchForwardService(unboundTarget, {
+        describeState: () => "SANDBOX BIND PORT PID STATUS",
+        getProcessIdentity: stableProcessIdentity,
+        isProcessRunning: () => running,
+        isReachable: () => false,
+        maxAttempts: 1,
+        sleep: () => {},
+        spawnDetached: () => ({ pid: 42, unref: () => {} }),
+        stopProcess,
+        timeoutMs: 0,
+      }),
+    ).toThrow(/was stopped after failing to bind.*forward list: SANDBOX BIND PORT PID STATUS/u);
+    expect(stopProcess).toHaveBeenCalledWith(42, "SIGTERM");
+  });
+
+  it("retries only after the prior service process exited (#11084)", () => {
+    const spawnDetached = vi
+      .fn()
+      .mockReturnValueOnce({ pid: 51, unref: vi.fn() })
+      .mockReturnValueOnce({ pid: 52, unref: vi.fn() });
+    launchForwardService(target, {
+      getProcessIdentity: stableProcessIdentity,
+      isProcessRunning: (pid) => pid === 52,
+      isReachable: () => spawnDetached.mock.calls.length === 2,
+      retryDelayMs: 0,
+      sleep: () => {},
+      spawnDetached,
+      timeoutMs: 1_000,
+    });
+
+    expect(spawnDetached).toHaveBeenCalledTimes(2);
+  });
+
+  it("does not retry when the timed-out service cannot be stopped (#11084)", () => {
+    const settlingTarget = { ...target, localPort: 18_790, targetPort: 18_790 };
+    const spawnDetached = vi.fn(() => ({ pid: 61, unref: vi.fn() }));
+    const stopProcess = vi.fn();
+
+    expect(() =>
+      launchForwardService(settlingTarget, {
+        getProcessIdentity: stableProcessIdentity,
+        isProcessRunning: () => true,
+        isReachable: () => false,
+        sleep: () => {},
+        spawnDetached,
+        stopProcess,
+        stopTimeoutMs: 0,
+        timeoutMs: 0,
+      }),
+    ).toThrow(/process 61.*could not stop.*refusing to signal or retry/u);
+    expect(spawnDetached).toHaveBeenCalledOnce();
+    expect(stopProcess.mock.calls).toEqual([
+      [61, "SIGTERM"],
+      [61, "SIGKILL"],
+    ]);
+  });
+
+  it("confirms exit after escalating a timed-out service to SIGKILL (#11084)", () => {
+    const signals: NodeJS.Signals[] = [];
+    let killChecks = 0;
+
     expect(() =>
       launchForwardService(target, {
+        getProcessIdentity: stableProcessIdentity,
+        isProcessRunning: () =>
+          signals.at(-1) === "SIGKILL" ? ++killChecks < 2 : true,
         isReachable: () => false,
+        maxAttempts: 1,
         sleep: () => {},
-        spawnDetached: () => ({ unref: () => {} }),
+        spawnDetached: () => ({ pid: 62, unref: vi.fn() }),
+        stopProcess: (_pid, signal) => signals.push(signal),
+        stopTimeoutMs: 1_000,
         timeoutMs: 0,
       }),
-    ).toThrow(/did not bind/u);
+    ).toThrow(/was stopped after failing to bind/u);
+    expect(signals).toEqual(["SIGTERM", "SIGKILL"]);
+    expect(killChecks).toBe(2);
+  });
+
+  it("does not signal or retry after the forward PID is reused (#11084)", () => {
+    const spawnDetached = vi.fn(() => ({ pid: 63, unref: vi.fn() }));
+    const stopProcess = vi.fn();
+    let identityReads = 0;
+
+    expect(() =>
+      launchForwardService(target, {
+        getProcessIdentity: () => (++identityReads === 1 ? "original" : "replacement"),
+        isProcessRunning: () => true,
+        isReachable: () => false,
+        sleep: () => {},
+        spawnDetached,
+        stopProcess,
+        timeoutMs: 0,
+      }),
+    ).toThrow(/process 63 changed identity.*refusing to signal or retry/u);
+    expect(stopProcess).not.toHaveBeenCalled();
+    expect(spawnDetached).toHaveBeenCalledOnce();
+  });
+
+  it("refuses an unknown listener that appears before a safe retry (#11084)", () => {
+    const spawnDetached = vi.fn(() => ({ pid: 71, unref: vi.fn() }));
+    let probes = 0;
+
+    expect(() =>
+      launchForwardService(target, {
+        getProcessIdentity: stableProcessIdentity,
+        isProcessRunning: () => false,
+        isReachable: () => ++probes >= 3,
+        retryDelayMs: 0,
+        sleep: () => {},
+        spawnDetached,
+        timeoutMs: 1_000,
+      }),
+    ).toThrow(/became occupied.*refusing to adopt/u);
+    expect(spawnDetached).toHaveBeenCalledOnce();
+  });
+
+  it.runIf(process.platform === "linux")(
+    "tracks the spawned process by its Linux start time (#11084)",
+    () => {
+      let probes = 0;
+
+      launchForwardService(target, {
+        isReachable: () => ++probes >= 2,
+        spawnDetached: () => ({ pid: process.pid, unref: vi.fn() }),
+      });
+
+      expect(probes).toBe(2);
+    },
+  );
+
+  it("does not retry when the service returns no process identity (#11084)", () => {
+    const spawnDetached = vi.fn(() => ({ unref: vi.fn() }));
+
+    expect(() =>
+      launchForwardService(target, {
+        describeState: () => {
+          throw new Error("forward list failed");
+        },
+        isReachable: () => false,
+        sleep: () => {},
+        spawnDetached,
+      }),
+    ).toThrow(/no process identity.*forward list: <unavailable>/u);
+    expect(spawnDetached).toHaveBeenCalledOnce();
   });
 });