fix(e2e): retry transient OpenClaw provider turns

docs/get-started/quickstart-hermes.mdx

@@ -381,8 +381,11 @@ Use these details when your first-run path needs more control.
     The onboard flow starts both port forwards automatically.
     For a new sandbox, NemoClaw reserves the selected dashboard loopback port through sandbox preparation and creation.
     If another listener claims the port before NemoClaw binds the reservation, NemoClaw selects another port before changing sandbox resources.
-    If OpenShell returns the `sandbox is not ready` response, NemoClaw waits 5 seconds and retries the affected forward up to 12 times.
-    The readiness-specific delays total at most 1 minute and preserve the existing sandbox and selected host port.
+    NemoClaw waits up to 90 seconds for each forward to bind.
+    If the exact forward process exits before it binds, NemoClaw waits 1 second and retries once.
+    If the process remains unbound at the deadline, NemoClaw stops that exact process and confirms its exit before retrying.
+    The complete two-attempt sequence can take just over 3 minutes for each forward.
+    NemoClaw does not adopt an unrelated listener or start another forward while it cannot confirm that the prior process stopped.
     NemoClaw releases the reservation immediately before OpenShell starts the dashboard forward.
     If forwarding then fails, onboarding removes the new sandbox and tells you to resolve the reported error before retrying.
     The Hermes dashboard URL does not include an OpenClaw `#token=` fragment.

docs/get-started/quickstart.mdx

@@ -458,8 +458,11 @@ Use these details when your first-run path needs more control.
     When that port is occupied, NemoClaw uses the next free dashboard port, such as `18790`, and includes the port in the URL.
     For a new sandbox, NemoClaw reserves the selected loopback port through sandbox preparation and creation.
     If another listener claims the port before NemoClaw binds the reservation, NemoClaw selects another port before changing sandbox resources.
-    If OpenShell returns the `sandbox is not ready` response, NemoClaw waits 5 seconds and retries the dashboard forward up to 12 times.
-    The readiness-specific delays total at most 1 minute and preserve the existing sandbox and selected port.
+    NemoClaw waits up to 90 seconds for the dashboard forward to bind.
+    If the exact forward process exits before it binds, NemoClaw waits 1 second and retries once.
+    If the process remains unbound at the deadline, NemoClaw stops that exact process and confirms its exit before retrying.
+    The complete two-attempt sequence can take just over 3 minutes.
+    NemoClaw does not adopt an unrelated listener or start another forward while it cannot confirm that the prior process stopped.
     NemoClaw releases the reservation immediately before OpenShell starts the dashboard forward.
     If forwarding then fails, onboarding removes the new sandbox and tells you to resolve the reported error before retrying.
     The installation transcript does not print the gateway token.

src/lib/adapters/openshell/forward-service.test.ts

@@ -47,11 +47,12 @@ describe("OpenShell forward service", () => {
 
   it("detaches the OpenShell child and waits for its local port", () => {
     const unref = vi.fn();
-    const spawnDetached = vi.fn(() => ({ unref }));
+    const spawnDetached = vi.fn(() => ({ pid: 41, unref }));
     let probes = 0;
 
     launchForwardService(target, {
       isReachable: () => ++probes >= 3,
+      isProcessRunning: () => true,
       sleep: () => {},
       spawnDetached,
       timeoutMs: 1_000,
@@ -74,14 +75,118 @@ describe("OpenShell forward service", () => {
     expect(spawnDetached).not.toHaveBeenCalled();
   });
 
-  it("fails when the detached service does not bind before the deadline", () => {
+  it("stops a running service that remains unbound at the deadline (#11084)", () => {
+    const unboundTarget = { ...target, localPort: 18_791, targetPort: 18_791 };
+    let running = true;
+    const stopProcess = vi.fn(() => {
+      running = false;
+    });
+    expect(() =>
+      launchForwardService(unboundTarget, {
+        describeState: () => "SANDBOX BIND PORT PID STATUS",
+        isProcessRunning: () => running,
+        isReachable: () => false,
+        maxAttempts: 1,
+        sleep: () => {},
+        spawnDetached: () => ({ pid: 42, unref: () => {} }),
+        stopProcess,
+        timeoutMs: 0,
+      }),
+    ).toThrow(/was stopped after failing to bind.*forward list: SANDBOX BIND PORT PID STATUS/u);
+    expect(stopProcess).toHaveBeenCalledWith(42, "SIGTERM");
+  });
+
+  it("retries only after the prior service process exited (#11084)", () => {
+    const spawnDetached = vi
+      .fn()
+      .mockReturnValueOnce({ pid: 51, unref: vi.fn() })
+      .mockReturnValueOnce({ pid: 52, unref: vi.fn() });
+    launchForwardService(target, {
+      isProcessRunning: (pid) => pid === 52,
+      isReachable: () => spawnDetached.mock.calls.length === 2,
+      retryDelayMs: 0,
+      sleep: () => {},
+      spawnDetached,
+      timeoutMs: 1_000,
+    });
+
+    expect(spawnDetached).toHaveBeenCalledTimes(2);
+  });
+
+  it("does not retry when the timed-out service cannot be stopped (#11084)", () => {
+    const settlingTarget = { ...target, localPort: 18_790, targetPort: 18_790 };
+    const spawnDetached = vi.fn(() => ({ pid: 61, unref: vi.fn() }));
+    const stopProcess = vi.fn();
+
+    expect(() =>
+      launchForwardService(settlingTarget, {
+        isProcessRunning: () => true,
+        isReachable: () => false,
+        sleep: () => {},
+        spawnDetached,
+        stopProcess,
+        stopTimeoutMs: 0,
+        timeoutMs: 0,
+      }),
+    ).toThrow(/process 61.*could not be stopped.*do not retry/u);
+    expect(spawnDetached).toHaveBeenCalledOnce();
+    expect(stopProcess.mock.calls).toEqual([
+      [61, "SIGTERM"],
+      [61, "SIGKILL"],
+    ]);
+  });
+
+  it("confirms exit after escalating a timed-out service to SIGKILL (#11084)", () => {
+    const signals: NodeJS.Signals[] = [];
+    let killChecks = 0;
+
     expect(() =>
       launchForwardService(target, {
+        isProcessRunning: () =>
+          signals.at(-1) === "SIGKILL" ? ++killChecks < 2 : true,
         isReachable: () => false,
+        maxAttempts: 1,
         sleep: () => {},
-        spawnDetached: () => ({ unref: () => {} }),
+        spawnDetached: () => ({ pid: 62, unref: vi.fn() }),
+        stopProcess: (_pid, signal) => signals.push(signal),
+        stopTimeoutMs: 1_000,
         timeoutMs: 0,
       }),
-    ).toThrow(/did not bind/u);
+    ).toThrow(/was stopped after failing to bind/u);
+    expect(signals).toEqual(["SIGTERM", "SIGKILL"]);
+    expect(killChecks).toBe(2);
+  });
+
+  it("refuses an unknown listener that appears before a safe retry (#11084)", () => {
+    const spawnDetached = vi.fn(() => ({ pid: 71, unref: vi.fn() }));
+    let probes = 0;
+
+    expect(() =>
+      launchForwardService(target, {
+        isProcessRunning: () => false,
+        isReachable: () => ++probes >= 3,
+        retryDelayMs: 0,
+        sleep: () => {},
+        spawnDetached,
+        timeoutMs: 1_000,
+      }),
+    ).toThrow(/became occupied.*refusing to adopt/u);
+    expect(spawnDetached).toHaveBeenCalledOnce();
+  });
+
+  it("does not retry when the service returns no process identity (#11084)", () => {
+    const spawnDetached = vi.fn(() => ({ unref: vi.fn() }));
+
+    expect(() =>
+      launchForwardService(target, {
+        describeState: () => {
+          throw new Error("forward list failed");
+        },
+        isReachable: () => false,
+        sleep: () => {},
+        spawnDetached,
+      }),
+    ).toThrow(/no process identity.*forward list: <unavailable>/u);
+    expect(spawnDetached).toHaveBeenCalledOnce();
   });
 });

src/lib/adapters/openshell/forward-service.ts

@@ -2,16 +2,25 @@
 // SPDX-License-Identifier: Apache-2.0
 
 import { spawn } from "node:child_process";
+import fs from "node:fs";
 import path from "node:path";
 
 import { isValidName } from "../../name-validation";
 import { buildOpenShellSubprocessEnv } from "./resolve-shared";
 import { probeLocalForwardListener } from "./local-forward-listener";
 
-const START_TIMEOUT_MS = 30_000;
+const START_TIMEOUT_MS = 90_000;
+const START_RETRY_DELAY_MS = 1_000;
+const START_ATTEMPTS = 2;
+const STOP_TIMEOUT_MS = 5_000;
 const POLL_INTERVAL_MS = 100;
 const sleepBuffer = new Int32Array(new SharedArrayBuffer(4));
 
+type ForwardServiceChild = {
+  readonly pid?: number;
+  unref(): void;
+};
+
 export interface ForwardServiceTarget {
   readonly executable: string;
   readonly gatewayName: string;
@@ -24,19 +33,101 @@ export interface ForwardServiceTarget {
 }
 
 export interface ForwardServiceLaunchOptions {
+  readonly describeState?: () => string | null | undefined;
+  readonly isProcessRunning?: (pid: number) => boolean;
   readonly isReachable?: (port: number) => boolean;
+  readonly maxAttempts?: number;
+  readonly retryDelayMs?: number;
   readonly sleep?: (milliseconds: number) => void;
   readonly sourceEnvironment?: NodeJS.ProcessEnv;
+  readonly stopProcess?: (pid: number, signal: NodeJS.Signals) => void;
+  readonly stopTimeoutMs?: number;
   readonly spawnDetached?: (
     executable: string,
     args: readonly string[],
     environment: NodeJS.ProcessEnv,
-  ) => {
-    unref(): void;
-  };
+  ) => ForwardServiceChild;
   readonly timeoutMs?: number;
 }
 
+function isProcessRunning(pid: number): boolean {
+  try {
+    if (process.platform === "linux") {
+      // Signal 0 still reports a zombie as present. Read its state so a failed
+      // forward may be retried without starting alongside a live process.
+      const stat = fs.readFileSync(`/proc/${String(pid)}/stat`, "utf8");
+      const state = stat.slice(stat.lastIndexOf(")") + 2).split(" ", 1)[0];
+      if (state === "Z") return false;
+    }
+    process.kill(pid, 0);
+    return true;
+  } catch (error) {
+    const code =
+      typeof error === "object" && error !== null && "code" in error ? error.code : undefined;
+    return code !== "ENOENT" && code !== "ESRCH";
+  }
+}
+
+function boundedState(description: string | null | undefined): string {
+  const compact = (description ?? "").replace(/\s+/gu, " ").trim();
+  return compact ? compact.slice(0, 240) : "<empty>";
+}
+
+function describeFailureState(describe: (() => string | null | undefined) | undefined): string {
+  try {
+    return boundedState(describe?.());
+  } catch {
+    return "<unavailable>";
+  }
+}
+
+function isProcessIdentity(pid: number | undefined): pid is number {
+  return Number.isSafeInteger(pid) && Number(pid) > 0;
+}
+
+function isMissingProcessError(error: unknown): boolean {
+  const code =
+    typeof error === "object" && error !== null && "code" in error ? error.code : undefined;
+  return code === "ENOENT" || code === "ESRCH";
+}
+
+function waitForProcessExit(
+  pid: number,
+  processIsRunning: (pid: number) => boolean,
+  sleep: (milliseconds: number) => void,
+  timeoutMs: number,
+): boolean {
+  const deadline = Date.now() + timeoutMs;
+  while (processIsRunning(pid)) {
+    if (Date.now() >= deadline) return false;
+    sleep(POLL_INTERVAL_MS);
+  }
+  return true;
+}
+
+function stopOwnedProcess(
+  pid: number,
+  processIsRunning: (pid: number) => boolean,
+  stopProcess: (pid: number, signal: NodeJS.Signals) => void,
+  sleep: (milliseconds: number) => void,
+  timeoutMs: number,
+): boolean {
+  try {
+    stopProcess(pid, "SIGTERM");
+  } catch (error) {
+    if (isMissingProcessError(error)) return true;
+    return false;
+  }
+  if (waitForProcessExit(pid, processIsRunning, sleep, timeoutMs)) return true;
+  try {
+    stopProcess(pid, "SIGKILL");
+  } catch (error) {
+    if (isMissingProcessError(error)) return true;
+    return false;
+  }
+  return waitForProcessExit(pid, processIsRunning, sleep, timeoutMs);
+}
+
 function isPort(value: unknown): value is number {
   return Number.isSafeInteger(value) && Number(value) >= 1 && Number(value) <= 65_535;
 }
@@ -101,28 +192,75 @@ export function launchForwardService(
 ): void {
   validateForwardServiceTarget(target);
   const isReachable = options.isReachable ?? probeLocalForwardListener;
+  const processIsRunning = options.isProcessRunning ?? isProcessRunning;
   if (isReachable(target.localPort)) {
     throw new Error(`Host port ${String(target.localPort)} is already occupied`);
   }
   const spawnDetached =
     options.spawnDetached ??
     ((executable, args, environment) =>
       spawn(executable, [...args], { detached: true, env: environment, stdio: "ignore" }));
-  const child = spawnDetached(
-    target.executable,
-    buildForwardServiceArgs(target),
-    buildOpenShellSubprocessEnv(options.sourceEnvironment ?? process.env),
-  );
-  child.unref();
-
   const sleep =
     options.sleep ?? ((milliseconds: number) => Atomics.wait(sleepBuffer, 0, 0, milliseconds));
-  const deadline = Date.now() + (options.timeoutMs ?? START_TIMEOUT_MS);
-  while (Date.now() < deadline) {
-    if (isReachable(target.localPort)) return;
-    sleep(POLL_INTERVAL_MS);
+  const stopProcess =
+    options.stopProcess ?? ((pid: number, signal: NodeJS.Signals) => process.kill(pid, signal));
+  const attempts = options.maxAttempts ?? START_ATTEMPTS;
+  if (!Number.isSafeInteger(attempts) || attempts < 1 || attempts > START_ATTEMPTS) {
+    throw new Error(`OpenShell forward service attempts must be between 1 and ${START_ATTEMPTS}`);
+  }
+  const args = buildForwardServiceArgs(target);
+  const environment = buildOpenShellSubprocessEnv(options.sourceEnvironment ?? process.env);
+  let finalFailure = "exited before binding";
+  for (let attempt = 1; attempt <= attempts; attempt += 1) {
+    finalFailure = "exited before binding";
+    if (isReachable(target.localPort)) {
+      throw new Error(
+        `Host port ${String(target.localPort)} became occupied before forward retry; refusing to adopt its listener`,
+      );
+    }
+    const child = spawnDetached(target.executable, args, environment);
+    child.unref();
+    if (!isProcessIdentity(child.pid)) {
+      throw new Error(
+        `OpenShell forward service returned no process identity for ${target.localHost}:${String(target.localPort)}; ` +
+          `refusing to start a duplicate service; forward list: ${describeFailureState(options.describeState)}`,
+      );
+    }
+    const deadline = Date.now() + (options.timeoutMs ?? START_TIMEOUT_MS);
+    let exited = false;
+    while (true) {
+      if (isProcessIdentity(child.pid) && !processIsRunning(child.pid)) {
+        exited = true;
+        break;
+      }
+      if (isReachable(target.localPort)) {
+        return;
+      }
+      if (Date.now() >= deadline) break;
+      sleep(POLL_INTERVAL_MS);
+    }
+    if (!exited) {
+      const stopped = stopOwnedProcess(
+        child.pid,
+        processIsRunning,
+        stopProcess,
+        sleep,
+        options.stopTimeoutMs ?? STOP_TIMEOUT_MS,
+      );
+      if (!stopped) {
+        throw new Error(
+          `OpenShell forward service process ${String(child.pid)} remained running after its ${String(options.timeoutMs ?? START_TIMEOUT_MS)}ms bind deadline and could not be stopped; ` +
+            `do not retry until that process exits; forward list: ${describeFailureState(options.describeState)}`,
+        );
+      }
+      finalFailure = "was stopped after failing to bind";
+    }
+    if (attempt < attempts) {
+      sleep(options.retryDelayMs ?? START_RETRY_DELAY_MS);
+    }
   }
   throw new Error(
-    `OpenShell forward service did not bind ${target.localHost}:${String(target.localPort)}`,
+    `OpenShell forward service ${finalFailure} ${target.localHost}:${String(target.localPort)} after ${String(attempts)} attempts; ` +
+      `forward list: ${describeFailureState(options.describeState)}`,
   );
 }