refactor(core): centralize result retries

src/lib/actions/sandbox/stopped-sandbox-backup.ts

@@ -3,6 +3,7 @@
 
 import { dockerContainerInspectFormat } from "../../adapters/docker/inspect";
 import { dockerCapture, dockerRun } from "../../adapters/docker/run";
+import { retryUntilAsync } from "../../core/retry";
 import { resolveSandboxContainerOwner } from "../../domain/sandbox/container-owner";
 import {
   findLabeledSandboxContainers,
@@ -208,14 +209,12 @@ export async function backupStartedSandboxState(
   depsOverride: Partial<BackupRetryDeps> = {},
 ): Promise<sandboxState.BackupResult> {
   const deps: BackupRetryDeps = { ...defaultBackupRetryDeps, ...depsOverride };
-  let result = deps.backup(sandboxName);
-  for (
-    let attempt = 1;
-    attempt < deps.attempts && !result.success && result.unreachable;
-    attempt++
-  ) {
-    await deps.sleep(deps.delayMs);
-    result = deps.backup(sandboxName);
-  }
-  return result;
+  return retryUntilAsync(() => deps.backup(sandboxName), {
+    accept: (result) => result.success || !result.unreachable,
+    retryDelaysMs: Array.from(
+      { length: Math.max(0, Math.ceil(deps.attempts) - 1) },
+      () => deps.delayMs,
+    ),
+    sleep: deps.sleep,
+  });
 }

src/lib/core/retry.ts

@@ -0,0 +1,58 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+interface RetryUntilBaseOptions<T> {
+  /** Return true to stop retrying after this result. */
+  accept: (result: T, attempt: number) => boolean;
+  /** Delay in milliseconds before each additional attempt. */
+  retryDelaysMs: readonly number[];
+}
+
+export type RetryUntilOptions<T> = RetryUntilBaseOptions<T> & {
+  /** Sleep for the specified number of milliseconds between attempts. */
+  sleep: (ms: number) => void;
+};
+
+export type RetryUntilAsyncOptions<T> = RetryUntilBaseOptions<T> & {
+  /** Sleep for the specified number of milliseconds between attempts. */
+  sleep: (ms: number) => Promise<void>;
+};
+
+/**
+ * Retry a synchronous operation until its result is accepted or the delay
+ * schedule is exhausted. Returns the final operation result.
+ */
+export function retryUntil<T>(operation: (attempt: number) => T, options: RetryUntilOptions<T>): T {
+  let attempt = 1;
+  let result = operation(attempt);
+  if (options.accept(result, attempt)) return result;
+
+  for (const delayMs of options.retryDelaysMs) {
+    options.sleep(delayMs);
+    attempt += 1;
+    result = operation(attempt);
+    if (options.accept(result, attempt)) return result;
+  }
+  return result;
+}
+
+/**
+ * Retry an asynchronous operation until its result is accepted or the delay
+ * schedule is exhausted. Returns the final operation result.
+ */
+export async function retryUntilAsync<T>(
+  operation: (attempt: number) => T | Promise<T>,
+  options: RetryUntilAsyncOptions<T>,
+): Promise<T> {
+  let attempt = 1;
+  let result = await operation(attempt);
+  if (options.accept(result, attempt)) return result;
+
+  for (const delayMs of options.retryDelaysMs) {
+    await options.sleep(delayMs);
+    attempt += 1;
+    result = await operation(attempt);
+    if (options.accept(result, attempt)) return result;
+  }
+  return result;
+}

src/lib/inference/https-pin-runtime-adapter.ts

@@ -55,6 +55,7 @@ import {
   VLLM_PORT,
   validateHttpsPinRuntimeAdapterPort,
 } from "../core/ports";
+import { retryUntilAsync } from "../core/retry";
 import { getVersion } from "../core/version";
 import { ROOT, run, runCapture } from "../runner";
 import { buildMinimalCredentialAdapterEnv } from "../subprocess-env";
@@ -845,11 +846,13 @@ async function waitForAdapterProcessExit(
   const sleep = options.sleep || sleepMs;
   const attempts = options.attempts || PROCESS_EXIT_WAIT_ATTEMPTS;
   const intervalMs = options.intervalMs || PROCESS_EXIT_WAIT_MS;
-  for (let attempt = 0; attempt < attempts; attempt++) {
-    if (!isRunning(pid)) return true;
-    if (attempt + 1 < attempts) await sleep(intervalMs);
-  }
-  return false;
+  if (attempts <= 0) return false;
+
+  return retryUntilAsync(() => !isRunning(pid), {
+    accept: (exited) => exited,
+    retryDelaysMs: Array.from({ length: Math.ceil(attempts) - 1 }, () => intervalMs),
+    sleep,
+  });
 }
 
 async function killStaleAdapter(): Promise<void> {
@@ -1380,11 +1383,11 @@ async function revokeRouteLocked(
   const allowedSourceCidrs = deps.readAllowedSourceCidrs();
   const authenticatedLiveAdapter = Boolean(
     controlToken &&
-      allowedSourceCidrs &&
-      (await deps.probeHealth({
-        controlToken: controlToken as string,
-        expectedSourceCidrs: allowedSourceCidrs,
-      })),
+    allowedSourceCidrs &&
+    (await deps.probeHealth({
+      controlToken: controlToken as string,
+      expectedSourceCidrs: allowedSourceCidrs,
+    })),
   );
   if (authenticatedLiveAdapter && controlToken) {
     await deps.deleteRoute(controlToken, routeId);

src/lib/onboard/docker-gpu-local-inference.test.ts

@@ -238,6 +238,9 @@ describe("verifyDockerGpuSandboxLocalInference", () => {
     }
     expect(execInSandbox).toHaveBeenCalledTimes(3);
     expect(sleep).toHaveBeenCalledTimes(2);
+
+    expect(sleep).toHaveBeenNthCalledWith(1, 2_000);
+    expect(sleep).toHaveBeenNthCalledWith(2, 2_000);
   });
 
   it("fails when the inference route is up but the local backend errors (HTTP 502)", () => {

src/lib/onboard/docker-gpu-local-inference.ts

@@ -1,6 +1,8 @@
 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
 
+import { retryUntil } from "../core/retry";
+
 import { getLocalProviderLabel } from "../inference/local";
 import type { SandboxGpuProofResult } from "../state/registry";
 import {
@@ -21,7 +23,7 @@ const {
 const DOCKER_GPU_INFERENCE_PROBE_CONNECT_TIMEOUT_SECS = 5;
 const DOCKER_GPU_INFERENCE_PROBE_MAX_TIME_SECS = 10;
 const DOCKER_GPU_INFERENCE_PROBE_MAX_ATTEMPTS = 3;
-const DOCKER_GPU_INFERENCE_PROBE_RETRY_DELAY_SECS = 2;
+const DOCKER_GPU_INFERENCE_PROBE_RETRY_DELAY_MS = 2_000;
 
 // The OpenShell inference route OpenClaw's LLM client actually uses inside the
 // sandbox. It is served by the OpenShell L7 proxy/router and routes to the
@@ -160,7 +162,7 @@ export type SandboxExecResult = {
 
 export type DockerGpuSandboxInferenceVerifyDeps = {
   execInSandbox?: (sandboxName: string, script: string) => SandboxExecResult;
-  sleep?: (seconds: number) => void;
+  sleep?: (milliseconds: number) => void;
 };
 
 export type DockerGpuSandboxInferenceVerification =
@@ -221,54 +223,55 @@ function probeSandboxRuntimeInference(
     `--connect-timeout ${DOCKER_GPU_INFERENCE_PROBE_CONNECT_TIMEOUT_SECS} ` +
     `--max-time ${DOCKER_GPU_INFERENCE_PROBE_MAX_TIME_SECS} ${safeEndpoint} 2>/dev/null || echo 000); ` +
     `echo "HTTP_$code"`;
-  let last: RuntimeProbeOutcome = {
-    kind: "exec-failed",
-    detail: "openshell sandbox exec did not run (sandbox unreachable or exec timed out)",
-  };
-  for (let attempt = 1; attempt <= DOCKER_GPU_INFERENCE_PROBE_MAX_ATTEMPTS; attempt++) {
+  const probe = (): RuntimeProbeOutcome => {
     const result = deps.execInSandbox(sandboxName, script);
     if (result === null) {
-      last = {
+      return {
         kind: "exec-failed",
         detail: "openshell sandbox exec did not run (sandbox unreachable or exec timed out)",
       };
-    } else {
-      const out = (result.stdout || "").trim();
-      if (out === "NO_CURL") return { kind: "no-curl" };
-      const match = out.match(/HTTP_(\d{3})/);
-      if (match) {
-        const httpCode = match[1];
-        // This gate is the #4509 runtime proof, so only a 2xx — the model list
-        // actually returned through the proxy with injected auth — counts as
-        // success. `000` is the reported failure (DNS / connection refused). A
-        // 4xx (wrong provider route, or auth the proxy failed to inject) or 5xx
-        // (local Ollama/vLLM backend down) means OpenClaw's real request would
-        // fail too, so do NOT report it as reachable.
-        if (/^2\d\d$/.test(httpCode)) return { kind: "ok", httpCode };
-        last = {
-          kind: "unreachable",
-          detail:
-            httpCode === "000"
-              ? `${endpoint} returned HTTP 000 (DNS failure or connection refused)`
-              : `${endpoint} returned HTTP ${httpCode} (inference route reached but not usable — provider route/auth misconfigured or the local backend is failing)`,
-        };
-      } else {
-        // The exec ran but produced no sentinel — the sandbox runtime exec
-        // path itself is broken (e.g. sandbox in Error, exec denied). Treat as
-        // an exec failure, NOT a missing-curl soft-skip, so we never declare
-        // success without actually exercising the runtime (#4509 review).
-        const noise = (out || result.stderr || "").slice(0, 160);
-        last = {
-          kind: "exec-failed",
-          detail: `unexpected sandbox exec output: ${noise}`,
-        };
-      }
     }
-    if (attempt < DOCKER_GPU_INFERENCE_PROBE_MAX_ATTEMPTS) {
-      deps.sleep(DOCKER_GPU_INFERENCE_PROBE_RETRY_DELAY_SECS);
+
+    const out = (result.stdout || "").trim();
+    if (out === "NO_CURL") return { kind: "no-curl" };
+    const match = out.match(/HTTP_(\d{3})/);
+    if (match) {
+      const httpCode = match[1];
+      // This gate is the #4509 runtime proof, so only a 2xx — the model list
+      // actually returned through the proxy with injected auth — counts as
+      // success. `000` is the reported failure (DNS / connection refused). A
+      // 4xx (wrong provider route, or auth the proxy failed to inject) or 5xx
+      // (local Ollama/vLLM backend down) means OpenClaw's real request would
+      // fail too, so do NOT report it as reachable.
+      if (/^2\d\d$/.test(httpCode)) return { kind: "ok", httpCode };
+      return {
+        kind: "unreachable",
+        detail:
+          httpCode === "000"
+            ? `${endpoint} returned HTTP 000 (DNS failure or connection refused)`
+            : `${endpoint} returned HTTP ${httpCode} (inference route reached but not usable — provider route/auth misconfigured or the local backend is failing)`,
+      };
     }
-  }
-  return last;
+
+    // The exec ran but produced no sentinel — the sandbox runtime exec
+    // path itself is broken (e.g. sandbox in Error, exec denied). Treat as
+    // an exec failure, NOT a missing-curl soft-skip, so we never declare
+    // success without actually exercising the runtime (#4509 review).
+    const noise = (out || result.stderr || "").slice(0, 160);
+    return {
+      kind: "exec-failed",
+      detail: `unexpected sandbox exec output: ${noise}`,
+    };
+  };
+
+  return retryUntil(probe, {
+    accept: (result) => result.kind === "ok" || result.kind === "no-curl",
+    retryDelaysMs: Array.from(
+      { length: DOCKER_GPU_INFERENCE_PROBE_MAX_ATTEMPTS - 1 },
+      () => DOCKER_GPU_INFERENCE_PROBE_RETRY_DELAY_MS,
+    ),
+    sleep: deps.sleep,
+  });
 }
 
 /**
@@ -308,8 +311,8 @@ export function verifyDockerGpuSandboxLocalInference(
   const execInSandbox = deps.execInSandbox ?? executeSandboxCommandForVerification;
   const sleep =
     deps.sleep ??
-    ((seconds: number) => {
-      Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, Math.max(0, seconds) * 1000);
+    ((milliseconds: number) => {
+      Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, Math.max(0, milliseconds));
     });
 
   const outcome = probeSandboxRuntimeInference(options.sandboxName, endpoint, {

src/lib/onboard/docker-gpu-patch-rollback.ts

@@ -8,6 +8,7 @@ import {
   dockerStart as defaultDockerStart,
   dockerStop as defaultDockerStop,
 } from "../adapters/docker";
+import { retryUntil } from "../core/retry";
 import { hasZeroDockerExitStatus } from "./docker-command-result";
 import { fullDockerContainerId } from "./docker-gpu-patch-clone";
 import { DOCKER_GPU_PATCH_TIMEOUT_MS } from "./docker-gpu-patch-constants";
@@ -29,7 +30,7 @@ type DockerRenameFn = (
 type DockerRunFn = (args: readonly string[], opts?: DockerRunOptions) => DockerRunResult;
 
 const REPLACEMENT_PRESENCE_ATTEMPTS = 3;
-const REPLACEMENT_PRESENCE_RETRY_SECONDS = 0.5;
+const REPLACEMENT_PRESENCE_RETRY_MS = 500;
 
 function sleepBeforeReplacementPresenceRetry(seconds: number): void {
   if (seconds <= 0 || !Number.isFinite(seconds)) return;
@@ -77,23 +78,28 @@ function observeReplacementPresence(
 ): DockerGpuPatchRollbackOutcome["replacementPresence"] {
   const exactId = fullDockerContainerId(containerId);
   if (!exactId) return "unknown";
-  for (let attempt = 0; attempt < REPLACEMENT_PRESENCE_ATTEMPTS; attempt += 1) {
-    const result = deps.dockerRun(
-      ["ps", "-a", "--no-trunc", "--filter", `id=${exactId}`, "--format", "{{.ID}}"],
-      options,
-    );
-    if (hasZeroDockerExitStatus(result)) {
-      const ids = outputText(result.stdout)
-        .split(/\r?\n/u)
-        .map((value) => value.trim())
-        .filter(Boolean);
-      return ids.some((id) => fullDockerContainerId(id) === exactId) ? "present" : "absent";
-    }
-    if (attempt + 1 < REPLACEMENT_PRESENCE_ATTEMPTS) {
-      deps.sleep(REPLACEMENT_PRESENCE_RETRY_SECONDS);
-    }
-  }
-  return "unknown";
+  const result = retryUntil(
+    () =>
+      deps.dockerRun(
+        ["ps", "-a", "--no-trunc", "--filter", `id=${exactId}`, "--format", "{{.ID}}"],
+        options,
+      ),
+    {
+      accept: hasZeroDockerExitStatus,
+      retryDelaysMs: Array.from(
+        { length: REPLACEMENT_PRESENCE_ATTEMPTS - 1 },
+        () => REPLACEMENT_PRESENCE_RETRY_MS,
+      ),
+      sleep: (milliseconds) => deps.sleep(milliseconds / 1000),
+    },
+  );
+  if (!hasZeroDockerExitStatus(result)) return "unknown";
+
+  const ids = outputText(result.stdout)
+    .split(/\r?\n/u)
+    .map((value) => value.trim())
+    .filter(Boolean);
+  return ids.some((id) => fullDockerContainerId(id) === exactId) ? "present" : "absent";
 }
 
 export function rollbackToBackupContainer(