NVIDIA · cv · May 6, 2026 · May 6, 2026 · May 6, 2026 · May 6, 2026
diff --git a/src/lib/onboard-inference-probes.test.ts b/src/lib/onboard-inference-probes.test.ts
@@ -13,6 +13,7 @@ const {
   getKimiK26ValidationProbeCurlArgs,
   isSandboxInternalUrl,
   probeOpenAiLikeEndpoint,
+  RETRIABLE_HTTP_PROBE_STATUSES,
 } = require("../../dist/lib/onboard-inference-probes");
 
 describe("OpenAI-compatible inference probes", () => {
@@ -130,6 +131,245 @@ describe("OpenAI-compatible inference probes", () => {
     });
   });
 
+  describe("retriable HTTP statuses (issues #2980, #3033)", () => {
+    it("retries 429 (rate limit)", () => {
+      expect(RETRIABLE_HTTP_PROBE_STATUSES.has(429)).toBe(true);
+    });
+
+    it("retries 502/503/504 (upstream gateway flakes)", () => {
+      expect(RETRIABLE_HTTP_PROBE_STATUSES.has(502)).toBe(true);
+      expect(RETRIABLE_HTTP_PROBE_STATUSES.has(503)).toBe(true);
+      expect(RETRIABLE_HTTP_PROBE_STATUSES.has(504)).toBe(true);
+    });
+
+    it("does not retry on client-side or non-transient statuses", () => {
+      expect(RETRIABLE_HTTP_PROBE_STATUSES.has(400)).toBe(false);
+      expect(RETRIABLE_HTTP_PROBE_STATUSES.has(401)).toBe(false);
+      expect(RETRIABLE_HTTP_PROBE_STATUSES.has(403)).toBe(false);
+      expect(RETRIABLE_HTTP_PROBE_STATUSES.has(404)).toBe(false);
+      expect(RETRIABLE_HTTP_PROBE_STATUSES.has(500)).toBe(false);
+      expect(RETRIABLE_HTTP_PROBE_STATUSES.has(200)).toBe(false);
+    });
+
+    it("recovers when an upstream 502 clears on retry (regression #2980)", () => {
+      const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-502-probe-"));
+      const fakeBin = path.join(tmpDir, "bin");
+      const counter = path.join(tmpDir, "counter");
+      fs.mkdirSync(fakeBin, { recursive: true });
+      fs.writeFileSync(counter, "0");
+      fs.writeFileSync(
+        path.join(fakeBin, "curl"),
+        `#!/usr/bin/env bash
+outfile=""
+while [ "$#" -gt 0 ]; do
+  case "$1" in
+    -o) outfile="$2"; shift 2 ;;
+    -w) shift 2 ;;
+    *) shift ;;
+  esac
+done
+n=$(cat "${counter}")
+n=$((n + 1))
+echo "$n" > "${counter}"
+if [ "$n" -lt 2 ]; then
+  if [ -n "$outfile" ]; then
+    printf '<html>502 Bad Gateway</html>' > "$outfile"
+  fi
+  printf '502'
+  exit 0
+fi
+if [ -n "$outfile" ]; then
+  cat <<'JSON' > "$outfile"
+{"choices":[{"message":{"content":"OK"}}]}
+JSON
+fi
+printf '200'
+exit 0
+`,
+        { mode: 0o755 },
+      );
+
+      const originalPath = process.env.PATH;
+      const originalNoSleep = process.env.NEMOCLAW_TEST_NO_SLEEP;
+      const originalLog = console.log;
+      const lines: string[] = [];
+      process.env.PATH = `${fakeBin}:${originalPath || ""}`;
+      process.env.NEMOCLAW_TEST_NO_SLEEP = "1";
+      console.log = (...args) => lines.push(args.join(" "));
+      try {
+        const result = probeOpenAiLikeEndpoint(
+          "https://integrate.api.nvidia.com/v1",
+          "nvidia/nemotron-3-super-120b-a12b",
+          "nvapi-test",
+          { skipResponsesProbe: true },
+        );
+
+        expect(result).toMatchObject({ ok: true, api: "openai-completions" });
+        expect(lines.join("\n")).toContain("HTTP 502");
+        expect(fs.readFileSync(counter, "utf8").trim()).toBe("2");
+      } finally {
+        console.log = originalLog;
+        process.env.PATH = originalPath;
+        if (originalNoSleep === undefined) {
+          delete process.env.NEMOCLAW_TEST_NO_SLEEP;
+        } else {
+          process.env.NEMOCLAW_TEST_NO_SLEEP = originalNoSleep;
+        }
+        fs.rmSync(tmpDir, { recursive: true, force: true });
+      }
+    });
+
+    it("retries chat-completions when /responses errors then chat-completions times out", () => {
+      const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mixed-probe-"));
+      const fakeBin = path.join(tmpDir, "bin");
+      const counter = path.join(tmpDir, "counter");
+      fs.mkdirSync(fakeBin, { recursive: true });
+      fs.writeFileSync(counter, "0");
+      fs.writeFileSync(
+        path.join(fakeBin, "curl"),
+        `#!/usr/bin/env bash
+outfile=""
+url=""
+while [ "$#" -gt 0 ]; do
+  case "$1" in
+    -o) outfile="$2"; shift 2 ;;
+    -w) shift 2 ;;
+    *) url="$1"; shift ;;
+  esac
+done
+n=$(cat "${counter}")
+n=$((n + 1))
+echo "$n" > "${counter}"
+if echo "$url" | grep -q '/responses'; then
+  if [ -n "$outfile" ]; then
+    printf '404 page not found' > "$outfile"
+  fi
+  printf '404'
+  exit 0
+fi
+if [ "$n" -le 2 ]; then
+  if [ -n "$outfile" ]; then
+    : > "$outfile"
+  fi
+  printf '000'
+  exit 28
+fi
+if [ -n "$outfile" ]; then
+  cat <<'JSON' > "$outfile"
+{"choices":[{"message":{"content":"OK"}}]}
+JSON
+fi
+printf '200'
+exit 0
+`,
+        { mode: 0o755 },
+      );
+
+      const originalPath = process.env.PATH;
+      const originalNoSleep = process.env.NEMOCLAW_TEST_NO_SLEEP;
+      const originalLog = console.log;
+      const lines: string[] = [];
+      process.env.PATH = `${fakeBin}:${originalPath || ""}`;
+      process.env.NEMOCLAW_TEST_NO_SLEEP = "1";
+      console.log = (...args) => lines.push(args.join(" "));
+      try {
+        const result = probeOpenAiLikeEndpoint(
+          "https://api.example.com/v1",
+          "test-model",
+          "sk-test",
+        );
+
+        expect(result).toMatchObject({ ok: true, api: "openai-completions" });
+        // /responses (404) + /chat/completions (28) + chat-completions retry (200)
+        expect(fs.readFileSync(counter, "utf8").trim()).toBe("3");
+      } finally {
+        console.log = originalLog;
+        process.env.PATH = originalPath;
+        if (originalNoSleep === undefined) {
+          delete process.env.NEMOCLAW_TEST_NO_SLEEP;
+        } else {
+          process.env.NEMOCLAW_TEST_NO_SLEEP = originalNoSleep;
+        }
+        fs.rmSync(tmpDir, { recursive: true, force: true });
+      }
+    });
+
+    it("keeps retrying when initial timeout is followed by a transient 502", () => {
+      const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-timeout-502-probe-"));
+      const fakeBin = path.join(tmpDir, "bin");
+      const counter = path.join(tmpDir, "counter");
+      fs.mkdirSync(fakeBin, { recursive: true });
+      fs.writeFileSync(counter, "0");
+      fs.writeFileSync(
+        path.join(fakeBin, "curl"),
+        `#!/usr/bin/env bash
+outfile=""
+while [ "$#" -gt 0 ]; do
+  case "$1" in
+    -o) outfile="$2"; shift 2 ;;
+    -w) shift 2 ;;
+    *) shift ;;
+  esac
+done
+n=$(cat "${counter}")
+n=$((n + 1))
+echo "$n" > "${counter}"
+if [ "$n" -eq 1 ]; then
+  if [ -n "$outfile" ]; then
+    : > "$outfile"
+  fi
+  printf '000'
+  exit 28
+fi
+if [ "$n" -eq 2 ]; then
+  if [ -n "$outfile" ]; then
+    printf '<html>502 Bad Gateway</html>' > "$outfile"
+  fi
+  printf '502'
+  exit 0
+fi
+if [ -n "$outfile" ]; then
+  cat <<'JSON' > "$outfile"
+{"choices":[{"message":{"content":"OK"}}]}
+JSON
+fi
+printf '200'
+exit 0
+`,
+        { mode: 0o755 },
+      );
+
+      const originalPath = process.env.PATH;
+      const originalNoSleep = process.env.NEMOCLAW_TEST_NO_SLEEP;
+      const originalLog = console.log;
+      const lines: string[] = [];
+      process.env.PATH = `${fakeBin}:${originalPath || ""}`;
+      process.env.NEMOCLAW_TEST_NO_SLEEP = "1";
+      console.log = (...args) => lines.push(args.join(" "));
+      try {
+        const result = probeOpenAiLikeEndpoint(
+          "https://integrate.api.nvidia.com/v1",
+          "nvidia/nemotron-3-super-120b-a12b",
+          "nvapi-test",
+          { skipResponsesProbe: true },
+        );
+
+        expect(result).toMatchObject({ ok: true, api: "openai-completions" });
+        expect(lines.join("\n")).toContain("HTTP 502");
+        expect(fs.readFileSync(counter, "utf8").trim()).toBe("3");
+      } finally {
+        console.log = originalLog;
+        process.env.PATH = originalPath;
+        if (originalNoSleep === undefined) {
+          delete process.env.NEMOCLAW_TEST_NO_SLEEP;
+        } else {
+          process.env.NEMOCLAW_TEST_NO_SLEEP = originalNoSleep;
+        }
+        fs.rmSync(tmpDir, { recursive: true, force: true });
+      }
+    });
+  });
+
   it("continues with openai-completions when DeepSeek V4 Pro stream validation times out", () => {
     const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-deepseek-probe-"));
     const fakeBin = path.join(tmpDir, "bin");

diff --git a/src/lib/onboard-inference-probes.ts b/src/lib/onboard-inference-probes.ts
@@ -117,11 +117,19 @@ function getProbeProcessTimeoutMs(args) {
   return (getCurlMaxTimeSeconds(args) + 5) * 1000;
 }
 
-const RETRIABLE_HTTP_PROBE_STATUSES = new Set([429]);
+// 429 = Too Many Requests; 502/503/504 = upstream gateway/availability flakes
+// (NVIDIA Endpoints and other hosted providers periodically emit these for
+// minutes at a time). All four are transient — retry with backoff before
+// surfacing a hard failure to the wizard. See issues #2980 and #3033.
+const RETRIABLE_HTTP_PROBE_STATUSES = new Set([429, 502, 503, 504]);
 const HTTP_PROBE_RETRY_DELAYS_MS = [5_000, 15_000, 30_000];
 
 function sleepSync(ms) {
   if (ms <= 0) return;
+  // Skip real waits under vitest so retry-loop coverage doesn't burn 50s of
+  // wall-clock per test. process.env.VITEST is set automatically by the
+  // test runner.
+  if (process.env.VITEST === "true" || process.env.NEMOCLAW_TEST_NO_SLEEP === "1") return;
   Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
 }
 
@@ -432,16 +440,24 @@ function probeOpenAiLikeEndpoint(endpointUrl, model, apiKey, options = {}) {
     });
   }
 
-  // Single retry with doubled timeouts on timeout/connection failure.
-  // WSL2's virtualized network stack can cause the initial probe to time out
-  // before the TLS handshake completes. See issue #987.
+  // Retry with doubled timeouts on timeout/connection failure, using the same
+  // backoff schedule as transient HTTP statuses. WSL2's virtualized network
+  // stack can cause the initial probe to time out before the TLS handshake
+  // completes (#987); hosted providers also occasionally drop connections for
+  // tens of seconds during incidents (#3033).
   const isTimeoutOrConnFailure = (cs) => cs === 28 || cs === 6 || cs === 7;
+  const isRetriableProbeResult = (result) =>
+    isTimeoutOrConnFailure(result.curlStatus) ||
+    RETRIABLE_HTTP_PROBE_STATUSES.has(result.httpStatus);
+  // Look across every failure entry rather than only failures[0] so a probe
+  // ordering like /responses (HTTP error) followed by /chat/completions
+  // (curl 28) still triggers the chat-completions retry path.
   let retriedAfterTimeout = false;
-  if (failures.length > 0 && isTimeoutOrConnFailure(failures[0].curlStatus)) {
+  if (failures.some((failure) => isTimeoutOrConnFailure(failure.curlStatus))) {
     retriedAfterTimeout = true;
     const baseArgs = getValidationProbeCurlArgs();
     const doubledArgs = baseArgs.map((arg) => (/^\d+$/.test(arg) ? String(Number(arg) * 2) : arg));
-    const retryResult = runCurlProbe([
+    const buildRetryArgs = () => [
       "-sS",
       ...doubledArgs,
       "-H",
@@ -450,10 +466,25 @@ function probeOpenAiLikeEndpoint(endpointUrl, model, apiKey, options = {}) {
       "-d",
       JSON.stringify(getChatCompletionsProbePayload(model)),
       `${String(endpointUrl).replace(/\/+$/, "")}/chat/completions`,
-    ]);
+    ];
+    let retryResult = runCurlProbe(buildRetryArgs());
     if (retryResult.ok) {
       return { ok: true, api: "openai-completions", label: "Chat Completions API" };
     }
+    for (const delayMs of HTTP_PROBE_RETRY_DELAYS_MS) {
+      if (!isRetriableProbeResult(retryResult)) break;
+      const reason = isTimeoutOrConnFailure(retryResult.curlStatus)
+        ? "timed out"
+        : `returned HTTP ${retryResult.httpStatus}`;
+      console.log(
+        `  Chat Completions API validation ${reason}; retrying in ${Math.round(delayMs / 1000)}s...`,
+      );
+      sleepSync(delayMs);
+      retryResult = runCurlProbe(buildRetryArgs());
+      if (retryResult.ok) {
+        return { ok: true, api: "openai-completions", label: "Chat Completions API" };
+      }
+    }
   }
 
   // Detect the NVCF "Function not found for account" error and reframe it
@@ -536,4 +567,5 @@ module.exports = {
   probeResponsesToolCalling,
   probeOpenAiLikeEndpoint,
   probeAnthropicEndpoint,
+  RETRIABLE_HTTP_PROBE_STATUSES,
 };