From 6d19fed0fcbf63ce7253719454c9bdb3171940a8 Mon Sep 17 00:00:00 2001 From: Tinson Lai Date: Wed, 6 May 2026 07:40:10 +0000 Subject: [PATCH 1/3] fix(onboard): retry inference validation on transient upstream failures Treat HTTP 502/503/504 as retriable in addition to 429, and extend the curl-timeout retry into the same backoff schedule. Hosted providers (NVIDIA Endpoints in particular) periodically emit these for tens of seconds at a time, currently failing onboard and nightly E2E on the first response. Closes #2980 Closes #3033 Signed-off-by: Tinson Lai --- src/lib/onboard-inference-probes.test.ts | 90 ++++++++++++++++++++++++ src/lib/onboard-inference-probes.ts | 32 +++++++-- 2 files changed, 116 insertions(+), 6 deletions(-) diff --git a/src/lib/onboard-inference-probes.test.ts b/src/lib/onboard-inference-probes.test.ts index 4ac57b184ac..44b8efbf91a 100644 --- a/src/lib/onboard-inference-probes.test.ts +++ b/src/lib/onboard-inference-probes.test.ts @@ -12,6 +12,7 @@ const { getDeepSeekV4ProValidationProbeCurlArgs, isSandboxInternalUrl, probeOpenAiLikeEndpoint, + RETRIABLE_HTTP_PROBE_STATUSES, } = require("../../dist/lib/onboard-inference-probes"); describe("OpenAI-compatible inference probes", () => { @@ -97,6 +98,95 @@ describe("OpenAI-compatible inference probes", () => { }); }); + describe("retriable HTTP statuses (issues #2980, #3033)", () => { + it("retries 429 (rate limit)", () => { + expect(RETRIABLE_HTTP_PROBE_STATUSES.has(429)).toBe(true); + }); + + it("retries 502/503/504 (upstream gateway flakes)", () => { + expect(RETRIABLE_HTTP_PROBE_STATUSES.has(502)).toBe(true); + expect(RETRIABLE_HTTP_PROBE_STATUSES.has(503)).toBe(true); + expect(RETRIABLE_HTTP_PROBE_STATUSES.has(504)).toBe(true); + }); + + it("does not retry on client-side or non-transient statuses", () => { + expect(RETRIABLE_HTTP_PROBE_STATUSES.has(400)).toBe(false); + expect(RETRIABLE_HTTP_PROBE_STATUSES.has(401)).toBe(false); + expect(RETRIABLE_HTTP_PROBE_STATUSES.has(403)).toBe(false); + expect(RETRIABLE_HTTP_PROBE_STATUSES.has(404)).toBe(false); + expect(RETRIABLE_HTTP_PROBE_STATUSES.has(500)).toBe(false); + expect(RETRIABLE_HTTP_PROBE_STATUSES.has(200)).toBe(false); + }); + + it("recovers when an upstream 502 clears on retry (regression #2980)", () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-502-probe-")); + const fakeBin = path.join(tmpDir, "bin"); + const counter = path.join(tmpDir, "counter"); + fs.mkdirSync(fakeBin, { recursive: true }); + fs.writeFileSync(counter, "0"); + fs.writeFileSync( + path.join(fakeBin, "curl"), + `#!/usr/bin/env bash +outfile="" +while [ "$#" -gt 0 ]; do + case "$1" in + -o) outfile="$2"; shift 2 ;; + -w) shift 2 ;; + *) shift ;; + esac +done +n=$(cat "${counter}") +n=$((n + 1)) +echo "$n" > "${counter}" +if [ "$n" -lt 2 ]; then + if [ -n "$outfile" ]; then + printf '502 Bad Gateway' > "$outfile" + fi + printf '502' + exit 0 +fi +if [ -n "$outfile" ]; then + cat <<'JSON' > "$outfile" +{"choices":[{"message":{"content":"OK"}}]} +JSON +fi +printf '200' +exit 0 +`, + { mode: 0o755 }, + ); + + const originalPath = process.env.PATH; + const originalNoSleep = process.env.NEMOCLAW_TEST_NO_SLEEP; + const originalLog = console.log; + const lines: string[] = []; + process.env.PATH = `${fakeBin}:${originalPath || ""}`; + process.env.NEMOCLAW_TEST_NO_SLEEP = "1"; + console.log = (...args) => lines.push(args.join(" ")); + try { + const result = probeOpenAiLikeEndpoint( + "https://integrate.api.nvidia.com/v1", + "nvidia/nemotron-3-super-120b-a12b", + "nvapi-test", + { skipResponsesProbe: true }, + ); + + expect(result).toMatchObject({ ok: true, api: "openai-completions" }); + expect(lines.join("\n")).toContain("HTTP 502"); + expect(fs.readFileSync(counter, "utf8").trim()).toBe("2"); + } finally { + console.log = originalLog; + process.env.PATH = originalPath; + if (originalNoSleep === undefined) { + delete process.env.NEMOCLAW_TEST_NO_SLEEP; + } else { + process.env.NEMOCLAW_TEST_NO_SLEEP = originalNoSleep; + } + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); + }); + it("continues with openai-completions when DeepSeek V4 Pro stream validation times out", () => { const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-deepseek-probe-")); const fakeBin = path.join(tmpDir, "bin"); diff --git a/src/lib/onboard-inference-probes.ts b/src/lib/onboard-inference-probes.ts index 1bb6569a19a..c3bd95c3689 100644 --- a/src/lib/onboard-inference-probes.ts +++ b/src/lib/onboard-inference-probes.ts @@ -110,11 +110,16 @@ function getProbeProcessTimeoutMs(args) { return (getCurlMaxTimeSeconds(args) + 5) * 1000; } -const RETRIABLE_HTTP_PROBE_STATUSES = new Set([429]); +// 429 = Too Many Requests; 502/503/504 = upstream gateway/availability flakes +// (NVIDIA Endpoints and other hosted providers periodically emit these for +// minutes at a time). All four are transient — retry with backoff before +// surfacing a hard failure to the wizard. See issues #2980 and #3033. +const RETRIABLE_HTTP_PROBE_STATUSES = new Set([429, 502, 503, 504]); const HTTP_PROBE_RETRY_DELAYS_MS = [5_000, 15_000, 30_000]; function sleepSync(ms) { if (ms <= 0) return; + if (process.env.NEMOCLAW_TEST_NO_SLEEP === "1") return; Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); } @@ -412,16 +417,18 @@ function probeOpenAiLikeEndpoint(endpointUrl, model, apiKey, options = {}) { }); } - // Single retry with doubled timeouts on timeout/connection failure. - // WSL2's virtualized network stack can cause the initial probe to time out - // before the TLS handshake completes. See issue #987. + // Retry with doubled timeouts on timeout/connection failure, using the same + // backoff schedule as transient HTTP statuses. WSL2's virtualized network + // stack can cause the initial probe to time out before the TLS handshake + // completes (#987); hosted providers also occasionally drop connections for + // tens of seconds during incidents (#3033). const isTimeoutOrConnFailure = (cs) => cs === 28 || cs === 6 || cs === 7; let retriedAfterTimeout = false; if (failures.length > 0 && isTimeoutOrConnFailure(failures[0].curlStatus)) { retriedAfterTimeout = true; const baseArgs = getValidationProbeCurlArgs(); const doubledArgs = baseArgs.map((arg) => (/^\d+$/.test(arg) ? String(Number(arg) * 2) : arg)); - const retryResult = runCurlProbe([ + const buildRetryArgs = () => [ "-sS", ...doubledArgs, "-H", @@ -430,10 +437,22 @@ function probeOpenAiLikeEndpoint(endpointUrl, model, apiKey, options = {}) { "-d", JSON.stringify(getChatCompletionsProbePayload(model)), `${String(endpointUrl).replace(/\/+$/, "")}/chat/completions`, - ]); + ]; + let retryResult = runCurlProbe(buildRetryArgs()); if (retryResult.ok) { return { ok: true, api: "openai-completions", label: "Chat Completions API" }; } + for (const delayMs of HTTP_PROBE_RETRY_DELAYS_MS) { + if (!isTimeoutOrConnFailure(retryResult.curlStatus)) break; + console.log( + ` Chat Completions API validation timed out; retrying in ${Math.round(delayMs / 1000)}s...`, + ); + sleepSync(delayMs); + retryResult = runCurlProbe(buildRetryArgs()); + if (retryResult.ok) { + return { ok: true, api: "openai-completions", label: "Chat Completions API" }; + } + } } // Detect the NVCF "Function not found for account" error and reframe it @@ -515,4 +534,5 @@ module.exports = { probeResponsesToolCalling, probeOpenAiLikeEndpoint, probeAnthropicEndpoint, + RETRIABLE_HTTP_PROBE_STATUSES, }; From c26d52fa0fa42bd6f162848fa182c4682439aace Mon Sep 17 00:00:00 2001 From: Tinson Lai Date: Wed, 6 May 2026 07:58:48 +0000 Subject: [PATCH 2/3] fix(onboard): skip retry sleeps under vitest The new curl-failure backoff loop ran real 5/15/30s sleeps under vitest, which exceeded the 15s test timeout in test/onboard-selection.test.ts:2565. Detect VITEST=true (set automatically by the test runner) and short-circuit sleepSync, in addition to the existing NEMOCLAW_TEST_NO_SLEEP opt-in. Signed-off-by: Tinson Lai --- src/lib/onboard-inference-probes.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/lib/onboard-inference-probes.ts b/src/lib/onboard-inference-probes.ts index c3bd95c3689..5ec083cfc4b 100644 --- a/src/lib/onboard-inference-probes.ts +++ b/src/lib/onboard-inference-probes.ts @@ -119,7 +119,10 @@ const HTTP_PROBE_RETRY_DELAYS_MS = [5_000, 15_000, 30_000]; function sleepSync(ms) { if (ms <= 0) return; - if (process.env.NEMOCLAW_TEST_NO_SLEEP === "1") return; + // Skip real waits under vitest so retry-loop coverage doesn't burn 50s of + // wall-clock per test. process.env.VITEST is set automatically by the + // test runner. + if (process.env.VITEST === "true" || process.env.NEMOCLAW_TEST_NO_SLEEP === "1") return; Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); } From 3c35f7c53da08fb1bdaa6b9b7fc1c41d9502e54a Mon Sep 17 00:00:00 2001 From: Tinson Lai Date: Wed, 6 May 2026 08:20:51 +0000 Subject: [PATCH 3/3] fix(onboard): broaden timeout-retry predicate and trigger condition Address two CodeRabbit findings on the inference probe retry loop: * The trigger condition checked only `failures[0].curlStatus`, so a probe ordering like /responses (HTTP error) followed by /chat/completions (curl 28) skipped the retry path entirely. Trigger on any timeout entry in the failure list. * The retry loop broke out as soon as the curl invocation succeeded at the transport level, even when the response was a retriable 5xx. Treat curl timeout/connection failure OR retriable HTTP status as a reason to keep retrying, and surface a status-aware "retrying in Ns..." log so the cause of each retry is clear. Add behavioural coverage for both flows: /responses 404 + chat-completions curl 28 + chat-completions 200 on retry, and curl 28 + 502 + 200. Signed-off-by: Tinson Lai --- src/lib/onboard-inference-probes.test.ts | 150 +++++++++++++++++++++++ src/lib/onboard-inference-probes.ts | 15 ++- 2 files changed, 162 insertions(+), 3 deletions(-) diff --git a/src/lib/onboard-inference-probes.test.ts b/src/lib/onboard-inference-probes.test.ts index 44b8efbf91a..fe82333e884 100644 --- a/src/lib/onboard-inference-probes.test.ts +++ b/src/lib/onboard-inference-probes.test.ts @@ -185,6 +185,156 @@ exit 0 fs.rmSync(tmpDir, { recursive: true, force: true }); } }); + + it("retries chat-completions when /responses errors then chat-completions times out", () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mixed-probe-")); + const fakeBin = path.join(tmpDir, "bin"); + const counter = path.join(tmpDir, "counter"); + fs.mkdirSync(fakeBin, { recursive: true }); + fs.writeFileSync(counter, "0"); + fs.writeFileSync( + path.join(fakeBin, "curl"), + `#!/usr/bin/env bash +outfile="" +url="" +while [ "$#" -gt 0 ]; do + case "$1" in + -o) outfile="$2"; shift 2 ;; + -w) shift 2 ;; + *) url="$1"; shift ;; + esac +done +n=$(cat "${counter}") +n=$((n + 1)) +echo "$n" > "${counter}" +if echo "$url" | grep -q '/responses'; then + if [ -n "$outfile" ]; then + printf '404 page not found' > "$outfile" + fi + printf '404' + exit 0 +fi +if [ "$n" -le 2 ]; then + if [ -n "$outfile" ]; then + : > "$outfile" + fi + printf '000' + exit 28 +fi +if [ -n "$outfile" ]; then + cat <<'JSON' > "$outfile" +{"choices":[{"message":{"content":"OK"}}]} +JSON +fi +printf '200' +exit 0 +`, + { mode: 0o755 }, + ); + + const originalPath = process.env.PATH; + const originalNoSleep = process.env.NEMOCLAW_TEST_NO_SLEEP; + const originalLog = console.log; + const lines: string[] = []; + process.env.PATH = `${fakeBin}:${originalPath || ""}`; + process.env.NEMOCLAW_TEST_NO_SLEEP = "1"; + console.log = (...args) => lines.push(args.join(" ")); + try { + const result = probeOpenAiLikeEndpoint( + "https://api.example.com/v1", + "test-model", + "sk-test", + ); + + expect(result).toMatchObject({ ok: true, api: "openai-completions" }); + // /responses (404) + /chat/completions (28) + chat-completions retry (200) + expect(fs.readFileSync(counter, "utf8").trim()).toBe("3"); + } finally { + console.log = originalLog; + process.env.PATH = originalPath; + if (originalNoSleep === undefined) { + delete process.env.NEMOCLAW_TEST_NO_SLEEP; + } else { + process.env.NEMOCLAW_TEST_NO_SLEEP = originalNoSleep; + } + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); + + it("keeps retrying when initial timeout is followed by a transient 502", () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-timeout-502-probe-")); + const fakeBin = path.join(tmpDir, "bin"); + const counter = path.join(tmpDir, "counter"); + fs.mkdirSync(fakeBin, { recursive: true }); + fs.writeFileSync(counter, "0"); + fs.writeFileSync( + path.join(fakeBin, "curl"), + `#!/usr/bin/env bash +outfile="" +while [ "$#" -gt 0 ]; do + case "$1" in + -o) outfile="$2"; shift 2 ;; + -w) shift 2 ;; + *) shift ;; + esac +done +n=$(cat "${counter}") +n=$((n + 1)) +echo "$n" > "${counter}" +if [ "$n" -eq 1 ]; then + if [ -n "$outfile" ]; then + : > "$outfile" + fi + printf '000' + exit 28 +fi +if [ "$n" -eq 2 ]; then + if [ -n "$outfile" ]; then + printf '502 Bad Gateway' > "$outfile" + fi + printf '502' + exit 0 +fi +if [ -n "$outfile" ]; then + cat <<'JSON' > "$outfile" +{"choices":[{"message":{"content":"OK"}}]} +JSON +fi +printf '200' +exit 0 +`, + { mode: 0o755 }, + ); + + const originalPath = process.env.PATH; + const originalNoSleep = process.env.NEMOCLAW_TEST_NO_SLEEP; + const originalLog = console.log; + const lines: string[] = []; + process.env.PATH = `${fakeBin}:${originalPath || ""}`; + process.env.NEMOCLAW_TEST_NO_SLEEP = "1"; + console.log = (...args) => lines.push(args.join(" ")); + try { + const result = probeOpenAiLikeEndpoint( + "https://integrate.api.nvidia.com/v1", + "nvidia/nemotron-3-super-120b-a12b", + "nvapi-test", + { skipResponsesProbe: true }, + ); + + expect(result).toMatchObject({ ok: true, api: "openai-completions" }); + expect(lines.join("\n")).toContain("HTTP 502"); + expect(fs.readFileSync(counter, "utf8").trim()).toBe("3"); + } finally { + console.log = originalLog; + process.env.PATH = originalPath; + if (originalNoSleep === undefined) { + delete process.env.NEMOCLAW_TEST_NO_SLEEP; + } else { + process.env.NEMOCLAW_TEST_NO_SLEEP = originalNoSleep; + } + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); }); it("continues with openai-completions when DeepSeek V4 Pro stream validation times out", () => { diff --git a/src/lib/onboard-inference-probes.ts b/src/lib/onboard-inference-probes.ts index 5ec083cfc4b..9b529233106 100644 --- a/src/lib/onboard-inference-probes.ts +++ b/src/lib/onboard-inference-probes.ts @@ -426,8 +426,14 @@ function probeOpenAiLikeEndpoint(endpointUrl, model, apiKey, options = {}) { // completes (#987); hosted providers also occasionally drop connections for // tens of seconds during incidents (#3033). const isTimeoutOrConnFailure = (cs) => cs === 28 || cs === 6 || cs === 7; + const isRetriableProbeResult = (result) => + isTimeoutOrConnFailure(result.curlStatus) || + RETRIABLE_HTTP_PROBE_STATUSES.has(result.httpStatus); + // Look across every failure entry rather than only failures[0] so a probe + // ordering like /responses (HTTP error) followed by /chat/completions + // (curl 28) still triggers the chat-completions retry path. let retriedAfterTimeout = false; - if (failures.length > 0 && isTimeoutOrConnFailure(failures[0].curlStatus)) { + if (failures.some((failure) => isTimeoutOrConnFailure(failure.curlStatus))) { retriedAfterTimeout = true; const baseArgs = getValidationProbeCurlArgs(); const doubledArgs = baseArgs.map((arg) => (/^\d+$/.test(arg) ? String(Number(arg) * 2) : arg)); @@ -446,9 +452,12 @@ function probeOpenAiLikeEndpoint(endpointUrl, model, apiKey, options = {}) { return { ok: true, api: "openai-completions", label: "Chat Completions API" }; } for (const delayMs of HTTP_PROBE_RETRY_DELAYS_MS) { - if (!isTimeoutOrConnFailure(retryResult.curlStatus)) break; + if (!isRetriableProbeResult(retryResult)) break; + const reason = isTimeoutOrConnFailure(retryResult.curlStatus) + ? "timed out" + : `returned HTTP ${retryResult.httpStatus}`; console.log( - ` Chat Completions API validation timed out; retrying in ${Math.round(delayMs / 1000)}s...`, + ` Chat Completions API validation ${reason}; retrying in ${Math.round(delayMs / 1000)}s...`, ); sleepSync(delayMs); retryResult = runCurlProbe(buildRetryArgs());