From 3a59afebee9c642b25824cadbb5f161c7665a845 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 28 May 2026 10:59:17 -0700 Subject: [PATCH 1/2] fix(onboard): honor extended validation timeout budgets Signed-off-by: Aaron Erickson --- src/lib/adapters/http/probe.test.ts | 48 +++++++++++++ src/lib/adapters/http/probe.ts | 39 +++++++++-- src/lib/inference/onboard-probes.test.ts | 48 +++++++++++++ src/lib/inference/onboard-probes.ts | 87 +++++++++++++++++------- 4 files changed, 195 insertions(+), 27 deletions(-) diff --git a/src/lib/adapters/http/probe.test.ts b/src/lib/adapters/http/probe.test.ts index ad07ed6f55f..e27aefb1865 100644 --- a/src/lib/adapters/http/probe.test.ts +++ b/src/lib/adapters/http/probe.test.ts @@ -91,6 +91,54 @@ describe("http-probe helpers", () => { expect(fs.existsSync(path.dirname(outputPath))).toBe(false); }); + it("lets the process wrapper outlive curl --max-time", () => { + let timeout: number | undefined; + const result = runCurlProbe(["-sS", "--max-time", "60", "https://example.test/models"], { + spawnSyncImpl: (_command, args, options) => { + timeout = options.timeout; + const outputPath = args[args.indexOf("-o") + 1]; + if (typeof outputPath === "string") { + fs.writeFileSync(outputPath, "{}"); + } + return { + pid: 1, + output: [], + stdout: "200", + stderr: "", + status: 0, + signal: null, + }; + }, + }); + + expect(result.ok).toBe(true); + expect(timeout).toBe(65_000); + }); + + it("honors an explicit process timeout over inferred curl --max-time", () => { + let timeout: number | undefined; + runCurlProbe(["-sS", "--max-time", "60", "https://example.test/models"], { + timeoutMs: 12_345, + spawnSyncImpl: (_command, args, options) => { + timeout = options.timeout; + const outputPath = args[args.indexOf("-o") + 1]; + if (typeof outputPath === "string") { + fs.writeFileSync(outputPath, "{}"); + } + return { + pid: 1, + output: [], + stdout: "200", + stderr: "", + status: 0, + signal: null, + }; + }, + }); + + expect(timeout).toBe(12_345); + }); + it("reports spawn errors as curl failures", () => { const result = runCurlProbe(["-sS", "https://example.test/models"], { spawnSyncImpl: () => { diff --git a/src/lib/adapters/http/probe.ts b/src/lib/adapters/http/probe.ts index e787e0dcad5..00c6c7f33bf 100644 --- a/src/lib/adapters/http/probe.ts +++ b/src/lib/adapters/http/probe.ts @@ -35,6 +35,9 @@ export interface StreamingProbeResult { message: string; } +const DEFAULT_CURL_PROCESS_TIMEOUT_MS = 30_000; +const CURL_PROCESS_TIMEOUT_SLACK_MS = 5_000; + function validateTempPrefix(prefix: string): string { if ( prefix.length === 0 || @@ -69,6 +72,31 @@ export function getCurlTimingArgs(): string[] { return ["--connect-timeout", "10", "--max-time", "60"]; } +function getCurlMaxTimeSeconds(argv: string[]): number | null { + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === "--max-time") { + const value = Number(argv[index + 1]); + return Number.isFinite(value) && value > 0 ? value : null; + } + if (arg.startsWith("--max-time=")) { + const value = Number(arg.slice("--max-time=".length)); + return Number.isFinite(value) && value > 0 ? value : null; + } + } + return null; +} + +function resolveCurlProcessTimeoutMs(argv: string[], opts: CurlProbeOptions): number { + if (opts.timeoutMs !== undefined) return opts.timeoutMs; + const maxTimeSeconds = getCurlMaxTimeSeconds(argv); + if (maxTimeSeconds === null) return DEFAULT_CURL_PROCESS_TIMEOUT_MS; + return Math.max( + DEFAULT_CURL_PROCESS_TIMEOUT_MS, + Math.ceil(maxTimeSeconds * 1000) + CURL_PROCESS_TIMEOUT_SLACK_MS, + ); +} + function sanitizeCurlUrl(value: string): string { try { const url = new URL(value); @@ -92,7 +120,7 @@ function getCurlProbeTraceAttributes(argv: string[], opts: CurlProbeOptions): Re return { "http.url": sanitizeCurlUrl(String(url)), "http.request.method": method, - "process.timeout_ms": opts.timeoutMs ?? 30_000, + "process.timeout_ms": resolveCurlProcessTimeoutMs(argv, opts), }; } @@ -173,13 +201,14 @@ function runCurlProbeImpl(argv: string[], opts: CurlProbeOptions = {}): CurlProb const args = [...argv]; const url = args.pop(); const spawnSyncImpl = opts.spawnSyncImpl ?? spawnSync; + const timeout = resolveCurlProcessTimeoutMs(argv, opts); const result = spawnSyncImpl( "curl", [...args, "-o", bodyFile, "-w", "%{http_code}", String(url || "")], { cwd: opts.cwd ?? ROOT, encoding: "utf8", - timeout: opts.timeoutMs ?? 30_000, + timeout, env: opts.replaceEnv ? (opts.env ?? {}) : { ...process.env, ...opts.env }, }, ); @@ -283,13 +312,14 @@ function runChatCompletionsStreamingProbeImpl( const args = [...argv]; const url = args.pop(); const spawnSyncImpl = opts.spawnSyncImpl ?? spawnSync; + const timeout = resolveCurlProcessTimeoutMs(argv, opts); const result = spawnSyncImpl( "curl", [...args, "-N", "-o", bodyFile, "-w", "%{http_code}", String(url || "")], { cwd: opts.cwd ?? ROOT, encoding: "utf8", - timeout: opts.timeoutMs ?? 30_000, + timeout, env: { ...process.env, ...opts.env, @@ -405,10 +435,11 @@ function runStreamingEventProbeImpl( const args = [...argv]; const url = args.pop(); const spawnSyncImpl = opts.spawnSyncImpl ?? spawnSync; + const timeout = resolveCurlProcessTimeoutMs(argv, opts); const result = spawnSyncImpl("curl", [...args, "-N", "-o", bodyFile, String(url || "")], { cwd: opts.cwd ?? ROOT, encoding: "utf8", - timeout: opts.timeoutMs ?? 30_000, + timeout, env: { ...process.env, ...opts.env, diff --git a/src/lib/inference/onboard-probes.test.ts b/src/lib/inference/onboard-probes.test.ts index 20a4a0cf3e0..562e23be07e 100644 --- a/src/lib/inference/onboard-probes.test.ts +++ b/src/lib/inference/onboard-probes.test.ts @@ -11,6 +11,7 @@ const { getChatCompletionsProbePayload, getDeepSeekV4ProValidationProbeCurlArgs, getKimiK26ValidationProbeCurlArgs, + getValidationProbeCurlArgs, hasChatCompletionsToolCall, hasChatCompletionsToolCallLeak, hasResponsesToolCall, @@ -289,6 +290,53 @@ describe("OpenAI-compatible inference probes", () => { }); }); + it("allows onboard validation max-time to be raised from the environment", () => { + const original = process.env.NEMOCLAW_ONBOARD_VALIDATION_TIMEOUT_SECONDS; + process.env.NEMOCLAW_ONBOARD_VALIDATION_TIMEOUT_SECONDS = "300"; + try { + expect(getValidationProbeCurlArgs({ isWsl: false })).toEqual([ + "--connect-timeout", + "10", + "--max-time", + "300", + ]); + expect(getKimiK26ValidationProbeCurlArgs({ isWsl: false })).toEqual([ + "--connect-timeout", + "10", + "--max-time", + "300", + ]); + } finally { + if (original === undefined) { + delete process.env.NEMOCLAW_ONBOARD_VALIDATION_TIMEOUT_SECONDS; + } else { + process.env.NEMOCLAW_ONBOARD_VALIDATION_TIMEOUT_SECONDS = original; + } + } + }); + + it("uses an extended validation budget for slow NVIDIA Build models", () => { + for (const model of ["qwen/qwen3.5-397b-a17b", "deepseek-ai/deepseek-v4-flash"]) { + const args = getChatCompletionsProbeCurlArgs({ + authHeader: ["-H", "Authorization: Bearer nvapi-test"], + model, + url: "https://integrate.api.nvidia.com/v1/chat/completions", + isWsl: false, + }); + expect(args[args.indexOf("--connect-timeout") + 1]).toBe("10"); + expect(args[args.indexOf("--max-time") + 1]).toBe("300"); + } + + const wslArgs = getChatCompletionsProbeCurlArgs({ + authHeader: ["-H", "Authorization: Bearer nvapi-test"], + model: "qwen/qwen3.5-397b-a17b", + url: "https://integrate.api.nvidia.com/v1/chat/completions", + isWsl: true, + }); + expect(wslArgs[wslArgs.indexOf("--connect-timeout") + 1]).toBe("30"); + expect(wslArgs[wslArgs.indexOf("--max-time") + 1]).toBe("300"); + }); + it("caps Kimi K2.6 probe output and gives it a slower validation budget", () => { expect(getChatCompletionsProbePayload("moonshotai/kimi-k2.6")).toEqual({ model: "moonshotai/kimi-k2.6", diff --git a/src/lib/inference/onboard-probes.ts b/src/lib/inference/onboard-probes.ts index a86a1afb957..918d481cd8f 100644 --- a/src/lib/inference/onboard-probes.ts +++ b/src/lib/inference/onboard-probes.ts @@ -28,6 +28,12 @@ const trace = require("../trace"); // ── Helpers ────────────────────────────────────────────────────── +const ONBOARD_VALIDATION_TIMEOUT_ENV = "NEMOCLAW_ONBOARD_VALIDATION_TIMEOUT_SECONDS"; +const EXTENDED_NVIDIA_ENDPOINT_VALIDATION_MODELS = new Set([ + "qwen/qwen3.5-397b-a17b", + "deepseek-ai/deepseek-v4-flash", +]); + // Hostnames that are normally meant for the sandbox/container host boundary. // host.openshell.internal only resolves inside the OpenShell sandbox network, // so host-side validation cannot prove reachability for that URL. For ordinary @@ -172,24 +178,31 @@ function getProbeAuthMode(_provider) { // getCurlTimingArgs() because validation must not hang the wizard for a // minute on a misbehaving model. See issue #1601 (Bug 3). function getValidationProbeCurlArgs(opts) { - if (isWsl(opts)) { - return ["--connect-timeout", "20", "--max-time", "30"]; - } - return ["--connect-timeout", "10", "--max-time", "15"]; + const args = isWsl(opts) + ? ["--connect-timeout", "20", "--max-time", "30"] + : ["--connect-timeout", "10", "--max-time", "15"]; + return withValidationMaxTimeOverride(args); } function getDeepSeekV4ProValidationProbeCurlArgs(opts) { - if (isWsl(opts)) { - return ["--connect-timeout", "30", "--max-time", "150"]; - } - return ["--connect-timeout", "20", "--max-time", "120"]; + const args = isWsl(opts) + ? ["--connect-timeout", "30", "--max-time", "150"] + : ["--connect-timeout", "20", "--max-time", "120"]; + return withValidationMaxTimeOverride(args); } function getKimiK26ValidationProbeCurlArgs(opts) { - if (isWsl(opts)) { - return ["--connect-timeout", "20", "--max-time", "90"]; - } - return ["--connect-timeout", "10", "--max-time", "60"]; + const args = isWsl(opts) + ? ["--connect-timeout", "20", "--max-time", "90"] + : ["--connect-timeout", "10", "--max-time", "60"]; + return withValidationMaxTimeOverride(args); +} + +function getExtendedNvidiaEndpointValidationProbeCurlArgs(opts) { + const args = isWsl(opts) + ? ["--connect-timeout", "30", "--max-time", "300"] + : ["--connect-timeout", "10", "--max-time", "300"]; + return withValidationMaxTimeOverride(args); } function getCurlMaxTimeSeconds(args) { @@ -199,6 +212,19 @@ function getCurlMaxTimeSeconds(args) { return Number.isFinite(value) && value > 0 ? value : 30; } +function withValidationMaxTimeOverride(args) { + const raw = (process.env[ONBOARD_VALIDATION_TIMEOUT_ENV] || "").trim(); + if (!raw) return args; + const overrideSeconds = Math.ceil(Number(raw)); + if (!Number.isFinite(overrideSeconds) || overrideSeconds <= 0) return args; + if (overrideSeconds <= getCurlMaxTimeSeconds(args)) return args; + const maxTimeIndex = args.indexOf("--max-time"); + if (maxTimeIndex === -1) return args; + const next = [...args]; + next[maxTimeIndex + 1] = String(overrideSeconds); + return next; +} + function getProbeProcessTimeoutMs(args) { return (getCurlMaxTimeSeconds(args) + 5) * 1000; } @@ -334,8 +360,8 @@ function probeChatCompletionsToolCalling(endpointUrl, model, apiKey, options = { const url = useQueryParam && normalizedKey ? `${baseUrl}/chat/completions?key=${encodeURIComponent(normalizedKey)}` : `${baseUrl}/chat/completions`; - const timingArgs = options.timingArgs ?? getValidationProbeCurlArgs(); - const result = runCurlProbe([ + const timingArgs = options.timingArgs ?? getChatCompletionsProbeTimingArgs(model); + const args = [ "-sS", ...timingArgs, "-H", @@ -401,7 +427,8 @@ function probeChatCompletionsToolCalling(endpointUrl, model, apiKey, options = { temperature: 0, }), url, - ]); + ]; + const result = runCurlProbe(args, { timeoutMs: getProbeProcessTimeoutMs(args) }); if (!result.ok) { return result; @@ -441,6 +468,19 @@ function isKimiK26Model(model) { return String(model || "").toLowerCase() === "moonshotai/kimi-k2.6"; } +function needsExtendedNvidiaEndpointValidationBudget(model) { + return EXTENDED_NVIDIA_ENDPOINT_VALIDATION_MODELS.has(String(model || "").toLowerCase()); +} + +function getChatCompletionsProbeTimingArgs(model, opts) { + if (isDeepSeekV4ProModel(model)) return getDeepSeekV4ProValidationProbeCurlArgs(opts); + if (isKimiK26Model(model)) return getKimiK26ValidationProbeCurlArgs(opts); + if (needsExtendedNvidiaEndpointValidationBudget(model)) { + return getExtendedNvidiaEndpointValidationProbeCurlArgs(opts); + } + return getValidationProbeCurlArgs(opts); +} + function getChatCompletionsProbePayload(model) { const payload = { model, @@ -477,11 +517,7 @@ export function getChatCompletionsProbeCurlArgs({ }) { const platformOptions = typeof isWslOverride === "boolean" ? { isWsl: isWslOverride } : undefined; - const timingArgs = (() => { - if (isDeepSeekV4ProModel(model)) return getDeepSeekV4ProValidationProbeCurlArgs(platformOptions); - if (isKimiK26Model(model)) return getKimiK26ValidationProbeCurlArgs(platformOptions); - return getValidationProbeCurlArgs(platformOptions); - })(); + const timingArgs = getChatCompletionsProbeTimingArgs(model, platformOptions); return [ "-sS", ...timingArgs, @@ -506,7 +542,7 @@ function runChatCompletionsProbe({ authHeader, model, url, isWsl: isWslOverride timeoutMs: getProbeProcessTimeoutMs(args), }); } - return runCurlProbe(args); + return runCurlProbe(args, { timeoutMs: getProbeProcessTimeoutMs(args) }); } function probeOpenAiLikeEndpoint(endpointUrl, model, apiKey, options = {}) { @@ -699,7 +735,9 @@ function probeOpenAiLikeEndpoint(endpointUrl, model, apiKey, options = {}) { let retriedAfterTimeout = false; if (failures.some((failure) => isTimeoutOrConnFailure(failure.curlStatus))) { retriedAfterTimeout = true; - const baseArgs = getValidationProbeCurlArgs(); + const platformOptions = + typeof options.isWsl === "boolean" ? { isWsl: options.isWsl } : undefined; + const baseArgs = getChatCompletionsProbeTimingArgs(model, platformOptions); const doubledArgs = baseArgs.map((arg) => (/^\d+$/.test(arg) ? String(Number(arg) * 2) : arg)); const buildRetryArgs = () => [ "-sS", @@ -717,7 +755,10 @@ function probeOpenAiLikeEndpoint(endpointUrl, model, apiKey, options = {}) { authMode: options.authMode, timingArgs: doubledArgs, }) - : runCurlProbe(buildRetryArgs()); + : (() => { + const retryArgs = buildRetryArgs(); + return runCurlProbe(retryArgs, { timeoutMs: getProbeProcessTimeoutMs(retryArgs) }); + })(); let retryResult = runRetryProbe(); if (retryResult.ok) { return { ok: true, api: "openai-completions", label: "Chat Completions API" }; From e1437478c387abc8f1bee19993f24d023373a07f Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 28 May 2026 12:18:05 -0700 Subject: [PATCH 2/2] fix(http): match curl max-time precedence Signed-off-by: Aaron Erickson --- src/lib/adapters/http/probe.test.ts | 26 ++++++++++++++++++++++++++ src/lib/adapters/http/probe.ts | 12 +++++++++--- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/src/lib/adapters/http/probe.test.ts b/src/lib/adapters/http/probe.test.ts index e27aefb1865..275e1479526 100644 --- a/src/lib/adapters/http/probe.test.ts +++ b/src/lib/adapters/http/probe.test.ts @@ -115,6 +115,32 @@ describe("http-probe helpers", () => { expect(timeout).toBe(65_000); }); + it("uses the last curl --max-time when the flag is repeated", () => { + let timeout: number | undefined; + runCurlProbe( + ["-sS", "--max-time", "15", "--max-time", "120", "https://example.test/models"], + { + spawnSyncImpl: (_command, args, options) => { + timeout = options.timeout; + const outputPath = args[args.indexOf("-o") + 1]; + if (typeof outputPath === "string") { + fs.writeFileSync(outputPath, "{}"); + } + return { + pid: 1, + output: [], + stdout: "200", + stderr: "", + status: 0, + signal: null, + }; + }, + }, + ); + + expect(timeout).toBe(125_000); + }); + it("honors an explicit process timeout over inferred curl --max-time", () => { let timeout: number | undefined; runCurlProbe(["-sS", "--max-time", "60", "https://example.test/models"], { diff --git a/src/lib/adapters/http/probe.ts b/src/lib/adapters/http/probe.ts index 00c6c7f33bf..8fbc2241959 100644 --- a/src/lib/adapters/http/probe.ts +++ b/src/lib/adapters/http/probe.ts @@ -73,18 +73,24 @@ export function getCurlTimingArgs(): string[] { } function getCurlMaxTimeSeconds(argv: string[]): number | null { + let maxTimeSeconds: number | null = null; for (let index = 0; index < argv.length; index += 1) { const arg = argv[index]; if (arg === "--max-time") { const value = Number(argv[index + 1]); - return Number.isFinite(value) && value > 0 ? value : null; + if (Number.isFinite(value) && value > 0) { + maxTimeSeconds = value; + } + continue; } if (arg.startsWith("--max-time=")) { const value = Number(arg.slice("--max-time=".length)); - return Number.isFinite(value) && value > 0 ? value : null; + if (Number.isFinite(value) && value > 0) { + maxTimeSeconds = value; + } } } - return null; + return maxTimeSeconds; } function resolveCurlProcessTimeoutMs(argv: string[], opts: CurlProbeOptions): number {