diff --git a/.env.example b/.env.example index 93ff922af2cb..201a60d3793c 100644 --- a/.env.example +++ b/.env.example @@ -1146,6 +1146,15 @@ PROVIDER_LIMITS_SYNC_SPACING_MS=1500 # Used by: src/lib/jobs/reasoningCacheCleanupJob.ts. #OMNIROUTE_REASONING_CACHE_CLEANUP_INTERVAL_MS=1800000 +# Recover from a reasoning_effort 4xx whose body names no accepted set, by +# probing one tier down and learning that tier when the probe is answered. +# Comma-separated provider ids, or * for all. Default: unset (disabled). +# Off by default because an opaque 4xx carries no signal that the *effort* was +# the cause — a 2xx on the probe could mask an unrelated validation failure and +# pin a reasoning cap that was never proven. Upstreams that DO name their +# accepted set are already handled without this flag. +#OMNIROUTE_REASONING_EFFORT_PROBE_PROVIDERS= + # Opt-in minimum output budget (tokens) for reasoning models (#10281 follow-up). # When set, a caller max_tokens in [256, floor) on a thinking-capable model is # raised to the floor so reasoning tokens cannot consume the whole budget diff --git a/changelog.d/fixes/14895-reasoning-effort-opaque-400-probe.md b/changelog.d/fixes/14895-reasoning-effort-opaque-400-probe.md new file mode 100644 index 000000000000..e141d728b3c0 --- /dev/null +++ b/changelog.d/fixes/14895-reasoning-effort-opaque-400-probe.md @@ -0,0 +1 @@ +- **fix(sse):** recover from a `reasoning_effort` 4xx that names no accepted set — opt in per provider with `OMNIROUTE_REASONING_EFFORT_PROBE_PROVIDERS` and the reactive path probes one tier down, learning that single tier when the probe is answered so later requests clamp up front instead of 400ing on every attempt diff --git a/docs/reference/ENVIRONMENT.md b/docs/reference/ENVIRONMENT.md index caeef040bf6d..ff6c7a7b962b 100644 --- a/docs/reference/ENVIRONMENT.md +++ b/docs/reference/ENVIRONMENT.md @@ -569,6 +569,7 @@ detection above). | `OMNIROUTE_DISABLE_CONNECTION_RECOVERY` | `false` | `src/lib/quota/connectionRecovery.ts` | Disable the proactive connection-cooldown recovery scheduler (lazy recovery in `getProviderCredentials` still applies). | | `OMNIROUTE_REASONING_CACHE_CLEANUP_INTERVAL_MS` | `1800000` | `src/lib/jobs/reasoningCacheCleanupJob.ts` | Reasoning cache cleanup cadence (ms). Floor `60000`. | | `OMNIROUTE_REASONING_MIN_BUDGET` | _unset_ (disabled) | `open-sse/services/reasoningTokenBuffer.ts` | Opt-in floor for thinking-model output budgets: caller `max_tokens` in `[256, floor)` is raised to the floor (capped by the model output cap). Unset = client budgets never enlarged (#9507). | +| `OMNIROUTE_REASONING_EFFORT_PROBE_PROVIDERS` | _unset_ (disabled) | `open-sse/services/learnedReasoningEffortCaps.ts` | Opt-in, comma-separated provider ids (or `*`): when a `reasoning_effort` 400/422 names no accepted set, retry once one tier lower (never below `low`) and learn that tier only if the probe is answered (#14895). | | `OMNIROUTE_LOG_EXPORT_CRON` | `0 * * * *` | `src/lib/jobs/logExportJob.ts` | Cron expression (UTC) for the call-log export job that drains every enabled log-export destination. | | `OMNIROUTE_CONFIG_HOT_RELOAD_MS` | `5000` | `src/lib/config/hotReload.ts` | Polling interval (ms) for config hot-reload. Lower than `1000` is rejected. | | `OMNIROUTE_DISABLE_REDIS_AUTH_CACHE` | _(enabled)_ | `src/lib/db/apiKeys.ts` | Set `1` to bypass the Redis-backed API-key auth cache (forces DB reads). | diff --git a/open-sse/executors/base/reasoningEffort.ts b/open-sse/executors/base/reasoningEffort.ts index e176a9a9d1aa..2120a062d024 100644 --- a/open-sse/executors/base/reasoningEffort.ts +++ b/open-sse/executors/base/reasoningEffort.ts @@ -260,6 +260,29 @@ function writeEffortValue( return next; } +/** + * The effort the outgoing body actually asks for, across all three carriers. + * Used by the reactive 4xx probe in `base.ts` to decide which tier to step + * down to, so it must report what is on the wire — not what the registry says + * the model supports. + */ +export function readBodyReasoningEffort(body: unknown): string | null { + if (!body || typeof body !== "object") return null; + const effort = readEffortCarriers(body as Record).effort; + return typeof effort === "string" ? effort : null; +} + +/** + * Write `value` onto every carrier the body already uses, leaving the shape of + * the body untouched — the reactive probe rewrites one field of the request + * that is already on its way upstream, so it must not reshape anything else. + */ +export function writeBodyReasoningEffort(body: unknown, value: string): unknown { + if (!body || typeof body !== "object") return body; + const record = body as Record; + return writeEffortValue(record, value, readEffortCarriers(record)); +} + /** Strip the effort field from every carrier that was present. */ function stripEffortValue(b: Record, c: EffortCarriers): Record { const next: Record = { ...b }; diff --git a/open-sse/executors/base/reasoningEffortRecovery.ts b/open-sse/executors/base/reasoningEffortRecovery.ts index 2a37f17b3bd9..eabb89f18ed0 100644 --- a/open-sse/executors/base/reasoningEffortRecovery.ts +++ b/open-sse/executors/base/reasoningEffortRecovery.ts @@ -10,10 +10,17 @@ // sanitizer, and retry the SAME url once. import { HTTP_STATUS } from "../../config/constants.ts"; import { + nextProbeReasoningEffort, parseReasoningEffortEnum, + reasoningEffortProbeEnabled, + recordLearnedProbeReasoningEffort, recordLearnedReasoningEffort, } from "../../services/learnedReasoningEffortCaps.ts"; -import { sanitizeReasoningEffortForProvider } from "./reasoningEffort.ts"; +import { + readBodyReasoningEffort, + sanitizeReasoningEffortForProvider, + writeBodyReasoningEffort, +} from "./reasoningEffort.ts"; type ReasoningRecoveryLog = { info?: (tag: string, message: string) => void; @@ -83,7 +90,7 @@ export async function applyReasoningEffortRecovery( .catch(() => ""); const acceptedValues = parseReasoningEffortEnum(errText); if (!acceptedValues) { - return notRecoverable; + return probeOneTierDown(params, body, notRecoverable); } const learned = recordLearnedReasoningEffort(provider, model as string, acceptedValues); @@ -111,3 +118,47 @@ export async function applyReasoningEffortRecovery( const retriedResponse = await fetchFn(url, { ...fetchOptions, body: retryBody }); return { response: retriedResponse, body: clampedBody, retried: true, attempted: true }; } + +/** + * #14895 — the 4xx body named no accepted set, so there is nothing to learn from + * it and no way to tell an effort rejection from an unrelated validation failure. + * Opt in per provider (OMNIROUTE_REASONING_EFFORT_PROBE_PROVIDERS, off by default) + * rather than guess for everyone: a 2xx on the probe would otherwise mask the real + * error and record a cap nothing was proven against. When enabled, step down ONE + * tier and retry the same URL once; only an ANSWERED probe is recorded (that one + * tier), so later requests clamp up front. A refused probe learns nothing and its + * response is surfaced. Lives here (not inline in BaseExecutor) so every caller of + * applyReasoningEffortRecovery — commandCode, cliproxyapi, glm — gets it too. + */ +async function probeOneTierDown( + params: ReasoningEffortRecoveryParams, + body: unknown, + notRecoverable: ReasoningEffortRecoveryResult +): Promise { + const { response, url, provider, model, fetchOptions, fetchFn, log } = params; + if (!reasoningEffortProbeEnabled(provider)) return notRecoverable; + const probe = nextProbeReasoningEffort(readBodyReasoningEffort(body) ?? ""); + if (!probe) return notRecoverable; + + const probeBody = writeBodyReasoningEffort(body, probe); + const serialize = params.serializeBody ?? ((b: unknown) => JSON.stringify(b)); + const serialized = await serialize(probeBody); + log?.info?.( + "REASONING_SANITIZE", + `Upstream ${response.status} refused reasoning_effort on ${url} without naming the accepted set — probing one step down (${probe}) for ${provider}/${model}` + ); + const probeResponse = await fetchFn(url, { ...fetchOptions, body: serialized }); + if (probeResponse.ok) { + const learned = recordLearnedProbeReasoningEffort(provider, model, probe); + log?.info?.( + "REASONING_SANITIZE", + `Probe accepted for ${provider}/${model} — learned ceiling up to ${[...(learned ?? [])].join(",") || probe}` + ); + } else { + log?.info?.( + "REASONING_SANITIZE", + `Probe ${probe} also refused for ${provider}/${model} — learned nothing, surfacing the probe's ${probeResponse.status}` + ); + } + return { response: probeResponse, body: probeBody, retried: true, attempted: true }; +} diff --git a/open-sse/services/learnedReasoningEffortCaps.ts b/open-sse/services/learnedReasoningEffortCaps.ts index bf2c077a2a57..15fdb2b4c45d 100644 --- a/open-sse/services/learnedReasoningEffortCaps.ts +++ b/open-sse/services/learnedReasoningEffortCaps.ts @@ -181,6 +181,101 @@ export function clampToLearned(effortStr: string, accepted: Set): string return nearestAbove ?? highest; } +// ── Single-step probe for 4xx bodies that name no enum ──────────────────── +// Most gateways that reject an out-of-range reasoning_effort DO advertise the +// accepted set ("expected one of `low`, `medium`, `high`"), and +// `parseReasoningEffortEnum` reads it. Some do not: the body is an opaque +// "Invalid request parameters" / "Streaming response failed: [400] ..." with +// no list at all. Nothing can be learned from that text, so the request 400s +// forever — the only remedy is to try a lower tier and see whether it is +// accepted. These two helpers drive that one-step probe: pick the tier to try, +// and — once a probe SUCCEEDS — remember the whole range it just proved +// accepted, so every later request clamps without another round trip. + +// The probe floor. `low` is the universally-accepted baseline on every +// OpenAI-compatible reasoning surface, so it is the safest rung to land on. The +// ladder deliberately stops HERE: the two rungs below it (`minimal`, `none`) are +// not just "less thinking", they are the carriers that switch thinking OFF. A +// probe that happened to be answered on `minimal` proves the model tolerates +// `minimal`, not that it wants reasoning disabled, so stepping down that far +// would silently downgrade a request's intent rather than repair it. +const PROBE_FLOOR = "low"; + +/** + * The next tier to try when a 4xx named no enum, given the effort that was sent. + * + * Steps down exactly one rung (xhigh → high, max → xhigh, ultra → max, …) so a + * probe costs the client as little reasoning as possible, and never below + * `PROBE_FLOOR`. Returns null when there is nothing to step down to (already at + * the floor, an unrecognized value, or nothing left but the thinking-off tiers): + * stripping the field entirely is the learned path's job, not the probe's. + */ +export function nextProbeReasoningEffort(sent: string): string | null { + const idx = REASONING_EFFORT_ORDER.indexOf(sent); + const floorIdx = REASONING_EFFORT_ORDER.indexOf(PROBE_FLOOR); + if (idx < 0 || floorIdx < 0 || idx <= floorIdx) return null; + const next = REASONING_EFFORT_ORDER[idx - 1]; + return next && next !== sent ? next : null; +} + +/** + * Whether the opaque-4xx probe may run for this provider. + * + * The probe is a *guess-driven* retry: when a 4xx body names no accepted set + * there is no way to tell an effort rejection from an unrelated validation + * failure (context too long, a malformed tool schema, a bad image URL). If the + * upstream happens to answer 2xx for the probe, an unrelated failure would be + * masked and a bogus cap recorded — permanently lowering reasoning quality for + * that provider/model in this process, for a tier nothing was ever proven + * against. + * + * So it is opt-in per provider. An empty/absent `OMNIROUTE_REASONING_EFFORT_PROBE_PROVIDERS` + * disables the probe everywhere, which is the default: the enum-naming path + * (#14013) already covers every upstream that cooperates, and this is only for + * gateways that answer opaquely. `*` enables it for all providers. + */ +export function reasoningEffortProbeEnabled(provider: string | null | undefined): boolean { + const raw = process.env.OMNIROUTE_REASONING_EFFORT_PROBE_PROVIDERS; + if (!raw) return false; + const entries = raw + .split(",") + .map((s) => s.trim().toLowerCase()) + .filter(Boolean); + if (entries.length === 0) return false; + if (entries.includes("*")) return true; + return provider != null && entries.includes(String(provider).toLowerCase()); +} + +/** + * Record that `acceptedProbe` was accepted, after a 4xx on a HIGHER tier proved + * the higher tier is refused. + * + * Only that one rung is recorded. A single accepted probe is proof about one + * value and says nothing about the tiers below it: a model may well answer + * `high` and still refuse `low` (the learned-cap tests already cover sparse + * accepted sets such as `{high,max}`), so inferring the whole + * `PROBE_FLOOR..acceptedProbe` slice would let later `low`/`medium` requests + * bypass clamping and go out unproven. Pinning just the probed tier is the + * conservative direction — the clamp then maps anything higher onto a value the + * upstream has actually acknowledged, and leaves anything lower alone. + * + * Only call this once the probe has actually succeeded: a failed probe proves + * nothing, and learning from it would pin the model to a ceiling it may well + * still support. Monotonic, like `recordLearnedReasoningEffort`. + */ +export function recordLearnedProbeReasoningEffort( + provider: string | null | undefined, + model: string | null | undefined, + acceptedProbe: string +): Set | null { + const idx = REASONING_EFFORT_ORDER.indexOf(acceptedProbe); + const floorIdx = REASONING_EFFORT_ORDER.indexOf(PROBE_FLOOR); + // A probe below the floor never happens (nextProbeReasoningEffort refuses to + // step there), so treat it as a caller error rather than learning it. + if (idx < 0 || floorIdx < 0 || idx < floorIdx) return null; + return recordLearnedReasoningEffort(provider, model, [acceptedProbe]); +} + // Matches prose shapes: OVH's "@ai-sdk/openai-compatible" deserializer // ("expected one of `a`, `b`"), generic ("Supported types are a, b, and c"), // and "please use a, b, or c". diff --git a/tests/unit/reasoning-effort-opaque-400-probe.test.ts b/tests/unit/reasoning-effort-opaque-400-probe.test.ts new file mode 100644 index 000000000000..22a1509d196a --- /dev/null +++ b/tests/unit/reasoning-effort-opaque-400-probe.test.ts @@ -0,0 +1,486 @@ +import { test, after, beforeEach } from "node:test"; +import assert from "node:assert/strict"; +import { BaseExecutor } from "../../open-sse/executors/base.ts"; +import { applyReasoningEffortRecovery } from "../../open-sse/executors/base/reasoningEffortRecovery.ts"; +import { + getLearnedReasoningEffort, + nextProbeReasoningEffort, + recordLearnedProbeReasoningEffort, + reasoningEffortProbeEnabled, + recordLearnedReasoningEffort, + parseReasoningEffortEnum, + __test_resetLearnedReasoningEffortCaps, +} from "../../open-sse/services/learnedReasoningEffortCaps.ts"; + +// The failure this covers: an upstream that refuses an out-of-range +// reasoning_effort with a body naming no accepted set. Measured against +// opencode.ai/zen/go (mimo-v2.6-flash / mimo-v2.5-pro), whose rejection is an +// opaque `{"type":"server_error","message":"Streaming response failed: [400] +// Invalid request parameters"}` wrapped in the SSE error frame — no list, so +// `parseReasoningEffortEnum` has nothing to learn from and the request 400s on +// every attempt. The probe is opt-in per provider +// (OMNIROUTE_REASONING_EFFORT_PROBE_PROVIDERS) because such a body gives no +// signal that the effort was the cause, so it is applied only where an operator +// has said this gateway is known to answer opaquely on the effort. +const OPAQUE_400_BODY = JSON.stringify({ + error: { + param: "", + type: "server_error", + message: "Streaming response failed: [400] Invalid request parameters", + }, +}); + +const ANY_400_BODY = JSON.stringify({ error: { message: "Invalid request parameters" } }); + +const PROVIDER = "openai-compatible-probe-test"; +const MODEL = "mimo-v2.6-flash"; + +const CREDENTIALS = { + providerSpecificData: { baseUrl: "https://example.invalid/v1" }, +}; + +class SimpleExecutor extends BaseExecutor { + constructor() { + super(PROVIDER, { + baseUrls: ["https://example.invalid/v1/chat/completions"], + }); + } + async transformRequest(_model: string, body: Record) { + return { ...body }; + } +} + +// The probe is opt-in per provider (see reasoningEffortProbeEnabled), and +// Copilot's review is why: an opaque 4xx gives no signal that the effort was the +// cause, so a 2xx on the probe could mask an unrelated validation failure and +// pin an unproven cap. The end-to-end tests below enable it for this fake +// provider; the gate tests that follow then turn it back off explicitly. +const PROBE_ENV_KEY = "OMNIROUTE_REASONING_EFFORT_PROBE_PROVIDERS"; +const savedProbeEnv = process.env[PROBE_ENV_KEY]; +beforeEach(() => { + __test_resetLearnedReasoningEffortCaps(); + process.env[PROBE_ENV_KEY] = PROVIDER; +}); + +after(() => { + __test_resetLearnedReasoningEffortCaps(); + if (savedProbeEnv === undefined) delete process.env[PROBE_ENV_KEY]; + else process.env[PROBE_ENV_KEY] = savedProbeEnv; +}); + +/** Mock fetch: first call 400s with `rejectBody`, later calls answer 200. */ +function mockFetchOnce(capturedBodies: Record[], rejectBody: string) { + globalThis.fetch = async (_url: string | URL | Request, init: RequestInit = {}) => { + capturedBodies.push(JSON.parse(String(init.body))); + if (capturedBodies.length === 1) { + return new Response(rejectBody, { + status: 400, + headers: { "Content-Type": "application/json" }, + }); + } + return new Response(JSON.stringify({ ok: true }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + }; +} + +/** Run `fn` with env overrides applied, then restore the previous values. */ +async function withEnv( + vars: Record, + fn: () => Promise +): Promise { + const saved = Object.fromEntries(Object.keys(vars).map((k) => [k, process.env[k]])); + for (const [k, v] of Object.entries(vars)) { + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + try { + return await fn(); + } finally { + for (const [k, v] of Object.entries(saved)) { + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + } +} + +async function withMockedFetch(fn: () => Promise): Promise { + const originalFetch = globalThis.fetch; + try { + return await fn(); + } finally { + globalThis.fetch = originalFetch; + } +} + +// ── nextProbeReasoningEffort ─────────────────────────────────────────────── + +test("the opaque 400 body carries no enum, so the learned path cannot fire", () => { + // Guards the premise of the whole probe: if this ever parses, the probe is + // redundant and this file's assertions below would be testing dead code. + assert.equal(parseReasoningEffortEnum(OPAQUE_400_BODY), null); + assert.equal(parseReasoningEffortEnum(ANY_400_BODY), null); +}); + +test("the probe steps down exactly one tier", () => { + assert.equal(nextProbeReasoningEffort("xhigh"), "high"); + assert.equal(nextProbeReasoningEffort("max"), "xhigh"); + assert.equal(nextProbeReasoningEffort("ultra"), "max"); + assert.equal(nextProbeReasoningEffort("medium"), "low"); + assert.equal(nextProbeReasoningEffort("high"), "medium"); +}); + +test("the probe declines at the floor and on values outside the scale", () => { + assert.equal(nextProbeReasoningEffort("low"), null, "low is the probe floor"); + assert.equal(nextProbeReasoningEffort("none"), null, "nothing below none"); + assert.equal(nextProbeReasoningEffort("minimal"), null); + assert.equal(nextProbeReasoningEffort("banana"), null); + assert.equal(nextProbeReasoningEffort(""), null); +}); + +// ── recordLearnedProbeReasoningEffort ────────────────────────────────────── + +test("a successful probe learns ONLY the tier it proved, never the tiers below it", () => { + // One accepted probe is proof about one value. A model can answer `high` and + // still refuse `low` (the learned-cap tests already cover sparse sets such as + // {high,max}), so the tiers below the probe stay unproven and unlearned. + const learned = recordLearnedProbeReasoningEffort(PROVIDER, MODEL, "high"); + assert.deepEqual([...(learned as Set)], ["high"]); + const stored = getLearnedReasoningEffort(PROVIDER, MODEL) as unknown as Set; + assert.equal(stored.has("medium"), false, "medium was never probed"); + assert.equal(stored.has("low"), false, "low was never probed — a sparse set is real"); + assert.equal(stored.has("xhigh"), false, "xhigh stayed refused"); + assert.equal(stored.has("none"), false, "none was not proven"); +}); + +test("a probe at the floor learns only low", () => { + const learned = recordLearnedProbeReasoningEffort(PROVIDER, MODEL, "low"); + assert.deepEqual([...(learned as Set)], ["low"]); +}); + +test("a refused 4xx on a HIGHER tier is not re-learned by a lower probe", () => { + // xhigh 400s, the probe at high is answered. The accepted set is then exactly + // {high} — xhigh stays out, which is the whole point of learning a ceiling + // rather than a floor. + recordLearnedReasoningEffort(PROVIDER, MODEL, ["high"]); + const stored = getLearnedReasoningEffort(PROVIDER, MODEL) as unknown as Set; + assert.deepEqual([...stored].sort(), ["high"]); +}); + +test("probing an unknown provider+model, or an unknown tier, learns nothing", () => { + assert.equal(recordLearnedProbeReasoningEffort(PROVIDER, MODEL, "banana"), null); + assert.equal(recordLearnedProbeReasoningEffort("", "", "high"), null); + assert.equal(getLearnedReasoningEffort(PROVIDER, MODEL), null); +}); + +// ── the reactive chain in base.ts ────────────────────────────────────────── + +test("an opaque 400 probes one step down, retries, and learns the ceiling", async () => { + const executor = new SimpleExecutor(); + const captured: Record[] = []; + + await withMockedFetch(async () => { + mockFetchOnce(captured, OPAQUE_400_BODY); + const result = await executor.execute({ + model: MODEL, + body: { reasoning_effort: "xhigh" }, + stream: false, + credentials: CREDENTIALS, + }); + assert.equal(result.response.status, 200, "the probe answered, so the client sees 200"); + }); + + assert.equal(captured.length, 2, "exactly one probe, not a ladder walk"); + assert.equal(captured[0].reasoning_effort, "xhigh", "the first attempt is untouched"); + assert.equal(captured[1].reasoning_effort, "high", "the probe steps down one tier"); + const stored = getLearnedReasoningEffort(PROVIDER, MODEL) as unknown as Set; + assert.equal(stored.has("high"), true, "learned from the probe that succeeded"); +}); + +test("after the probe, a later request clamps on the FIRST attempt", async () => { + const executor = new SimpleExecutor(); + const captured: Record[] = []; + + await withMockedFetch(async () => { + mockFetchOnce(captured, OPAQUE_400_BODY); + await executor.execute({ + model: MODEL, + body: { reasoning_effort: "xhigh" }, + stream: false, + credentials: CREDENTIALS, + }); + // Second request, same provider+model: the learned set must apply up front. + captured.length = 0; + globalThis.fetch = async (_url: string | URL | Request, init: RequestInit = {}) => { + captured.push(JSON.parse(String(init.body))); + return new Response(JSON.stringify({ ok: true }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + }; + await executor.execute({ + model: MODEL, + body: { reasoning_effort: "xhigh" }, + stream: false, + credentials: CREDENTIALS, + }); + }); + + assert.equal(captured.length, 1, "no 4xx round-trip needed any more"); + assert.equal(captured[0].reasoning_effort, "high", "clamped proactively from the learned set"); +}); + +test("a failed probe learns nothing and surfaces the upstream error", async () => { + const executor = new SimpleExecutor(); + const captured: Record[] = []; + + const result = await withMockedFetch(async () => { + // 400 on the original AND on the probe: the model refuses every tier. + globalThis.fetch = async (_url: string | URL | Request, init: RequestInit = {}) => { + captured.push(JSON.parse(String(init.body))); + return new Response(ANY_400_BODY, { + status: 400, + headers: { "Content-Type": "application/json" }, + }); + }; + return executor.execute({ + model: MODEL, + body: { reasoning_effort: "xhigh" }, + stream: false, + credentials: CREDENTIALS, + }); + }); + + assert.equal(captured.length, 2, "probed once, then stopped"); + assert.equal(result.response.status, 400, "the probe's own status is surfaced"); + assert.equal( + getLearnedReasoningEffort(PROVIDER, MODEL), + null, + "a failed probe proves nothing, so nothing is pinned" + ); +}); + +test("an effort at the probe floor is not probed — nothing to step down to", async () => { + const executor = new SimpleExecutor(); + const captured: Record[] = []; + + await withMockedFetch(async () => { + globalThis.fetch = async (_url: string | URL | Request, init: RequestInit = {}) => { + captured.push(JSON.parse(String(init.body))); + return new Response(ANY_400_BODY, { + status: 400, + headers: { "Content-Type": "application/json" }, + }); + }; + await executor.execute({ + model: MODEL, + body: { reasoning_effort: "low" }, + stream: false, + credentials: CREDENTIALS, + }); + }); + + assert.equal(captured.length, 1, "no probe: low is already the floor"); + assert.equal(getLearnedReasoningEffort(PROVIDER, MODEL), null); +}); + +test("a 400 that names no enum but carries no effort is left alone", async () => { + const executor = new SimpleExecutor(); + const captured: Record[] = []; + + await withMockedFetch(async () => { + globalThis.fetch = async (_url: string | URL | Request, init: RequestInit = {}) => { + captured.push(JSON.parse(String(init.body))); + return new Response(ANY_400_BODY, { + status: 400, + headers: { "Content-Type": "application/json" }, + }); + }; + await executor.execute({ + model: MODEL, + body: { messages: [{ role: "user", content: "hi" }] }, + stream: false, + credentials: CREDENTIALS, + }); + }); + + assert.equal(captured.length, 1, "no effort to probe for, so no probe"); +}); + +test("the probe rewrites only the effort, on the Responses carrier too", async () => { + const executor = new SimpleExecutor(); + const captured: Record[] = []; + + await withMockedFetch(async () => { + mockFetchOnce(captured, ANY_400_BODY); + await executor.execute({ + model: MODEL, + body: { reasoning: { effort: "max" }, messages: [{ role: "user", content: "hi" }] }, + stream: false, + credentials: CREDENTIALS, + }); + }); + + assert.equal(captured.length, 2); + assert.deepEqual(captured[0].reasoning, { effort: "max" }); + assert.deepEqual( + captured[1].reasoning, + { effort: "xhigh" }, + "one rung down from max, on the same carrier" + ); + assert.equal( + captured[1].reasoning_effort, + undefined, + "a carrier the request did not use is not invented" + ); +}); + +// ── the opt-in gate ──────────────────────────────────────────────────────── +// These exist because of review: an opaque 4xx names no enum, so nothing in it +// says the *effort* was the cause. Applying the probe to every such 400 would +// re-POST any unrelated validation failure (context too long, malformed tool +// schema) with a different body, and a 2xx on that probe would mask the real +// error while recording a reasoning cap nothing was ever proven against. + +test("the probe is off by default, so an opaque 400 is surfaced unchanged", async () => { + const executor = new SimpleExecutor(); + const captured: Record[] = []; + + await withMockedFetch(async () => { + mockFetchOnce(captured, OPAQUE_400_BODY); + const result = await withEnv({ [PROBE_ENV_KEY]: undefined }, () => + executor.execute({ + model: MODEL, + body: { reasoning_effort: "xhigh" }, + stream: false, + credentials: CREDENTIALS, + }) + ); + assert.equal(result.response.status, 400, "the original error is not masked"); + }); + + assert.equal(captured.length, 1, "no second POST for an unrelated provider"); + assert.equal( + getLearnedReasoningEffort(PROVIDER, MODEL), + null, + "no cap is recorded from a probe that never ran" + ); +}); + +test("an empty or blank provider list is treated as disabled", () => { + // beforeEach enables the probe for PROVIDER, so each value is set explicitly. + for (const value of ["", " ", ",,"]) { + assert.equal( + withEnvSync(value, () => reasoningEffortProbeEnabled(PROVIDER)), + false, + JSON.stringify(value) + ); + } +}); + +test("the opt-in matches the provider, case- and space-insensitively", () => { + const mixed = ` ${PROVIDER.toUpperCase()} , other-provider `; + assert.equal( + withEnvSync(mixed, () => reasoningEffortProbeEnabled(PROVIDER)), + true, + "a padded, differently-cased entry still matches" + ); + assert.equal( + withEnvSync("some-other-provider", () => reasoningEffortProbeEnabled(PROVIDER)), + false, + "a provider that was not opted in stays off" + ); +}); + +test("`*` enables the probe for every provider", () => { + assert.equal( + withEnvSync("*", () => reasoningEffortProbeEnabled("anything-at-all")), + true + ); + assert.equal( + withEnvSync("openai, *", () => reasoningEffortProbeEnabled("anything-at-all")), + true, + "a wildcard alongside real entries still wins" + ); +}); + +test("a null provider is never probed even under a wildcard-free list", () => { + assert.equal( + withEnvSync(PROVIDER, () => reasoningEffortProbeEnabled(null)), + false + ); +}); + +/** Set the env var, read the result synchronously, restore. */ +function withEnvSync(value: string, fn: () => T): T { + const saved = process.env[PROBE_ENV_KEY]; + process.env[PROBE_ENV_KEY] = value; + try { + return fn(); + } finally { + if (saved === undefined) delete process.env[PROBE_ENV_KEY]; + else process.env[PROBE_ENV_KEY] = saved; + } +} + +// #14629 extracted the clamp-and-retry chain into applyReasoningEffortRecovery() so +// executors that never call super.execute() (commandCode, cliproxyapi, glm) reach it. +// The probe must live there too, or those callers keep 400ing on opaque bodies. +test("applyReasoningEffortRecovery probes one tier down for direct callers", async () => { + const sent: Record[] = []; + const recovery = await applyReasoningEffortRecovery({ + response: new Response(OPAQUE_400_BODY, { status: 400 }), + url: "https://example.invalid/v1/chat/completions", + provider: PROVIDER, + model: MODEL, + body: { model: MODEL, reasoning_effort: "xhigh", messages: [] }, + fetchOptions: { method: "POST" }, + fetchFn: async (_url, init) => { + sent.push(JSON.parse(String(init.body))); + return new Response("{}", { status: 200 }); + }, + }); + + assert.equal(recovery.retried, true); + assert.equal(recovery.attempted, true); + assert.equal(recovery.response.status, 200); + assert.equal(sent.length, 1); + assert.equal(sent[0].reasoning_effort, "high"); + assert.deepEqual([...(getLearnedReasoningEffort(PROVIDER, MODEL) ?? [])], ["high"]); +}); + +test("applyReasoningEffortRecovery leaves an opaque 4xx alone when the probe is off", async () => { + let calls = 0; + const original = new Response(OPAQUE_400_BODY, { status: 400 }); + const recovery = await withEnvAsync("", () => + applyReasoningEffortRecovery({ + response: original, + url: "https://example.invalid/v1/chat/completions", + provider: PROVIDER, + model: MODEL, + body: { model: MODEL, reasoning_effort: "xhigh", messages: [] }, + fetchOptions: { method: "POST" }, + fetchFn: async () => { + calls += 1; + return new Response("{}", { status: 200 }); + }, + }) + ); + + assert.equal(calls, 0); + assert.equal(recovery.retried, false); + assert.equal(recovery.response, original); + assert.equal(getLearnedReasoningEffort(PROVIDER, MODEL), null); +}); + +async function withEnvAsync(value: string, fn: () => Promise): Promise { + const saved = process.env[PROBE_ENV_KEY]; + process.env[PROBE_ENV_KEY] = value; + try { + return await fn(); + } finally { + if (saved === undefined) delete process.env[PROBE_ENV_KEY]; + else process.env[PROBE_ENV_KEY] = saved; + } +}