From 88e159900bda69db2f70b435b6f90d8afa195123 Mon Sep 17 00:00:00 2001 From: Minxi Hou Date: Mon, 28 Sep 2026 03:04:19 -0400 Subject: [PATCH] fix(combo): trust finish_reason stop over the reasoning-ratio heuristic The 90% reasoning-ratio check rejects responses where reasoning fills the budget and visible content is empty. That is correct for models that optionally think and run out of budget mid-reasoning, but it misfires on models that cannot disable thinking: Claude Opus 4.7+ and GLM-5.3 both spend most of their output on reasoning and still finish normally with finish_reason "stop". OpenAI, Anthropic, and Z.ai all document "stop" as a natural end of turn and "length"/"max_tokens" as truncation. Trust the stop signal and keep the ratio check only as the fallback when the provider reports no finish_reason. Signed-off-by: Minxi Hou (cherry picked from commit 4a03936ad6a5d8fd6bbe26190b5e57dd51e3dc0a) --- open-sse/services/combo/validateQuality.ts | 8 +++++--- .../combo-quality-validator-reasoning.test.ts | 17 +++++++++++++++-- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/open-sse/services/combo/validateQuality.ts b/open-sse/services/combo/validateQuality.ts index df275ba02190..915e4379465e 100644 --- a/open-sse/services/combo/validateQuality.ts +++ b/open-sse/services/combo/validateQuality.ts @@ -945,10 +945,12 @@ export async function validateResponseQuality( reason: `reasoning truncated at token limit (finish_reason: ${finishReason}) — no content output`, }; } - if (usage) { + if (usage && finishReason !== "stop") { const reasoningTokens = getReasoningTokens(usage); - // If reasoning consumed 90%+ of completion tokens, the model ran out of - // budget before producing any content output. + // finish_reason "stop" means the model ended on its own (OpenAI, Anthropic, + // Z.ai all document this). A high reasoning ratio on a clean stop is normal + // for models that cannot disable thinking (Claude Opus 4.7+, GLM-5.3). The + // 90% check stays as the fallback when the provider reports no finish_reason. if (completionTokens > 0 && reasoningTokens >= completionTokens * 0.9) { if (isTinyBudgetTruncation(completionTokens)) return { valid: true }; return { diff --git a/tests/unit/combo-quality-validator-reasoning.test.ts b/tests/unit/combo-quality-validator-reasoning.test.ts index 64609d3ebf6c..8ef7b4af3feb 100644 --- a/tests/unit/combo-quality-validator-reasoning.test.ts +++ b/tests/unit/combo-quality-validator-reasoning.test.ts @@ -249,7 +249,7 @@ test("client-audit-2026-09-01: no finish_reason + reasoning <90% of tokens → s assert.equal(out.valid, true, "should stay valid: no finish_reason signal, ratio under 90%"); }); -test("client-audit-2026-09-01: finish_reason:stop + empty content + reasoning → ratio heuristic still applies unchanged", async () => { +test("finish_reason:stop + reasoning over 90% is a clean stop, not exhaustion", async () => { const res = makeResponse({ choices: [ { @@ -260,7 +260,20 @@ test("client-audit-2026-09-01: finish_reason:stop + empty content + reasoning usage: { completion_tokens: 4096, reasoning_tokens: 3800 }, }); const out = await validateResponseQuality(res, false, silentLog); - assert.equal(out.valid, false, "finish_reason:stop doesn't short-circuit — ratio (>90%) still applies"); + assert.equal(out.valid, true, "finish_reason:stop means the model ended on its own"); +}); + +test("no finish_reason + reasoning over 90% stays invalid (fallback heuristic)", async () => { + const res = makeResponse({ + choices: [ + { + message: { content: null, reasoning_content: "Deep reasoning" }, + }, + ], + usage: { completion_tokens: 4096, reasoning_tokens: 3800 }, + }); + const out = await validateResponseQuality(res, false, silentLog); + assert.equal(out.valid, false, "no finish_reason still falls back to the 90% check"); assert.match(out.reason ?? "", /reasoning consumed/i); });