diff --git a/open-sse/services/combo/validateQuality.ts b/open-sse/services/combo/validateQuality.ts index df275ba02190..915e4379465e 100644 --- a/open-sse/services/combo/validateQuality.ts +++ b/open-sse/services/combo/validateQuality.ts @@ -945,10 +945,12 @@ export async function validateResponseQuality( reason: `reasoning truncated at token limit (finish_reason: ${finishReason}) — no content output`, }; } - if (usage) { + if (usage && finishReason !== "stop") { const reasoningTokens = getReasoningTokens(usage); - // If reasoning consumed 90%+ of completion tokens, the model ran out of - // budget before producing any content output. + // finish_reason "stop" means the model ended on its own (OpenAI, Anthropic, + // Z.ai all document this). A high reasoning ratio on a clean stop is normal + // for models that cannot disable thinking (Claude Opus 4.7+, GLM-5.3). The + // 90% check stays as the fallback when the provider reports no finish_reason. if (completionTokens > 0 && reasoningTokens >= completionTokens * 0.9) { if (isTinyBudgetTruncation(completionTokens)) return { valid: true }; return { diff --git a/tests/unit/combo-quality-validator-reasoning.test.ts b/tests/unit/combo-quality-validator-reasoning.test.ts index 64609d3ebf6c..8ef7b4af3feb 100644 --- a/tests/unit/combo-quality-validator-reasoning.test.ts +++ b/tests/unit/combo-quality-validator-reasoning.test.ts @@ -249,7 +249,7 @@ test("client-audit-2026-09-01: no finish_reason + reasoning <90% of tokens → s assert.equal(out.valid, true, "should stay valid: no finish_reason signal, ratio under 90%"); }); -test("client-audit-2026-09-01: finish_reason:stop + empty content + reasoning → ratio heuristic still applies unchanged", async () => { +test("finish_reason:stop + reasoning over 90% is a clean stop, not exhaustion", async () => { const res = makeResponse({ choices: [ { @@ -260,7 +260,20 @@ test("client-audit-2026-09-01: finish_reason:stop + empty content + reasoning usage: { completion_tokens: 4096, reasoning_tokens: 3800 }, }); const out = await validateResponseQuality(res, false, silentLog); - assert.equal(out.valid, false, "finish_reason:stop doesn't short-circuit — ratio (>90%) still applies"); + assert.equal(out.valid, true, "finish_reason:stop means the model ended on its own"); +}); + +test("no finish_reason + reasoning over 90% stays invalid (fallback heuristic)", async () => { + const res = makeResponse({ + choices: [ + { + message: { content: null, reasoning_content: "Deep reasoning" }, + }, + ], + usage: { completion_tokens: 4096, reasoning_tokens: 3800 }, + }); + const out = await validateResponseQuality(res, false, silentLog); + assert.equal(out.valid, false, "no finish_reason still falls back to the 90% check"); assert.match(out.reason ?? "", /reasoning consumed/i); });