Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions open-sse/services/combo/validateQuality.ts
Original file line number Diff line number Diff line change
Expand Up @@ -945,10 +945,12 @@ export async function validateResponseQuality(
reason: `reasoning truncated at token limit (finish_reason: ${finishReason}) — no content output`,
};
}
if (usage) {
if (usage && finishReason !== "stop") {
const reasoningTokens = getReasoningTokens(usage);
// If reasoning consumed 90%+ of completion tokens, the model ran out of
// budget before producing any content output.
// finish_reason "stop" means the model ended on its own (OpenAI, Anthropic,
// Z.ai all document this). A high reasoning ratio on a clean stop is normal
// for models that cannot disable thinking (Claude Opus 4.7+, GLM-5.3). The
// 90% check stays as the fallback when the provider reports no finish_reason.
if (completionTokens > 0 && reasoningTokens >= completionTokens * 0.9) {
if (isTinyBudgetTruncation(completionTokens)) return { valid: true };
return {
Expand Down
17 changes: 15 additions & 2 deletions tests/unit/combo-quality-validator-reasoning.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -249,7 +249,7 @@ test("client-audit-2026-09-01: no finish_reason + reasoning <90% of tokens → s
assert.equal(out.valid, true, "should stay valid: no finish_reason signal, ratio under 90%");
});

test("client-audit-2026-09-01: finish_reason:stop + empty content + reasoning → ratio heuristic still applies unchanged", async () => {
test("finish_reason:stop + reasoning over 90% is a clean stop, not exhaustion", async () => {
const res = makeResponse({
choices: [
{
Expand All @@ -260,7 +260,20 @@ test("client-audit-2026-09-01: finish_reason:stop + empty content + reasoning
usage: { completion_tokens: 4096, reasoning_tokens: 3800 },
});
const out = await validateResponseQuality(res, false, silentLog);
assert.equal(out.valid, false, "finish_reason:stop doesn't short-circuit — ratio (>90%) still applies");
assert.equal(out.valid, true, "finish_reason:stop means the model ended on its own");
});

test("no finish_reason + reasoning over 90% stays invalid (fallback heuristic)", async () => {
const res = makeResponse({
choices: [
{
message: { content: null, reasoning_content: "Deep reasoning" },
},
],
usage: { completion_tokens: 4096, reasoning_tokens: 3800 },
});
const out = await validateResponseQuality(res, false, silentLog);
assert.equal(out.valid, false, "no finish_reason still falls back to the 90% check");
assert.match(out.reason ?? "", /reasoning consumed/i);
});

Expand Down
Loading