diff --git a/packages/core/src/__tests__/message-runtime-stage1.test.ts b/packages/core/src/__tests__/message-runtime-stage1.test.ts index 825f84a4f3a59..7c200f1baf293 100644 --- a/packages/core/src/__tests__/message-runtime-stage1.test.ts +++ b/packages/core/src/__tests__/message-runtime-stage1.test.ts @@ -603,8 +603,79 @@ describe("runV5MessageRuntimeStage1", () => { } }); + it("delivers bare-code and structured Stage 1 replies verbatim", async () => { + // #11504: the old junk heuristic flagged ANY character repeated 5+ times + // anywhere in the reply, so the 8+ consecutive spaces of two-level code + // indentation (a gemma-4-31b HumanEval-style bare function body), markdown + // "-----" dividers, and pretty-printed JSON all dead-ended into "I'm not + // sure how to answer that." — depressing eliza-harness HumanEval to 0.40 + // vs 1.00 for the same model on raw harnesses. + const bareCodeBody = [ + "def has_close_elements(numbers: List[float], threshold: float) -> bool:", + " for idx, elem in enumerate(numbers):", + " for idx2, elem2 in enumerate(numbers):", + " if idx != idx2:", + " distance = abs(elem - elem2)", + " if distance < threshold:", + " return True", + " return False", + ].join("\n"); + const fencedCode = `\`\`\`python\n${bareCodeBody}\n\`\`\``; + const proseThenFencedCode = `Here's the implementation:\n\n${fencedCode}`; + const prettyPrintedJson = [ + "{", + ' "name": "config",', + ' "nested": {', + ' "deep": {', + ' "value": 1', + " }", + " }", + "}", + ].join("\n"); + const markdownWithDivider = "Results\n-------\nAll checks passed."; + for (const reply of [ + bareCodeBody, + fencedCode, + proseThenFencedCode, + prettyPrintedJson, + markdownWithDivider, + ]) { + const runtime = makeRuntime([ + stage1Response({ + contexts: ["simple"], + replyText: reply, + }), + ]); + + const result = await runV5MessageRuntimeStage1({ + runtime, + message: makeMessage({ + text: "Write a Python function that checks whether any two numbers in a list are closer than a threshold.", + }), + state: makeState(), + responseId: "00000000-0000-0000-0000-000000000005" as UUID, + }); + + expect(result.kind).toBe("direct_reply"); + if (result.kind === "direct_reply") { + expect(result.result.responseContent?.text).toBe(reply); + } + expect(useModelCalls(runtime).length).toBe(1); + } + }); + it("does not keep known-junk Stage 1 fragments when regeneration returns empty", async () => { - for (const badReply of ["RPPY", "{}", "aaaaa", "::::"]) { + for (const badReply of [ + "RPPY", + "{}", + "aaaaa", + "::::", + // whitespace-only reply trims to empty + " ", + // degenerate single-character spam, including across whitespace + "!!!!!!!!", + "aaaaa aaaaa", + ]) { const runtime = makeRuntime([ stage1Response({ contexts: ["simple"], diff --git a/packages/core/src/services/message.ts b/packages/core/src/services/message.ts index cf50e81e9aa2e..99ccf789c9a2d 100644 --- a/packages/core/src/services/message.ts +++ b/packages/core/src/services/message.ts @@ -3073,13 +3073,16 @@ function isUnusableStage1Reply(reply: string | undefined): boolean { if (/^```[a-z0-9_-]*\s+/iu.test(trimmed)) return false; if (/^[\s{}[\]":,]+$/.test(trimmed)) return true; if (/^\d+$/.test(trimmed)) return true; - // A reply that is ENTIRELY 5+ of the same glyph = a model-glitch reply - // ("aaaaaa", "......"). ANCHORED (like the siblings above): the run must be - // the whole reply, not merely present inside it — a real answer that happens - // to contain a run (a "XXXXXXXX" placeholder, a "--------" divider, aligned - // `df -h` columns) is legitimate and must not be blanked to "I'm not sure how - // to answer that." `\S` also keeps a pure-whitespace string from matching. - if (/^(\S)\1{4,}$/u.test(trimmed)) return true; + // Degenerate single-character spam: the WHOLE reply is one code point + // repeated 5+ times ("aaaaa", "!!!!!", "aaaaa aaaaa" across whitespace). + // A repeated run INSIDE a longer reply is legitimate — nested code + // indentation, aligned `df -h` columns, markdown "-----" dividers, an + // "XXXXXXXX" placeholder, pretty-printed JSON — and matching those blanked + // valid replies to "I'm not sure how to answer that." (#11504). + const nonWhitespace = [...trimmed.replace(/\s+/gu, "")]; + if (nonWhitespace.length >= 5 && new Set(nonWhitespace).size === 1) { + return true; + } if (/^[A-Z]{2,8}$/.test(trimmed)) { const allowed = new Set(["OK", "YES", "NO", "STOP"]); return !allowed.has(trimmed);