Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 72 additions & 1 deletion packages/core/src/__tests__/message-runtime-stage1.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -603,8 +603,79 @@ describe("runV5MessageRuntimeStage1", () => {
}
});

it("delivers bare-code and structured Stage 1 replies verbatim", async () => {
// #11504: the old junk heuristic flagged ANY character repeated 5+ times
// anywhere in the reply, so the 8+ consecutive spaces of two-level code
// indentation (a gemma-4-31b HumanEval-style bare function body), markdown
// "-----" dividers, and pretty-printed JSON all dead-ended into "I'm not
// sure how to answer that." — depressing eliza-harness HumanEval to 0.40
// vs 1.00 for the same model on raw harnesses.
const bareCodeBody = [
"def has_close_elements(numbers: List[float], threshold: float) -> bool:",
" for idx, elem in enumerate(numbers):",
" for idx2, elem2 in enumerate(numbers):",
" if idx != idx2:",
" distance = abs(elem - elem2)",
" if distance < threshold:",
" return True",
" return False",
].join("\n");
const fencedCode = `\`\`\`python\n${bareCodeBody}\n\`\`\``;
const proseThenFencedCode = `Here's the implementation:\n\n${fencedCode}`;
const prettyPrintedJson = [
"{",
' "name": "config",',
' "nested": {',
' "deep": {',
' "value": 1',
" }",
" }",
"}",
].join("\n");
const markdownWithDivider = "Results\n-------\nAll checks passed.";
for (const reply of [
bareCodeBody,
fencedCode,
proseThenFencedCode,
prettyPrintedJson,
markdownWithDivider,
]) {
const runtime = makeRuntime([
stage1Response({
contexts: ["simple"],
replyText: reply,
}),
]);

const result = await runV5MessageRuntimeStage1({
runtime,
message: makeMessage({
text: "Write a Python function that checks whether any two numbers in a list are closer than a threshold.",
}),
state: makeState(),
responseId: "00000000-0000-0000-0000-000000000005" as UUID,
});

expect(result.kind).toBe("direct_reply");
if (result.kind === "direct_reply") {
expect(result.result.responseContent?.text).toBe(reply);
}
expect(useModelCalls(runtime).length).toBe(1);
}
});

it("does not keep known-junk Stage 1 fragments when regeneration returns empty", async () => {
for (const badReply of ["RPPY", "{}", "aaaaa", "::::"]) {
for (const badReply of [
"RPPY",
"{}",
"aaaaa",
"::::",
// whitespace-only reply trims to empty
" ",
// degenerate single-character spam, including across whitespace
"!!!!!!!!",
"aaaaa aaaaa",
]) {
const runtime = makeRuntime([
stage1Response({
contexts: ["simple"],
Expand Down
17 changes: 10 additions & 7 deletions packages/core/src/services/message.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3073,13 +3073,16 @@ function isUnusableStage1Reply(reply: string | undefined): boolean {
if (/^```[a-z0-9_-]*\s+/iu.test(trimmed)) return false;
if (/^[\s{}[\]":,]+$/.test(trimmed)) return true;
if (/^\d+$/.test(trimmed)) return true;
// A reply that is ENTIRELY 5+ of the same glyph = a model-glitch reply
// ("aaaaaa", "......"). ANCHORED (like the siblings above): the run must be
// the whole reply, not merely present inside it — a real answer that happens
// to contain a run (a "XXXXXXXX" placeholder, a "--------" divider, aligned
// `df -h` columns) is legitimate and must not be blanked to "I'm not sure how
// to answer that." `\S` also keeps a pure-whitespace string from matching.
if (/^(\S)\1{4,}$/u.test(trimmed)) return true;
// Degenerate single-character spam: the WHOLE reply is one code point
// repeated 5+ times ("aaaaa", "!!!!!", "aaaaa aaaaa" across whitespace).
// A repeated run INSIDE a longer reply is legitimate — nested code
// indentation, aligned `df -h` columns, markdown "-----" dividers, an
// "XXXXXXXX" placeholder, pretty-printed JSON — and matching those blanked
// valid replies to "I'm not sure how to answer that." (#11504).
const nonWhitespace = [...trimmed.replace(/\s+/gu, "")];
if (nonWhitespace.length >= 5 && new Set(nonWhitespace).size === 1) {
return true;
}
if (/^[A-Z]{2,8}$/.test(trimmed)) {
const allowed = new Set(["OK", "YES", "NO", "STOP"]);
return !allowed.has(trimmed);
Expand Down
Loading