From 0a59dc3343dcece1041e30aaba121674343b8148 Mon Sep 17 00:00:00 2001 From: Koosha Pari Date: Sat, 12 Sep 2026 23:28:12 -0700 Subject: [PATCH 1/2] fix(compression): preserve polarity words and newlines in ultra heuristic (#13454) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ultra heuristic pruner scored polarity words (never, always, no, not, must, do) as stopwords (0.1) or short tokens (0.2), making them the first tokens pruned. This inverted instruction meaning: 'NEVER run rm -rf' → 'run rm -rf' 'must never be deleted' → 'must deleted' Also collapsed newlines via .replace(/\s{2,}/g, ' '), destroying bullet lists, headings, and code fences. Changes: - Remove polarity/modality words from STOPWORDS set (do, does, did, should, can, need, nor, always, never) - Add POLARITY_WORDS force-preserve set (score 1.0) for never, always, no, not, nor, must, do, can, should, need, and contractions - Only collapse spaces/tabs, not newlines, in pruneByScore output 8 tests in tests/unit/compression/ultra-heuristic-polarity.test.ts. Fixes #13454 --- .../services/compression/ultraHeuristic.ts | 41 ++++-- .../ultra-heuristic-polarity.test.ts | 118 ++++++++++++++++++ 2 files changed, 149 insertions(+), 10 deletions(-) create mode 100644 tests/unit/compression/ultra-heuristic-polarity.test.ts diff --git a/open-sse/services/compression/ultraHeuristic.ts b/open-sse/services/compression/ultraHeuristic.ts index 8a827159183e..546830a48b70 100644 --- a/open-sse/services/compression/ultraHeuristic.ts +++ b/open-sse/services/compression/ultraHeuristic.ts @@ -3,6 +3,10 @@ * * Scores tokens by information density and prunes low-value tokens * to achieve a target compression rate. + * + * #13454: Polarity/modality words (never, always, no, not, must, etc.) must + * NOT be prunable — dropping them flips the meaning of the sentence. + * "must never be deleted" → "must deleted" is worse than no compression. */ export const STOPWORDS = new Set([ @@ -19,18 +23,17 @@ export const STOPWORDS = new Set([ "have", "has", "had", - "do", - "does", - "did", + // #13454: "do/does/did" removed — carry polarity in imperatives + // ("do not push") and negations ("don't"). Dropping them flips + // instruction meaning. "will", "would", "could", - "should", + // #13454: "should" removed — modality word in instructions. "may", "might", "shall", - "can", - "need", + // #13454: "can/need" removed — modal auxiliaries in instructions. "dare", "ought", "used", @@ -59,7 +62,7 @@ export const STOPWORDS = new Set([ "and", "but", "or", - "nor", + // #13454: "nor" removed — negation word. "for", "yet", "so", @@ -88,8 +91,8 @@ export const STOPWORDS = new Set([ "even", "still", "already", - "always", - "never", + // #13454: "always/never" removed — polarity words, highest-value tokens + // in instructions. "never" → score 0.1 was the root cause of #13454. "often", "usually", "sometimes", @@ -100,6 +103,20 @@ export const STOPWORDS = new Set([ /** Regex for tokens that must never be pruned */ export const FORCE_PRESERVE_RE = /\d|https?:\/\/|[._\/\\]|Error:|Exception:|```/i; +// #13454: Polarity, modality, and negation words that must never be pruned. +// Dropping these flips the meaning of the sentence they appear in. +const POLARITY_WORDS = new Set([ + "never", "always", "no", "not", "nor", + "must", "shall", "shall not", + "do", "does", "did", + "don't", "doesn't", "didn't", + "can", "cannot", "can't", + "should", "shouldn't", + "need", "needs", "mustn't", + "won't", "wouldn't", + "could", "couldn't", +]); + /** * Score a single token (word/symbol) for information value. * Returns 0.0 (prune candidate) to 1.0 (must keep). @@ -107,6 +124,8 @@ export const FORCE_PRESERVE_RE = /\d|https?:\/\/|[._\/\\]|Error:|Exception:|```/ export function scoreToken(token: string): number { if (FORCE_PRESERVE_RE.test(token)) return 1.0; const lower = token.toLowerCase(); + // #13454: polarity words always score 1.0 — never prunable + if (POLARITY_WORDS.has(lower)) return 1.0; if (STOPWORDS.has(lower)) return 0.1; if (token.length <= 2) return 0.2; if (/^[A-Z]/.test(token)) return 0.8; // proper nouns / identifiers @@ -152,6 +171,8 @@ export function pruneByScore(text: string, keepRate = 0.5, minScore = 0.3): stri return keep ? t : ""; }) .join("") - .replace(/\s{2,}/g, " ") + // #13454: Only collapse spaces/tabs, NOT newlines. + // Collapsing newlines destroys bullet lists, headings, and code fences. + .replace(/[ \t]{2,}/g, " ") .trim(); } diff --git a/tests/unit/compression/ultra-heuristic-polarity.test.ts b/tests/unit/compression/ultra-heuristic-polarity.test.ts new file mode 100644 index 000000000000..de9c1dde45c9 --- /dev/null +++ b/tests/unit/compression/ultra-heuristic-polarity.test.ts @@ -0,0 +1,118 @@ +/** + * Tests for #13454: ultra heuristic must not prune polarity/modality words. + * + * The ultra heuristic engine scores tokens and prunes the lowest-scoring 50%. + * Before the fix, polarity words like "never", "always", "no", "not", "must" + * scored 0.1 (stopwords) or 0.2 (length ≤ 2), making them the first tokens + * pruned. This inverted instruction meaning: + * "must never be deleted" → "must deleted" + * "NEVER run rm -rf" → "run rm -rf" + * + * The fix adds polarity words to a force-preserve set (score 1.0) and stops + * collapsing newlines (which destroyed bullet lists and code fences). + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { scoreToken, pruneByScore, STOPWORDS } from "../../open-sse/services/compression/ultraHeuristic.ts"; + +test("scoreToken: polarity words score 1.0 (never prunable)", () => { + // These words MUST survive compression — they carry instruction polarity + const polarityWords = ["never", "always", "no", "not", "nor", "must", "do", "does", "did"]; + for (const word of polarityWords) { + assert.equal(scoreToken(word), 1.0, `"${word}" should score 1.0 (force-preserved)`); + } +}); + +test("scoreToken: modal auxiliaries score 1.0 (never prunable)", () => { + // Modal auxiliaries in instructions must not be pruned + const modals = ["can", "should", "need", "shall"]; + for (const word of modals) { + assert.equal(scoreToken(word), 1.0, `"${word}" should score 1.0 (modal auxiliary)`); + } +}); + +test("scoreToken: contractions score 1.0", () => { + const contractions = ["don't", "doesn't", "didn't", "can't", "cannot", "won't"]; + for (const word of contractions) { + assert.equal(scoreToken(word), 1.0, `"${word}" should score 1.0 (contraction)`); + } +}); + +test("scoreToken: regular stopwords still score 0.1", () => { + // Words that are genuinely low-value should still be prunable + const stopwords = ["a", "the", "is", "are", "was", "were", "in", "of", "on"]; + for (const word of stopwords) { + assert.equal(scoreToken(word), 0.1, `"${word}" should still score 0.1`); + } +}); + +test("pruneByScore: polarity words survive pruning", () => { + const block = `- NEVER run \`rm -rf\` on the target host. Always ask first. +- Do not push to \`main\` directly; open a PR. +- The backup files \`.app-prev-*\` must never be deleted. +- Never store the SSH password on disk. +- Always run \`npm test\` before \`npm run build\`. +- Do NOT edit files under \`/etc\` by hand.`; + + // Default engine settings: keepRate 0.5, minScore 0.3 + const result = pruneByScore(block, 0.5, 0.3); + + // All polarity words MUST survive + assert.ok(result.includes("never") || result.includes("NEVER"), "MUST preserve 'never'/'NEVER'"); + assert.ok(result.includes("always") || result.includes("Always"), "MUST preserve 'always'/'Always'"); + assert.ok(result.includes("not") || result.includes("NOT"), "MUST preserve 'not'/'NOT'"); + assert.ok(result.includes("Do"), "MUST preserve 'Do'"); +}); + +test("pruneByScore: newlines are preserved (not collapsed to spaces)", () => { + const block = `Line one +Line two +Line three`; + + const result = pruneByScore(block, 1.0); // keepRate=1.0 means keep everything + + // With keepRate=1.0 nothing is pruned, but we verify newlines survive + assert.ok(result.includes("\n"), "Newlines must be preserved when keepRate=1.0"); + assert.equal(result, block, "Full keepRate should return identical text"); +}); + +test("pruneByScore: newlines survive even with pruning", () => { + const block = `- NEVER do X +- ALWAYS do Y +- NEVER do Z`; + + const result = pruneByScore(block, 0.7, 0.3); + + // The line breaks between bullets should survive + const lines = result.split("\n"); + assert.ok(lines.length >= 2, "Line breaks between bullets must be preserved"); +}); + +test("pruneByScore: sample from #13454 issue preserves meaning", () => { + const block = `- NEVER run \`rm -rf\` on the target host. Always ask first. +- Do not push to \`main\` directly; open a PR. +- The backup files \`.app-prev-*\` must never be deleted. +- Never store the SSH password on disk. +- Always run \`npm test\` before \`npm run build\`. +- Do NOT edit files under \`/etc\` by hand.`; + + const result = pruneByScore(block, 0.5, 0.3); + + // After the fix, NONE of these meaning-critical words should be pruned: + assert.ok(result.includes("NEVER") || result.includes("never"), + "NEVER must survive"); + assert.ok(result.includes("Always") || result.includes("always"), + "Always must survive"); + assert.ok(result.includes("not") || result.includes("NOT"), + "not/NOT must survive"); + assert.ok(result.includes("Do") || result.includes("do"), + "Do/do must survive"); + assert.ok(result.includes("must") || result.includes("MUST"), + "must/MUST must survive"); + + // The critical test: "must never" must NOT become "must" alone + assert.ok( + !result.match(/\bmust\b(?![\s\S]*never)/) || result.includes("never"), + "must and never must both survive together" + ); +}); From 0fdd3d34082cfb427581cb1a8d963f913f356eb1 Mon Sep 17 00:00:00 2001 From: diegosouzapw <8016841+diegosouzapw@users.noreply.github.com> Date: Mon, 14 Sep 2026 23:43:59 -0300 Subject: [PATCH 2/2] chore(compression): fix polarity test import; refresh budget baseline for the intentional ultra trade-off --- .../check/compression-budget-baseline.json | 12 +++++----- .../ultra-heuristic-polarity.test.ts | 22 +++++++++---------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/scripts/check/compression-budget-baseline.json b/scripts/check/compression-budget-baseline.json index f94829362309..c9a6ed5f5952 100644 --- a/scripts/check/compression-budget-baseline.json +++ b/scripts/check/compression-budget-baseline.json @@ -8,9 +8,9 @@ }, "caveman": { "tasks": { - "prose": 129, - "tool-output": 127, - "json": 160 + "prose": 119, + "tool-output": 114, + "json": 136 } }, "aggressive": { @@ -22,9 +22,9 @@ }, "ultra": { "tasks": { - "prose": 92, - "tool-output": 116, - "json": 117 + "prose": 97, + "tool-output": 117, + "json": 126 } }, "rtk": { diff --git a/tests/unit/compression/ultra-heuristic-polarity.test.ts b/tests/unit/compression/ultra-heuristic-polarity.test.ts index de9c1dde45c9..4c46f507d63b 100644 --- a/tests/unit/compression/ultra-heuristic-polarity.test.ts +++ b/tests/unit/compression/ultra-heuristic-polarity.test.ts @@ -13,7 +13,7 @@ */ import { test } from "node:test"; import assert from "node:assert/strict"; -import { scoreToken, pruneByScore, STOPWORDS } from "../../open-sse/services/compression/ultraHeuristic.ts"; +import { scoreToken, pruneByScore } from "../../../open-sse/services/compression/ultraHeuristic.ts"; test("scoreToken: polarity words score 1.0 (never prunable)", () => { // These words MUST survive compression — they carry instruction polarity @@ -59,7 +59,10 @@ test("pruneByScore: polarity words survive pruning", () => { // All polarity words MUST survive assert.ok(result.includes("never") || result.includes("NEVER"), "MUST preserve 'never'/'NEVER'"); - assert.ok(result.includes("always") || result.includes("Always"), "MUST preserve 'always'/'Always'"); + assert.ok( + result.includes("always") || result.includes("Always"), + "MUST preserve 'always'/'Always'" + ); assert.ok(result.includes("not") || result.includes("NOT"), "MUST preserve 'not'/'NOT'"); assert.ok(result.includes("Do"), "MUST preserve 'Do'"); }); @@ -99,16 +102,11 @@ test("pruneByScore: sample from #13454 issue preserves meaning", () => { const result = pruneByScore(block, 0.5, 0.3); // After the fix, NONE of these meaning-critical words should be pruned: - assert.ok(result.includes("NEVER") || result.includes("never"), - "NEVER must survive"); - assert.ok(result.includes("Always") || result.includes("always"), - "Always must survive"); - assert.ok(result.includes("not") || result.includes("NOT"), - "not/NOT must survive"); - assert.ok(result.includes("Do") || result.includes("do"), - "Do/do must survive"); - assert.ok(result.includes("must") || result.includes("MUST"), - "must/MUST must survive"); + assert.ok(result.includes("NEVER") || result.includes("never"), "NEVER must survive"); + assert.ok(result.includes("Always") || result.includes("always"), "Always must survive"); + assert.ok(result.includes("not") || result.includes("NOT"), "not/NOT must survive"); + assert.ok(result.includes("Do") || result.includes("do"), "Do/do must survive"); + assert.ok(result.includes("must") || result.includes("MUST"), "must/MUST must survive"); // The critical test: "must never" must NOT become "must" alone assert.ok(