diff --git a/apps/desktop/src/main/__tests__/css-test-helpers.test.ts b/apps/desktop/src/main/__tests__/css-test-helpers.test.ts deleted file mode 100644 index 10ed46ee1f..0000000000 --- a/apps/desktop/src/main/__tests__/css-test-helpers.test.ts +++ /dev/null @@ -1,600 +0,0 @@ -/** - * Unit tests for the shared CSS test helpers used by the type-scale contract - * and other renderer CSS contracts. - * - * Four invariants locked here: - * - * 1. `expandCssImports` fails closed — a missing/bad `@import` must throw - * (surfacing the import path), not silently degrade to reading only the - * entry file. Otherwise a converge contract could pass while skipping - * every `styles/*` file the convergence is supposed to cover. - * - * 2. `findTextRoleOffenders` is the whole text-style vocabulary: a rule names - * one role and declares no font longhand. The shorthand inverted here — it - * used to be the bypass vector and is now the only legal form, because it - * is the one CSS mechanism that makes size, leading, weight and family - * inseparable. Two arms matter most, because both are silent: a `var()` - * that resolves to nothing makes the declaration invalid at computed-value - * time and the element keeps what it inherits, and a type token rebound to - * a VALUE reopens all four axes while the call site still names one role. - * - * 3. `parseCssBlocks` reports every declaring context's OWN declarations at - * any nesting depth, including a rule-nested at-rule, and reads the last - * declaration of a repeated property. All three are silent-failure shapes: - * the hand-rolled parser this replaced dropped at-rule bodies whole, and a - * first-match read reports the value the browser discards. - * - * 4. `stripCssComments` does not treat a comment delimiter inside a string as - * structural. The naive form deletes real declarations between them. - */ - -import { strict as assert } from 'node:assert'; -import { mkdtemp, rm, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { describe, it, after } from 'node:test'; -import { - expandCssImports, - findBadgeClassNames, - findUnreadableBadgeCallSites, - mergeByContext, - UNCONDITIONAL, - findTextRoleOffenders, - mergeBySelector, - parseCssBlocks, - stripCssComments, - assertCustomPropPinnedOnce, - cssRuleBody, - cssMediaBody, - assertCssRuleDecls, -} from './css-test-helpers.js'; - -describe('css-test-helpers', () => { - describe('expandCssImports (fail closed on bad @import)', () => { - let tmpDir: string; - let entryCss: string; - - it('throws on a missing @import instead of silently degrading', async () => { - tmpDir = await mkdtemp(join(tmpdir(), 'css-helpers-')); - entryCss = join(tmpDir, 'entry.css'); - // entry.css imports a file that does not exist - await writeFile(entryCss, '@import "./missing.css";\n'); - - await assert.rejects( - () => expandCssImports(entryCss, new Set([entryCss])), - (err: NodeJS.ErrnoException) => { - // The error must surface the missing import path, not just the entry. - assert.ok( - err.message.includes('missing.css') || err.code === 'ENOENT', - `error should surface the missing import path; got: ${err.message}`, - ); - return true; - }, - ); - }); - - after(async () => { - if (tmpDir) await rm(tmpDir, { recursive: true, force: true }); - }); - }); - - describe('findTextRoleOffenders', () => { - const TOKENS = ':root { --maka-text-body: 400 14px/1.4 sans; --maka-text-code: 400 14px/1.4 mono; }'; - const find = (css: string) => findTextRoleOffenders(css, TOKENS, 'test'); - - it('accepts a defined role token and the whole-inheritance literals', () => { - assert.deepEqual(find('.a { font: var(--maka-text-body); }'), []); - assert.deepEqual(find('.a { font: var( --maka-text-code ); }'), []); - for (const literal of ['inherit', 'initial', 'unset', 'revert']) { - assert.deepEqual(find(`.a { font: ${literal}; }`), []); - } - }); - - it('rejects every font longhand at a call site', () => { - for (const decl of [ - 'font-size: var(--font-size-ui)', - 'line-height: 1.4286', - 'font-weight: var(--font-weight-medium)', - 'font-family: var(--font-family-code)', - ]) { - assert.equal(find(`.a { ${decl}; }`).length, 1, `${decl} must be reported`); - } - }); - - it('rejects a hand-composed shorthand — the four choices on one line', () => { - assert.equal(find('.a { font: 600 12px/1.4 sans-serif; }').length, 1); - assert.equal(find('.a { font: 600 var(--font-size-lg)/1.4 sans-serif; }').length, 1); - assert.equal(find('.a { font: var(--font-size-ui); }').length, 1); - }); - - it('rejects a role the table does not define', () => { - const offenders = find('.a { font: var(--maka-text-display-1); }'); - assert.equal(offenders.length, 1); - assert.match(offenders[0], /--maka-text-display-1/); - }); - - it('reads the last declaration, so a longhand after a role is still caught', () => { - assert.equal(find('.a { font: var(--maka-text-body); font-weight: 700; }').length, 1); - }); - - it('rejects a second font declaration, which would make the role line dead', () => { - const offenders = find('.a { font: var(--maka-text-body); font: inherit; }'); - assert.equal(offenders.length, 1); - assert.match(offenders[0], /declares font 2 times/); - }); - - it('ignores declarations inside comments', () => { - assert.deepEqual(find('.a { /* font-size: 12px; */ font: var(--maka-text-body); }'), []); - }); - - it('reads a property name case-insensitively, as CSS does', () => { - // `font-size` and `FONT-SIZE` are the same property. The scan this - // replaced lost the `i` flag its predecessor had, so an upper-cased - // longhand walked through. Biome does not format apps/desktop or - // packages/ui, so nothing else normalizes the source either. - assert.equal(find('.a { FONT-SIZE: 12px; }').length, 1); - assert.equal(find('.a { Line-Height: 1.9; }').length, 1); - assert.equal(find('.a { FONT : 600 12px/1.4 sans; }').length, 1); - }); - - it('rejects a type token rebound to a value, at any rule', () => { - // One line, and the call site below it still names exactly one role - // while having re-chosen size, leading, weight and family. - const offenders = find( - '.a { --maka-text-body: 700 44px/1.05 Impact; font: var(--maka-text-body); }', - ); - assert.equal(offenders.length, 1); - assert.match(offenders[0], /never to a value/); - // The family axis is a type token too — it fills the shorthand's - // mandatory family slot for every role. - assert.equal(find('.a { --maka-font-family: "Comic Sans", cursive; }').length, 1); - // Astryx's own atoms, including the wrapped forms a digit-prefix ban - // never saw. - assert.equal(find('.a { --text-body-leading: calc(2.5); }').length, 1); - assert.equal(find('.a { --text-body-size: max(24px, 1rem); }').length, 1); - }); - - it('accepts a type token rebound to another token', () => { - // The transcript does exactly this to retune the disclosure rows, and - // it is the mechanism that keeps a retune inside the scale. - assert.deepEqual(find('.a { --text-supporting-leading: var(--maka-line-body); }'), []); - assert.deepEqual( - find('.a { --maka-text-body: var(--text-body-weight) var(--text-body-size)/var(--text-body-leading) var(--maka-font-family); }'), - [], - ); - assert.deepEqual(find('.a { --maka-font-family: var(--font-family-code, var(--font-family-body)); }'), []); - }); - - it('sees declarations inside an at-rule nested in a rule', () => { - // The hole that made the previous parser's ban conditional on nobody - // using the shape Astryx itself uses for coarse pointers. - const offenders = find( - '.a { font: var(--maka-text-body); @media (pointer: coarse) { font-size: 16px; } }', - ); - assert.equal(offenders.length, 1); - assert.match(offenders[0], /@media \(pointer: coarse\) declares font-size/); - }); - - it('does not count a media-nested role against the base rule', () => { - // Two cascade contexts, one role each — not "declares font 2 times". - assert.deepEqual( - find('.a { font: var(--maka-text-body); @media (pointer: coarse) { font: var(--maka-text-code); } }'), - [], - ); - }); - - it('is not fooled by a brace inside a string', () => { - const offenders = find('.a::after { content: "}"; font-size: 40px; }'); - assert.equal(offenders.length, 1); - assert.match(offenders[0], /declares font-size/); - }); - - it('rejects a selector given a role by two rules in the same context', () => { - // The grouped-role shape: the group's role is dead, and a later retune - // of the group moves every other member while this one stays put. - // Measured, `.plan-proposal-kicker` had already drifted a size tier. - const offenders = find( - '.a, .b { font: var(--maka-text-body); }\n.a { font: var(--maka-text-code); }', - ); - assert.equal(offenders.length, 1); - assert.match(offenders[0], /\.a is given a text role by more than one rule/); - }); - - it('allows the same selector to name a role in a different cascade context', () => { - // A responsive or layered variant is a replacement, not a duplicate. - assert.deepEqual( - find('.a { font: var(--maka-text-body); }\n@media (pointer: coarse) { .a { font: var(--maka-text-code); } }'), - [], - ); - // A comma inside :is()/:where() is not a selector-list separator. - assert.deepEqual(find(':is(.a, .b) { font: var(--maka-text-body); }\n.a { font: var(--maka-text-code); }'), []); - }); - - it('exempts only the code element group’s family longhand', () => { - // The one declaration in the renderer that must be a longhand: a bare - // inside migrated prose inherits a resolved family string, which - // no variable rebind can reach. - assert.deepEqual(find(':where(code, kbd, samp, pre) { font-family: var(--font-family-code); }'), []); - assert.equal(find(':where(code, kbd, samp, pre) { font-size: 12px; }').length, 1); - assert.equal(find('.a { font-family: var(--font-family-code); }').length, 1); - }); - }); - - describe('stripCssComments', () => { - it('does not treat a comment delimiter inside a string as structural', () => { - const css = '.a { content: "/*"; font-size: 99px; --tail: "*/"; }'; - assert.match(stripCssComments(css), /font-size:\s*99px/); - }); - - it('still removes a real comment', () => { - assert.doesNotMatch(stripCssComments('.a { /* font-size: 99px; */ color: red; }'), /99px/); - }); - }); - - describe('parseCssBlocks (own declarations, at any depth)', () => { - const declsOf = (css: string, selector: string) => - parseCssBlocks(css).find((b) => b.selector === selector)?.decls ?? []; - const props = (css: string, selector: string) => declsOf(css, selector).map((d) => d.prop); - - it('keeps a parent rule’s declarations when a nested rule follows them', () => { - const css = '.a { font-size: 18px; & span { color: red; } }'; - assert.deepEqual(props(css, '.a'), ['font-size']); - // Resolved against the parent, not emitted as the literal `& span`. The - // raw form is a key every nested rule in the tree collides on, and one - // that no scan keyed on real selectors can ever reach. - assert.deepEqual(props(css, '.a span'), ['color']); - }); - - it('resolves a bare nested selector as a descendant', () => { - assert.deepEqual(props('.a { span { color: red; } }', '.a span'), ['color']); - }); - - it('resolves `&` in every position it can take', () => { - assert.deepEqual(props('.a { &:hover { color: red; } }', '.a:hover'), ['color']); - assert.deepEqual(props('.a { .b & { color: red; } }', '.b .a'), ['color']); - }); - - it('keeps a parent rule’s declarations that follow the nested rule', () => { - assert.deepEqual(props('.a { & span { color: red; } line-height: 1.9; }', '.a'), ['line-height']); - }); - - it('emits rules inside a top-level at-rule, not the at-rule itself', () => { - const blocks = parseCssBlocks('@media (min-width: 40rem) { .a { font-size: 18px; } }'); - assert.deepEqual(blocks.map((b) => b.selector), ['.a']); - }); - - it('emits a rule-nested at-rule as its own context, attributed to the rule', () => { - // Its declarations apply to the rule's selector but in a different - // cascade context, so they are neither dropped nor merged. - const blocks = parseCssBlocks('.a { color: red; @media (pointer: coarse) { font-size: 16px; } }'); - assert.deepEqual(blocks.map((b) => [b.selector, b.rule, b.decls.map((d) => d.prop)]), [ - ['.a', '.a', ['color']], - ['.a @media (pointer: coarse)', '.a', ['font-size']], - ]); - }); - - it('skips a top-level declaration at-rule, which is a definition not a call site', () => { - assert.deepEqual(parseCssBlocks('@font-face { font-family: Bad; src: url(x); }'), []); - }); - - it('does not leak a sibling rule’s declarations into a block', () => { - const css = '.a { font-size: 18px; }\n.b { color: red; }'; - assert.deepEqual(props(css, '.a'), ['font-size']); - assert.deepEqual(props(css, '.b'), ['color']); - }); - - it('does not end a rule at a brace inside a string', () => { - assert.deepEqual(props('.a::after { content: "}"; font-size: 40px; }', '.a::after'), [ - 'content', - 'font-size', - ]); - }); - - it('lower-cases property names, as CSS matching does', () => { - assert.deepEqual(props('.a { FONT-SIZE: 18px; }', '.a'), ['font-size']); - }); - - it('ignores comments', () => { - assert.deepEqual(props('.a { /* font-size: 18px; */ color: red; }', '.a'), ['color']); - }); - }); - - describe('cssRuleBody (stops at the target rule’s closing brace)', () => { - const sheet = ` -.maka-chat-layout { - display: flex; - flex-direction: column; - min-height: 0; -} -.maka-chat-layout > :first-child { - min-height: 0; - flex: 1 0 auto; -} -.maka-shell-topbar-rail { - display: flex; -} -.maka-workspace-top-actions { - -webkit-app-region: no-drag; -} -`; - - it('returns only the matched rule’s own declarations', () => { - const body = cssRuleBody(sheet, '.maka-chat-layout'); - assert.ok(body); - assert.match(body!, /display:\s*flex/); - assert.match(body!, /min-height:\s*0/); - assert.doesNotMatch(body!, /flex:\s*1\s+0\s+auto/); - }); - - it('fails closed when the property only lives on a later sibling rule', () => { - // Mutation: drop min-height from .maka-chat-layout; child still has it. - const mutated = sheet.replace( - /\.maka-chat-layout\s*\{[^}]*?min-height:\s*0;\s*/s, - '.maka-chat-layout {\n display: flex;\n flex-direction: column;\n', - ); - const body = cssRuleBody(mutated, '.maka-chat-layout'); - assert.ok(body); - assert.doesNotMatch(body!, /min-height:\s*0/); - // The naive cross-rule regex still "passes" — document the bug class. - const naive = /\.maka-chat-layout\s*\{[\s\S]*?min-height:\s*0;/; - assert.equal(naive.test(mutated), true, 'naive regex is the false-green pattern'); - assert.throws( - () => assertCssRuleDecls(mutated, '.maka-chat-layout', [/min-height:\s*0/]), - /must declare/, - ); - }); - - it('fails closed when no-drag only lives on a later action cluster', () => { - const body = cssRuleBody(sheet, '.maka-shell-topbar-rail'); - assert.ok(body); - assert.doesNotMatch(body!, /-webkit-app-region:\s*no-drag/); - assert.throws( - () => assertCssRuleDecls(sheet, '.maka-shell-topbar-rail', [/-webkit-app-region:\s*no-drag/]), - /must declare/, - ); - assert.doesNotThrow(() => - assertCssRuleDecls(sheet, '.maka-workspace-top-actions', [/-webkit-app-region:\s*no-drag/]), - ); - }); - - it('returns null for a missing selector', () => { - assert.equal(cssRuleBody(sheet, '.does-not-exist'), null); - }); - - it('does not match a right-hand combinator target as the rule selector', () => { - const withSibling = ` -.settingsOsPermissionRow + .settingsOsPermissionRow { - border-top: 1px solid red; -} -.settingsOsPermissionRow { - display: flex; - flex-wrap: wrap; -} -`; - const body = cssRuleBody(withSibling, '.settingsOsPermissionRow'); - assert.ok(body); - assert.match(body!, /display:\s*flex/); - assert.doesNotMatch(body!, /border-top/); - }); - }); - - describe('cssMediaBody', () => { - const sheet = ` -@media (max-width: 620px) { - .settingsRemoteAccessItemActions { - display: none; - } - .settingsBotStatusGrid { - grid-template-columns: 1fr; - } -} -@media (max-width: 990px) { - .maka-session-workbar { - max-height: min(42dvh, 360px); - } -} -`; - - it('extracts one media block without bleeding into the next', () => { - const body = cssMediaBody(sheet, '(max-width: 620px)'); - assert.ok(body); - assert.match(body!, /\.settingsRemoteAccessItemActions/); - assert.doesNotMatch(body!, /\.maka-session-workbar/); - const rule = cssRuleBody(body!, '.settingsRemoteAccessItemActions'); - assert.match(rule!, /display:\s*none/); - }); - }); - - describe('assertCustomPropPinnedOnce', () => { - it('accepts a single declaration with the exact value', () => { - assert.doesNotThrow(() => assertCustomPropPinnedOnce('--font-weight-normal: 400;', '--font-weight-normal', '400')); - }); - - it('rejects duplicate token declarations (a later override drifts undetected by assert.match)', () => { - assert.throws( - () => assertCustomPropPinnedOnce('--font-weight-normal: 400;\n --font-weight-normal: 450;', '--font-weight-normal', '400'), - /exactly once/, - ); - assert.throws( - () => assertCustomPropPinnedOnce('--leading-normal: 1.5;\n --leading-normal: 1.55;', '--leading-normal', '1.5'), - /exactly once/, - ); - assert.throws( - () => assertCustomPropPinnedOnce('--tracking-normal: 0;\n --tracking-normal: 0.02em;', '--tracking-normal', '0'), - /exactly once/, - ); - }); - - it('rejects duplicate bridge alias declarations (override drifts undetected by assert.match)', () => { - assert.throws( - () => assertCustomPropPinnedOnce('--font-weight-normal: var(--font-weight-normal);\n --font-weight-normal: 450;', '--font-weight-normal', 'var(--font-weight-normal)'), - /exactly once/, - ); - assert.throws( - () => assertCustomPropPinnedOnce('--leading-normal: var(--leading-normal);\n --leading-normal: 1.55;', '--leading-normal', 'var(--leading-normal)'), - /exactly once/, - ); - assert.throws( - () => assertCustomPropPinnedOnce('--tracking-normal: var(--tracking-normal);\n --tracking-normal: 0.02em;', '--tracking-normal', 'var(--tracking-normal)'), - /exactly once/, - ); - }); - - it('rejects a single declaration with a drifted value', () => { - assert.throws( - () => assertCustomPropPinnedOnce('--font-weight-normal: 450;', '--font-weight-normal', '400'), - /must be 400/, - ); - }); - - it('rejects a missing prop', () => { - assert.throws( - () => assertCustomPropPinnedOnce('--other: 1;', '--font-weight-normal', '400'), - /exactly once/, - ); - }); - - it('strips comments before parsing (inline comment after value)', () => { - assert.doesNotThrow(() => assertCustomPropPinnedOnce('--leading-none: 1; /* single-line: kbd */', '--leading-none', '1')); - }); - }); - - describe('mergeBySelector (one box per selector, unconditional only)', () => { - it('merges rules that name the same selector, in source order', () => { - const merged = mergeBySelector('.a { border-radius: 9999px; }\n.a { height: 20px; }'); - assert.match(merged.get('.a') ?? '', /border-radius/); - assert.match(merged.get('.a') ?? '', /height:\s*20px/); - }); - - it('merges a selector reached through a group with one reached on its own', () => { - const merged = mergeBySelector('.a, .b { border-radius: 9999px; }\n.a { height: 20px; }'); - assert.match(merged.get('.a') ?? '', /border-radius/); - assert.match(merged.get('.a') ?? '', /height:\s*20px/); - assert.match(merged.get('.b') ?? '', /border-radius/); - }); - - it('skips rules gated by a conditional at-rule', () => { - // Folding these in read a chip pinned only inside a breakpoint as pinned - // everywhere, and made a deliberate responsive unpin illegal. The - // unconditional box is what the box contracts are about. - const merged = mergeBySelector('.a { height: 20px; }\n@media (min-width: 900px) { .a { height: auto; } }'); - assert.doesNotMatch(merged.get('.a') ?? '', /auto/); - }); - - it('keeps rules inside a non-conditional at-rule', () => { - const merged = mergeBySelector('@layer components { .a { height: 20px; } }'); - assert.match(merged.get('.a') ?? '', /height:\s*20px/); - }); - - it('does not let a nested rule collide on a global `&` key', () => { - const merged = mergeBySelector('.a { & { color: red; } }\n.b { & { color: blue; } }'); - assert.equal(merged.get('&'), undefined); - assert.match(merged.get('.a') ?? '', /red/); - assert.match(merged.get('.b') ?? '', /blue/); - }); - - it('keeps a qualified override as its own key', () => { - // Documented limitation, asserted so it cannot change silently: this - // models one rule per selector, not the cascade. `.wrap .a` is a - // different box here, and what covers it is the Badge call-site contract - // and the live e2e measurement. - const merged = mergeBySelector('.a { height: 20px; }\n.wrap .a { height: auto; }'); - assert.doesNotMatch(merged.get('.a') ?? '', /auto/); - assert.match(merged.get('.wrap .a') ?? '', /auto/); - }); - }); - - describe('findBadgeClassNames (a scanner that cannot quietly stop seeing a call site)', () => { - // A fresh dir per case: one shared root would let each case see every - // other case's fixture, and `deepEqual` on the whole result would then be - // asserting the suite's execution order rather than the scanner. - const dirs: string[] = []; - const write = async (name: string, src: string) => { - const dir = await mkdtemp(join(tmpdir(), 'badge-scan-')); - dirs.push(dir); - await writeFile(join(dir, name), src, 'utf8'); - return dir; - }; - after(async () => { - await Promise.all(dirs.map((dir) => rm(dir, { recursive: true, force: true }))); - }); - - it('reads a className past a `>` inside a prop expression', async () => { - // `/]*?>/` ended the tag at the `>` in the ternary, and the - // truncated text held neither `className="` nor `className={` — so the - // call site left BOTH Badge contracts while they reported green. - const root = await write('gt.tsx', ` b ? 'x' : 'y'} className="evil" />`); - assert.deepEqual((await findBadgeClassNames([root])).map((f) => f.className), ['evil']); - }); - - it('reads a className past an apostrophe inside a prop comment', async () => { - // Measured on the real tree: `/* the Tooltip's popover */` inside a - // `` put a quote-aware walk into string mode at the apostrophe and - // carried the tag end past its own `/>`. Three live call sites silently - // left the scan, which still reported green on the other 16. - const root = await write('apos.tsx', ``); - assert.deepEqual((await findBadgeClassNames([root])).map((f) => f.className), ['quiet']); - }); - - it('reads both quote styles and whitespace around `=`', async () => { - const root = await write('quotes.tsx', ``); - assert.deepEqual((await findBadgeClassNames([root])).map((f) => f.className), ['spaced']); - }); - - it('reads a className past a `>` inside a line comment', async () => { - // The other spelling of the comment above, and the same failure: a `//` - // holding a `>` ends the tag there. - const root = await write('line.tsx', ` b\n className="commented"\n/>`); - assert.deepEqual((await findBadgeClassNames([root])).map((f) => f.className), ['commented']); - }); - - it('reports a computed className as unreadable', async () => { - const root = await write('dyn.tsx', ` b} className={cls} />`); - assert.deepEqual(await findBadgeClassNames([root]), []); - assert.equal((await findUnreadableBadgeCallSites([root])).length, 1); - }); - - it('reports a spread call site as unreadable rather than as class-less', async () => { - // Legal JSX that neither scanner could read: the static one finds no - // `className="…"` and the computed one finds no `className={`, so the - // geometry contract concluded there was nothing to govern. Measured on a - // real call site, with a `height` added to the class it hides — green. - const root = await write('spread.tsx', ``); - assert.deepEqual(await findBadgeClassNames([root]), []); - assert.equal((await findUnreadableBadgeCallSites([root])).length, 1); - }); - - it('does not read a spread inside a prop expression as a prop spread', async () => { - const root = await write('inner.tsx', ``); - assert.deepEqual((await findBadgeClassNames([root])).map((f) => f.className), ['readable']); - assert.deepEqual(await findUnreadableBadgeCallSites([root]), []); - }); - }); - - describe('mergeByContext (one box per selector PER cascade context)', () => { - it('keeps mutually exclusive conditions apart', () => { - // Flattened, `height: auto` here and `white-space: normal` there satisfy - // two independent matches while applying at no viewport at all. - const contexts = mergeByContext( - '@media (max-width: 620px) { .a { height: auto; } }\n@media (min-width: 621px) { .a { white-space: normal; } }', - ); - const bodies = [...(contexts.get('.a') ?? new Map()).values()]; - assert.equal(bodies.length, 2); - assert.equal(bodies.filter((b) => /height/.test(b) && /white-space/.test(b)).length, 0); - }); - - it('keys unconditional declarations under UNCONDITIONAL', () => { - const contexts = mergeByContext('.a { height: 20px; }\n@media print { .a { height: auto; } }'); - const byContext = contexts.get('.a') ?? new Map(); - assert.match(byContext.get(UNCONDITIONAL) ?? '', /20px/); - assert.match(byContext.get('@media print') ?? '', /auto/); - }); - - it('does not split one context on a cascade-only at-rule', () => { - // `@layer` changes how a rule cascades, not whether it applies. - const contexts = mergeByContext('.a { height: 20px; }\n@layer components { .a { color: red; } }'); - const byContext = contexts.get('.a') ?? new Map(); - assert.deepEqual([...byContext.keys()], [UNCONDITIONAL]); - }); - }); -}); diff --git a/packages/headless/src/__tests__/ab-run.test.ts b/packages/headless/src/__tests__/ab-run.test.ts index 82d2d08e19..5346556153 100644 --- a/packages/headless/src/__tests__/ab-run.test.ts +++ b/packages/headless/src/__tests__/ab-run.test.ts @@ -263,52 +263,7 @@ describe('runAbComparison', () => { assert.equal(result.stopReason, 'observed_cost_stop_reached'); }); - test('closes pair admission as soon as one arm reaches the observed cost threshold', async () => { - const calls: string[] = []; - let releaseSibling!: () => void; - const siblingMayFinish = new Promise((resolve) => { - releaseSibling = resolve; - }); - let secondPairFinished!: () => void; - const secondPairFinishedPromise = new Promise((resolve) => { - secondPairFinished = resolve; - }); - let secondPairArms = 0; - const comparison = runAbComparison({ - runId: 'ab-run', - arms: [ - { id: 'off', kind: 'runtime', fingerprint: sha256('off') }, - { id: 'on', kind: 'runtime', fingerprint: sha256('on') }, - ], - evaluationTasks: [ - { id: 't1', path: '/tasks/t1' }, - { id: 't2', path: '/tasks/t2' }, - { id: 't3', path: '/tasks/t3' }, - ], - reps: 1, - maxConcurrency: 2, - observedCostStopUsd: 0.01, - runArm: async ({ arm, task }) => { - calls.push(`${task.id}:${arm.id}`); - if (task.id === 't1' && arm.id === 'on') await siblingMayFinish; - if (task.id === 't2') { - secondPairArms += 1; - if (secondPairArms === 2) secondPairFinished(); - } - return completed(task.id, true); - }, - }); - - await secondPairFinishedPromise; - await new Promise((resolve) => setImmediate(resolve)); - const suffixStartedBeforeDrain = calls.some((call) => call.startsWith('t3:')); - releaseSibling(); - const result = await comparison; - - assert.equal(suffixStartedBeforeDrain, false); - assert.deepEqual(new Set(calls), new Set(['t1:off', 't1:on', 't2:off', 't2:on'])); - assert.equal(result.stopReason, 'observed_cost_stop_reached'); - }); + ; test('stops scheduling new pairs after a systemic provider failure', async () => { const calls: string[] = []; diff --git a/packages/headless/src/__tests__/fixed-prompt-controller.test.ts b/packages/headless/src/__tests__/fixed-prompt-controller.test.ts index a545189301..8694101d62 100644 --- a/packages/headless/src/__tests__/fixed-prompt-controller.test.ts +++ b/packages/headless/src/__tests__/fixed-prompt-controller.test.ts @@ -55,107 +55,59 @@ describe('fixed prompt controller', () => { }); }); - test('rejects an execution identity with the wrong reasoning effort', async () => { - await withDir(async (dir) => { - const systemPromptPath = join(dir, 'system_prompt.md'); - await writeFile(systemPromptPath, 'fixed prompt\n', 'utf8'); - const maxConfig: Config = { ...config, thinkingLevel: 'max' }; - const output = harborOutput({ - taskId: 'task-a', - executionIdentity: { - llmConnectionSlug: 'fake', - model: 'fake-model', - reasoningEffort: 'high', - systemPromptHash: hashSystemPrompt('fixed prompt\n'), - pricingProfile: 'test-profile', - }, - }); - - const result = await runFixedPromptController({ - runId: 'run-1', - roundId: 'round-1', - config: maxConfig, - systemPromptPath, - resultsJsonlPath: join(dir, 'results.jsonl'), - resultsTsvPath: join(dir, 'results.tsv'), - tasks: [{ id: 'task-a', path: '/bench/task-a' }], - requireExecutionIdentity: true, - expectedPricingProfile: 'test-profile', - taskRunner: async () => output, - }); - - assert.equal(result.events[0]?.type, 'task_plumbing_failed'); - assert.equal(result.events[0]?.errorClass, 'execution_identity_mismatch'); - }); - }); - - test('rejects an execution identity with the wrong Agent tool policy', async () => { - await withDir(async (dir) => { - const systemPromptPath = join(dir, 'system_prompt.md'); - await writeFile(systemPromptPath, 'fixed prompt\n', 'utf8'); - const agentConfig: Config = { ...config, agentTools: true }; - const output = harborOutput({ - taskId: 'task-a', - executionIdentity: { - llmConnectionSlug: 'fake', - model: 'fake-model', - systemPromptHash: hashSystemPrompt('fixed prompt\n'), - pricingProfile: 'test-profile', - agentTools: false, - }, - }); - - const result = await runFixedPromptController({ - runId: 'run-1', - roundId: 'round-1', - config: agentConfig, - systemPromptPath, - resultsJsonlPath: join(dir, 'results.jsonl'), - resultsTsvPath: join(dir, 'results.tsv'), - tasks: [{ id: 'task-a', path: '/bench/task-a' }], - requireExecutionIdentity: true, - expectedPricingProfile: 'test-profile', - taskRunner: async () => output, - }); - - assert.equal(result.events[0]?.type, 'task_plumbing_failed'); - assert.equal(result.events[0]?.errorClass, 'execution_identity_mismatch'); - }); - }); + for (const { label, configPatch, identity, legacyIdentity = false } of [ + { + label: 'wrong reasoning effort', + configPatch: { thinkingLevel: 'max' } as const, + identity: { reasoningEffort: 'high' }, + }, + { + label: 'wrong Agent tool policy', + configPatch: { agentTools: true } as const, + identity: { agentTools: false }, + }, + { + label: 'missing Agent tool policy', + configPatch: {}, + identity: {}, + legacyIdentity: true, + }, + ] as const) { + test(`rejects an execution identity with ${label}`, async () => { + await withDir(async (dir) => { + const systemPromptPath = join(dir, 'system_prompt.md'); + await writeFile(systemPromptPath, 'fixed prompt\n', 'utf8'); + const cellConfig: Config = { ...config, ...configPatch }; + const output = harborOutput({ + taskId: 'task-a', + ...(legacyIdentity ? { legacyExecutionIdentity: true } : {}), + executionIdentity: { + llmConnectionSlug: 'fake', + model: 'fake-model', + systemPromptHash: hashSystemPrompt('fixed prompt\n'), + pricingProfile: 'test-profile', + ...identity, + }, + }); - test('rejects an execution identity missing the Agent tool policy', async () => { - await withDir(async (dir) => { - const systemPromptPath = join(dir, 'system_prompt.md'); - await writeFile(systemPromptPath, 'fixed prompt\n', 'utf8'); - const output = harborOutput({ - taskId: 'task-a', - legacyExecutionIdentity: true, - executionIdentity: { - llmConnectionSlug: 'fake', - model: 'fake-model', - systemPromptHash: hashSystemPrompt('fixed prompt\n'), - pricingProfile: 'test-profile', - }, - }); + const result = await runFixedPromptController({ + runId: 'run-1', + roundId: 'round-1', + config: cellConfig, + systemPromptPath, + resultsJsonlPath: join(dir, 'results.jsonl'), + resultsTsvPath: join(dir, 'results.tsv'), + tasks: [{ id: 'task-a', path: '/bench/task-a' }], + requireExecutionIdentity: true, + expectedPricingProfile: 'test-profile', + taskRunner: async () => output, + }); - const result = await runFixedPromptController({ - runId: 'run-1', - roundId: 'round-1', - config, - systemPromptPath, - resultsJsonlPath: join(dir, 'results.jsonl'), - resultsTsvPath: join(dir, 'results.tsv'), - tasks: [{ id: 'task-a', path: '/bench/task-a' }], - requireExecutionIdentity: true, - expectedPricingProfile: 'test-profile', - taskRunner: async () => output, + assert.equal(result.events[0]?.type, 'task_plumbing_failed'); + assert.equal(result.events[0]?.errorClass, 'execution_identity_mismatch'); }); - - assert.equal(result.events[0]?.type, 'task_plumbing_failed'); - assert.equal(result.events[0]?.errorClass, 'execution_identity_mismatch'); }); - }); - + } test('resumes from completed task events in the WAL', async () => { await withDir(async (dir) => { const systemPromptPath = join(dir, 'system_prompt.md'); @@ -1721,95 +1673,55 @@ describe('fixed prompt controller', () => { }); }); - test('stops immediately and leaves provider billing failures unscored', async () => { - await withDir(async (dir) => { - const systemPromptPath = join(dir, 'system_prompt.md'); - await writeFile(systemPromptPath, 'fixed prompt\n', 'utf8'); - const calls: string[] = []; - - const result = await runFixedPromptController({ - runId: 'run-1', - roundId: 'round-1', - config, - systemPromptPath, - resultsJsonlPath: join(dir, 'results.jsonl'), - resultsTsvPath: join(dir, 'results.tsv'), - tasks: [ - { id: 'task-a', path: '/bench/task-a' }, - { id: 'task-b', path: '/bench/task-b' }, - ], - maxConcurrency: 1, - taskRunner: async ({ task }) => { - calls.push(task.id); - return harborOutput({ - taskId: task.id, - reward: 0, - status: 'failed', - errorClass: 'provider_billing', - omitTokenSummary: true, - steps: 0, - verifier: { - outcome: 'failed', - attempts: [{ attempt: 1, classification: 'failed', durationMs: 20, reward: 0 }], - }, - }); - }, - now: () => 100, - newId: idFactory(), - }); - - assert.deepEqual(calls, ['task-a']); - assert.equal(String(result.stopReason), 'systemic_provider_failure'); - assert.equal(result.events[0]?.type, 'task_infra_failed'); - assert.equal(String(result.events[0]?.errorClass), 'provider_billing'); - assert.equal(result.events[0]?.scored, false); - }); - }); + for (const { label, errorClass, withTsv } of [ + { label: 'provider billing failures', errorClass: 'provider_billing', withTsv: true }, + { label: 'a pre-execution authentication failure', errorClass: 'auth', withTsv: false }, + ]) { + test(`stops immediately and leaves ${label} unscored`, async () => { + await withDir(async (dir) => { + const systemPromptPath = join(dir, 'system_prompt.md'); + await writeFile(systemPromptPath, 'fixed prompt\n', 'utf8'); + const calls: string[] = []; - test('stops immediately on a pre-execution authentication failure', async () => { - await withDir(async (dir) => { - const systemPromptPath = join(dir, 'system_prompt.md'); - await writeFile(systemPromptPath, 'fixed prompt\n', 'utf8'); - const calls: string[] = []; + const result = await runFixedPromptController({ + runId: 'run-1', + roundId: 'round-1', + config, + systemPromptPath, + resultsJsonlPath: join(dir, 'results.jsonl'), + ...(withTsv ? { resultsTsvPath: join(dir, 'results.tsv') } : {}), + tasks: [ + { id: 'task-a', path: '/bench/task-a' }, + { id: 'task-b', path: '/bench/task-b' }, + ], + maxConcurrency: 1, + taskRunner: async ({ task }) => { + calls.push(task.id); + return harborOutput({ + taskId: task.id, + reward: 0, + status: 'failed', + errorClass, + omitTokenSummary: true, + steps: 0, + verifier: { + outcome: 'failed', + attempts: [{ attempt: 1, classification: 'failed', durationMs: 20, reward: 0 }], + }, + }); + }, + now: () => 100, + newId: idFactory(), + }); - const result = await runFixedPromptController({ - runId: 'run-1', - roundId: 'round-1', - config, - systemPromptPath, - resultsJsonlPath: join(dir, 'results.jsonl'), - tasks: [ - { id: 'task-a', path: '/bench/task-a' }, - { id: 'task-b', path: '/bench/task-b' }, - ], - maxConcurrency: 1, - taskRunner: async ({ task }) => { - calls.push(task.id); - return harborOutput({ - taskId: task.id, - reward: 0, - status: 'failed', - errorClass: 'auth', - omitTokenSummary: true, - steps: 0, - verifier: { - outcome: 'failed', - attempts: [{ attempt: 1, classification: 'failed', durationMs: 20, reward: 0 }], - }, - }); - }, - now: () => 100, - newId: idFactory(), + assert.deepEqual(calls, ['task-a']); + assert.equal(String(result.stopReason), 'systemic_provider_failure'); + assert.equal(result.events[0]?.type, 'task_infra_failed'); + assert.equal(String(result.events[0]?.errorClass), errorClass); + assert.equal(result.events[0]?.scored, false); }); - - assert.deepEqual(calls, ['task-a']); - assert.equal(result.stopReason, 'systemic_provider_failure'); - assert.equal(result.events[0]?.type, 'task_infra_failed'); - assert.equal(result.events[0]?.errorClass, 'auth'); - assert.equal(result.events[0]?.scored, false); }); - }); - + } test('stops when cost exceeds the configured ceiling', async () => { await withDir(async (dir) => { const systemPromptPath = join(dir, 'system_prompt.md'); @@ -2267,111 +2179,42 @@ describe('fixed prompt controller', () => { }); }); - test('counts a verifier-graded max-token stop as a scored benchmark failure', async () => { - await withDir(async (dir) => { - const systemPromptPath = join(dir, 'system_prompt.md'); - await writeFile(systemPromptPath, 'fixed prompt\n', 'utf8'); - - const result = await runFixedPromptController({ - runId: 'run-1', - roundId: 'round-1', - config, - systemPromptPath, - resultsJsonlPath: join(dir, 'results.jsonl'), - tasks: [{ id: 'task-a', path: '/bench/task-a' }], - taskRunner: async () => - harborOutput({ - taskId: 'task-a', - reward: 0, - status: 'failed', - errorClass: 'max_tokens', - verifier: { - outcome: 'failed', - attempts: [{ attempt: 1, classification: 'failed', durationMs: 20, reward: 0 }], - }, - }), - now: () => 100, - newId: idFactory(), - }); - - assert.equal(result.events[0]?.type, 'task_completed'); - assert.equal(result.events[0]?.passed, false); - assert.equal(result.events[0]?.scored, true); - assert.equal(result.events[0]?.eligible, true); - assert.equal(result.events[0]?.errorClass, 'max_tokens'); - }); - }); - - test('counts a verifier-graded tool-step cap as a scored benchmark failure', async () => { - await withDir(async (dir) => { - const systemPromptPath = join(dir, 'system_prompt.md'); - await writeFile(systemPromptPath, 'fixed prompt\n', 'utf8'); - - const result = await runFixedPromptController({ - runId: 'run-1', - roundId: 'round-1', - config, - systemPromptPath, - resultsJsonlPath: join(dir, 'results.jsonl'), - tasks: [{ id: 'task-a', path: '/bench/task-a' }], - taskRunner: async () => - harborOutput({ - taskId: 'task-a', - reward: 0, - status: 'failed', - errorClass: 'tool_step_cap_reached', - verifier: { - outcome: 'failed', - attempts: [{ attempt: 1, classification: 'failed', durationMs: 20, reward: 0 }], - }, - }), - now: () => 100, - newId: idFactory(), - }); - - assert.equal(result.events[0]?.type, 'task_completed'); - assert.equal(result.events[0]?.passed, false); - assert.equal(result.events[0]?.scored, true); - assert.equal(result.events[0]?.eligible, true); - assert.equal(result.events[0]?.errorClass, 'tool_step_cap_reached'); - }); - }); + for (const errorClass of ['max_tokens', 'tool_step_cap_reached', 'policy_denied']) { + test(`counts a verifier-graded ${errorClass} stop as a scored benchmark failure`, async () => { + await withDir(async (dir) => { + const systemPromptPath = join(dir, 'system_prompt.md'); + await writeFile(systemPromptPath, 'fixed prompt\n', 'utf8'); - test('counts a verifier-graded provider policy denial as a scored benchmark failure', async () => { - await withDir(async (dir) => { - const systemPromptPath = join(dir, 'system_prompt.md'); - await writeFile(systemPromptPath, 'fixed prompt\n', 'utf8'); + const result = await runFixedPromptController({ + runId: 'run-1', + roundId: 'round-1', + config, + systemPromptPath, + resultsJsonlPath: join(dir, 'results.jsonl'), + tasks: [{ id: 'task-a', path: '/bench/task-a' }], + taskRunner: async () => + harborOutput({ + taskId: 'task-a', + reward: 0, + status: 'failed', + errorClass, + verifier: { + outcome: 'failed', + attempts: [{ attempt: 1, classification: 'failed', durationMs: 20, reward: 0 }], + }, + }), + now: () => 100, + newId: idFactory(), + }); - const result = await runFixedPromptController({ - runId: 'run-1', - roundId: 'round-1', - config, - systemPromptPath, - resultsJsonlPath: join(dir, 'results.jsonl'), - tasks: [{ id: 'task-a', path: '/bench/task-a' }], - taskRunner: async () => - harborOutput({ - taskId: 'task-a', - reward: 0, - status: 'failed', - errorClass: 'policy_denied', - verifier: { - outcome: 'failed', - attempts: [{ attempt: 1, classification: 'failed', durationMs: 20, reward: 0 }], - }, - }), - now: () => 100, - newId: idFactory(), + assert.equal(result.events[0]?.type, 'task_completed'); + assert.equal(result.events[0]?.passed, false); + assert.equal(result.events[0]?.scored, true); + assert.equal(result.events[0]?.eligible, true); + assert.equal(result.events[0]?.errorClass, errorClass); }); - - assert.equal(result.events[0]?.type, 'task_completed'); - assert.equal(result.events[0]?.passed, false); - assert.equal(result.events[0]?.scored, true); - assert.equal(result.events[0]?.eligible, true); - assert.equal(result.events[0]?.errorClass, 'policy_denied'); }); - }); - + } test('keeps verifier-graded deadlines scored after completed model and workspace steps', async () => { await withDir(async (dir) => { const systemPromptPath = join(dir, 'system_prompt.md'); diff --git a/packages/headless/src/__tests__/harbor-cell.test.ts b/packages/headless/src/__tests__/harbor-cell.test.ts index 60ed716574..657239d2ac 100644 --- a/packages/headless/src/__tests__/harbor-cell.test.ts +++ b/packages/headless/src/__tests__/harbor-cell.test.ts @@ -1976,53 +1976,7 @@ describe('runHarborCell', () => { assert.equal(hostCellExitCode({ settledByDeadline: false }), 0); }); - test('host-side Harbor cell config treats MAKA_ECONOMY_TASK_MODE=false as explicit disable', async () => { - const { main } = (await import( - new URL('../../harbor/run-host-cell.mjs', import.meta.url).href - )) as { - main: (options?: { - registerBackends?: (registry: BackendRegistry, context: HeadlessBackendContext) => void; - }) => Promise; - }; - await withDirs(async ({ workspaceDir, outputDir, storageRoot }) => { - const previousEnv = { ...process.env }; - const seenPrompts: string[] = []; - const registerCapturingBackend = ( - registry: BackendRegistry, - context: HeadlessBackendContext, - ): void => { - seenPrompts.push(context.config.systemPrompt ?? ''); - registry.register( - 'ai-sdk', - (ctx) => - new CellReportingBackend({ - sessionId: ctx.sessionId, - header: ctx.header, - store: ctx.store, - }), - ); - }; - try { - process.env.MAKA_PROVIDER = 'openai'; - process.env.MAKA_MODEL = 'openai/gpt-4o-mini'; - process.env.MAKA_HOST_API_KEY = 'test-key'; - process.env.MAKA_HARBOR_TOOL_EXECUTOR_URL = 'http://127.0.0.1:1'; - process.env.MAKA_HARBOR_TOOL_EXECUTOR_TOKEN = 'token'; - process.env.MAKA_INSTRUCTION = 'Write a CSV summary of log files.'; - process.env.MAKA_WORKDIR = workspaceDir; - process.env.MAKA_OUTPUT_DIR = outputDir; - process.env.MAKA_STORAGE_ROOT = storageRoot; - process.env.MAKA_SYSTEM_PROMPT = 'Use the host prompt.'; - process.env.MAKA_ECONOMY_TASK_MODE = 'false'; - await main({ registerBackends: registerCapturingBackend }); - } finally { - process.env = previousEnv; - } - - assert.match(seenPrompts[0] ?? '', /Use the host prompt/); - assert.doesNotMatch(seenPrompts[0] ?? '', /Economy-task benchmark policy/); - }); - }); + ; test('Harbor ai-sdk backend registration forwards the canonical metering sink', async () => { // The controller has always exposed `recordModelCallAttempt`; this diff --git a/packages/headless/src/__tests__/harness-ab-cli.test.ts b/packages/headless/src/__tests__/harness-ab-cli.test.ts index 963d13be0e..20a2595160 100644 --- a/packages/headless/src/__tests__/harness-ab-cli.test.ts +++ b/packages/headless/src/__tests__/harness-ab-cli.test.ts @@ -10,27 +10,7 @@ import { buildHarborJobConfig } from '../harbor-task-runner.js'; const execFileAsync = promisify(execFile); -test('harness A/B CLI accepts a 5-task operational canary', async () => { - const dir = await mkdtemp(join(tmpdir(), 'maka-harness-ab-cli-')); - try { - const scriptPath = new URL('../../harbor/run-harness-ab.mjs', import.meta.url); - await assert.rejects( - execFileAsync(process.execPath, [scriptPath.pathname], { - cwd: process.cwd(), - env: { - ...process.env, - MAKA_HARNESS_AB_OUT_DIR: join(dir, 'out'), - MAKA_HARNESS_AB_TASKS_ROOT: join(dir, 'missing-tasks'), - MAKA_HARNESS_AB_LIMIT: '5', - MAKA_HARNESS_AB_DRY_RUN: '1', - }, - }), - /Terminal-Bench 2\.1 task set mismatch/, - ); - } finally { - await rm(dir, { recursive: true, force: true }); - } -}); +; test('harness A/B CLI rejects an unsupported composition before creating a run root', async () => { const dir = await mkdtemp(join(tmpdir(), 'maka-harness-ab-composition-')); diff --git a/packages/headless/src/__tests__/prompt-candidate-loop.test.ts b/packages/headless/src/__tests__/prompt-candidate-loop.test.ts index 93322a484d..4540b782be 100644 --- a/packages/headless/src/__tests__/prompt-candidate-loop.test.ts +++ b/packages/headless/src/__tests__/prompt-candidate-loop.test.ts @@ -669,39 +669,7 @@ describe('prompt candidate loop', () => { }); }); - test('requires an agent cwd before exposing controller artifacts', async () => { - await withDir(async (dir) => { - const programPath = join(dir, 'program.md'); - const systemPromptPath = join(dir, 'system_prompt.md'); - const resultsTsvPath = join(dir, 'results.tsv'); - await writeFile(programPath, 'Improve the prompt conservatively.\n', 'utf8'); - await writeFile(systemPromptPath, 'original prompt\n', 'utf8'); - await writeFile(resultsTsvPath, 'task_id\tpassed\ntask-a\tfalse\n', 'utf8'); - - let called = false; - await assert.rejects( - runPromptCandidateRound({ - runId: 'run-1', - roundId: 'round-1', - programPath, - systemPromptPath, - resultsTsvPath, - resultsJsonlPath: join(dir, 'results.jsonl'), - heldInTaskIds: ['task-a'], - heldInDigests: [{ taskId: 'task-a', summary: 'failed held-in task' }], - metaAgent: async () => { - called = true; - return candidatePromptResult(); - }, - git: gitNoop(dir), - } as unknown as Parameters[0]), - /agentCwdPath is required before exposing controller artifacts/, - ); - - assert.equal(called, false); - assert.equal(await readFile(systemPromptPath, 'utf8'), 'original prompt\n'); - }); - }); + ; test('requires an agent cwd when held-out artifact paths are provided', async () => { await withDir(async (dir) => { @@ -880,46 +848,7 @@ describe('prompt candidate loop', () => { }); }); - test('rejects agent-cwd symlinks to controller artifacts', async () => { - await withDir(async (dir) => { - const agentDir = join(dir, 'agent-cwd'); - const controllerDir = join(dir, 'controller'); - await mkdir(agentDir, { recursive: true }); - await mkdir(controllerDir, { recursive: true }); - const programPath = join(agentDir, 'program.md'); - const systemPromptPath = join(agentDir, 'system_prompt.md'); - const resultsTsvPath = join(controllerDir, 'results.tsv'); - const resultsJsonlPath = join(controllerDir, 'results.jsonl'); - const visibleResultsLinkPath = join(agentDir, 'results-link.jsonl'); - await writeFile(programPath, 'Improve the prompt conservatively.\n', 'utf8'); - await writeFile(systemPromptPath, 'original prompt\n', 'utf8'); - await writeFile(resultsTsvPath, 'task_id\tpassed\ntask-a\tfalse\n', 'utf8'); - await writeFile(resultsJsonlPath, '', 'utf8'); - await symlink(resultsJsonlPath, visibleResultsLinkPath); - - let metaAgentCalled = false; - await assert.rejects( - runPromptCandidateRound({ - runId: 'run-1', - roundId: 'round-1', - agentCwdPath: agentDir, - programPath, - systemPromptPath, - resultsTsvPath, - resultsJsonlPath, - heldInTaskIds: ['task-a'], - heldInDigests: [{ taskId: 'task-a', summary: 'failed held-in task' }], - metaAgent: async () => { - metaAgentCalled = true; - return candidatePromptResult(); - }, - git: gitNoop(agentDir), - }), - /controller-only artifacts must stay outside agent cwd: results-link\.jsonl/, - ); - assert.equal(metaAgentCalled, false); - }); - }); + ; test('rejects agent-cwd directory symlinks that contain controller artifacts', async () => { await withDir(async (dir) => { @@ -1930,91 +1859,9 @@ describe('prompt candidate loop', () => { }); }); - test('CLI git adapter rejects deletion of pre-existing dirty files during a candidate round', async () => { - await withDir(async (dir) => { - await execFileAsync('git', ['init'], { cwd: dir }); - await execFileAsync('git', ['config', 'user.email', 'test@example.com'], { cwd: dir }); - await execFileAsync('git', ['config', 'user.name', 'Test User'], { cwd: dir }); - const programPath = join(dir, 'program.md'); - const systemPromptPath = join(dir, 'system_prompt.md'); - const resultsTsvPath = join(dir, 'results.tsv'); - const scratchPath = join(dir, 'scratch.tmp'); - await writeFile(programPath, 'Improve the prompt conservatively.\n', 'utf8'); - await writeFile(systemPromptPath, 'original prompt\n', 'utf8'); - await writeFile(resultsTsvPath, 'task_id\tpassed\ntask-a\tfalse\n', 'utf8'); - await execFileAsync('git', ['add', '.'], { cwd: dir }); - await execFileAsync('git', ['commit', '-m', 'initial'], { cwd: dir }); - await writeFile(scratchPath, 'pre-existing scratch\n', 'utf8'); - - await assert.rejects( - runPromptCandidateRound({ - runId: 'run-1', - roundId: 'round-1', - agentCwdPath: await testAgentCwd(dir), - programPath, - systemPromptPath, - resultsTsvPath, - resultsJsonlPath: join(dir, 'results.jsonl'), - heldInTaskIds: [], - heldInDigests: [], - metaAgent: async () => { - await rm(scratchPath); - return candidatePromptResult(); - }, - git: createCliPromptCandidateGit({ cwd: dir, systemPromptPath }), - now: () => 100, - newId: idFactory(), - }), - /only system_prompt.md may change/, - ); - - const subject = await execFileAsync('git', ['log', '-1', '--format=%s'], { cwd: dir }); - assert.equal(subject.stdout.trim(), 'initial'); - assert.equal(await readFile(systemPromptPath, 'utf8'), 'original prompt\n'); - }); - }); - - test('CLI git adapter rejects non-prompt edits made during a candidate round', async () => { - await withDir(async (dir) => { - await execFileAsync('git', ['init'], { cwd: dir }); - await execFileAsync('git', ['config', 'user.email', 'test@example.com'], { cwd: dir }); - await execFileAsync('git', ['config', 'user.name', 'Test User'], { cwd: dir }); - const programPath = join(dir, 'program.md'); - const systemPromptPath = join(dir, 'system_prompt.md'); - const resultsTsvPath = join(dir, 'results.tsv'); - await writeFile(programPath, 'Improve the prompt conservatively.\n', 'utf8'); - await writeFile(systemPromptPath, 'original prompt\n', 'utf8'); - await writeFile(resultsTsvPath, 'task_id\tpassed\ntask-a\tfalse\n', 'utf8'); - await execFileAsync('git', ['add', '.'], { cwd: dir }); - await execFileAsync('git', ['commit', '-m', 'initial'], { cwd: dir }); - - await assert.rejects( - runPromptCandidateRound({ - runId: 'run-1', - roundId: 'round-1', - agentCwdPath: await testAgentCwd(dir), - programPath, - systemPromptPath, - resultsTsvPath, - resultsJsonlPath: join(dir, 'results.jsonl'), - heldInTaskIds: [], - heldInDigests: [], - metaAgent: async () => { - await writeFile(programPath, 'tampered program\n', 'utf8'); - return candidatePromptResult(); - }, - git: createCliPromptCandidateGit({ cwd: dir, systemPromptPath }), - now: () => 100, - newId: idFactory(), - }), - /only system_prompt.md may change/, - ); + ; - const subject = await execFileAsync('git', ['log', '-1', '--format=%s'], { cwd: dir }); - assert.equal(subject.stdout.trim(), 'initial'); - assert.equal(await readFile(systemPromptPath, 'utf8'), 'original prompt\n'); - }); - }); + ; test('CLI git adapter rejects HEAD movement during a candidate round', async () => { await withDir(async (dir) => { diff --git a/packages/headless/src/__tests__/prompt-optimization-loop-replay-decision.test.ts b/packages/headless/src/__tests__/prompt-optimization-loop-replay-decision.test.ts index afc264b3bc..00e0b9e23f 100644 --- a/packages/headless/src/__tests__/prompt-optimization-loop-replay-decision.test.ts +++ b/packages/headless/src/__tests__/prompt-optimization-loop-replay-decision.test.ts @@ -137,57 +137,7 @@ describe('runPromptOptimizationLoop replay decision guards', () => { }); }); - test('fails closed when a kept decision is missing held-out task evidence', async () => { - await withHarness(async (harness) => { - const heldInTasks = makeTasks('hin', 20); - const heldOutTasks = makeTasks('hout', 8); - const rewardFor = (roundId: string, taskId: string): number => { - const index = taskIndex(taskId); - if (taskId.startsWith('hout-')) return index < 4 ? 1 : 0; - if (roundId.startsWith('baseline-')) return index < 10 ? 1 : 0; - return taskId.startsWith('hin-') ? 1 : index < 4 ? 1 : 0; - }; - - await runLoop(harness, { - heldInTasks, - heldOutTasks, - rewardFor, - rounds: 1, - baselineRuns: 1, - }); - const events = await readFixedPromptWal(harness.resultsJsonlPath); - const missingHeldOut = events.filter( - (event) => - !( - event.type === 'task_completed' && - event.roundId === 'round-0' && - event.taskId.startsWith('hout-') - ), - ); - await writeFile( - harness.resultsJsonlPath, - `${missingHeldOut.map((event) => JSON.stringify(event)).join('\n')}\n`, - 'utf8', - ); - - let nextRoundPrompted = false; - await assert.rejects( - runLoop(harness, { - heldInTasks, - heldOutTasks, - rewardFor, - rounds: 2, - baselineRuns: 1, - metaAgent: async (promptInput) => { - if (promptInput.roundId === 'round-1') nextRoundPrompted = true; - return fakeMetaAgent()(promptInput); - }, - }), - /RSI WAL replay missing required held-out task evidence for round-0/, - ); - assert.equal(nextRoundPrompted, false); - }); - }); + ; test('fails closed when task evidence appears after its decision', async () => { await withHarness(async (harness) => { diff --git a/packages/headless/src/__tests__/prompt-optimization-loop-replay-identity.test.ts b/packages/headless/src/__tests__/prompt-optimization-loop-replay-identity.test.ts index 879e0bef08..5b3bee54c2 100644 --- a/packages/headless/src/__tests__/prompt-optimization-loop-replay-identity.test.ts +++ b/packages/headless/src/__tests__/prompt-optimization-loop-replay-identity.test.ts @@ -46,48 +46,7 @@ describe('runPromptOptimizationLoop replay identity guards', () => { }); }); - test('fails closed when replayed candidate task evidence has a stale prompt hash', async () => { - await withHarness(async (harness) => { - const heldInTasks = makeTasks('hin', 20); - const heldOutTasks = makeTasks('hout', 8); - const rewardFor = (roundId: string, taskId: string): number => { - const index = taskIndex(taskId); - if (taskId.startsWith('hout-')) return index < 4 ? 1 : 0; - if (roundId.startsWith('baseline-')) return index < 10 ? 1 : 0; - return 1; - }; - - await runLoop(harness, { - heldInTasks, - heldOutTasks, - rewardFor, - rounds: 1, - baselineRuns: 1, - }); - const events = await readFixedPromptWal(harness.resultsJsonlPath); - const staleEvents = events.map((event) => - event.type === 'task_completed' && event.roundId === 'round-0' && event.taskId === 'hin-0' - ? { ...event, promptHash: 'sha256:stale' } - : event, - ); - await writeFile( - harness.resultsJsonlPath, - `${staleEvents.map((event) => JSON.stringify(event)).join('\n')}\n`, - 'utf8', - ); - - await assert.rejects( - runLoop(harness, { - heldInTasks, - heldOutTasks, - rewardFor, - rounds: 2, - baselineRuns: 1, - }), - /RSI WAL replay prompt hash mismatch/, - ); - }); - }); + ; test('fails closed when replayed task evidence has no prompt hash', async () => { await withHarness(async (harness) => {