Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 53 additions & 0 deletions apps/desktop/src/lib/markdown-blocks.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -108,4 +108,57 @@ describe('parseMarkdownIntoBlocksCached', () => {

expect(parseMarkdownIntoBlocksCached(rewritten)).toEqual(parseMarkdownIntoBlocks(rewritten))
})

it('matches a full lex when a trailing setext underline merges the previous block (regression)', () => {
// A trailing `-`/`=` line is a setext underline of the block ABOVE it, so
// appending to it can retroactively merge the previous parse's LAST TWO
// blocks into one. Cached `"…#e\n5\n-"` lexes to [ …, "#e\n", "5\n-" ], but
// grown to `"…#e\n5\n-p2=kj:c"` collapses `#e`/`5\n-` into one block. The
// old boundary dropped only the single last content block, so it reused a
// `"#e\n"` block that no longer exists. `blocks.join('') === text` still
// holds for the wrong split, so the reconstruction guard cannot catch it.
// The settled prefix pushes the text past the append-cache threshold so
// the incremental path actually engages.
const settled = 'settled line paragraph text.\n\n'.repeat(80)
const prev = `${settled}#e\n5\n-`
const grown = `${prev}p2=kj:c`

// Seed the append cache with `prev`, then grow it — the exact two-call
// sequence a streaming flush produces.
parseMarkdownIntoBlocksCached(prev)

expect(parseMarkdownIntoBlocksCached(grown)).toEqual(parseMarkdownIntoBlocks(grown))
})

// 12 seeds × 500 growing prefixes is ~6000 full+cached lexes; it first trips
// the pre-fix boundary at seed 11 / step 257, so the workload can't shrink
// without gutting the guard. The work is bounded but exceeds one test's 5s
// default budget, so raise the timeout rather than weaken the coverage.
it('matches a full lex at every char-level streaming cut over noisy markdown (property fuzz)', () => {
// Character-level append fuzz over the markdown control alphabet — the
// harness that surfaced the setext-underline merge above. Growing a single
// lineage one small chunk at a time keeps `startsWith` lineage intact so
// the incremental path runs on nearly every step; each prefix must
// deep-equal a fresh full lex.
const alphabet = '\n `#*-_>[]()|~:=abcdefghijklmnopqrstuvwxyz0123456789'

for (let seed = 1; seed <= 12; seed++) {
const rand = mulberry32(seed)
// Seed past the append-cache threshold so the incremental path engages.
let text = `seed ${seed}\n\n`.repeat(180)

for (let step = 0; step < 500; step++) {
const n = 1 + Math.floor(rand() * 24)
let chunk = ''

for (let j = 0; j < n; j++) {
chunk += alphabet[Math.floor(rand() * alphabet.length)]
}

text += chunk

expect(parseMarkdownIntoBlocksCached(text)).toEqual(parseMarkdownIntoBlocks(text))
}
}
}, 30_000)
})
25 changes: 18 additions & 7 deletions apps/desktop/src/lib/markdown-blocks.ts
Original file line number Diff line number Diff line change
Expand Up @@ -65,16 +65,27 @@ function lexIncrementally(text: string): null | string[] {
return null
}

// Settled boundary: drop trailing whitespace-only blocks, then the last
// content block (the only one appended text can reinterpret).
// Settled boundary: drop the last TWO content blocks (skipping any
// whitespace-only blocks around them). Dropping only the single last content
// block is unsound: appended text can retroactively merge the previous
// parse's last two blocks into one. The trigger is a trailing Setext
// underline — `marked` only treats `-`/`=` as an underline for the paragraph
// ABOVE it, so a settled `"#e\n5\n-"` lexes as ["#e\n", "5\n-"], but growing
// the tail to `"#e\n5\n-p2=kj:c"` collapses both into one paragraph. The
// block before the last is the deepest an append can reach (the underline
// consumes exactly one preceding block), so re-lexing the last two is safe;
// earlier blocks are fenced off by settled blank lines. join('') === text
// still holds either way, so the reconstruction check below can't catch this.
let keep = entry.blocks.length

while (keep > 0 && !entry.blocks[keep - 1].trim()) {
keep -= 1
}
for (let dropped = 0; dropped < 2 && keep > 0; dropped += 1) {
while (keep > 0 && !entry.blocks[keep - 1].trim()) {
keep -= 1
}

if (keep > 0) {
keep -= 1
if (keep > 0) {
keep -= 1
}
}

if (keep === 0) {
Expand Down
Loading