diff --git a/apps/web/package.json b/apps/web/package.json index c2ce73b42b11..069a222ca122 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -59,6 +59,7 @@ "@types/babel__core": "^7.20.5", "@types/compression": "^1.8.1", "@types/culori": "^4.0.1", + "@types/mdast": "^4.0.4", "@types/react": "~19.2.14", "@types/react-dom": "~19.2.3", "@types/react-test-renderer": "19.1.0", @@ -68,6 +69,7 @@ "compression": "^1.8.1", "react-test-renderer": "19.2.6", "tailwindcss": "^4.0.0", + "unified": "^11.0.5", "vite": "catalog:", "vite-plus": "catalog:" } diff --git a/apps/web/src/components/ChatMarkdown.tsx b/apps/web/src/components/ChatMarkdown.tsx index 192abaf385f7..88d34b8d58bc 100644 --- a/apps/web/src/components/ChatMarkdown.tsx +++ b/apps/web/src/components/ChatMarkdown.tsx @@ -69,6 +69,7 @@ import React, { } from "react"; import type { Components, Options as ReactMarkdownOptions } from "react-markdown"; import ReactMarkdown from "react-markdown"; +import { createIncrementalMarkdownPlugin } from "../markdown-incremental"; import { defaultUrlTransform } from "react-markdown"; import rehypeRaw from "rehype-raw"; import rehypeSanitize, { defaultSchema } from "rehype-sanitize"; @@ -3104,12 +3105,17 @@ function ChatMarkdown({ localMediaPreview, setLocalMediaPreview, } = useChatMarkdownState({ text, ...props }); + const incrementalParsing = + props.isStreaming === true && + extraRemarkPlugins.length === 0 && + /(?:^|\n) {0,3}(?:`{3}|~{3})/.test(text); const remarkPlugins = useMemo( () => [ ...(lineBreaks ? CHAT_MARKDOWN_REMARK_PLUGINS_WITH_BREAKS : CHAT_MARKDOWN_REMARK_PLUGINS), ...extraRemarkPlugins, + ...(incrementalParsing ? [createIncrementalMarkdownPlugin()] : []), ], - [extraRemarkPlugins, lineBreaks], + [extraRemarkPlugins, incrementalParsing, lineBreaks], ); // react-markdown converts unparsed HTML nodes to text when skipHtml is false. diff --git a/apps/web/src/markdown-incremental.test.tsx b/apps/web/src/markdown-incremental.test.tsx new file mode 100644 index 000000000000..cf06355f3b49 --- /dev/null +++ b/apps/web/src/markdown-incremental.test.tsx @@ -0,0 +1,136 @@ +import type { Root } from "mdast"; +import { renderToStaticMarkup } from "react-dom/server"; +import ReactMarkdown from "react-markdown"; +import rehypeRaw from "rehype-raw"; +import rehypeSanitize from "rehype-sanitize"; +import remarkGfm from "remark-gfm"; +import type { Plugin } from "unified"; +import { describe, expect, it } from "vite-plus/test"; + +import { remarkCodexDirectives } from "@t3tools/client-runtime/codex-markdown-directives"; +import { remarkGithubAlerts } from "./markdown-github-alerts"; +import { createIncrementalMarkdownPlugin } from "./markdown-incremental"; +import { remarkNormalizeListItemIndentation } from "./markdown-list-indentation"; + +function render(source: string, incremental?: Plugin<[], Root>, parsedSources?: string[]) { + let tree: Root | undefined; + const observeParsing: Plugin<[], Root> = function () { + const original = this.parser; + if (original) { + this.parser = (text, file) => { + parsedSources?.push(text); + return original(text, file); + }; + } + }; + const capture: Plugin<[], Root> = () => (root) => { + tree = structuredClone(root); + }; + const html = renderToStaticMarkup( + + {source} + , + ); + return { html, tree }; +} + +const prefix = "# Before\n\n```ts\nconst values = [1, 2];\n```\n\n"; + +describe("incremental Markdown parsing", () => { + it("keeps the document prefix cached when list recovery parses contain fences", () => { + const source = + prefix + + "- first block\n\n ```ts\n const nested = 1;\n ```\n\n tail"; + const incremental = createIncrementalMarkdownPlugin(); + const parsedSources: string[] = []; + expect(render(source, incremental, parsedSources)).toEqual(render(source)); + parsedSources.length = 0; + const next = source + " more"; + expect(render(next, incremental, parsedSources)).toEqual(render(next)); + expect(parsedSources).not.toContain(next); + expect(parsedSources.some((text) => text.startsWith("t3-markdown-inline-prefix:"))).toBe(true); + }); + + it.each([ + "a\n===\n\nb\n---\n", + "- first\n\n continued\n\n- next\n", + "> quoted\n>\n> ```js\n> abc\n> ```\n\nend", + "
\nhello\n\n
\n\nend", + "[ref]\n\n[ref]: /later", + "a[^x]\n\n[^x]: note", + "a | b\n--|--\na | b\n", + "```\na\n```\n\nnext\n\n~~~\nb\n~~~\n\nmore", + "\n\n\tcode\n\nmore", + "text *bold*", + "> [!NOTE]\n> alert\n\n- [ ] task", + "\uFEFFtext after a byte-order mark", + ])("preserves the parse tree, positions, and HTML while streaming %j", (tail) => { + const source = prefix + tail; + const incremental = createIncrementalMarkdownPlugin(); + for (let end = 0; end <= source.length; end++) { + const text = source.slice(0, end); + expect(render(text, incremental), `prefix ${end}`).toEqual(render(text)); + } + }); + + it.each(["\r\n", "\r"])("preserves partial %j line endings", (newline) => { + const source = (prefix + "next\n\n```\nlast\n```\n\nend").replaceAll("\n", newline); + const incremental = createIncrementalMarkdownPlugin(); + for (let end = 0; end <= source.length; end++) { + const text = source.slice(0, end); + expect(render(text, incremental)).toEqual(render(text)); + } + }); + + it("updates earlier references when definitions arrive after the cached prefix", () => { + const before = "[later] and footnote[^note]\n\n" + prefix; + const incremental = createIncrementalMarkdownPlugin(); + for (const tail of ["text", "[later]: /target", "[later]: /target\n\n[^note]: a note"]) { + expect(render(before + tail, incremental)).toEqual(render(before + tail)); + } + }); + + it("handles edits, replacements, and repeated renders without leaking transformed nodes", () => { + const incremental = createIncrementalMarkdownPlugin(); + const documents = [ + prefix + "- first\n - second", + prefix + "> [!NOTE]\n> transformed alert", + prefix + "plain text", + "replacement without fences", + prefix.replace("Before", "Edited") + "edited prefix", + prefix + "plain text", + prefix + "plain text", + ]; + for (const document of documents) { + expect(render(document, incremental)).toEqual(render(document)); + } + }); + + it("does not freeze unclosed, nested, indented, or mismatched fences", () => { + const prefixes = [ + "```\nopen\n\n", + "````\n```\n\n", + "> ```\n> code\n> ```\n\n", + "- ```\n code\n ```\n\n", + " ```\n code\n ```\n\n", + "