`, `| `), mis-nested inline formatting, and table sections are recovered the way a browser recovers them. The conversion never throws on bad HTML. Only `` content is converted. ` --> after ",
+ "\\-->\n\nafter",
+ ],
+ ["script comment without close", "ok ", "ok"],
+ [
+ "plaintext swallows everything",
+ "a <![CDATA[<b>&</b></plaintext><p>b",
+ "a\n\n\\<b>&\\</b>\\</plaintext>\\<p>b",
+ ],
+ [
+ "CDATA only in foreign content",
+ "<p><![CDATA[x<y]]]]><![CDATA[>a</p><p><svg><![CDATA[x<y]]]]><![CDATA[></svg>b</p><p><math><mrow><![CDATA[z]]]]><![CDATA[></mrow></math></p>",
+ "a\n\nx\\<yb\n\nz",
+ ],
+ [
+ "doctype decides quirks: table inside p",
+ "<!DOCTYPE html><p>a<table><tr><td>b</td><td>c</td></tr></table>",
+ "a\n\n| b | c |\n| --- | --- |",
+ ],
+ [
+ "quirks mode keeps table in p",
+ "<p>a<table><tr><td>b</td><td>c</td></tr></table>",
+ "a\n\n| b | c |\n| --- | --- |",
+ ],
+ [
+ "legacy doctype public id",
+ '<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd"><p>x</p>',
+ "x",
+ ],
+ ["BOM at start is dropped, elsewhere kept", "<p>ab</p>", "ab"],
+ ["EOF inside a tag drops it", '<p>kept</p><a href="/x', "kept"],
+ ["EOF inside comment", "<p>a</p><!-- b", "a"],
+ ["end tag with attributes still closes", "<p><b>a</b x=1>b</p>", "**a**b"],
+ ["NUL bytes", "<p>a\u0000b</p><pre>c\u0000d</pre><p><b x\u0000y=1>e</b></p>", "ab\n\n```\ncd\n```\n\n**e**"],
+ ["less-than that does not start a tag", "<p>1 < 2 <3 a<=b </3 <</p>", "1 < 2 <3 a<=b"],
+ ["non-ASCII tag and attribute names", '<p><é>x</é> <a hréf="/x" href="/y">z</a></p>', "<é>x [z](/y)"],
+ ["form feed and tab as attribute separators", "<p><a\fhref='/x'\ttitle='t'>y</a></p>", '[y](/x "t")'],
+ ];
+
+ for (const [name, html, md] of cases) {
+ test(name, () => expect(fromHTML(html)).toBe(md));
+ }
+ test("escape precision: only text that starts a line is escaped", () => {
+ expect(fromHTML("<p>a<span>- b</span> <b>1. c</b> <em>#</em> d</p><p><span>- e</span></p>")).toBe(
+ "a- b **1. c** _#_ d\n\n\\- e",
+ );
+ });
+ });
+
+ describe("tables", () => {
+ test("thead/tbody with alignment", () => {
+ expect(
+ fromHTML(
+ '<table><thead><tr><th>A</th><th align="center">B</th><th style="text-align: right">C</th></tr></thead>' +
+ "<tbody><tr><td>1</td><td>2</td><td>3</td></tr></tbody></table>",
+ ),
+ ).toBe("| A | B | C |\n| --- | :-: | --: |\n| 1 | 2 | 3 |");
+ });
+ test("first row becomes the header when there is no thead/th", () => {
+ expect(fromHTML("<table><tr><td>a</td><td>b</td></tr><tr><td>1</td><td>2</td></tr></table>")).toBe(
+ "| a | b |\n| --- | --- |\n| 1 | 2 |",
+ );
+ });
+ test("single-row multi-cell table still converts", () => {
+ expect(fromHTML("<table><tr><td>a</td><td>b</td></tr></table>")).toBe("| a | b |\n| --- | --- |");
+ });
+ test("rows are padded to the widest row and colspan adds cells", () => {
+ expect(
+ fromHTML(
+ '<table><tr><th colspan="2">ab</th><th>c</th></tr><tr><td>1</td><td>2</td><td>3</td></tr><tr><td>x</td></tr></table>',
+ ),
+ ).toBe("| ab | | c |\n| --- | --- | --- |\n| 1 | 2 | 3 |\n| x | | |");
+ });
+ test("pipes in cells are escaped and block content is flattened", () => {
+ expect(
+ fromHTML("<table><tr><th>a|b</th><th>c</th></tr><tr><td><p>one</p><p>two</p></td><td>x<br>y</td></tr></table>"),
+ ).toBe("| a\\|b | c |\n| --- | --- |\n| one two | x y |");
+ });
+ test("list inside a cell", () => {
+ expect(
+ fromHTML(
+ "<table><tr><th>a</th><th>b</th></tr><tr><td><ul><li>x</li><li>y</li></ul></td><td>z</td></tr></table>",
+ ),
+ ).toBe("| a | b |\n| --- | --- |\n| - x - y | z |");
+ });
+ test("caption precedes the table", () => {
+ expect(fromHTML("<table><caption>Cap <b>bold</b></caption><tr><th>a</th><th>b</th></tr></table>")).toBe(
+ "Cap **bold**\n\n| a | b |\n| --- | --- |",
+ );
+ });
+ test("tfoot rows come last, thead rows first", () => {
+ expect(
+ fromHTML(
+ "<table><tfoot><tr><td>f1</td><td>f2</td></tr></tfoot><tbody><tr><td>b1</td><td>b2</td></tr></tbody><thead><tr><th>h1</th><th>h2</th></tr></thead></table>",
+ ),
+ ).toBe("| h1 | h2 |\n| --- | --- |\n| b1 | b2 |\n| f1 | f2 |");
+ });
+ test("single-cell table is unwrapped", () => {
+ expect(fromHTML("<table><tr><td><p>just layout</p><p>more</p></td></tr></table>")).toBe("just layout\n\nmore");
+ });
+ test("layout table wrapping another table is unwrapped, inner table converts", () => {
+ expect(
+ fromHTML(
+ "<table><tr><td><h2>Title</h2></td><td><table><tr><th>k</th><th>v</th></tr><tr><td>1</td><td>2</td></tr></table></td></tr></table>",
+ ),
+ ).toBe("## Title\n\n| k | v |\n| --- | --- |\n| 1 | 2 |");
+ });
+ test("empty table produces nothing", () => {
+ expect(fromHTML("<p>a</p><table></table><table><tr></tr></table><p>b</p>")).toBe("a\n\nb");
+ });
+ test("tables: false unwraps cells into blocks", () => {
+ expect(
+ fromHTML("<table><tr><th>a</th><th>b</th></tr><tr><td>1</td><td>2</td></tr></table>", { tables: false }),
+ ).toBe("a\n\nb\n\n1\n\n2");
+ });
+ });
+
+ describe("table width bounds", () => {
+ const repeat = (s: string, n: number) => Buffer.alloc(s.length * n, s).toString();
+ test("colspan is clamped to 64 columns", () => {
+ const md = fromHTML('<table><tr><th colspan="999">h</th></tr><tr><td>a</td><td>b</td></tr></table>');
+ expect(md.split("\n")[1]).toBe("|" + repeat(" --- |", 64));
+ });
+ test("body rows of a very wide table are not padded to its width", () => {
+ const md = fromHTML("<table><tr>" + repeat("<th>h</th>", 100) + "</tr><tr><td>x</td></tr></table>");
+ const lines = md.split("\n");
+ expect(lines[0]).toBe("|" + repeat(" h |", 100));
+ expect(lines[2]).toBe("| x |");
+ });
+ test("repeated <body> tags with attributes do not accumulate", () => {
+ expect(fromHTML(repeat("<body a=1 b=2 c=3>", 20_000) + "<p>ok</p>")).toBe("ok");
+ });
+ test("nested framesets are depth-capped like everything else", () => {
+ // `<frameset>` is ignored outright (no frameset insertion mode), so the
+ // divs and the text are ordinary body content.
+ expect(fromHTML(repeat("<frameset>", 50_000) + repeat("<div>", 50_000) + "x")).toBe("x");
+ });
+ });
+
+ describe("task lists", () => {
+ test("checkbox leading a list item", () => {
+ expect(
+ fromHTML('<ul><li><input type="checkbox" checked> done</li><li><input type="checkbox"> todo</li></ul>'),
+ ).toBe("- [x] done\n- [ ] todo");
+ });
+ test("checkbox elsewhere is dropped", () => {
+ expect(fromHTML('<p>a<input type="checkbox" checked>b <input type="text" value="v"></p>')).toBe("ab");
+ });
+ test("tasklists: false drops the checkbox", () => {
+ expect(fromHTML('<ul><li><input type="checkbox" checked> done</li></ul>', { tasklists: false })).toBe("- done");
+ });
+ });
+
+ describe("document handling", () => {
+ test("full document: only body content, head is ignored", () => {
+ expect(
+ fromHTML(
+ "<!DOCTYPE html><html><head><title>T</title><style>p{}</style><script>var x</script></head><body><p>Body</p></body></html>",
+ ),
+ ).toBe("Body");
+ });
+ test("script/style/noscript/template in body are dropped", () => {
+ expect(
+ fromHTML(
+ "<p>a</p><script>alert('*')</script><style>b{}</style><noscript><p>enable js</p></noscript><template><p>t</p></template><p>b</p>",
+ ),
+ ).toBe("a\n\nb");
+ });
+ test("dropped elements do not glue words or double spaces", () => {
+ expect(fromHTML("<p>a <script>x</script> b<script>y</script>c</p>")).toBe("a bc");
+ });
+ test("svg/math text content passes through", () => {
+ expect(fromHTML('<p>icon<svg><title>t</title><path d="M0"/></svg> and <math><mi>x</mi></math></p>')).toBe(
+ "icont and x",
+ );
+ });
+ test("iframe/video/audio/canvas produce nothing but do not swallow siblings", () => {
+ expect(fromHTML('<p>a</p><iframe src="x"></iframe><video src="v"></video><canvas></canvas><p>b</p>')).toBe(
+ "a\n\nb",
+ );
+ });
+ test("frameset document", () => {
+ expect(fromHTML("<html><frameset><frame src=a></frameset></html>")).toBe("");
+ });
+ test("empty and whitespace-only input", () => {
+ expect(fromHTML("")).toBe("");
+ expect(fromHTML(" \n\t")).toBe("");
+ expect(fromHTML("<!-- only a comment -->")).toBe("");
+ });
+ });
+
+ describe("options", () => {
+ test("headingStyle: setext", () => {
+ expect(fromHTML("<h1>One</h1><h2>Two</h2><h3>Three</h3>", { headingStyle: "setext" })).toBe(
+ "One\n===\n\nTwo\n---\n\n### Three",
+ );
+ // Underline is as wide as the (flattened) text, in characters not bytes.
+ expect(fromHTML("<h1>caf\u00e9<br>ok</h1>", { headingStyle: "setext" })).toBe("caf\u00e9 ok\n=======");
+ });
+ test("hr", () => {
+ for (const hr of ["---", "***", "___", "- - -", "* * *", "_ _ _"] as const) {
+ expect(fromHTML("<hr>", { hr })).toBe(hr);
+ }
+ });
+ test("bulletListMarker", () => {
+ expect(fromHTML("<ul><li>a</li></ul>", { bulletListMarker: "*" })).toBe("* a");
+ expect(fromHTML("<ul><li>a</li></ul>", { bulletListMarker: "+" })).toBe("+ a");
+ });
+ test("codeBlockStyle: indented", () => {
+ expect(
+ fromHTML('<pre><code class="language-js">a\n b</code></pre><p>x</p>', { codeBlockStyle: "indented" }),
+ ).toBe(" a\n b\n\nx");
+ });
+ test("fence: ~~~", () => {
+ expect(fromHTML("<pre><code>~~~\nx</code></pre>", { fence: "~~~" })).toBe("~~~~\n~~~\nx\n~~~~");
+ });
+ test("emDelimiter / strongDelimiter", () => {
+ expect(fromHTML("<p><em>a</em> <strong>b</strong></p>", { emDelimiter: "*", strongDelimiter: "__" })).toBe(
+ "*a* __b__",
+ );
+ });
+ test("br: backslash", () => {
+ expect(fromHTML("<p>a<br>b</p>", { br: "\\" })).toBe("a\\\nb");
+ });
+ test("strikethrough: false keeps the text", () => {
+ expect(fromHTML("<p><del>a</del></p>", { strikethrough: false })).toBe("a");
+ });
+ test("undefined/null options mean defaults; unknown keys are ignored", () => {
+ expect(fromHTML("<h1>a</h1>", undefined)).toBe("# a");
+ expect(fromHTML("<h1>a</h1>", null as any)).toBe("# a");
+ expect(fromHTML("<h1>a</h1>", { headingStyle: undefined, whatever: 1 } as any)).toBe("# a");
+ });
+ test("invalid enum values throw", () => {
+ expect(() => fromHTML("<h1>a</h1>", { headingStyle: "loud" } as any)).toThrow(/headingStyle/);
+ expect(() => fromHTML("<hr>", { hr: "-*-" } as any)).toThrow(/hr/);
+ expect(() => fromHTML("<br>", { br: " " } as any)).toThrow(/br/);
+ expect(() => fromHTML("<ul><li>a</li></ul>", { bulletListMarker: 1 } as any)).toThrow(/bulletListMarker/);
+ });
+ test("missing input throws", () => {
+ expect(() => (fromHTML as any)()).toThrow();
+ expect(() => fromHTML(null as any)).toThrow();
+ expect(() => fromHTML(123 as any)).toThrow();
+ });
+ });
+
+ describe("input types", () => {
+ test("Uint8Array / ArrayBuffer / Buffer", () => {
+ const bytes = new TextEncoder().encode("<p>caf\u00e9 <b>ok</b></p>");
+ expect(fromHTML(bytes)).toBe("caf\u00e9 **ok**");
+ expect(fromHTML(bytes.buffer)).toBe("caf\u00e9 **ok**");
+ expect(fromHTML(Buffer.from(bytes))).toBe("caf\u00e9 **ok**");
+ });
+ test("invalid UTF-8 bytes become U+FFFD instead of throwing", () => {
+ expect(fromHTML(new Uint8Array([0x3c, 0x70, 0x3e, 0x61, 0xff, 0x62]))).toBe("a\ufffdb");
+ });
+ test("lone surrogate in a JS string does not throw", () => {
+ expect(typeof fromHTML("<p>a\ud800b</p>")).toBe("string");
+ });
+ test("option getters run before the input buffer is read", () => {
+ // A getter that detaches the buffer must not leave the converter
+ // reading freed memory: options are evaluated first, then the (now
+ // empty) buffer is read.
+ const buf = new Uint8Array(new TextEncoder().encode("<p>hello</p>"));
+ const opts = {
+ get headingStyle() {
+ buf.buffer.transfer();
+ return "atx";
+ },
+ };
+ expect(fromHTML(buf, opts as any)).toBe("");
+ });
+ test("NUL bytes", () => {
+ expect(fromHTML("<p>a\u0000b</p>")).toBe("ab");
+ });
+ test("SharedArrayBuffer-backed input is snapshotted", () => {
+ const src = new TextEncoder().encode("<p>caf\u00e9 <b>ok</b></p>");
+ const shared = new Uint8Array(new SharedArrayBuffer(src.length));
+ shared.set(src);
+ expect(fromHTML(shared)).toBe("caf\u00e9 **ok**");
+ });
+ test("negative <ol start> clamps to 0", () => {
+ expect(fromHTML('<ol start="-2"><li>a</li><li>b</li><li>c</li><li>d</li></ol>')).toBe("0. a\n0. b\n0. c\n1. d");
+ });
+ });
+
+ // Structure recovery (the tree-construction rules layered over the
+ // HTMLRewriter tokenizer): implied end tags, scope-limited end-tag matching,
+ // table fix-up and foster parenting, reopened and adopted formatting
+ // elements, foreign content.
+ describe("tree construction", () => {
+ const cases: [string, string, string][] = [
+ ["<p> closes an open <p>", "<p>a<p>b", "a\n\nb"],
+ ["<li> closes an open <li>", "<ul><li>one<li>two</ul>", "- one\n- two"],
+ ["<dt>/<dd> close each other", "<dl><dt>t<dd>d<dt>t2</dl>", "t\n\nd\n\nt2"],
+ ["a heading closes an open heading", "<h1>a<h2>b", "# a\n\n## b"],
+ ["a block start closes <p> through phrasing content", "<p>a<span>b<div>c</div>", "ab\n\nc"],
+ ["<li> outside a list", "<p>x</p><li>y", "x\n\n- y"],
+ ["nested <button> closes the outer one", "<p>a<button>b<button>c</p>", "abc"],
+ ["<option> closes <option>", "<select><option>a<option>b</select>", "ab"],
+ ["ruby annotations close each other", "<ruby>b<rt>t<rb>b2</ruby>", "btb2"],
+ ["end tag blocked by a special element", "<div><span>a<div>b</span>c</div></div>", "a\n\nbc"],
+ [
+ "cells and rows imply <tr>/<tbody> and close each other",
+ "<table><tr><td>1<td>2<tr><td>3</table>",
+ "| 1 | 2 |\n| --- | --- |\n| 3 | |",
+ ],
+ ["<td> straight inside <table>", "<table><td>x</table>", "x"],
+ [
+ "text inside <table> is foster-parented before it",
+ "<table>oops<tr><td>a</td><td>b</td></tr></table>",
+ "oops\n\n| a | b |\n| --- | --- |",
+ ],
+ [
+ "elements inside <table> are foster-parented too",
+ "<table><p>para<tr><td>a</td><td>b</td></tr></table>",
+ "para\n\n| a | b |\n| --- | --- |",
+ ],
+ [
+ "foster-parented text reopens formatting",
+ "<table><b><tr><td>a</td></tr>bold</table>after",
+ "**bold**\n\na\n\n**after**",
+ ],
+ [
+ "<table> inside <table> closes the first",
+ "<table><tr><td>a<table><tr><td>b</td><td>c</td></tr></table>",
+ "a\n\n| b | c |\n| --- | --- |",
+ ],
+ ["table tags outside a table are ignored", "<p>a<tr><td>b</td></tr>c</p>", "abc"],
+ [
+ "<col>, <colgroup>, <caption>",
+ "<table><caption>cap</caption><colgroup><col></colgroup><tr><th>h</th></tr><tr><td>d</td></tr></table>",
+ "cap\n\n| h |\n| --- |\n| d |",
+ ],
+ [
+ "closing a cell drops formatting opened inside it",
+ "<table><tr><td><b>x</td><td>y</td></tr></table>z",
+ "| **x** | y |\n| --- | --- |\n\nz",
+ ],
+ ["a block inside inline formatting", "<b>1<p>2</p>3", "**1\n\n2\n\n3**"],
+ ["formatting reopens in the next list item", "<ul><li><b>1<li>2</ul>", "- **1**\n- **2**"],
+ ["adoption agency: </b> across a <p>", "<b>1<p>2</b>3", "**1**\n\n**2**3"],
+ ["<a> inside <a> closes the first", "<a href=x>1<a href=y>2</a>", "[1](x)[2](y)"],
+ [
+ "<a> left open is reopened in later blocks, as in a browser",
+ "<ul><li><a href=x>one<li>two</ul><p>after</p>",
+ "- [one](x)\n- [two](x)\n\n[after](x)",
+ ],
+ [
+ "identical formatting elements are capped at three (Noah's Ark)",
+ "<b><b><b><b>x</b></b></b></b><p>y",
+ "********x********\n\ny",
+ ],
+ [
+ "<a> inside <a> inside <object> leaves the outer link alone",
+ "<a href=/x>o<object><a href=/y>y<a href=/z>z</object>after</a>",
+ "[o[y](/y)[z](/z)after](/x)",
+ ],
+ [
+ "dropped siblings do not shift list numbering",
+ "<ol><script></script><li>a</li><template></template><li>b</ol>",
+ "1. a\n2. b",
+ ],
+ ["<image> is <img>", "<image src=a.png alt=i>", ""],
+ ["first newline after <pre> is dropped, even as a reference", "<pre> x\n</pre>", "```\nx\n```"],
+ ["<textarea> is RCDATA", "<textarea>\n<b><i></textarea>", "\\<b>\\<i>"],
+ [
+ "svg/math integration points carry HTML text",
+ "<svg><title>T</title><desc>D</desc></svg>|<math><mi>x</mi></math>",
+ "TD|x",
+ ],
+ ["text inside svg", "<svg><g><rect/><text>in svg</text></g></svg> after", "in svg after"],
+ ["an HTML block start breaks out of svg", "<svg><g>a<p>para</svg>b", "a\n\nparab"],
+ ["NUL is dropped in body text, U+FFFD in foreign content", "<p>a\u0000b<svg>c\u0000d</svg></p>", "abc\ufffdd"],
+ ["head-only elements before any content are not body content", "<meta charset=utf-8><link rel=x><p>a</p>", "a"],
+ [
+ "<body>/<html>/<head> tags mid-document are ignored",
+ "<p>a</p><body class=x><html lang=y><head><p>b</p>",
+ "a\n\nb",
+ ],
+ ["stray </p> makes an empty paragraph", "x</p>y", "x\n\ny"],
+ ["stray </br> is a <br>", "<p>a</br>b</p>", "a \nb"],
+ ["adoption agency: </b> inside <i> reopens the <i>", "<p><b>a<i>b</b>c</i>d</p>", "**a_b_**_c_d"],
+ ["<i> mis-closed by </p> reopens until its own end tag", "<p><i>a</p><p>b</i>c</p><p>d</p>", "_a_\n\n_b_c\n\nd"],
+ ["<i> never closed reopens in every later block", "<p><i>a</p><p>b</p>", "_a_\n\n_b_"],
+ ];
+ for (const [name, html, md] of cases) {
+ test(name, () => expect(fromHTML(html)).toBe(md));
+ }
+ });
+
+ describe("malformed input never throws", () => {
+ const nasties: [string, string][] = [
+ ["unclosed tags", "<p>a<b>b<i>c"],
+ ["stray end tags", "</div></p></b>text</table>"],
+ ["misnested formatting (adoption agency)", "<p><b>a<i>b</b>c</i>d</p>"],
+ ["table junk (foster parenting)", "<table>oops<tr>more<td>cell</td>bad</tr></table>"],
+ ["p inside button inside a", "<a href=x><button><p>hi</a></button>"],
+ ["unterminated comment", "<p>a<!-- forever"],
+ ["unterminated tag", "<p>a<b"],
+ ["unterminated attribute", '<a href="x'],
+ ["bogus doctype and PI", "<!DOCTYPE whatever SYSTEM><?xml version=1?><p>x</p>"],
+ ["CDATA outside foreign content", "<p><![CDATA[ x ]]]]><![CDATA[></p>"],
+ ["nested forms and selects", "<form><form><select><select><option>x</select>"],
+ ["plaintext element swallows the rest", "<plaintext><p>not a tag</p>"],
+ ["deeply broken lists", "<li><li><ul></li><ol><li></ul></ol>x"],
+ ["null in tag name", "<p\u0000>x</p\u0000>"],
+ ["isindex/keygen/legacy", "<isindex><keygen><frame><frameset><noframes>x</noframes>"],
+ ["huge attribute count", "<p " + Array.from({ length: 5000 }, (_, i) => `a${i}=v`).join(" ") + ">x</p>"],
+ ];
+ for (const [name, html] of nasties) {
+ test(name, () => {
+ expect(typeof fromHTML(html)).toBe("string");
+ });
+ }
+ test("misnested formatting keeps all text in order", () => {
+ expect(fromHTML("<p><b>a<i>b</b>c</i>d</p>").replace(/[*_]/g, "")).toBe("abcd");
+ });
+ test("foster-parented text keeps all words", () => {
+ const md = fromHTML("<table>oops<tr><td>cell</td><td>two</td></tr></table>");
+ expect(md).toContain("oops");
+ expect(md).toContain("| cell | two |");
+ });
+ });
+
+ describe("pathological nesting", () => {
+ // Each of these is quadratic (or a stack overflow) in a naive
+ // implementation. They must finish and keep the innermost text. The
+ // depth is far past both the parser's tree-depth cap (512) and the point
+ // where a recursive converter would exhaust the stack.
+ const depth = 20_000;
+ // `String.prototype.repeat` is slow in debug JSC.
+ const repeat = (s: string, n: number) => Buffer.alloc(s.length * n, s).toString();
+ const cases: [string, () => string][] = [
+ ["div", () => repeat("<div>", depth) + "deep" + repeat("</div>", depth)],
+ ["unclosed div", () => repeat("<div>", depth) + "deep"],
+ ["span", () => repeat("<span>", depth) + "deep"],
+ ["b", () => repeat("<b>", depth) + "deep"],
+ ["ul/li", () => repeat("<ul><li>", depth / 2) + "deep"],
+ ["blockquote", () => repeat("<blockquote>", depth) + "deep"],
+ ["table", () => repeat("<table><tr><td>", depth / 4) + "deep"],
+ ["a", () => repeat("<a href=x>", depth) + "deep"],
+ [
+ "custom elements with distinct names",
+ () => Array.from({ length: depth }, (_, i) => `<x-${i}>`).join("") + "deep",
+ ],
+ ["svg", () => "<svg>" + repeat("<g>", depth) + "<text>deep</text>"],
+ // Void / raw-text names are only special in HTML content; inside <svg>
+ // they nest like anything else and must be capped like anything else.
+ ["svg + html void names", () => "<svg>" + repeat("<wbr><textarea><col>", depth / 3) + "<text>deep</text>"],
+ ["formatting elements across blocks", () => repeat("<b><i><s><u>", 500) + repeat("<p>x</p>", 2000) + "deep"],
+ // Every implied </li> closes ~500 formatting elements that the next item
+ // would reopen; the reopen budget keeps the node count linear.
+ [
+ "formatting reopened by every list item",
+ () =>
+ "<ul><li>" +
+ Array.from({ length: 500 }, (_, i) => `<b x=${i}>`).join("") +
+ repeat("<li>x", depth) +
+ "<li>deep",
+ ],
+ // Noah's Ark compares attribute lists of same-named formatting elements.
+ [
+ "formatting elements with thousands of attributes",
+ () => repeat("<b " + Array.from({ length: 3000 }, (_, i) => `a${i}=1`).join(" ") + ">", 8) + "deep",
+ ],
+ // Each nested inline element re-derives its edge whitespace.
+ [
+ "long non-ASCII whitespace run inside nested inlines",
+ () => repeat("<span>", 500) + repeat("\u00a0", 200_000) + "deep",
+ ],
+ ["long list with omitted </li> and inline content", () => repeat("<li><a href=x>x</a>", depth) + "<li>deep"],
+ // A closed formatting list that can no longer be reopened (budget spent,
+ // deep stack) must not be re-walked for every later token.
+ [
+ "unreopenable formatting list under a deep stack",
+ () =>
+ "<ul>" +
+ Array.from({ length: 1022 }, (_, i) => `<li><b x=${i}>`).join("") +
+ "</ul>" +
+ repeat("<div>", 500) +
+ repeat("<wbr>", 100_000) +
+ "deep",
+ ],
+ ["stray end tags against many unclosed <p>", () => repeat("<p>x</q>", depth) + "<p>deep"],
+ ];
+ for (const [name, html] of cases) {
+ test(name, () => {
+ expect(fromHTML(html())).toContain("deep");
+ });
+ }
+ test("links still convert after the reopen budget is spent", () => {
+ const prefix =
+ "<ul><li>" + Array.from({ length: 509 }, (_, i) => `<b x=${i}>`).join("") + repeat("<li>x", 400) + "</ul>";
+ const links = Array.from({ length: 1200 }, (_, i) => `<p><a href=/p${i}>t${i}</a></p>`).join("");
+ const md = fromHTML(prefix + links);
+ expect(md).toContain("[t0](/p0)");
+ expect(md).toEndWith("[t1199](/p1199)");
+ });
+ test("content after a deep region is still structured", () => {
+ const md = fromHTML(
+ repeat("<div>", 5000) + "deep" + repeat("</div>", 5000) + "<h2>After</h2><ul><li>ok</li></ul>",
+ );
+ expect(md).toContain("deep");
+ expect(md).toEndWith("## After\n\n- ok");
+ });
+ test("deep inline formatting keeps text in order", () => {
+ const md = fromHTML(
+ repeat("<em><strong>", 600) + "one <b>two</b>" + repeat("</strong></em>", 600) + "<p>three</p>",
+ );
+ const words = md
+ .replace(/[*_\\]/g, "")
+ .split(/\s+/)
+ .filter(Boolean);
+ expect(words).toEqual(["one", "two", "three"]);
+ });
+ test("block boundaries past the depth cap still separate words", () => {
+ const md = fromHTML(repeat("<div>", 600) + "<p>one</p><p>two<br>three</p><ul><li>a</li><li>b</li></ul>");
+ expect(md.split(/\s+/)).toEqual(["one", "two", "three", "a", "b"]);
+ });
+ // Mis-nested formatting tags make the HTML adoption agency algorithm
+ // re-parent nodes towards the root, so the parser's stack of open
+ // elements grows while the tree stays shallow; a cap that only watched
+ // tree depth let this go quadratic (seconds for a few hundred KB).
+ test("adoption-agency churn is bounded by the open-element cap", () => {
+ const md = fromHTML(repeat(repeat("<a><b><div><a>", 8) + "x", 4000));
+ expect(md.replace(/[*\s]/g, "")).toBe(repeat("x", 4000));
+ });
+ test("a tag with 50k attributes is linear", () => {
+ // (Building the input dominates in a debug build; the conversions
+ // themselves take milliseconds.)
+ const attrs = Array.from({ length: 50_000 }, (_, i) => `a${i}=${i}`).join(" ");
+ expect(fromHTML(`<p><a ${attrs} href="/last">x</a></p>`)).toBe("[x](/last)");
+ expect(fromHTML(`<p><a ${repeat("href=/first ", 50_000)}href=/dup>y</a></p>`)).toBe("[y](/first)");
+ });
+ // Every level of a nested quote re-prefixes all of its lines; scratch
+ // buffers for that must not be retained per level or peak memory becomes
+ // depth × output.
+ test("nested blockquotes: output is prefixed per level, memory stays proportional", () => {
+ const md = fromHTML(repeat("<blockquote>", 50) + repeat("<p>line</p>", 2000));
+ const lines = md.split("\n").filter(l => l.endsWith("line"));
+ expect(lines.length).toBe(2000);
+ expect(lines[0]).toBe(repeat("> ", 50) + "line");
+ });
+ });
+
+ // -------------------------------------------------------------------------
+ // Round trip: canonical Markdown → Bun.markdown.html() → fromHTML() should
+ // give back the same Markdown. This checks the two directions agree on
+ // structure without involving any third-party implementation.
+ // -------------------------------------------------------------------------
+ describe("round-trips Bun.markdown.html output", () => {
+ const docs: [string, string][] = [
+ ["headings and paragraphs", "# Title\n\nSome text here.\n\n## Sub\n\nMore text."],
+ ["emphasis", "A **bold** and _em_ and `code` and ~~del~~ word."],
+ ["links and images", '[Bun](https://bun.com "Bun") and '],
+ ["lists", "- one\n- two\n - nested\n - again\n- three\n\n1. a\n2. b"],
+ ["loose list", "- para one\n\n para two\n\n- next"],
+ ["blockquote", "> quoted **text**\n>\n> - with\n> - list"],
+ ["code block", "```js\nconst x = 1 * 2;\nif (x) {\n y();\n}\n```"],
+ ["table", "| a | b |\n| --- | :-: |\n| 1 | 2 |\n| 3 | 4 |"],
+ ["task list", "- [x] done\n- [ ] todo"],
+ ["hr", "above\n\n---\n\nbelow"],
+ ["escapes", "\\*not em\\* and \\<tag> and \\_x\\_ but a_b and 3 < 4"],
+ ["hard break", "line one \nline two"],
+ ];
+ for (const [name, md] of docs) {
+ test(name, () => {
+ expect(fromHTML(Bun.markdown.html(md))).toBe(md);
+ });
+ }
+ });
+
+ // -------------------------------------------------------------------------
+ // Differential tests against turndown + turndown-plugin-gfm (the reference
+ // rule set), configured with the same spellings as our defaults. Outputs
+ // are compared after normalizing the differences we take deliberately:
+ // backslash escapes, `~`/`~~`, list-marker padding, indentation width and
+ // blank-line placement.
+ // -------------------------------------------------------------------------
+ describe("differential vs turndown", () => {
+ const td = new TurndownService({
+ headingStyle: "atx",
+ codeBlockStyle: "fenced",
+ bulletListMarker: "-",
+ hr: "---",
+ });
+ td.use(gfm);
+ td.remove(["script", "style", "noscript", "template", "title", "head"]);
+ const turndownCache = new Map<string, string>();
+ function turndown(html: string): string {
+ let md = turndownCache.get(html);
+ if (md === undefined) {
+ md = td.turndown(html);
+ turndownCache.set(html, md);
+ }
+ return md;
+ }
+ function normalize(md: string): string[] {
+ return md
+ .replace(/\\(.)/g, "$1")
+ .replace(/~+/g, "~")
+ .split("\n")
+ .map(l =>
+ l
+ .replace(/^(\s*)([-*+]|\d+\.)\s+/, "$1$2 ")
+ .replace(/\s+/g, " ")
+ .trim(),
+ )
+ .filter(l => l.length > 0 && !/^>+$/.test(l));
+ }
+ function expectEquivalent(html: string) {
+ const ours = normalize(fromHTML(html));
+ const theirs = normalize(turndown(html));
+ if (ours.join("\n") !== theirs.join("\n")) {
+ // Show the HTML in the failure so a fuzz case is reproducible.
+ expect({ html, markdown: ours }).toEqual({ html, markdown: theirs });
+ }
+ }
+
+ describe("curated", () => {
+ const snippets: [string, string][] = [
+ [
+ "article",
+ "<article><h1>Title</h1><p>By <a href='/me'>me</a>, <time>today</time>.</p><p>First <strong>para</strong>.</p><h2>Part</h2><p>Second.</p></article>",
+ ],
+ [
+ "nav list of links",
+ "<nav><ul><li><a href='/'>Home</a></li><li><a href='/docs'>Docs</a><ul><li><a href='/docs/a'>A</a></li></ul></li></ul></nav>",
+ ],
+ [
+ "definition-ish markup",
+ "<p><dfn>Term</dfn>: <abbr title='x'>abbr</abbr>, <cite>cite</cite>, <q>quote</q>, <mark>mark</mark>, <small>small</small></p>",
+ ],
+ [
+ "code in links and emphasis",
+ "<p><a href='/x'><code>fn()</code></a> and <em><code>y</code></em> and <strong><a href='/z'>z</a></strong></p>",
+ ],
+ [
+ "mixed list content",
+ "<ol><li><p>Para</p><pre><code>code\nblock</code></pre></li><li><blockquote>q</blockquote></li><li>plain<ul><li>n</li></ul>tail</li></ol>",
+ ],
+ [
+ "blockquote with everything",
+ "<blockquote><h3>H</h3><ul><li>a</li></ul><pre><code>x</code></pre><p>p<br>q</p></blockquote>",
+ ],
+ [
+ "gfm table with inline markup",
+ "<table><thead><tr><th><em>a</em></th><th align='right'>b</th></tr></thead><tbody><tr><td><a href='/x'>x</a></td><td><code>y|z</code></td></tr></tbody></table>",
+ ],
+ [
+ "images and figures",
+ "<figure><img src='/a.png' alt='A'><figcaption>Caption <em>here</em></figcaption></figure>",
+ ],
+ ["whitespace soup", "<div>\n <p>\n a\n <span> b </span>\n <b> c</b><i>d </i>\n e\n </p>\n</div>"],
+ ["inline-block mix", "<div>text <div>block</div> more <span>inline</span><p>para</p>tail</div>"],
+ [
+ "headings with links (wiki style)",
+ "<h2><span id='History'>History</span><span><a href='/edit'>[edit]</a></span></h2><p>x<sup><a href='#cite-1'>[1]</a></sup></p>",
+ ],
+ [
+ "entities and unicode",
+ "<p><div> & " ' café 日本語 emoji 😀 — …</p>",
+ ],
+ [
+ "line-start escapes",
+ "<p>- a</p><ul><li>+ b</li><li>1. c</li></ul><h2># d</h2><blockquote>> e</blockquote>",
+ ],
+ [
+ "adjacent code blocks and hr",
+ "<pre><code>a</code></pre><pre><code class='language-b'>b</code></pre><hr><hr>",
+ ],
+ ["strikethrough nesting", "<p><del>a <b>b</b></del> <s><a href='/x'>c</a></s></p>"],
+ [
+ "task list variants",
+ "<ul><li><input type=checkbox checked>a</li><li><input type=checkbox> <b>b</b></li><li>c</li></ul>",
+ ],
+ ];
+ for (const [name, html] of snippets) {
+ test(name, () => expectEquivalent(html));
+ }
+ });
+
+ test("Bun docs page fixture", () => {
+ // bun.com/docs/runtime/markdown (this repo's own docs page) with
+ // scripts, SVGs and styling attributes stripped. turndown's output for
+ // it is checked in (see fixtures/regenerate-turndown.mjs) because the JS
+ // converter takes seconds on a whole page under a debug build.
+ const dir = join(import.meta.dir, "fixtures");
+ const ours = normalize(fromHTML(readFileSync(join(dir, "bun-docs-markdown.html"), "utf8")));
+ const theirs = normalize(readFileSync(join(dir, "bun-docs-markdown.turndown.md"), "utf8"));
+ expect(ours).toEqual(theirs);
+ // And it is real content, not two empty outputs agreeing.
+ expect(ours.length).toBeGreaterThan(300);
+ });
+
+ // Seeded structural fuzzing: random documents built from the elements
+ // both implementations treat the same way. The generator avoids the
+ // documented deviations (br as the only content of an inline element,
+ // newlines inside table cells and headings, header-less tables, bare
+ // <pre>).
+ describe("fuzz", () => {
+ function makeRng(seed: number) {
+ let s = seed >>> 0;
+ return () => {
+ // xorshift32
+ s ^= s << 13;
+ s >>>= 0;
+ s ^= s >>> 17;
+ s ^= s << 5;
+ s >>>= 0;
+ return s / 0x100000000;
+ };
+ }
+ function generate(seed: number): string {
+ const rnd = makeRng(seed);
+ const pick = <T>(a: T[]): T => a[Math.floor(rnd() * a.length)];
+ const WORDS = [
+ "foo",
+ "bar",
+ "baz qux",
+ "a*b",
+ "x_y",
+ "1. one",
+ "- dash",
+ "# hash",
+ "[br]",
+ "back\\slash",
+ "tick`s",
+ " spaced ",
+ "\n\t nl \n",
+ "&",
+ "<tag>",
+ "caf\u00e9",
+ "\u00a0nbsp\u00a0",
+ "> gt",
+ "+ plus",
+ "===",
+ "~~~",
+ "snake_case_word",
+ ""q"",
+ ];
+ const ws = () => pick(["", "", "", " ", "\n", " \n "]);
+ const text = () => pick(WORDS) + (rnd() < 0.5 ? " " : "") + (rnd() < 0.3 ? pick(WORDS) : "");
+ let oneLine = false; // inside a table cell or heading
+ const inline = (d: number): string => {
+ if (d > 4 || rnd() < 0.35) return text();
+ let k = pick(["b", "strong", "i", "em", "code", "a", "span", "del", "br", "img", "sup", "kbd"]);
+ if ((oneLine || d > 0) && k === "br") k = "span";
+ if (k === "br") return "<br>";
+ if (k === "img")
+ return `<img src="${pick(["a.png", "", "b c.png", "x(1).png"])}" alt="${pick(["alt", "", "a*b"])}"${rnd() < 0.3 ? ' title="t"' : ""}>`;
+ if (k === "a")
+ return `<a${rnd() < 0.9 ? ` href="${pick(["/x", "http://e.com/a b", "", "#f(1)"])}"` : ""}${!oneLine && rnd() < 0.2 ? ' title="T\n t"' : ""}>${ws()}${inline(d + 1)}${ws()}</a>`;
+ return `<${k}>${ws()}${inline(d + 1)}${rnd() < 0.4 ? inline(d + 1) : ""}${ws()}</${k}>`;
+ };
+ const inlines = (d: number) => {
+ let s = "";
+ const n = 1 + Math.floor(rnd() * 3);
+ for (let i = 0; i < n; i++) s += inline(d) + ws();
+ return s;
+ };
+ const block = (d: number): string => {
+ if (d > 3 || rnd() < 0.25) return `<p>${ws()}${inlines(0)}${ws()}</p>`;
+ const k = pick([
+ "div",
+ "h1",
+ "h2",
+ "h3",
+ "blockquote",
+ "ul",
+ "ol",
+ "pre",
+ "hr",
+ "table",
+ "section",
+ "p",
+ "dl",
+ ]);
+ switch (k) {
+ case "hr":
+ return "<hr>";
+ case "h1":
+ case "h2":
+ case "h3": {
+ oneLine = true;
+ const h = `<${k}>${inlines(0)}</${k}>`;
+ oneLine = false;
+ return h;
+ }
+ case "pre":
+ return `<pre><code${rnd() < 0.5 ? ' class="language-js"' : ""}>${pick(["let x = 1;\n y();\n", "```\ninner\n```", "plain", "a\n\n\nb\n"])}</code></pre>`;
+ case "ul":
+ case "ol": {
+ let s = `<${k}${k === "ol" && rnd() < 0.3 ? ' start="4"' : ""}>`;
+ const n = 1 + Math.floor(rnd() * 3);
+ for (let i = 0; i < n; i++)
+ s += `${ws()}<li>${ws()}${rnd() < 0.6 ? inlines(0) : block(d + 1) + (rnd() < 0.5 ? block(d + 1) : "")}${rnd() < 0.3 ? `<ul><li>n1</li><li>n2</li></ul>` : ""}${ws()}</li>${ws()}`;
+ return s + `</${k}>`;
+ }
+ case "table": {
+ oneLine = true;
+ const cols = 1 + Math.floor(rnd() * 3);
+ let s = "<table><thead><tr>";
+ for (let c = 0; c < cols; c++) s += `<th${rnd() < 0.3 ? ' align="center"' : ""}>${inline(2)}</th>`;
+ s += "</tr></thead><tbody>";
+ const rows = 1 + Math.floor(rnd() * 2);
+ for (let r = 0; r < rows; r++) {
+ s += "<tr>";
+ for (let c = 0; c < cols; c++) s += `<td>${inline(2)}</td>`;
+ s += "</tr>";
+ }
+ oneLine = false;
+ return s + "</tbody></table>";
+ }
+ case "dl":
+ return `<dl><dt>${inlines(0)}</dt><dd>${inlines(0)}</dd></dl>`;
+ default: {
+ let s = `<${k}>${ws()}`;
+ const n = 1 + Math.floor(rnd() * 3);
+ for (let i = 0; i < n; i++) s += (rnd() < 0.3 ? inlines(0) : block(d + 1)) + ws();
+ return s + `</${k}>`;
+ }
+ }
+ };
+ let s = "";
+ const n = 1 + Math.floor(rnd() * 4);
+ for (let i = 0; i < n; i++) s += block(0) + ws();
+ return s;
+ }
+
+ // Whitespace-only `<del>` (turndown emits an empty `~~` pair, we emit
+ // nothing) is the one deviation the generator can still hit; skip
+ // those cases, and make sure that stays a small minority.
+ // (The plugin writes `~`, so that output is `~~` or `~ ~` with no
+ // tilde on either side; literal `~~~` text does not count.)
+ const deviates = (html: string) => /(^|[^~])~\s*~(?!~)/.test(turndown(html));
+ let skipped = 0;
+
+ for (let seed = 1; seed <= 250; seed++) {
+ test(`seed ${seed}`, () => {
+ const html = generate(seed);
+ if (deviates(html)) {
+ skipped++;
+ return;
+ }
+ expectEquivalent(html);
+ });
+ }
+ test("most seeds are comparable", () => {
+ expect(skipped).toBeLessThan(40);
+ });
+ });
+ });
+});
diff --git a/test/package.json b/test/package.json
index b9bc51f5a99d..8e7402991103 100644
--- a/test/package.json
+++ b/test/package.json
@@ -97,6 +97,8 @@
"svelte": "5.20.4",
"tsyringe": "4.8.0",
"tunnel": "0.0.6",
+ "turndown": "7.2.4",
+ "turndown-plugin-gfm": "1.0.2",
"type-graphql": "2.0.0-rc.2",
"typeorm": "0.3.20",
"typescript": "6.0.2",
]]></plaintext></td></li></div> | 
Figure ${i}.