From 45803ca5040bb51f94cfa6953ec8ad98b13fbe9a Mon Sep 17 00:00:00 2001 From: SUZUKI Sosuke Date: Sat, 5 Sep 2026 17:49:33 +0000 Subject: [PATCH 01/19] Add Bun.markdown.fromHTML(): convert HTML to Markdown natively Adds the inverse of Bun.markdown.html(): a synchronous HTML -> Markdown converter. It takes a document or fragment (string or UTF-8 buffer) and returns GitHub Flavored Markdown. - Parses with html5ever (spec-compliant HTML5 tree construction) into an arena DOM, collapses whitespace the way turndown does, then applies turndown's rule set plus turndown-plugin-gfm (tables, strikethrough, task lists) with a fixed rule per tag. No JS callbacks; options are enums/booleans only (headingStyle, hr, bulletListMarker, codeBlockStyle, fence, emDelimiter, strongDelimiter, br, tables, strikethrough, tasklists). - Deliberate differences from turndown, all in the reader's favour: header-less tables still become pipe tables (first row is the header), bare
 is a code block, headings and table cells are kept on one
  line, script/style/noscript/template are dropped, line-start escapes
  only apply at line start, and intraword underscores are not escaped.
- Never throws on malformed input. Tree depth is capped at 512 during
  parsing (as browsers do) so hostile nesting stays linear, and the
  emitter has its own recursion backstop.
- ~12-20 ms for 800 KB-1 MB real pages in a release build (7-15x faster
  than turndown + gfm on the same inputs), memory linear in input.

Tests cover each rule, options, malformed and pathological input,
round-tripping Bun.markdown.html output, and differential comparison
against vendored turndown (curated snippets, a real docs page, and a
seeded structural fuzzer).
---
 Cargo.lock                                    | 109 +++
 Cargo.toml                                    |   2 +
 bench/bun.lock                                |   8 +
 bench/package.json                            |   2 +
 bench/snippets/html-to-markdown.mjs           |  98 ++
 docs/project/license.mdx                      |   1 +
 docs/runtime/markdown.mdx                     | 106 ++-
 packages/bun-types/bun.d.ts                   | 118 +++
 src/md/Cargo.toml                             |   2 +
 src/md/from_html/depth.rs                     | 160 ++++
 src/md/from_html/dom.rs                       | 786 +++++++++++++++
 src/md/from_html/emit.rs                      | 896 ++++++++++++++++++
 src/md/from_html/mod.rs                       | 230 +++++
 src/md/from_html/text.rs                      | 331 +++++++
 src/md/from_html/whitespace.rs                | 165 ++++
 src/md/lib.rs                                 |   1 +
 src/runtime/api/MarkdownObject.rs             | 182 ++++
 test/bun.lock                                 |   8 +
 test/integration/bun-types/fixture/bun.ts     |  21 +
 .../js/bun/md/fixtures/bun-docs-markdown.html | 128 +++
 .../md/fixtures/bun-docs-markdown.turndown.md | 528 +++++++++++
 .../bun/md/fixtures/regenerate-turndown.mjs   |  22 +
 test/js/bun/md/md-from-html.test.ts           | 747 +++++++++++++++
 test/package.json                             |   2 +
 24 files changed, 4652 insertions(+), 1 deletion(-)
 create mode 100644 bench/snippets/html-to-markdown.mjs
 create mode 100644 src/md/from_html/depth.rs
 create mode 100644 src/md/from_html/dom.rs
 create mode 100644 src/md/from_html/emit.rs
 create mode 100644 src/md/from_html/mod.rs
 create mode 100644 src/md/from_html/text.rs
 create mode 100644 src/md/from_html/whitespace.rs
 create mode 100644 test/js/bun/md/fixtures/bun-docs-markdown.html
 create mode 100644 test/js/bun/md/fixtures/bun-docs-markdown.turndown.md
 create mode 100644 test/js/bun/md/fixtures/regenerate-turndown.mjs
 create mode 100644 test/js/bun/md/md-from-html.test.ts

diff --git a/Cargo.lock b/Cargo.lock
index 20012ce3d0cf..c028058bdb18 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -987,8 +987,10 @@ dependencies = [
  "bun_paths",
  "bun_sys",
  "bun_url",
+ "html5ever",
  "scopeguard",
  "strum",
+ "typed-arena",
 ]
 
 [[package]]
@@ -2250,6 +2252,16 @@ version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c"
 
+[[package]]
+name = "html5ever"
+version = "0.39.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "46a1761807faccc9a19e86944bbf40610014066306f96edcdedc2fb714bcb7b8"
+dependencies = [
+ "log",
+ "markup5ever",
+]
+
 [[package]]
 name = "id-arena"
 version = "2.3.0"
@@ -2363,6 +2375,15 @@ version = "0.4.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab"
 
+[[package]]
+name = "lock_api"
+version = "0.4.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965"
+dependencies = [
+ "scopeguard",
+]
+
 [[package]]
 name = "log"
 version = "0.4.29"
@@ -2386,6 +2407,17 @@ dependencies = [
  "thiserror",
 ]
 
+[[package]]
+name = "markup5ever"
+version = "0.39.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7122d987ec5f704ee56f6e5b41a7d93722e9aae27ae07cafa4036c4d3f9757de"
+dependencies = [
+ "log",
+ "tendril",
+ "web_atoms",
+]
+
 [[package]]
 name = "memchr"
 version = "2.8.0"
@@ -2425,6 +2457,29 @@ version = "11.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e"
 
+[[package]]
+name = "parking_lot"
+version = "0.12.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a"
+dependencies = [
+ "lock_api",
+ "parking_lot_core",
+]
+
+[[package]]
+name = "parking_lot_core"
+version = "0.9.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "redox_syscall",
+ "smallvec",
+ "windows-link",
+]
+
 [[package]]
 name = "paste"
 version = "1.0.15"
@@ -2587,6 +2642,15 @@ dependencies = [
  "crossbeam-utils",
 ]
 
+[[package]]
+name = "redox_syscall"
+version = "0.5.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
+dependencies = [
+ "bitflags",
+]
+
 [[package]]
 name = "regex"
 version = "1.12.3"
@@ -2792,6 +2856,30 @@ version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"
 
+[[package]]
+name = "string_cache"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a18596f8c785a729f2819c0f6a7eae6ebeebdfffbfe4214ae6b087f690e31901"
+dependencies = [
+ "new_debug_unreachable",
+ "parking_lot",
+ "phf_shared",
+ "precomputed-hash",
+]
+
+[[package]]
+name = "string_cache_codegen"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "585635e46db231059f76c5849798146164652513eb9e8ab2685939dd90f29b69"
+dependencies = [
+ "phf_generator",
+ "phf_shared",
+ "proc-macro2",
+ "quote",
+]
+
 [[package]]
 name = "strum"
 version = "0.26.3"
@@ -2831,6 +2919,15 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "tendril"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5fed54709c5b3a53d09bb1c113ea4f5ceafd1e772ddcb0030a82e1d56c087b08"
+dependencies = [
+ "new_debug_unreachable",
+]
+
 [[package]]
 name = "thiserror"
 version = "2.0.18"
@@ -3002,6 +3099,18 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "web_atoms"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba8b815c1b593dc0baf78dd0f4fc8fdb2de53198fb1163738093e9a311c33fb3"
+dependencies = [
+ "phf",
+ "phf_codegen",
+ "string_cache",
+ "string_cache_codegen",
+]
+
 [[package]]
 name = "winapi-util"
 version = "0.1.11"
diff --git a/Cargo.toml b/Cargo.toml
index 012f23a486ab..5cdec9b87bce 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -364,6 +364,8 @@ thiserror = "2"
 smallvec = "1"
 itoa = "1"
 encoding_rs = "0.8.35"
+# Spec-compliant HTML5 parser behind `Bun.markdown.fromHTML` (src/md/from_html).
+html5ever = "0.39"
 # Vendored, not crates.io — see the `exclude = ["vendor"]` note above.
 lol_html = { path = "vendor/lolhtml" }
 bun_opaque = { path = "src/opaque" }
diff --git a/bench/bun.lock b/bench/bun.lock
index 2692ed9c3aac..bd5c1f278480 100644
--- a/bench/bun.lock
+++ b/bench/bun.lock
@@ -31,6 +31,8 @@
         "strip-ansi": "^7.1.0",
         "tar": "^7.4.3",
         "tinycolor2": "^1.6.0",
+        "turndown": "^7.2.4",
+        "turndown-plugin-gfm": "^1.0.2",
         "wrap-ansi": "^9.0.0",
         "zx": "^7.2.3",
       },
@@ -128,6 +130,8 @@
 
     "@jridgewell/trace-mapping": ["@jridgewell/trace-mapping@0.3.17", "", { "dependencies": { "@jridgewell/resolve-uri": "3.1.0", "@jridgewell/sourcemap-codec": "1.4.14" } }, "sha512-MCNzAp77qzKca9+W/+I0+sEpaUnZoeasnghNeVc41VZCEKaCH73Vq3BZZ/SzWIgrqE4H4ceI+p+b6C0mHf9T4g=="],
 
+    "@mixmark-io/domino": ["@mixmark-io/domino@2.2.0", "", {}, "sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw=="],
+
     "@nodelib/fs.scandir": ["@nodelib/fs.scandir@2.1.5", "", { "dependencies": { "@nodelib/fs.stat": "2.0.5", "run-parallel": "^1.1.9" } }, "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g=="],
 
     "@nodelib/fs.stat": ["@nodelib/fs.stat@2.0.5", "", {}, "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A=="],
@@ -648,6 +652,10 @@
 
     "trough": ["trough@2.2.0", "", {}, "sha512-tmMpK00BjZiUyVyvrBK7knerNgmgvcV/KLVyuma/SC+TQN167GrMRciANTz09+k3zW8L8t60jWO1GpfkZdjTaw=="],
 
+    "turndown": ["turndown@7.2.4", "", { "dependencies": { "@mixmark-io/domino": "^2.2.0" } }, "sha512-I8yFsfRzmzK0WV1pNNOA4A7y4RDfFxPRxb3t+e3ui14qSGOxGtiSP6GjeX+Y6CHb7HYaFj7ECUD7VE5kQMZWGQ=="],
+
+    "turndown-plugin-gfm": ["turndown-plugin-gfm@1.0.2", "", {}, "sha512-vwz9tfvF7XN/jE0dGoBei3FXWuvll78ohzCZQuOb+ZjWrs3a0XhQVomJEb2Qh4VHTPNRO4GPZh0V7VRbiWwkRg=="],
+
     "undici-types": ["undici-types@5.26.5", "", {}, "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA=="],
 
     "unified": ["unified@11.0.5", "", { "dependencies": { "@types/unist": "^3.0.0", "bail": "^2.0.0", "devlop": "^1.0.0", "extend": "^3.0.0", "is-plain-obj": "^4.0.0", "trough": "^2.0.0", "vfile": "^6.0.0" } }, "sha512-xKvGhPWw3k84Qjh8bI3ZeJjqnyadK+GEFtazSfZv/rKeTkTjOJho6mFqh2SM96iIcZokxiOpg78GazTSg8+KHA=="],
diff --git a/bench/package.json b/bench/package.json
index 9bed18fa6801..97ce07b0aee6 100644
--- a/bench/package.json
+++ b/bench/package.json
@@ -27,6 +27,8 @@
     "strip-ansi": "^7.1.0",
     "tar": "^7.4.3",
     "tinycolor2": "^1.6.0",
+    "turndown": "^7.2.4",
+    "turndown-plugin-gfm": "^1.0.2",
     "wrap-ansi": "^9.0.0",
     "zx": "^7.2.3"
   },
diff --git a/bench/snippets/html-to-markdown.mjs b/bench/snippets/html-to-markdown.mjs
new file mode 100644
index 000000000000..8b6673c43132
--- /dev/null
+++ b/bench/snippets/html-to-markdown.mjs
@@ -0,0 +1,98 @@
+// HTML → Markdown: Bun.markdown.fromHTML vs turndown (+ turndown-plugin-gfm).
+//
+//   bun bench/snippets/html-to-markdown.mjs            # synthetic pages
+//   bun bench/snippets/html-to-markdown.mjs page.html  # your own saved pages
+//
+// turndown is what most JS apps use for this today; it builds a DOM with a
+// pure-JS HTML parser (domino) and walks it with regex-heavy rules, so large
+// pages cost hundreds of milliseconds of main-thread time and a lot of
+// transient garbage. The synthetic input below is shaped like a docs/article
+// page (nav, prose with inline markup, code blocks, tables, lists) and is
+// repeated to reach the sizes real pages have.
+import { readFileSync } from "node:fs";
+import { basename } from "node:path";
+import TurndownService from "turndown";
+import { gfm } from "turndown-plugin-gfm";
+import { bench, group, run } from "../runner.mjs";
+
+function turndown(html) {
+  const td = new TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced", bulletListMarker: "-", hr: "---" });
+  td.use(gfm);
+  td.remove(["script", "style"]);
+  return td.turndown(html);
+}
+
+const section = i => `
+
+

Section ${i}: configuring the thing

+

+ Lorem ipsum dolor sit amet, consectetur adipiscing elit. Use + the thing() API to + frobnicate the widget — it’s slow fast now. See issue + #${1000 + i} for details, and note that + 2 * 3 = 6 and snake_case_names are left alone. +

+
import { thing } from "pkg";
+
+export async function run${i}(input: string): Promise<number> {
+  const result = await thing(input, { retries: ${i % 5}, verbose: true });
+  if (!result.ok) throw new Error(\`failed: \${result.error}\`);
+  return result.value * 2;
+}
+
+ + + + + + + +
OptionTypeDefaultDescription
retriesnumber3How many times to retry before giving up.
verbosebooleanfalseLog each attempt to stderr.
signalAbortSignal—Cancels the in-flight attempt | and any pending retries.
+
    +
  • First point with bold and a link
  • +
  • Second point +
    1. nested three
    2. nested four
      • deeper
    +
  • +
  • done item
  • +
  • Loose item with a paragraph.

    And a quote
    with a break.

  • +
+

Diagram ${i} Figure ${i}.

+
+
`; + +function page(sections) { + let nav = ""; + for (let i = 0; i < sections; i++) nav += `
  • Section ${i}
  • `; + let body = ""; + for (let i = 0; i < sections; i++) body += section(i); + return `Bench page + + +

    Benchmark page

    ${body}
    + +`; +} + +const inputs = []; +const files = process.argv.slice(2); +if (files.length) { + for (const f of files) inputs.push([basename(f), readFileSync(f, "utf8")]); +} else { + for (const n of [2, 40, 400]) { + const html = page(n); + inputs.push([`${(html.length / 1024).toFixed(0)} KB page`, html]); + } +} + +const hasBun = typeof Bun !== "undefined" && Bun.markdown && typeof Bun.markdown.fromHTML === "function"; + +for (const [name, html] of inputs) { + group(name, () => { + if (hasBun) { + bench("Bun.markdown.fromHTML", () => Bun.markdown.fromHTML(html)); + } + bench("turndown + gfm", () => turndown(html)); + }); +} + +await run(); diff --git a/docs/project/license.mdx b/docs/project/license.mdx index 67dc787c9943..d72aa76962d2 100644 --- a/docs/project/license.mdx +++ b/docs/project/license.mdx @@ -29,6 +29,7 @@ Bun statically links these libraries: | [`brotli`](https://github.com/google/brotli) | MIT | | [`libarchive`](https://github.com/libarchive/libarchive) | [several licenses](https://github.com/libarchive/libarchive/blob/master/COPYING) | | [`lol-html`](https://github.com/cloudflare/lol-html/tree/master/c-api) | BSD 3-Clause | +| [`html5ever`](https://github.com/servo/html5ever) | dual-licensed under the MIT License or Apache 2.0 | | [`ls-hpack`](https://github.com/litespeedtech/ls-hpack) | MIT (bundled xxhash is BSD 2-Clause) | | [`ls-qpack`](https://github.com/litespeedtech/ls-qpack) | MIT | | [`lsquic`](https://github.com/litespeedtech/lsquic) | MIT (portions derived from Chromium proto-quic, BSD 3-Clause) | diff --git a/docs/runtime/markdown.mdx b/docs/runtime/markdown.mdx index 3a30fb9990bd..673b16069c00 100644 --- a/docs/runtime/markdown.mdx +++ b/docs/runtime/markdown.mdx @@ -7,11 +7,12 @@ description: Parse and render Markdown with Bun's built-in Markdown API, support **Unstable API** — This API is under active development and may change in future versions of Bun. -Bun includes a fast, built-in Markdown parser written in Rust. It supports GitHub Flavored Markdown (GFM) extensions and provides three APIs: +Bun includes a fast, built-in Markdown parser written in Rust. It supports GitHub Flavored Markdown (GFM) extensions and provides these APIs: - `Bun.markdown.html()` — render Markdown to an HTML string - `Bun.markdown.render()` — render Markdown with custom callbacks for each element - `Bun.markdown.react()` — render Markdown to React JSX elements +- `Bun.markdown.fromHTML()` — convert HTML to Markdown --- @@ -379,3 +380,106 @@ const el = Bun.markdown.react("## Hello World", undefined, { autolinks: true, }); ``` + +--- + +## `Bun.markdown.fromHTML()` + +Convert an HTML string to Markdown. This is the inverse of `Bun.markdown.html()`: it takes a document or fragment of HTML and produces GitHub Flavored Markdown. + +```ts +const md = Bun.markdown.fromHTML("

    Hello world

    See bun.com.

    "); +// "# Hello _world_\n\nSee [bun.com](https://bun.com)." +``` + +Whole pages work the same way as fragments: + +```ts +const res = await fetch("https://bun.com/docs/runtime/markdown"); +const md = Bun.markdown.fromHTML(await res.text()); +``` + +Bun parses the input with a spec-compliant HTML5 parser, so malformed markup is recovered the same way a browser recovers it. The conversion never throws on bad HTML. Only `` content is converted. `` so a browser could run it; nothing to + // do here but resume. + while !matches!(tokenizer.feed(&input), TokenizerResult::Done) {} + tokenizer.end(); + let document = tokenizer.sink.inner.sink.get_document(); + + // Convert `` when the parser produced one (it always does for + // document input); fall back to `` for frameset documents. + let html_el = document.children().find(|c| c.tag() == dom::Tag::Html); + let root = html_el + .and_then(|h| h.children().find(|c| c.tag() == dom::Tag::Body)) + .or(html_el) + .unwrap_or(document); + + whitespace::collapse_whitespace(root); + dom::compute_subtree_flags(root); + + let mut out = String::with_capacity(html.len() / 2); + emit::Converter::new(options).convert(root, &mut out); + out +} diff --git a/src/md/from_html/text.rs b/src/md/from_html/text.rs new file mode 100644 index 000000000000..9e086f3e0899 --- /dev/null +++ b/src/md/from_html/text.rs @@ -0,0 +1,331 @@ +//! String helpers that mirror the JavaScript semantics turndown relies on +//! (`String.prototype.trim`, `\s`), plus Markdown escaping. + +use bun_core::strings; + +use super::dom::{NodeData, Ref, next_in_preorder}; + +/// Splits on `\n`. (`str::split` is off-limits in this tree; the SIMD byte +/// search finds each newline and `\n` is always a char boundary.) +pub(crate) fn lines(s: &str) -> impl Iterator { + let mut rest = Some(s); + core::iter::from_fn(move || { + let cur = rest?; + match strings::index_of_char_usize(cur.as_bytes(), b'\n') { + Some(i) => { + rest = Some(&cur[i + 1..]); + Some(&cur[..i]) + } + None => { + rest = None; + Some(cur) + } + } + }) +} + +/// JavaScript `\s` / `String.prototype.trim` whitespace: WhiteSpace + +/// LineTerminator code points. +#[inline] +pub(crate) fn is_js_whitespace(c: char) -> bool { + matches!( + c, + '\u{0009}'..='\u{000D}' + | ' ' + | '\u{00A0}' + | '\u{1680}' + | '\u{2000}'..='\u{200A}' + | '\u{2028}' + | '\u{2029}' + | '\u{202F}' + | '\u{205F}' + | '\u{3000}' + | '\u{FEFF}' + ) +} + +/// The subset turndown's whitespace collapsing treats as inter-word space +/// (`/[ \r\n\t]+/`). Everything else in `\s` (nbsp, etc.) is preserved. +#[inline] +pub(crate) fn is_ascii_ws(c: char) -> bool { + matches!(c, ' ' | '\r' | '\n' | '\t') +} + +pub(crate) fn js_trim(s: &str) -> &str { + s.trim_matches(is_js_whitespace) +} + +/// `String.prototype.trim` applied in place. +pub(crate) fn js_trim_in_place(s: &mut String) { + let end = s.trim_end_matches(is_js_whitespace).len(); + s.truncate(end); + let start = s.len() - s.trim_start_matches(is_js_whitespace).len(); + if start > 0 { + s.drain(..start); + } +} + +pub(crate) fn leading_newlines(s: &str) -> usize { + s.bytes().take_while(|&b| b == b'\n').count() +} + +/// Appends the concatenated text of every text node under `node` (DOM +/// `textContent`). +pub(crate) fn push_text_content(node: Ref<'_>, out: &mut String) { + if let Some(t) = node.as_text() { + out.push_str(&t.borrow()); + return; + } + let mut cur = node.first_child.get(); + while let Some(n) = cur { + if let Some(t) = n.as_text() { + out.push_str(&t.borrow()); + } + cur = next_in_preorder(n, node, true); + } +} + +/// Iterates the text nodes under `root` in document order (forwards) or +/// reverse document order (backwards). +fn text_nodes<'a>(root: Ref<'a>, backwards: bool) -> impl Iterator> { + // Reverse pre-order: last_child chain, then previous siblings, then up. + fn prev_in_order<'a>(node: Ref<'a>, root: Ref<'a>) -> Option> { + if let Some(c) = node.last_child.get() { + return Some(c); + } + let mut cur = node; + loop { + if core::ptr::eq(cur, root) { + return None; + } + if let Some(s) = cur.previous_sibling.get() { + return Some(s); + } + cur = cur.parent.get()?; + } + } + let first = if root.as_text().is_some() { + Some(root) + } else if backwards { + root.last_child.get() + } else { + root.first_child.get() + }; + let mut cur = first; + core::iter::from_fn(move || { + loop { + let n = cur?; + cur = if core::ptr::eq(n, root) { + None + } else if backwards { + prev_in_order(n, root) + } else { + next_in_preorder(n, root, true) + }; + if matches!(n.data, NodeData::Text(_)) { + return Some(n); + } + } + }) +} + +/// The leading whitespace run of `node.textContent`, split the way +/// turndown's `edgeWhitespace` regex does: `ascii` is the initial +/// `[ \t\r\n]*` run and `rest` is whatever `\s*` follows it (nbsp and +/// friends, possibly with more ASCII space after them). For all-whitespace +/// content the whole text counts as leading. +pub(crate) fn leading_whitespace(node: Ref<'_>, ascii: &mut String, rest: &mut String) { + ascii.clear(); + rest.clear(); + for t in text_nodes(node, false) { + let t = t.as_text().unwrap().borrow(); + for c in t.chars() { + if !is_js_whitespace(c) { + return; + } + if rest.is_empty() && is_ascii_ws(c) { + ascii.push(c); + } else { + rest.push(c); + } + } + } +} + +/// The trailing whitespace run of `node.textContent`: `ascii` is the final +/// `[ \t\r\n]*` run and `rest` is the non-ASCII-whitespace part before it. +/// Empty for all-whitespace content (turndown attributes it all to the +/// leading side). +pub(crate) fn trailing_whitespace(node: Ref<'_>, rest: &mut String, ascii: &mut String) { + ascii.clear(); + rest.clear(); + // Collected in reverse; flipped at the end. + let mut saw_non_ws = false; + 'outer: for t in text_nodes(node, true) { + let t = t.as_text().unwrap().borrow(); + for c in t.chars().rev() { + if !is_js_whitespace(c) { + saw_non_ws = true; + break 'outer; + } + if rest.is_empty() && is_ascii_ws(c) { + ascii.push(c); + } else { + rest.push(c); + } + } + } + if !saw_non_ws { + ascii.clear(); + rest.clear(); + return; + } + reverse_in_place(ascii); + reverse_in_place(rest); +} + +fn reverse_in_place(s: &mut String) { + if s.len() > 1 { + let rev: String = s.chars().rev().collect(); + *s = rev; + } +} + +/// Whether `node.textContent` ends with an ASCII space (`/ $/`). +pub(crate) fn text_content_ends_with_space(node: Ref<'_>) -> bool { + for t in text_nodes(node, true) { + let t = t.as_text().unwrap().borrow(); + if !t.is_empty() { + return t.ends_with(' '); + } + } + false +} + +/// Whether `node.textContent` starts with an ASCII space (`/^ /`). +pub(crate) fn text_content_starts_with_space(node: Ref<'_>) -> bool { + for t in text_nodes(node, false) { + let t = t.as_text().unwrap().borrow(); + if !t.is_empty() { + return t.starts_with(' '); + } + } + false +} + +/// Escapes Markdown-significant characters in a run of prose so the text +/// round-trips as literal text. Follows turndown's `escapeMarkdown`, with +/// two deliberate differences that only ever remove backslashes a reader +/// would otherwise see: +/// +/// - line-start constructs (`-`, `+ `, `#`, `>`, `1. `, `=`, `~~~`) are only +/// escaped when the text actually lands at the start of a line, and +/// - `_` between two ASCII alphanumerics is left alone, since CommonMark +/// never treats an intraword underscore as emphasis. +/// +/// `<` followed by something that could open a tag or autolink is escaped +/// too (turndown leaves it, which lets literal `<div>` text turn back +/// into markup). +pub(crate) fn escape_markdown_into(text: &str, at_line_start: bool, out: &mut String) { + let bytes = text.as_bytes(); + if bytes.is_empty() { + return; + } + out.reserve(text.len()); + + let mut i = 0; + if at_line_start { + i = escape_line_start(text, out); + } + + // Everything below is ASCII, so byte-wise copying keeps UTF-8 intact. + let mut run_start = i; + while i < bytes.len() { + let b = bytes[i]; + let esc = match b { + b'\\' | b'*' | b'`' | b'[' | b']' => true, + b'_' => { + let prev_alnum = i > 0 && bytes[i - 1].is_ascii_alphanumeric(); + let next_alnum = i + 1 < bytes.len() && bytes[i + 1].is_ascii_alphanumeric(); + !(prev_alnum && next_alnum) + } + b'<' => { + i + 1 < bytes.len() + && (bytes[i + 1].is_ascii_alphabetic() + || matches!(bytes[i + 1], b'/' | b'!' | b'?')) + } + _ => false, + }; + if esc { + out.push_str(&text[run_start..i]); + out.push('\\'); + run_start = i; + } + i += 1; + } + out.push_str(&text[run_start..]); +} + +/// Handles turndown's `^`-anchored escapes. Returns how many input bytes +/// were consumed (always a prefix of ASCII digits / `=`). +fn escape_line_start(text: &str, out: &mut String) -> usize { + let bytes = text.as_bytes(); + match bytes[0] { + b'-' | b'>' => { + out.push('\\'); + 0 + } + b'+' if bytes.get(1) == Some(&b' ') => { + out.push('\\'); + 0 + } + b'=' => { + // `/^(=+)/` → `\$1` + out.push('\\'); + let n = bytes.iter().take_while(|&&b| b == b'=').count(); + out.push_str(&text[..n]); + n + } + b'#' => { + // `/^(#{1,6}) /` + let n = bytes.iter().take_while(|&&b| b == b'#').count(); + if n <= 6 && bytes.get(n) == Some(&b' ') { + out.push('\\'); + } + 0 + } + b'~' if bytes.starts_with(b"~~~") => { + out.push('\\'); + 0 + } + b'0'..=b'9' => { + // `/^(\d+)\. /` → `$1\. ` + let n = bytes.iter().take_while(|b| b.is_ascii_digit()).count(); + if bytes.get(n) == Some(&b'.') && bytes.get(n + 1) == Some(&b' ') { + out.push_str(&text[..n]); + out.push_str("\\."); + n + 1 + } else { + 0 + } + } + _ => 0, + } +} + +#[inline] +pub(crate) fn needs_escape_scan(text: &str, at_line_start: bool) -> bool { + let bytes = text.as_bytes(); + if bytes.is_empty() { + return false; + } + if at_line_start + && matches!( + bytes[0], + b'-' | b'>' | b'+' | b'=' | b'#' | b'~' | b'0'..=b'9' + ) + { + return true; + } + strings::index_of_any(bytes, b"\\*`[]_<").is_some() +} diff --git a/src/md/from_html/whitespace.rs b/src/md/from_html/whitespace.rs new file mode 100644 index 000000000000..8d223ab3c320 --- /dev/null +++ b/src/md/from_html/whitespace.rs @@ -0,0 +1,165 @@ +//! The whitespace-collapsing pre-pass turndown runs over the DOM before +//! converting (its `collapse-whitespace` module): runs of ASCII whitespace in +//! text become single spaces, spaces that would double up across inline +//! element boundaries are dropped, and text is trimmed against block +//! boundaries and `
    `. `
    ` subtrees are left untouched.
    +//!
    +//! This is a direct transcription of that state machine (including the way
    +//! it re-visits an element when the walk ascends back through it) so the
    +//! resulting text nodes match what turndown's rules later see.
    +
    +use core::ptr;
    +
    +use bun_core::strings;
    +use html5ever::tendril::StrTendril;
    +
    +use super::dom::{NodeData, Ref, Tag};
    +
    +/// Elements whose children the walk never enters: `
    ` (whitespace is
    +/// significant) and the subtrees the converter drops wholesale, so their
    +/// text cannot perturb spacing of the surrounding prose.
    +#[inline]
    +fn is_opaque(node: Ref<'_>) -> bool {
    +    let tag = node.tag();
    +    tag == Tag::Pre || tag.is_skipped()
    +}
    +
    +pub(crate) fn collapse_whitespace(root: Ref<'_>) {
    +    if root.first_child.get().is_none() || is_opaque(root) {
    +        return;
    +    }
    +
    +    let mut prev_text: Option> = None;
    +    let mut keep_leading_ws = false;
    +    let mut prev: Option> = None;
    +    let mut node = next(prev, root);
    +
    +    while !ptr::eq(node, root) {
    +        match &node.data {
    +            NodeData::Text(cell) => {
    +                let mut data = cell.borrow_mut();
    +                let strip_leading = !keep_leading_ws
    +                    && prev_text.is_none_or(|p| p.as_text().unwrap().borrow().ends_with(' '));
    +                collapse_text(&mut data, strip_leading);
    +                if data.is_empty() {
    +                    drop(data);
    +                    node = remove(node);
    +                    continue;
    +                }
    +                drop(data);
    +                prev_text = Some(node);
    +            }
    +            NodeData::Element { tag, .. } => {
    +                let tag = *tag;
    +                if tag.is_skipped() {
    +                    // Dropped subtrees behave as if absent.
    +                } else if tag.is_block() || tag == Tag::Br {
    +                    if let Some(p) = prev_text {
    +                        trim_trailing_space(p);
    +                    }
    +                    prev_text = None;
    +                    keep_leading_ws = false;
    +                } else if tag.is_void() || tag == Tag::Pre {
    +                    // Keep the space around inline voids (``) and inline `
    `.
    +                    prev_text = None;
    +                    keep_leading_ws = true;
    +                } else if prev_text.is_some() {
    +                    keep_leading_ws = false;
    +                }
    +            }
    +            NodeData::Document | NodeData::Ignored => {
    +                node = remove(node);
    +                continue;
    +            }
    +        }
    +        let next_node = next(prev, node);
    +        prev = Some(node);
    +        node = next_node;
    +    }
    +
    +    if let Some(p) = prev_text {
    +        trim_trailing_space(p);
    +        if p.as_text().unwrap().borrow().is_empty() {
    +            p.detach();
    +        }
    +    }
    +}
    +
    +/// turndown's `next(prev, current)`: pre-order, except that arriving back at
    +/// a node from its child (or sitting on an opaque node) moves on to the next
    +/// sibling / parent instead of descending.
    +fn next<'a>(prev: Option>, current: Ref<'a>) -> Ref<'a> {
    +    let came_from_child =
    +        prev.is_some_and(|p| p.parent.get().is_some_and(|pp| ptr::eq(pp, current)));
    +    if came_from_child || is_opaque(current) {
    +        return current
    +            .next_sibling
    +            .get()
    +            .or_else(|| current.parent.get())
    +            .expect("walk is bounded by root");
    +    }
    +    current
    +        .first_child
    +        .get()
    +        .or_else(|| current.next_sibling.get())
    +        .or_else(|| current.parent.get())
    +        .expect("walk is bounded by root")
    +}
    +
    +fn remove<'a>(node: Ref<'a>) -> Ref<'a> {
    +    let next = node
    +        .next_sibling
    +        .get()
    +        .or_else(|| node.parent.get())
    +        .expect("removed node has a parent");
    +    node.detach();
    +    next
    +}
    +
    +fn trim_trailing_space(text_node: Ref<'_>) {
    +    let cell = text_node.as_text().unwrap();
    +    let mut t = cell.borrow_mut();
    +    if t.ends_with(' ') {
    +        let n = t.len32() - 1;
    +        *t = t.subtendril(0, n);
    +    }
    +}
    +
    +/// `text.replace(/[ \r\n\t]+/g, ' ')`, then drop one leading space if
    +/// `strip_leading`.
    +fn collapse_text(t: &mut StrTendril, strip_leading: bool) {
    +    let text: &str = t;
    +    let bytes = text.as_bytes();
    +    // Most text nodes in real markup are either clean already or are the
    +    // indentation between tags; both are settled by two SIMD scans.
    +    let needs_rewrite =
    +        strings::index_of_any(bytes, b"\t\n\r").is_some() || strings::contains(bytes, b"  ");
    +
    +    if needs_rewrite {
    +        let mut out = String::with_capacity(bytes.len());
    +        let mut rest = text;
    +        while !rest.is_empty() {
    +            match strings::index_of_any(rest.as_bytes(), b" \t\n\r") {
    +                Some(i) => {
    +                    out.push_str(&rest[..i]);
    +                    out.push(' ');
    +                    let run = rest.as_bytes()[i..]
    +                        .iter()
    +                        .take_while(|&&b| matches!(b, b' ' | b'\t' | b'\n' | b'\r'))
    +                        .count();
    +                    rest = &rest[i + run..];
    +                }
    +                None => {
    +                    out.push_str(rest);
    +                    break;
    +                }
    +            }
    +        }
    +        *t = StrTendril::from(out);
    +    }
    +
    +    if strip_leading && t.starts_with(' ') {
    +        let n = t.len32();
    +        *t = t.subtendril(1, n - 1);
    +    }
    +}
    diff --git a/src/md/lib.rs b/src/md/lib.rs
    index 38d0c83b296c..ada76516f652 100644
    --- a/src/md/lib.rs
    +++ b/src/md/lib.rs
    @@ -6,6 +6,7 @@ pub mod autolinks;
     pub(crate) mod blocks;
     pub(crate) mod containers;
     pub(crate) mod entity;
    +pub mod from_html;
     pub mod helpers;
     pub mod html_renderer;
     pub mod inlines;
    diff --git a/src/runtime/api/MarkdownObject.rs b/src/runtime/api/MarkdownObject.rs
    index f8d50c54a3ce..87bba138b7df 100644
    --- a/src/runtime/api/MarkdownObject.rs
    +++ b/src/runtime/api/MarkdownObject.rs
    @@ -83,10 +83,192 @@ pub(crate) fn create(global_this: &JSGlobalObject) -> JSValue {
                 ("ansi", __jsc_host_render_to_ansi, 2),
                 ("render", __jsc_host_render, 3),
                 ("react", __jsc_host_render_react, 3),
    +            ("fromHTML", __jsc_host_from_html, 2),
             ],
         )
     }
     
    +/// `Bun.markdown.fromHTML(html, options?)` — convert an HTML document or
    +/// fragment to Markdown. `options` picks between equivalent Markdown
    +/// spellings (`{ headingStyle, hr, bulletListMarker, codeBlockStyle, fence,
    +/// emDelimiter, strongDelimiter, br }`) and toggles the GFM extensions
    +/// (`{ tables, strikethrough, tasklists }`, all on by default).
    +#[bun_jsc::host_fn]
    +fn from_html(global_this: &JSGlobalObject, callframe: &CallFrame) -> JsResult {
    +    use bun_md::from_html as h;
    +
    +    let [input_value, opts_value] = callframe.arguments_as_array::<2>();
    +
    +    if input_value.is_empty_or_undefined_or_null() {
    +        return Err(global_this
    +            .throw_invalid_arguments(format_args!("Expected a string or buffer to convert")));
    +    }
    +    let Some(buffer) = StringOrBuffer::from_js(global_this, input_value)? else {
    +        return Err(global_this
    +            .throw_invalid_arguments(format_args!("Expected a string or buffer to convert")));
    +    };
    +    // The conversion never re-enters JS, so a buffer input needs no pinning.
    +    let bytes: &[u8] = buffer.slice();
    +    if bytes.len() > h::MAX_INPUT_LEN {
    +        return Err(global_this.throw_range_error(
    +            bytes.len() as i64,
    +            RangeErrorOptions {
    +                max: h::MAX_INPUT_LEN as i64,
    +                field_name: b"input.byteLength",
    +                ..Default::default()
    +            },
    +        ));
    +    }
    +
    +    let mut options = h::Options::default();
    +    if opts_value.is_object() {
    +        if let Some(v) = opts_value.get_optional_enum_from_map(
    +            global_this,
    +            "headingStyle",
    +            &from_html_options::HEADING_STYLE,
    +            "\"atx\" or \"setext\"",
    +        )? {
    +            options.heading_style = v;
    +        }
    +        if let Some(v) = opts_value.get_optional_enum_from_map(
    +            global_this,
    +            "hr",
    +            &from_html_options::HR,
    +            "\"---\", \"***\", \"___\", \"- - -\", \"* * *\", or \"_ _ _\"",
    +        )? {
    +            options.thematic_break = v;
    +        }
    +        if let Some(v) = opts_value.get_optional_enum_from_map(
    +            global_this,
    +            "bulletListMarker",
    +            &from_html_options::BULLET,
    +            "\"-\", \"*\", or \"+\"",
    +        )? {
    +            options.bullet_list_marker = v;
    +        }
    +        if let Some(v) = opts_value.get_optional_enum_from_map(
    +            global_this,
    +            "codeBlockStyle",
    +            &from_html_options::CODE_BLOCK_STYLE,
    +            "\"fenced\" or \"indented\"",
    +        )? {
    +            options.code_block_style = v;
    +        }
    +        if let Some(v) = opts_value.get_optional_enum_from_map(
    +            global_this,
    +            "fence",
    +            &from_html_options::FENCE,
    +            "\"```\" or \"~~~\"",
    +        )? {
    +            options.fence = v;
    +        }
    +        if let Some(v) = opts_value.get_optional_enum_from_map(
    +            global_this,
    +            "emDelimiter",
    +            &from_html_options::EM,
    +            "\"_\" or \"*\"",
    +        )? {
    +            options.em_delimiter = v;
    +        }
    +        if let Some(v) = opts_value.get_optional_enum_from_map(
    +            global_this,
    +            "strongDelimiter",
    +            &from_html_options::STRONG,
    +            "\"**\" or \"__\"",
    +        )? {
    +            options.strong_delimiter = v;
    +        }
    +        if let Some(v) = opts_value.get_optional_enum_from_map(
    +            global_this,
    +            "br",
    +            &from_html_options::BR,
    +            "\"  \" (two spaces) or \"\\\\\"",
    +        )? {
    +            options.line_break = v;
    +        }
    +        if let Some(v) = opts_value.get_boolean_loose(global_this, "tables")? {
    +            options.tables = v;
    +        }
    +        if let Some(v) = opts_value.get_boolean_loose(global_this, "strikethrough")? {
    +            options.strikethrough = v;
    +        }
    +        if let Some(v) = opts_value.get_boolean_loose(global_this, "tasklists")? {
    +            options.tasklists = v;
    +        }
    +    }
    +
    +    // html5ever consumes `&str`. JS strings arrive as valid UTF-8; a raw
    +    // buffer may not be, in which case invalid sequences become U+FFFD —
    +    // what a browser does when it decodes the same bytes as UTF-8.
    +    let markdown = if bun_core::strings::is_valid_utf8(bytes) {
    +        // SAFETY: validated on the line above.
    +        h::convert(unsafe { core::str::from_utf8_unchecked(bytes) }, &options)
    +    } else {
    +        #[allow(clippy::disallowed_methods)]
    +        let lossy = String::from_utf8_lossy(bytes);
    +        h::convert(&lossy, &options)
    +    };
    +
    +    bun_string_jsc::owned_utf8_into_js(global_this, markdown.into_bytes())
    +}
    +
    +mod from_html_options {
    +    use bun_md::from_html::*;
    +    bun_core::comptime_string_map! {
    +        pub(super) static HEADING_STYLE: HeadingStyle = {
    +            b"atx" => HeadingStyle::Atx,
    +            b"setext" => HeadingStyle::Setext,
    +        };
    +    }
    +    bun_core::comptime_string_map! {
    +        pub(super) static HR: ThematicBreak = {
    +            b"---" => ThematicBreak::Dashes,
    +            b"***" => ThematicBreak::Asterisks,
    +            b"___" => ThematicBreak::Underscores,
    +            b"- - -" => ThematicBreak::SpacedDashes,
    +            b"* * *" => ThematicBreak::SpacedAsterisks,
    +            b"_ _ _" => ThematicBreak::SpacedUnderscores,
    +        };
    +    }
    +    bun_core::comptime_string_map! {
    +        pub(super) static BULLET: BulletListMarker = {
    +            b"-" => BulletListMarker::Dash,
    +            b"*" => BulletListMarker::Asterisk,
    +            b"+" => BulletListMarker::Plus,
    +        };
    +    }
    +    bun_core::comptime_string_map! {
    +        pub(super) static CODE_BLOCK_STYLE: CodeBlockStyle = {
    +            b"fenced" => CodeBlockStyle::Fenced,
    +            b"indented" => CodeBlockStyle::Indented,
    +        };
    +    }
    +    bun_core::comptime_string_map! {
    +        pub(super) static FENCE: Fence = {
    +            b"```" => Fence::Backticks,
    +            b"~~~" => Fence::Tildes,
    +        };
    +    }
    +    bun_core::comptime_string_map! {
    +        pub(super) static EM: EmDelimiter = {
    +            b"_" => EmDelimiter::Underscore,
    +            b"*" => EmDelimiter::Asterisk,
    +        };
    +    }
    +    bun_core::comptime_string_map! {
    +        pub(super) static STRONG: StrongDelimiter = {
    +            b"**" => StrongDelimiter::Asterisks,
    +            b"__" => StrongDelimiter::Underscores,
    +        };
    +    }
    +    bun_core::comptime_string_map! {
    +        pub(super) static BR: LineBreak = {
    +            b"  " => LineBreak::Spaces,
    +            b"\\" => LineBreak::Backslash,
    +        };
    +    }
    +}
    +
     /// `bun:internal-for-testing`'s `setMaxMarkdownBlockBytesForTesting(limit)`:
     /// shrink the parser's block-metadata cap so its `TooManyBlocks` error is
     /// testable without 4 GiB of input. Returns the previous limit.
    diff --git a/test/bun.lock b/test/bun.lock
    index 0dcc5c65ac03..1d3e78dffef0 100644
    --- a/test/bun.lock
    +++ b/test/bun.lock
    @@ -95,6 +95,8 @@
             "svelte": "5.20.4",
             "tsyringe": "4.8.0",
             "tunnel": "0.0.6",
    +        "turndown": "7.2.4",
    +        "turndown-plugin-gfm": "1.0.2",
             "type-graphql": "2.0.0-rc.2",
             "typeorm": "0.3.20",
             "typescript": "6.0.2",
    @@ -425,6 +427,8 @@
     
         "@lukeed/csprng": ["@lukeed/csprng@1.1.0", "", {}, "sha512-Z7C/xXCiGWsg0KuKsHTKJxbWhpI3Vs5GwLfOean7MGyVFGqdRgBbAjOCh6u4bbjPc/8MJ2pZmK/0DLdCbivLDA=="],
     
    +    "@mixmark-io/domino": ["@mixmark-io/domino@2.2.0", "", {}, "sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw=="],
    +
         "@mongodb-js/saslprep": ["@mongodb-js/saslprep@1.1.7", "", { "dependencies": { "sparse-bitfield": "^3.0.3" } }, "sha512-dCHW/oEX0KJ4NjDULBo3JiOaK5+6axtpBbS+ao2ZInoAL9/YRQLhXzSNAFz7hP4nzLkIqsfYAK/PDE3+XHny0Q=="],
     
         "@msgpackr-extract/msgpackr-extract-darwin-arm64": ["@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.2", "", { "os": "darwin", "cpu": "arm64" }, "sha512-9bfjwDxIDWmmOKusUcqdS4Rw+SETlp9Dy39Xui9BEGEk19dDwH0jhipwFzEff/pFg95NKymc6TOTbRKcWeRqyQ=="],
    @@ -2377,6 +2381,10 @@
     
         "turbo-stream": ["turbo-stream@2.2.0", "", {}, "sha512-FKFg7A0To1VU4CH9YmSMON5QphK0BXjSoiC7D9yMh+mEEbXLUP9qJ4hEt1qcjKtzncs1OpcnjZO8NgrlVbZH+g=="],
     
    +    "turndown": ["turndown@7.2.4", "", { "dependencies": { "@mixmark-io/domino": "^2.2.0" } }, "sha512-I8yFsfRzmzK0WV1pNNOA4A7y4RDfFxPRxb3t+e3ui14qSGOxGtiSP6GjeX+Y6CHb7HYaFj7ECUD7VE5kQMZWGQ=="],
    +
    +    "turndown-plugin-gfm": ["turndown-plugin-gfm@1.0.2", "", {}, "sha512-vwz9tfvF7XN/jE0dGoBei3FXWuvll78ohzCZQuOb+ZjWrs3a0XhQVomJEb2Qh4VHTPNRO4GPZh0V7VRbiWwkRg=="],
    +
         "tweetnacl": ["tweetnacl@0.14.5", "", {}, "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA=="],
     
         "typanion": ["typanion@3.14.0", "", {}, "sha512-ZW/lVMRabETuYCd9O9ZvMhAh8GslSqaUjxmK/JLPCh6l73CvLBiuXswj/+7LdnWOgYsQ130FqLzFz5aGT4I3Ug=="],
    diff --git a/test/integration/bun-types/fixture/bun.ts b/test/integration/bun-types/fixture/bun.ts
    index eb51a0135022..6b371f2ec3c7 100644
    --- a/test/integration/bun-types/fixture/bun.ts
    +++ b/test/integration/bun-types/fixture/bun.ts
    @@ -93,3 +93,24 @@ tsd
     
     tsd.expectType(Bun.mmap("./data.bin", { offset: 4096 })).is>();
     tsd.expectType(Bun.mmap("./data.bin", { size: 1024 })).is>();
    +
    +tsd.expectType(Bun.markdown.fromHTML("

    hi

    ")).is(); +tsd + .expectType( + Bun.markdown.fromHTML(new TextEncoder().encode("

    x

    "), { + headingStyle: "setext", + hr: "* * *", + bulletListMarker: "*", + codeBlockStyle: "indented", + fence: "~~~", + emDelimiter: "*", + strongDelimiter: "__", + br: "\\", + tables: false, + strikethrough: false, + tasklists: false, + }), + ) + .is(); +// @ts-expect-error - only the two-space and backslash spellings exist +Bun.markdown.fromHTML("
    ", { br: " " }); diff --git a/test/js/bun/md/fixtures/bun-docs-markdown.html b/test/js/bun/md/fixtures/bun-docs-markdown.html new file mode 100644 index 000000000000..7808c64cf7d0 --- /dev/null +++ b/test/js/bun/md/fixtures/bun-docs-markdown.html @@ -0,0 +1,128 @@ +Markdown | Bun Docs
    Skip to content

    Markdown

    Parse and render Markdown with Bun's built-in Markdown API, supporting GFM extensions and custom rendering callbacks

    Unstable API — This API is under active development and may change in future versions of Bun.

    Bun includes a fast, built-in Markdown parser written in Rust. It supports GitHub Flavored Markdown (GFM) extensions and provides three APIs:

    • Bun.markdown.html() — render Markdown to an HTML string
    • Bun.markdown.render() — render Markdown with custom callbacks for each element
    • Bun.markdown.react() — render Markdown to React JSX elements

    Bun.markdown.html()#

    Convert a Markdown string to HTML.

    const html = Bun.markdown.html("# Hello **world**");
    +// "<h1>Hello <strong>world</strong></h1>\n"

    GFM extensions like tables, strikethrough, and task lists are enabled by default:

    const html = Bun.markdown.html(`
    +| Feature      | Status |
    +|-------------|--------|
    +| Tables       | ~~done~~ |
    +| Strikethrough| ~~done~~ |
    +| Task lists   | done |
    +`);

    Options#

    Pass an options object as the second argument to configure the parser:

    const html = Bun.markdown.html("some markdown", {
    +  tables: true, // GFM tables (default: true)
    +  strikethrough: true, // GFM strikethrough (default: true)
    +  tasklists: true, // GFM task lists (default: true)
    +  tagFilter: true, // GFM tag filter for disallowed HTML tags
    +  autolinks: true, // Autolink URLs, emails, and www. links
    +});

    All available options:

    OptionDefaultDescription
    tablestrueGFM tables
    strikethroughtrueGFM strikethrough (~~text~~)
    taskliststrueGFM task lists (- [x] item)
    autolinksfalseEnable autolinks — see Autolinks
    headingsfalseHeading IDs and autolinks — see Heading IDs
    hardSoftBreaksfalseTreat soft line breaks as hard breaks
    wikiLinksfalseEnable [[wiki links]]
    underlinefalse__text__ renders as <u> instead of <strong>
    latexMathfalseEnable $inline$ and $$display$$ math
    collapseWhitespacefalseCollapse whitespace in text
    permissiveAtxHeadersfalseATX headers without space after #
    noIndentedCodeBlocksfalseDisable indented code blocks
    noHtmlBlocksfalseDisable HTML blocks
    noHtmlSpansfalseDisable inline HTML
    tagFilterfalseGFM tag filter for disallowed HTML tags

    Autolinks#

    Pass true to enable all autolink types, or an object for granular control:

    // Enable all autolinks (URL, WWW, email)
    +Bun.markdown.html("Visit www.example.com", { autolinks: true });
    +
    +// Enable only specific types
    +Bun.markdown.html("Visit www.example.com", {
    +  autolinks: { url: true, www: true },
    +});

    Heading IDs#

    Pass true to enable both heading IDs and autolink headings, or an object for granular control:

    // Enable heading IDs and autolink headings
    +Bun.markdown.html("## Hello World", { headings: true });
    +// '<h2 id="hello-world"><a href="#hello-world">Hello World</a></h2>\n'
    +
    +// Enable only heading IDs (no autolink)
    +Bun.markdown.html("## Hello World", { headings: { ids: true } });
    +// '<h2 id="hello-world">Hello World</h2>\n'

    Bun.markdown.render()#

    Parse Markdown and render it using custom JavaScript callbacks. The callbacks give you full control over the output format. You can generate HTML with custom classes, ANSI terminal output, or any other string format.

    const result = Bun.markdown.render("# Hello **world**", {
    +  heading: (children, { level }) => `<h${level} class="title">${children}</h${level}>`,
    +  strong: children => `<b>${children}</b>`,
    +  paragraph: children => `<p>${children}</p>`,
    +});
    +// '<h1 class="title">Hello <b>world</b></h1>'

    Callback signature#

    Each callback receives:

    1. children — the accumulated content of the element as a string
    2. meta (optional) — an object with element-specific metadata

    Return a string to replace the element's rendering. Return null or undefined to omit the element from the output entirely. If an element has no callback, its children pass through unchanged.

    Block callbacks#

    CallbackMetaDescription
    heading{ level, id? }Heading level 1–6. id is set when headings: { ids: true } is enabled
    paragraph—Paragraph block
    blockquote—Blockquote block
    code{ language? }Fenced or indented code block. language is the info-string when specified on the fence
    list{ ordered, start?, depth }depth is nesting level (0 = top-level). start is set for ordered lists
    listItem{ index, depth, ordered, start?, checked? }See List item meta below
    hr—Horizontal rule
    table—Table block
    thead—Table head
    tbody—Table body
    tr—Table row
    th{ align? }Table header cell. align is "left", "center", "right", or absent
    td{ align? }Table data cell
    html—Raw HTML content

    List item meta#

    The listItem callback receives everything needed to render markers directly:

    • index — 0-based position within the parent list
    • depth — the parent list's nesting level (0 = top-level)
    • ordered — whether the parent list is ordered
    • start — the parent list's start number (only when ordered is true)
    • checked — task list state (only for - [x] / - [ ] items)

    Inline callbacks#

    CallbackMetaDescription
    strong—Strong emphasis (**text**)
    emphasis—Emphasis (*text*)
    link{ href: string, title?: string }Link
    image{ src: string, title?: string }Image
    codespan—Inline code (`code`)
    strikethrough—Strikethrough (~~text~~)
    text—Plain text content

    Examples#

    Custom HTML with classes#

    const html = Bun.markdown.render("# Title\n\nHello **world**", {
    +  heading: (children, { level }) => `<h${level} class="heading heading-${level}">${children}</h${level}>`,
    +  paragraph: children => `<p class="body">${children}</p>`,
    +  strong: children => `<strong class="bold">${children}</strong>`,
    +});

    Stripping all formatting#

    const plaintext = Bun.markdown.render("# Hello **world**", {
    +  heading: children => children,
    +  paragraph: children => children,
    +  strong: children => children,
    +  emphasis: children => children,
    +  link: children => children,
    +  image: () => "",
    +  code: children => children,
    +  codespan: children => children,
    +});
    +// "Hello world"

    Omitting elements#

    Return null or undefined to remove an element from the output:

    const result = Bun.markdown.render("# Title\n\n![logo](img.png)\n\nHello", {
    +  image: () => null, // Remove all images
    +  heading: children => children,
    +  paragraph: children => children + "\n",
    +});
    +// "Title\nHello\n"

    ANSI terminal output#

    const ansi = Bun.markdown.render("# Hello\n\nThis is **bold** and *italic*", {
    +  heading: (children, { level }) => `\x1b[1;4m${children}\x1b[0m\n`,
    +  paragraph: children => children + "\n",
    +  strong: children => `\x1b[1m${children}\x1b[22m`,
    +  emphasis: children => `\x1b[3m${children}\x1b[23m`,
    +});

    Nested list numbering#

    The listItem callback receives everything needed to render markers directly — no post-processing:

    const result = Bun.markdown.render("1. first\n   1. sub-a\n   2. sub-b\n2. second", {
    +  listItem: (children, { index, depth, ordered, start }) => {
    +    const n = (start ?? 1) + index;
    +    // 1. 2. 3. at depth 0, a. b. c. at depth 1, i. ii. iii. at depth 2
    +    const marker = !ordered
    +      ? "-"
    +      : depth === 0
    +        ? `${n}.`
    +        : depth === 1
    +          ? `${String.fromCharCode(96 + n)}.`
    +          : `${toRoman(n)}.`;
    +    return "  ".repeat(depth) + marker + " " + children.trimEnd() + "\n";
    +  },
    +  // Prepend a newline so nested lists are separated from their parent item's text
    +  list: children => "\n" + children,
    +});
    +// 1. first
    +//   a. sub-a
    +//   b. sub-b
    +// 2. second

    Code block syntax highlighting#

    const result = Bun.markdown.render("```js\nconsole.log('hi')\n```", {
    +  code: (children, meta) => {
    +    const lang = meta?.language ?? "";
    +    return `<pre><code class="language-${lang}">${children}</code></pre>`;
    +  },
    +});

    Parser options#

    Pass parser options as a separate third argument:

    const result = Bun.markdown.render(
    +  "Visit www.example.com",
    +  {
    +    link: (children, { href }) => `[${children}](${href})`,
    +    paragraph: children => children,
    +  },
    +  { autolinks: true },
    +);

    Bun.markdown.react()#

    Render Markdown directly to React elements. Returns a <Fragment> that you can use as a component return value.

    function Markdown({ text }: { text: string }) {
    +  return Bun.markdown.react(text);
    +}

    Server-side rendering#

    Works with renderToString() and React Server Components:

    import { renderToString } from "react-dom/server";
    +
    +const html = renderToString(Bun.markdown.react("# Hello **world**"));
    +// "<h1>Hello <strong>world</strong></h1>"

    Component overrides#

    Replace any HTML element with a custom React component by passing it in the second argument, keyed by tag name:

    function Code({ language, children }) {
    +  return (
    +    <pre data-language={language}>
    +      <code>{children}</code>
    +    </pre>
    +  );
    +}
    +
    +function Link({ href, title, children }) {
    +  return (
    +    <a href={href} title={title} target="_blank" rel="noopener noreferrer">
    +      {children}
    +    </a>
    +  );
    +}
    +
    +function Heading({ id, children }) {
    +  return (
    +    <h2 id={id}>
    +      <a href={`#${id}`}>{children}</a>
    +    </h2>
    +  );
    +}
    +
    +const el = Bun.markdown.react(
    +  content,
    +  {
    +    pre: Code,
    +    a: Link,
    +    h2: Heading,
    +  },
    +  { headings: { ids: true } },
    +);

    Available overrides#

    You can override every HTML tag the parser produces:

    OptionPropsDescription
    h1–h6{ id?, children }Headings. id is set when headings: { ids: true } is enabled
    p{ children }Paragraph
    blockquote{ children }Blockquote
    pre{ language?, children }Code block. language is the info string (e.g. "js")
    hr{}Horizontal rule (no children)
    ul{ children }Unordered list
    ol{ start, children }Ordered list. start is the first item number
    li{ checked?, children }List item. checked is set for task list items
    table{ children }Table
    thead{ children }Table head
    tbody{ children }Table body
    tr{ children }Table row
    th{ align?, children }Table header cell
    td{ align?, children }Table data cell
    em{ children }Emphasis (*text*)
    strong{ children }Strong (**text**)
    a{ href, title?, children }Link
    img{ src, alt?, title? }Image (no children)
    code{ children }Inline code
    del{ children }Strikethrough (~~text~~)
    br{}Hard line break (no children)

    React 18 and older#

    By default, elements use Symbol.for('react.transitional.element') as the $$typeof symbol. For React 18 and older, pass reactVersion: 18 in the options (third argument):

    function Markdown({ text }: { text: string }) {
    +  return Bun.markdown.react(text, undefined, { reactVersion: 18 });
    +}

    Parser options#

    Pass any of the parser options as the third argument:

    const el = Bun.markdown.react("## Hello World", undefined, {
    +  headings: { ids: true },
    +  autolinks: true,
    +});
    \ No newline at end of file diff --git a/test/js/bun/md/fixtures/bun-docs-markdown.turndown.md b/test/js/bun/md/fixtures/bun-docs-markdown.turndown.md new file mode 100644 index 000000000000..d29a096a6b5f --- /dev/null +++ b/test/js/bun/md/fixtures/bun-docs-markdown.turndown.md @@ -0,0 +1,528 @@ +[Skip to content](#main) + +[![](/logo.svg)](/) + +[Docs](/docs)[Guides](/guides)[Reference](/reference)[Blog](/blog) + +Search docs, APIs, guides…/ + +[](https://github.com/oven-sh/bun)[](/discord)[Install](/get) + +[![](/logo.svg)](/) + +[Home](/)[Docs](/docs)[Guides](/guides)[Reference](/reference)[Blog](/blog) + +[Install Bun v1.4.2](/get) + +[](https://github.com/oven-sh/bun)[](/discord)[](https://x.com/bunjavascript) + +[Runtime](/docs)[Package Manager](/docs/pm/cli/install)[Bundler](/docs/bundler)[Test Runner](/docs/test)[Guides](/guides)[Reference](/reference)[Blog](/blog)[Feedback](/docs/feedback) + +Browse docs + +Runtime + +- Get Started + + - [Welcome to Bun](/docs) + - [Installation](/docs/installation) + - [Quickstart](/docs/quickstart) + - [TypeScript](/docs/typescript) + - [TypeScript 6 and 7](/docs/typescript-6) + - [bun init](/docs/runtime/templating/init) + - [bun create](/docs/runtime/templating/create) +- Core Runtime + + - [Bun Runtime](/docs/runtime) + - [Watch Mode](/docs/runtime/watch-mode) + - [Debugging](/docs/runtime/debugger) + - [REPL](/docs/runtime/repl) + - [bunfig.toml](/docs/runtime/bunfig) +- File & Module System + + - [File Types](/docs/runtime/file-types) + - [Module Resolution](/docs/runtime/module-resolution) + - [JSX](/docs/runtime/jsx) + - [Auto-install](/docs/runtime/auto-install) + - [Plugins](/docs/runtime/plugins) + - [File System Router](/docs/runtime/file-system-router) +- HTTP server + + - [Server](/docs/runtime/http/server) + - [Routing](/docs/runtime/http/routing) + - [Cookies](/docs/runtime/http/cookies) + - [TLS](/docs/runtime/http/tls) + - [Error Handling](/docs/runtime/http/error-handling) + - [Metrics](/docs/runtime/http/metrics) +- Networking + + - [Fetch](/docs/runtime/networking/fetch) + - [WebSockets](/docs/runtime/http/websockets) + - [TCP](/docs/runtime/networking/tcp) + - [UDP](/docs/runtime/networking/udp) + - [DNS](/docs/runtime/networking/dns) +- Data & Storage + + - [Cookies](/docs/runtime/cookies) + - [File I/O](/docs/runtime/file-io) + - [Streams](/docs/runtime/streams) + - [Binary Data](/docs/runtime/binary-data) + - [Archive](/docs/runtime/archive) + - [SQL](/docs/runtime/sql) + - [SQLite](/docs/runtime/sqlite) + - [S3](/docs/runtime/s3) + - [Redis](/docs/runtime/redis) +- Concurrency + + - [Workers](/docs/runtime/workers) +- Process & System + + - [Environment Variables](/docs/runtime/environment-variables) + - [Shell](/docs/runtime/shell) + - [Spawn](/docs/runtime/child-process) + - [WebView](/docs/runtime/webview) + - [Cron](/docs/runtime/cron) +- Interop & Tooling + + - [Node-API](/docs/runtime/node-api) + - [FFI](/docs/runtime/ffi) + - [C Compiler](/docs/runtime/c-compiler) + - [Transpiler](/docs/runtime/transpiler) +- Utilities + + - [CSRF Protection](/docs/runtime/csrf) + - [Secrets](/docs/runtime/secrets) + - [Console](/docs/runtime/console) + - [TOML](/docs/runtime/toml) + - [YAML](/docs/runtime/yaml) + - [Markdown](/docs/runtime/markdown) + - [JSON5](/docs/runtime/json5) + - [XML](/docs/runtime/xml) + - [JSONL](/docs/runtime/jsonl) + - [HTMLRewriter](/docs/runtime/html-rewriter) + - [Image](/docs/runtime/image) + - [Hashing](/docs/runtime/hashing) + - [Glob](/docs/runtime/glob) + - [Semver](/docs/runtime/semver) + - [Color](/docs/runtime/color) + - [Utils](/docs/runtime/utils) +- Standards & Compatibility + + - [Globals](/docs/runtime/globals) + - [Bun APIs](/docs/runtime/bun-apis) + - [Web APIs](/docs/runtime/web-apis) + - [Node.js Compatibility](/docs/runtime/nodejs-compat) +- Contributing + + - [Roadmap](/docs/project/roadmap) + - [Benchmarking](/docs/project/benchmarking) + - [Contributing](/docs/project/contributing) + - [Building Windows](/docs/project/building-windows) + - [Bindgen](/docs/project/bindgen) + - [License](/docs/project/license) + +1. [Runtime](/docs) +2. ›Utilities + +Copy page + +# Markdown + +Parse and render Markdown with Bun's built-in Markdown API, supporting GFM extensions and custom rendering callbacks + +**Unstable API** — This API is under active development and may change in future versions of Bun. + +Bun includes a fast, built-in Markdown parser written in Rust. It supports GitHub Flavored Markdown (GFM) extensions and provides three APIs: + +- `Bun.markdown.html()` — render Markdown to an HTML string +- `Bun.markdown.render()` — render Markdown with custom callbacks for each element +- `Bun.markdown.react()` — render Markdown to React JSX elements + +--- + +## `Bun.markdown.html()`[#](#bun-markdown-html) + +Convert a Markdown string to HTML. + +``` +const html = Bun.markdown.html("# Hello **world**"); +// "

    Hello world

    \n" +``` + +GFM extensions like tables, strikethrough, and task lists are enabled by default: + +``` +const html = Bun.markdown.html(` +| Feature | Status | +|-------------|--------| +| Tables | ~~done~~ | +| Strikethrough| ~~done~~ | +| Task lists | done | +`); +``` + +### Options[#](#options) + +Pass an options object as the second argument to configure the parser: + +``` +const html = Bun.markdown.html("some markdown", { + tables: true, // GFM tables (default: true) + strikethrough: true, // GFM strikethrough (default: true) + tasklists: true, // GFM task lists (default: true) + tagFilter: true, // GFM tag filter for disallowed HTML tags + autolinks: true, // Autolink URLs, emails, and www. links +}); +``` + +All available options: + +| Option | Default | Description | +| --- | --- | --- | +| `tables` | `true` | GFM tables | +| `strikethrough` | `true` | GFM strikethrough (`~~text~~`) | +| `tasklists` | `true` | GFM task lists (`- [x] item`) | +| `autolinks` | `false` | Enable autolinks — see [Autolinks](#autolinks) | +| `headings` | `false` | Heading IDs and autolinks — see [Heading IDs](#heading-ids) | +| `hardSoftBreaks` | `false` | Treat soft line breaks as hard breaks | +| `wikiLinks` | `false` | Enable `[[wiki links]]` | +| `underline` | `false` | `__text__` renders as `` instead of `` | +| `latexMath` | `false` | Enable `$inline$` and `$$display$$` math | +| `collapseWhitespace` | `false` | Collapse whitespace in text | +| `permissiveAtxHeaders` | `false` | ATX headers without space after `#` | +| `noIndentedCodeBlocks` | `false` | Disable indented code blocks | +| `noHtmlBlocks` | `false` | Disable HTML blocks | +| `noHtmlSpans` | `false` | Disable inline HTML | +| `tagFilter` | `false` | GFM tag filter for disallowed HTML tags | + +#### Autolinks[#](#autolinks) + +Pass `true` to enable all autolink types, or an object for granular control: + +``` +// Enable all autolinks (URL, WWW, email) +Bun.markdown.html("Visit www.example.com", { autolinks: true }); + +// Enable only specific types +Bun.markdown.html("Visit www.example.com", { + autolinks: { url: true, www: true }, +}); +``` + +#### Heading IDs[#](#heading-ids) + +Pass `true` to enable both heading IDs and autolink headings, or an object for granular control: + +``` +// Enable heading IDs and autolink headings +Bun.markdown.html("## Hello World", { headings: true }); +// '

    Hello World

    \n' + +// Enable only heading IDs (no autolink) +Bun.markdown.html("## Hello World", { headings: { ids: true } }); +// '

    Hello World

    \n' +``` + +--- + +## `Bun.markdown.render()`[#](#bun-markdown-render) + +Parse Markdown and render it using custom JavaScript callbacks. The callbacks give you full control over the output format. You can generate HTML with custom classes, ANSI terminal output, or any other string format. + +``` +const result = Bun.markdown.render("# Hello **world**", { + heading: (children, { level }) => `${children}`, + strong: children => `${children}`, + paragraph: children => `

    ${children}

    `, +}); +// '

    Hello world

    ' +``` + +### Callback signature[#](#callback-signature) + +Each callback receives: + +1. **`children`** — the accumulated content of the element as a string +2. **`meta`** (optional) — an object with element-specific metadata + +Return a string to replace the element's rendering. Return `null` or `undefined` to omit the element from the output entirely. If an element has no callback, its children pass through unchanged. + +### Block callbacks[#](#block-callbacks) + +| Callback | Meta | Description | +| --- | --- | --- | +| `heading` | `{ level, id? }` | Heading level 1–6. `id` is set when `headings: { ids: true }` is enabled | +| `paragraph` | — | Paragraph block | +| `blockquote` | — | Blockquote block | +| `code` | `{ language? }` | Fenced or indented code block. `language` is the info-string when specified on the fence | +| `list` | `{ ordered, start?, depth }` | `depth` is nesting level (0 = top-level). `start` is set for ordered lists | +| `listItem` | `{ index, depth, ordered, start?, checked? }` | See [List item meta](#list-item-meta) below | +| `hr` | — | Horizontal rule | +| `table` | — | Table block | +| `thead` | — | Table head | +| `tbody` | — | Table body | +| `tr` | — | Table row | +| `th` | `{ align? }` | Table header cell. `align` is `"left"`, `"center"`, `"right"`, or absent | +| `td` | `{ align? }` | Table data cell | +| `html` | — | Raw HTML content | + +#### List item meta[#](#list-item-meta) + +The `listItem` callback receives everything needed to render markers directly: + +- `index` — 0-based position within the parent list +- `depth` — the parent list's nesting level (0 = top-level) +- `ordered` — whether the parent list is ordered +- `start` — the parent list's start number (only when `ordered` is true) +- `checked` — task list state (only for `- [x]` / `- [ ]` items) + +### Inline callbacks[#](#inline-callbacks) + +| Callback | Meta | Description | +| --- | --- | --- | +| `strong` | — | Strong emphasis (`**text**`) | +| `emphasis` | — | Emphasis (`*text*`) | +| `link` | `{ href: string, title?: string }` | Link | +| `image` | `{ src: string, title?: string }` | Image | +| `codespan` | — | Inline code (`` `code` ``) | +| `strikethrough` | — | Strikethrough (`~~text~~`) | +| `text` | — | Plain text content | + +### Examples[#](#examples) + +#### Custom HTML with classes[#](#custom-html-with-classes) + +``` +const html = Bun.markdown.render("# Title\n\nHello **world**", { + heading: (children, { level }) => `${children}`, + paragraph: children => `

    ${children}

    `, + strong: children => `${children}`, +}); +``` + +#### Stripping all formatting[#](#stripping-all-formatting) + +``` +const plaintext = Bun.markdown.render("# Hello **world**", { + heading: children => children, + paragraph: children => children, + strong: children => children, + emphasis: children => children, + link: children => children, + image: () => "", + code: children => children, + codespan: children => children, +}); +// "Hello world" +``` + +#### Omitting elements[#](#omitting-elements) + +Return `null` or `undefined` to remove an element from the output: + +``` +const result = Bun.markdown.render("# Title\n\n![logo](img.png)\n\nHello", { + image: () => null, // Remove all images + heading: children => children, + paragraph: children => children + "\n", +}); +// "Title\nHello\n" +``` + +#### ANSI terminal output[#](#ansi-terminal-output) + +``` +const ansi = Bun.markdown.render("# Hello\n\nThis is **bold** and *italic*", { + heading: (children, { level }) => `\x1b[1;4m${children}\x1b[0m\n`, + paragraph: children => children + "\n", + strong: children => `\x1b[1m${children}\x1b[22m`, + emphasis: children => `\x1b[3m${children}\x1b[23m`, +}); +``` + +#### Nested list numbering[#](#nested-list-numbering) + +The `listItem` callback receives everything needed to render markers directly — no post-processing: + +``` +const result = Bun.markdown.render("1. first\n 1. sub-a\n 2. sub-b\n2. second", { + listItem: (children, { index, depth, ordered, start }) => { + const n = (start ?? 1) + index; + // 1. 2. 3. at depth 0, a. b. c. at depth 1, i. ii. iii. at depth 2 + const marker = !ordered + ? "-" + : depth === 0 + ? `${n}.` + : depth === 1 + ? `${String.fromCharCode(96 + n)}.` + : `${toRoman(n)}.`; + return " ".repeat(depth) + marker + " " + children.trimEnd() + "\n"; + }, + // Prepend a newline so nested lists are separated from their parent item's text + list: children => "\n" + children, +}); +// 1. first +// a. sub-a +// b. sub-b +// 2. second +``` + +#### Code block syntax highlighting[#](#code-block-syntax-highlighting) + +``` +const result = Bun.markdown.render("```js\nconsole.log('hi')\n```", { + code: (children, meta) => { + const lang = meta?.language ?? ""; + return `
    ${children}
    `; + }, +}); +``` + +### Parser options[#](#parser-options) + +Pass parser options as a separate third argument: + +``` +const result = Bun.markdown.render( + "Visit www.example.com", + { + link: (children, { href }) => `[${children}](${href})`, + paragraph: children => children, + }, + { autolinks: true }, +); +``` + +--- + +## `Bun.markdown.react()`[#](#bun-markdown-react) + +Render Markdown directly to React elements. Returns a `` that you can use as a component return value. + +``` +function Markdown({ text }: { text: string }) { + return Bun.markdown.react(text); +} +``` + +### Server-side rendering[#](#server-side-rendering) + +Works with `renderToString()` and React Server Components: + +``` +import { renderToString } from "react-dom/server"; + +const html = renderToString(Bun.markdown.react("# Hello **world**")); +// "

    Hello world

    " +``` + +### Component overrides[#](#component-overrides) + +Replace any HTML element with a custom React component by passing it in the second argument, keyed by tag name: + +``` +function Code({ language, children }) { + return ( +
    +      {children}
    +    
    + ); +} + +function Link({ href, title, children }) { + return ( + + {children} + + ); +} + +function Heading({ id, children }) { + return ( +

    + {children} +

    + ); +} + +const el = Bun.markdown.react( + content, + { + pre: Code, + a: Link, + h2: Heading, + }, + { headings: { ids: true } }, +); +``` + +#### Available overrides[#](#available-overrides) + +You can override every HTML tag the parser produces: + +| Option | Props | Description | +| --- | --- | --- | +| `h1`–`h6` | `{ id?, children }` | Headings. `id` is set when `headings: { ids: true }` is enabled | +| `p` | `{ children }` | Paragraph | +| `blockquote` | `{ children }` | Blockquote | +| `pre` | `{ language?, children }` | Code block. `language` is the info string (e.g. `"js"`) | +| `hr` | `{}` | Horizontal rule (no children) | +| `ul` | `{ children }` | Unordered list | +| `ol` | `{ start, children }` | Ordered list. `start` is the first item number | +| `li` | `{ checked?, children }` | List item. `checked` is set for task list items | +| `table` | `{ children }` | Table | +| `thead` | `{ children }` | Table head | +| `tbody` | `{ children }` | Table body | +| `tr` | `{ children }` | Table row | +| `th` | `{ align?, children }` | Table header cell | +| `td` | `{ align?, children }` | Table data cell | +| `em` | `{ children }` | Emphasis (`*text*`) | +| `strong` | `{ children }` | Strong (`**text**`) | +| `a` | `{ href, title?, children }` | Link | +| `img` | `{ src, alt?, title? }` | Image (no children) | +| `code` | `{ children }` | Inline code | +| `del` | `{ children }` | Strikethrough (`~~text~~`) | +| `br` | `{}` | Hard line break (no children) | + +### React 18 and older[#](#react-18-and-older) + +By default, elements use `Symbol.for('react.transitional.element')` as the `$$typeof` symbol. For React 18 and older, pass `reactVersion: 18` in the options (third argument): + +``` +function Markdown({ text }: { text: string }) { + return Bun.markdown.react(text, undefined, { reactVersion: 18 }); +} +``` + +### Parser options[#](#parser-options-1) + +Pass any of the [parser options](#options) as the third argument: + +``` +const el = Bun.markdown.react("## Hello World", undefined, { + headings: { ids: true }, + autolinks: true, +}); +``` + +[PreviousYAML](/docs/runtime/yaml)[NextJSON5](/docs/runtime/json5) + +[Edit this page on GitHub](https://github.com/oven-sh/bun/blob/main/docs/runtime/markdown.mdx)[View as Markdown](/docs/runtime/markdown.md) + +On this page + +- [`Bun.markdown.html()`](#bun-markdown-html) + - [Options](#options) +- [`Bun.markdown.render()`](#bun-markdown-render) + - [Callback signature](#callback-signature) + - [Block callbacks](#block-callbacks) + - [Inline callbacks](#inline-callbacks) + - [Examples](#examples) + - [Parser options](#parser-options) +- [`Bun.markdown.react()`](#bun-markdown-react) + - [Server-side rendering](#server-side-rendering) + - [Component overrides](#component-overrides) + - [React 18 and older](#react-18-and-older) + - [Parser options](#parser-options-1) + +Ask AI diff --git a/test/js/bun/md/fixtures/regenerate-turndown.mjs b/test/js/bun/md/fixtures/regenerate-turndown.mjs new file mode 100644 index 000000000000..605358314be2 --- /dev/null +++ b/test/js/bun/md/fixtures/regenerate-turndown.mjs @@ -0,0 +1,22 @@ +// Regenerates `*.turndown.md` next to each `*.html` fixture: the reference +// output of turndown@7.2.4 + turndown-plugin-gfm@1.0.2 configured with the +// same spellings as Bun.markdown.fromHTML's defaults. md-from-html.test.ts +// compares against these files so the (slow under debug builds) JS converter +// does not have to run over whole pages at test time. +// +// bun test/js/bun/md/fixtures/regenerate-turndown.mjs +import { readdirSync, readFileSync, writeFileSync } from "fs"; +import { join } from "path"; +import TurndownService from "turndown"; +import { gfm } from "turndown-plugin-gfm"; + +const dir = import.meta.dir; +for (const name of readdirSync(dir)) { + if (!name.endsWith(".html")) continue; + const td = new TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced", bulletListMarker: "-", hr: "---" }); + td.use(gfm); + td.remove(["script", "style", "noscript", "template", "title", "head"]); + const out = td.turndown(readFileSync(join(dir, name), "utf8")); + writeFileSync(join(dir, name.replace(/\.html$/, ".turndown.md")), out + "\n"); + console.log(name, "->", out.length, "chars"); +} diff --git a/test/js/bun/md/md-from-html.test.ts b/test/js/bun/md/md-from-html.test.ts new file mode 100644 index 000000000000..341ac9c6a7f6 --- /dev/null +++ b/test/js/bun/md/md-from-html.test.ts @@ -0,0 +1,747 @@ +import { describe, expect, test } from "bun:test"; +import { readFileSync } from "fs"; +import { join } from "path"; +import TurndownService from "turndown"; +// @ts-ignore untyped +import { gfm } from "turndown-plugin-gfm"; + +const { fromHTML } = Bun.markdown; + +// --------------------------------------------------------------------------- +// Rule-by-rule expectations. These pin the exact output for the default +// options; they follow turndown's rule set with the documented deviations +// (single space after list markers, `~~` strikethrough, header-less tables +// still become pipe tables, bare `
    ` is a code block).
    +// ---------------------------------------------------------------------------
    +describe("Bun.markdown.fromHTML", () => {
    +  describe("block rules", () => {
    +    const cases: [string, string, string][] = [
    +      ["paragraphs", "

    one

    two

    ", "one\n\ntwo"], + ["div is a block boundary", "
    one
    two
    ", "one\n\ntwo"], + [ + "headings atx", + "

    a

    b

    c

    d

    e
    f
    ", + "# a\n\n## b\n\n### c\n\n#### d\n\n##### e\n\n###### f", + ], + ["heading with inline markup", "

    Hello there x

    ", "## Hello _there_ `x`"], + ["blockquote", "

    one

    two

    ", "> one\n>\n> two"], + ["nested blockquote", "

    a

    b

    ", "> a\n>\n> > b"], + ["hr", "

    a


    b

    ", "a\n\n---\n\nb"], + ["br", "

    a
    b

    ", "a \nb"], + ["unordered list", "
    • one
    • two
    ", "- one\n- two"], + ["ordered list", "
    1. one
    2. two
    ", "1. one\n2. two"], + ["ordered list start", '
    1. a
    2. b
    ', "7. a\n8. b"], + [ + "nested list", + "
    • a
      • b
        • c
    • d
    ", + "- a\n - b\n - c\n- d", + ], + ["nested ordered indent follows marker width", "
    1. a
      • b
    ", "1. a\n - b"], + ["loose list items", "
    • a

    • b

      c

    ", "- a\n\n- b\n\n c"], + ["list after paragraph", "

    x

    • a

    y

    ", "x\n\n- a\n\ny"], + ["whitespace between li is ignored", "
      \n
    • a
    • \n
    • b
    • \n
    ", "- a\n- b"], + [ + "fenced code block with language", + '
    const x: number = 1;\n
    ', + "```ts\nconst x: number = 1;\n```", + ], + [ + "code block keeps markdown characters literally", + "
    *a* _b_ [c]\n  # not a heading
    ", + "```\n*a* _b_ [c]\n # not a heading\n```", + ], + [ + "fence grows past backtick runs in the code", + "
    ```\ninner\n```
    ", + "````\n```\ninner\n```\n````", + ], + ["bare pre is a code block", "
    line 1\n  line 2
    ", "```\nline 1\n line 2\n```"], + [ + "pre with highlighted spans", + '
    let x;
    ', + "```js\nlet x;\n```", + ], + ["lang- class prefix", '
    fn f() {}
    ', "```rust\nfn f() {}\n```"], + [ + "pre lang attribute (GitHub API HTML)", + '
    [a]\nb = 1
    ', + "```toml\n[a]\nb = 1\n```", + ], + [ + "heading content is one line", + "

    Brand
    tag line

    x

    ", + "# Brand tag _line_ x", + ], + ["leading newline after
     is dropped by the HTML parser", "
    \nx\n
    ", "```\nx\n```"], + ["dl/dt/dd are blocks", "
    term
    def
    ", "term\n\ndef"], + ["details/summary are blocks", "
    MoreHidden text
    ", "More\n\nHidden text"], + ["empty blocks vanish", "

    x

    ", "x"], + ["unknown block-ish custom element is inline", "ab", "ab"], + ]; + for (const [name, html, md] of cases) { + test(name, () => expect(fromHTML(html)).toBe(md)); + } + }); + + describe("inline rules", () => { + const cases: [string, string, string][] = [ + ["strong/b", "

    a b

    ", "**a** **b**"], + ["em/i", "

    a b

    ", "_a_ _b_"], + ["nested emphasis", "

    a b

    ", "**a _b_**"], + ["empty emphasis is dropped", "

    ab c

    ", "ab c"], + ["emphasis hoists edge whitespace", "

    a bold b

    ", "a **bold** b"], + ["hoisted whitespace collapses against neighbours", "

    a bold b

    ", "a **bold** b"], + ["inline code", "

    Use foo() now

    ", "Use `foo()` now"], + ["inline code containing backticks", "

    a`b ``

    ", "``a`b`` ` `` `"], + ["inline code edge backtick gets padding", "

    `x

    ", "`` `x ``"], + ["inline code is not escaped", "

    *a* [b]

    ", "`*a* [b]`"], + ["kbd/samp/var are plain", "

    Ctrl+C n

    ", "Ctrl+C n"], + ["link", '

    Bun

    ', "[Bun](https://bun.com)"], + ["link with title", '

    x

    ', '[x](/x "The X")'], + ["link title quotes are escaped", '

    x

    ', '[x](/x "say \\"hi\\"")'], + ["link destination parens/angles are escaped", '

    x

    ', "[x](/a_\\(b\\)\\)"], + ["link destination with spaces is bracketed", '

    x

    ', "[x]()"], + ["link destination drops tabs/newlines", '

    x

    ', "[x](/ab)"], + ["anchor without href is plain text", '

    x y

    ', "x y"], + [ + "link wrapping an image", + '

    pic

    ', + "[![pic](/small.png)](/big)", + ], + ["empty link is kept (href is meaningful)", '

    ab

    ', "a[](/x)b"], + ["image", '

    Alt text

    ', '![Alt text](a.png "T")'], + ["image without src is dropped", '

    axb

    ', "ab"], + ["image alt is escaped", '

    *x* [y]

    ', "![\\*x\\* \\[y\\]](a.png)"], + ["image keeps surrounding spaces", '

    a b

    ', "a ![](x.png) b"], + ["strikethrough", "

    a b c

    ", "~~a~~ ~~b~~ ~~c~~"], + [ + "sub/sup/span/font pass through", + '

    H2O x2 sf

    ', + "H2O x2 sf", + ], + ["br inside otherwise-empty inline is kept", "

    a
    b

    ", "a \nb"], + ]; + for (const [name, html, md] of cases) { + test(name, () => expect(fromHTML(html)).toBe(md)); + } + }); + + describe("whitespace", () => { + const cases: [string, string, string][] = [ + ["runs collapse to one space", "

    a \n\t b

    ", "a b"], + ["leading/trailing document whitespace is trimmed", "\n\n

    x

    \n\n", "x"], + ["space between inline siblings survives", "

    a b

    ", "**a** _b_"], + ["no space is invented between touching inlines", "

    ab

    ", "**a**_b_"], + [ + "newline between inline elements becomes a space", + "

    a\nb

    ", + "[a](/a) [b](/b)", + ], + ["nbsp is preserved, not collapsed", "

    a  b

    ", "a\u00a0\u00a0b"], + ["nbsp at element edge is hoisted, not dropped", "

    a x

    ", "a\u00a0**x**"], + ["text directly in body", "hello world", "hello **world**"], + ["comments are dropped", "

    ab

    ", "ab"], + ["at most one blank line between blocks", "

    a

    \n\n\n

    b

    ", "a\n\nb"], + ]; + for (const [name, html, md] of cases) { + test(name, () => expect(fromHTML(html)).toBe(md)); + } + }); + + describe("escaping", () => { + const cases: [string, string, string][] = [ + ["emphasis characters", "

    2 * 3 * 4 and _x_

    ", "2 \\* 3 \\* 4 and \\_x\\_"], + [ + "intraword underscores are left alone", + "

    snake_case_name and __dunder__

    ", + "snake_case_name and \\_\\_dunder\\_\\_", + ], + ["brackets and backslashes", "

    a[1] b\\c `d`

    ", "a\\[1\\] b\\\\c \\`d\\`"], + [ + "line-start markers", + "

    - a

    + b

    1. c

    # d

    > e

    =f

    ~~~g

    ", + "\\- a\n\n\\+ b\n\n1\\. c\n\n\\# d\n\n\\> e\n\n\\=f\n\n\\~~~g", + ], + ["line-start markers mid-line are not escaped", "

    a - b + c 1. d # e > f

    ", "a - b + c 1. d # e > f"], + ["line-start marker after a
    is escaped", "

    a
    - b

    ", "a \n\\- b"], + ["line-start marker at the start of a list item", "
    • - x
    • 3. y
    ", "- \\- x\n- 3\\. y"], + ["hash without following space is literal", "

    #tag and ## x

    ", "#tag and ## x"], + [ + "literal tags in text are escaped", + "

    a <b> c </b> <!-- d & 1 < 2

    ", + "a \\ c \\ \\")).toBe(""); + }); + }); + + describe("options", () => { + test("headingStyle: setext", () => { + expect(fromHTML("

    One

    Two

    Three

    ", { headingStyle: "setext" })).toBe( + "One\n===\n\nTwo\n---\n\n### Three", + ); + // Underline is as wide as the (flattened) text, in characters not bytes. + expect(fromHTML("

    caf\u00e9
    ok

    ", { headingStyle: "setext" })).toBe("caf\u00e9 ok\n======="); + }); + test("hr", () => { + for (const hr of ["---", "***", "___", "- - -", "* * *", "_ _ _"] as const) { + expect(fromHTML("
    ", { hr })).toBe(hr); + } + }); + test("bulletListMarker", () => { + expect(fromHTML("
    • a
    ", { bulletListMarker: "*" })).toBe("* a"); + expect(fromHTML("
    • a
    ", { bulletListMarker: "+" })).toBe("+ a"); + }); + test("codeBlockStyle: indented", () => { + expect( + fromHTML('
    a\n  b

    x

    ', { codeBlockStyle: "indented" }), + ).toBe(" a\n b\n\nx"); + }); + test("fence: ~~~", () => { + expect(fromHTML("
    ~~~\nx
    ", { fence: "~~~" })).toBe("~~~~\n~~~\nx\n~~~~"); + }); + test("emDelimiter / strongDelimiter", () => { + expect(fromHTML("

    a b

    ", { emDelimiter: "*", strongDelimiter: "__" })).toBe( + "*a* __b__", + ); + }); + test("br: backslash", () => { + expect(fromHTML("

    a
    b

    ", { br: "\\" })).toBe("a\\\nb"); + }); + test("strikethrough: false keeps the text", () => { + expect(fromHTML("

    a

    ", { strikethrough: false })).toBe("a"); + }); + test("undefined/null options mean defaults; unknown keys are ignored", () => { + expect(fromHTML("

    a

    ", undefined)).toBe("# a"); + expect(fromHTML("

    a

    ", null as any)).toBe("# a"); + expect(fromHTML("

    a

    ", { headingStyle: undefined, whatever: 1 } as any)).toBe("# a"); + }); + test("invalid enum values throw", () => { + expect(() => fromHTML("

    a

    ", { headingStyle: "loud" } as any)).toThrow(/headingStyle/); + expect(() => fromHTML("
    ", { hr: "-*-" } as any)).toThrow(/hr/); + expect(() => fromHTML("
    ", { br: " " } as any)).toThrow(/br/); + expect(() => fromHTML("
    • a
    ", { bulletListMarker: 1 } as any)).toThrow(/bulletListMarker/); + }); + test("missing input throws", () => { + expect(() => (fromHTML as any)()).toThrow(); + expect(() => fromHTML(null as any)).toThrow(); + expect(() => fromHTML(123 as any)).toThrow(); + }); + }); + + describe("input types", () => { + test("Uint8Array / ArrayBuffer / Buffer", () => { + const bytes = new TextEncoder().encode("

    caf\u00e9 ok

    "); + expect(fromHTML(bytes)).toBe("caf\u00e9 **ok**"); + expect(fromHTML(bytes.buffer)).toBe("caf\u00e9 **ok**"); + expect(fromHTML(Buffer.from(bytes))).toBe("caf\u00e9 **ok**"); + }); + test("invalid UTF-8 bytes become U+FFFD instead of throwing", () => { + expect(fromHTML(new Uint8Array([0x3c, 0x70, 0x3e, 0x61, 0xff, 0x62]))).toBe("a\ufffdb"); + }); + test("lone surrogate in a JS string does not throw", () => { + expect(typeof fromHTML("

    a\ud800b

    ")).toBe("string"); + }); + test("NUL bytes", () => { + expect(fromHTML("

    a\u0000b

    ")).toBe("ab"); + }); + }); + + describe("malformed input never throws", () => { + const nasties: [string, string][] = [ + ["unclosed tags", "

    abc"], + ["stray end tags", "

    text"], + ["misnested formatting (adoption agency)", "

    abcd

    "], + ["table junk (foster parenting)", "oopsmorebad
    cell
    "], + ["p inside button inside a", ""], + ["unterminated comment", "

    a`: back to plain script data. + Script, + /// The script element ended (its end tag was emitted), or input did. + Done(Option), +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Mode { + Data, + Raw(RawKind), + Plaintext, +} + +pub(crate) struct FastTokenizer<'t, S: TokenSink> { + src: &'t str, + /// The same bytes as `src`, as a tendril, so text and attribute values + /// can be handed over as shared sub-slices instead of copies. + input: &'t StrTendril, + sink: &'t S, + pos: usize, + mode: Mode, + last_start_tag: Option, + names: NameCache<'t>, +} + +/// Tag and attribute names repeat heavily within a document; interning one +/// (`LocalName::from`) hashes it with SipHash and probes the static atom +/// table (or locks the dynamic one). A small direct-mapped cache of +/// `input slice → atom` in front of that turns nearly every lookup into a +/// short byte comparison and an atom clone. +struct NameCache<'t> { + slots: [Option<(&'t str, LocalName)>; 256], +} + +impl<'t> NameCache<'t> { + fn new() -> Self { + NameCache { + slots: [const { None }; 256], + } + } + + #[inline] + fn intern(&mut self, name: &'t str) -> LocalName { + let b = name.as_bytes(); + // Cheap spread over length and the first/last bytes; collisions just + // cost a real intern. + let h = (b.len().wrapping_mul(31)) + ^ (b[0] as usize).wrapping_mul(7) + ^ (b[b.len() - 1] as usize).wrapping_mul(131) + ^ (b[b.len() / 2] as usize).wrapping_mul(17); + let slot = &mut self.slots[h & 255]; + if let Some((k, atom)) = slot { + if *k == name { + return atom.clone(); + } + } + let atom = LocalName::from(name); + *slot = Some((name, atom.clone())); + atom + } +} + +/// HTML whitespace as the tokenizer sees it after input preprocessing: +/// TAB, LF, FF, SPACE, and CR (which preprocessing turns into LF). +#[inline] +fn is_ws(b: u8) -> bool { + matches!(b, b'\t' | b'\n' | b'\x0C' | b' ' | b'\r') +} + +impl<'t, S: TokenSink> FastTokenizer<'t, S> { + pub(crate) fn new(input: &'t StrTendril, sink: &'t S) -> Self { + let src: &str = input; + // Input-stream preprocessing: a leading BOM is not content. + let pos = if src.starts_with('\u{FEFF}') { 3 } else { 0 }; + FastTokenizer { + src, + input, + sink, + pos, + mode: Mode::Data, + last_start_tag: None, + names: NameCache::new(), + } + } + + #[inline] + fn bytes(&self) -> &'t [u8] { + self.src.as_bytes() + } + + #[inline] + fn peek(&self) -> Option { + self.bytes().get(self.pos).copied() + } + + // ───────────────────────── emission ───────────────────────── + + #[inline] + fn process(&mut self, token: Token) { + match self.sink.process_token(token, 1) { + TokenSinkResult::Continue | TokenSinkResult::EncodingIndicator(_) => {} + // The tree builder only redirects the tokenizer in response to + // tags; see `emit_tag`. + TokenSinkResult::Script(_) => { + self.mode = Mode::Data; + // html5ever's driver re-enters `Tokenizer::feed` after every + // ``, and `feed` discards a BOM at the front of the + // remaining input each time. Match that so the output does + // not depend on which tokenizer handled the document. + if self.bytes()[self.pos..].starts_with("\u{FEFF}".as_bytes()) { + self.pos += 3; + } + } + TokenSinkResult::Plaintext => self.mode = Mode::Plaintext, + TokenSinkResult::RawData(kind) => self.mode = Mode::Raw(kind), + } + } + + /// `input[start..end]` as a tendril sharing the input's buffer. + #[inline] + fn slice(&self, start: usize, end: usize) -> StrTendril { + debug_assert!(self.src.is_char_boundary(start) && self.src.is_char_boundary(end)); + // SAFETY: `start`/`end` are within bounds and on char boundaries — + // every position this tokenizer produces is either 0, the input + // length, or adjacent to an ASCII byte it matched — and the input is + // valid UTF-8, so the sub-slice is too. (`subtendril` would re-derive + // this by classifying the bytes at both ends on every call.) + unsafe { + self.input + .unsafe_subtendril(start as u32, (end - start) as u32) + } + } + + /// Characters `start..end` of the input, verbatim (zero-copy). + #[inline] + fn emit_slice(&mut self, start: usize, end: usize) { + if end > start { + let t = self.slice(start, end); + self.process(Token::CharacterTokens(t)); + } + } + + #[inline] + fn emit_str(&mut self, s: &str) { + self.process(Token::CharacterTokens(StrTendril::from_slice(s))); + } + + fn emit_tag(&mut self, tag: Tag) { + if tag.kind == TagKind::StartTag { + self.last_start_tag = Some(tag.name.clone()); + } + // A tag always returns the tokenizer to the data state unless the + // tree builder answers with a different one. + self.mode = Mode::Data; + self.process(Token::TagToken(tag)); + } + + fn emit_comment(&mut self, start: usize, end: usize) { + // Comment text is carried for fidelity with html5ever's stream (the + // tree builder passes it to the sink, which discards it). CR and NUL + // are left raw: nothing downstream reads comment data. + let t = if end > start { + self.slice(start, end) + } else { + StrTendril::new() + }; + self.process(Token::CommentToken(t)); + } + + // ───────────────────────── driver ───────────────────────── + + /// Tokenizes the whole input into the sink, EOF token included, and + /// tells the sink the input has ended (what `Tokenizer::feed` + `end` do + /// for html5ever's own tokenizer). + pub(crate) fn run(mut self) { + loop { + let halt = match self.mode { + Mode::Data => self.data(), + Mode::Raw(kind) => self.raw(kind), + Mode::Plaintext => self.plaintext(), + }; + match halt { + Some(Halt::Eof) => break, + // The tree builder switched modes; dispatch again. + None => {} + } + } + self.process(Token::EOFToken); + self.sink.end(); + } + + /// Data state. Returns when input ends, or (with `None`) when the tree + /// builder switched the tokenizer to another mode. + fn data(&mut self) -> Option { + let bytes = self.bytes(); + loop { + let start = self.pos; + let Some(off) = strings::index_of_any(&bytes[start..], b"<&\r\0") else { + self.emit_slice(start, bytes.len()); + self.pos = bytes.len(); + return Some(Halt::Eof); + }; + let i = start + off; + self.emit_slice(start, i); + self.pos = i + 1; + match bytes[i] { + b'<' => { + if let Some(h) = self.tag_open() { + return Some(h); + } + if self.mode != Mode::Data { + return None; + } + } + b'&' => self.char_ref_in_text(), + b'\r' => self.carriage_return(), + _ => self.process(Token::NullCharacterToken), + } + } + } + + /// After a CR at `pos - 1`: CRLF becomes the LF (left for the next text + /// run to pick up), a lone CR becomes LF. + #[inline] + fn carriage_return(&mut self) { + if self.peek() != Some(b'\n') { + self.emit_str("\n"); + } + } + + /// RCDATA / RAWTEXT / script data: text until the matching end tag. + fn raw(&mut self, kind: RawKind) -> Option { + let bytes = self.bytes(); + let set: &[u8] = if kind == RawKind::Rcdata { + b"<&\r\0" + } else { + b"<\r\0" + }; + loop { + let start = self.pos; + let Some(off) = strings::index_of_any(&bytes[start..], set) else { + self.emit_slice(start, bytes.len()); + self.pos = bytes.len(); + return Some(Halt::Eof); + }; + let i = start + off; + match bytes[i] { + b'<' => { + if kind == RawKind::ScriptData && bytes[i + 1..].starts_with(b"!--") { + // `` drops back to plain script data. + fn script_data_escaped(&mut self) -> Escaped { + let bytes = self.bytes(); + let mut text_start = self.pos; + let mut double = false; + let mut dashes: u32 = 2; + loop { + // Skip ordinary bytes in bulk. + if dashes == 0 { + match strings::index_of_any(&bytes[self.pos..], b"-<\r\0") { + Some(off) => self.pos += off, + None => self.pos = bytes.len(), + } + } + let Some(&b) = bytes.get(self.pos) else { + self.emit_slice(text_start, self.pos); + return Escaped::Done(Some(Halt::Eof)); + }; + match b { + b'-' => { + dashes += 1; + self.pos += 1; + } + b'>' if dashes >= 2 => { + self.pos += 1; + self.emit_slice(text_start, self.pos); + return Escaped::Script; + } + b'<' => { + dashes = 0; + let next = bytes.get(self.pos + 1).copied(); + if !double { + if next == Some(b'/') { + if let Some((name_end, name)) = self.appropriate_end_tag(self.pos + 2) { + self.emit_slice(text_start, self.pos); + self.pos = name_end; + return Escaped::Done(self.tag_rest(TagKind::EndTag, name)); + } + // `' => { + double = self.src[word_start..j].eq_ignore_ascii_case("script"); + // The delimiter is consumed as text too — a + // CR via the newline handling below. + self.pos = if d == b'\r' { j } else { j + 1 }; + } + _ => self.pos = j, + } + } else { + self.pos += 1; + } + } else if next == Some(b'/') { + // Double escape end: `' => { + if self.src[word_start..j].eq_ignore_ascii_case("script") { + double = false; + } + self.pos = if d == b'\r' { j } else { j + 1 }; + } + _ => self.pos = j, + } + } else { + self.pos += 1; + } + } + 0 => { + self.emit_slice(text_start, self.pos); + self.pos += 1; + self.emit_str("\u{FFFD}"); + text_start = self.pos; + dashes = 0; + } + b'\r' => { + self.emit_slice(text_start, self.pos); + self.pos += 1; + self.carriage_return(); + text_start = self.pos; + dashes = 0; + } + _ => { + dashes = 0; + self.pos += 1; + } + } + } + } + + /// In a raw-text mode, checks whether ``. Returns the + /// position after the name and the (lowercase) name. + fn appropriate_end_tag(&self, at: usize) -> Option<(usize, LocalName)> { + let last = self.last_start_tag.as_ref()?; + let bytes = self.bytes(); + let mut end = at; + while end < bytes.len() && bytes[end].is_ascii_alphabetic() { + end += 1; + } + let delim = *bytes.get(end)?; + if !(is_ws(delim) || delim == b'/' || delim == b'>') { + return None; + } + let candidate = &self.src[at..end]; + if candidate.len() == last.len() && candidate.eq_ignore_ascii_case(last) { + Some((end, last.clone())) + } else { + None + } + } + + fn plaintext(&mut self) -> Option { + let bytes = self.bytes(); + loop { + let start = self.pos; + let Some(off) = strings::index_of_any(&bytes[start..], b"\r\0") else { + self.emit_slice(start, bytes.len()); + self.pos = bytes.len(); + return Some(Halt::Eof); + }; + let i = start + off; + self.emit_slice(start, i); + self.pos = i + 1; + if bytes[i] == b'\r' { + self.carriage_return(); + } else { + self.emit_str("\u{FFFD}"); + } + } + } + + // ───────────────────────── tags ───────────────────────── + + /// Tag open state; `pos` is just past `<`. `Some` only at EOF. + fn tag_open(&mut self) -> Option { + match self.peek() { + None => { + self.emit_str("<"); + Some(Halt::Eof) + } + Some(b'!') => { + self.pos += 1; + self.markup_declaration_open() + } + Some(b'/') => { + self.pos += 1; + self.end_tag_open() + } + Some(b'?') => { + // ``: bogus comment starting at the `?`. + self.bogus_comment(self.pos) + } + Some(c) if c.is_ascii_alphabetic() => self.tag(TagKind::StartTag), + Some(_) => { + // Not a tag: the `<` was text; reconsume this character as data. + self.emit_str("<"); + None + } + } + } + + /// End tag open state; `pos` is just past ` Option { + match self.peek() { + None => { + self.emit_str("') => { + // `` is swallowed whole. + self.pos += 1; + None + } + Some(c) if c.is_ascii_alphabetic() => self.tag(TagKind::EndTag), + Some(_) => self.bogus_comment(self.pos), + } + } + + /// Tag name state onwards; `pos` is at the first (ASCII alpha) name byte. + fn tag(&mut self, kind: TagKind) -> Option { + let bytes = self.bytes(); + let start = self.pos; + let mut end = start; + let mut clean = true; // no uppercase / NUL: name can be interned from the slice + while end < bytes.len() { + let b = bytes[end]; + if is_ws(b) || b == b'/' || b == b'>' { + break; + } + if b.is_ascii_uppercase() || b == 0 { + clean = false; + } + end += 1; + } + if end == bytes.len() { + // EOF in tag name: the tag is dropped. + self.pos = end; + return Some(Halt::Eof); + } + let name = if clean { + self.names.intern(&self.src[start..end]) + } else { + LocalName::from(lowercase_with_replacement(&self.src[start..end])) + }; + self.pos = end; + self.tag_rest(kind, name) + } + + /// Everything after the tag name: attributes, `/`, `>`. Emits the tag. + /// `Some` only for EOF inside the tag (the tag is then dropped, as the + /// spec requires). + fn tag_rest(&mut self, kind: TagKind, name: LocalName) -> Option { + let bytes = self.bytes(); + let mut attrs: Vec = Vec::new(); + let mut had_duplicate_attributes = false; + let mut self_closing = false; + + // Each iteration starts in the "before attribute name" state. + 'attrs: loop { + while self.peek().is_some_and(is_ws) { + self.pos += 1; + } + match self.peek() { + None => return Some(Halt::Eof), + Some(b'/') => { + self.pos += 1; + // Self-closing start tag state. + match self.peek() { + None => return Some(Halt::Eof), + Some(b'>') => { + self_closing = true; + self.pos += 1; + break 'attrs; + } + // Anything else: back to "before attribute name" + // without consuming. + Some(_) => continue 'attrs, + } + } + Some(b'>') => { + self.pos += 1; + break 'attrs; + } + Some(_) => {} + } + + // Attribute name state. The first byte is always part of the + // name, even `=` (that is how `` gets an attribute named + // "=x"); after it, `=` ends the name. + let name_start = self.pos; + let mut name_end = name_start + 1; + let mut clean = !bytes[name_start].is_ascii_uppercase() && bytes[name_start] != 0; + while name_end < bytes.len() { + let b = bytes[name_end]; + if is_ws(b) || b == b'/' || b == b'=' || b == b'>' { + break; + } + if b.is_ascii_uppercase() || b == 0 { + clean = false; + } + name_end += 1; + } + // `name_start` may sit inside a multi-byte character only if the + // byte there is a UTF-8 continuation byte, which cannot happen: + // we arrive here at an ASCII delimiter boundary or at a byte the + // previous loop stopped *before*. + let attr_name = if clean { + self.names.intern(&self.src[name_start..name_end]) + } else { + LocalName::from(lowercase_with_replacement(&self.src[name_start..name_end])) + }; + self.pos = name_end; + + // After attribute name / before attribute value. + let mut value: Option = None; + loop { + match self.peek() { + None => return Some(Halt::Eof), + Some(b) if is_ws(b) => self.pos += 1, + Some(b'=') => { + self.pos += 1; + match self.attribute_value() { + Ok(v) => value = Some(v), + Err(h) => return Some(h), + } + break; + } + // `/`, `>`, or the start of the next attribute: this one + // has no value. Leave the byte for the outer loop. + Some(_) => break, + } + } + + // Duplicate attributes are dropped (first one wins). + if attrs.iter().any(|a| a.name.local == attr_name) { + had_duplicate_attributes = true; + } else { + attrs.push(Attribute { + name: QualName::new(None, ns!(), attr_name), + value: value.unwrap_or_default(), + }); + } + + // After a quoted value the spec wants whitespace, `/` or `>`; + // anything else is a parse error and is reconsumed as the start + // of the next attribute — which is what looping does. + } + + self.emit_tag(Tag { + kind, + name, + self_closing, + attrs, + had_duplicate_attributes, + }); + None + } + + /// Before attribute value state onwards; `pos` is just past `=`. + /// `Err` for EOF inside the value (the whole tag is dropped). + fn attribute_value(&mut self) -> Result { + let bytes = self.bytes(); + while self.peek().is_some_and(is_ws) { + self.pos += 1; + } + match self.peek() { + // EOF here reconsumes in the unquoted state, which then hits EOF + // in a tag: dropped either way. + None => Err(Halt::Eof), + Some(q @ (b'"' | b'\'')) => { + self.pos += 1; + let start = self.pos; + let set: &[u8] = if q == b'"' { b"\"&\r\0" } else { b"'&\r\0" }; + // Fast path: nothing to decode before the closing quote. + match strings::index_of_any(&bytes[start..], set) { + None => { + self.pos = bytes.len(); + Err(Halt::Eof) + } + Some(off) if bytes[start + off] == q => { + self.pos = start + off + 1; + Ok(self.slice(start, start + off)) + } + Some(_) => self.attribute_value_slow(q, set), + } + } + Some(b'>') => { + // ``: missing value; the tag ends here. Leave `>` + // for the caller. + Ok(StrTendril::new()) + } + Some(_) => { + // Unquoted: up to whitespace or `>`. + let start = self.pos; + match strings::index_of_any(&bytes[start..], b"\t\n\x0C \r>&\0") { + None => { + self.pos = bytes.len(); + Err(Halt::Eof) + } + Some(off) if !matches!(bytes[start + off], b'&' | 0) => { + self.pos = start + off; + Ok(self.slice(start, start + off)) + } + Some(_) => self.attribute_value_slow(0, b"\t\n\x0C \r>&\0"), + } + } + } + } + + /// Attribute value containing character references, CRs or NULs. + /// `quote` is the closing quote, or 0 for an unquoted value. `pos` is at + /// the start of the value. + fn attribute_value_slow(&mut self, quote: u8, set: &[u8]) -> Result { + let bytes = self.bytes(); + let mut out = StrTendril::new(); + loop { + let start = self.pos; + let Some(off) = strings::index_of_any(&bytes[start..], set) else { + self.pos = bytes.len(); + return Err(Halt::Eof); + }; + let i = start + off; + out.push_slice(&self.src[start..i]); + let b = bytes[i]; + self.pos = i + 1; + match b { + b'&' => self.char_ref(true, &mut out), + 0 => out.push_char('\u{FFFD}'), + _ if quote != 0 && b == quote => return Ok(out), + b'\r' if quote != 0 => { + if self.peek() == Some(b'\n') { + self.pos += 1; + } + out.push_char('\n'); + } + // Unquoted terminator (whitespace, CR included, or `>`): not + // consumed. + _ => { + self.pos = i; + return Ok(out); + } + } + } + } + + // ───────────────────── character references ───────────────────── + + /// `&` in text (data or RCDATA); `pos` is just past it. + fn char_ref_in_text(&mut self) { + let mut out = StrTendril::new(); + self.char_ref(false, &mut out); + self.process(Token::CharacterTokens(out)); + } + + /// Character reference state; `pos` is just past the `&`. Appends the + /// expansion — or `&` itself when this is not a reference — to `out` and + /// leaves `pos` after whatever was consumed. Mirrors html5ever's + /// `CharRefTokenizer`, including where it deviates from a naive reading + /// of the spec (e.g. overflow handling of numeric references). + fn char_ref(&mut self, in_attribute: bool, out: &mut StrTendril) { + let bytes = self.bytes(); + match self.peek() { + Some(b'#') => { + let after_hash = self.pos + 1; + let (base, digits_start) = match bytes.get(after_hash) { + Some(b'x' | b'X') => (16u32, after_hash + 1), + _ => (10u32, after_hash), + }; + let mut i = digits_start; + let mut num: u32 = 0; + let mut too_big = false; + while let Some(d) = bytes.get(i).and_then(|&b| (b as char).to_digit(base)) { + num = num.wrapping_mul(base); + if num > 0x10FFFF { + too_big = true; + } + num = num.wrapping_add(d); + i += 1; + } + if i == digits_start { + // `&#` / `&#x` with no digits: not a reference. + out.push_char('&'); + return; + } + if bytes.get(i) == Some(&b';') { + i += 1; + } + self.pos = i; + out.push_char(numeric_char_ref(num, too_big)); + } + Some(b) if b.is_ascii_alphanumeric() => { + // Longest match against the entity table, which also holds + // every proper prefix of every name (mapped to 0) so the + // scan knows when to stop. + let name_start = self.pos; + let mut len = 0usize; + let mut matched: Option<(usize, (u32, u32))> = None; + while let Some(&b) = bytes.get(name_start + len) { + if !b.is_ascii() { + break; + } + let Some(&m) = NAMED_ENTITIES.get(&self.src[name_start..name_start + len + 1]) + else { + break; + }; + len += 1; + if m.0 != 0 { + matched = Some((len, m)); + } + } + let Some((mlen, (c1, c2))) = matched else { + out.push_char('&'); + return; + }; + let last_matched = bytes[name_start + mlen - 1]; + let next_after = bytes.get(name_start + mlen).copied(); + // Historical attribute rule: `¬=` / `¬x` in an + // attribute value are left alone. + if in_attribute + && last_matched != b';' + && next_after.is_some_and(|c| c == b'=' || c.is_ascii_alphanumeric()) + { + out.push_char('&'); + return; + } + self.pos = name_start + mlen; + out.push_char(char::from_u32(c1).unwrap_or('\u{FFFD}')); + if c2 != 0 { + out.push_char(char::from_u32(c2).unwrap_or('\u{FFFD}')); + } + } + _ => out.push_char('&'), + } + } + + // ─────────────── comments, doctype, CDATA ─────────────── + + /// Markup declaration open state; `pos` is just past ` Option { + let rest = &self.bytes()[self.pos..]; + if rest.starts_with(b"--") { + self.pos += 2; + return self.comment(); + } + if rest.len() >= 7 && rest[..7].eq_ignore_ascii_case(b"doctype") { + self.pos += 7; + return self.doctype(); + } + if rest.starts_with(b"[CDATA[") + && self + .sink + .adjusted_current_node_present_but_not_in_html_namespace() + { + self.pos += 7; + return self.cdata_section(); + } + self.bogus_comment(self.pos) + } + + /// Bogus comment: everything up to the next `>` (or EOF). + fn bogus_comment(&mut self, data_start: usize) -> Option { + let bytes = self.bytes(); + match strings::index_of_char_usize(&bytes[data_start..], b'>') { + Some(off) => { + self.emit_comment(data_start, data_start + off); + self.pos = data_start + off + 1; + None + } + None => { + self.emit_comment(data_start, bytes.len()); + self.pos = bytes.len(); + Some(Halt::Eof) + } + } + } + + /// Comment states; `pos` is just past `` and `` close immediately. + if bytes[data_start..].starts_with(b">") { + self.emit_comment(data_start, data_start); + self.pos = data_start + 1; + return None; + } + if bytes[data_start..].starts_with(b"->") { + self.emit_comment(data_start, data_start); + self.pos = data_start + 2; + return None; + } + // Otherwise the comment runs to the first `--` (plus any further + // `-`) that is followed by `>` or `!>`. + let mut search = data_start; + loop { + let Some(off) = strings::index_of(&bytes[search..], b"--") else { + self.emit_comment( + data_start, + comment_data_end_at_eof(&bytes[data_start..]) + data_start, + ); + self.pos = bytes.len(); + return Some(Halt::Eof); + }; + let dashes = search + off; + let mut k = dashes + 2; + while bytes.get(k) == Some(&b'-') { + k += 1; + } + match bytes.get(k) { + Some(b'>') => { + self.emit_comment(data_start, k - 2); + self.pos = k + 1; + return None; + } + Some(b'!') if bytes.get(k + 1) == Some(&b'>') => { + self.emit_comment(data_start, k - 2); + self.pos = k + 2; + return None; + } + None => { + self.emit_comment( + data_start, + comment_data_end_at_eof(&bytes[data_start..]) + data_start, + ); + self.pos = bytes.len(); + return Some(Halt::Eof); + } + Some(_) => search = k, + } + } + } + + /// CDATA section (foreign content only); `pos` is just past ` Option { + let bytes = self.bytes(); + let end = strings::index_of(&bytes[self.pos..], b"]]>").map(|off| self.pos + off); + let stop = end.unwrap_or(bytes.len()); + // The section's text, with the same NUL / CR treatment as data. + while self.pos < stop { + let start = self.pos; + match strings::index_of_any(&bytes[start..stop], b"\r\0") { + None => { + self.emit_slice(start, stop); + self.pos = stop; + } + Some(off) => { + let i = start + off; + self.emit_slice(start, i); + self.pos = i + 1; + if bytes[i] == b'\r' { + self.carriage_return(); + } else { + self.process(Token::NullCharacterToken); + } + } + } + } + match end { + Some(e) => { + self.pos = e + 3; + None + } + None => Some(Halt::Eof), + } + } + + /// DOCTYPE states; `pos` is just past ` Option { + let mut d = Doctype::default(); + let halt = self.doctype_into(&mut d); + self.process(Token::DoctypeToken(d)); + halt + } + + fn doctype_into(&mut self, d: &mut Doctype) -> Option { + #[derive(Clone, Copy, PartialEq)] + enum Id { + Public, + System, + } + let eof = |d: &mut Doctype| { + d.force_quirks = true; + Some(Halt::Eof) + }; + + // DOCTYPE state / before DOCTYPE name state. + match self.peek() { + None => return eof(d), + Some(b) if is_ws(b) => self.pos += 1, + Some(_) => {} + } + while self.peek().is_some_and(is_ws) { + self.pos += 1; + } + match self.peek() { + None => return eof(d), + Some(b'>') => { + self.pos += 1; + d.force_quirks = true; + return None; + } + Some(_) => {} + } + // DOCTYPE name state. + let mut name = String::new(); + loop { + match self.peek() { + None => { + d.name = Some(name.into()); + return eof(d); + } + Some(b) if is_ws(b) => break, + Some(b'>') => { + d.name = Some(name.into()); + self.pos += 1; + return None; + } + Some(_) => self.push_lower_char(&mut name), + } + } + d.name = Some(name.into()); + + // After DOCTYPE name state. + while self.peek().is_some_and(is_ws) { + self.pos += 1; + } + let rest = &self.bytes()[self.pos..]; + let mut which = if rest.is_empty() { + return eof(d); + } else if rest[0] == b'>' { + self.pos += 1; + return None; + } else if rest.len() >= 6 && rest[..6].eq_ignore_ascii_case(b"public") { + self.pos += 6; + Id::Public + } else if rest.len() >= 6 && rest[..6].eq_ignore_ascii_case(b"system") { + self.pos += 6; + Id::System + } else { + d.force_quirks = true; + return self.bogus_doctype(); + }; + + loop { + // After DOCTYPE public/system keyword state. + let saw_ws = self.peek().is_some_and(is_ws); + while self.peek().is_some_and(is_ws) { + self.pos += 1; + } + let _ = saw_ws; // missing whitespace is only a parse error + let quote = match self.peek() { + None => return eof(d), + Some(q @ (b'"' | b'\'')) => q, + Some(b'>') => { + self.pos += 1; + d.force_quirks = true; + return None; + } + Some(_) => { + d.force_quirks = true; + return self.bogus_doctype(); + } + }; + self.pos += 1; + // DOCTYPE public/system identifier (quoted) state. + let mut id = String::new(); + loop { + match self.peek() { + None => { + set_id(d, which == Id::Public, id); + return eof(d); + } + Some(b) if b == quote => { + self.pos += 1; + break; + } + Some(b'>') => { + set_id(d, which == Id::Public, id); + self.pos += 1; + d.force_quirks = true; + return None; + } + Some(0) => { + id.push('\u{FFFD}'); + self.pos += 1; + } + Some(b'\r') => { + id.push('\n'); + self.pos += 1; + if self.peek() == Some(b'\n') { + self.pos += 1; + } + } + Some(_) => self.push_char(&mut id), + } + } + set_id(d, which == Id::Public, id); + + // After DOCTYPE public/system identifier state. + if which == Id::System { + while self.peek().is_some_and(is_ws) { + self.pos += 1; + } + return match self.peek() { + None => eof(d), + Some(b'>') => { + self.pos += 1; + None + } + // No force-quirks here, per spec. + Some(_) => self.bogus_doctype(), + }; + } + // After public identifier: optional whitespace, then either `>` + // or a system identifier. + let saw_ws = self.peek().is_some_and(is_ws); + while self.peek().is_some_and(is_ws) { + self.pos += 1; + } + let _ = saw_ws; + match self.peek() { + None => return eof(d), + Some(b'>') => { + self.pos += 1; + return None; + } + Some(b'"' | b'\'') => { + which = Id::System; + // Loop: parse the system identifier (quote consumed there). + continue; + } + Some(_) => { + d.force_quirks = true; + return self.bogus_doctype(); + } + } + } + + fn set_id(d: &mut Doctype, public: bool, id: String) { + if public { + d.public_id = Some(id.into()); + } else { + d.system_id = Some(id.into()); + } + } + } + + /// Bogus DOCTYPE state: skip to `>`. + fn bogus_doctype(&mut self) -> Option { + let bytes = self.bytes(); + match strings::index_of_char_usize(&bytes[self.pos..], b'>') { + Some(off) => { + self.pos += off + 1; + None + } + None => { + self.pos = bytes.len(); + Some(Halt::Eof) + } + } + } + + /// Appends the character at `pos` (lowercasing ASCII, NUL → U+FFFD) and + /// advances past it. + fn push_lower_char(&mut self, s: &mut String) { + let b = self.bytes()[self.pos]; + if b.is_ascii() { + s.push(if b == 0 { + '\u{FFFD}' + } else { + b.to_ascii_lowercase() as char + }); + self.pos += 1; + } else { + let c = self.src[self.pos..].chars().next().unwrap(); + s.push(c); + self.pos += c.len_utf8(); + } + } + + fn push_char(&mut self, s: &mut String) { + let c = self.src[self.pos..].chars().next().unwrap(); + s.push(c); + self.pos += c.len_utf8(); + } +} + +/// For a comment cut off by EOF, how much of `data` html5ever reports: a +/// trailing `-`, `--` or `--!` is still pending in the comment-end states +/// and never makes it into the comment text. +fn comment_data_end_at_eof(data: &[u8]) -> usize { + if data.ends_with(b"--!") { + return data.len() - 3; + } + let dashes = data.iter().rev().take_while(|&&b| b == b'-').count(); + data.len() - dashes.min(2) +} + +/// ASCII-lowercases and maps NUL to U+FFFD. +fn lowercase_with_replacement(s: &str) -> String { + let mut out = String::with_capacity(s.len() + 2); + for c in s.chars() { + out.push(match c { + '\0' => '\u{FFFD}', + c => c.to_ascii_lowercase(), + }); + } + out +} + +/// The numeric character reference end state's mapping. +fn numeric_char_ref(num: u32, too_big: bool) -> char { + match num { + n if n > 0x10FFFF || too_big => '\u{FFFD}', + 0x00 | 0xD800..=0xDFFF => '\u{FFFD}', + 0x80..=0x9F => { + C1_REPLACEMENTS[(num - 0x80) as usize].unwrap_or_else(|| char::from_u32(num).unwrap()) + } + n => char::from_u32(n).expect("range checked above"), + } +} + +/// Token-stream equivalence with html5ever's own tokenizer: both are run in +/// front of the same tree builder (so state switches come from the real +/// insertion modes) and the tokens each delivers are recorded and compared. +/// +/// Miri-only (`bun run rust:miri -p bun_md`): a native `cargo test` binary +/// cannot link the highway kernels behind `bun_core::strings`, which Miri +/// replaces with scalar fallbacks. The same comparison was run natively out of +/// tree over the html5lib tokenizer and tree-construction inputs (with every +/// prefix, in every raw-text context, with CR/NUL variants), a corpus of real +/// pages and random windows into them, and millions of generated documents; +/// the generator below is that one with a Miri-sized iteration count. +#[cfg(all(test, miri))] +mod tests { + use core::cell::RefCell; + + use html5ever::TokenizerResult; + use html5ever::tendril::StrTendril; + use html5ever::tokenizer::{ + BufferQueue, TagKind, Token, TokenSink, TokenSinkResult, Tokenizer, TokenizerOpts, + }; + use html5ever::tree_builder::{TreeBuilder, TreeBuilderOpts}; + + use super::super::{depth, dom}; + use super::FastTokenizer; + + /// Records a normalized transcript of every token (and every tokenizer + /// state switch the tree builder requests) on its way through. + struct Recorder { + inner: S, + log: RefCell>, + text: RefCell, + } + + impl Recorder { + fn new(inner: S) -> Self { + Recorder { + inner, + log: RefCell::new(Vec::new()), + text: RefCell::new(String::new()), + } + } + fn flush(&self) { + let mut t = self.text.borrow_mut(); + if !t.is_empty() { + self.log.borrow_mut().push(format!("text {:?}", *t)); + t.clear(); + } + } + } + + impl TokenSink for Recorder { + type Handle = S::Handle; + fn process_token(&self, token: Token, line: u64) -> TokenSinkResult { + match &token { + // Chunking of character tokens is allowed to differ. + Token::CharacterTokens(t) => self.text.borrow_mut().push_str(t), + Token::NullCharacterToken => self.text.borrow_mut().push('\0'), + Token::ParseError(_) => {} + Token::TagToken(tag) => { + self.flush(); + let mut s = format!( + "{} {}{}{}", + if tag.kind == TagKind::StartTag { + "start" + } else { + "end" + }, + &*tag.name, + if tag.self_closing { " /" } else { "" }, + if tag.had_duplicate_attributes { + " (dup)" + } else { + "" + } + ); + for a in &tag.attrs { + s.push_str(&format!(" [{}={:?}]", &*a.name.local, &*a.value)); + } + self.log.borrow_mut().push(s); + } + Token::CommentToken(c) => { + self.flush(); + // Comment text is discarded downstream; ours leaves CR and + // NUL raw in it, so apply input preprocessing before + // comparing. + let mut norm = String::new(); + let mut chars = c.chars().peekable(); + while let Some(c) = chars.next() { + match c { + '\r' => { + if chars.peek() == Some(&'\n') { + chars.next(); + } + norm.push('\n'); + } + '\0' => norm.push('\u{FFFD}'), + c => norm.push(c), + } + } + self.log.borrow_mut().push(format!("comment {norm:?}")); + } + Token::DoctypeToken(d) => { + self.flush(); + self.log.borrow_mut().push(format!( + "doctype {:?} {:?} {:?} quirks={}", + d.name, d.public_id, d.system_id, d.force_quirks + )); + } + Token::EOFToken => { + self.flush(); + self.log.borrow_mut().push("eof".into()); + } + } + let r = self.inner.process_token(token, line); + match &r { + TokenSinkResult::Continue | TokenSinkResult::EncodingIndicator(_) => {} + TokenSinkResult::Script(_) => self.log.borrow_mut().push("-> script".into()), + TokenSinkResult::Plaintext => self.log.borrow_mut().push("-> plaintext".into()), + TokenSinkResult::RawData(k) => self.log.borrow_mut().push(format!("-> {k:?}")), + } + r + } + fn end(&self) { + self.flush(); + self.inner.end(); + } + fn adjusted_current_node_present_but_not_in_html_namespace(&self) -> bool { + self.inner + .adjusted_current_node_present_but_not_in_html_namespace() + } + } + + fn opts() -> TreeBuilderOpts { + TreeBuilderOpts { + scripting_enabled: true, + drop_doctype: true, + ..Default::default() + } + } + + fn reference_stream(html: &str) -> Vec { + let arena = typed_arena::Arena::new(); + let rec = Recorder::new(depth::DepthLimiter::new(TreeBuilder::new( + dom::Sink::new(&arena), + opts(), + ))); + let tok = Tokenizer::new(rec, TokenizerOpts::default()); + let q = BufferQueue::default(); + q.push_back(StrTendril::from(html)); + while !matches!(tok.feed(&q), TokenizerResult::Done) {} + tok.end(); + tok.sink.log.take() + } + + fn our_stream(html: &str) -> Vec { + let arena = typed_arena::Arena::new(); + let rec = Recorder::new(depth::DepthLimiter::new(TreeBuilder::new( + dom::Sink::new(&arena), + opts(), + ))); + let input = StrTendril::from(html); + FastTokenizer::new(&input, &rec).run(); + rec.log.take() + } + + #[track_caller] + fn assert_same(html: &str) { + let reference = reference_stream(html); + let ours = our_stream(html); + assert_eq!(reference, ours, "token streams differ for input {html:?}"); + } + + /// One input per tokenizer corner (the out-of-tree runs also fed every + /// prefix of each, inside every raw-text context). + const CASES: &[&str] = &[ + "

    text

    ", + "Y
    ", + "e", + "", + "", + "", + "", + "& & & <<> ∉ ¬it; ¬i; &xyz; &; & & &", + "�€Ÿ��􏿿��﷐ ", + "a\r\nb\rc\n\rd\0e", + "t&<b>x", + "", + "x", + "< !DOCTYPE html publicx>", + "", + "x", + " c --><

    ", + "", + " --> x", + "", + "x", + "c", + "", + "<a>nf", + "x", + "