From c9aea5e2e47807aae0109fbb0b656fcb2cf8c1bc Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Thu, 23 Jul 2026 19:37:03 +0000 Subject: [PATCH 1/7] printer: decode ill-formed UTF-8 as U+FFFD when quoting byte strings The text loader feeds raw file bytes into an E::String, which the printer serializes via the Encoding::Utf8 path of write_pre_quoted_string_inner. That path used the WTF-8 stepper (wtf8_byte_sequence_length_with_invalid + decode_wtf8_rune_t with zero=0), which for ill-formed input: * widened an invalid lead or lone continuation byte to its Latin-1 code point (0xFF -> \xFF), * returned 0 for a bad multibyte sequence and still advanced by the lead-byte-implied width, dropping the following byte(s), and * let WTF-8-encoded surrogates through verbatim. So importing a .txt file disagreed with Bun.file().text(), fs.readFileSync(p, 'utf8'), and TextDecoder on the same bytes. Switch the Utf8 arm to strings::convert_utf8_bytes_into_utf16, the same WHATWG decoder Bun.file().text() already uses, which yields U+FFFD with maximal-subpart advancement. On replacement.fail emit \uFFFD (ascii-only output) or the raw EF BF BD bytes and continue, so neither the raw-byte fast path nor the \xHH branch ever sees the invalid input. Valid sequences keep the existing behaviour. --- src/bun_core/string/immutable.rs | 6 +-- src/bun_core/string/immutable/unicode.rs | 2 +- src/js_printer/lib.rs | 50 ++++++++++------- .../bun/util/text-loader-invalid-utf8.test.ts | 53 +++++++++++++++++++ 4 files changed, 87 insertions(+), 24 deletions(-) create mode 100644 test/js/bun/util/text-loader-invalid-utf8.test.ts diff --git a/src/bun_core/string/immutable.rs b/src/bun_core/string/immutable.rs index 187ca16cca28..0a10f2c93f80 100644 --- a/src/bun_core/string/immutable.rs +++ b/src/bun_core/string/immutable.rs @@ -38,9 +38,9 @@ pub use crate::strings_impl::{ // Transcoding helpers from `unicode_draft` — re-exported so downstream // `bun_core::strings::*` callers (e.g. runtime/webcore/encoding.rs) resolve. pub use unicode_draft::{ - BOM, UTF16Replacement, allocate_latin1_into_utf8, copy_cp1252_into_utf16, - copy_latin1_into_ascii, copy_latin1_into_utf8_stop_on_non_ascii, copy_latin1_into_utf16, - copy_u8_into_u16, copy_u16_into_u8, copy_utf16_into_utf8_impl, + BOM, UTF16Replacement, allocate_latin1_into_utf8, convert_utf8_bytes_into_utf16, + copy_cp1252_into_utf16, copy_latin1_into_ascii, copy_latin1_into_utf8_stop_on_non_ascii, + copy_latin1_into_utf16, copy_u8_into_u16, copy_u16_into_u8, copy_utf16_into_utf8_impl, element_length_cp1252_into_utf16, element_length_utf8_into_utf16, to_utf8_list_with_type_bun, to_utf16_alloc_maybe_buffered, u16_is_lead, u16_is_trail, utf16_codepoint, utf16_codepoint_with_fffd, wtf8_sequence, diff --git a/src/bun_core/string/immutable/unicode.rs b/src/bun_core/string/immutable/unicode.rs index b22a3554f57b..4fe83ac8c6c1 100644 --- a/src/bun_core/string/immutable/unicode.rs +++ b/src/bun_core/string/immutable/unicode.rs @@ -410,7 +410,7 @@ pub(super) fn convert_utf8_bytes_into_utf16_with_length( // This variation matches WebKit behavior. // fn convertUTF8BytesIntoUTF16(sequence: *const [4]u8, remaining_len: usize) UTF16Replacement { -pub(super) fn convert_utf8_bytes_into_utf16(bytes: &[u8]) -> UTF16Replacement { +pub fn convert_utf8_bytes_into_utf16(bytes: &[u8]) -> UTF16Replacement { let sequence: [u8; 4] = match bytes.len() { 0 => unreachable!(), 1 => [bytes[0], 0, 0, 0], diff --git a/src/js_printer/lib.rs b/src/js_printer/lib.rs index a0c4584f3afe..f646a20ea209 100644 --- a/src/js_printer/lib.rs +++ b/src/js_printer/lib.rs @@ -781,35 +781,45 @@ where } while i < n { - let width: u8 = match ENCODING { - Encoding::Latin1 | Encoding::Ascii => 1, - Encoding::Utf8 => strings::wtf8_byte_sequence_length_with_invalid(text[i]), - Encoding::Utf16 => 1, - }; - let clamped_width = (width as usize).min(n.saturating_sub(i)); - let c: i32 = match ENCODING { - Encoding::Utf8 => { - let bytes: [u8; 4] = match clamped_width { - 1 => [text[i], 0, 0, 0], - 2 => [text[i], text[i + 1], 0, 0], - 3 => [text[i], text[i + 1], text[i + 2], 0], - 4 => [text[i], text[i + 1], text[i + 2], text[i + 3]], - _ => unreachable!(), - }; - strings::decode_wtf8_rune_t::(bytes, width, 0) - } + let (width, c): (u8, i32) = match ENCODING { + Encoding::Latin1 => (1, text[i] as i32), Encoding::Ascii => { debug_assert!(text[i] <= 0x7F); - text[i] as i32 + (1, text[i] as i32) } - Encoding::Latin1 => text[i] as i32, Encoding::Utf16 => { // TODO: if this is a part of a surrogate pair, we could parse the whole codepoint in order // to emit it as a single \u{result} rather than two paired \uLOW\uHIGH. // eg: "\u{10334}" will convert to "𐌴" without this. - code_unit_at!(i) + (1, code_unit_at!(i)) + } + Encoding::Utf8 => { + let first = text[i]; + if first < 0x80 { + (1, first as i32) + } else { + // WHATWG UTF-8 decode so ill-formed sequences become U+FFFD with + // maximal-subpart advancement, matching `Bun.file().text()` / + // `TextDecoder` / `fs.readFileSync(..., "utf8")`. The previous + // WTF-8 stepper here widened invalid lead bytes to their Latin-1 + // code point and emitted NUL for bad multibyte sequences while + // over-advancing past following bytes. + let replacement = strings::convert_utf8_bytes_into_utf16(&text[i..]); + let len = (replacement.len as usize).max(1); + if replacement.fail { + i += len; + if ascii_only { + writer.write_all(&bmp_escape(strings::UNICODE_REPLACEMENT))?; + } else { + writer.write_all(b"\xEF\xBF\xBD")?; + } + continue; + } + (len as u8, replacement.code_point as i32) + } } }; + let clamped_width = (width as usize).min(n.saturating_sub(i)); if can_print_without_escape(c, ascii_only) { match ENCODING { diff --git a/test/js/bun/util/text-loader-invalid-utf8.test.ts b/test/js/bun/util/text-loader-invalid-utf8.test.ts new file mode 100644 index 000000000000..3a08ec31c200 --- /dev/null +++ b/test/js/bun/util/text-loader-invalid-utf8.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from "bun:test"; +import { bunEnv, bunExe, tempDir } from "harness"; + +// `import "./file.txt"` must agree with `Bun.file(f).text()`, `fs.readFileSync(f, "utf8")` +// and `TextDecoder` on files that are not valid UTF-8. Previously the loader widened +// invalid lead bytes to their Latin-1 code point and emitted NUL for other ill-formed +// sequences (over-advancing past following bytes) instead of U+FFFD. +describe("text loader decodes ill-formed UTF-8 as U+FFFD", () => { + const cases: { name: string; bytes: number[] }[] = [ + { name: "invalid lead bytes", bytes: [0x61, 0xff, 0xfe, 0x62] }, + { name: "lone continuation byte", bytes: [0x61, 0x80, 0x62] }, + { name: "2-byte lead then non-continuation", bytes: [0x61, 0xc2, 0x20, 0x62] }, + { name: "3-byte lead then 1 continuation then ASCII", bytes: [0x61, 0xe0, 0x80, 0x62] }, + { name: "truncated 2-byte sequence at EOF", bytes: [0x61, 0xc2] }, + { name: "overlong 4-byte sequence", bytes: [0x61, 0xf0, 0x80, 0x80, 0x80, 0x62] }, + { name: "surrogate encoded in UTF-8", bytes: [0x61, 0xed, 0xa0, 0x80, 0x62] }, + { name: "valid 2-byte", bytes: [0x61, 0xc3, 0xa9, 0x62] }, + { name: "valid 3-byte", bytes: [0x61, 0xe2, 0x82, 0xac, 0x62] }, + { name: "valid 4-byte", bytes: [0x61, 0xf0, 0x9f, 0x98, 0x80, 0x62] }, + ]; + + const files: Record = { + "entry.ts": ` + import { readFileSync } from "node:fs"; + const name = process.argv[2]; + const { default: loaded } = await import("./" + name + ".txt"); + const read = readFileSync("./" + name + ".txt", "utf8"); + const cps = (s: string) => [...s].map(c => c.codePointAt(0)); + console.log(JSON.stringify({ loaded: cps(loaded), read: cps(read), same: loaded === read })); + `, + }; + for (let i = 0; i < cases.length; i++) { + files[`c${i}.txt`] = Buffer.from(cases[i].bytes); + } + + for (let i = 0; i < cases.length; i++) { + const { name, bytes } = cases[i]; + it.concurrent(name, async () => { + using dir = tempDir("text-loader-invalid-utf8", files); + await using proc = Bun.spawn({ + cmd: [bunExe(), "entry.ts", `c${i}`], + env: bunEnv, + cwd: String(dir), + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + const expected = [...new TextDecoder().decode(Uint8Array.from(bytes))].map(c => c.codePointAt(0)); + expect({ stderr, ...JSON.parse(stdout) }).toEqual({ stderr: "", loaded: expected, read: expected, same: true }); + expect(exitCode).toBe(0); + }); + } +}); From 525417399804f4e56722e7cb82c3a0d788159db5 Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Thu, 23 Jul 2026 19:48:38 +0000 Subject: [PATCH 2/7] review: doc precondition on convert_utf8_bytes_into_utf16; test Bun.file().text() parity and assert stderr before JSON.parse --- src/bun_core/string/immutable/unicode.rs | 6 +++++- test/js/bun/util/text-loader-invalid-utf8.test.ts | 6 ++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/bun_core/string/immutable/unicode.rs b/src/bun_core/string/immutable/unicode.rs index 4fe83ac8c6c1..69696e7d84b0 100644 --- a/src/bun_core/string/immutable/unicode.rs +++ b/src/bun_core/string/immutable/unicode.rs @@ -408,7 +408,11 @@ pub(super) fn convert_utf8_bytes_into_utf16_with_length( } } -// This variation matches WebKit behavior. +/// Decode one UTF-8 sequence at `bytes[0..]` with WHATWG maximal-subpart +/// replacement, matching WebKit's `TextCodecUTF8`. +/// +/// Preconditions: `bytes` is non-empty and `bytes[0] >= 0x80`. ASCII must be +/// handled by the caller. // fn convertUTF8BytesIntoUTF16(sequence: *const [4]u8, remaining_len: usize) UTF16Replacement { pub fn convert_utf8_bytes_into_utf16(bytes: &[u8]) -> UTF16Replacement { let sequence: [u8; 4] = match bytes.len() { diff --git a/test/js/bun/util/text-loader-invalid-utf8.test.ts b/test/js/bun/util/text-loader-invalid-utf8.test.ts index 3a08ec31c200..a37c455793a6 100644 --- a/test/js/bun/util/text-loader-invalid-utf8.test.ts +++ b/test/js/bun/util/text-loader-invalid-utf8.test.ts @@ -25,8 +25,9 @@ describe("text loader decodes ill-formed UTF-8 as U+FFFD", () => { const name = process.argv[2]; const { default: loaded } = await import("./" + name + ".txt"); const read = readFileSync("./" + name + ".txt", "utf8"); + const blob = await Bun.file("./" + name + ".txt").text(); const cps = (s: string) => [...s].map(c => c.codePointAt(0)); - console.log(JSON.stringify({ loaded: cps(loaded), read: cps(read), same: loaded === read })); + console.log(JSON.stringify({ loaded: cps(loaded), read: cps(read), blob: cps(blob) })); `, }; for (let i = 0; i < cases.length; i++) { @@ -45,8 +46,9 @@ describe("text loader decodes ill-formed UTF-8 as U+FFFD", () => { stderr: "pipe", }); const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + expect(stderr).toBe(""); const expected = [...new TextDecoder().decode(Uint8Array.from(bytes))].map(c => c.codePointAt(0)); - expect({ stderr, ...JSON.parse(stdout) }).toEqual({ stderr: "", loaded: expected, read: expected, same: true }); + expect(JSON.parse(stdout)).toEqual({ loaded: expected, read: expected, blob: expected }); expect(exitCode).toBe(0); }); } From a571bfba04bb32d7a627ddf07df29b2c6ecfedc3 Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Thu, 23 Jul 2026 20:01:19 +0000 Subject: [PATCH 3/7] bun_core::printer: apply the same U+FFFD decode to the sibling write_pre_quoted_string The bun_core copy (used by format_escapes, quote_for_json, snapshot keys, sourcemap JSON) is documented as 'Same algorithm as bun_js_printer::write_pre_quoted_string'; keep that true by giving its Utf8 arm the same convert_utf8_bytes_into_utf16 replacement decode. --- src/bun_core/string/mod.rs | 41 +++++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/src/bun_core/string/mod.rs b/src/bun_core/string/mod.rs index 5244df099cfb..d0eba000e591 100644 --- a/src/bun_core/string/mod.rs +++ b/src/bun_core/string/mod.rs @@ -2256,24 +2256,37 @@ pub mod printer { let mut i: usize = 0; while i < n { - let width: u8 = match encoding { - StrEncoding::Latin1 | StrEncoding::Ascii | StrEncoding::Utf16 => 1, - StrEncoding::Utf8 => strings::wtf8_byte_sequence_length_with_invalid(text_in[i]), - }; - let clamped_width = (width as usize).min(n.saturating_sub(i)); - let c: i32 = match encoding { - StrEncoding::Utf8 => { - let mut buf = [0u8; 4]; - buf[..clamped_width].copy_from_slice(&text_in[i..i + clamped_width]); - strings::decode_wtf8_rune_t::(buf, width, 0) - } + let (width, c): (u8, i32) = match encoding { + StrEncoding::Latin1 => (1, text_in[i] as i32), StrEncoding::Ascii => { debug_assert!(text_in[i] <= 0x7F); - text_in[i] as i32 + (1, text_in[i] as i32) + } + StrEncoding::Utf16 => (1, text16[i] as i32), + StrEncoding::Utf8 => { + let first = text_in[i]; + if first < 0x80 { + (1, first as i32) + } else { + // Same WHATWG UTF-8 decode as + // `bun_js_printer::write_pre_quoted_string_inner`: ill-formed + // sequences become U+FFFD with maximal-subpart advancement. + let replacement = strings::convert_utf8_bytes_into_utf16(&text_in[i..]); + let len = (replacement.len as usize).max(1); + if replacement.fail { + i += len; + if ascii_only { + writer.write_all(&bmp_escape(strings::UNICODE_REPLACEMENT))?; + } else { + writer.write_all(b"\xEF\xBF\xBD")?; + } + continue; + } + (len as u8, replacement.code_point as i32) + } } - StrEncoding::Latin1 => text_in[i] as i32, - StrEncoding::Utf16 => text16[i] as i32, }; + let clamped_width = (width as usize).min(n.saturating_sub(i)); if can_print_without_escape(c, ascii_only) { match encoding { From 8140ce328fb69a7df15bcc3ba75a186076474e9f Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Thu, 23 Jul 2026 20:24:58 +0000 Subject: [PATCH 4/7] js_printer: tighten print_string_literal_utf8 debug_assert to strict UTF-8 The Encoding::Utf8 path now does a strict WHATWG decode, so the WTF-8-shaped is_valid_wtf8 guard (which accepted ED A0 80..ED BF BF) documented an invariant the decoder no longer honours. Replace it with strings::is_valid_utf8 and note that raw text-loader bytes reach the printer via print_string_characters_e_string without this assert. --- src/js_printer/lib.rs | 37 +++++++------------------------------ 1 file changed, 7 insertions(+), 30 deletions(-) diff --git a/src/js_printer/lib.rs b/src/js_printer/lib.rs index f646a20ea209..bfb0e81d3ebf 100644 --- a/src/js_printer/lib.rs +++ b/src/js_printer/lib.rs @@ -2716,36 +2716,13 @@ pub mod __gated_printer { } pub fn print_string_literal_utf8(&mut self, str: &[u8], allow_backtick: bool) { - // WTF-8 = UTF-8 plus surrogate code points (U+D800..U+DFFF as - // `ED A0 80`..`ED BF BF`), so validate UTF-8 shape minus the - // surrogate exclusion. - fn is_valid_wtf8(mut s: &[u8]) -> bool { - while let Some(&b0) = s.first() { - let len = match b0 { - 0x00..=0x7F => 1, - 0xC2..=0xDF => 2, - 0xE0..=0xEF => 3, - 0xF0..=0xF4 => 4, - _ => return false, - }; - if s.len() < len { - return false; - } - let cont_ok = s[1..len].iter().all(|&b| b & 0xC0 == 0x80); - if !cont_ok { - return false; - } - match (len, b0) { - (3, 0xE0) if s[1] < 0xA0 => return false, // overlong - (4, 0xF0) if s[1] < 0x90 => return false, // overlong - (4, 0xF4) if s[1] > 0x8F => return false, // > U+10FFFF - _ => {} - } - s = &s[len..]; - } - true - } - debug_assert!(is_valid_wtf8(str)); + // The `Encoding::Utf8` path below does a strict WHATWG decode, so + // callers must pass well-formed UTF-8. This entry point is only used + // for parser-derived text (aliases, property keys, `path.pretty`, + // directives); raw file bytes from the text loader reach the printer + // via `print_string_characters_e_string` without this assert and are + // handled by the U+FFFD replacement branch there. + debug_assert!(strings::is_valid_utf8(str)); let quote = if !IS_JSON { best_quote_char_for_string(str, allow_backtick) From 5853f83cdc6fc9659d24331f7ad8571c048e3eb5 Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Thu, 23 Jul 2026 20:36:58 +0000 Subject: [PATCH 5/7] js_printer: drop bug-history sentence from the Utf8 decode comment --- src/js_printer/lib.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/js_printer/lib.rs b/src/js_printer/lib.rs index bfb0e81d3ebf..21adffdc5e46 100644 --- a/src/js_printer/lib.rs +++ b/src/js_printer/lib.rs @@ -800,10 +800,7 @@ where } else { // WHATWG UTF-8 decode so ill-formed sequences become U+FFFD with // maximal-subpart advancement, matching `Bun.file().text()` / - // `TextDecoder` / `fs.readFileSync(..., "utf8")`. The previous - // WTF-8 stepper here widened invalid lead bytes to their Latin-1 - // code point and emitted NUL for bad multibyte sequences while - // over-advancing past following bytes. + // `TextDecoder` / `fs.readFileSync(..., "utf8")`. let replacement = strings::convert_utf8_bytes_into_utf16(&text[i..]); let len = (replacement.len as usize).max(1); if replacement.fail { From d7725e17be6a6a73a027e84a06d57c18ba4ca0e6 Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Thu, 23 Jul 2026 20:52:48 +0000 Subject: [PATCH 6/7] ci: retrigger From ee9d06e06297b3e054adc07ba1f0bc3f53f03ca1 Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Thu, 23 Jul 2026 20:58:56 +0000 Subject: [PATCH 7/7] test: bundler coverage for text-loader U+FFFD replacement; trim header comment Add itBundled cases for target=bun (ascii-only output, \uFFFD arm) and target=browser (raw EF BF BD arm) so both replacement.fail branches and the bun build entry point for #12981 have fail-before coverage. Drop the bug-history sentence from the runtime test header and replace it with the issue URL. --- test/bundler/bundler_loader.test.ts | 15 +++++++++++++++ test/js/bun/util/text-loader-invalid-utf8.test.ts | 5 ++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/test/bundler/bundler_loader.test.ts b/test/bundler/bundler_loader.test.ts index daeccfa1acfe..5bf4a3d1192e 100644 --- a/test/bundler/bundler_loader.test.ts +++ b/test/bundler/bundler_loader.test.ts @@ -87,6 +87,21 @@ describe("bundler", async () => { }, }); + // https://github.com/oven-sh/bun/issues/12981 + for (const target of ["bun", "browser"] as const) { + itBundled(`bun/loader-text-file-invalid-utf8-${target}`, { + target, + files: { + "/entry.ts": /* js */ ` + import t from './f.txt'; + console.log(JSON.stringify([...t].map(c => c.codePointAt(0)))); + `, + "/f.txt": Buffer.from([0x61, 0xff, 0xfe, 0x62]), + }, + run: { stdout: "[97,65533,65533,98]" }, + }); + } + itBundled("bun/loader-json-proto-key-is-own-property", { target: "bun", files: { diff --git a/test/js/bun/util/text-loader-invalid-utf8.test.ts b/test/js/bun/util/text-loader-invalid-utf8.test.ts index a37c455793a6..ba6a2df760cb 100644 --- a/test/js/bun/util/text-loader-invalid-utf8.test.ts +++ b/test/js/bun/util/text-loader-invalid-utf8.test.ts @@ -2,9 +2,8 @@ import { describe, expect, it } from "bun:test"; import { bunEnv, bunExe, tempDir } from "harness"; // `import "./file.txt"` must agree with `Bun.file(f).text()`, `fs.readFileSync(f, "utf8")` -// and `TextDecoder` on files that are not valid UTF-8. Previously the loader widened -// invalid lead bytes to their Latin-1 code point and emitted NUL for other ill-formed -// sequences (over-advancing past following bytes) instead of U+FFFD. +// and `TextDecoder` on files that are not valid UTF-8. +// https://github.com/oven-sh/bun/issues/12981 describe("text loader decodes ill-formed UTF-8 as U+FFFD", () => { const cases: { name: string; bytes: number[] }[] = [ { name: "invalid lead bytes", bytes: [0x61, 0xff, 0xfe, 0x62] },