printer: decode ill-formed UTF-8 as U+FFFD when quoting byte strings

src/bun_core/string/immutable.rs

@@ -38,9 +38,9 @@ pub use crate::strings_impl::{
 // Transcoding helpers from `unicode_draft` — re-exported so downstream
 // `bun_core::strings::*` callers (e.g. runtime/webcore/encoding.rs) resolve.
 pub use unicode_draft::{
-    BOM, UTF16Replacement, allocate_latin1_into_utf8, copy_cp1252_into_utf16,
-    copy_latin1_into_ascii, copy_latin1_into_utf8_stop_on_non_ascii, copy_latin1_into_utf16,
-    copy_u8_into_u16, copy_u16_into_u8, copy_utf16_into_utf8_impl,
+    BOM, UTF16Replacement, allocate_latin1_into_utf8, convert_utf8_bytes_into_utf16,
+    copy_cp1252_into_utf16, copy_latin1_into_ascii, copy_latin1_into_utf8_stop_on_non_ascii,
+    copy_latin1_into_utf16, copy_u8_into_u16, copy_u16_into_u8, copy_utf16_into_utf8_impl,
     element_length_cp1252_into_utf16, element_length_utf8_into_utf16, to_utf8_list_with_type_bun,
     to_utf16_alloc_maybe_buffered, u16_is_lead, u16_is_trail, utf16_codepoint,
     utf16_codepoint_with_fffd, wtf8_sequence,

src/bun_core/string/immutable/unicode.rs

@@ -408,9 +408,13 @@ pub(super) fn convert_utf8_bytes_into_utf16_with_length(
     }
 }
 
-// This variation matches WebKit behavior.
+/// Decode one UTF-8 sequence at `bytes[0..]` with WHATWG maximal-subpart
+/// replacement, matching WebKit's `TextCodecUTF8`.
+///
+/// Preconditions: `bytes` is non-empty and `bytes[0] >= 0x80`. ASCII must be
+/// handled by the caller.
 // fn convertUTF8BytesIntoUTF16(sequence: *const [4]u8, remaining_len: usize) UTF16Replacement {
-pub(super) fn convert_utf8_bytes_into_utf16(bytes: &[u8]) -> UTF16Replacement {
+pub fn convert_utf8_bytes_into_utf16(bytes: &[u8]) -> UTF16Replacement {
     let sequence: [u8; 4] = match bytes.len() {
         0 => unreachable!(),
         1 => [bytes[0], 0, 0, 0],

src/bun_core/string/mod.rs

@@ -2256,24 +2256,37 @@ pub mod printer {
         let mut i: usize = 0;
 
         while i < n {
-            let width: u8 = match encoding {
-                StrEncoding::Latin1 | StrEncoding::Ascii | StrEncoding::Utf16 => 1,
-                StrEncoding::Utf8 => strings::wtf8_byte_sequence_length_with_invalid(text_in[i]),
-            };
-            let clamped_width = (width as usize).min(n.saturating_sub(i));
-            let c: i32 = match encoding {
-                StrEncoding::Utf8 => {
-                    let mut buf = [0u8; 4];
-                    buf[..clamped_width].copy_from_slice(&text_in[i..i + clamped_width]);
-                    strings::decode_wtf8_rune_t::<i32>(buf, width, 0)
-                }
+            let (width, c): (u8, i32) = match encoding {
+                StrEncoding::Latin1 => (1, text_in[i] as i32),
                 StrEncoding::Ascii => {
                     debug_assert!(text_in[i] <= 0x7F);
-                    text_in[i] as i32
+                    (1, text_in[i] as i32)
+                }
+                StrEncoding::Utf16 => (1, text16[i] as i32),
+                StrEncoding::Utf8 => {
+                    let first = text_in[i];
+                    if first < 0x80 {
+                        (1, first as i32)
+                    } else {
+                        // Same WHATWG UTF-8 decode as
+                        // `bun_js_printer::write_pre_quoted_string_inner`: ill-formed
+                        // sequences become U+FFFD with maximal-subpart advancement.
+                        let replacement = strings::convert_utf8_bytes_into_utf16(&text_in[i..]);
+                        let len = (replacement.len as usize).max(1);
+                        if replacement.fail {
+                            i += len;
+                            if ascii_only {
+                                writer.write_all(&bmp_escape(strings::UNICODE_REPLACEMENT))?;
+                            } else {
+                                writer.write_all(b"\xEF\xBF\xBD")?;
+                            }
+                            continue;
+                        }
+                        (len as u8, replacement.code_point as i32)
+                    }
                 }
-                StrEncoding::Latin1 => text_in[i] as i32,
-                StrEncoding::Utf16 => text16[i] as i32,
             };
+            let clamped_width = (width as usize).min(n.saturating_sub(i));
 
             if can_print_without_escape(c, ascii_only) {
                 match encoding {

src/js_printer/lib.rs

@@ -781,35 +781,42 @@ where
     }
 
     while i < n {
-        let width: u8 = match ENCODING {
-            Encoding::Latin1 | Encoding::Ascii => 1,
-            Encoding::Utf8 => strings::wtf8_byte_sequence_length_with_invalid(text[i]),
-            Encoding::Utf16 => 1,
-        };
-        let clamped_width = (width as usize).min(n.saturating_sub(i));
-        let c: i32 = match ENCODING {
-            Encoding::Utf8 => {
-                let bytes: [u8; 4] = match clamped_width {
-                    1 => [text[i], 0, 0, 0],
-                    2 => [text[i], text[i + 1], 0, 0],
-                    3 => [text[i], text[i + 1], text[i + 2], 0],
-                    4 => [text[i], text[i + 1], text[i + 2], text[i + 3]],
-                    _ => unreachable!(),
-                };
-                strings::decode_wtf8_rune_t::<i32>(bytes, width, 0)
-            }
+        let (width, c): (u8, i32) = match ENCODING {
+            Encoding::Latin1 => (1, text[i] as i32),
             Encoding::Ascii => {
                 debug_assert!(text[i] <= 0x7F);
-                text[i] as i32
+                (1, text[i] as i32)
             }
-            Encoding::Latin1 => text[i] as i32,
             Encoding::Utf16 => {
                 // TODO: if this is a part of a surrogate pair, we could parse the whole codepoint in order
                 // to emit it as a single \u{result} rather than two paired \uLOW\uHIGH.
                 // eg: "\u{10334}" will convert to "𐌴" without this.
-                code_unit_at!(i)
+                (1, code_unit_at!(i))
+            }
+            Encoding::Utf8 => {
+                let first = text[i];
+                if first < 0x80 {
+                    (1, first as i32)
+                } else {
+                    // WHATWG UTF-8 decode so ill-formed sequences become U+FFFD with
+                    // maximal-subpart advancement, matching `Bun.file().text()` /
+                    // `TextDecoder` / `fs.readFileSync(..., "utf8")`.
+                    let replacement = strings::convert_utf8_bytes_into_utf16(&text[i..]);
+                    let len = (replacement.len as usize).max(1);
+                    if replacement.fail {
+                        i += len;
+                        if ascii_only {
+                            writer.write_all(&bmp_escape(strings::UNICODE_REPLACEMENT))?;
+                        } else {
+                            writer.write_all(b"\xEF\xBF\xBD")?;
+                        }
+                        continue;
+                    }
+                    (len as u8, replacement.code_point as i32)
+                }
             }
         };
+        let clamped_width = (width as usize).min(n.saturating_sub(i));
 
         if can_print_without_escape(c, ascii_only) {
             match ENCODING {
@@ -2706,36 +2713,13 @@ pub mod __gated_printer {
         }
 
         pub fn print_string_literal_utf8(&mut self, str: &[u8], allow_backtick: bool) {
-            // WTF-8 = UTF-8 plus surrogate code points (U+D800..U+DFFF as
-            // `ED A0 80`..`ED BF BF`), so validate UTF-8 shape minus the
-            // surrogate exclusion.
-            fn is_valid_wtf8(mut s: &[u8]) -> bool {
-                while let Some(&b0) = s.first() {
-                    let len = match b0 {
-                        0x00..=0x7F => 1,
-                        0xC2..=0xDF => 2,
-                        0xE0..=0xEF => 3,
-                        0xF0..=0xF4 => 4,
-                        _ => return false,
-                    };
-                    if s.len() < len {
-                        return false;
-                    }
-                    let cont_ok = s[1..len].iter().all(|&b| b & 0xC0 == 0x80);
-                    if !cont_ok {
-                        return false;
-                    }
-                    match (len, b0) {
-                        (3, 0xE0) if s[1] < 0xA0 => return false, // overlong
-                        (4, 0xF0) if s[1] < 0x90 => return false, // overlong
-                        (4, 0xF4) if s[1] > 0x8F => return false, // > U+10FFFF
-                        _ => {}
-                    }
-                    s = &s[len..];
-                }
-                true
-            }
-            debug_assert!(is_valid_wtf8(str));
+            // The `Encoding::Utf8` path below does a strict WHATWG decode, so
+            // callers must pass well-formed UTF-8. This entry point is only used
+            // for parser-derived text (aliases, property keys, `path.pretty`,
+            // directives); raw file bytes from the text loader reach the printer
+            // via `print_string_characters_e_string` without this assert and are
+            // handled by the U+FFFD replacement branch there.
+            debug_assert!(strings::is_valid_utf8(str));
 
             let quote = if !IS_JSON {
                 best_quote_char_for_string(str, allow_backtick)

test/bundler/bundler_loader.test.ts

@@ -87,6 +87,21 @@ describe("bundler", async () => {
     },
   });
 
+  // https://github.com/oven-sh/bun/issues/12981
+  for (const target of ["bun", "browser"] as const) {
+    itBundled(`bun/loader-text-file-invalid-utf8-${target}`, {
+      target,
+      files: {
+        "/entry.ts": /* js */ `
+          import t from './f.txt';
+          console.log(JSON.stringify([...t].map(c => c.codePointAt(0))));
+        `,
+        "/f.txt": Buffer.from([0x61, 0xff, 0xfe, 0x62]),
+      },
+      run: { stdout: "[97,65533,65533,98]" },
+    });
+  }
+
   itBundled("bun/loader-json-proto-key-is-own-property", {
     target: "bun",
     files: {

test/js/bun/util/text-loader-invalid-utf8.test.ts

@@ -0,0 +1,54 @@
+import { describe, expect, it } from "bun:test";
+import { bunEnv, bunExe, tempDir } from "harness";
+
+// `import "./file.txt"` must agree with `Bun.file(f).text()`, `fs.readFileSync(f, "utf8")`
+// and `TextDecoder` on files that are not valid UTF-8.
+// https://github.com/oven-sh/bun/issues/12981
+describe("text loader decodes ill-formed UTF-8 as U+FFFD", () => {
+  const cases: { name: string; bytes: number[] }[] = [
+    { name: "invalid lead bytes", bytes: [0x61, 0xff, 0xfe, 0x62] },
+    { name: "lone continuation byte", bytes: [0x61, 0x80, 0x62] },
+    { name: "2-byte lead then non-continuation", bytes: [0x61, 0xc2, 0x20, 0x62] },
+    { name: "3-byte lead then 1 continuation then ASCII", bytes: [0x61, 0xe0, 0x80, 0x62] },
+    { name: "truncated 2-byte sequence at EOF", bytes: [0x61, 0xc2] },
+    { name: "overlong 4-byte sequence", bytes: [0x61, 0xf0, 0x80, 0x80, 0x80, 0x62] },
+    { name: "surrogate encoded in UTF-8", bytes: [0x61, 0xed, 0xa0, 0x80, 0x62] },
+    { name: "valid 2-byte", bytes: [0x61, 0xc3, 0xa9, 0x62] },
+    { name: "valid 3-byte", bytes: [0x61, 0xe2, 0x82, 0xac, 0x62] },
+    { name: "valid 4-byte", bytes: [0x61, 0xf0, 0x9f, 0x98, 0x80, 0x62] },
+  ];
+
+  const files: Record<string, Buffer | string> = {
+    "entry.ts": `
+      import { readFileSync } from "node:fs";
+      const name = process.argv[2];
+      const { default: loaded } = await import("./" + name + ".txt");
+      const read = readFileSync("./" + name + ".txt", "utf8");
+      const blob = await Bun.file("./" + name + ".txt").text();
+      const cps = (s: string) => [...s].map(c => c.codePointAt(0));
+      console.log(JSON.stringify({ loaded: cps(loaded), read: cps(read), blob: cps(blob) }));
+    `,
+  };
+  for (let i = 0; i < cases.length; i++) {
+    files[`c${i}.txt`] = Buffer.from(cases[i].bytes);
+  }
+
+  for (let i = 0; i < cases.length; i++) {
+    const { name, bytes } = cases[i];
+    it.concurrent(name, async () => {
+      using dir = tempDir("text-loader-invalid-utf8", files);
+      await using proc = Bun.spawn({
+        cmd: [bunExe(), "entry.ts", `c${i}`],
+        env: bunEnv,
+        cwd: String(dir),
+        stdout: "pipe",
+        stderr: "pipe",
+      });
+      const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]);
+      expect(stderr).toBe("");
+      const expected = [...new TextDecoder().decode(Uint8Array.from(bytes))].map(c => c.codePointAt(0));
+      expect(JSON.parse(stdout)).toEqual({ loaded: expected, read: expected, blob: expected });
+      expect(exitCode).toBe(0);
+    });
+  }
+});