transpiler: preserve non-ASCII source text at runtime

src/js_printer/lib.rs

@@ -1009,12 +1009,12 @@ where
     }
 
     while i < n {
-        let width: u8 = match ENCODING {
+        let mut width: u8 = match ENCODING {
             Encoding::Latin1 | Encoding::Ascii => 1,
             Encoding::Utf8 => strings::wtf8_byte_sequence_length_with_invalid(text[i]),
             Encoding::Utf16 => 1,
         };
-        let clamped_width = (width as usize).min(n.saturating_sub(i));
+        let mut clamped_width = (width as usize).min(n.saturating_sub(i));
         let c: i32 = match ENCODING {
             Encoding::Utf8 => {
                 let bytes: [u8; 4] = match clamped_width {
@@ -1032,10 +1032,19 @@ where
             }
             Encoding::Latin1 => text[i] as i32,
             Encoding::Utf16 => {
-                // TODO: if this is a part of a surrogate pair, we could parse the whole codepoint in order
-                // to emit it as a single \u{result} rather than two paired \uLOW\uHIGH.
-                // eg: "\u{10334}" will convert to "𐌴" without this.
-                code_unit_at!(i)
+                let unit = code_unit_at!(i);
+                if strings::u16_is_lead(unit as u16) && i + 1 < n {
+                    let trail = code_unit_at!(i + 1);
+                    if strings::u16_is_trail(trail as u16) {
+                        width = 2;
+                        clamped_width = 2;
+                        strings::u16_get_supplementary(unit as u16, trail as u16) as i32
+                    } else {
+                        unit
+                    }
+                } else {
+                    unit
+                }
             }
         };
 
@@ -4598,7 +4607,7 @@ pub mod __gated_printer {
                 self.print(b" ");
             }
 
-            if IS_BUN_PLATFORM {
+            if ASCII_ONLY {
                 // Translate any non-ASCII to unicode escape sequences
                 let mut ascii_start: usize = 0;
                 let mut is_ascii = false;
@@ -7794,11 +7803,10 @@ pub type BufferPrinter = Writer<BufferWriter>;
 pub enum Format {
     Esm,
     Cjs,
-    // bun.js must escape non-latin1 identifiers in the output This is because
-    // we load JavaScript as a UTF-8 buffer instead of a UTF-16 buffer
-    // JavaScriptCore does not support UTF-8 identifiers when the source code
-    // string is loaded as const char* We don't want to double the size of code
-    // in memory...
+    // Runtime-transpiler path. The printer now emits UTF-8 for this variant and
+    // consumers call `String::clone_utf8`, so non-ASCII source text is preserved
+    // verbatim (observable via `Function.prototype.toString`, `RegExp#source`,
+    // tagged-template `.raw`). The name is kept for churn avoidance.
     EsmAscii,
     CjsAscii,
 }
@@ -7903,7 +7911,12 @@ pub fn get_source_map_builder<const IS_BUN_PLATFORM: bool>(
 // Top-level print entry points
 // ───────────────────────────────────────────────────────────────────────────
 
-pub fn print_ast<'a, W: WriterTrait, const ASCII_ONLY: bool, const GENERATE_SOURCE_MAP: bool>(
+pub fn print_ast<
+    'a,
+    W: WriterTrait,
+    const IS_BUN_PLATFORM: bool,
+    const GENERATE_SOURCE_MAP: bool,
+>(
     _writer: W,
     bump: &'a bun_alloc::Arena,
     tree: &'a Ast,
@@ -8015,9 +8028,11 @@ pub fn print_ast<'a, W: WriterTrait, const ASCII_ONLY: bool, const GENERATE_SOUR
 
     // defer: if minify_identifiers { renamer.deinit() } — Drop handles.
 
-    // `is_bun_platform = ascii_only` for printAst.
-    type PrinterType<'a, W, const A: bool, const G: bool> =
-        Printer<'a, W, A, false, /*IS_BUN_PLATFORM=*/ A, false, G>;
+    // Runtime-transpiler path: emit UTF-8 so `Function.prototype.toString`,
+    // `RegExp#source` and tagged-template `.raw` reflect the author's source
+    // text. Consumers pass the output through `String::clone_utf8`.
+    type PrinterType<'a, W, const B: bool, const G: bool> =
+        Printer<'a, W, /*ASCII_ONLY=*/ false, false, /*IS_BUN_PLATFORM=*/ B, false, G>;
     let mut writer = _writer;
     // Pre-size the output buffer ~proportional to the source. Transpiled output
     // is almost always within a small factor of the input, so reserving up front
@@ -8026,13 +8041,13 @@ pub fn print_ast<'a, W: WriterTrait, const ASCII_ONLY: bool, const GENERATE_SOUR
     let _ = writer.reserve(source.contents().len() as u64);
 
     let mut opts = opts;
-    let source_map_builder = get_source_map_builder::<ASCII_ONLY>(
+    let source_map_builder = get_source_map_builder::<IS_BUN_PLATFORM>(
         GenerateSourceMap::lazy_if(GENERATE_SOURCE_MAP),
         &mut opts,
         source,
         tree,
     );
-    let mut printer = PrinterType::<W, ASCII_ONLY, GENERATE_SOURCE_MAP>::init(
+    let mut printer = PrinterType::<W, IS_BUN_PLATFORM, GENERATE_SOURCE_MAP>::init(
         writer,
         bump,
         tree.import_records.as_slice(),
@@ -8050,7 +8065,7 @@ pub fn print_ast<'a, W: WriterTrait, const ASCII_ONLY: bool, const GENERATE_SOUR
     // Borrowck: `opts` was moved into `Printer::init`; populate
     // `printer.module_info` by taking it back out of `printer.options`
     // (see `print_with_writer_and_platform`).
-    if PrinterType::<W, ASCII_ONLY, GENERATE_SOURCE_MAP>::MAY_HAVE_MODULE_INFO {
+    if PrinterType::<W, IS_BUN_PLATFORM, GENERATE_SOURCE_MAP>::MAY_HAVE_MODULE_INFO {
         printer.module_info = printer.options.module_info.take();
     }
     printer.binary_expression_stack = Vec::new();
@@ -8070,7 +8085,7 @@ pub fn print_ast<'a, W: WriterTrait, const ASCII_ONLY: bool, const GENERATE_SOUR
         // `require` must be an unbound variable.
         printer.print(b"var {require}=import.meta;");
 
-        if PrinterType::<W, ASCII_ONLY, GENERATE_SOURCE_MAP>::MAY_HAVE_MODULE_INFO {
+        if PrinterType::<W, IS_BUN_PLATFORM, GENERATE_SOURCE_MAP>::MAY_HAVE_MODULE_INFO {
             if let Some(mi) = printer.module_info.as_deref_mut() {
                 mi.flags.contains_import_meta = true;
                 let s = mi.str(b"require");
@@ -8088,8 +8103,9 @@ pub fn print_ast<'a, W: WriterTrait, const ASCII_ONLY: bool, const GENERATE_SOUR
     }
     printer.check_stack_overflow()?;
 
-    let have_module_info = PrinterType::<W, ASCII_ONLY, GENERATE_SOURCE_MAP>::MAY_HAVE_MODULE_INFO
-        && printer.module_info.is_some();
+    let have_module_info =
+        PrinterType::<W, IS_BUN_PLATFORM, GENERATE_SOURCE_MAP>::MAY_HAVE_MODULE_INFO
+            && printer.module_info.is_some();
     if have_module_info {
         printer
             .module_info

src/jsc/AsyncModule.rs

@@ -1375,7 +1375,7 @@ impl AsyncModule {
         }
 
         Ok(ResolvedSource {
-            source_code: BunString::clone_latin1(printer.ctx.get_written()),
+            source_code: BunString::clone_utf8(printer.ctx.get_written()),
             specifier: BunString::init(specifier),
             source_url: BunString::init(path.text),
             is_commonjs_module,

src/jsc/RuntimeTranspilerCache.rs

@@ -41,7 +41,8 @@ bun_core::declare_scope!(cache, visible);
 /// Version 22: Serialize `has_tla` in the cached ESM record flags byte. Entries
 /// written before #30888 carried `has_tla=false` for every module; the cache-HIT
 /// path reinstates the bug for any previously-cached TLA module (#30887).
-const EXPECTED_VERSION: u32 = 22;
+/// Version 23: Runtime transpiler emits UTF-8 instead of ASCII-escaped output.
+const EXPECTED_VERSION: u32 = 23;
 
 /// Source files smaller than this are not written to / read from the on-disk
 /// transpiler cache. Originally 50 KiB, which excluded almost every file in a
@@ -1014,7 +1015,7 @@ impl RuntimeTranspilerCache {
             return;
         }
         debug_assert!(self.entry.is_none());
-        let output_code = BunString::clone_latin1(output_code_bytes);
+        let output_code = BunString::clone_utf8(output_code_bytes);
         // Refcount stays at 1, sole owner.
         // BunString is Copy with no Drop, so an extra dupe_ref here would leak.
         self.output_code = Some(output_code);
@@ -1033,7 +1034,7 @@ impl RuntimeTranspilerCache {
         }
         #[cfg(debug_assertions)]
         {
-            bun_core::scoped_log!(cache, "put() = {} bytes", output_code.latin1().len());
+            bun_core::scoped_log!(cache, "put() = {} bytes", output_code_bytes.len());
         }
     }
 }
@@ -1083,10 +1084,9 @@ bun_ast::link_impl_TranspilerCacheImpl! {
             }
             debug_assert!(this.entry.is_none());
 
-            // Borrowed Latin-1 view: `to_file` only reads `byte_slice()` + the encoding
-            // tag (unmarked 8-bit ZigString -> Encoding::LATIN1, same as clone_latin1),
-            // and `output_code_bytes` outlives the synchronous `to_file` call.
-            let output_code = BunString::ascii(output_code_bytes);
+            // UTF-8-tagged so `to_file` records Encoding::UTF8 (it then
+            // Box-copies into `OutputCode::Utf8`; see that fn's PERF note).
+            let output_code = BunString::borrow_utf8(output_code_bytes);
             let result = RuntimeTranspilerCache::to_file(
                 this.input_byte_length.unwrap(),
                 this.input_hash.unwrap(),

src/jsc/RuntimeTranspilerStore.rs

@@ -1181,7 +1181,7 @@ impl TranspilerJob {
             // `cache.output_code` (only the `r#impl == None` fallback does,
             // and `r#impl` is `Some(Jsc)` here), so it is always `None`.
             debug_assert!(cache.output_code.is_none());
-            let result = String::clone_latin1(written);
+            let result = String::clone_utf8(written);
 
             // SAFETY: leaf scalar field read on `*vm`; see `vm` note above.
             if written.len() > 1024 * 1024 * 2 || unsafe { (*vm).smol } {

src/jsc/VirtualMachine.rs

@@ -3785,6 +3785,18 @@ impl VirtualMachine {
                 ..Default::default()
             };
         }
+        // `ref_counted_string` wraps the bytes as a Latin-1 external string.
+        // The runtime transpiler now emits UTF-8, so non-ASCII output would be
+        // mojibaked; transcode it instead of interning on that (rare) path.
+        if !bun_core::strings::is_all_ascii(code) {
+            return ResolvedSource {
+                source_code: bun_core::String::clone_utf8(code),
+                specifier,
+                source_url: create_if_different(&specifier, source_url),
+                source_code_needs_deref: true,
+                ..Default::default()
+            };
+        }
         // Const-generic bool can't be `!ADD_DOUBLE_REF`, so branch.
         let source = if ADD_DOUBLE_REF {
             self.ref_counted_string::<false>(code, hash_)

src/runtime/cli/repl.rs

@@ -1862,7 +1862,7 @@ impl<'a> Repl<'a> {
 
         if bun_js_printer::print_ast::<
             _,
-            /* ASCII_ONLY */ true,
+            /* IS_BUN_PLATFORM */ true,
             /* GENERATE_SOURCE_MAP */ false,
         >(
             &mut buffer_printer,

src/runtime/jsc_hooks.rs

@@ -3059,7 +3059,7 @@ fn transpile_source_code_inner(
                 // `None`.
                 debug_assert!(cache.output_code.is_none());
                 let written_len = written.len();
-                let source_code = bun_core::String::clone_latin1(written);
+                let source_code = bun_core::String::clone_utf8(written);
                 // `printer.ctx.buffer.deinit()`: release the
                 // large/--smol print buffer now instead of holding it until the
                 // next transpile. Replacing the printer drops the old buffer

test/bundler/transpiler/transpiler.test.js

@@ -2308,13 +2308,16 @@ console.log(<div {...obj} key="after" />);`),
     });
 
     it("unicode surrogates", () => {
-      expectPrinted_(`console.log("𐌴")`, 'console.log("\\uD800\\uDF34")');
-      expectPrinted_(`console.log("\\u{10334}")`, 'console.log("\\uD800\\uDF34")');
-      expectPrinted_(`console.log("\\uD800\\uDF34")`, 'console.log("\\uD800\\uDF34")');
+      expectPrinted_(`console.log("𐌴")`, 'console.log("𐌴")');
+      expectPrinted_(`console.log("\\u{10334}")`, 'console.log("𐌴")');
+      expectPrinted_(`console.log("\\uD800\\uDF34")`, 'console.log("𐌴")');
       expectPrinted_(`console.log("\\u{10334}" === "\\uD800\\uDF34")`, "console.log(true)");
       expectPrinted_(`console.log("\\u{10334}" === "\\uDF34\\uD800")`, "console.log(false)");
       expectPrintedMin_(`console.log("abc" + "def")`, 'console.log("abcdef")');
+      // Lone surrogates stay escaped; concatenation must not combine across operands.
       expectPrintedMin_(`console.log("\\uD800" + "\\uDF34")`, 'console.log("\\uD800" + "\\uDF34")');
+      expectPrinted_(`console.log("\\uD800")`, 'console.log("\\uD800")');
+      expectPrinted_(`console.log("\\uDF34")`, 'console.log("\\uDF34")');
     });
 
     it("fold string addition", () => {
@@ -3945,7 +3948,7 @@ console.log(foo, array);
 
       const input = `let list = ["•", "-", "◦", "▪", "▫"];\n`;
       const result = await transpiler.transform(input);
-      expect(result).toBe(`let list = [\"\\u2022\", \"-\", \"\\u25E6\", \"\\u25AA\", \"\\u25AB\"];\n`);
+      expect(result).toBe(`let list = ["•", "-", "◦", "▪", "▫"];\n`);
     });
   });
 

test/js/bun/transpiler/transpiler-non-ascii-source-text.test.ts

@@ -0,0 +1,106 @@
+import { describe, expect, test } from "bun:test";
+import { bunEnv, bunExe, tempDir } from "harness";
+
+// ES2026 §20.2.3.5: Function.prototype.toString must return the source text
+// the host has for the function. The runtime transpiler used to rewrite every
+// non-ASCII codepoint to an escape sequence, which leaked into toString(),
+// RegExp#source, and tagged-template .raw.
+
+const fixture = `
+function f() { return "café"; }
+class Café { méth(x) { return \`é\${x}é\`; } }
+const rx = () => /π+/u.test("ππ");
+const tag = (a) => a.raw[0];
+const raw = () => tag\`你好𐃘\\\\\`;
+const dyn = new Function('return "café";');
+
+console.log(JSON.stringify({
+  value: f(),
+  fn: f.toString(),
+  cls: Café.toString(),
+  rx: rx.toString(),
+  raw: raw(),
+  dyn: dyn.toString(),
+  emoji: (() => "👍").toString(),
+}));
+process.exit(0);
+`;
+
+async function run(cmd: string[], cwd: string) {
+  await using proc = Bun.spawn({
+    cmd,
+    env: bunEnv,
+    cwd,
+    stdout: "pipe",
+    stderr: "pipe",
+  });
+  const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]);
+  return { stdout, stderr, exitCode };
+}
+
+describe("runtime transpiler preserves non-ASCII source text", () => {
+  for (const [label, args] of [
+    [".mjs", ["entry.mjs"]],
+    [".cjs", ["entry.cjs"]],
+    [".ts", ["entry.ts"]],
+    [".mjs --hot", ["--hot", "entry.mjs"]],
+  ] as const) {
+    test(label, async () => {
+      const ext = args[args.length - 1].split(".").pop()!;
+      using dir = tempDir("non-ascii-src", { [`entry.${ext}`]: fixture });
+      const { stdout, stderr, exitCode } = await run([bunExe(), ...args], String(dir));
+      expect(stderr).toBe("");
+      const out = JSON.parse(stdout);
+
+      // Evaluated value — must be the real string, not mojibake
+      expect(out.value).toBe("café");
+
+      // Function.prototype.toString — string literal preserved verbatim
+      expect(out.fn).toContain('"café"');
+      expect(out.fn).not.toContain("\\xE9");
+      expect(out.fn).not.toContain("\\u");
+
+      // Class source — identifier, method name, template literal all preserved
+      expect(out.cls).toContain("Café");
+      expect(out.cls).toContain("méth");
+      expect(out.cls).toContain("`é${x}é`");
+      expect(out.cls).not.toContain("\\u{e9}");
+      expect(out.cls).not.toContain("\\xE9");
+
+      // RegExp literal inside a function body
+      expect(out.rx).toContain("/π+/u");
+      expect(out.rx).not.toContain("\\u03C0");
+
+      // Tagged template .raw — non-ASCII plus a raw backslash preserved
+      expect(out.raw).toBe("你好𐃘\\\\");
+
+      // Astral codepoint in a string literal
+      expect(out.emoji).toContain('"👍"');
+
+      // Control: new Function bodies never pass through the file transpiler
+      expect(out.dyn).toContain('"café"');
+
+      expect(exitCode).toBe(0);
+    });
+  }
+
+  test("RegExp.prototype.source", async () => {
+    using dir = tempDir("non-ascii-rx", {
+      "entry.mjs": `console.log(/π+/u.source);`,
+    });
+    const { stdout, stderr, exitCode } = await run([bunExe(), "entry.mjs"], String(dir));
+    expect(stderr).toBe("");
+    expect(stdout).toBe("π+\n");
+    expect(exitCode).toBe(0);
+  });
+
+  test("String.raw with non-ASCII", async () => {
+    using dir = tempDir("non-ascii-raw", {
+      "entry.mjs": `console.log(String.raw\`你好\\n𐃘\`);`,
+    });
+    const { stdout, stderr, exitCode } = await run([bunExe(), "entry.mjs"], String(dir));
+    expect(stderr).toBe("");
+    expect(stdout).toBe("你好\\n𐃘\n");
+    expect(exitCode).toBe(0);
+  });
+});

test/regression/issue/14976/14976.test.ts

@@ -70,7 +70,7 @@ test("constant-folded equals doesn't lie", async () => {
   console.log("\"" === '"');
 });
 
-test.skip("template literal raw property with unicode in an ascii-only build", async () => {
+test("template literal raw property with unicode", async () => {
   expect(String.raw`你好𐃘\\`).toBe("你好𐃘\\\\");
   expect((await $`echo 你好𐃘`.text()).trim()).toBe("你好𐃘");
 });