diff --git a/src/js_printer/lib.rs b/src/js_printer/lib.rs index a0c4584f3afe..979b9d37ea3e 100644 --- a/src/js_printer/lib.rs +++ b/src/js_printer/lib.rs @@ -4284,41 +4284,11 @@ pub mod __gated_printer { self.print(b" "); } - if IS_BUN_PLATFORM { - // Translate any non-ASCII to unicode escape sequences - let mut ascii_start: usize = 0; - let mut is_ascii = false; - let iter = CodepointIterator::init(&e.value); - let mut cursor = strings::Cursor::default(); - while iter.next(&mut cursor) { - match cursor.c as u32 { - FIRST_ASCII..=LAST_ASCII => { - if !is_ascii { - ascii_start = cursor.i as usize; - is_ascii = true; - } - } - _ => { - if is_ascii { - self.print(&e.value[ascii_start..(cursor.i as usize)]); - is_ascii = false; - } - - match cursor.c as u32 { - c @ 0..=0xFFFF => self.print(&bmp_escape(c)[..]), - c => self.print(&surrogate_pair_escape(c)[..]), - } - } - } - } - - if is_ascii { - self.print(&e.value[ascii_start..]); - } - } else { - // UTF8 sequence is fine - self.print(&e.value[..]); - } + // The pattern is printed verbatim (UTF-8), even under `IS_BUN_PLATFORM`: + // rewriting `/¶/u` as `/\u00B6/u` changes `RegExp.prototype.source` at + // runtime. The consumers of the transpiled buffer treat it as UTF-8 + // (see `String::clone_utf8` at the `ResolvedSource` construction sites). + self.print(&e.value[..]); // Need a space before the next identifier to avoid it turning into flags self.prev_reg_exp_end = self.writer.written(); diff --git a/src/jsc/AsyncModule.rs b/src/jsc/AsyncModule.rs index 18d2be1e5dba..a6936a364a80 100644 --- a/src/jsc/AsyncModule.rs +++ b/src/jsc/AsyncModule.rs @@ -1375,7 +1375,10 @@ impl AsyncModule { } Ok(ResolvedSource { - source_code: BunString::clone_latin1(printer.ctx.get_written()), + // `clone_utf8`: RegExp literals are printed verbatim (non-ASCII + // bytes possible); `BunString__fromBytes` stays Latin-1 when the + // output is all-ASCII. + source_code: BunString::clone_utf8(printer.ctx.get_written()), specifier: BunString::init(specifier), source_url: BunString::init(path.text), is_commonjs_module, diff --git a/src/jsc/RuntimeTranspilerCache.rs b/src/jsc/RuntimeTranspilerCache.rs index 1f2447a9ac9e..1d5c2dab613f 100644 --- a/src/jsc/RuntimeTranspilerCache.rs +++ b/src/jsc/RuntimeTranspilerCache.rs @@ -43,7 +43,10 @@ bun_core::declare_scope!(cache, visible); /// path reinstates the bug for any previously-cached TLA module (#30887). /// Version 23: `jsx.runtime`/`jsx.development` participate in the features hash, /// and tsconfig `"jsx": "react-jsx"` now emits the production runtime (#4227). -const EXPECTED_VERSION: u32 = 23; +/// Version 24: RegExp literals are printed verbatim (no `\uXXXX` escaping of +/// non-ASCII) so `RegExp.prototype.source` matches the source text (#13853); +/// cached output is now tagged `Encoding::UTF8`. +const EXPECTED_VERSION: u32 = 24; /// Source files smaller than this are not written to / read from the on-disk /// transpiler cache. Originally 50 KiB, which excluded almost every file in a @@ -1009,7 +1012,10 @@ impl RuntimeTranspilerCache { return; } debug_assert!(self.entry.is_none()); - let output_code = BunString::clone_latin1(output_code_bytes); + // Printer output is ASCII except for RegExp literals (printed verbatim + // so `.source` is preserved); `clone_utf8` keeps the Latin-1 fast path + // for the common all-ASCII case and transcodes to UTF-16 otherwise. + let output_code = BunString::clone_utf8(output_code_bytes); // Refcount stays at 1, sole owner. // BunString is Copy with no Drop, so an extra dupe_ref here would leak. self.output_code = Some(output_code); @@ -1028,7 +1034,7 @@ impl RuntimeTranspilerCache { } #[cfg(debug_assertions)] { - bun_core::scoped_log!(cache, "put() = {} bytes", output_code.latin1().len()); + bun_core::scoped_log!(cache, "put() = {} bytes", output_code_bytes.len()); } } } @@ -1078,10 +1084,13 @@ bun_ast::link_impl_TranspilerCacheImpl! { } debug_assert!(this.entry.is_none()); - // Borrowed Latin-1 view: `to_file` only reads `byte_slice()` + the encoding - // tag (unmarked 8-bit ZigString -> Encoding::LATIN1, same as clone_latin1), - // and `output_code_bytes` outlives the synchronous `to_file` call. - let output_code = BunString::ascii(output_code_bytes); + // Borrowed UTF-8 view: `to_file` only reads `byte_slice()` + the + // encoding tag, and `output_code_bytes` outlives the synchronous + // `to_file` call. Printer output is ASCII except for RegExp + // literals (printed verbatim so `.source` is preserved), so tag as + // UTF-8; the read path's `Encoding::UTF8` branch handles both the + // all-ASCII fast path and the rare multi-byte case. + let output_code = BunString::borrow_utf8(output_code_bytes); let result = RuntimeTranspilerCache::to_file( this.input_byte_length.unwrap(), this.input_hash.unwrap(), diff --git a/src/jsc/RuntimeTranspilerStore.rs b/src/jsc/RuntimeTranspilerStore.rs index acd5bb4d9438..65665312e2ca 100644 --- a/src/jsc/RuntimeTranspilerStore.rs +++ b/src/jsc/RuntimeTranspilerStore.rs @@ -1149,7 +1149,11 @@ impl TranspilerJob { // `cache.output_code` (only the `r#impl == None` fallback does, // and `r#impl` is `Some(Jsc)` here), so it is always `None`. debug_assert!(cache.output_code.is_none()); - let result = String::clone_latin1(written); + // `clone_utf8`: the printer emits ASCII-only output except for + // RegExp literals, which are printed verbatim so their `.source` + // is preserved; `BunString__fromBytes` keeps the Latin-1 fast + // path for the common all-ASCII case. + let result = String::clone_utf8(written); // SAFETY: leaf scalar field read on `*vm`; see `vm` note above. if written.len() > 1024 * 1024 * 2 || unsafe { (*vm).smol } { diff --git a/src/jsc/VirtualMachine.rs b/src/jsc/VirtualMachine.rs index 191c40190c98..fa0e491cbb89 100644 --- a/src/jsc/VirtualMachine.rs +++ b/src/jsc/VirtualMachine.rs @@ -3744,6 +3744,18 @@ impl VirtualMachine { ..Default::default() }; } + // The ref-string cache wraps `code` in a Latin-1 external string. When + // the printer emitted non-ASCII UTF-8 (currently only RegExp literals, + // printed verbatim so `.source` is preserved), interning as Latin-1 + // would corrupt those bytes, so fall back to a plain UTF-8 copy. + if !bun_core::strings::is_all_ascii(code) { + return ResolvedSource { + source_code: bun_core::String::clone_utf8(code), + specifier, + source_url: create_if_different(&specifier, source_url), + ..Default::default() + }; + } // Const-generic bool can't be `!ADD_DOUBLE_REF`, so branch. let source = if ADD_DOUBLE_REF { self.ref_counted_string::(code, hash_) diff --git a/src/runtime/jsc_hooks.rs b/src/runtime/jsc_hooks.rs index 0c719212132e..458b86d62d88 100644 --- a/src/runtime/jsc_hooks.rs +++ b/src/runtime/jsc_hooks.rs @@ -3154,7 +3154,11 @@ fn transpile_source_code_inner( // `None`. debug_assert!(cache.output_code.is_none()); let written_len = written.len(); - let source_code = bun_core::String::clone_latin1(written); + // `clone_utf8`: the printer emits ASCII-only output except for + // RegExp literals, which are printed verbatim so their `.source` + // is preserved; `BunString__fromBytes` keeps the Latin-1 fast + // path for the common all-ASCII case. + let source_code = bun_core::String::clone_utf8(written); // `printer.ctx.buffer.deinit()`: release the // large/--smol print buffer now instead of holding it until the // next transpile. Replacing the printer drops the old buffer diff --git a/test/regression/issue/13853/13853.test.ts b/test/regression/issue/13853/13853.test.ts new file mode 100644 index 000000000000..0bf74a98ae76 --- /dev/null +++ b/test/regression/issue/13853/13853.test.ts @@ -0,0 +1,149 @@ +// https://github.com/oven-sh/bun/issues/13853 +// RegExp literal .source must preserve non-ASCII characters from the original +// source text. Bun's runtime transpiler used to rewrite /¶/u as /\u00B6/u +// (to keep the printed output ASCII-only for a Latin-1 source pipeline), +// which changed the observable value of RegExp.prototype.source and broke +// packages such as parsel-js/Puppeteer that string-replace on .source. +import { expect, test } from "bun:test"; +import { bunEnv, bunExe, tempDir } from "harness"; +import { join } from "node:path"; + +test("RegExp literal .source preserves non-ASCII characters (#13853)", async () => { + // Spawn a fresh process so the fixture is run through the runtime transpiler + // (this test file itself is also transpiled, but the fixture's bytes are + // what we want the assertion to observe). + using dir = tempDir("issue-13853", { + "index.js": ` + const results = { + latin1_no_u: /\u00b6/.source, + latin1_u: /\u00b6/u.source, + latin1_v: /\u00b6/v.source, + cjk: /\u8981\u66ff\u6362/u.source, + astral: /\u{1d54f}/u.source, + // via new RegExp the source text is already a runtime string, + // so this was never broken; kept as a sanity check + runtime: new RegExp("\u00b6", "u").source, + }; + process.stdout.write(JSON.stringify(results)); + `, + }); + + await using proc = Bun.spawn({ + cmd: [bunExe(), "run", join(String(dir), "index.js")], + env: bunEnv, + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + + expect(stderr).toBe(""); + const got = JSON.parse(stdout); + expect(got).toEqual({ + latin1_no_u: "\u00b6", + latin1_u: "\u00b6", + latin1_v: "\u00b6", + cjk: "\u8981\u66ff\u6362", + astral: "\u{1d54f}", + runtime: "\u00b6", + }); + expect(exitCode).toBe(0); +}); + +test("parsel-js .source.replace pattern works (#13853)", async () => { + // Minimal reduction of what Puppeteer's bundled parsel-js does for + // ::-p-xpath() / ::-p-text(): build a RegExp with a literal PILCROW SIGN + // placeholder, then .source.replace("\u00b6*", ".*") to derive a second + // pattern. If .source escaped \u00b6 to "\\u00B6" the replace would miss + // and the derived pattern would fail to capture the argument. + using dir = tempDir("issue-13853-parsel", { + "index.js": ` + const TOKEN = /::(?[-\\w]+)(?:\\((?\u00b6*)\\))?/gu; + const src = TOKEN.source.replace("(?\u00b6*)", "(?.*)"); + const derived = new RegExp(src, "gu"); + derived.lastIndex = 0; + const m = derived.exec("::-p-xpath(//div)"); + process.stdout.write(JSON.stringify(m && m.groups)); + `, + }); + + await using proc = Bun.spawn({ + cmd: [bunExe(), "run", join(String(dir), "index.js")], + env: bunEnv, + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + + expect(stderr).toBe(""); + expect(JSON.parse(stdout)).toEqual({ name: "-p-xpath", argument: "//div" }); + expect(exitCode).toBe(0); +}); + +test("transpiler cache round-trip preserves non-ASCII RegExp .source (#13853)", async () => { + // The on-disk transpiler cache used to tag printer output as Latin-1, so a + // non-ASCII RegExp literal would be corrupted when read back on a cache hit. + // The file must exceed the 4 KiB minimum cache size. + const pad = Buffer.alloc(8 * 1024, "a").toString(); + using dir = tempDir("issue-13853-cache", { + "a.js": `/* ${pad} */\nprocess.stdout.write(/\u00b6\u65e5/u.source);\n`, + }); + const cacheDir = join(String(dir), ".cache"); + const env = { + ...bunEnv, + BUN_RUNTIME_TRANSPILER_CACHE_PATH: cacheDir, + BUN_DEBUG_ENABLE_RESTORE_FROM_TRANSPILER_CACHE: "1", + }; + + // First run: cache miss, writes the cache entry. + { + await using proc = Bun.spawn({ + cmd: [bunExe(), "run", join(String(dir), "a.js")], + env, + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + expect(stderr).toBe(""); + expect(stdout).toBe("\u00b6\u65e5"); + expect(exitCode).toBe(0); + } + + // Second run: cache hit, reads the entry back. + { + await using proc = Bun.spawn({ + cmd: [bunExe(), "run", join(String(dir), "a.js")], + env, + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + expect(stderr).toBe(""); + expect(stdout).toBe("\u00b6\u65e5"); + expect(exitCode).toBe(0); + } +}); + +test("non-ASCII RegExp literal still matches correctly (#2005 stays fixed)", async () => { + using dir = tempDir("issue-13853-match", { + "index.js": ` + const text = "\u8fd9\u662f\u4e00\u6bb5\u8981\u66ff\u6362\u7684\u6587\u5b57"; + process.stdout.write(JSON.stringify({ + literal: text.replace(/\u8981\u66ff\u6362/, ""), + ctor: text.replace(new RegExp("\u8981\u66ff\u6362"), ""), + })); + `, + }); + await using proc = Bun.spawn({ + cmd: [bunExe(), "run", join(String(dir), "index.js")], + env: bunEnv, + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + expect(stderr).toBe(""); + expect(JSON.parse(stdout)).toEqual({ + literal: "\u8fd9\u662f\u4e00\u6bb5\u7684\u6587\u5b57", + ctor: "\u8fd9\u662f\u4e00\u6bb5\u7684\u6587\u5b57", + }); + expect(exitCode).toBe(0); +});