diff --git a/src/bun_core/string/mod.rs b/src/bun_core/string/mod.rs index f9fbc7d42d79..fbf58b54a232 100644 --- a/src/bun_core/string/mod.rs +++ b/src/bun_core/string/mod.rs @@ -1833,7 +1833,7 @@ pub mod printer { const MALFORMED: i32 = -1; - /// Same algorithm as `bun_js_printer::write_pre_quoted_string`, except malformed UTF-8 becomes U+FFFD. + /// Same algorithm as `bun_js_printer::write_pre_quoted_string`. /// PERF: (quote_char, ascii_only, json, encoding) are runtime params — /// profile if it shows up on a hot path. pub fn write_pre_quoted_string( diff --git a/src/js_printer/lib.rs b/src/js_printer/lib.rs index 5ec6076920e5..acb06c9c9acd 100644 --- a/src/js_printer/lib.rs +++ b/src/js_printer/lib.rs @@ -1031,6 +1031,8 @@ where write_pre_quoted_string_inner::(text_in, writer, QUOTE_CHAR, ASCII_ONLY, JSON) } +const MALFORMED: i32 = -1; + /// `quote_char` / `ascii_only` / `json` are runtime args (were `const`): the /// branches on them are cheap and well-predicted, and collapsing the /// monomorphizations keeps the hot transpile pages dense (see the facade above). @@ -1083,14 +1085,23 @@ where let clamped_width = (width as usize).min(n.saturating_sub(i)); let c: i32 = match ENCODING { Encoding::Utf8 => { - let bytes: [u8; 4] = match clamped_width { - 1 => [text[i], 0, 0, 0], - 2 => [text[i], text[i + 1], 0, 0], - 3 => [text[i], text[i + 1], text[i + 2], 0], - 4 => [text[i], text[i + 1], text[i + 2], text[i + 3]], - _ => unreachable!(), - }; - strings::decode_wtf8_rune_t::(bytes, width, 0) + if width == 1 { + // width 1 with a byte >= 0x80 is a stray continuation byte or an invalid lead. + if text[i] >= 0x80 { + MALFORMED + } else { + text[i] as i32 + } + } else { + let bytes: [u8; 4] = match clamped_width { + 1 => [text[i], 0, 0, 0], + 2 => [text[i], text[i + 1], 0, 0], + 3 => [text[i], text[i + 1], text[i + 2], 0], + 4 => [text[i], text[i + 1], text[i + 2], text[i + 3]], + _ => unreachable!(), + }; + strings::decode_wtf8_rune_t::(bytes, width, MALFORMED) + } } Encoding::Ascii => { debug_assert!(text[i] <= 0x7F); @@ -1105,6 +1116,17 @@ where } }; + if c == MALFORMED { + if ascii_only { + writer.write_all(&bmp_escape(0xFFFD))?; + } else { + writer.write_all("\u{FFFD}".as_bytes())?; + } + // One byte, not `width`, so the bytes after a truncated sequence survive. + i += 1; + continue; + } + if can_print_without_escape(c, ascii_only) { match ENCODING { Encoding::Ascii | Encoding::Utf8 => { diff --git a/test/bundler/bundler_loader.test.ts b/test/bundler/bundler_loader.test.ts index bab25c3a0726..6c95789b9fc0 100644 --- a/test/bundler/bundler_loader.test.ts +++ b/test/bundler/bundler_loader.test.ts @@ -432,6 +432,46 @@ describe("bundler", async () => { }, }); + // A byte that is not valid UTF-8 in a text import or in a string of a data + // file prints as U+FFFD. The bytes around it are kept and the output stays + // valid UTF-8. "bun" prints the literal with ASCII escapes, "browser" writes + // the characters as UTF-8. + { + // "A" E9 "B" C3 "Z" 80 FF "[" C3 A9, E2 82 AC, F0 9F 98 80 "]" E2 + // E9 and C3 start a sequence that the next byte does not continue, 80 is a + // continuation byte on its own, FF is never valid, the bracket holds valid + // 2, 3 and 4 byte sequences, and the E2 at the end is cut off by the end + // of the file. Every bad sequence here breaks at its second byte, so one + // U+FFFD per bad byte (the printer) and one per maximal subpart + // (TextDecoder) give the same string. + const body = Buffer.from("41e942c35a80ff5bc3a9e282acf09f98805d", "hex"); + const text = Buffer.concat([body, Buffer.from([0xe2])]); + const json = Buffer.concat([Buffer.from('{"key":"'), body, Buffer.from('"}')]); + const expected = "41 fffd 42 fffd 5a fffd fffd 5b e9 20ac 1f600 5d"; + for (const target of ["bun", "browser"] as const) { + itBundled(`${target}/loader-ill-formed-utf8`, { + target, + files: { + "/entry.ts": /* js */ ` + import text from "./chars.txt" with { type: "text" }; + import json from "./chars.json"; + const codePoints = (s) => [...s].map(c => c.codePointAt(0).toString(16)).join(" "); + console.log(codePoints(text)); + console.log(codePoints(json.key)); + `, + "/chars.txt": text, + "/chars.json": json, + }, + onAfterBundle(api) { + const output = fs.readFileSync(api.outfile); + expect(() => new TextDecoder("utf-8", { fatal: true }).decode(output)).not.toThrow(); + expect(output.includes(target === "bun" ? "\\uFFFD" : "\uFFFD")).toBe(true); + }, + run: { stdout: `${expected} fffd\n${expected}` }, + }); + } + } + const loaders: Loader[] = ["wasm", "json", "file" /* "napi" */, "text"]; const exts = ["wasm", "json", "lmao" /* ".node" */, "txt"]; for (let i = 0; i < loaders.length; i++) { diff --git a/test/js/bun/import-attributes/import-attributes.test.ts b/test/js/bun/import-attributes/import-attributes.test.ts index 2c1888662f74..2ab670a409e4 100644 --- a/test/js/bun/import-attributes/import-attributes.test.ts +++ b/test/js/bun/import-attributes/import-attributes.test.ts @@ -99,7 +99,7 @@ type Tests = Record< const default_tests = Object.fromEntries( loaders.map(loader => [loader, { loader, filename: "no_extension" }]), ) as Tests; -async function compileAndTest(code: string, tests: Tests = default_tests): Promise> { +async function compileAndTest(code: string | Buffer, tests: Tests = default_tests): Promise> { const [v1, v2, v3] = await Promise.all([ compileAndTest_inner(code, tests, testBunRun), compileAndTest_inner(code, tests, testBunRunAwaitImport), @@ -114,7 +114,7 @@ async function compileAndTest(code: string, tests: Tests = default_tests): Promi return v1; } async function compileAndTest_inner( - code: string, + code: string | Buffer, tests: Tests, cb: (dir: string, loader: string | null, filename: string) => Promise, ): Promise> { @@ -131,7 +131,7 @@ async function compileAndTest_inner( ); let res: Record = Object.fromEntries(results); if (Object.hasOwn(res, "text")) { - expect(res.text).toEqual({ default: code }); + expect(res.text).toEqual({ default: typeof code === "string" ? code : code.toString("utf8") }); delete res.text; } if (Object.hasOwn(res, "yaml")) { @@ -292,6 +292,27 @@ test("yaml", async () => { `); }); +// A byte that is not valid UTF-8 becomes U+FFFD and the bytes around it are +// kept, both in a text import and in a string value of a data file. +test("ill-formed UTF-8", async () => { + // {"key":"A E9 B C3 Z 80 FF [é€😀]"}: E9 and C3 start a sequence that the + // next byte does not continue, 80 is a continuation byte on its own, FF is + // never valid, and the bracket holds valid 2, 3 and 4 byte sequences. Every + // bad sequence breaks at its second byte, so one U+FFFD per bad byte (the + // printer) and one per maximal subpart (TextDecoder, the runtime JSON + // loader) give the same string and all import paths agree. + const code = Buffer.concat([ + Buffer.from('{"key":"'), + Buffer.from("41e942c35a80ff5bc3a9e282acf09f98805d", "hex"), + Buffer.from('"}'), + ]); + const key = "A\uFFFDB\uFFFDZ\uFFFD\uFFFD[\u00E9\u20AC\u{1F600}]"; + expect(await compileAndTest(code)).toEqual({ + "js,jsx,ts,tsx,toml": "error", + "json,jsonc,yaml": { default: { key }, key }, + }); +}); + test("tsconfig.json is assumed jsonc", async () => { const tests: Tests = { "tsconfig.json": { loader: null, filename: "tsconfig.json" },