From ed157778fbfee64f22e115a6f464e17f739dcba2 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Sat, 22 Aug 2026 23:21:17 -0700 Subject: [PATCH 1/6] WIP: --compile embeds text imports as pre-encoded string modules (unbuilt draft) --- src/bun_core/string/mod.rs | 8 + src/bundler/ParseTask.rs | 77 +++++++++- src/bundler/bundle_v2.rs | 9 ++ src/runtime/jsc_hooks.rs | 29 +++- src/standalone_graph/StandaloneModuleGraph.rs | 140 +++++++++++++++--- 5 files changed, 236 insertions(+), 27 deletions(-) diff --git a/src/bun_core/string/mod.rs b/src/bun_core/string/mod.rs index 2eae4e7d1ea8..496ee1f7802c 100644 --- a/src/bun_core/string/mod.rs +++ b/src/bun_core/string/mod.rs @@ -355,6 +355,14 @@ impl String { // without copying and never frees it. unsafe { BunString__createStaticExternal(bytes.as_ptr(), bytes.len(), is_latin1) } } + /// UTF-16 form of [`Self::create_static_external`]: `units` must be + /// 2-byte aligned and live for the rest of the process. + pub fn create_static_external_utf16(units: &[u16]) -> Self { + debug_assert!(!units.is_empty()); + // SAFETY: the C++ side takes the length in code units and stores + // ptr/len without copying or freeing. + unsafe { BunString__createStaticExternal(units.as_ptr().cast::(), units.len(), false) } + } /// `bun.String.createFormat` — formats `args` into a temporary buffer and /// copies the result into a fresh WTF-backed string. pub fn create_format(args: core::fmt::Arguments<'_>) -> Self { diff --git a/src/bundler/ParseTask.rs b/src/bundler/ParseTask.rs index 7c72ac077fb4..97d65ca85a8a 100644 --- a/src/bundler/ParseTask.rs +++ b/src/bundler/ParseTask.rs @@ -886,6 +886,74 @@ pub mod parse_worker { return result; } Loader::Text => { + if topts.compile_mode.is_executable() { + // In a standalone executable the text is embedded as an + // asset and the module becomes + // `export default import.meta.require("")`; the + // runtime answers with a string that aliases the section + // bytes (`StandaloneModuleGraph::encode_text_module`). + // Inlining it as a string literal instead would ship the + // text twice (source + bytecode constant) and parse it on + // load. + let mut buf = bun_alloc::ArenaString::new_in(bump); + write!( + &mut buf, + "{}", + crate::chunk::UniqueKey { + prefix: unique_key_prefix, + kind: crate::chunk::QueryKind::Asset, + index: source.index.0, + }, + ) + .expect("unreachable"); + let unique_key = buf.into_bump_str().as_bytes(); + *unique_key_for_additional_file = FileLoaderHash { + key: ast::StoreStr::new(unique_key), + content_hash: ContentHasher::run(&source.contents), + }; + let import_path = Expr::init( + E::String { + data: unique_key.into(), + ..Default::default() + }, + Loc { start: 0 }, + ); + let import_meta = Expr::init(E::ImportMeta {}, Loc { start: 0 }); + let require_property = Expr::init( + E::Dot { + target: import_meta, + name_loc: Loc::EMPTY, + name: b"require".into(), + ..Default::default() + }, + Loc { start: 0 }, + ); + let require_args = bump.alloc_slice_fill_default::(1); + require_args[0] = import_path; + let root = Expr::init( + E::Call { + target: require_property, + // SAFETY: bump-owned slice; never grown via this Vec. + args: unsafe { bun_ast::ExprNodeList::from_bump_slice(require_args) }, + ..Default::default() + }, + Loc { start: 0 }, + ); + let mut ast = JSAst::init( + js_parser::new_lazy_export_ast( + bump, + &mut topts.define, + opts, + log, + root, + source, + b"", + )? + .ok_or(AnyError::ParserError)?, + ); + ast.add_url_for_css(bump, source, None, Some(unique_key), false); + return Ok(ast); + } let root = Expr::init( E::String { data: source.contents().into(), @@ -2676,12 +2744,9 @@ pub mod parse_worker { loader, package_name: task.package_name, - // Hash the files in here so that we do it in parallel. - content_hash_for_additional_file: if loader.should_copy_for_bundling() { - unique_key_for_additional_file.content_hash - } else { - 0 - }, + // Hash the files in here so that we do it in parallel. Zero unless + // the loader registered an asset (`FileLoaderHash` default). + content_hash_for_additional_file: unique_key_for_additional_file.content_hash, }) } diff --git a/src/bundler/bundle_v2.rs b/src/bundler/bundle_v2.rs index 24c9a1d4508e..3592bab478dd 100644 --- a/src/bundler/bundle_v2.rs +++ b/src/bundler/bundle_v2.rs @@ -7097,6 +7097,15 @@ pub mod bv2_impl { .input_files .items_content_hash_for_additional_file_mut()[result_source_index] = result.content_hash_for_additional_file; + if !result.unique_key_for_additional_file.is_empty() + && result.loader == Loader::Text + { + // Text embedded as an asset by `--compile` (ParseTask); + // the importer-side bookkeeping for copied loaders + // never saw it, so count it here or + // `process_files_to_copy` skips the whole pass. + this.graph.estimated_file_loader_count += 1; + } if !result.unique_key_for_additional_file.is_empty() && result.loader.should_copy_for_bundling() { diff --git a/src/runtime/jsc_hooks.rs b/src/runtime/jsc_hooks.rs index 932e1dbce99d..4d999c3d2ecc 100644 --- a/src/runtime/jsc_hooks.rs +++ b/src/runtime/jsc_hooks.rs @@ -3896,7 +3896,7 @@ fn get_hardcoded_module( /// `jsc_vm` is the live per-thread VM; `out` is a valid out-param. unsafe fn fetch_builtin_module( jsc_vm: *mut VirtualMachine, - _global: *mut JSGlobalObject, + global: *mut JSGlobalObject, specifier: &bun_core::String, _referrer: &bun_core::String, out: *mut ErrorableResolvedSource, @@ -4008,6 +4008,33 @@ export default db; return FetchBuiltinResult::Found; } + if file.loader == Loader::Text { + // A text import embedded by `--compile`. The bytes are already + // a Latin-1/UTF-16 string body (`encode_text_module`), so the + // default export is a JSString over the section — no parse, no + // bytecode, no copy. + // SAFETY: per fn contract — `global` is the live global object. + let global = unsafe { &*global }; + let string = file.to_wtf_string(); + // `jsString(vm, String(impl))` over the cached static impl; + // only a `Dead` string throws and `to_wtf_string` never + // produces one. + let value = bun_jsc::bun_string_jsc::to_js(&string, global) + .expect("embedded text module string is never dead"); + string.deref(); + // SAFETY: per fn contract — `out` is a valid out-param. + unsafe { + *out = ErrorableResolvedSource::ok(ResolvedSource { + jsvalue_for_export: value, + specifier: specifier.dupe_ref(), + source_url: specifier.dupe_ref(), + tag: bun_jsc::resolved_source::Tag::ExportDefaultObject, + ..ResolvedSource::default() + }); + } + return FetchBuiltinResult::Found; + } + let bytecode_len = file.bytecode.len(); let module_info_len = file.module_info.len(); // SAFETY: per fn contract — `out` is a valid out-param. diff --git a/src/standalone_graph/StandaloneModuleGraph.rs b/src/standalone_graph/StandaloneModuleGraph.rs index 27d7aaeab3df..4cfd94ab03a0 100644 --- a/src/standalone_graph/StandaloneModuleGraph.rs +++ b/src/standalone_graph/StandaloneModuleGraph.rs @@ -2,7 +2,7 @@ //! But this incurred a fixed 350ms overhead on every build, which is unacceptable //! so we give up on codesigning support on macOS for now until we can find a better solution -use core::mem::size_of; +use core::mem::{align_of, size_of}; use core::ptr::NonNull; use std::io::Write as _; use std::sync::Arc; @@ -351,6 +351,9 @@ pub enum Encoding { Latin1 = 1, // Not used yet. Utf8 = 2, + /// Native-endian UTF-16 code units, 2-byte aligned in the section. Only + /// produced for `Loader::Text` modules whose text does not fit in Latin-1. + Utf16 = 3, } #[repr(u8)] @@ -502,6 +505,9 @@ impl File { } pub fn to_wtf_string(&mut self) -> BunString { + if self.contents.is_empty() { + return BunString::EMPTY; + } if self.wtf_string.is_empty() { match self.encoding { Encoding::Binary | Encoding::Utf8 => { @@ -511,6 +517,19 @@ impl File { self.wtf_string = BunString::create_static_external(self.contents.as_bytes(), true); } + Encoding::Utf16 => { + let bytes = self.contents.as_bytes(); + // `to_bytes` pads to an even section offset; the section + // base is page-aligned on every platform (the bytecode + // cache relies on the same property at 128 bytes). + debug_assert!(bytes.as_ptr().addr() % align_of::() == 0); + // SAFETY: even byte count at a 2-byte-aligned offset of a + // section that is never freed. + let units = unsafe { + core::slice::from_raw_parts(bytes.as_ptr().cast::(), bytes.len() / 2) + }; + self.wtf_string = BunString::create_static_external_utf16(units); + } } } // We don't want this to free. @@ -819,6 +838,73 @@ unsafe fn slice_to_z(base: *const u8, len: usize, ptr: StringPointer) -> &'stati unsafe { ZStr::from_raw(base.add(off), n) } } +/// A `Loader::Text` import that the bundler emitted as an embedded asset (see +/// the `Loader::Text` arm of `ParseTask` under `CompileMode::Executable`). The +/// chunk that imports it does `import.meta.require("")`, and the +/// runtime answers with a string that aliases the section bytes, so the bytes +/// are stored already in WTF string form rather than as UTF-8. +fn is_text_module(output_file: &OutputFile) -> bool { + output_file.loader == Loader::Text && output_file.output_kind == options::OutputKind::Asset +} + +/// Write `utf8` as the in-memory representation of a `WTF::StringImpl`: Latin-1 +/// when every code point fits in a byte (the all-ASCII case needs no +/// transcoding at all), else native-endian UTF-16 at an even offset so the +/// runtime can point a `char16_t*` at it. Returns the contents pointer and the +/// encoding to record on the module. +fn encode_text_module( + string_builder: &mut bun_core::StringBuilder, + utf8: &[u8], +) -> (StringPointer, Encoding) { + if strings::first_non_ascii(utf8).is_none() { + return (string_builder.append_count_z(utf8), Encoding::Latin1); + } + // Invalid UTF-8 becomes U+FFFD, matching what the text loader's + // `export default "..."` string would have held. + let units = match strings::to_utf16_alloc(utf8, false, false) { + Ok(Some(units)) => units, + // `None` only for all-ASCII input, handled above. + Ok(None) => unreachable!("to_utf16_alloc returned None for non-ASCII input"), + Err(_) => bun_alloc::out_of_memory(), + }; + if units.iter().all(|&unit| unit <= 0xFF) { + let start = string_builder.len; + let dst = string_builder.writable(); + for (dst, &unit) in dst.iter_mut().zip(units.iter()) { + *dst = unit as u8; + } + dst[units.len()] = 0; + string_builder.len += units.len() + 1; + return ( + StringPointer { + offset: start as u32, + length: units.len() as u32, + }, + Encoding::Latin1, + ); + } + if string_builder.len % align_of::() != 0 { + string_builder.writable()[0] = 0; + string_builder.len += 1; + } + let start = string_builder.len; + let byte_len = units.len() * 2; + let dst = string_builder.writable(); + for (dst, unit) in dst.chunks_exact_mut(2).zip(units.iter()) { + dst.copy_from_slice(&unit.to_ne_bytes()); + } + dst[byte_len] = 0; + dst[byte_len + 1] = 0; + string_builder.len += byte_len + 2; + ( + StringPointer { + offset: start as u32, + length: byte_len as u32, + }, + Encoding::Utf16, + ) +} + pub(crate) fn to_bytes( prefix: &[u8], output_files: &[OutputFile], @@ -858,6 +944,11 @@ pub(crate) fn to_bytes( } string_builder.count_z(bytes); + if is_text_module(output_file) { + // Worst case for `encode_text_module`: every byte widens to + // a UTF-16 unit, plus alignment padding and a 2-byte NUL. + string_builder.cap += bytes.len() + 3; + } module_count += 1; } } @@ -1026,27 +1117,36 @@ pub(crate) fn to_bytes( StringPointer::default() }; + let name = string_builder.fmt_append_count_z(format_args!( + "{}{}", + bstr::BStr::new(prefix), + bstr::BStr::new(dest_path) + )); + let (contents, encoding) = if is_text_module(output_file) { + encode_text_module(&mut string_builder, buf_bytes) + } else { + ( + string_builder.append_count_z(buf_bytes), + // Latin1 lets the runtime wrap the mmapped section bytes in a + // zero-copy ExternalStringImpl. The printer escapes non-ASCII for + // server-side JS, but `--banner`/`--footer`/hashbang and + // client-side (target=browser) chunks are concatenated verbatim + // as UTF-8, so verify the final bytes before committing to Latin1. + match output_file.loader { + Loader::Js | Loader::Jsx | Loader::Ts | Loader::Tsx + if strings::first_non_ascii(buf_bytes).is_none() => + { + Encoding::Latin1 + } + _ => Encoding::Binary, + }, + ) + }; let mut module = CompiledModuleGraphFile { - name: string_builder.fmt_append_count_z(format_args!( - "{}{}", - bstr::BStr::new(prefix), - bstr::BStr::new(dest_path) - )), + name, loader: output_file.loader, - contents: string_builder.append_count_z(buf_bytes), - // Latin1 lets the runtime wrap the mmapped section bytes in a - // zero-copy ExternalStringImpl. The printer escapes non-ASCII for - // server-side JS, but `--banner`/`--footer`/hashbang and - // client-side (target=browser) chunks are concatenated verbatim - // as UTF-8, so verify the final bytes before committing to Latin1. - encoding: match output_file.loader { - Loader::Js | Loader::Jsx | Loader::Ts | Loader::Tsx - if strings::first_non_ascii(buf_bytes).is_none() => - { - Encoding::Latin1 - } - _ => Encoding::Binary, - }, + contents, + encoding, module_format: if output_file.loader.is_javascript_like() { match output_format { Format::Cjs => ModuleFormat::Cjs, From 9ce8bb23b016d631f4f2ac863eef8968caa3f7e3 Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Sun, 23 Aug 2026 07:30:08 +0000 Subject: [PATCH 2/6] bun build --compile: embed text imports as pre-encoded strings A `Loader::Text` import in a standalone executable used to become a JS module with a string literal. The binary carried the text as module source and, with --bytecode, again as a UTF-16 constant in the bytecode cache, and the runtime copied it onto the heap on load. The bundler now emits the text as an embedded asset in compile mode. `StandaloneModuleGraph::to_bytes` stores it as a WTF::StringImpl body: Latin-1 when every code point fits, UTF-16LE at an even offset otherwise. The chunk holds `require("/$bunfs/root/-.txt")`, and `fetch_builtin_module` answers with a JSString over the section bytes (ExportDefaultObject), so there is no parse, no bytecode, and no copy. Also: - ModuleLoader.cpp handles the ExportDefaultObject tag from Bun__fetchBuiltinModule in both the require() and the ESM path. - The lazy-export ESM path imports the runtime `__require` when the export is a `require()` call. `import addon from "./x.node"` in ESM output threw "__require is not defined" before. - Text modules ignore --asset-naming so two same-named files never share a path. `to_bytes` skips a repeated path instead of miscounting. - `Bun.embeddedFiles` does not list text modules. - Browser chunks of a full-stack executable keep the inline literal. Co-authored-by: Jarred Sumner --- docs/bundler/executables.mdx | 13 + src/bundler/ParseTask.rs | 223 +++++++----------- src/bundler/bundle_v2.rs | 62 +++-- .../generateCodeForLazyExport.rs | 23 +- src/jsc/bindings/ModuleLoader.cpp | 21 ++ src/runtime/jsc_hooks.rs | 4 +- src/standalone_graph/StandaloneModuleGraph.rs | 90 ++++--- test/bundler/bundler_compile.test.ts | 145 ++++++++++++ test/bundler/bundler_loader.test.ts | 20 ++ 9 files changed, 401 insertions(+), 200 deletions(-) diff --git a/docs/bundler/executables.mdx b/docs/bundler/executables.mdx index 9e81077bb05e..d5ae8754fd66 100644 --- a/docs/bundler/executables.mdx +++ b/docs/bundler/executables.mdx @@ -758,6 +758,19 @@ const wasmModule = await WebAssembly.instantiate(wasmBytes); const fontData = await file(fontPath).bytes(); ``` +### Embed text as a string + +Use the `text` loader when you want the contents of a file as a string. `.txt` files use it by default. For other extensions, add `with { type: "text" }` or pass `--loader .md:text`. + +```ts index.ts icon="/icons/typescript.svg" +import notes from "./notes.txt"; +import readme from "./README.md" with { type: "text" }; + +console.log(notes); // the contents of notes.txt +``` + +In a compiled executable the text is stored once, in the form the JavaScript engine uses for strings (Latin-1 when every character fits, UTF-16 otherwise). The runtime hands the string back without a copy, so a large text import costs its size in the binary and no extra copy on the heap. A text import is a module, not a file: `Bun.embeddedFiles` does not list it. Use `with { type: "file" }` when you need a file path instead. + ### Embed SQLite databases To embed a SQLite database into the compiled executable, set `type: "sqlite"` in the import attribute and the `embed` attribute to `"true"`. diff --git a/src/bundler/ParseTask.rs b/src/bundler/ParseTask.rs index 97d65ca85a8a..e0275e796db8 100644 --- a/src/bundler/ParseTask.rs +++ b/src/bundler/ParseTask.rs @@ -653,6 +653,60 @@ pub mod parse_worker { pub(crate) content_hash: u64, } + /// Registers `source` as an embedded asset: the returned unique key is + /// replaced with the asset's final path when the chunk is printed, and + /// `process_files_to_copy` emits the bytes as an `OutputKind::Asset`. + fn register_embedded_asset<'b>( + bump: &'b Bump, + source: &Source, + unique_key_prefix: u64, + unique_key_for_additional_file: &mut FileLoaderHash, + ) -> &'b [u8] { + use core::fmt::Write as _; + let mut buf = bun_alloc::ArenaString::new_in(bump); + write!( + &mut buf, + "{}", + crate::chunk::UniqueKey { + prefix: unique_key_prefix, + kind: crate::chunk::QueryKind::Asset, + index: source.index.0, + }, + ) + .expect("unreachable"); + let unique_key = buf.into_bump_str().as_bytes(); + *unique_key_for_additional_file = FileLoaderHash { + key: ast::StoreStr::new(unique_key), + content_hash: ContentHasher::run(&source.contents), + }; + unique_key + } + + /// `require("")`. The printer turns the call target into the + /// CommonJS `require` or the runtime's `__require` to match the output + /// format (`generateCodeForLazyExport` imports the latter), so the chunk + /// stays free of `import.meta` and `--bytecode` can compile it. + fn require_embedded_asset(unique_key: &[u8]) -> Expr { + let import_path = Expr::init( + E::String { + data: unique_key.into(), + ..Default::default() + }, + Loc { start: 0 }, + ); + Expr::init( + E::Call { + target: Expr { + data: ast::ExprData::ERequireCallTarget, + loc: Loc { start: 0 }, + }, + args: bun_ast::ExprNodeList::from_arena_slice(&[import_path]), + ..Default::default() + }, + Loc { start: 0 }, + ) + } + // ─────────────────────────────────────────────────────────────────────────── // CSS Symbol bridge — `bun_ast::Symbol` ↔ `bun_ast::Symbol` // @@ -886,81 +940,30 @@ pub mod parse_worker { return result; } Loader::Text => { - if topts.compile_mode.is_executable() { - // In a standalone executable the text is embedded as an - // asset and the module becomes - // `export default import.meta.require("")`; the - // runtime answers with a string that aliases the section - // bytes (`StandaloneModuleGraph::encode_text_module`). - // Inlining it as a string literal instead would ship the - // text twice (source + bytecode constant) and parse it on - // load. - let mut buf = bun_alloc::ArenaString::new_in(bump); - write!( - &mut buf, - "{}", - crate::chunk::UniqueKey { - prefix: unique_key_prefix, - kind: crate::chunk::QueryKind::Asset, - index: source.index.0, - }, - ) - .expect("unreachable"); - let unique_key = buf.into_bump_str().as_bytes(); - *unique_key_for_additional_file = FileLoaderHash { - key: ast::StoreStr::new(unique_key), - content_hash: ContentHasher::run(&source.contents), - }; - let import_path = Expr::init( + // In a standalone executable the text is embedded as an asset + // and the module becomes `export default require("")`. + // The runtime answers with a string that aliases the section + // bytes (`StandaloneModuleGraph::encode_text_module`). A string + // literal would ship the text twice (module source + bytecode + // constant) and parse it on load. Browser chunks of a + // full-stack build cannot reach the embedded graph and keep + // the literal. + let root = if topts.compile_mode.is_executable() && topts.target.is_bun() { + require_embedded_asset(register_embedded_asset( + bump, + source, + unique_key_prefix, + unique_key_for_additional_file, + )) + } else { + Expr::init( E::String { - data: unique_key.into(), - ..Default::default() - }, - Loc { start: 0 }, - ); - let import_meta = Expr::init(E::ImportMeta {}, Loc { start: 0 }); - let require_property = Expr::init( - E::Dot { - target: import_meta, - name_loc: Loc::EMPTY, - name: b"require".into(), - ..Default::default() - }, - Loc { start: 0 }, - ); - let require_args = bump.alloc_slice_fill_default::(1); - require_args[0] = import_path; - let root = Expr::init( - E::Call { - target: require_property, - // SAFETY: bump-owned slice; never grown via this Vec. - args: unsafe { bun_ast::ExprNodeList::from_bump_slice(require_args) }, + data: source.contents().into(), ..Default::default() }, Loc { start: 0 }, - ); - let mut ast = JSAst::init( - js_parser::new_lazy_export_ast( - bump, - &mut topts.define, - opts, - log, - root, - source, - b"", - )? - .ok_or(AnyError::ParserError)?, - ); - ast.add_url_for_css(bump, source, None, Some(unique_key), false); - return Ok(ast); - } - let root = Expr::init( - E::String { - data: source.contents().into(), - ..Default::default() - }, - Loc { start: 0 }, - ); + ) + }; let mut ast = JSAst::init( js_parser::new_lazy_export_ast( bump, @@ -1035,29 +1038,15 @@ pub mod parse_worker { return Err(crate::Error::ParserError); } - let path_to_use: &[u8] = 'brk: { - // Implements embedded sqlite - if loader == Loader::SqliteEmbedded { - let mut buf = bun_alloc::ArenaString::new_in(bump); - write!( - &mut buf, - "{}", - crate::chunk::UniqueKey { - prefix: unique_key_prefix, - kind: crate::chunk::QueryKind::Asset, - index: source.index.0, - }, - ) - .expect("unreachable"); - let embedded_path = buf.into_bump_str().as_bytes(); - *unique_key_for_additional_file = FileLoaderHash { - key: ast::StoreStr::new(embedded_path), - content_hash: ContentHasher::run(&source.contents), - }; - break 'brk embedded_path; - } - - break 'brk source.path.text; + let path_to_use: &[u8] = if loader == Loader::SqliteEmbedded { + register_embedded_asset( + bump, + source, + unique_key_prefix, + unique_key_for_additional_file, + ) + } else { + source.path.text }; // This injects the following code: @@ -1156,50 +1145,16 @@ pub mod parse_worker { return Err(crate::Error::ParserError); } - let mut buf = bun_alloc::ArenaString::new_in(bump); - write!( - &mut buf, - "{}", - crate::chunk::UniqueKey { - prefix: unique_key_prefix, - kind: crate::chunk::QueryKind::Asset, - index: source.index.0, - }, - ) - .expect("unreachable"); - let unique_key = buf.into_bump_str().as_bytes(); // This injects the following code: // // require(unique_key) // - let import_path = Expr::init( - E::String { - data: unique_key.into(), - ..Default::default() - }, - Loc { start: 0 }, - ); - - let require_args = bump.alloc_slice_fill_default::(1); - require_args[0] = import_path; - - let root = Expr::init( - E::Call { - target: Expr { - data: ast::ExprData::ERequireCallTarget, - loc: Loc { start: 0 }, - }, - // SAFETY: bump-owned slice; never grown via this Vec. - args: unsafe { bun_ast::ExprNodeList::from_bump_slice(require_args) }, - ..Default::default() - }, - Loc { start: 0 }, - ); - - *unique_key_for_additional_file = FileLoaderHash { - key: ast::StoreStr::new(unique_key), - content_hash: ContentHasher::run(&source.contents), - }; + let root = require_embedded_asset(register_embedded_asset( + bump, + source, + unique_key_prefix, + unique_key_for_additional_file, + )); return Ok(JSAst::init( js_parser::new_lazy_export_ast( bump, diff --git a/src/bundler/bundle_v2.rs b/src/bundler/bundle_v2.rs index 3592bab478dd..dda0bad394c2 100644 --- a/src/bundler/bundle_v2.rs +++ b/src/bundler/bundle_v2.rs @@ -4191,28 +4191,39 @@ pub mod bv2_impl { let index = reachable_source.get() as usize; let key: &[u8] = &unique_key_for_additional_files[index]; if !key.is_empty() { - let mut template: options::PathTemplate = - if self.graph.html_imports.server_source_indices.len() != 0 - && self.transpiler.options.asset_naming.is_empty() - { - options::PathTemplate::ASSET_WITH_TARGET.into() - } else { - options::PathTemplate::ASSET.into() - }; - + let loader = loaders[index]; let target = targets[index]; - // SAFETY: see `self_ptr` note above — `transpiler_for_target` needs - // `&mut self` only to pick between two stored `*mut Transpiler`s; it - // never touches `graph.input_files`. - let asset_naming = unsafe { - &(*self_ptr) - .transpiler_for_target(target) - .options - .asset_naming + let mut template: options::PathTemplate = if loader == Loader::Text { + // A text module embedded by `--compile` (ParseTask's + // `Loader::Text` arm) is only ever addressed through + // the bunfs path the bundler prints into the chunk, so + // `--asset-naming` does not apply: without `[hash]` + // two same-named files would silently share one path. + options::PathTemplate::ASSET.into() + } else { + let mut template: options::PathTemplate = + if self.graph.html_imports.server_source_indices.len() != 0 + && self.transpiler.options.asset_naming.is_empty() + { + options::PathTemplate::ASSET_WITH_TARGET.into() + } else { + options::PathTemplate::ASSET.into() + }; + + // SAFETY: see `self_ptr` note above — `transpiler_for_target` needs + // `&mut self` only to pick between two stored `*mut Transpiler`s; it + // never touches `graph.input_files`. + let asset_naming = unsafe { + &(*self_ptr) + .transpiler_for_target(target) + .options + .asset_naming + }; + if !asset_naming.is_empty() { + template.data.clone_from(asset_naming); + } + template }; - if !asset_naming.is_empty() { - template.data.clone_from(asset_naming); - } let source = &mut sources[index]; @@ -4253,8 +4264,6 @@ pub mod bv2_impl { v.into_boxed_slice() }; - let loader = loaders[index]; - // Hand the existing `source.contents` buffer to the // OutputFile — no copy: move the contents // out instead of `to_vec()`-cloning, @@ -7100,10 +7109,11 @@ pub mod bv2_impl { if !result.unique_key_for_additional_file.is_empty() && result.loader == Loader::Text { - // Text embedded as an asset by `--compile` (ParseTask); - // the importer-side bookkeeping for copied loaders - // never saw it, so count it here or - // `process_files_to_copy` skips the whole pass. + // A text module embedded as an asset by `--compile`. The + // importer-side count in `process_resolve_queue` only + // covers `should_copy_for_bundling()` loaders, and + // `process_files_to_copy` skips the whole pass when the + // count is zero. this.graph.estimated_file_loader_count += 1; } if !result.unique_key_for_additional_file.is_empty() diff --git a/src/bundler/linker_context/generateCodeForLazyExport.rs b/src/bundler/linker_context/generateCodeForLazyExport.rs index ba694c1c49d5..14b464a4f11c 100644 --- a/src/bundler/linker_context/generateCodeForLazyExport.rs +++ b/src/bundler/linker_context/generateCodeForLazyExport.rs @@ -373,6 +373,13 @@ pub(crate) fn generate_code_for_lazy_export( loc: stmt.loc, }; + // `require()` from the napi loader and the `--compile` text + // loader prints as the runtime's `__require` outside CommonJS output, so + // the part that holds the call has to import that symbol. + let calls_runtime_require = matches!(expr.data, ExprData::ECall(ref c) + if matches!(c.target.data, ExprData::ERequireCallTarget)) + && this.options.output_format != crate::options::OutputFormat::Cjs; + match exports_kind { bun_ast::ExportsKind::Cjs => { part.stmts.slice_mut()[0] = Stmt::assign( @@ -395,12 +402,7 @@ pub(crate) fn generate_code_for_lazy_export( Index::init(source_index), )?; - // If this is a .napi addon and it's not node, we need to generate a require() call to the runtime - if matches!(expr.data, ExprData::ECall(ref c) - if matches!(c.target.data, ExprData::ERequireCallTarget)) - // if it's commonjs, use require() - && this.options.output_format != crate::options::OutputFormat::Cjs - { + if calls_runtime_require { this.graph.generate_runtime_symbol_import_and_use( source_index, Index::part(1u32), @@ -507,6 +509,15 @@ pub(crate) fn generate_code_for_lazy_export( ))); let parts = this.graph.ast.items_parts_mut()[source_index as usize].as_mut_slice(); parts[generated.1 as usize].stmts = bun_ast::StoreSlice::new_mut(new_stmts); + + if calls_runtime_require { + this.graph.generate_runtime_symbol_import_and_use( + source_index, + Index::part(generated.1), + b"__require", + 1, + )?; + } } } } diff --git a/src/jsc/bindings/ModuleLoader.cpp b/src/jsc/bindings/ModuleLoader.cpp index b68327d1f900..e2b65481dcb5 100644 --- a/src/jsc/bindings/ModuleLoader.cpp +++ b/src/jsc/bindings/ModuleLoader.cpp @@ -568,6 +568,12 @@ JSValue fetchBuiltinModuleWithoutResolution( RELEASE_AND_RETURN(scope, jsNumber(-1)); } + // A text file embedded by `bun build --compile`: the value is the + // string itself, which becomes `module.exports` directly. + case SyntheticModuleType::ExportDefaultObject: { + return JSC::JSValue::decode(res->result.value.jsvalue_for_export); + } + default: { if (tag & SyntheticModuleType::InternalModuleRegistryFlag) { constexpr auto mask = (SyntheticModuleType::InternalModuleRegistryFlag - 1); @@ -1065,6 +1071,21 @@ static JSValue fetchESMSourceCode( BUN_FOREACH_LAZY_ESM_NATIVE_MODULE(LAZY_CASE) #undef LAZY_CASE + // A text file embedded by `bun build --compile`: a module whose + // default export is the already-decoded string. + case SyntheticModuleType::ExportDefaultObject: { + JSC::JSValue value = JSC::JSValue::decode(res->result.value.jsvalue_for_export); + if (!value) { + RELEASE_AND_RETURN(scope, reject(JSC::createSyntaxError(globalObject, "Failed to parse Object"_s))); + } + auto function = generateJSValueExportDefaultObjectSourceCode(globalObject, value); + auto source = JSC::SourceCode( + JSC::SyntheticSourceProvider::create(WTF::move(function), + JSC::SourceOrigin(), WTF::move(moduleKey))); + JSC::ensureStillAliveHere(value); + RELEASE_AND_RETURN(scope, rejectOrResolve(JSSourceCode::create(vm, WTF::move(source)))); + } + // CommonJS modules from src/js/* default: { if (tag & SyntheticModuleType::InternalModuleRegistryFlag) { diff --git a/src/runtime/jsc_hooks.rs b/src/runtime/jsc_hooks.rs index 4d999c3d2ecc..fb60116550d3 100644 --- a/src/runtime/jsc_hooks.rs +++ b/src/runtime/jsc_hooks.rs @@ -4008,10 +4008,10 @@ export default db; return FetchBuiltinResult::Found; } - if file.loader == Loader::Text { + if file.is_text_module() { // A text import embedded by `--compile`. The bytes are already // a Latin-1/UTF-16 string body (`encode_text_module`), so the - // default export is a JSString over the section — no parse, no + // default export is a JSString over the section: no parse, no // bytecode, no copy. // SAFETY: per fn contract — `global` is the live global object. let global = unsafe { &*global }; diff --git a/src/standalone_graph/StandaloneModuleGraph.rs b/src/standalone_graph/StandaloneModuleGraph.rs index 4cfd94ab03a0..f2d1544d8d35 100644 --- a/src/standalone_graph/StandaloneModuleGraph.rs +++ b/src/standalone_graph/StandaloneModuleGraph.rs @@ -349,11 +349,13 @@ pub enum Encoding { Binary = 0, #[default] Latin1 = 1, - // Not used yet. - Utf8 = 2, - /// Native-endian UTF-16 code units, 2-byte aligned in the section. Only - /// produced for `Loader::Text` modules whose text does not fit in Latin-1. - Utf16 = 3, + /// Little-endian UTF-16 code units (the native order of every compile + /// target), 2-byte aligned in the section. Only produced for + /// `Loader::Text` modules whose text does not fit in Latin-1. Takes the + /// value of the never-written `Utf8` variant, so a runtime from before + /// this variant reads it through its plain-copy arm instead of an invalid + /// discriminant. + Utf16 = 2, } #[repr(u8)] @@ -486,8 +488,17 @@ pub struct File { } impl File { + /// A `Loader::Text` import that `--compile` embedded as a module (see + /// `encode_text_module`). Its bytes are a string body, not the file. + pub fn is_text_module(&self) -> bool { + self.loader == Loader::Text + } + pub fn appears_in_embedded_files_array(&self) -> bool { - self.side == FileSide::Client || !self.loader.is_javascript_like() + // Text modules are imports, not assets, and their bytes are not the + // file's UTF-8: a `Blob` over them would decode wrong. + !self.is_text_module() + && (self.side == FileSide::Client || !self.loader.is_javascript_like()) } pub fn stat(&self) -> Stat { @@ -510,7 +521,7 @@ impl File { } if self.wtf_string.is_empty() { match self.encoding { - Encoding::Binary | Encoding::Utf8 => { + Encoding::Binary => { self.wtf_string = BunString::clone_utf8(self.contents.as_bytes()); } Encoding::Latin1 => { @@ -838,20 +849,20 @@ unsafe fn slice_to_z(base: *const u8, len: usize, ptr: StringPointer) -> &'stati unsafe { ZStr::from_raw(base.add(off), n) } } -/// A `Loader::Text` import that the bundler emitted as an embedded asset (see -/// the `Loader::Text` arm of `ParseTask` under `CompileMode::Executable`). The -/// chunk that imports it does `import.meta.require("")`, and the -/// runtime answers with a string that aliases the section bytes, so the bytes -/// are stored already in WTF string form rather than as UTF-8. -fn is_text_module(output_file: &OutputFile) -> bool { +/// A `Loader::Text` import that the bundler emitted as an embedded asset (the +/// `Loader::Text` arm of `ParseTask` under `CompileMode::Executable`). The +/// chunk that imports it does `require("")`, and the runtime +/// answers with a string that aliases the section bytes, so the bytes are +/// stored already in WTF string form rather than as UTF-8. +fn is_text_module_output(output_file: &OutputFile) -> bool { output_file.loader == Loader::Text && output_file.output_kind == options::OutputKind::Asset } /// Write `utf8` as the in-memory representation of a `WTF::StringImpl`: Latin-1 /// when every code point fits in a byte (the all-ASCII case needs no -/// transcoding at all), else native-endian UTF-16 at an even offset so the -/// runtime can point a `char16_t*` at it. Returns the contents pointer and the -/// encoding to record on the module. +/// transcoding at all), else UTF-16 at an even offset so the runtime can point +/// a `char16_t*` at it. Returns the contents pointer and the encoding to record +/// on the module. fn encode_text_module( string_builder: &mut bun_core::StringBuilder, utf8: &[u8], @@ -859,8 +870,7 @@ fn encode_text_module( if strings::first_non_ascii(utf8).is_none() { return (string_builder.append_count_z(utf8), Encoding::Latin1); } - // Invalid UTF-8 becomes U+FFFD, matching what the text loader's - // `export default "..."` string would have held. + // Invalid UTF-8 becomes U+FFFD, as with `TextDecoder`. let units = match strings::to_utf16_alloc(utf8, false, false) { Ok(Some(units)) => units, // `None` only for all-ASCII input, handled above. @@ -890,8 +900,10 @@ fn encode_text_module( let start = string_builder.len; let byte_len = units.len() * 2; let dst = string_builder.writable(); + // Every `--compile` target is little-endian, so this is the byte order the + // runtime's `char16_t*` view expects whatever the build host is. for (dst, unit) in dst.chunks_exact_mut(2).zip(units.iter()) { - dst.copy_from_slice(&unit.to_ne_bytes()); + dst.copy_from_slice(&unit.to_le_bytes()); } dst[byte_len] = 0; dst[byte_len + 1] = 0; @@ -915,7 +927,12 @@ pub(crate) fn to_bytes( // RAII trace handle ends on drop. let _serialize_trace = bun_perf::trace(bun_perf::PerfEvent::StandaloneModuleGraphSerialize); - let mut entry_point_id: Option = None; + let is_entry_point = |output_file: &OutputFile| { + output_file.output_kind == options::OutputKind::EntryPoint + && (output_file.side.is_none() || output_file.side == Some(options::Side::Server)) + }; + + let mut has_entry_point = false; let mut string_builder = bun_core::StringBuilder::default(); let mut module_count: usize = 0; for output_file in output_files { @@ -934,17 +951,10 @@ pub(crate) fn to_bytes( } else if output_file.output_kind == options::OutputKind::ModuleInfo { string_builder.cap += bytes.len(); } else { - if entry_point_id.is_none() { - if output_file.side.is_none() || output_file.side == Some(options::Side::Server) - { - if output_file.output_kind == options::OutputKind::EntryPoint { - entry_point_id = Some(module_count); - } - } - } + has_entry_point |= is_entry_point(output_file); string_builder.count_z(bytes); - if is_text_module(output_file) { + if is_text_module_output(output_file) { // Worst case for `encode_text_module`: every byte widens to // a UTF-16 unit, plus alignment padding and a 2-byte NUL. string_builder.cap += bytes.len() + 3; @@ -954,7 +964,7 @@ pub(crate) fn to_bytes( } } - if module_count == 0 || entry_point_id.is_none() { + if module_count == 0 || !has_entry_point { return Ok(Vec::new()); } @@ -967,6 +977,10 @@ pub(crate) fn to_bytes( string_builder.allocate()?; let mut modules: Vec = Vec::with_capacity(module_count); + let mut entry_point_id: Option = None; + // `Graph::from_bytes` keys files by path, so a repeated path would shift + // `entry_point_id` and every later index by one. + let mut seen_paths: StringArrayHashMap<()> = StringArrayHashMap::new(); let mut source_map_header_list: Vec = Vec::new(); let mut source_map_string_list: Vec = Vec::new(); @@ -992,6 +1006,15 @@ pub(crate) fn to_bytes( let dest_path: &[u8] = path::resolve_path::platform_to_posix_buf::(dest_path, &mut dest_path_buf); + // Two assets can print to one path: same `[name]` and `[hash]` means the + // same bytes, so the first copy serves both importers. + if seen_paths.get_or_put(dest_path)?.found_existing { + continue; + } + if entry_point_id.is_none() && is_entry_point(output_file) { + entry_point_id = Some(modules.len()); + } + let bytecode: StringPointer = 'brk: { if output_file.bytecode_index != u32::MAX { // Bytecode alignment for JSC bytecode cache deserialization. @@ -1122,7 +1145,7 @@ pub(crate) fn to_bytes( bstr::BStr::new(prefix), bstr::BStr::new(dest_path) )); - let (contents, encoding) = if is_text_module(output_file) { + let (contents, encoding) = if is_text_module_output(output_file) { encode_text_module(&mut string_builder, buf_bytes) } else { ( @@ -1190,8 +1213,11 @@ pub(crate) fn to_bytes( modules.len() * size_of::(), ) }; + let Some(entry_point_id) = entry_point_id else { + return Ok(Vec::new()); + }; let offsets = Offsets { - entry_point_id: entry_point_id.unwrap() as u32, + entry_point_id: entry_point_id as u32, modules_ptr: string_builder.append_count(modules_as_bytes), compile_exec_argv_ptr: string_builder.append_count_z(compile_exec_argv), byte_count: string_builder.len, diff --git a/test/bundler/bundler_compile.test.ts b/test/bundler/bundler_compile.test.ts index 01b514dbbe54..c731fdd76eeb 100644 --- a/test/bundler/bundler_compile.test.ts +++ b/test/bundler/bundler_compile.test.ts @@ -911,6 +911,151 @@ describe("bundler", () => { }, run: { stdout: "Hello, world!", setCwd: true }, }); + + // A text import in a compiled executable is embedded as a string body + // (Latin-1 when every code point fits, UTF-16LE otherwise) that the runtime + // hands back without a parse or a copy, instead of a JS module with a string + // literal. The same source also checks `require()`, `import()`, and that + // `Bun.embeddedFiles` keeps listing only real assets. + // Buffers: `files` strings go through dedent(), which would trim them. + const textImportFiles = { + "/ascii.txt": Buffer.from("hello world\nline 2\n"), + "/latin1.txt": Buffer.from("caf\u00e9 na\u00efve\n"), + "/wide.txt": Buffer.from("em \u2014 dash \u{1F600} emoji \u65e5\u672c\n"), + "/empty.txt": Buffer.alloc(0), + "/invalid.txt": Buffer.from([0x62, 0x61, 0x64, 0x20, 0xff, 0xfe, 0x20, 0xc3, 0x28, 0x0a]), + "/doc.md": Buffer.from("# Title\n\nsome *markdown* \u2014 with dash\n"), + "/asset.file": "abcd", + }; + const textImportEntry = /* js */ ` + import ascii from "./ascii.txt"; + import latin1 from "./latin1.txt"; + import wide from "./wide.txt"; + import empty from "./empty.txt"; + import invalid from "./invalid.txt"; + import doc from "./doc.md"; + import asset from "./asset.file" with { type: "file" }; + import { readdirSync, readFileSync } from "node:fs"; + import { dirname, join } from "node:path"; + + // No top-level await: the bytecode variant is CommonJS output. + async function main() { + // The embedded root. \`import.meta.dir\` is inlined at build time in CommonJS output. + const root = dirname(Bun.main); + const expected = { + ascii: "hello world\\nline 2\\n", + latin1: "caf\\u00e9 na\\u00efve\\n", + wide: "em \\u2014 dash \\u{1F600} emoji \\u65e5\\u672c\\n", + empty: "", + // Invalid UTF-8 decodes like TextDecoder: one U+FFFD per bad byte. + invalid: "bad \\ufffd\\ufffd \\ufffd(\\n", + doc: "# Title\\n\\nsome *markdown* \\u2014 with dash\\n", + }; + const actual = { ascii, latin1, wide, empty, invalid, doc }; + for (const [name, value] of Object.entries(actual)) { + if (typeof value !== "string") throw new Error(name + " is a " + typeof value); + if (value !== expected[name]) throw new Error(name + " mismatch: " + JSON.stringify(value)); + } + if (require("./ascii.txt") !== ascii) throw new Error("require() returned " + JSON.stringify(require("./ascii.txt"))); + if ((await import("./wide.txt")).default !== wide) throw new Error("import() mismatch"); + + // Text modules are not assets; only the file loader import is listed. + const embedded = Bun.embeddedFiles.map(blob => blob.name); + if (embedded.length !== 1 || !embedded[0].startsWith("asset-")) throw new Error("embeddedFiles: " + embedded); + if ((await Bun.file(asset).text()) !== "abcd") throw new Error("asset: " + asset); + + // The embedded bytes are the string body itself. + const encoded = { + "ascii.txt": Buffer.from(expected.ascii, "latin1"), + "latin1.txt": Buffer.from(expected.latin1, "latin1"), + "wide.txt": Buffer.from(expected.wide, "utf16le"), + "empty.txt": Buffer.alloc(0), + "invalid.txt": Buffer.from(expected.invalid, "utf16le"), + "doc.md": Buffer.from(expected.doc, "utf16le"), + }; + const embeddedNames = readdirSync(root); + for (const [name, bytes] of Object.entries(encoded)) { + const [base, ext] = name.split("."); + const file = embeddedNames.find(entry => entry.startsWith(base + "-") && entry.endsWith("." + ext)); + if (!file) throw new Error("no embedded module for " + name + " in " + JSON.stringify(embeddedNames)); + const got = readFileSync(join(root, file)); + if (!got.equals(bytes)) throw new Error(name + ": " + got.toString("hex") + " != " + bytes.toString("hex")); + } + console.log("PASS"); + } + main(); + `; + for (const [suffix, options] of [ + ["", {}], + ["Bytecode", { bytecode: true }], + ["BytecodeESM", { bytecode: true, format: "esm" }], + ] as const) { + itBundled(`compile/TextImport${suffix}`, { + compile: true, + ...options, + loader: { ".md": "text" }, + files: { "/entry.ts": textImportEntry, ...textImportFiles }, + run: { stdout: "PASS" }, + }); + } + + // Text modules keep their own `[name]-[hash]` path: a user `--asset-naming` + // without `[hash]` must not make two same-named text files share one path. + itBundled("compile/TextImportSameBasename", { + compile: true, + assetNaming: "[name].[ext]", + files: { + "/entry.ts": /* js */ ` + import a from "./a/readme.txt"; + import b from "./b/readme.txt"; + import sameA from "./a/same.txt"; + import sameB from "./b/same.txt"; + import icon from "./a/icon.file" with { type: "file" }; + console.log(JSON.stringify({ a, b, sameA, sameB, icon: await Bun.file(icon).text() })); + `, + "/a/readme.txt": "from a", + "/b/readme.txt": "from b", + "/a/same.txt": "same", + "/b/same.txt": "same", + "/a/icon.file": "icon", + }, + run: { stdout: JSON.stringify({ a: "from a", b: "from b", sameA: "same", sameB: "same", icon: "icon" }) }, + }); + + // A browser chunk of a full-stack executable cannot reach the embedded + // module graph, so its text imports stay inline string literals. + itBundled("compile/TextImportClientChunkStaysInline", { + compile: true, + files: { + "/entry.ts": /* js */ ` + import index from "./index.html"; + import note from "./note.txt"; + using server = Bun.serve({ port: 0, routes: { "/": index } }); + const html = await (await fetch(server.url)).text(); + const src = html.match(/]*src="([^"]+)"/)[1]; + const js = await (await fetch(new URL(src, server.url))).text(); + console.log(JSON.stringify({ + server: note, + clientHasLiteral: js.includes("client sees the text"), + clientHasBunfs: js.includes("$bunfs"), + })); + `, + "/index.html": /* html */ ` + + + + + + + `, + "/app.ts": /* js */ ` + import note from "./note.txt"; + document.body.textContent = note; + `, + "/note.txt": "client sees the text", + }, + run: { stdout: JSON.stringify({ server: "client sees the text", clientHasLiteral: true, clientHasBunfs: false }) }, + }); itBundled("compile/Utf8", { compile: true, files: { diff --git a/test/bundler/bundler_loader.test.ts b/test/bundler/bundler_loader.test.ts index e7a7d02051d5..6e33cdf018e7 100644 --- a/test/bundler/bundler_loader.test.ts +++ b/test/bundler/bundler_loader.test.ts @@ -488,6 +488,26 @@ describe("bundler", async () => { }); } + // `import addon from "./addon.node"` prints as `__require("./addon-[hash].node")` + // in ESM output, so the chunk has to define the runtime helper. + itBundled("bun/loader-napi-esm-runtime-require", { + target: "bun", + format: "esm", + outdir: "/out", + files: { + "/entry.ts": /* js */ ` + import addon from "./addon.node"; + export default addon; + `, + "/addon.node": "not a real addon", + }, + onAfterBundle(api) { + const js = api.readFile("/out/entry.js"); + expect(js).toContain("var __require = import.meta.require;"); + expect(js).toMatch(/__require\("\.\/addon-[a-z0-9]+\.node"\)/); + }, + }); + describe("handles empty files", () => { for (const target of ["bun", "node", "browser"] as const) { itBundled(`${target}/loader-empty-text-file`, { From 82866ec7f046d21d275c789d97e87aee72c08462 Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Sun, 23 Aug 2026 07:56:12 +0000 Subject: [PATCH 3/6] Tighten comments --- src/bundler/ParseTask.rs | 25 ++++------ src/bundler/bundle_v2.rs | 14 ++---- .../generateCodeForLazyExport.rs | 5 +- src/jsc/bindings/ModuleLoader.cpp | 6 +-- src/runtime/jsc_hooks.rs | 10 ++-- src/standalone_graph/StandaloneModuleGraph.rs | 46 ++++++------------- 6 files changed, 34 insertions(+), 72 deletions(-) diff --git a/src/bundler/ParseTask.rs b/src/bundler/ParseTask.rs index e0275e796db8..2248a6632596 100644 --- a/src/bundler/ParseTask.rs +++ b/src/bundler/ParseTask.rs @@ -653,9 +653,7 @@ pub mod parse_worker { pub(crate) content_hash: u64, } - /// Registers `source` as an embedded asset: the returned unique key is - /// replaced with the asset's final path when the chunk is printed, and - /// `process_files_to_copy` emits the bytes as an `OutputKind::Asset`. + /// Returns the unique key the printer replaces with the asset's final path. fn register_embedded_asset<'b>( bump: &'b Bump, source: &Source, @@ -682,10 +680,8 @@ pub mod parse_worker { unique_key } - /// `require("")`. The printer turns the call target into the - /// CommonJS `require` or the runtime's `__require` to match the output - /// format (`generateCodeForLazyExport` imports the latter), so the chunk - /// stays free of `import.meta` and `--bytecode` can compile it. + /// `require("")`. Unlike `import.meta.require`, the call target + /// prints per output format, so `--bytecode` (CommonJS) can compile it. fn require_embedded_asset(unique_key: &[u8]) -> Expr { let import_path = Expr::init( E::String { @@ -940,14 +936,10 @@ pub mod parse_worker { return result; } Loader::Text => { - // In a standalone executable the text is embedded as an asset - // and the module becomes `export default require("")`. - // The runtime answers with a string that aliases the section - // bytes (`StandaloneModuleGraph::encode_text_module`). A string - // literal would ship the text twice (module source + bytecode - // constant) and parse it on load. Browser chunks of a - // full-stack build cannot reach the embedded graph and keep - // the literal. + // A standalone executable embeds the text as a string body the + // runtime aliases without a copy (`encode_text_module`), so the + // module becomes `export default require("")`. + // Browser chunks cannot reach the embedded graph. let root = if topts.compile_mode.is_executable() && topts.target.is_bun() { require_embedded_asset(register_embedded_asset( bump, @@ -2699,8 +2691,7 @@ pub mod parse_worker { loader, package_name: task.package_name, - // Hash the files in here so that we do it in parallel. Zero unless - // the loader registered an asset (`FileLoaderHash` default). + // Hash the files in here so that we do it in parallel. content_hash_for_additional_file: unique_key_for_additional_file.content_hash, }) } diff --git a/src/bundler/bundle_v2.rs b/src/bundler/bundle_v2.rs index dda0bad394c2..97cd0b348178 100644 --- a/src/bundler/bundle_v2.rs +++ b/src/bundler/bundle_v2.rs @@ -4194,11 +4194,8 @@ pub mod bv2_impl { let loader = loaders[index]; let target = targets[index]; let mut template: options::PathTemplate = if loader == Loader::Text { - // A text module embedded by `--compile` (ParseTask's - // `Loader::Text` arm) is only ever addressed through - // the bunfs path the bundler prints into the chunk, so - // `--asset-naming` does not apply: without `[hash]` - // two same-named files would silently share one path. + // Text modules ignore `--asset-naming`: without `[hash]` + // two same-named files would share one path. options::PathTemplate::ASSET.into() } else { let mut template: options::PathTemplate = @@ -7109,11 +7106,8 @@ pub mod bv2_impl { if !result.unique_key_for_additional_file.is_empty() && result.loader == Loader::Text { - // A text module embedded as an asset by `--compile`. The - // importer-side count in `process_resolve_queue` only - // covers `should_copy_for_bundling()` loaders, and - // `process_files_to_copy` skips the whole pass when the - // count is zero. + // `process_resolve_queue` only counts `should_copy_for_bundling()` + // loaders, and a zero count skips `process_files_to_copy`. this.graph.estimated_file_loader_count += 1; } if !result.unique_key_for_additional_file.is_empty() diff --git a/src/bundler/linker_context/generateCodeForLazyExport.rs b/src/bundler/linker_context/generateCodeForLazyExport.rs index 14b464a4f11c..44c9a36142c8 100644 --- a/src/bundler/linker_context/generateCodeForLazyExport.rs +++ b/src/bundler/linker_context/generateCodeForLazyExport.rs @@ -373,9 +373,8 @@ pub(crate) fn generate_code_for_lazy_export( loc: stmt.loc, }; - // `require()` from the napi loader and the `--compile` text - // loader prints as the runtime's `__require` outside CommonJS output, so - // the part that holds the call has to import that symbol. + // `require()` prints as the runtime's `__require` outside CommonJS + // output, so the part that holds the call must import it. let calls_runtime_require = matches!(expr.data, ExprData::ECall(ref c) if matches!(c.target.data, ExprData::ERequireCallTarget)) && this.options.output_format != crate::options::OutputFormat::Cjs; diff --git a/src/jsc/bindings/ModuleLoader.cpp b/src/jsc/bindings/ModuleLoader.cpp index e2b65481dcb5..79c18a0526cf 100644 --- a/src/jsc/bindings/ModuleLoader.cpp +++ b/src/jsc/bindings/ModuleLoader.cpp @@ -568,8 +568,7 @@ JSValue fetchBuiltinModuleWithoutResolution( RELEASE_AND_RETURN(scope, jsNumber(-1)); } - // A text file embedded by `bun build --compile`: the value is the - // string itself, which becomes `module.exports` directly. + // A text file embedded by `bun build --compile`: the string is `module.exports`. case SyntheticModuleType::ExportDefaultObject: { return JSC::JSValue::decode(res->result.value.jsvalue_for_export); } @@ -1071,8 +1070,7 @@ static JSValue fetchESMSourceCode( BUN_FOREACH_LAZY_ESM_NATIVE_MODULE(LAZY_CASE) #undef LAZY_CASE - // A text file embedded by `bun build --compile`: a module whose - // default export is the already-decoded string. + // A text file embedded by `bun build --compile`: the string is the default export. case SyntheticModuleType::ExportDefaultObject: { JSC::JSValue value = JSC::JSValue::decode(res->result.value.jsvalue_for_export); if (!value) { diff --git a/src/runtime/jsc_hooks.rs b/src/runtime/jsc_hooks.rs index fb60116550d3..644982ea2907 100644 --- a/src/runtime/jsc_hooks.rs +++ b/src/runtime/jsc_hooks.rs @@ -4009,16 +4009,12 @@ export default db; } if file.is_text_module() { - // A text import embedded by `--compile`. The bytes are already - // a Latin-1/UTF-16 string body (`encode_text_module`), so the - // default export is a JSString over the section: no parse, no - // bytecode, no copy. + // The bytes are already a string body (`encode_text_module`): + // the default export is a JSString over the section, no copy. // SAFETY: per fn contract — `global` is the live global object. let global = unsafe { &*global }; let string = file.to_wtf_string(); - // `jsString(vm, String(impl))` over the cached static impl; - // only a `Dead` string throws and `to_wtf_string` never - // produces one. + // Only a `Dead` string throws; `to_wtf_string` never yields one. let value = bun_jsc::bun_string_jsc::to_js(&string, global) .expect("embedded text module string is never dead"); string.deref(); diff --git a/src/standalone_graph/StandaloneModuleGraph.rs b/src/standalone_graph/StandaloneModuleGraph.rs index f2d1544d8d35..34a98d8da38c 100644 --- a/src/standalone_graph/StandaloneModuleGraph.rs +++ b/src/standalone_graph/StandaloneModuleGraph.rs @@ -349,12 +349,9 @@ pub enum Encoding { Binary = 0, #[default] Latin1 = 1, - /// Little-endian UTF-16 code units (the native order of every compile - /// target), 2-byte aligned in the section. Only produced for - /// `Loader::Text` modules whose text does not fit in Latin-1. Takes the - /// value of the never-written `Utf8` variant, so a runtime from before - /// this variant reads it through its plain-copy arm instead of an invalid - /// discriminant. + /// Little-endian UTF-16 code units at an even section offset. Reuses the + /// value of the never-written `Utf8` variant so an older runtime reads it + /// through its plain-copy arm, not as an invalid discriminant. Utf16 = 2, } @@ -488,15 +485,13 @@ pub struct File { } impl File { - /// A `Loader::Text` import that `--compile` embedded as a module (see - /// `encode_text_module`). Its bytes are a string body, not the file. + /// A text import stored as a string body by `encode_text_module`. pub fn is_text_module(&self) -> bool { self.loader == Loader::Text } pub fn appears_in_embedded_files_array(&self) -> bool { - // Text modules are imports, not assets, and their bytes are not the - // file's UTF-8: a `Blob` over them would decode wrong. + // A text module's bytes are not the file's UTF-8. !self.is_text_module() && (self.side == FileSide::Client || !self.loader.is_javascript_like()) } @@ -530,9 +525,8 @@ impl File { } Encoding::Utf16 => { let bytes = self.contents.as_bytes(); - // `to_bytes` pads to an even section offset; the section - // base is page-aligned on every platform (the bytecode - // cache relies on the same property at 128 bytes). + // `to_bytes` pads to an even offset and the section base is + // page-aligned (the 128-byte bytecode alignment relies on it). debug_assert!(bytes.as_ptr().addr() % align_of::() == 0); // SAFETY: even byte count at a 2-byte-aligned offset of a // section that is never freed. @@ -849,20 +843,14 @@ unsafe fn slice_to_z(base: *const u8, len: usize, ptr: StringPointer) -> &'stati unsafe { ZStr::from_raw(base.add(off), n) } } -/// A `Loader::Text` import that the bundler emitted as an embedded asset (the -/// `Loader::Text` arm of `ParseTask` under `CompileMode::Executable`). The -/// chunk that imports it does `require("")`, and the runtime -/// answers with a string that aliases the section bytes, so the bytes are -/// stored already in WTF string form rather than as UTF-8. +/// A text import the bundler emitted as an asset (`Loader::Text` arm of +/// `ParseTask` in compile mode). The runtime aliases its bytes as a string. fn is_text_module_output(output_file: &OutputFile) -> bool { output_file.loader == Loader::Text && output_file.output_kind == options::OutputKind::Asset } -/// Write `utf8` as the in-memory representation of a `WTF::StringImpl`: Latin-1 -/// when every code point fits in a byte (the all-ASCII case needs no -/// transcoding at all), else UTF-16 at an even offset so the runtime can point -/// a `char16_t*` at it. Returns the contents pointer and the encoding to record -/// on the module. +/// Writes `utf8` as a `WTF::StringImpl` body: Latin-1 when every code point +/// fits, else UTF-16 at an even offset so the runtime can alias a `char16_t*`. fn encode_text_module( string_builder: &mut bun_core::StringBuilder, utf8: &[u8], @@ -900,8 +888,7 @@ fn encode_text_module( let start = string_builder.len; let byte_len = units.len() * 2; let dst = string_builder.writable(); - // Every `--compile` target is little-endian, so this is the byte order the - // runtime's `char16_t*` view expects whatever the build host is. + // Every `--compile` target is little-endian, whatever the build host is. for (dst, unit) in dst.chunks_exact_mut(2).zip(units.iter()) { dst.copy_from_slice(&unit.to_le_bytes()); } @@ -955,8 +942,7 @@ pub(crate) fn to_bytes( string_builder.count_z(bytes); if is_text_module_output(output_file) { - // Worst case for `encode_text_module`: every byte widens to - // a UTF-16 unit, plus alignment padding and a 2-byte NUL. + // UTF-16 worst case: 2 bytes per byte, padding, 2-byte NUL. string_builder.cap += bytes.len() + 3; } module_count += 1; @@ -978,8 +964,7 @@ pub(crate) fn to_bytes( let mut modules: Vec = Vec::with_capacity(module_count); let mut entry_point_id: Option = None; - // `Graph::from_bytes` keys files by path, so a repeated path would shift - // `entry_point_id` and every later index by one. + // `Graph::from_bytes` keys files by path; a repeat would shift `entry_point_id`. let mut seen_paths: StringArrayHashMap<()> = StringArrayHashMap::new(); let mut source_map_header_list: Vec = Vec::new(); @@ -1006,8 +991,7 @@ pub(crate) fn to_bytes( let dest_path: &[u8] = path::resolve_path::platform_to_posix_buf::(dest_path, &mut dest_path_buf); - // Two assets can print to one path: same `[name]` and `[hash]` means the - // same bytes, so the first copy serves both importers. + // Same `[name]` and `[hash]` means the same bytes: keep the first copy. if seen_paths.get_or_put(dest_path)?.found_existing { continue; } From ceff847a9da76b2a1dc310795250d355da626c9c Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Sun, 23 Aug 2026 08:03:45 +0000 Subject: [PATCH 4/6] Fix clippy lints in encode_text_module and to_wtf_string --- src/standalone_graph/StandaloneModuleGraph.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/standalone_graph/StandaloneModuleGraph.rs b/src/standalone_graph/StandaloneModuleGraph.rs index 34a98d8da38c..14d38979b045 100644 --- a/src/standalone_graph/StandaloneModuleGraph.rs +++ b/src/standalone_graph/StandaloneModuleGraph.rs @@ -525,9 +525,11 @@ impl File { } Encoding::Utf16 => { let bytes = self.contents.as_bytes(); - // `to_bytes` pads to an even offset and the section base is - // page-aligned (the 128-byte bytecode alignment relies on it). - debug_assert!(bytes.as_ptr().addr() % align_of::() == 0); + debug_assert!(bytes.as_ptr().addr().is_multiple_of(align_of::())); + #[expect( + clippy::cast_ptr_alignment, + reason = "`to_bytes` writes UTF-16 at an even offset and the section base is page-aligned (the 128-byte bytecode alignment relies on the same property)" + )] // SAFETY: even byte count at a 2-byte-aligned offset of a // section that is never freed. let units = unsafe { @@ -881,7 +883,7 @@ fn encode_text_module( Encoding::Latin1, ); } - if string_builder.len % align_of::() != 0 { + if !string_builder.len.is_multiple_of(align_of::()) { string_builder.writable()[0] = 0; string_builder.len += 1; } @@ -889,8 +891,8 @@ fn encode_text_module( let byte_len = units.len() * 2; let dst = string_builder.writable(); // Every `--compile` target is little-endian, whatever the build host is. - for (dst, unit) in dst.chunks_exact_mut(2).zip(units.iter()) { - dst.copy_from_slice(&unit.to_le_bytes()); + for (dst, unit) in dst.as_chunks_mut::<2>().0.iter_mut().zip(units.iter()) { + *dst = unit.to_le_bytes(); } dst[byte_len] = 0; dst[byte_len + 1] = 0; From 4566d5d9136d2a4f7306923be885b0fc03790fb7 Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Sun, 23 Aug 2026 08:30:15 +0000 Subject: [PATCH 5/6] encode_text_module: transcode with simdutf instead of per-unit loops UTF-8 to UTF-16 already went through simdutf; the Latin-1 decision and the narrowing copy were scalar loops over every code unit, and the UTF-16 write was a per-unit loop. Bind simdutf's convert_utf16le_to_latin1_with_errors (one pass, fails at the first unit above U+00FF) and memcpy the UTF-16 units. Drop the unused specifier/source_url refs from the text module ResolvedSource: the ExportDefaultObject consumers never deref them. --- src/bun_core/string/immutable.rs | 18 ++++++++++++ src/runtime/jsc_hooks.rs | 4 +-- src/simdutf_sys/bun-simdutf.cpp | 7 +++++ src/simdutf_sys/simdutf.rs | 25 +++++++++++++++++ src/standalone_graph/StandaloneModuleGraph.rs | 28 +++++++++---------- 5 files changed, 65 insertions(+), 17 deletions(-) diff --git a/src/bun_core/string/immutable.rs b/src/bun_core/string/immutable.rs index 8e2b777a1f57..2125eaddbd13 100644 --- a/src/bun_core/string/immutable.rs +++ b/src/bun_core/string/immutable.rs @@ -2493,6 +2493,24 @@ pub fn try_convert_utf8_to_utf16_in_buffer<'a>( Some(&mut buf[..written]) } +/// UTF-16 → Latin-1 into `buf` (capacity ≥ `input.len()`), or `None` when a +/// unit is above U+00FF. SIMD via simdutf; `buf` may hold a partial prefix +/// after `None`. +pub fn try_convert_utf16_to_latin1_in_buffer<'a>( + buf: &'a mut [u8], + input: &[u16], +) -> Option<&'a mut [u8]> { + if input.len() > buf.len() { + return None; + } + let r = simdutf::convert::utf16::to::latin1::with_errors::le(input, buf); + if !r.is_successful() { + return None; + } + debug_assert_eq!(r.count, input.len()); + Some(&mut buf[..input.len()]) +} + /// Decode one WTF-8 sequence at the head of `s`; invalid lead/truncated → (U+FFFD, 1). /// Lone surrogates pass through (WTF-8). Helper for [`convert_utf8_to_utf16_in_buffer`]. fn decode_wtf8_one(s: &[u8]) -> (u32, usize) { diff --git a/src/runtime/jsc_hooks.rs b/src/runtime/jsc_hooks.rs index 644982ea2907..829b0aca05b4 100644 --- a/src/runtime/jsc_hooks.rs +++ b/src/runtime/jsc_hooks.rs @@ -4018,12 +4018,12 @@ export default db; let value = bun_jsc::bun_string_jsc::to_js(&string, global) .expect("embedded text module string is never dead"); string.deref(); + // No `specifier`/`source_url`: the ExportDefaultObject consumers + // never read or deref them. // SAFETY: per fn contract — `out` is a valid out-param. unsafe { *out = ErrorableResolvedSource::ok(ResolvedSource { jsvalue_for_export: value, - specifier: specifier.dupe_ref(), - source_url: specifier.dupe_ref(), tag: bun_jsc::resolved_source::Tag::ExportDefaultObject, ..ResolvedSource::default() }); diff --git a/src/simdutf_sys/bun-simdutf.cpp b/src/simdutf_sys/bun-simdutf.cpp index 43a361be9562..43554a5cf289 100644 --- a/src/simdutf_sys/bun-simdutf.cpp +++ b/src/simdutf_sys/bun-simdutf.cpp @@ -42,6 +42,13 @@ simdutf__convert_utf8_to_utf16le_with_errors(const char* buf, size_t len, return { res.error, res.count }; } +SIMDUTFResult simdutf__convert_utf16le_to_latin1_with_errors(const char16_t* buf, size_t len, + char* latin1_output) +{ + auto res = simdutf::convert_utf16le_to_latin1_with_errors(buf, len, latin1_output); + return { res.error, res.count }; +} + SIMDUTFResult simdutf__convert_utf16le_to_utf8_with_errors(const char16_t* buf, size_t len, char* utf8_buffer) diff --git a/src/simdutf_sys/simdutf.rs b/src/simdutf_sys/simdutf.rs index 47be59bcb46d..acfc68a11544 100644 --- a/src/simdutf_sys/simdutf.rs +++ b/src/simdutf_sys/simdutf.rs @@ -41,6 +41,11 @@ unsafe extern "C" { len: usize, utf16_output: *mut u16, ) -> SIMDUTFResult; + pub(crate) fn simdutf__convert_utf16le_to_latin1_with_errors( + buf: *const u16, + len: usize, + latin1_output: *mut u8, + ) -> SIMDUTFResult; pub fn simdutf__convert_utf16le_to_utf8_with_errors( buf: *const u16, len: usize, @@ -143,6 +148,26 @@ pub mod convert { } } } + pub mod latin1 { + use super::*; + pub mod with_errors { + use super::*; + /// Fails with `TOO_LARGE` at the first unit above U+00FF. + /// `output` needs `input.len()` bytes. + pub fn le(input: &[u16], output: &mut [u8]) -> SIMDUTFResult { + debug_assert!(output.len() >= input.len()); + // SAFETY: one output byte per input unit; the caller + // provides `input.len()` bytes. + unsafe { + simdutf__convert_utf16le_to_latin1_with_errors( + input.as_ptr(), + input.len(), + output.as_mut_ptr(), + ) + } + } + } + } } } } diff --git a/src/standalone_graph/StandaloneModuleGraph.rs b/src/standalone_graph/StandaloneModuleGraph.rs index 14d38979b045..1c83a1de8c5b 100644 --- a/src/standalone_graph/StandaloneModuleGraph.rs +++ b/src/standalone_graph/StandaloneModuleGraph.rs @@ -853,6 +853,7 @@ fn is_text_module_output(output_file: &OutputFile) -> bool { /// Writes `utf8` as a `WTF::StringImpl` body: Latin-1 when every code point /// fits, else UTF-16 at an even offset so the runtime can alias a `char16_t*`. +/// `to_bytes` reserves `2 * utf8.len() + 4` bytes for it. fn encode_text_module( string_builder: &mut bun_core::StringBuilder, utf8: &[u8], @@ -863,22 +864,20 @@ fn encode_text_module( // Invalid UTF-8 becomes U+FFFD, as with `TextDecoder`. let units = match strings::to_utf16_alloc(utf8, false, false) { Ok(Some(units)) => units, - // `None` only for all-ASCII input, handled above. - Ok(None) => unreachable!("to_utf16_alloc returned None for non-ASCII input"), + Ok(None) => unreachable!("non-ASCII input always converts"), Err(_) => bun_alloc::out_of_memory(), }; - if units.iter().all(|&unit| unit <= 0xFF) { - let start = string_builder.len; - let dst = string_builder.writable(); - for (dst, &unit) in dst.iter_mut().zip(units.iter()) { - *dst = unit as u8; - } - dst[units.len()] = 0; - string_builder.len += units.len() + 1; + let start = string_builder.len; + if let Some(latin1) = + strings::try_convert_utf16_to_latin1_in_buffer(string_builder.writable(), &units) + { + let len = latin1.len(); + string_builder.writable()[len] = 0; + string_builder.len += len + 1; return ( StringPointer { offset: start as u32, - length: units.len() as u32, + length: len as u32, }, Encoding::Latin1, ); @@ -889,11 +888,10 @@ fn encode_text_module( } let start = string_builder.len; let byte_len = units.len() * 2; + // SAFETY: a `u8` view over initialized `u16`s is in bounds and aligned. + let bytes = unsafe { core::slice::from_raw_parts(units.as_ptr().cast::(), byte_len) }; let dst = string_builder.writable(); - // Every `--compile` target is little-endian, whatever the build host is. - for (dst, unit) in dst.as_chunks_mut::<2>().0.iter_mut().zip(units.iter()) { - *dst = unit.to_le_bytes(); - } + dst[..byte_len].copy_from_slice(bytes); dst[byte_len] = 0; dst[byte_len + 1] = 0; string_builder.len += byte_len + 2; From 4db71948d6e0682fea2b833a3eb81350c5913506 Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Sun, 23 Aug 2026 08:38:46 +0000 Subject: [PATCH 6/6] encode_text_module: 8-bit for ASCII, UTF-16 otherwise to_utf16_alloc already answers the width question: None means the bytes are ASCII and stay 8-bit, Some means UTF-16. Drop the Latin-1 narrowing pass and the simdutf binding it needed; this is the same split String::clone_utf8 makes at runtime. --- docs/bundler/executables.mdx | 2 +- src/bun_core/string/immutable.rs | 18 ------------- src/simdutf_sys/bun-simdutf.cpp | 7 ----- src/simdutf_sys/simdutf.rs | 25 ------------------ src/standalone_graph/StandaloneModuleGraph.rs | 26 +++---------------- test/bundler/bundler_compile.test.ts | 7 +++-- 6 files changed, 8 insertions(+), 77 deletions(-) diff --git a/docs/bundler/executables.mdx b/docs/bundler/executables.mdx index d5ae8754fd66..1698d0ed5095 100644 --- a/docs/bundler/executables.mdx +++ b/docs/bundler/executables.mdx @@ -769,7 +769,7 @@ import readme from "./README.md" with { type: "text" }; console.log(notes); // the contents of notes.txt ``` -In a compiled executable the text is stored once, in the form the JavaScript engine uses for strings (Latin-1 when every character fits, UTF-16 otherwise). The runtime hands the string back without a copy, so a large text import costs its size in the binary and no extra copy on the heap. A text import is a module, not a file: `Bun.embeddedFiles` does not list it. Use `with { type: "file" }` when you need a file path instead. +In a compiled executable the text is stored once, in the form the JavaScript engine uses for strings (one byte per character for ASCII text, UTF-16 otherwise). The runtime hands the string back without a copy, so a large text import costs its size in the binary and no extra copy on the heap. A text import is a module, not a file: `Bun.embeddedFiles` does not list it. Use `with { type: "file" }` when you need a file path instead. ### Embed SQLite databases diff --git a/src/bun_core/string/immutable.rs b/src/bun_core/string/immutable.rs index 2125eaddbd13..8e2b777a1f57 100644 --- a/src/bun_core/string/immutable.rs +++ b/src/bun_core/string/immutable.rs @@ -2493,24 +2493,6 @@ pub fn try_convert_utf8_to_utf16_in_buffer<'a>( Some(&mut buf[..written]) } -/// UTF-16 → Latin-1 into `buf` (capacity ≥ `input.len()`), or `None` when a -/// unit is above U+00FF. SIMD via simdutf; `buf` may hold a partial prefix -/// after `None`. -pub fn try_convert_utf16_to_latin1_in_buffer<'a>( - buf: &'a mut [u8], - input: &[u16], -) -> Option<&'a mut [u8]> { - if input.len() > buf.len() { - return None; - } - let r = simdutf::convert::utf16::to::latin1::with_errors::le(input, buf); - if !r.is_successful() { - return None; - } - debug_assert_eq!(r.count, input.len()); - Some(&mut buf[..input.len()]) -} - /// Decode one WTF-8 sequence at the head of `s`; invalid lead/truncated → (U+FFFD, 1). /// Lone surrogates pass through (WTF-8). Helper for [`convert_utf8_to_utf16_in_buffer`]. fn decode_wtf8_one(s: &[u8]) -> (u32, usize) { diff --git a/src/simdutf_sys/bun-simdutf.cpp b/src/simdutf_sys/bun-simdutf.cpp index 43554a5cf289..43a361be9562 100644 --- a/src/simdutf_sys/bun-simdutf.cpp +++ b/src/simdutf_sys/bun-simdutf.cpp @@ -42,13 +42,6 @@ simdutf__convert_utf8_to_utf16le_with_errors(const char* buf, size_t len, return { res.error, res.count }; } -SIMDUTFResult simdutf__convert_utf16le_to_latin1_with_errors(const char16_t* buf, size_t len, - char* latin1_output) -{ - auto res = simdutf::convert_utf16le_to_latin1_with_errors(buf, len, latin1_output); - return { res.error, res.count }; -} - SIMDUTFResult simdutf__convert_utf16le_to_utf8_with_errors(const char16_t* buf, size_t len, char* utf8_buffer) diff --git a/src/simdutf_sys/simdutf.rs b/src/simdutf_sys/simdutf.rs index acfc68a11544..47be59bcb46d 100644 --- a/src/simdutf_sys/simdutf.rs +++ b/src/simdutf_sys/simdutf.rs @@ -41,11 +41,6 @@ unsafe extern "C" { len: usize, utf16_output: *mut u16, ) -> SIMDUTFResult; - pub(crate) fn simdutf__convert_utf16le_to_latin1_with_errors( - buf: *const u16, - len: usize, - latin1_output: *mut u8, - ) -> SIMDUTFResult; pub fn simdutf__convert_utf16le_to_utf8_with_errors( buf: *const u16, len: usize, @@ -148,26 +143,6 @@ pub mod convert { } } } - pub mod latin1 { - use super::*; - pub mod with_errors { - use super::*; - /// Fails with `TOO_LARGE` at the first unit above U+00FF. - /// `output` needs `input.len()` bytes. - pub fn le(input: &[u16], output: &mut [u8]) -> SIMDUTFResult { - debug_assert!(output.len() >= input.len()); - // SAFETY: one output byte per input unit; the caller - // provides `input.len()` bytes. - unsafe { - simdutf__convert_utf16le_to_latin1_with_errors( - input.as_ptr(), - input.len(), - output.as_mut_ptr(), - ) - } - } - } - } } } } diff --git a/src/standalone_graph/StandaloneModuleGraph.rs b/src/standalone_graph/StandaloneModuleGraph.rs index 1c83a1de8c5b..76d28f831e5b 100644 --- a/src/standalone_graph/StandaloneModuleGraph.rs +++ b/src/standalone_graph/StandaloneModuleGraph.rs @@ -851,37 +851,19 @@ fn is_text_module_output(output_file: &OutputFile) -> bool { output_file.loader == Loader::Text && output_file.output_kind == options::OutputKind::Asset } -/// Writes `utf8` as a `WTF::StringImpl` body: Latin-1 when every code point -/// fits, else UTF-16 at an even offset so the runtime can alias a `char16_t*`. -/// `to_bytes` reserves `2 * utf8.len() + 4` bytes for it. +/// Writes `utf8` as a `WTF::StringImpl` body: 8-bit when ASCII, else UTF-16 +/// at an even offset so the runtime can alias a `char16_t*` (the same split +/// as `String::clone_utf8`). `to_bytes` reserves `2 * utf8.len() + 4` bytes. fn encode_text_module( string_builder: &mut bun_core::StringBuilder, utf8: &[u8], ) -> (StringPointer, Encoding) { - if strings::first_non_ascii(utf8).is_none() { - return (string_builder.append_count_z(utf8), Encoding::Latin1); - } // Invalid UTF-8 becomes U+FFFD, as with `TextDecoder`. let units = match strings::to_utf16_alloc(utf8, false, false) { + Ok(None) => return (string_builder.append_count_z(utf8), Encoding::Latin1), Ok(Some(units)) => units, - Ok(None) => unreachable!("non-ASCII input always converts"), Err(_) => bun_alloc::out_of_memory(), }; - let start = string_builder.len; - if let Some(latin1) = - strings::try_convert_utf16_to_latin1_in_buffer(string_builder.writable(), &units) - { - let len = latin1.len(); - string_builder.writable()[len] = 0; - string_builder.len += len + 1; - return ( - StringPointer { - offset: start as u32, - length: len as u32, - }, - Encoding::Latin1, - ); - } if !string_builder.len.is_multiple_of(align_of::()) { string_builder.writable()[0] = 0; string_builder.len += 1; diff --git a/test/bundler/bundler_compile.test.ts b/test/bundler/bundler_compile.test.ts index c731fdd76eeb..5cc760ed5172 100644 --- a/test/bundler/bundler_compile.test.ts +++ b/test/bundler/bundler_compile.test.ts @@ -913,9 +913,8 @@ describe("bundler", () => { }); // A text import in a compiled executable is embedded as a string body - // (Latin-1 when every code point fits, UTF-16LE otherwise) that the runtime - // hands back without a parse or a copy, instead of a JS module with a string - // literal. The same source also checks `require()`, `import()`, and that + // (8-bit when ASCII, UTF-16LE otherwise) that the runtime hands back without + // a parse or a copy, instead of a JS module with a string literal. The same source also checks `require()`, `import()`, and that // `Bun.embeddedFiles` keeps listing only real assets. // Buffers: `files` strings go through dedent(), which would trim them. const textImportFiles = { @@ -967,7 +966,7 @@ describe("bundler", () => { // The embedded bytes are the string body itself. const encoded = { "ascii.txt": Buffer.from(expected.ascii, "latin1"), - "latin1.txt": Buffer.from(expected.latin1, "latin1"), + "latin1.txt": Buffer.from(expected.latin1, "utf16le"), "wide.txt": Buffer.from(expected.wide, "utf16le"), "empty.txt": Buffer.alloc(0), "invalid.txt": Buffer.from(expected.invalid, "utf16le"),