diff --git a/Cargo.lock b/Cargo.lock index 25d7c449b58d..948a2de2eca9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -260,6 +260,7 @@ dependencies = [ "bun_errno", "bun_event_loop", "bun_exe_format", + "bun_glob", "bun_hash", "bun_http_types", "bun_io", diff --git a/docs/bundler/index.mdx b/docs/bundler/index.mdx index 4279eaddb3c3..117412d0c8f7 100644 --- a/docs/bundler/index.mdx +++ b/docs/bundler/index.mdx @@ -1478,6 +1478,25 @@ interface BuildMetafile { } ``` +## Glob imports + +A `require()` or `import()` whose argument is a relative path built from a template literal or string concatenation is bundled like in [esbuild](https://esbuild.github.io/api/#glob). The static parts become a glob pattern, every file that matches is bundled, and the call becomes a lookup in a map from specifier to module: + +```ts title="index.ts" icon="/icons/typescript.svg" +const lang = process.env.LANG ?? "en"; +const messages = require(`./locales/${lang}.json`); // bundles every ./locales/**/*.json +``` + +The rules: + +- The path must start with `./` or `../` and name a directory or file prefix before the first `${}` or `+`. A bare `require("./" + name)` is left alone. +- Static text that contains a glob metacharacter (`*`, `?`, `[`, `{`, `!`, or `\`), or a `.`, `..`, or empty path segment past the leading `./` or `../`, leaves the call as a runtime call. +- A placeholder right after a `/` matches across directories (`**/*`). Anywhere else it matches one path segment (`*`). +- A `const` whose initializer is such a template or concatenation is followed, so ``const file = `./bin/${arch}/addon.node`; require(file)`` is bundled too. +- When the path ends in a placeholder, each file can also be reached without its extension: `require("./src/" + name)` finds `./src/cat.js` for `"cat"`. +- A specifier that is not in the map falls back to the runtime `require()` or `import()`. With `allowUnresolved: []` (or `--reject-unresolved`) the map is a closed set and a miss throws `MODULE_NOT_FOUND`. +- A pattern that matches no files, or more than 1024 files, leaves the call as a runtime call. + ## Outputs The `Bun.build` function returns a `Promise`, defined as: diff --git a/docs/snippets/cli/build.mdx b/docs/snippets/cli/build.mdx index bf26360f7483..57ab1e404d59 100644 --- a/docs/snippets/cli/build.mdx +++ b/docs/snippets/cli/build.mdx @@ -104,12 +104,14 @@ bun build - Allow unresolved dynamic import()/require() specifiers matching these glob patterns. Pass '' to allow - opaque specifiers + Allow unresolved dynamic import()/require() specifiers matching these glob patterns. A pattern is matched against the + specifier's static parts (a template literal, a string concatenation, or a const bound to one) with + * in place of each dynamic part. Pass '' to allow opaque specifiers - Fail the build on any dynamic import()/require() specifier that cannot be resolved at build time + Fail the build on any dynamic import()/require() specifier that cannot be resolved at build time. A relative template + or concatenation that matches files on disk is bundled as a closed glob map and passes diff --git a/packages/bun-types/bun.d.ts b/packages/bun-types/bun.d.ts index 74c1aaf5070f..d852bd8037ad 100644 --- a/packages/bun-types/bun.d.ts +++ b/packages/bun-types/bun.d.ts @@ -3254,10 +3254,17 @@ declare module "bun" { * - `["*"]` (default) — allow all dynamic specifiers * - `[]` — fail the build on any dynamic specifier * - `["./locales/*.json", ...]` — allow only specifiers whose static - * template parts match one of these glob patterns + * parts match one of these glob patterns * + * The static parts come from a template literal, a string concatenation, + * or a `const` bound to one of those; each dynamic part becomes `*`. * Add `""` to the list to allow fully opaque specifiers like `import(fn())`. * + * A relative template or concatenation that matches files on disk is + * bundled as a glob map and always passes. The map keeps a runtime + * fallback for specifiers it does not contain only when this option + * allows the shape; with `[]` a miss throws `MODULE_NOT_FOUND`. + * * @default ["*"] */ allowUnresolved?: string[]; diff --git a/src/ast/runtime.rs b/src/ast/runtime.rs index 2295ab6ef4b4..3d9b7ed9f119 100644 --- a/src/ast/runtime.rs +++ b/src/ast/runtime.rs @@ -173,10 +173,11 @@ pub struct Imports { pub(crate) __promiseAll: Ref, pub(crate) __MEMO_CACHE_SENTINEL: Ref, pub(crate) __EARLY_RETURN_SENTINEL: Ref, + pub(crate) __glob: Ref, } impl Imports { - pub const ALL: [&'static [u8]; 27] = [ + pub const ALL: [&'static [u8]; 28] = [ b"__name", b"__require", b"__export", @@ -204,12 +205,13 @@ impl Imports { b"__promiseAll", b"__MEMO_CACHE_SENTINEL", b"__EARLY_RETURN_SENTINEL", + b"__glob", ]; /// Rust stable cannot sort in `const`; precomputed here and verified by /// the test in `tests` below. #[cfg_attr(not(test), allow(dead_code))] - const ALL_SORTED: [&'static [u8]; 27] = [ + const ALL_SORTED: [&'static [u8]; 28] = [ b"$$typeof", b"__EARLY_RETURN_SENTINEL", b"__MEMO_CACHE_SENTINEL", @@ -220,6 +222,7 @@ impl Imports { b"__export", b"__exportDefault", b"__exportValue", + b"__glob", b"__jsonParse", b"__legacyDecorateClassTS", b"__legacyDecorateParamTS", @@ -241,34 +244,35 @@ impl Imports { /// When generating the list of runtime imports, we sort it for determinism. /// This is a lookup table so we don't need to resort the strings each time - pub const ALL_SORTED_INDEX: [usize; 27] = [ - 15, // __name - 24, // __require + pub const ALL_SORTED_INDEX: [usize; 28] = [ + 16, // __name + 25, // __require 7, // __export - 23, // __reExport + 24, // __reExport 9, // __exportValue 8, // __exportDefault - 14, // __merge - 11, // __legacyDecorateClassTS - 12, // __legacyDecorateParamTS - 13, // __legacyMetadataTS - 22, // __publicField - 18, // __privateIn - 17, // __privateGet - 16, // __privateAdd - 20, // __privateSet - 19, // __privateMethod + 15, // __merge + 12, // __legacyDecorateClassTS + 13, // __legacyDecorateParamTS + 14, // __legacyMetadataTS + 23, // __publicField + 19, // __privateIn + 18, // __privateGet + 17, // __privateAdd + 21, // __privateSet + 20, // __privateMethod 6, // __decoratorStart 5, // __decoratorMetadata - 25, // __runInitializers + 26, // __runInitializers 4, // __decorateElement 0, // $$typeof - 26, // __using + 27, // __using 3, // __callDispose - 10, // __jsonParse - 21, // __promiseAll + 11, // __jsonParse + 22, // __promiseAll 2, // __MEMO_CACHE_SENTINEL 1, // __EARLY_RETURN_SENTINEL + 10, // __glob ]; pub const NAME: &'static [u8] = b"bun:wrap"; @@ -304,6 +308,7 @@ impl Imports { 24 => self.__promiseAll, 25 => self.__MEMO_CACHE_SENTINEL, 26 => self.__EARLY_RETURN_SENTINEL, + 27 => self.__glob, _ => return None, }; r.to_nullable() @@ -339,6 +344,7 @@ impl Imports { 24 => Some(&mut self.__promiseAll), 25 => Some(&mut self.__MEMO_CACHE_SENTINEL), 26 => Some(&mut self.__EARLY_RETURN_SENTINEL), + 27 => Some(&mut self.__glob), _ => None, } } diff --git a/src/bundler/Cargo.toml b/src/bundler/Cargo.toml index 3f9660ede8ff..99fb1a46169e 100644 --- a/src/bundler/Cargo.toml +++ b/src/bundler/Cargo.toml @@ -39,6 +39,7 @@ bun_collections.workspace = true bun_css.workspace = true bun_dotenv.workspace = true bun_event_loop.workspace = true +bun_glob.workspace = true bun_http_types.workspace = true bun_parsers.workspace = true bun_io.workspace = true diff --git a/src/bundler/ParseTask.rs b/src/bundler/ParseTask.rs index 20f6ea89b3b1..1a2f74064239 100644 --- a/src/bundler/ParseTask.rs +++ b/src/bundler/ParseTask.rs @@ -2445,6 +2445,14 @@ pub mod parse_worker { // a worker-owned `Transpiler` that outlives the parse. // SAFETY: ARENA — `topts` outlives `opts` (worker-owned for the bundle pass). opts.allow_unresolved = unsafe { bun_collections::detach_ref(&topts.allow_unresolved) }; + opts.glob_resolver = Some(crate::options::parser_glob_resolver); + // SAFETY: ARENA — `topts` outlives `opts` (worker-owned for the bundle pass). + opts.glob_stem_extension_orders = unsafe { + [ + bun_collections::detach_ref(&*topts.extension_order.default.default), + bun_collections::detach_ref(&*topts.extension_order.default.esm), + ] + }; // `Transpiler.macro_context` is `Option` // (same nominal type as `ParserOptions.macro_context`'s pointee). Reborrow // through the raw `*mut Transpiler` so the `&mut MacroContext` is disjoint diff --git a/src/bundler/options.rs b/src/bundler/options.rs index 292026162119..ef3769785bc4 100644 --- a/src/bundler/options.rs +++ b/src/bundler/options.rs @@ -78,6 +78,54 @@ pub(crate) fn validate_path( // `&transpiler.options.allow_unresolved` straight through. pub use bun_js_parser::options::AllowUnresolved; +/// `glob_resolver` for the parser's template-literal `require()` / `import()` +/// support: every file under `source_dir` matching `pattern`, each as the +/// `./`- or `../`-prefixed POSIX-separated relative specifier the runtime +/// string would have to equal. Hidden entries are skipped (`dot: false`). +pub fn parser_glob_resolver(source_dir: &[u8], pattern: &[u8]) -> Vec> { + let mut out: Vec> = Vec::new(); + if !(pattern.starts_with(b"./") || pattern.starts_with(b"../")) { + return out; + } + + let walker = match bun_glob::BunGlobWalker::init_with_cwd( + pattern, source_dir, /* dot */ false, /* absolute */ true, + /* follow_symlinks */ true, /* error_on_broken_symlinks */ false, + /* only_files */ true, None, + ) { + Ok(Ok(w)) => w, + _ => return out, + }; + let mut walker = Box::new(walker); + let mut iter = bun_glob::walk::Iterator::new(&mut *walker); + if iter.init().map(|m| m.is_err()).unwrap_or(true) { + return out; + } + loop { + match iter.next() { + Ok(Ok(Some(abs))) => { + // `Loose` emits `/` separators, so the key matches the + // string the source builds at runtime on every platform. + let rel = bun_paths::resolve_path::relative_platform::< + bun_paths::resolve_path::platform::Loose, + false, + >(source_dir, &abs); + let mut key: Vec = Vec::with_capacity(rel.len() + 2); + if !(rel.starts_with(b"./") || rel.starts_with(b"../")) { + key.extend_from_slice(b"./"); + } + key.extend_from_slice(rel); + out.push(key.into_boxed_slice()); + } + Ok(Ok(None)) => break, + // A mid-walk failure would otherwise emit a partial map whose + // keys can never cover the runtime specifier. + Ok(Err(_)) | Err(_) => return Vec::new(), + } + } + out +} + // Canonical defs live in `bun_resolver::options` (lower tier; resolver is the // runtime consumer of `.patterns`/`.abs_paths`/`.node_modules`). Re-export so // `BundleOptions.external` and `Resolver.opts.external` are the SAME nominal diff --git a/src/bundler/transpiler.rs b/src/bundler/transpiler.rs index 18c768c20b49..2535a21c4ca1 100644 --- a/src/bundler/transpiler.rs +++ b/src/bundler/transpiler.rs @@ -1580,6 +1580,8 @@ impl<'a> Transpiler<'a> { macro_context: None, warn_about_unbundled_modules: !target.is_bun(), allow_unresolved: &p_opts::AllowUnresolved::DEFAULT, + glob_resolver: None, + glob_stem_extension_orders: [&[], &[]], module_type: to_parser_module_type(this_parse.module_type), output_format: p_opts::Format::Esm, transform_only: self.options.transform_only, diff --git a/src/js_parser/p.rs b/src/js_parser/p.rs index 0f7eab70d7a2..27df02c499bf 100644 --- a/src/js_parser/p.rs +++ b/src/js_parser/p.rs @@ -563,6 +563,10 @@ pub struct P<'a, const TYPESCRIPT: bool, const SCAN_ONLY: bool> { pub(crate) const_values: bun_ast::ast_result::ConstValuesMap, + /// `const x = ` initializers, consulted by + /// `append_dynamic_specifier_shape` so `require(x)` can glob. + pub(crate) glob_specifier_values: bun_ast::ast_result::ConstValuesMap, + // These are backed by stack fallback allocators in _parse, and are uninitialized until then. pub(crate) binary_expression_stack: ListManaged<'a, BinaryExpressionVisitor>, // Reusable stack for `SideEffects::simplify_unused_binary_comma_expr`; @@ -782,6 +786,19 @@ impl<'a, const TYPESCRIPT: bool, const SCAN_ONLY: bool> P<'a, TYPESCRIPT, SCAN_O } } +/// How a sub-expression contributes to a dynamic specifier shape. +#[derive(Clone, Copy, PartialEq, Eq)] +enum ShapePart { + /// The expression's static shape was appended to the buffer (it may + /// contain placeholders from nested dynamic parts). + Appended, + /// Unknown at compile time: the caller adds one placeholder for it. + Opaque, + /// A literal NUL would be confused with the placeholder marker, so the + /// whole specifier cannot be shaped. + Poison, +} + impl<'a, const TYPESCRIPT: bool, const SCAN_ONLY: bool> P<'a, TYPESCRIPT, SCAN_ONLY> { pub(crate) const ALLOW_MACROS: bool = !cfg!(target_family = "wasm"); @@ -794,35 +811,467 @@ impl<'a, const TYPESCRIPT: bool, const SCAN_ONLY: bool> P<'a, TYPESCRIPT, SCAN_O } /// Extracts a matchable "shape" from a dynamic import argument. - /// Template literals: static parts joined by \x00 placeholders. - /// Everything else: empty string. + /// Template literals and string concatenation: static parts joined by + /// \x00 placeholders. Everything else: empty string. fn extract_dynamic_specifier_shape<'b>( &mut self, arg: Expr, buf: &'b mut BumpVec<'a, u8>, ) -> Result<&'b [u8], crate::Error> { - if let Some(tmpl) = arg.data.e_template() { - if tmpl.tag.is_some() { - return Ok(b""); // tagged template — opaque + if self.append_dynamic_specifier_shape(arg, buf, 0)? == ShapePart::Appended { + Ok(buf.as_slice()) + } else { + buf.clear(); + Ok(b"") + } + } + + fn append_estring_rope( + &self, + s: &js_ast::e::EString, + buf: &mut BumpVec<'a, u8>, + ) -> Result { + let start = buf.len(); + let mut cur: Option<&js_ast::e::EString> = Some(s); + while let Some(seg) = cur { + let bytes = seg.string(self.arena)?; + // A literal NUL would be confused with the placeholder marker. + if strings::contains_char(bytes, 0) { + buf.truncate(start); + return Ok(false); + } + buf.extend_from_slice(bytes); + cur = seg.next.as_ref().map(|r| r.get()); + } + Ok(true) + } + + /// Appends `arg`'s static shape to `buf`. On `Opaque` and `Poison` + /// nothing is appended (`buf` is truncated back to its entry length). + fn append_dynamic_specifier_shape( + &mut self, + arg: Expr, + buf: &mut BumpVec<'a, u8>, + depth: u32, + ) -> Result { + use ShapePart::{Appended, Opaque, Poison}; + if depth >= 32 { + return Ok(Opaque); + } + let start = buf.len(); + let part = match &arg.data { + js_ast::ExprData::EString(s) => { + if self.append_estring_rope(s, buf)? { + Appended + } else { + Poison + } } - match &tmpl.head { - js_ast::e::TemplateContents::Cooked(head) => { - buf.extend_from_slice(head.string(self.arena)?); + js_ast::ExprData::ETemplate(tmpl) => 'tmpl: { + if tmpl.tag.is_some() { + break 'tmpl Opaque; } - js_ast::e::TemplateContents::Raw(_) => return Ok(b""), // shouldn't happen post-visit but be safe + match &tmpl.head { + js_ast::e::TemplateContents::Cooked(head) => { + if !self.append_estring_rope(head, buf)? { + break 'tmpl Poison; + } + } + js_ast::e::TemplateContents::Raw(_) => break 'tmpl Opaque, + } + let mut out = Appended; + for part in tmpl.parts().iter() { + let value = part.value; + match self.append_dynamic_specifier_shape(value, buf, depth + 1)? { + Appended => {} + Opaque => buf.push(0), + Poison => { + out = Poison; + break; + } + } + match &part.tail { + js_ast::e::TemplateContents::Cooked(tail) => { + if !self.append_estring_rope(tail, buf)? { + out = Poison; + break; + } + } + js_ast::e::TemplateContents::Raw(_) => { + out = Opaque; + break; + } + } + } + out + } + js_ast::ExprData::EBinary(bin) if bin.op == js_ast::Op::Code::BinAdd => 'add: { + let (l, r) = (bin.left, bin.right); + let part_l = self.append_dynamic_specifier_shape(l, buf, depth + 1)?; + if part_l == Poison { + break 'add Poison; + } + if part_l == Opaque { + buf.push(0); + } + let part_r = self.append_dynamic_specifier_shape(r, buf, depth + 1)?; + if part_r == Poison { + break 'add Poison; + } + if part_r == Opaque { + buf.push(0); + } + if part_l == Opaque && part_r == Opaque { + Opaque + } else { + Appended + } + } + js_ast::ExprData::EIdentifier(id) => { + let ref_ = id.ref_; + if ref_.source_index() == self.source.index.0 + && self.symbols[ref_.inner_index() as usize].kind + == js_ast::symbol::Kind::Constant + { + // Taken out while recursing so a self-referential + // initializer (const a = `./x/${a}`) terminates. + if let Some(init) = self.glob_specifier_values.remove(&ref_) { + let r = self.append_dynamic_specifier_shape(init, buf, depth + 1); + self.glob_specifier_values.put(ref_, init).expect("oom"); + return r; + } + } + Opaque + } + _ => Opaque, + }; + if part != Appended { + buf.truncate(start); + } + Ok(part) + } + + /// Relative, has a placeholder, and the static text before the first + /// placeholder names a path component: `./${x}` would otherwise glob the + /// importer's whole directory tree. + fn glob_shape_is_eligible(shape: &[u8]) -> bool { + if !(shape.starts_with(b"./") || shape.starts_with(b"../")) { + return false; + } + let Some(first_placeholder) = strings::index_of_char_usize(shape, 0) else { + return false; + }; + if !shape[..first_placeholder] + .iter() + .any(|&b| b != b'.' && b != b'/') + { + return false; + } + if shape.last() == Some(&b'/') { + return false; + } + // Past the leading relative prefix, a `.`, `..`, or empty segment in + // the static text is normalized away in the map keys, so the runtime + // string could never equal a key and every lookup would miss. + let mut rest = shape; + if let Some(s) = rest.strip_prefix(b"./") { + rest = s; + } else { + while let Some(s) = rest.strip_prefix(b"../") { + rest = s; + } + } + for seg in strings::split(rest, b"/") { + if seg.is_empty() || strings::eql(seg, b".") || strings::eql(seg, b"..") { + return false; + } + } + // Glob metacharacters in the literal text would change the match. + strings::index_of_any(shape, b"*?[{!\\").is_none() + } + + /// Like esbuild, a placeholder right after `/` becomes `**/*` (it may + /// span directories); anywhere else it becomes `*` (one path segment). + fn glob_shape_to_pattern(shape: &[u8], out: &mut Vec) { + out.clear(); + let mut i = 0; + while i < shape.len() { + let b = shape[i]; + if b != 0 { + out.push(b); + i += 1; + continue; + } + while i < shape.len() && shape[i] == 0 { + i += 1; + } + if out.last() == Some(&b'/') { + out.extend_from_slice(b"**/*"); + } else { + out.push(b'*'); + } + } + } + + /// A stem key (`./src/cat` for `./src/cat.js`) resolves like a + /// hand-written `require("./src/cat")`, so the extension must be in the + /// effective resolver order or the probe could not find the file. + fn glob_resolvable_stem<'k>(key: &'k [u8], extension_order: &[Box<[u8]>]) -> Option<&'k [u8]> { + let base_start = strings::last_index_of_char(key, b'/').map_or(0, |i| i + 1); + let base = &key[base_start..]; + let dot = strings::last_index_of_char(base, b'.')?; + if dot == 0 { + return None; + } + let ext = &base[dot..]; + if !extension_order.iter().any(|e| strings::eql(e, ext)) { + return None; + } + Some(&key[..base_start + dot]) + } + + /// Record `const x =