diff --git a/src/bun_alloc/lib.rs b/src/bun_alloc/lib.rs index 0cae99668a53..6cbca63fefce 100644 --- a/src/bun_alloc/lib.rs +++ b/src/bun_alloc/lib.rs @@ -2196,7 +2196,7 @@ impl /// is never needed by callers, only the hash. `#[inline(always)]` + the /// const-generic branch fold to identical codegen at each monomorphization. #[inline(always)] - fn key_hash(denormalized_key: &[u8]) -> u64 { + pub fn key_hash(denormalized_key: &[u8]) -> u64 { let key = if REMOVE_TRAILING_SLASHES { trim_right(denormalized_key, SEP_STR.as_bytes()) } else { @@ -2312,6 +2312,55 @@ impl let _key = Self::key_hash(denormalized_key); self.index.remove(&_key).is_some() } + + /// [`remove`](Self::remove) that also returns the hash of the key and what the key mapped + /// to: a slot, `NOT_FOUND` or `UNASSIGNED`. A slot stays allocated, so + /// [`publish`](Self::publish) can map the hash to it again. + pub fn unpublish(&mut self, denormalized_key: &[u8]) -> (u64, Option) { + let _guard = self.mutex.lock(); + let hash = Self::key_hash(denormalized_key); + (hash, self.index.remove(&hash)) + } + + /// Unmaps every key whose slot satisfies `pred` and passes its hash and slot to `unpublished`. + /// A key that maps to `NOT_FOUND` or `UNASSIGNED` has no slot and stays. + pub fn unpublish_where( + &mut self, + mut pred: impl FnMut(&ValueType) -> bool, + mut unpublished: impl FnMut(u64, IndexType), + ) { + let _guard = self.mutex.lock(); + let Self { + index, + overflow_list, + backing_buf, + .. + } = self; + index.retain(|&hash, &mut slot| { + if slot.index() == NOT_FOUND.index() || slot.index() == UNASSIGNED.index() { + return true; + } + let value: &ValueType = if slot.is_overflow() { + overflow_list.at_index_mut(slot) + } else { + // SAFETY: a non-sentinel, non-overflow index was assigned by `put`, which + // initialized this slot via `.write()`. + unsafe { backing_buf[slot.index() as usize].assume_init_ref() } + }; + if !pred(value) { + return true; + } + unpublished(hash, slot); + false + }); + } + + /// Maps `hash` to a slot that [`unpublish`](Self::unpublish) or + /// [`unpublish_where`](Self::unpublish_where) returned. + pub fn publish(&mut self, hash: u64, slot: IndexType) { + let _guard = self.mutex.lock(); + self.index.insert(hash, slot); + } } // ────────────────────────────────────────────────────────────────────────── diff --git a/src/resolver/dir_info.rs b/src/resolver/dir_info.rs index b90cba7326e7..ac021fda1f7d 100644 --- a/src/resolver/dir_info.rs +++ b/src/resolver/dir_info.rs @@ -25,7 +25,8 @@ pub type Index = IndexType; // `&mut DirInfo` is ever materialized concurrently with a read — writes happen // only inside `dir_info_uncached` while filling a freshly-`put` slot, before // any handle to that slot escapes. All access is additionally serialized under -// the resolver mutex. +// the resolver mutex. The one field that changes later is `entries`, which is +// atomic: see `ListingIndex`. // // `as_ptr()` exposes the raw `*mut` for the few callers that still need it // (the `dir_info_uncached` fill path and `MatchResult.dir_info` round-trip). @@ -134,7 +135,7 @@ pub struct DirInfo { // lifetime — slice into BSS-backed path storage; never individually freed pub abs_path: &'static [u8], - pub(crate) entries: Index, + pub(crate) entries: ListingIndex, /// Is there a "package.json" file? // `NonNull` (not `&'static`) preserves mut-provenance. Read via `.package_json()`. pub package_json: Option>, @@ -148,6 +149,38 @@ pub struct DirInfo { pub(crate) flags: Flags, } +// One slot per directory that any resolver of the process reads, never freed. +const _: () = assert!(core::mem::size_of::() == 104); + +/// `DirInfo::entries`: where the listing of the directory is in `RealFS::entries`. A bust drops +/// the listing. A re-read that finds the directory as it was keeps the slot, because its +/// subdirectories hold the index of the slot, and stores the index of the new listing here. +#[derive(Default)] +pub(crate) struct ListingIndex(core::sync::atomic::AtomicU32); + +impl ListingIndex { + const OVERFLOW_SHIFT: u32 = 31; + + pub(crate) fn new(index: Index) -> Self { + Self(core::sync::atomic::AtomicU32::new(Self::bits(index))) + } + + #[inline] + pub(crate) fn get(&self) -> Index { + let bits = self.0.load(core::sync::atomic::Ordering::Relaxed); + Index::new(bits, bits >> Self::OVERFLOW_SHIFT != 0) + } + + pub(crate) fn set(&self, index: Index) { + self.0 + .store(Self::bits(index), core::sync::atomic::Ordering::Relaxed); + } + + fn bits(index: Index) -> u32 { + index.index() | (u32::from(index.is_overflow()) << Self::OVERFLOW_SHIFT) + } +} + impl Default for DirInfo { fn default() -> Self { Self { @@ -159,7 +192,7 @@ impl Default for DirInfo { package_json_for_module_type: None, package_json_for_dependencies: None, abs_path: b"", - entries: Index::default(), + entries: ListingIndex::default(), package_json: None, tsconfig_json: None, abs_real_path: b"", @@ -183,6 +216,51 @@ fn arena_ref(p: NonNull) -> &'static T { } impl DirInfo { + /// Whether a re-read of a busted directory (`self`) left it as `before`, the slot it had when + /// it was busted. A subdirectory copies the pointers and `InsideNodeModules` when it is + /// filled, and reads the rest through `parent` and `enclosing_browser_scope` later. So when + /// this is true the directory keeps `before`, and what was filled below it still holds. Every + /// field is named here so that a new one does not compile until it has an answer. + pub(crate) fn is_same_after_reread(&self, before: &DirInfo) -> bool { + fn same(a: Option<&'static T>, b: Option<&'static T>) -> bool { + a.map(core::ptr::from_ref) == b.map(core::ptr::from_ref) + } + let DirInfo { + parent, + enclosing_browser_scope, + package_json_for_browser_field, + enclosing_tsconfig_json, + enclosing_package_json, + package_json_for_module_type, + package_json_for_dependencies, + // The key is the same path. + abs_path: _, + // A re-read has a new listing. + entries: _, + package_json, + tsconfig_json, + abs_real_path, + flags, + } = self; + *parent == before.parent + && *enclosing_browser_scope == before.enclosing_browser_scope + && same( + *package_json_for_browser_field, + before.package_json_for_browser_field, + ) + && same(*enclosing_tsconfig_json, before.enclosing_tsconfig_json) + && same(*enclosing_package_json, before.enclosing_package_json) + && same( + *package_json_for_module_type, + before.package_json_for_module_type, + ) + && *package_json_for_dependencies == before.package_json_for_dependencies + && *package_json == before.package_json + && *tsconfig_json == before.tsconfig_json + && *abs_real_path == before.abs_real_path + && *flags == before.flags + } + /// Is there a "node_modules" subdirectory? #[inline] pub(crate) fn has_node_modules(&self) -> bool { @@ -242,7 +320,7 @@ impl DirInfo { pub fn get_entries_ref_locked(&self, generation: Generation) -> Option<&'static fs::DirEntry> { fs::FileSystem::instance() .fs - .entries_at_locked(self.entries, generation)? + .entries_at_locked(self.entries.get(), generation)? .as_entries() } @@ -263,7 +341,7 @@ impl DirInfo { // `MutexGuard` stores the mutex by raw pointer, so holding it does not // keep `rfs` borrowed. let _lock = rfs.entries_mutex.lock_guard(); - rfs.entries_at_locked(self.entries, generation)? + rfs.entries_at_locked(self.entries.get(), generation)? .as_entries()? .get(query) } @@ -276,7 +354,7 @@ impl DirInfo { fs::FileSystem::instance() .fs .entries - .at_index(self.entries)? + .at_index(self.entries.get())? .as_entries() } @@ -378,6 +456,166 @@ pub(crate) unsafe fn slot_ptr_at(index: Index) -> Option<*mut DirInfo> { } } +/// A slot that a bust took out of the index. A lookup of its key settles it: see +/// `Resolver::dir_info_cached_miss`. +#[derive(Clone, Copy)] +pub(crate) struct Parked { + pub(crate) slot: Index, + /// True: this directory was busted. It is read again and compared with the slot. + /// False: a directory above it was. The slot is good while its parent has the slot it had. + pub(crate) busted: bool, +} + +type ParkedMap = std::collections::HashMap< + u64, + Parked, + core::hash::BuildHasherDefault, +>; + +/// By the hash of the key, as in the index. Every access holds `RealFS::entries_mutex`, which +/// also keeps a fill from writing the slots that are read here. +static PARKED: bun_threading::Guarded = + bun_threading::Guarded::new(ParkedMap::with_hasher(core::hash::BuildHasherDefault::new())); + +/// `PARKED.len()`, for `any_parked`. +static PARKED_COUNT: core::sync::atomic::AtomicUsize = core::sync::atomic::AtomicUsize::new(0); + +/// Whether a bust left a slot out of the index. False in a process that never busts. +#[inline] +pub(crate) fn any_parked() -> bool { + PARKED_COUNT.load(core::sync::atomic::Ordering::Relaxed) != 0 +} + +/// Whether `abs_path` names a directory below `dir`. Either can end in a separator. +fn is_below(abs_path: &[u8], dir: &[u8]) -> bool { + let is_separator = |byte: u8| bun_paths::Platform::AUTO.is_separator(byte); + let dir = match dir.split_last() { + Some((&last, rest)) if is_separator(last) => rest, + _ => dir, + }; + abs_path.len() > dir.len() + 1 && abs_path.starts_with(dir) && is_separator(abs_path[dir.len()]) +} + +/// Takes the directory `key` out of the index, with every directory below it: each one holds +/// what it took from the directory above it when it was filled. Returns whether `key` was in +/// the index, as `BSSMapInner::remove` does. +/// +/// The caller holds `RealFS::entries_mutex`. +pub(crate) fn bust(map: &mut HashMap, key: &[u8]) -> bool { + let (hash, unpublished) = map.unpublish(key); + let mut parked = PARKED.lock(); + let slot = unpublished + .filter(|it| it.index() != NOT_FOUND.index() && it.index() != UNASSIGNED.index()); + let Some(slot) = slot else { + // A bust of a directory above this one parked it. Now it is busted too. + let was_parked = parked + .get_mut(&hash) + .is_some_and(|it| !core::mem::replace(&mut it.busted, true)); + return unpublished.is_some() || was_parked; + }; + parked.insert(hash, Parked { slot, busted: true }); + map.unpublish_where( + |info| is_below(info.abs_path, key), + |hash, slot| { + parked.insert( + hash, + Parked { + slot, + busted: false, + }, + ); + }, + ); + PARKED_COUNT.store(parked.len(), core::sync::atomic::Ordering::Relaxed); + true +} + +/// The slot that a bust took out of the index for the key with this hash, if one did. The +/// caller publishes the slot again or leaves it out for good. +/// +/// The caller holds `RealFS::entries_mutex`. +pub(crate) fn take_parked(hash: u64) -> Option { + let mut parked = PARKED.lock(); + let taken = parked.remove(&hash)?; + PARKED_COUNT.store(parked.len(), core::sync::atomic::Ordering::Relaxed); + Some(taken) +} + +/// Leaves every parked directory below `dir` out of the index for good: `dir` is not what it +/// was when they were filled. +/// +/// The caller holds `RealFS::entries_mutex`. +pub(crate) fn discard_parked_below(dir: &[u8]) { + let mut parked = PARKED.lock(); + parked.retain(|_, it| { + // SAFETY: a parked slot was assigned by `put`. `entries_mutex` is held: see `PARKED`. + let slot = unsafe { slot_ptr_at(it.slot) }; + // SAFETY: `slot_ptr_at` returns a pointer to an initialized slot. + !slot.is_some_and(|slot| is_below(unsafe { (*slot).abs_path }, dir)) + }); + PARKED_COUNT.store(parked.len(), core::sync::atomic::Ordering::Relaxed); +} + +/// A file that the tsconfig.json of a cached directory extends. +pub(crate) struct ExtendedFile { + /// In `DirnameStore`. + pub(crate) path: &'static [u8], + /// `SourceFile::hash` of the file when the configurations that extend it read it. `None`: + /// two of them read different bytes, so one of them is not what the file has now. + pub(crate) seen: Option>, +} + +type ExtendedMap = std::collections::HashMap< + u64, + Vec, + core::hash::BuildHasherDefault, +>; + +/// By the hash of the key of the directory that has the file. A re-read of a busted directory +/// leaves its slot as it is when its own configuration did not change. A change to one of these +/// files is what tells it that a directory below it is still not to be published as it was. +/// Every access holds `RealFS::entries_mutex`. +static EXTENDED: bun_threading::Guarded = bun_threading::Guarded::new( + ExtendedMap::with_hasher(core::hash::BuildHasherDefault::new()), +); + +/// `EXTENDED.len()`, for `any_extended`. +static EXTENDED_COUNT: core::sync::atomic::AtomicUsize = core::sync::atomic::AtomicUsize::new(0); + +/// Whether any cached tsconfig.json extends another file. +#[inline] +pub(crate) fn any_extended() -> bool { + EXTENDED_COUNT.load(core::sync::atomic::Ordering::Relaxed) != 0 +} + +/// Records what a tsconfig.json that was just loaded read from `path`, a file it extends. +/// `dir_hash` is the key hash of the directory of `path`, and `hash` is `SourceFile::hash`. +pub(crate) fn note_extended(dir_hash: u64, path: &'static [u8], hash: Option) { + let mut extended = EXTENDED.lock(); + let files = extended.entry(dir_hash).or_default(); + match files.iter_mut().find(|it| *it.path == *path) { + Some(file) => { + if file.seen != Some(hash) { + file.seen = None; + } + } + None => files.push(ExtendedFile { + path, + seen: Some(hash), + }), + } + EXTENDED_COUNT.store(extended.len(), core::sync::atomic::Ordering::Relaxed); +} + +/// The extended files in the directory with this key hash. `put_extended` gives them back. +pub(crate) fn take_extended(dir_hash: u64) -> Option> { + EXTENDED.lock().remove(&dir_hash) +} + +pub(crate) fn put_extended(dir_hash: u64, files: Vec) { + EXTENDED.lock().insert(dir_hash, files); +} + bitflags::bitflags! { #[derive(Clone, Copy, Default, PartialEq, Eq, Debug)] pub struct Flags: u8 { diff --git a/src/resolver/lib.rs b/src/resolver/lib.rs index a9872c36e891..7ec91711b68e 100644 --- a/src/resolver/lib.rs +++ b/src/resolver/lib.rs @@ -1196,14 +1196,13 @@ pub mod fs { /// Evicts `file_path` from the directory-entry cache; returns whether /// an entry was removed. - pub(crate) fn bust_entries_cache(&mut self, file_path: &[u8]) -> bool { - // `entries` is the process-global - // BSSMap singleton and `remove` mutates it; callers (transpiler / - // hot-reloader / VM) reach this without `RESOLVER_MUTEX`, so take - // `entries_mutex` to satisfy `EntriesMap::inner`'s aliasing - // invariant. No caller already holds it (no re-entry from - // `read_directory`/`dir_info_cached_maybe_log`). - let _g = self.entries_mutex.lock_guard(); + /// + /// The caller holds `entries_mutex`: `entries` is the process-global + /// BSSMap singleton, and the callers of `Resolver::bust_dir_cache` + /// (transpiler / hot-reloader / VM) reach it without `RESOLVER_MUTEX`. + /// `bust_dir_cache` evicts the directory from its own cache under the + /// same lock. + pub(crate) fn bust_entries_cache_locked(&mut self, file_path: &[u8]) -> bool { self.entries.remove(file_path) } diff --git a/src/resolver/package_json.rs b/src/resolver/package_json.rs index 1aa43e1a9881..20f404f0b8f7 100644 --- a/src/resolver/package_json.rs +++ b/src/resolver/package_json.rs @@ -94,6 +94,9 @@ pub struct PackageJSON { pub config: Option>>, pub(crate) package_manager_package_id: PackageID, + /// Whether this parse read the dependency groups (`IncludeDependencies::Main` or `Local`). + /// A parse that did not cannot stand in for one that has to. + pub(crate) parsed_dependencies: bool, pub dependencies: DependencyMap, /// Used instead of `dependencies` for a package in node_modules, so that auto-install only /// parses the row it looks up. Each range is where a group's rows are in `json_tape`. @@ -151,6 +154,7 @@ impl Default for PackageJSON { scripts: None, config: None, package_manager_package_id: INVALID_PACKAGE_ID, + parsed_dependencies: false, dependencies: DependencyMap::default(), installed_dependency_rows: Default::default(), side_effects: SideEffects::default(), @@ -393,8 +397,24 @@ impl PackageJSON { package_id: Option, include_scripts_: IncludeScripts, ) -> Option { - let include_scripts = include_scripts_ == IncludeScripts::IncludeScripts; + let (package_json_path, entry_contents) = Self::read(r, input_path, dirname_fd)?; + Self::parse_contents::( + r, + package_json_path, + entry_contents, + package_id, + include_scripts_, + ) + } + /// The path and the bytes of the package.json of the directory `input_path`. Apart from + /// [`parse_contents`](Self::parse_contents) so that a re-read of the directory can compare the + /// bytes with those of the parse it already has. + pub(crate) fn read( + r: &mut resolver::Resolver<'_>, + input_path: &[u8], + dirname_fd: Fd, + ) -> Option<(&'static [u8], Box<[u8]>)> { // SAFETY: PORT (Stacked Borrows) — `r.fs()`/`r.log()` return RAW `*mut` // (see `Resolver::fs()` note in lib.rs). `fs` and `log` are DISTINCT // singletons so the two `&mut` projections below do not alias each other, @@ -473,6 +493,25 @@ impl PackageJSON { )); } + Some((package_json_path, entry_contents)) + } + + /// [`parse`](Self::parse) for bytes that [`read`](Self::read) returned. + pub(crate) fn parse_contents( + r: &mut resolver::Resolver<'_>, + package_json_path: &'static [u8], + entry_contents: Box<[u8]>, + package_id: Option, + include_scripts_: IncludeScripts, + ) -> Option { + let include_scripts = include_scripts_ == IncludeScripts::IncludeScripts; + + // SAFETY: see `read`: `r.fs()` and `r.log()` are distinct singletons, and nothing else + // retags either while these are live. + let r_fs: &mut fs::FileSystem = unsafe { &mut *r.fs() }; + // SAFETY: see above. + let r_log: &mut bun_ast::Log = unsafe { &mut *r.log() }; + // `bun_ast::Source.path` is the lightweight `bun_paths::fs::Path<'static>` (no // `pretty`/`is_node_module`); `key_path` is only used for `text`, so init the // source directly from the interned path. @@ -528,6 +567,7 @@ impl PackageJSON { scripts: None, config: None, package_manager_package_id: INVALID_PACKAGE_ID, + parsed_dependencies: false, dependencies: DependencyMap::default(), installed_dependency_rows: Default::default(), side_effects: SideEffects::Unspecified, @@ -814,6 +854,7 @@ impl PackageJSON { if INCLUDE_DEPENDENCIES == IncludeDependencies::Main || INCLUDE_DEPENDENCIES == IncludeDependencies::Local { + package_json.parsed_dependencies = true; 'update_dependencies: { if let Some(pkg) = package_id { package_json.package_manager_package_id = pkg; diff --git a/src/resolver/resolver.rs b/src/resolver/resolver.rs index 08cac08e4ce9..2ef6377fdf2c 100644 --- a/src/resolver/resolver.rs +++ b/src/resolver/resolver.rs @@ -2476,11 +2476,18 @@ impl<'a> Resolver<'a> { } /// Bust the directory cache for the given path, which may end in a separator. + /// + /// The cached directories below the path leave the index too: each one holds what it took + /// from the directory above it. The next lookup of one puts it back as it is, unless a + /// directory between it and the path, or the path, is different when it is read again. pub fn bust_dir_cache(&mut self, path: &[u8]) -> bool { let path = strings::without_trailing_slash_windows_path(path); Self::assert_valid_cache_key(path); - let first_bust = self.fs_mut().fs.bust_entries_cache(path); - let second_bust = self.dir_cache_mut().remove(path); + let rfs = &mut self.fs_mut().fs; + // The guard keeps the mutex by raw pointer, so `self` is free again below. + let _entries_lock = rfs.entries_mutex.lock_guard(); + let first_bust = rfs.bust_entries_cache_locked(path); + let second_bust = DirInfo::bust(self.dir_cache_mut(), path); bun_core::scoped_log!( ResolverDev, "Bust {} = {}, {}", @@ -3433,6 +3440,31 @@ impl<'a> Resolver<'a> { // the `dir_info_uncached` call, mirroring `dir_info_cached_miss`: the // route loaders iterate the `DirEntry.data` map under this lock. let _entries_unlock = rfs!().entries_mutex.lock_guard(); + + // The slot that this directory had when it was busted: see `fill_dir_info`. + let mut before_bust: Option = None; + if DirInfo::any_parked() + && let Some(parked) = DirInfo::take_parked(dir_cache_info_result.hash) + { + // SAFETY: a parked index was assigned by `put`; `entries_mutex` held. + let slot = unsafe { DirInfo::slot_ptr_at(parked.slot) } + .map(|slot| unsafe { DirInfoRef::from_raw(slot) }) + // A slot that this function filled has no parent. One that `dir_info_cached` + // filled is not what this function fills: it stays out of the index. + .filter(|slot| slot.parent == allocators::NOT_FOUND); + match slot { + Some(_) if parked.busted => before_bust = Some(parked.slot), + // A bust of a directory above this one took it out of the index, and it took + // nothing from that directory. + Some(slot) => { + self.dir_cache_mut() + .publish(dir_cache_info_result.hash, parked.slot); + return Ok(Some(slot)); + } + None => DirInfo::discard_parked_below(dir_path), + } + } + let mut cached_dir_entry_result = rfs!().entries.get_or_put(dir_path)?; // NOTE: always assigned by either the cached-hit arm or the @@ -3545,18 +3577,6 @@ impl<'a> Resolver<'a> { .expect("unreachable"); } - // We must initialize it as empty so that the result index is correct. - // This is important so that browser_scope has a valid index. - // SAFETY: `dir_cache()` is the live singleton; resolver mutex held. - let dir_info_ptr: *mut DirInfo::DirInfo = unsafe { - DirInfo::put_slot( - self.dir_cache(), - &mut dir_cache_info_result, - DirInfo::DirInfo::default(), - ) - } - .expect("unreachable"); - // `dir_path` is a slice into the threadlocal `bufs(.path_in_global_disk_cache)` buffer, // which gets overwritten on the next auto-install resolution. `dir_info_uncached` stores // its `path` argument directly as `DirInfo.abs_path` in the permanent `dir_cache`, so @@ -3566,12 +3586,12 @@ impl<'a> Resolver<'a> { // temporary is live when the raw `*mut` is passed below (avoids a needless Unique // retag that would pop the shared tag mid-argument-list under Stacked Borrows). let dir_entries_dir = unsafe { &*dir_entries_option }.entries().dir; - self.dir_info_uncached( - dir_info_ptr, + let dir_info_ptr: *mut DirInfo::DirInfo = self.fill_dir_info( + &mut dir_cache_info_result, + before_bust, dir_entries_dir, // already `*mut EntriesOption` — pass raw, no intermediate `&mut` retag dir_entries_option, - dir_cache_info_result, cached_dir_entry_result.index, // Packages in the global disk cache are top-level, we shouldn't try // to check for a parent package.json @@ -4057,11 +4077,8 @@ impl<'a> Resolver<'a> { } } - pub(crate) fn parse_tsconfig( - &mut self, - file: &[u8], - dirname_fd: FD, - ) -> crate::CrateResult>> { + /// The bytes of a configuration file. + fn read_config_file(&mut self, file: &[u8], dirname_fd: FD) -> crate::CrateResult> { // Since tsconfig.json is cached permanently, in our DirEntries cache // we must use the global allocator let mut entry = self.caches.fs.read_file_with_allocator( @@ -4080,6 +4097,23 @@ impl<'a> Resolver<'a> { let _close_guard = scopeguard::guard(entry, |mut e| { let _ = e.close_fd(); }); + // `use_shared_buffer = false` above, so `entry_contents` is + // `Contents::Owned`/`Empty`. + Ok(match entry_contents { + crate::cache::Contents::Owned(v) => v, + crate::cache::Contents::Empty => Vec::new(), + other => other.as_slice().to_vec(), + }) + } + + /// `sources` gets the file, for `TSConfigJSON::sources`. + pub(crate) fn parse_tsconfig( + &mut self, + file: &[u8], + dirname_fd: FD, + sources: &mut Vec, + ) -> crate::CrateResult>> { + let contents = self.read_config_file(file, dirname_fd); // The file name needs to be persistent because it can have errors // and if those errors need to print the filename @@ -4089,18 +4123,15 @@ impl<'a> Resolver<'a> { .dirname_store .append_slice(file) .expect("unreachable"); + sources.push(crate::tsconfig_json::SourceFile { + path: key_path, + hash: contents.as_deref().ok().map(bun_wyhash::hash), + }); - // `use_shared_buffer = false` above, so `entry_contents` is - // `Contents::Owned`/`Empty`. `TSConfigJSON` owns `Box<[u8]>` copies of - // every field, so the source bytes are dead once `parse` returns and - // can be dropped with the local `Source`. - let contents = match entry_contents { - crate::cache::Contents::Owned(v) => v, - crate::cache::Contents::Empty => Vec::new(), - other => other.as_slice().to_vec(), - }; - - let source = bun_ast::Source::init_path_string_owned(key_path, contents); + // `TSConfigJSON` owns `Box<[u8]>` copies of every field, so the source + // bytes are dead once `parse` returns and can be dropped with the + // local `Source`. + let source = bun_ast::Source::init_path_string_owned(key_path, contents?); let file_dir = source.path.source_dir(); // SAFETY: BACKREF — `self.log` (see `log()` NOTE); disjoint from `self.caches`, @@ -4145,13 +4176,32 @@ impl<'a> Resolver<'a> { unsafe { (*BIN_FOLDERS.get()).assume_init_ref().const_slice() } } + /// `before_bust`: the parse that the directory had when it was busted. The re-read keeps it + /// when the file has the same bytes, so that the directory compares equal to its slot + /// (`DirInfo::is_same_after_reread`) and the parse is not made and kept a second time. The + /// options of the resolver that made it stay, as they do for a directory that is not busted. pub(crate) fn parse_package_json( &mut self, file: &[u8], dirname_fd: FD, package_id: Option, + before_bust: Option>, ) -> crate::CrateResult>> { use crate::package_json::{IncludeDependencies, IncludeScripts}; + let Some((path, contents)) = PackageJSON::read(self, file, dirname_fd) else { + return Ok(None); + }; + if let Some(kept) = before_bust { + // SAFETY: interned for the life of the process: see `intern_package_json`. + let before = unsafe { kept.as_ref() }; + if before.source_contents == contents + && (!ALLOW_DEPENDENCIES + || (before.parsed_dependencies + && package_id.is_none_or(|id| id == before.package_manager_package_id))) + { + return Ok(Some(kept)); + } + } // NOTE: `IncludeDependencies` is a // const generic on `PackageJSON::parse`, `IncludeScripts` is runtime (it only // gates one branch). @@ -4161,18 +4211,18 @@ impl<'a> Resolver<'a> { IncludeScripts::IgnoreScripts }; let pkg = if ALLOW_DEPENDENCIES { - PackageJSON::parse::<{ IncludeDependencies::Local }>( + PackageJSON::parse_contents::<{ IncludeDependencies::Local }>( self, - file, - dirname_fd, + path, + contents, package_id, include_scripts, ) } else { - PackageJSON::parse::<{ IncludeDependencies::None }>( + PackageJSON::parse_contents::<{ IncludeDependencies::None }>( self, - file, - dirname_fd, + path, + contents, package_id, include_scripts, ) @@ -4487,6 +4537,37 @@ impl<'a> Resolver<'a> { let queue_top_safe_path: &[u8] = qt_safe_path.slice(); queue_slice_len -= 1; + // The slot that this directory had when it was busted: see `fill_dir_info`. + let mut before_bust: Option = None; + if DirInfo::any_parked() + && let Some(parked) = DirInfo::take_parked(queue_top.result.hash) + { + // SAFETY: a parked index was assigned by `put`; resolver mutex held. + let slot = unsafe { DirInfo::slot_ptr_at(parked.slot) } + .map(|slot| unsafe { DirInfoRef::from_raw(slot) }); + if parked.busted { + before_bust = Some(parked.slot); + } else if let Some(slot) = slot + && slot.parent == top_parent.index + { + // A bust of a directory above this one took it out of the index. What is + // above it now has the slots it had then, so this one is as it was filled. + self.dir_cache_mut() + .publish(queue_top.result.hash, parked.slot); + top_parent = allocators::Result { + hash: queue_top.result.hash, + index: parked.slot, + status: allocators::ItemStatus::Exists, + }; + if queue_slice_len == 0 { + return Ok(Some(slot)); + } + continue; + } else { + DirInfo::discard_parked_below(queue_top_unsafe_path); + } + } + let open_dir: FD = if queue_top.fd.is_valid() { queue_top.fd } else { @@ -4547,6 +4628,9 @@ impl<'a> Resolver<'a> { if err == crate::Error::Sys(bun_errno::SystemErrno::ENOTDIR) || err == crate::Error::Sys(bun_errno::SystemErrno::EISDIR) { + if DirInfo::any_parked() { + DirInfo::discard_parked_below(queue_top_unsafe_path); + } return Ok(None); } // A permission-denied ancestor (sandboxed drive roots, x-only @@ -4577,6 +4661,9 @@ impl<'a> Resolver<'a> { // ... self.dir_cache_mut().mark_not_found(queue_top.result); rfs!().entries.mark_not_found(cached_dir_entry_result); + if DirInfo::any_parked() { + DirInfo::discard_parked_below(queue_top_unsafe_path); + } if err != crate::Error::Sys(bun_errno::SystemErrno::ENOENT) { if enable_logging { let pretty = queue_top_unsafe_path; @@ -4759,29 +4846,18 @@ impl<'a> Resolver<'a> { )?; } - // We must initialize it as empty so that the result index is correct. - // This is important so that browser_scope has a valid index. - // SAFETY: `dir_cache()` is the live singleton; resolver mutex held. - // Both slot pointers are rooted at the singleton, so the map - // reborrows inside `dir_info_uncached` cannot pop them. - let dir_info_ptr: *mut DirInfo::DirInfo = unsafe { - DirInfo::put_slot( - self.dir_cache(), - &mut queue_top.result, - DirInfo::DirInfo::default(), - ) - }?; // SAFETY: `top_parent.index` is a sentinel or a slot assigned by - // `put`; resolver mutex held. + // `put`; resolver mutex held. The pointer is rooted at the singleton, + // so the map reborrows inside `fill_dir_info` cannot pop it. let parent_dir_ptr = unsafe { DirInfo::slot_ptr_at(top_parent.index) } .map(|p| unsafe { DirInfoRef::from_raw(p) }); - self.dir_info_uncached( - dir_info_ptr, + let dir_info_ptr: *mut DirInfo::DirInfo = self.fill_dir_info( + &mut queue_top.result, + before_bust, dir_path, // SAFETY: ARENA — `dir_entries_option` is a slot in `rfs.entries` (BSSMap) and outlives the resolver. dir_entries_option, - queue_top.result, cached_dir_entry_result.index, parent_dir_ptr, top_parent.index, @@ -6153,6 +6229,91 @@ impl<'a> Resolver<'a> { None } + /// Fills a slot for a directory that is not in the index, and puts it in the index. + /// + /// `before_bust` is the slot that the directory had when `bust_dir_cache` took it out of the + /// index. The directories below it hold the index of that slot and what they took from it. + /// So when the directory is as that slot has it, that slot goes back with the new listing, + /// and they are still right. When it is not, it gets a new slot, and those that the bust + /// took out with it stay out: the next lookup of one reads it again. + fn fill_dir_info( + &mut self, + result: &mut allocators::Result, + before_bust: Option, + path: &'static [u8], + entries: *mut Fs::file_system::real_fs::EntriesOption, + dir_entry_index: allocators::IndexType, + parent: Option, + parent_index: allocators::IndexType, + fd: FD, + package_id: Option, + ) -> crate::CrateResult<*mut DirInfo::DirInfo> { + let Some(before_index) = before_bust else { + // We must initialize it as empty so that the result index is correct. + // This is important so that browser_scope has a valid index. + // SAFETY: `dir_cache()` is the live singleton; resolver mutex held. + let slot = unsafe { + DirInfo::put_slot(self.dir_cache(), result, DirInfo::DirInfo::default()) + }?; + self.dir_info_uncached( + slot, + path, + entries, + *result, + dir_entry_index, + parent, + parent_index, + fd, + package_id, + None, + )?; + return Ok(slot); + }; + + // SAFETY: a parked index was assigned by `put`; resolver mutex held. + let before_ptr = unsafe { DirInfo::slot_ptr_at(before_index) }.expect("slot of a bust"); + // SAFETY: a slot of the singleton. + let before = unsafe { DirInfoRef::from_raw(before_ptr) }; + let mut reread = DirInfo::DirInfo::default(); + self.dir_info_uncached( + &raw mut reread, + path, + entries, + // A directory that is its own browser scope has the index of its slot. + allocators::Result { + index: before_index, + ..*result + }, + dir_entry_index, + parent, + parent_index, + fd, + package_id, + Some(before), + )?; + let same = reread.is_same_after_reread(&before); + if self.extended_file_changed(result.hash) || !same { + DirInfo::discard_parked_below(path); + } + if same { + before.entries.set(reread.entries.get()); + self.dir_cache_mut().publish(result.hash, before_index); + result.index = before_index; + return Ok(before_ptr); + } + // SAFETY: `dir_cache()` is the live singleton; resolver mutex held. + let slot = unsafe { DirInfo::put_slot(self.dir_cache(), result, reread) }?; + // SAFETY: `put_slot` returned a pointer to the slot it initialized, and nothing else + // has it yet. + let info = unsafe { &mut *slot }; + if info.enclosing_browser_scope == before_index { + info.enclosing_browser_scope = result.index; + } + Ok(slot) + } + + /// `before_bust`: the slot that the directory had when it was busted. Its package.json and + /// tsconfig.json parses are kept when the files are as they were. fn dir_info_uncached( &mut self, info: *mut DirInfo::DirInfo, @@ -6164,6 +6325,7 @@ impl<'a> Resolver<'a> { parent_index: allocators::IndexType, fd: FD, package_id: Option, + before_bust: Option, ) -> crate::CrateResult<()> { let result = _result; @@ -6200,7 +6362,7 @@ impl<'a> Resolver<'a> { abs_path: path, // .abs_real_path = path, parent: parent_index, - entries: dir_entry_index, + entries: DirInfo::ListingIndex::new(dir_entry_index), ..Default::default() }; @@ -6445,6 +6607,7 @@ impl<'a> Resolver<'a> { FD::INVALID }, package_id, + before_bust.and_then(|it| it.package_json), ) .ok() .flatten() @@ -6457,6 +6620,7 @@ impl<'a> Resolver<'a> { FD::INVALID }, None, + before_bust.and_then(|it| it.package_json), ) .ok() .flatten() @@ -6528,23 +6692,40 @@ impl<'a> Resolver<'a> { } if let Some(tsconfigpath) = tsconfig_path { - let fd = if FeatureFlags::STORE_FILE_DESCRIPTORS { - fd + let kept = before_bust + .and_then(|it| it.tsconfig_json) + .filter(|it| self.tsconfig_sources_unchanged(it.as_ptr(), tsconfigpath)); + if kept.is_some() { + info.tsconfig_json = kept; } else { - FD::ZERO - }; - // It is for the resolvers that read it. What is wrong with it is not an error of - // one that reads another in its place. - let (mut ignored, log) = (bun_ast::Log::init(), self.log); - if self.opts.tsconfig_override.is_some() { - self.log = NonNull::from(&mut ignored); + let fd = if FeatureFlags::STORE_FILE_DESCRIPTORS { + fd + } else { + FD::ZERO + }; + // It is for the resolvers that read it. What is wrong with it is not an error of + // one that reads another in its place. + let (mut ignored, log) = (bun_ast::Log::init(), self.log); + if self.opts.tsconfig_override.is_some() { + self.log = NonNull::from(&mut ignored); + } + let loaded = self.load_tsconfig(tsconfigpath, fd); + self.log = log; + let loaded = loaded?; + if let Some(it) = &loaded { + for extended in it.sources.iter().skip(1) { + DirInfo::note_extended( + DirInfo::HashMap::key_hash(Dirname::dirname(extended.path)), + extended.path, + extended.hash, + ); + } + } + // A leaked Box, interned into DirInfo: it outlives the resolver. + info.tsconfig_json = loaded.map(|it| { + NonNull::new(bun_core::heap::into_raw(it)).expect("heap::alloc is non-null") + }); } - let loaded = self.load_tsconfig(tsconfigpath, fd); - self.log = log; - // A leaked Box, interned into DirInfo: it outlives the resolver. - info.tsconfig_json = loaded?.map(|it| { - NonNull::new(bun_core::heap::into_raw(it)).expect("heap::alloc is non-null") - }); info.enclosing_tsconfig_json = info.tsconfig_json(); } } @@ -6558,33 +6739,37 @@ impl<'a> Resolver<'a> { tsconfigpath: &[u8], fd: FD, ) -> crate::CrateResult>> { - let parsed_tsconfig: Option<*mut TSConfigJSON> = match self.parse_tsconfig(tsconfigpath, fd) - { - Ok(v) => v.map(bun_core::heap::into_raw), - Err(err) => { - let pretty = tsconfigpath; - if err == crate::Error::Sys(bun_errno::SystemErrno::ENOENT) { - let _ = self.log_mut().add_error_fmt( - None, - bun_ast::Loc::EMPTY, - format_args!("Cannot find tsconfig file {}", bun_core::fmt::quote(pretty)), - ); - } else if err != crate::Error::ParseErrorAlreadyLogged - && err != crate::Error::Sys(bun_errno::SystemErrno::EISDIR) - { - let _ = self.log_mut().add_error_fmt( - None, - bun_ast::Loc::EMPTY, - format_args!( - "Cannot read file {}: {}", - bun_core::fmt::quote(pretty), - bstr::BStr::new(err.name()) - ), - ); + let mut sources = Vec::new(); + let parsed_tsconfig: Option<*mut TSConfigJSON> = + match self.parse_tsconfig(tsconfigpath, fd, &mut sources) { + Ok(v) => v.map(bun_core::heap::into_raw), + Err(err) => { + let pretty = tsconfigpath; + if err == crate::Error::Sys(bun_errno::SystemErrno::ENOENT) { + let _ = self.log_mut().add_error_fmt( + None, + bun_ast::Loc::EMPTY, + format_args!( + "Cannot find tsconfig file {}", + bun_core::fmt::quote(pretty) + ), + ); + } else if err != crate::Error::ParseErrorAlreadyLogged + && err != crate::Error::Sys(bun_errno::SystemErrno::EISDIR) + { + let _ = self.log_mut().add_error_fmt( + None, + bun_ast::Loc::EMPTY, + format_args!( + "Cannot read file {}: {}", + bun_core::fmt::quote(pretty), + bstr::BStr::new(err.name()) + ), + ); + } + None } - None - } - }; + }; if let Some(tsconfig_json) = parsed_tsconfig { let mut parent_configs: BoundedArray<*mut TSConfigJSON, 64> = BoundedArray::default(); parent_configs.append(tsconfig_json)?; @@ -6605,7 +6790,7 @@ impl<'a> Resolver<'a> { bun_paths::Platform::AUTO, ) { Some(abs_path) => self - .parse_tsconfig(abs_path, FD::INVALID) + .parse_tsconfig(abs_path, FD::INVALID, &mut sources) .map_err(|err| (err, abs_path)), None => Err(( crate::Error::Sys(bun_errno::SystemErrno::ENAMETOOLONG), @@ -6690,11 +6875,50 @@ impl<'a> Resolver<'a> { } // SAFETY: from `parse_tsconfig` (heap::into_raw) above; the walk is over, so this // is the only pointer to it. - return Ok(Some(unsafe { bun_core::heap::take(merged_config) })); + let mut merged_config = unsafe { bun_core::heap::take(merged_config) }; + merged_config.sources = sources; + return Ok(Some(merged_config)); } Ok(None) } + /// Whether each file that `config` was read from is as it was then, and the first one is + /// `path`: a re-read of the directory then keeps `config`. + fn tsconfig_sources_unchanged(&mut self, config: *const TSConfigJSON, path: &[u8]) -> bool { + // SAFETY: interned into a `DirInfo` slot for the life of the process. + let sources = unsafe { &(*config).sources }; + sources.first().is_some_and(|first| *first.path == *path) + && sources + .iter() + .all(|file| self.hash_config_file(file.path) == file.hash) + } + + /// `SourceFile::hash` of what the file has now. + fn hash_config_file(&mut self, path: &[u8]) -> Option { + let contents = self.read_config_file(path, FD::INVALID).ok()?; + Some(bun_wyhash::hash(&contents)) + } + + /// Whether a file in the directory with this key hash, which the tsconfig.json of another + /// directory extends, is not what that one read. The directories below this one that a bust + /// parked are not published again then. + fn extended_file_changed(&mut self, dir_hash: u64) -> bool { + if !DirInfo::any_extended() { + return false; + } + let Some(mut files) = DirInfo::take_extended(dir_hash) else { + return false; + }; + let mut changed = false; + for file in &mut files { + let hash = self.hash_config_file(file.path); + changed |= file.seen != Some(hash); + file.seen = Some(hash); + } + DirInfo::put_extended(dir_hash, files); + changed + } + /// `opts.tsconfig_override`, which this resolver and its workers read in place of the /// configuration file of every directory. It is not in `DirInfo`, which every resolver of the /// process shares: the runtime, and every `Bun.build`. `None`: there is none. diff --git a/src/resolver/tsconfig_json.rs b/src/resolver/tsconfig_json.rs index 753c29d75067..bb76047ad98c 100644 --- a/src/resolver/tsconfig_json.rs +++ b/src/resolver/tsconfig_json.rs @@ -163,6 +163,19 @@ pub struct TSConfigJSON { pub experimental_decorators: bool, /// `None` = unset (keeps native [[Define]] class-field semantics). pub use_define_for_class_fields: Option, + + /// The files that `Resolver::load_tsconfig` read for this configuration: the file of the + /// directory, then each file that the one before it extends. A re-read of the directory + /// keeps the configuration while each file hashes to the same value. + pub(crate) sources: Vec, +} + +/// One file of `TSConfigJSON::sources`. +pub(crate) struct SourceFile { + /// In `DirnameStore`. + pub(crate) path: &'static [u8], + /// `bun_wyhash::hash` of the bytes. `None`: the file could not be read. + pub(crate) hash: Option, } impl Default for TSConfigJSON { @@ -179,6 +192,7 @@ impl Default for TSConfigJSON { emit_decorator_metadata: false, experimental_decorators: false, use_define_for_class_fields: None, + sources: Vec::new(), } } } diff --git a/test/bake/dev/bundle.test.ts b/test/bake/dev/bundle.test.ts index 6e7e8359aa46..e5b410b0566c 100644 --- a/test/bake/dev/bundle.test.ts +++ b/test/bake/dev/bundle.test.ts @@ -1926,3 +1926,54 @@ devTest("recovers from an unresolved css url() and html script src longer than t expect((await dev.fetch("/")).status).toBe(200); }, }); + +// A deleted file drops its directory from the resolver's cache. The directories below it were cached with what they +// took from it. +devTest("a new package.json reaches a file in a subdirectory after a file next to it is deleted", { + framework: minimalFramework, + files: { + "pkg/package.json": JSON.stringify({ name: "pkg", imports: { "#lib": "./a.js" } }), + "pkg/a.js": `export default "a";`, + "pkg/b.js": `export default "b";`, + "pkg/deleted.js": `export {};`, + "pkg/root.js": ` + import lib from "#lib"; + import "./deleted.js"; + export const root = lib; + `, + "pkg/src/sub.js": ` + import lib from "#lib"; + export const sub = lib; + `, + "routes/index.ts": ` + import { root } from "../pkg/root.js"; + import { sub } from "../pkg/src/sub.js"; + export default function (req, meta) { + return new Response("root imports " + root + ", src/sub imports " + sub); + } + `, + }, + async test(dev) { + await dev.fetch("/").equals("root imports a, src/sub imports a"); + + fs.writeFileSync(dev.join("pkg/package.json"), JSON.stringify({ name: "pkg", imports: { "#lib": "./b.js" } })); + { + await using batch = await dev.batchChanges(); + await dev.write( + "pkg/root.js", + ` + import lib from "#lib"; + export const root = lib; + `, + ); + await dev.delete("pkg/deleted.js"); + } + // The first build after the delete can resolve "#lib" before or after it sees that the file is gone. + { + await using batch = await dev.batchChanges(); + await dev.patch("pkg/root.js", { find: "export", replace: "/* saved */ export" }); + await dev.patch("pkg/src/sub.js", { find: "export", replace: "/* saved */ export" }); + } + await dev.fetch("/").equals("root imports b, src/sub imports b"); + }, +}); diff --git a/test/bundler/resolver/cache-invalidation.test.ts b/test/bundler/resolver/cache-invalidation.test.ts index 2ea15366e14b..d042cc0cf878 100644 --- a/test/bundler/resolver/cache-invalidation.test.ts +++ b/test/bundler/resolver/cache-invalidation.test.ts @@ -392,4 +392,47 @@ describe("resolver cache invalidation", () => { const text3 = await result3.outputs[0].text(); expect(text3).toContain("recreated"); }); + + // A build whose entry point does not exist drops the directory of the entry point from the cache. The directories + // below it were cached with what they took from it. + test.each([ + [ + "imports", + { name: "app", imports: { "#lib": "./a.js" } }, + { name: "app", imports: { "#lib": "./b.js" } }, + /from [ab]/, + "from a", + "from b", + ], + [ + "type", + { name: "app" }, + { name: "app", type: "module" }, + /__toESM\(require_dep\(\)(, 1)?\)/, + "__toESM(require_dep())", + "__toESM(require_dep(), 1)", + ], + ])('a new package.json "%s" reaches the files of a subdirectory', async (_, before, after, printed, old, updated) => { + const source = `import dep from "dep"; import lib from "#lib"; console.log(dep, lib);`; + using dir = tempDir("cache-test-config-subdir", { + "package.json": JSON.stringify({ imports: { "#lib": "./a.js" }, ...before }), + "a.js": `export default "from a";`, + "b.js": `export default "from b";`, + "node_modules/dep/package.json": JSON.stringify({ name: "dep", main: "index.js" }), + "node_modules/dep/index.js": `exports.__esModule = true; exports.default = function dep() {};`, + "root.js": source, + "src/sub.js": source, + }); + const root = String(dir); + const built = async (file: string) => { + const result = await Bun.build({ entrypoints: [path.join(root, file)] }); + return printed.exec(await result.outputs[0].text())?.[0]; + }; + expect({ root: await built("root.js"), sub: await built("src/sub.js") }).toEqual({ root: old, sub: old }); + + await fs.writeFile(path.join(root, "package.json"), JSON.stringify({ imports: { "#lib": "./a.js" }, ...after })); + const failed = await Bun.build({ entrypoints: [path.join(root, "not-there.js")], throw: false }); + expect(failed.success).toBe(false); + expect({ root: await built("root.js"), sub: await built("src/sub.js") }).toEqual({ root: updated, sub: updated }); + }); }); diff --git a/test/bundler/resolver/cache-runtime.test.ts b/test/bundler/resolver/cache-runtime.test.ts index 1e03e191d077..8e13e9c94255 100644 --- a/test/bundler/resolver/cache-runtime.test.ts +++ b/test/bundler/resolver/cache-runtime.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "bun:test"; -import { promises as fs } from "fs"; +import { promises as fs, mkdirSync, rmSync, writeFileSync } from "fs"; import { tempDir } from "harness"; import * as path from "path"; @@ -118,3 +118,120 @@ describe.concurrent("runtime cache invalidation", () => { expect(result3.value).toBe("recreated"); }); }); + +// A cached directory holds what it took from the directories above it: the package.json and the tsconfig.json that +// apply to it, and a link to its parent. A require that fails drops one directory from the cache. +describe("a directory dropped from the cache", () => { + const drop = (dir: string) => expect(() => require(path.join(dir, "not-there.js"))).toThrow(); + const probe = (specifier: string) => ` + const { basename } = require("path"); + module.exports = () => { + try { + return basename(require.resolve("${specifier}")); + } catch (e) { + return e.code; + } + }; + `; + const probes = (specifier: string) => ({ + "a.js": "", + "b.js": "", + "probe.js": probe(specifier), + "src/probe.js": probe(specifier), + "src/deep/probe.js": probe(specifier), + }); + /** What the specifier resolves to from the project root, from src and from src/deep, asked in that order. */ + function resolved(root: string, order: ("root" | "src" | "deep")[] = ["root", "src", "deep"]) { + const dirs = { root: "", src: "src", deep: "src/deep" }; + const from = Object.fromEntries(order.map(at => [at, require(path.join(root, dirs[at], "probe.js"))()])); + return { root: from.root, src: from.src, deep: from.deep }; + } + const everywhere = (file: string) => ({ root: file, src: file, deep: file }); + const imports = (target: string) => JSON.stringify({ name: "app", imports: { "#lib": target } }); + const paths = (target: string) => JSON.stringify({ compilerOptions: { paths: { "@lib": [target] } } }); + + test.each([ + ["package.json", "#lib", imports], + ["tsconfig.json", "@lib", paths], + ])("a new %s reaches the directories below it", (config, specifier, contents) => { + using dir = tempDir("runtime-cache-config", { ...probes(specifier), [config]: contents("./a.js") }); + const root = String(dir); + expect(resolved(root)).toEqual(everywhere("a.js")); + + writeFileSync(path.join(root, config), contents("./b.js")); + drop(root); + expect(resolved(root, ["deep", "src", "root"])).toEqual(everywhere("b.js")); + + writeFileSync(path.join(root, config), contents("./a.js")); + drop(root); + expect(resolved(root, ["root", "src", "deep"])).toEqual(everywhere("a.js")); + }); + + test("a package.json that is created or deleted reaches the directories below it", () => { + using dir = tempDir("runtime-cache-config-created", probes("#lib")); + const root = String(dir); + expect(resolved(root)).toEqual(everywhere("MODULE_NOT_FOUND")); + + writeFileSync(path.join(root, "package.json"), imports("./a.js")); + drop(root); + expect(resolved(root, ["deep", "src", "root"])).toEqual(everywhere("a.js")); + + rmSync(path.join(root, "package.json")); + drop(root); + expect(resolved(root, ["deep", "src", "root"])).toEqual(everywhere("MODULE_NOT_FOUND")); + }); + + test("a new node_modules folder reaches the directories below it", () => { + using dir = tempDir("runtime-cache-node-modules", probes("installed-later")); + const root = String(dir); + expect(resolved(root)).toEqual(everywhere("MODULE_NOT_FOUND")); + + mkdirSync(path.join(root, "node_modules", "installed-later"), { recursive: true }); + writeFileSync(path.join(root, "node_modules", "installed-later", "index.js"), ""); + drop(root); + expect(resolved(root, ["deep", "src", "root"])).toEqual(everywhere("index.js")); + }); + + test("a new file that a tsconfig.json below extends reaches that directory", () => { + using dir = tempDir("runtime-cache-extends", { + ...probes("@lib"), + "tsconfig.base.json": paths("./a.js"), + "src/tsconfig.json": JSON.stringify({ extends: "../tsconfig.base.json" }), + }); + const root = String(dir); + expect(resolved(root, ["src", "deep"])).toEqual({ root: undefined, src: "a.js", deep: "a.js" }); + + writeFileSync(path.join(root, "tsconfig.base.json"), paths("./b.js")); + drop(root); + expect(resolved(root, ["deep", "src"])).toEqual({ root: undefined, src: "b.js", deep: "b.js" }); + }); + + test.each([ + ["the directory above it first", ["", "src"]], + ["the directory above it last", ["src", ""]], + ])("a directory that is dropped with %s is read again", (_, order) => { + using dir = tempDir("runtime-cache-two-drops", { + ...probes("#lib"), + "package.json": JSON.stringify({ name: "app" }), + "src/package.json": imports("./a.js"), + "src/a.js": "", + "src/b.js": "", + }); + const root = String(dir); + expect(resolved(root, ["src", "deep"])).toEqual({ root: undefined, src: "a.js", deep: "a.js" }); + + writeFileSync(path.join(root, "src", "package.json"), imports("./b.js")); + for (const sub of order) drop(path.join(root, sub)); + expect(resolved(root, ["deep", "src"])).toEqual({ root: undefined, src: "b.js", deep: "b.js" }); + }); + + test("the directories below it stay as they are when nothing changed", () => { + using dir = tempDir("runtime-cache-no-change", { ...probes("#lib"), "package.json": imports("./a.js") }); + const root = String(dir); + expect(resolved(root)).toEqual(everywhere("a.js")); + for (const order of [[""], ["src", ""], ["", "src"], ["src/deep"], ["", "src", "src/deep"]]) { + for (const sub of order) drop(path.join(root, sub)); + expect(resolved(root, ["deep", "src", "root"])).toEqual(everywhere("a.js")); + } + }); +}); diff --git a/test/cli/hot/hot.test.ts b/test/cli/hot/hot.test.ts index f8c6887f5d9c..1463b1ccc8a4 100644 --- a/test/cli/hot/hot.test.ts +++ b/test/cli/hot/hot.test.ts @@ -1332,3 +1332,40 @@ it("--hot does not emit beforeExit to the listeners of a failed run", async () = expect(await stdout("still running")).toBe(true); expect(await runner.exited).toBe(0); }); + +// The resolver caches, for each directory, pointers to the package.json and the tsconfig.json of the directories above it. +it.each([ + ["tsconfig.json", "@lib", (target: string) => JSON.stringify({ compilerOptions: { paths: { "@lib": [target] } } })], + ["package.json", "#lib", (target: string) => JSON.stringify({ name: "app", imports: { "#lib": target } })], +])("--hot applies a new %s to the files of a subdirectory", async (config, alias, contents) => { + const main = (run: string) => ` + import { root } from "./root.ts"; + import { sub } from "./src/sub.ts"; + console.log("${run} run: root imports " + root + ", src/sub imports " + sub); + `; + using dir = tempDir("hot-config-edit", { + [config]: contents("./a.ts"), + "a.ts": `export default "a";`, + "b.ts": `export default "b";`, + "root.ts": `import lib from "${alias}";\nexport const root = lib;`, + "src/sub.ts": `import lib from "${alias}";\nexport const sub = lib;`, + "main.ts": main("first"), + }); + const cwd = String(dir); + await using runner = spawn({ + cmd: [bunExe(), "--hot", "--no-clear-screen", "main.ts"], + env: bunEnv, + cwd, + stdout: "pipe", + stderr: "inherit", + stdin: "ignore", + }); + const stdout = follow(runner.stdout); + expect(await stdout("first run: root imports a, src/sub imports a\n")).toBe(true); + + // By rename: kqueue reports a directory only when one of its entries is replaced. + writeHotFileAtomicSync(join(cwd, config), contents("./b.ts")); + writeFileSync(join(cwd, "main.ts"), main("second") + "process.exit(0);\n"); + expect(await stdout("second run: root imports b, src/sub imports b\n")).toBe(true); + expect(await runner.exited).toBe(0); +}); diff --git a/test/js/bun/resolve/tsconfig-extends-leak.test.ts b/test/js/bun/resolve/tsconfig-extends-leak.test.ts index 65b8ef8e72ac..9c8fc210e68f 100644 --- a/test/js/bun/resolve/tsconfig-extends-leak.test.ts +++ b/test/js/bun/resolve/tsconfig-extends-leak.test.ts @@ -122,3 +122,51 @@ test("tsconfig 'extends' merge still works after freeing intermediates", async ( expect(stdout.trim()).toBe("leaf"); expect(exitCode).toBe(0); }); + +// A require that fails drops the cached directory of the path it asked for, and the next lookup reads the directory +// again. Each parse of its tsconfig.json and package.json was kept for the life of the process, and the directories +// below it were not read again, which left them with the old parses. +test.skipIf(!isDebug)("a directory that is read again keeps its parses and the directories below it", async () => { + const drops = 20; + using dir = tempDir("tsconfig-reread", { + "package.json": JSON.stringify({ name: "app", imports: { "#lib": "./lib.ts" } }), + "tsconfig.json": JSON.stringify({ extends: "./tsconfig.base.json" }), + "tsconfig.base.json": JSON.stringify({ compilerOptions: { paths: { "@lib": ["./lib.ts"] } } }), + "lib.ts": "export default 1;\n", + "src/deep/probe.cjs": `module.exports = () => [require.resolve("@lib"), require.resolve("#lib")];`, + "main.cjs": ` + const { join } = require("path"); + const probe = require("./src/deep/probe.cjs"); + const first = probe(); + for (let i = 0; i < ${drops}; i++) { + try { + require(join(__dirname, "not-there.js")); + } catch {} + if (String(probe()) !== String(first)) throw new Error("resolved to another file"); + } + `, + }); + + await using proc = Bun.spawn({ + cmd: [bunExe(), "main.cjs"], + env: { ...bunEnv, BUN_DEBUG_alloc: "1", BUN_DEBUG_Fs: "1" }, + cwd: String(dir), + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + + const output = stdout + stderr; + // The log has a directory with or without a separator at the end. + const opens = (directory: string) => + [...output.matchAll(/open\((.*)\)/g)].filter(([, opened]) => opened.replace(/[\\/]$/, "") === directory).length; + expect({ + // tsconfig.json and the file it extends, once. Before, once for each drop. + tsconfigParses: output.split("new(TSConfigJSON)").length - 1 < drops, + // The project root is opened again after each drop. A directory below it is not. + rootOpens: opens(String(dir)) > drops, + srcOpens: opens(path.join(String(dir), "src")), + deepOpens: opens(path.join(String(dir), "src", "deep")), + }).toEqual({ tsconfigParses: true, rootOpens: true, srcOpens: 1, deepOpens: 1 }); + expect(exitCode).toBe(0); +}); diff --git a/test/js/bun/util/filesystem_router.test.ts b/test/js/bun/util/filesystem_router.test.ts index 8886fb09657b..ca4c6facb38e 100644 --- a/test/js/bun/util/filesystem_router.test.ts +++ b/test/js/bun/util/filesystem_router.test.ts @@ -489,6 +489,27 @@ it("reload() works with new dirs/files", () => { expect(router.match("/test/test2")!.name).toBe("/test/test2"); }); +// reload() drops the directory of the routes, and each directory below it that can have a route, from the resolver's +// cache. A directory that cannot have one, like one whose name starts with a dot, stays cached with what it took from +// the directories above it. +it("reload() applies a new package.json to a directory that has no route", () => { + const imports = (target: string) => JSON.stringify({ name: "app", imports: { "#lib": target } }); + using dir = tempDir("fsr-reload-config", { + "package.json": imports("./a.js"), + "a.js": "", + "b.js": "", + "index.tsx": "export default 1;", + ".private/probe.js": `module.exports = () => require("path").basename(require.resolve("#lib"));`, + }); + const router = new FileSystemRouter({ dir: String(dir), fileExtensions: [".tsx"], style: "nextjs" }); + const probe = require(path.join(String(dir), ".private", "probe.js")); + expect(probe()).toBe("a.js"); + + fs.writeFileSync(path.join(String(dir), "package.json"), imports("./b.js")); + router.reload(); + expect(probe()).toBe("b.js"); +}); + it("the constructor and reload() don't create allocator heaps", () => { const { dir } = make(["index.tsx", "posts/[id].tsx"]); const heapsCreatedBy = (fn: () => void) => {