dev server: keep a barrel's requests across builds, or never defer it again

src/bundler/barrel_imports.rs

@@ -11,12 +11,14 @@ use crate::bundled_ast as JSAst;
 use crate::mal_prelude::*;
 use bun_alloc::AllocError;
 use bun_ast::{ImportKind, import_record};
-use bun_collections::{ArrayHashMap, AutoBitSet, StringArrayHashMap};
+use bun_collections::{ArrayHashMap, ArrayIdentityContextU64, AutoBitSet, StringArrayHashMap};
 
 use crate::Graph::InputFileFlags;
 use crate::Index;
+use crate::bake_types;
 use crate::bundle_v2::BundleV2;
 use crate::bundle_v2::bv2_impl::{PatchImportRecordsCtx, ResolveImportRecordCtx};
+use crate::options_impl::TargetExt as _;
 use crate::parse_task;
 
 bun_core::declare_scope!(barrel, hidden);
@@ -32,6 +34,102 @@ impl Default for RequestedExports {
     }
 }
 
+/// Dev server: what every build requested from each barrel that has deferred a record.
+#[derive(Default)]
+pub struct TrackedBarrels {
+    /// By `bake_types::Side`: the server and SSR copies of a file share one graph record.
+    by_side: [StringArrayHashMap<RequestedExports>; 2],
+}
+
+impl TrackedBarrels {
+    #[inline]
+    fn get(&self, side: bake_types::Side, path: &[u8]) -> Option<&RequestedExports> {
+        let barrels = &self.by_side[side as usize];
+        // A lookup hashes the path: skip it for the projects that track no barrel.
+        if barrels.is_empty() {
+            return None;
+        }
+        barrels.get(path)
+    }
+
+    /// True if the copy of this file on `side` has an entry.
+    pub fn is_tracked(&self, side: bake_types::Side, path: &[u8]) -> bool {
+        self.get(side, path).is_some()
+    }
+
+    /// A client component boundary reads the copy of the other side: either side's entry counts.
+    pub fn must_reparse(&self, path: &[u8]) -> bool {
+        self.by_side
+            .iter()
+            .any(|barrels| !barrels.is_empty() && barrels.contains_key(path))
+    }
+
+    pub fn memory_cost(&self) -> usize {
+        let mut cost = 0;
+        for barrels in &self.by_side {
+            cost += barrels.capacity()
+                * (size_of::<Box<[u8]>>() + size_of::<RequestedExports>() + size_of::<u32>());
+            for (path, requested) in barrels.iter() {
+                cost += path.len();
+                if let RequestedExports::Partial(names) = requested {
+                    cost += names.capacity() * (size_of::<Box<[u8]>>() + size_of::<u32>());
+                    cost += names.keys().iter().map(|name| name.len()).sum::<usize>();
+                }
+            }
+        }
+        cost
+    }
+}
+
+/// Dev server: the source indices of a build by the hash of their path.
+#[derive(Default)]
+pub struct SourcesByPath {
+    newest: ArrayHashMap<u64, u32, ArrayIdentityContextU64>,
+    /// For each indexed source, the next older source with the same hash, or `u32::MAX`.
+    older: Vec<u32>,
+}
+
+impl SourcesByPath {
+    fn index_new_sources(&mut self, sources: &[bun_ast::Source]) -> Result<(), AllocError> {
+        for index in self.older.len()..sources.len() {
+            let gop = self
+                .newest
+                .get_or_put(bun_wyhash::hash(sources[index].path.text))?;
+            self.older.push(if gop.found_existing {
+                *gop.value_ptr
+            } else {
+                u32::MAX
+            });
+            *gop.value_ptr = index as u32;
+        }
+        Ok(())
+    }
+
+    /// The other indexed sources that have the path of `source_index`.
+    fn others_with_path<'a>(
+        &'a self,
+        sources: &'a [bun_ast::Source],
+        path: &'a [u8],
+        source_index: u32,
+    ) -> impl Iterator<Item = u32> + 'a {
+        let mut next = self
+            .newest
+            .get(&bun_wyhash::hash(path))
+            .copied()
+            .unwrap_or(u32::MAX);
+        core::iter::from_fn(move || {
+            while next != u32::MAX {
+                let index = next;
+                next = self.older[index as usize];
+                if index != source_index && sources[index as usize].path.text == path {
+                    return Some(index);
+                }
+            }
+            None
+        })
+    }
+}
+
 impl RequestedExports {
     /// `get_or_put`-shaped entry on the dense `Vec<Option<Self>>` storage.
     /// Returns `(found_existing, &mut value)`; inserts `Default` when absent.
@@ -50,6 +148,23 @@ impl RequestedExports {
     fn lookup(map: &[Option<Self>], idx: u32) -> Option<&Self> {
         map.get(idx as usize).and_then(Option::as_ref)
     }
+
+    /// Adds what `other` requests. Returns the number of names that are new.
+    fn add(&mut self, other: &Self) -> Result<usize, AllocError> {
+        let RequestedExports::Partial(kept) = self else {
+            return Ok(0);
+        };
+        let mut added = 0;
+        match other {
+            RequestedExports::All => *self = RequestedExports::All,
+            RequestedExports::Partial(names) => {
+                for name in names.keys() {
+                    added += !kept.get_or_put(name)?.found_existing as usize;
+                }
+            }
+        }
+        Ok(added)
+    }
 }
 
 // `original_alias` is stored as the raw arena `*const [u8]` (the
@@ -190,19 +305,56 @@ fn apply_barrel_optimization_impl(
     // `DevServerHandle` is `Copy`; copied out so the `&self` borrow
     // doesn't conflict with later `&mut this.requested_exports`.
     let dev_handle = this.dev_server_handle().copied();
+    let side = ast.target.bake_graph().side();
     if let Some(dev) = dev_handle {
-        // SAFETY: barrel_needed_exports is owned by DevServer; bundler runs on the bundle
-        // thread which holds the DevServer lock during this callback.
-        let needed = unsafe { &*dev.barrel_needed_exports() };
-        if let Some(persisted) = needed.get(result.source.path.text) {
-            for alias in persisted.keys() {
-                if let Some(resolution) =
-                    resolve_barrel_export(alias, &ast.named_exports, &ast.named_imports)
-                {
-                    needed_records.put(resolution.import_record_index, ())?;
+        // A file that was not there to read has a second index: count the requests under the first.
+        let sources = this.graph.input_files.items_source();
+        this.sources_by_path.index_new_sources(sources)?;
+        for other in
+            this.sources_by_path
+                .others_with_path(sources, result.source.path.text, source_index)
+        {
+            let Some(requested) = RequestedExports::lookup(&this.requested_exports, other) else {
+                continue;
+            };
+            if copy_side(this, other).is_some_and(|other_side| other_side != side) {
+                continue;
+            }
+            match requested {
+                RequestedExports::All => return Ok(()),
+                RequestedExports::Partial(partial) => {
+                    for key in partial.keys() {
+                        if let Some(resolution) =
+                            resolve_barrel_export(key, &ast.named_exports, &ast.named_imports)
+                        {
+                            needed_records.put(resolution.import_record_index, ())?;
+                        }
+                    }
                 }
             }
         }
+
+        // SAFETY: the DevServer owns `tracked_barrels` and outlives the bundle. Only the JS thread
+        // writes it, and no reference from this pointer is held across a call into the DevServer.
+        let is_tracked = match unsafe { &*dev.tracked_barrels() }.get(side, result.source.path.text)
+        {
+            Some(RequestedExports::All) => return Ok(()),
+            Some(RequestedExports::Partial(persisted)) => {
+                for alias in persisted.keys() {
+                    if let Some(resolution) =
+                        resolve_barrel_export(alias, &ast.named_exports, &ast.named_imports)
+                    {
+                        needed_records.put(resolution.import_record_index, ())?;
+                    }
+                }
+                true
+            }
+            None => false,
+        };
+        // Bundled and not tracked: no copy deferred a record, and no build recorded a request.
+        if !is_tracked && dev.has_file(result.source.path.key_for_incremental_graph(), side) {
+            return Ok(());
+        }
     }
 
     // When HMR is active, ConvertESMExportsForHmr deduplicates import records
@@ -284,16 +436,90 @@ fn apply_barrel_optimization_impl(
 
         // Merge with existing entry (keep already-requested names) or create new
         let _ = RequestedExports::entry(&mut this.requested_exports, source_index);
+    }
 
-        // Register with DevServer so isFileCached returns null for this barrel,
-        // ensuring it gets re-parsed on every incremental build. This is needed
-        // because the set of needed exports can change when importing files change.
-        if let Some(dev) = dev_handle {
-            dev.register_barrel_with_deferrals(result.source.path.text)
-                .map_err(|_| AllocError)?;
-        }
+    if has_deferrals && let Some(dev) = dev_handle {
+        // SAFETY: see the read of `tracked_barrels` above.
+        unsafe { &mut *dev.tracked_barrels() }.by_side[side as usize]
+            .get_or_put(result.source.path.text)?;
+    }
+
+    Ok(())
+}
+
+/// Dev server: the side of the copy with this source index. `None`: the file was not there to read.
+fn copy_side(this: &BundleV2, source_index: u32) -> Option<bake_types::Side> {
+    let index = source_index as usize;
+    if this.graph.ast.items_parts()[index].len() != 0 {
+        return Some(this.graph.ast.items_target()[index].bake_graph().side());
     }
+    // A copy that did not parse has no target: the path map that still has it tells the side.
+    let path = this.graph.input_files.items_source()[index].path.text;
+    this.graph
+        .build_graphs
+        .iter()
+        .find_map(|(target, path_to_source_index)| {
+            (path_to_source_index.get(path) == Some(source_index))
+                .then(|| target.bake_graph().side())
+        })
+}
 
+/// Dev server, once per build: adds the requests of this build to the entries of `TrackedBarrels`.
+pub(crate) fn persist_requests_of_tracked_barrels(
+    this: &BundleV2,
+    dev: &crate::dispatch::DevServerHandle,
+) -> Result<(), AllocError> {
+    // SAFETY: see the read of `tracked_barrels` in `apply_barrel_optimization_impl`.
+    let tracked = unsafe { &mut *dev.tracked_barrels() };
+    if tracked.by_side.iter().all(|barrels| barrels.is_empty()) {
+        return Ok(());
+    }
+    let sources = this.graph.input_files.items_source();
+    let mut lookups: usize = 0;
+    let mut names_added: usize = 0;
+    // By source index: the path maps can point a path at another copy within a build.
+    for (index, requested) in this.requested_exports.iter().enumerate() {
+        let Some(requested) = requested else {
+            continue;
+        };
+        let side = copy_side(this, index as u32);
+        for (barrels_side, barrels) in tracked.by_side.iter_mut().enumerate() {
+            if barrels.is_empty() || side.is_some_and(|side| side as usize != barrels_side) {
+                continue;
+            }
+            lookups += 1;
+            if let Some(kept) = barrels.get_mut(sources[index].path.text) {
+                names_added += kept.add(requested)?;
+            }
+        }
+    }
+    // A request under the generated copy of a client component boundary is for the original file.
+    let boundaries = this.graph.server_component_boundaries.slice();
+    for (&original, &generated) in boundaries
+        .list
+        .items_source_index()
+        .iter()
+        .zip(boundaries.list.items_reference_source_index())
+    {
+        let Some(requested) = RequestedExports::lookup(&this.requested_exports, generated) else {
+            continue;
+        };
+        let Some(side) = copy_side(this, original) else {
+            continue;
+        };
+        lookups += 1;
+        if let Some(kept) =
+            tracked.by_side[side as usize].get_mut(sources[original as usize].path.text)
+        {
+            names_added += kept.add(requested)?;
+        }
+    }
+    bun_core::scoped_log!(
+        barrel,
+        "persisted requests of tracked barrels: {} lookups, {} names added",
+        lookups,
+        names_added,
+    );
     Ok(())
 }
 
@@ -556,10 +782,6 @@ pub(crate) fn schedule_barrel_deferred_imports(
                     p.put(alias, ())?;
                 }
             }
-            // Persist the export request on DevServer so it survives across builds.
-            if let Some(dev) = dev_handle {
-                persist_barrel_export(&dev, resolved_path_text, alias);
-            }
         }
     }
 
@@ -995,10 +1217,54 @@ pub(crate) fn schedule_barrel_deferred_imports(
     Ok(newly_scheduled)
 }
 
-/// Persist an export name for a barrel file on the DevServer. Called during
-/// seeding so that exports requested in previous builds are not lost when the
-/// barrel is re-parsed in an incremental build where the requesting file is
-/// not stale.
-fn persist_barrel_export(dev: &crate::dispatch::DevServerHandle, barrel_path: &[u8], alias: &[u8]) {
-    dev.register_barrel_export(barrel_path, alias)
+#[cfg(test)]
+mod tests {
+    use super::SourcesByPath;
+
+    fn sources(paths: &[&'static [u8]]) -> Vec<bun_ast::Source> {
+        paths
+            .iter()
+            .map(|&text| {
+                let mut source = bun_ast::Source::default();
+                source.path.text = text;
+                source
+            })
+            .collect()
+    }
+
+    fn others(index: &SourcesByPath, sources: &[bun_ast::Source], of: u32) -> Vec<u32> {
+        index
+            .others_with_path(sources, sources[of as usize].path.text, of)
+            .collect()
+    }
+
+    #[test]
+    fn sources_by_path_finds_the_other_indices_of_a_path() {
+        let mut index = SourcesByPath::default();
+        let mut all = sources(&[b"/p/a.js", b"/p/b.js"]);
+        index.index_new_sources(&all).unwrap();
+        assert_eq!(others(&index, &all, 0), [0u32; 0]);
+
+        all.extend(sources(&[b"/p/a.js", b"/p/c.js", b"/p/a.js"]));
+        index.index_new_sources(&all).unwrap();
+        assert_eq!(others(&index, &all, 4), [2, 0]);
+        assert_eq!(others(&index, &all, 2), [4, 0]);
+        assert_eq!(others(&index, &all, 0), [4, 2]);
+        assert_eq!(others(&index, &all, 1), [0u32; 0]);
+        assert_eq!(others(&index, &all, 3), [0u32; 0]);
+    }
+
+    #[test]
+    fn sources_by_path_keeps_two_paths_with_one_hash_apart() {
+        // A 16-byte chunk that starts with wyhash's secret word zeroes the state: the next word is lost.
+        const A: &[u8] = b"\xdb\x28\xb4\xa0\xd1\x7e\x03\xe7AAAAAAAA/node_modules/lib/index.js";
+        const B: &[u8] = b"\xdb\x28\xb4\xa0\xd1\x7e\x03\xe7BBBBBBBB/node_modules/lib/index.js";
+        assert_eq!(bun_wyhash::hash(A), bun_wyhash::hash(B));
+
+        let all = sources(&[A, B, A]);
+        let mut index = SourcesByPath::default();
+        index.index_new_sources(&all).unwrap();
+        assert_eq!(others(&index, &all, 2), [0]);
+        assert_eq!(others(&index, &all, 1), [0u32; 0]);
+    }
 }