perf(cli): lock-free storage, and reduce system calls

.changeset/wet-grapes-cross.md

@@ -0,0 +1,5 @@
+---
+"@biomejs/biome": patch
+---
+
+Slightly increased the performance of the CLI in projects that have more than ~2K files.

crates/biome_cli/Cargo.toml

@@ -56,6 +56,7 @@ bpaf                     = { workspace = true, features = ["bright-color"] }
 camino                   = { workspace = true }
 crossbeam                = { workspace = true }
 dashmap                  = { workspace = true }
+papaya                   = { workspace = true }
 path-absolutize          = { workspace = true }
 quick-junit              = "0.5.2"
 rayon                    = { workspace = true }

crates/biome_cli/src/reporter/mod.rs

@@ -21,7 +21,6 @@ use biome_json_factory::make::{
 use biome_json_syntax::{AnyJsonMemberName, AnyJsonValue, JsonMember, T};
 use camino::Utf8Path;
 use serde::Serialize;
-use std::collections::BTreeSet;
 use std::io;
 use std::time::Duration;
 
@@ -183,7 +182,7 @@ pub(crate) trait ReporterVisitor {
     fn report_handled_paths(
         &mut self,
         _writer: &mut dyn ReporterWriter,
-        _evaluated_paths: BTreeSet<BiomePath>,
+        _evaluated_paths: Vec<BiomePath>,
         _working_directory: Option<&Utf8Path>,
     ) -> io::Result<()> {
         Ok(())

crates/biome_cli/src/reporter/summary.rs

@@ -24,7 +24,7 @@ pub(crate) struct SummaryReporter<'a> {
     pub(crate) summary: TraversalSummary,
     pub(crate) diagnostics_payload: &'a DiagnosticsPayload,
     pub(crate) execution: &'a dyn Execution,
-    pub(crate) evaluated_paths: BTreeSet<BiomePath>,
+    pub(crate) evaluated_paths: Vec<BiomePath>,
     pub(crate) working_directory: Option<Utf8PathBuf>,
     pub(crate) verbose: bool,
 }
@@ -93,7 +93,7 @@ impl ReporterVisitor for SummaryReporterVisitor {
     fn report_handled_paths(
         &mut self,
         writer: &mut dyn ReporterWriter,
-        evaluated_paths: BTreeSet<BiomePath>,
+        evaluated_paths: Vec<BiomePath>,
         working_directory: Option<&Utf8Path>,
     ) -> io::Result<()> {
         let evaluated_paths_diagnostic = EvaluatedPathsDiagnostic {

crates/biome_cli/src/reporter/terminal.rs

@@ -11,15 +11,14 @@ use biome_diagnostics::PrintDiagnostic;
 use biome_diagnostics::advice::ListAdvice;
 use biome_fs::BiomePath;
 use camino::{Utf8Path, Utf8PathBuf};
-use std::collections::BTreeSet;
 use std::io;
 use std::time::Duration;
 
 pub(crate) struct ConsoleReporter<'a> {
     pub(crate) summary: TraversalSummary,
     pub(crate) diagnostics_payload: &'a DiagnosticsPayload,
     pub(crate) execution: &'a dyn Execution,
-    pub(crate) evaluated_paths: BTreeSet<BiomePath>,
+    pub(crate) evaluated_paths: Vec<BiomePath>,
     pub(crate) working_directory: Option<Utf8PathBuf>,
     pub(crate) verbose: bool,
 }
@@ -87,7 +86,7 @@ impl ReporterVisitor for ConsoleReporterVisitor {
     fn report_handled_paths(
         &mut self,
         writer: &mut dyn ReporterWriter,
-        evaluated_paths: BTreeSet<BiomePath>,
+        evaluated_paths: Vec<BiomePath>,
         working_directory: Option<&Utf8Path>,
     ) -> io::Result<()> {
         let evaluated_paths_diagnostic = EvaluatedPathsDiagnostic {

crates/biome_cli/src/runner/crawler.rs

@@ -9,9 +9,9 @@ use biome_service::Workspace;
 use biome_service::projects::ProjectKey;
 use camino::Utf8PathBuf;
 use crossbeam::channel::{Sender, unbounded};
-use std::collections::BTreeSet;
+use papaya::{HashSet, HashSetRef, LocalGuard};
+use std::hash::RandomState;
 use std::marker::PhantomData;
-use std::sync::RwLock;
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::thread;
 use std::time::{Duration, Instant};
@@ -24,7 +24,7 @@ pub trait Crawler<Output> {
 
     fn output(
         collector_result: <Self::Collector as Collector>::Result,
-        evaluated_paths: BTreeSet<BiomePath>,
+        evaluated_paths: Vec<BiomePath>,
         duration: Duration,
     ) -> Output;
 
@@ -68,26 +68,31 @@ pub trait Crawler<Output> {
 
     /// Initiate the filesystem traversal tasks with the provided input paths and
     /// run it to completion, returning the duration of the process and the evaluated paths
-    fn crawl_inputs(
-        fs: &dyn FileSystem,
+    fn crawl_inputs<'a>(
+        fs: &'a dyn FileSystem,
         inputs: Vec<String>,
-        ctx: &CrawlerOptions<Self::Handler, Self::ProcessFile>,
-    ) -> (Duration, BTreeSet<BiomePath>) {
+        ctx: &'a CrawlerOptions<Self::Handler, Self::ProcessFile>,
+    ) -> (Duration, Vec<BiomePath>) {
         let start = Instant::now();
         fs.traversal(Box::new(move |scope: &dyn TraversalScope| {
             for input in inputs {
                 scope.evaluate(ctx, Utf8PathBuf::from(input));
             }
         }));
 
-        let paths = ctx.evaluated_paths();
+        let mut handle_paths: Vec<_> = ctx.evaluated_paths().into_iter().cloned().collect();
+        handle_paths.sort_unstable();
         fs.traversal(Box::new(|scope: &dyn TraversalScope| {
-            for path in paths {
+            for path in &handle_paths {
                 scope.handle(ctx, path.to_path_buf());
             }
         }));
 
-        (start.elapsed(), ctx.evaluated_paths())
+        // Re-collect after handle phase to capture was_written mutations
+        let mut evaluated_paths: Vec<_> = ctx.evaluated_paths().into_iter().cloned().collect();
+        evaluated_paths.sort_unstable();
+
+        (start.elapsed(), evaluated_paths)
     }
 }
 
@@ -125,7 +130,7 @@ pub(crate) struct CrawlerOptions<'ctx, 'app, H, P> {
     /// Channel sending messages to the display thread
     pub(crate) messages: Sender<Message>,
     /// List of paths that should be processed
-    pub(crate) evaluated_paths: RwLock<BTreeSet<BiomePath>>,
+    pub(crate) evaluated_paths: papaya::HashSet<BiomePath>,
 
     execution: &'app dyn Execution,
 
@@ -141,10 +146,9 @@ where
 {
     fn increment_changed(&self, path: &BiomePath) {
         self.changed.fetch_add(1, Ordering::Relaxed);
-        self.evaluated_paths
-            .write()
-            .unwrap()
-            .replace(path.to_written());
+        let guard = self.evaluated_paths.pin();
+        guard.remove(path);
+        guard.insert(path.to_written());
         self.push_message(Message::Stats(MessageStat::Changed));
     }
     fn increment_unchanged(&self) {
@@ -203,7 +207,7 @@ where
             project_key,
             interner,
             messages: sender,
-            evaluated_paths: RwLock::default(),
+            evaluated_paths: HashSet::default(),
             handler: I::default(),
             changed: AtomicUsize::new(0),
             unchanged: AtomicUsize::new(0),
@@ -224,8 +228,8 @@ where
         &self.interner
     }
 
-    fn evaluated_paths(&self) -> BTreeSet<BiomePath> {
-        self.evaluated_paths.read().unwrap().clone()
+    fn evaluated_paths(&self) -> HashSetRef<'_, BiomePath, RandomState, LocalGuard<'_>> {
+        self.evaluated_paths.pin()
     }
 
     fn push_diagnostic(&self, error: Error) {
@@ -242,9 +246,6 @@ where
     }
 
     fn store_path(&self, path: BiomePath) {
-        self.evaluated_paths
-            .write()
-            .unwrap()
-            .insert(BiomePath::new(path.as_path()));
+        self.evaluated_paths.pin().insert(path);
     }
 }

crates/biome_cli/src/runner/handler.rs

@@ -35,9 +35,8 @@ pub trait Handler: Default + Send + Sync + Debug + std::panic::RefUnwindSafe {
         let fs = ctx.fs();
         let workspace = ctx.workspace();
         let execution = ctx.execution();
-        let path = biome_path.as_path();
         let project_key = ctx.project_key();
-        if fs.path_is_dir(path) || fs.path_is_symlink(path) {
+        if biome_path.path_kind().is_dir() || biome_path.path_kind().is_symlink() {
             // handle:
             // - directories
             // - symlinks
@@ -48,6 +47,9 @@ pub trait Handler: Default + Send + Sync + Debug + std::panic::RefUnwindSafe {
                 .is_path_ignored(PathIsIgnoredParams {
                     project_key,
                     path: biome_path.clone(),
+                    // Treat symlinks the same as directories for include/ignore matching,
+                    // since symlinks in this branch point to directories.
+                    is_dir: true,
                     features: execution.wanted_features(),
                     ignore_kind: IgnoreKind::Ancestors,
                 })
@@ -60,7 +62,7 @@ pub trait Handler: Default + Send + Sync + Debug + std::panic::RefUnwindSafe {
         }
 
         // bail on fifo and socket files
-        if !fs.path_is_file(path) {
+        if !biome_path.path_kind().is_file() {
             return false;
         }
 

crates/biome_cli/src/runner/impls/commands/traversal.rs

@@ -16,7 +16,6 @@ use biome_service::configuration::load_editorconfig;
 use biome_service::workspace::ScanKind;
 use biome_service::{Workspace, WorkspaceError};
 use camino::Utf8PathBuf;
-use std::collections::BTreeSet;
 use std::ffi::OsString;
 use std::ops::{Deref, DerefMut};
 
@@ -209,6 +208,6 @@ where
 
 pub(crate) struct TraverseResult {
     pub(crate) summary: TraversalSummary,
-    pub(crate) evaluated_paths: BTreeSet<BiomePath>,
+    pub(crate) evaluated_paths: Vec<BiomePath>,
     pub(crate) diagnostics: Vec<Error>,
 }

crates/biome_cli/src/runner/impls/crawlers/default.rs

@@ -6,7 +6,6 @@ use crate::runner::impls::commands::traversal::TraverseResult;
 use crate::runner::impls::handlers::default::DefaultHandler;
 use crate::runner::process_file::ProcessFile;
 use biome_fs::BiomePath;
-use std::collections::BTreeSet;
 use std::time::Duration;
 
 pub(crate) struct DefaultCrawler<P>(P);
@@ -21,7 +20,7 @@ where
 
     fn output(
         collector_result: CollectorSummary,
-        evaluated_paths: BTreeSet<BiomePath>,
+        evaluated_paths: Vec<BiomePath>,
         _duration: Duration,
     ) -> TraverseResult {
         let CollectorSummary {
@@ -64,5 +63,5 @@ impl Crawler<()> for () {
     type ProcessFile = ();
     type Collector = ();
 
-    fn output(_: <Self::Collector as Collector>::Result, _: BTreeSet<BiomePath>, _: Duration) {}
+    fn output(_: <Self::Collector as Collector>::Result, _: Vec<BiomePath>, _: Duration) {}
 }

crates/biome_cli/src/runner/impls/finalizers/default.rs

@@ -19,7 +19,6 @@ use biome_fs::{BiomePath, FileSystem, OpenOptions};
 use biome_json_formatter::context::JsonFormatOptions;
 use biome_rowan::AstNode;
 use std::cmp::Ordering;
-use std::collections::BTreeSet;
 
 pub(crate) struct DefaultFinalizer;
 
@@ -156,7 +155,7 @@ struct PrintToReporter<'a> {
     cli_options: &'a CliOptions,
     diagnostics_payload: &'a DiagnosticsPayload,
     summary: TraversalSummary,
-    evaluated_paths: BTreeSet<BiomePath>,
+    evaluated_paths: Vec<BiomePath>,
     file_reporter_writer: &'a mut FileReporterWriter,
     console: &'a mut dyn Console,
     fs: &'a dyn FileSystem,

crates/biome_fs/src/fs.rs

@@ -5,9 +5,10 @@ use camino::{Utf8Path, Utf8PathBuf};
 use directories::ProjectDirs;
 pub use memory::{ErrorEntry, MemoryFileSystem};
 pub use os::{OsFileSystem, TemporaryFs};
+use papaya::{HashSetRef, LocalGuard};
 use serde::{Deserialize, Serialize};
-use std::collections::BTreeSet;
 use std::fmt::{Debug, Display, Formatter};
+use std::hash::RandomState;
 use std::panic::RefUnwindSafe;
 use std::path::Path;
 use std::sync::Arc;
@@ -37,12 +38,18 @@ impl ConfigName {
 }
 
 /// Represents the kind of filesystem entry a path points at.
-#[derive(Clone, Copy, Debug)]
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
 pub enum PathKind {
     File { is_symlink: bool },
     Directory { is_symlink: bool },
 }
 
+impl Default for PathKind {
+    fn default() -> Self {
+        Self::File { is_symlink: false }
+    }
+}
+
 impl PathKind {
     pub fn is_file(self) -> bool {
         matches!(self, Self::File { .. })
@@ -424,7 +431,7 @@ pub trait TraversalContext: Sync {
     fn store_path(&self, path: BiomePath);
 
     /// Returns the paths that should be handled
-    fn evaluated_paths(&self) -> BTreeSet<BiomePath>;
+    fn evaluated_paths(&self) -> HashSetRef<'_, BiomePath, RandomState, LocalGuard<'_>>;
 
     /// Returns whether directories are stored and returned by
     /// `Self::evaluated_paths()`.