diff --git a/Cargo.lock b/Cargo.lock index cd51940252d1f..852c1a08e9a77 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1185,20 +1185,6 @@ dependencies = [ "syn", ] -[[package]] -name = "dashmap" -version = "6.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" -dependencies = [ - "cfg-if", - "crossbeam-utils", - "hashbrown 0.14.5", - "lock_api", - "once_cell", - "parking_lot_core", -] - [[package]] name = "data-encoding" version = "2.11.0" @@ -3079,6 +3065,16 @@ version = "4.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d211803b9b6b570f68772237e415a029d5a50c65d382910b879fb19d3271f94d" +[[package]] +name = "papaya" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "997ee03cd38c01469a7046643714f0ad28880bcb9e6679ff0666e24817ca19b7" +dependencies = [ + "equivalent", + "seize", +] + [[package]] name = "parking" version = "2.2.1" @@ -4396,6 +4392,16 @@ dependencies = [ "libc", ] +[[package]] +name = "seize" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b55fb86dfd3a2f5f76ea78310a88f96c4ea21a3031f8d212443d56123fd0521" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "self-replace" version = "1.5.0" @@ -6526,9 +6532,9 @@ dependencies = [ "anyhow", "astral-reqwest-middleware", "cargo-util", - "dashmap", "fs-err", "owo-colors", + "papaya", "reqwest", "thiserror", "tokio", @@ -6734,8 +6740,8 @@ dependencies = [ name = "uv-once-map" version = "0.0.54" dependencies = [ - "dashmap", "futures", + "papaya", "tokio", ] @@ -7072,7 +7078,6 @@ dependencies = [ "astral-pubgrub", "clap", "cyclonedx-bom", - "dashmap", "either", "fs-err", "futures", @@ -7082,6 +7087,7 @@ dependencies = [ "itertools 0.14.0", "jiff", "owo-colors", + "papaya", "percent-encoding", "petgraph", "rkyv", @@ -7365,7 +7371,7 @@ name = "uv-types" version = "0.0.54" dependencies = [ "anyhow", - "dashmap", + "papaya", "rustc-hash", "thiserror", "uv-cache", diff --git a/Cargo.toml b/Cargo.toml index e555db8d27d96..1c1b486d818a1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -137,7 +137,6 @@ console = { version = "0.16.0", default-features = false, features = ["std"] } csv = { version = "1.3.0" } ctrlc = { version = "3.4.5" } cyclonedx-bom = { version = "0.8.1" } -dashmap = { version = "6.1.0" } data-encoding = { version = "2.6.0" } diskus = { version = "0.9.0", default-features = false } dotenvy = { version = "0.15.7" } @@ -183,6 +182,7 @@ miette = { version = "7.2.0", features = ["fancy-no-backtrace"] } nix = { version = "0.31.2", features = ["resource", "signal"] } open = { version = "5.3.2" } owo-colors = { version = "4.1.0" } +papaya = { version = "0.2.4" } path-slash = { version = "0.2.1" } pathdiff = { version = "0.2.1" } percent-encoding = { version = "2.3.1" } diff --git a/crates/uv-distribution/src/source/mod.rs b/crates/uv-distribution/src/source/mod.rs index f651ddd821f8f..58e95e60f7bd8 100644 --- a/crates/uv-distribution/src/source/mod.rs +++ b/crates/uv-distribution/src/source/mod.rs @@ -3025,7 +3025,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { }; if let Some(builder) = self.build_context.build_arena().remove(&build_key) { - debug!("Creating build environment for: {source}"); + debug!("Reusing existing build environment for: {source}"); let wheel = builder.wheel(temp_dir.path()).await.map_err(Error::Build)?; // Store the build context. @@ -3033,7 +3033,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { wheel } else { - debug!("Reusing existing build environment for: {source}"); + debug!("Creating build environment for: {source}"); let builder = self .build_context diff --git a/crates/uv-git/Cargo.toml b/crates/uv-git/Cargo.toml index afb5dcb6b2dac..cdd48629efc2a 100644 --- a/crates/uv-git/Cargo.toml +++ b/crates/uv-git/Cargo.toml @@ -27,7 +27,7 @@ uv-warnings = { workspace = true } anyhow = { workspace = true } cargo-util = { workspace = true } -dashmap = { workspace = true } +papaya = { workspace = true } fs-err = { workspace = true, features = ["tokio"] } owo-colors = { workspace = true } reqwest = { workspace = true } diff --git a/crates/uv-git/src/resolver.rs b/crates/uv-git/src/resolver.rs index fb36d67fd45b7..78f075285162e 100644 --- a/crates/uv-git/src/resolver.rs +++ b/crates/uv-git/src/resolver.rs @@ -3,9 +3,8 @@ use std::path::PathBuf; use std::str::FromStr; use std::sync::Arc; -use dashmap::DashMap; -use dashmap::mapref::one::Ref; use fs_err::tokio as fs; +use papaya::{HashMap, ResizeMode}; use reqwest_middleware::ClientWithMiddleware; use tracing::debug; @@ -60,18 +59,26 @@ impl GitHttpSettings { } /// A resolver for Git repositories. -#[derive(Default, Clone)] -pub struct GitResolver(Arc>); +#[derive(Clone)] +pub struct GitResolver(Arc>); + +impl Default for GitResolver { + fn default() -> Self { + Self(Arc::new( + HashMap::builder().resize_mode(ResizeMode::Blocking).build(), + )) + } +} impl GitResolver { /// Inserts a new [`GitOid`] for the given [`RepositoryReference`]. pub fn insert(&self, reference: RepositoryReference, sha: GitOid) { - self.0.insert(reference, sha); + self.0.pin().insert(reference, sha); } /// Returns the [`GitOid`] for the given [`RepositoryReference`], if it exists. - fn get(&self, reference: &RepositoryReference) -> Option> { - self.0.get(reference) + fn get(&self, reference: &RepositoryReference) -> Option { + self.0.pin().get(reference).copied() } /// Return the [`GitOid`] for the given [`GitUrl`], if it is already known. @@ -84,7 +91,7 @@ impl GitResolver { // If we know the precise commit already, return it. let reference = RepositoryReference::from(url); if let Some(precise) = self.get(&reference) { - return Some(*precise); + return Some(precise); } None @@ -181,7 +188,7 @@ impl GitResolver { // single process are consistent. let url = { if let Some(precise) = self.get(&reference) { - Cow::Owned(url.clone().with_precise(*precise)) + Cow::Owned(url.clone().with_precise(precise)) } else { Cow::Borrowed(url) } @@ -241,7 +248,7 @@ impl GitResolver { pub fn precise(&self, url: GitUrl) -> Option { let reference = RepositoryReference::from(&url); let precise = self.get(&reference)?; - Some(url.with_precise(*precise)) + Some(url.with_precise(precise)) } /// Returns `true` if the two Git URLs refer to the same precise commit. @@ -263,11 +270,11 @@ impl GitResolver { } // Otherwise, the URLs must resolve to the same precise commit. - let Some(a_precise) = a.precise().or_else(|| self.get(&a_ref).map(|sha| *sha)) else { + let Some(a_precise) = a.precise().or_else(|| self.get(&a_ref)) else { return false; }; - let Some(b_precise) = b.precise().or_else(|| self.get(&b_ref).map(|sha| *sha)) else { + let Some(b_precise) = b.precise().or_else(|| self.get(&b_ref)) else { return false; }; diff --git a/crates/uv-once-map/Cargo.toml b/crates/uv-once-map/Cargo.toml index d9c7f3f15e02f..679a0049bf9e1 100644 --- a/crates/uv-once-map/Cargo.toml +++ b/crates/uv-once-map/Cargo.toml @@ -17,6 +17,6 @@ test = false workspace = true [dependencies] -dashmap = { workspace = true } futures = { workspace = true } +papaya = { workspace = true } tokio = { workspace = true } diff --git a/crates/uv-once-map/src/lib.rs b/crates/uv-once-map/src/lib.rs index 1e79585ffb3a5..11f9e977da471 100644 --- a/crates/uv-once-map/src/lib.rs +++ b/crates/uv-once-map/src/lib.rs @@ -1,9 +1,9 @@ use std::borrow::Borrow; use std::fmt::{Debug, Display, Formatter}; use std::hash::{BuildHasher, Hash, RandomState}; -use std::sync::Arc; +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; -use dashmap::{DashMap, Entry}; +use papaya::{HashMap, ResizeMode}; use tokio::sync::Notify; /// The caller tried to wait for a task that was never registered. @@ -28,7 +28,7 @@ impl std::error::Error for UnregisteredTask {} /// Note that this always clones the value out of the underlying map. Because /// of this, it's common to wrap the `V` in an `Arc` to make cloning cheap. pub struct OnceMap { - items: DashMap, S>, + items: HashMap, S>, } impl Debug for OnceMap { @@ -44,14 +44,10 @@ impl OnceMap { /// or other tasks will hang. If it returns `false`, this job is already in progress and you /// can [`OnceMap::wait`] for the result. pub fn register(&self, key: K) -> bool { - let entry = self.items.entry(key); - match entry { - Entry::Occupied(_) => false, - Entry::Vacant(entry) => { - entry.insert(Value::Waiting(Arc::new(Notify::new()))); - true - } - } + self.items + .pin() + .try_insert(key, Value::Waiting(Arc::new(Notify::new()))) + .is_ok() } /// Register that you want to start a job, unless it was already started, then wait for its @@ -74,19 +70,13 @@ impl OnceMap { /// ``` pub async fn register_or_wait(&self, key: &K) -> Option { let notify = { - let entry = self.items.entry(key.clone()); - match entry { - Entry::Occupied(value) => match value.get() { - Value::Filled(value) => return Some(value.clone()), + let items = self.items.pin(); + match items.try_insert_with(key.clone(), || Value::Waiting(Arc::new(Notify::new()))) { + Ok(_) => return None, + Err(value) => match value { + Value::Filled(_) => return value.get(), Value::Waiting(notify) => notify.clone(), }, - Entry::Vacant(entry) => { - // We insert the notify even if the caller is `wait`. Calling `wait` without - // a previous `register` is a fatal error, so the state of the map doesn't - // matter. - entry.insert(Value::Waiting(Arc::new(Notify::new()))); - return None; - } } }; @@ -94,23 +84,24 @@ impl OnceMap { let notification = notify.notified(); // Make sure the value wasn't inserted in-between us checking the map and registering the waiter. - if let Value::Filled(value) = self.items.get(key).expect("map is append-only").value() { - return Some(value.clone()); + if let Some(value) = self.items.pin().get(key).expect("map is append-only").get() { + return Some(value); } // Wait until the value is inserted. notification.await; - let entry = self.items.get(key).expect("map is append-only"); - match entry.value() { - Value::Filled(value) => Some(value.clone()), + let items = self.items.pin(); + let value = items.get(key).expect("map is append-only"); + match value { + Value::Filled(_) => value.get(), Value::Waiting(_) => unreachable!("notify was called"), } } /// Submit the result of a job you registered. pub fn done(&self, key: K, value: V) { - if let Some(Value::Waiting(notify)) = self.items.insert(key, Value::Filled(value)) { + if let Some(Value::Waiting(notify)) = self.items.pin().insert(key, Value::filled(value)) { notify.notify_waiters(); } } @@ -139,11 +130,8 @@ impl OnceMap { where K: Borrow, { - let entry = self.items.get(key)?; - match entry.value() { - Value::Filled(value) => Some(value.clone()), - Value::Waiting(_) => None, - } + let items = self.items.pin(); + items.get(key)?.get() } /// Remove the result of a previous job, if any. @@ -151,18 +139,18 @@ impl OnceMap { where K: Borrow, { - let entry = self.items.remove(key)?; - match entry { - (_, Value::Filled(value)) => Some(value), - (_, Value::Waiting(_)) => None, - } + let items = self.items.pin(); + items.remove(key)?.take() } } impl Default for OnceMap { fn default() -> Self { Self { - items: DashMap::with_hasher(H::default()), + items: HashMap::builder() + .hasher(H::default()) + .resize_mode(ResizeMode::Blocking) + .build(), } } } @@ -176,7 +164,7 @@ where Self { items: iter .into_iter() - .map(|(k, v)| (k, Value::Filled(v))) + .map(|(k, v)| (k, Value::filled(v))) .collect(), } } @@ -185,5 +173,32 @@ where #[derive(Debug)] enum Value { Waiting(Arc), - Filled(V), + /// The mutex is a workaround to papaya always returning borrowed instead of owned values. + Filled(Mutex>), +} + +impl Value { + fn filled(value: V) -> Self { + Self::Filled(Mutex::new(Some(value))) + } + + fn lock(value: &Mutex>) -> MutexGuard<'_, Option> { + value.lock().unwrap_or_else(PoisonError::into_inner) + } + + fn take(&self) -> Option { + match self { + Self::Filled(value) => Self::lock(value).take(), + Self::Waiting(_) => None, + } + } +} + +impl Value { + fn get(&self) -> Option { + match self { + Self::Filled(value) => Self::lock(value).clone(), + Self::Waiting(_) => None, + } + } } diff --git a/crates/uv-resolver/Cargo.toml b/crates/uv-resolver/Cargo.toml index 0989d9ff8e6e9..9fd15571e005a 100644 --- a/crates/uv-resolver/Cargo.toml +++ b/crates/uv-resolver/Cargo.toml @@ -50,7 +50,7 @@ uv-workspace = { workspace = true } arcstr = { workspace = true } clap = { workspace = true, features = ["derive"], optional = true } cyclonedx-bom = { workspace = true } -dashmap = { workspace = true } +papaya = { workspace = true } either = { workspace = true } fs-err = { workspace = true, features = ["tokio"] } futures = { workspace = true } diff --git a/crates/uv-resolver/src/resolver/mod.rs b/crates/uv-resolver/src/resolver/mod.rs index 92c5a31e67f08..68659dd7e5959 100644 --- a/crates/uv-resolver/src/resolver/mod.rs +++ b/crates/uv-resolver/src/resolver/mod.rs @@ -9,10 +9,10 @@ use std::sync::Arc; use std::time::Instant; use std::{iter, slice, thread}; -use dashmap::DashMap; use either::Either; use futures::{FutureExt, StreamExt}; use itertools::Itertools; +use papaya::{HashMap, ResizeMode}; use pubgrub::{Id, IncompId, Incompatibility, Kind, Range, Ranges, State}; use rustc_hash::{FxHashMap, FxHashSet}; use tokio::sync::mpsc::{self, Receiver, Sender}; @@ -129,10 +129,11 @@ struct ResolverState { selector: CandidateSelector, index: InMemoryIndex, installed_packages: InstalledPackages, + // Papaya's maps are large on Windows, so box them to keep resolver futures small. /// Incompatibilities for packages that are entirely unavailable. - unavailable_packages: DashMap, + unavailable_packages: Box>, /// Incompatibilities for packages that are unavailable at specific versions. - incomplete_packages: DashMap>, + incomplete_packages: Box>>, /// The options that were used to configure this resolver. options: Options, /// The reporter to use for this resolver. @@ -251,8 +252,8 @@ impl python_requirement: python_requirement.clone(), conflicts, installed_packages, - unavailable_packages: DashMap::default(), - incomplete_packages: DashMap::default(), + unavailable_packages: Box::default(), + incomplete_packages: Box::default(), options, reporter: None, }; @@ -538,13 +539,13 @@ impl ResolverState ResolverState &archive.metadata, MetadataResponse::Unavailable(reason) => { self.unavailable_packages + .pin() .insert(name.clone(), reason.into()); return Ok(None); } @@ -1284,16 +1286,19 @@ impl ResolverState version_maps.as_slice(), VersionsResponse::NoIndex => { self.unavailable_packages + .pin() .insert(name.clone(), UnavailablePackage::NoIndex); &[] } VersionsResponse::Offline => { self.unavailable_packages + .pin() .insert(name.clone(), UnavailablePackage::Offline); &[] } VersionsResponse::NotFound => { self.unavailable_packages + .pin() .insert(name.clone(), UnavailablePackage::NotFound); &[] } @@ -1834,7 +1839,7 @@ impl ResolverState ResolverState { @@ -2523,18 +2530,21 @@ impl ResolverState { self.unavailable_packages + .pin() .insert(package_name.clone(), UnavailablePackage::NoIndex); return Ok(None); } VersionsResponse::Offline => { self.unavailable_packages + .pin() .insert(package_name.clone(), UnavailablePackage::Offline); return Ok(None); } VersionsResponse::NotFound => { self.unavailable_packages + .pin() .insert(package_name.clone(), UnavailablePackage::NotFound); return Ok(None); @@ -2702,23 +2712,23 @@ impl ResolverState(Arc>); +pub struct BuildArena(Arc>>); impl Default for BuildArena { fn default() -> Self { - Self(Arc::new(DashMap::new())) + Self(Arc::new( + HashMap::builder().resize_mode(ResizeMode::Blocking).build(), + )) } } @@ -73,12 +75,12 @@ impl Clone for BuildArena { impl BuildArena { /// Insert a build entry into the arena. - pub fn insert(&self, key: BuildKey, value: T) { - self.0.insert(key, value); + pub fn insert(&self, key: BuildKey, value: impl Into>) { + self.0.pin().insert(key, value.into()); } /// Remove a build entry from the arena. - pub fn remove(&self, key: &BuildKey) -> Option { - self.0.remove(key).map(|entry| entry.1) + pub fn remove(&self, key: &BuildKey) -> Option> { + self.0.pin().remove(key).cloned() } }