diff --git a/Cargo.lock b/Cargo.lock index 2ba91d96710..41fd9751a62 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6614,6 +6614,7 @@ dependencies = [ "encoding_rs_io", "fs-err", "junction", + "libc", "path-slash", "percent-encoding", "reflink-copy", diff --git a/Cargo.toml b/Cargo.toml index 8feee2bf0be..004eb49e339 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -171,6 +171,7 @@ indoc = { version = "2.0.5" } itertools = { version = "0.14.0" } jiff = { version = "0.2.0", features = ["serde"] } junction = { version = "2.0.0" } +libc = { version = "0.2.0" } mailparse = { version = "0.16.0" } md-5 = { version = "0.11.0" } memchr = { version = "2.7.4" } diff --git a/crates/uv-cache/src/lib.rs b/crates/uv-cache/src/lib.rs index d5eaa50f9d5..5592c042233 100644 --- a/crates/uv-cache/src/lib.rs +++ b/crates/uv-cache/src/lib.rs @@ -18,7 +18,7 @@ pub use crate::by_timestamp::CachedByTimestamp; #[cfg(feature = "clap")] pub use crate::cli::CacheArgs; use crate::removal::Remover; -pub use crate::removal::{Removal, rm_rf}; +pub use crate::removal::{Removal, RemovalMode}; pub use crate::wheel::WheelCache; use crate::wheel::WheelCacheKind; pub use archive::ArchiveId; @@ -174,6 +174,8 @@ pub struct Cache { /// Ensure that `uv cache` operations don't remove items from the cache that are used by another /// uv process. lock_file: Option>, + /// The storage accounting used when removing cache entries. + removal_mode: RemovalMode, } impl Cache { @@ -184,6 +186,7 @@ impl Cache { refresh: Refresh::None(Timestamp::now()), temp_dir: None, lock_file: None, + removal_mode: RemovalMode::Logical, } } @@ -195,6 +198,7 @@ impl Cache { refresh: Refresh::None(Timestamp::now()), temp_dir: Some(Arc::new(temp_dir)), lock_file: None, + removal_mode: RemovalMode::Logical, }) } @@ -204,6 +208,26 @@ impl Cache { Self { refresh, ..self } } + /// Set the storage accounting used when removing cache entries. + /// + /// Falls back to logical accounting when physical accounting is unsupported. + #[must_use] + pub fn with_removal_mode(self, removal_mode: RemovalMode) -> Self { + let removal_mode = match removal_mode { + RemovalMode::Physical if !uv_fs::supports_physical_space() => RemovalMode::Logical, + removal_mode => removal_mode, + }; + Self { + removal_mode, + ..self + } + } + + /// Create an empty removal summary using the cache's configured accounting mode. + pub fn removal(&self) -> Removal { + Removal::new(self.removal_mode) + } + /// Acquire a lock that allows removing entries from the cache. pub async fn with_exclusive_lock(self) -> Result { let Self { @@ -211,6 +235,7 @@ impl Cache { refresh, temp_dir, lock_file, + removal_mode, } = self; // Release the existing lock, avoid deadlocks from a cloned cache. @@ -233,6 +258,7 @@ impl Cache { refresh, temp_dir, lock_file: Some(Arc::new(lock_file)), + removal_mode, }) } @@ -245,6 +271,7 @@ impl Cache { refresh, temp_dir, lock_file, + removal_mode, } = self; match LockedFile::acquire_no_wait( @@ -257,12 +284,14 @@ impl Cache { refresh, temp_dir, lock_file: Some(Arc::new(lock_file)), + removal_mode, }), None => Err(Self { root, refresh, temp_dir, lock_file, + removal_mode, }), } } @@ -521,7 +550,9 @@ impl Cache { /// Clear the cache, removing all entries. pub fn clear(self, reporter: Box) -> Result { // Remove everything but `.lock`, Windows does not allow removal of a locked file - let mut removal = Remover::new(reporter).rm_rf(&self.root, true)?; + let mut removal = Remover::new(reporter) + .with_removal_mode(self.removal_mode) + .rm_rf(&self.root, true)?; let Self { root, lock_file, .. } = self; @@ -557,7 +588,7 @@ impl Cache { let references = self.find_archive_references()?; // Remove any entries for the package from the cache. - let mut summary = Removal::default(); + let mut summary = self.removal(); for bucket in CacheBucket::iter() { summary += bucket.remove(self, name)?; } @@ -574,7 +605,7 @@ impl Cache { for (target, references) in references { if target.starts_with(&archive_root) && references.iter().all(|path| !path.exists()) { debug!("Removing dangling cache entry: {}", target.display()); - summary += rm_rf(target)?; + summary += self.remove_path(target)?; } } @@ -583,7 +614,7 @@ impl Cache { /// Prune dangling cache entries and cached environments. pub fn prune(&self, ci: bool) -> Result { - let mut summary = Removal::default(); + let mut summary = self.removal(); // First, remove any top-level directories that are unused. These typically represent // outdated cache buckets (e.g., `wheels-v0`, when latest is `wheels-v1`). @@ -604,13 +635,13 @@ impl Cache { if CacheBucket::iter().all(|bucket| entry.file_name() != bucket.to_str()) { let path = entry.path(); debug!("Removing dangling cache bucket: {}", path.display()); - summary += rm_rf(path)?; + summary += self.remove_path(path)?; } } else { // If the file is not a marker file, remove it. let path = entry.path(); debug!("Removing dangling cache bucket: {}", path.display()); - summary += rm_rf(path)?; + summary += self.remove_path(path)?; } } @@ -622,7 +653,7 @@ impl Cache { let entry = entry?; let path = entry.path(); debug!("Removing cached environment: {}", path.display()); - summary += rm_rf(path)?; + summary += self.remove_path(path)?; } } Err(err) if err.kind() == io::ErrorKind::NotFound => (), @@ -639,7 +670,7 @@ impl Cache { let path = entry.path(); if path.is_dir() { debug!("Removing unzipped wheel entry: {}", path.display()); - summary += rm_rf(path)?; + summary += self.remove_path(path)?; } } } @@ -683,7 +714,7 @@ impl Cache { } debug!("Removing unzipped built wheel entry: {}", path.display()); - summary += rm_rf(path)?; + summary += self.remove_path(path)?; } } } @@ -700,7 +731,7 @@ impl Cache { let target = fs_err::canonicalize(&path)?; if !references.contains_key(&target) { debug!("Removing dangling cache archive: {}", path.display()); - summary += rm_rf(path)?; + summary += self.remove_path(path)?; } } } @@ -711,6 +742,13 @@ impl Cache { Ok(summary) } + /// Remove a cache path using the cache's configured storage accounting. + pub fn remove_path(&self, path: impl AsRef) -> io::Result { + Remover::default() + .with_removal_mode(self.removal_mode) + .rm_rf(path, false) + } + /// Find all references to entries in the archive bucket. /// /// Archive entries are often referenced by symlinks in other cache buckets. This method @@ -1233,37 +1271,37 @@ impl CacheBucket { metadata.name == *name } - let mut summary = Removal::default(); + let mut summary = cache.removal(); match self { Self::Wheels => { // For `pypi` wheels, we expect a directory per package (indexed by name). let root = cache.bucket(self).join(WheelCacheKind::Pypi); - summary += rm_rf(root.join(name.to_string()))?; + summary += cache.remove_path(root.join(name.to_string()))?; // For alternate indices, we expect a directory for every index (under an `index` // subdirectory), followed by a directory per package (indexed by name). let root = cache.bucket(self).join(WheelCacheKind::Index); for directory in directories(root)? { - summary += rm_rf(directory.join(name.to_string()))?; + summary += cache.remove_path(directory.join(name.to_string()))?; } // For direct URLs, we expect a directory for every URL, followed by a // directory per package (indexed by name). let root = cache.bucket(self).join(WheelCacheKind::Url); for directory in directories(root)? { - summary += rm_rf(directory.join(name.to_string()))?; + summary += cache.remove_path(directory.join(name.to_string()))?; } } Self::SourceDistributions => { // For `pypi` wheels, we expect a directory per package (indexed by name). let root = cache.bucket(self).join(WheelCacheKind::Pypi); - summary += rm_rf(root.join(name.to_string()))?; + summary += cache.remove_path(root.join(name.to_string()))?; // For alternate indices, we expect a directory for every index (under an `index` // subdirectory), followed by a directory per package (indexed by name). let root = cache.bucket(self).join(WheelCacheKind::Index); for directory in directories(root)? { - summary += rm_rf(directory.join(name.to_string()))?; + summary += cache.remove_path(directory.join(name.to_string()))?; } // For direct URLs, we expect a directory for every URL, followed by a @@ -1272,7 +1310,7 @@ impl CacheBucket { let root = cache.bucket(self).join(WheelCacheKind::Url); for url in directories(root)? { if directories(&url)?.any(|version| is_match(&version, name)) { - summary += rm_rf(url)?; + summary += cache.remove_path(url)?; } } @@ -1282,7 +1320,7 @@ impl CacheBucket { let root = cache.bucket(self).join(WheelCacheKind::Path); for path in directories(root)? { if directories(&path)?.any(|version| is_match(&version, name)) { - summary += rm_rf(path)?; + summary += cache.remove_path(path)?; } } @@ -1293,7 +1331,7 @@ impl CacheBucket { for repository in directories(root)? { for sha in directories(repository)? { if is_match(&sha, name) { - summary += rm_rf(sha)?; + summary += cache.remove_path(sha)?; } } } @@ -1301,20 +1339,20 @@ impl CacheBucket { Self::Simple => { // For `pypi` wheels, we expect a rkyv file per package, indexed by name. let root = cache.bucket(self).join(WheelCacheKind::Pypi); - summary += rm_rf(root.join(format!("{name}.rkyv")))?; + summary += cache.remove_path(root.join(format!("{name}.rkyv")))?; // For alternate indices, we expect a directory for every index (under an `index` // subdirectory), followed by a directory per package (indexed by name). let root = cache.bucket(self).join(WheelCacheKind::Index); for directory in directories(root)? { - summary += rm_rf(directory.join(format!("{name}.rkyv")))?; + summary += cache.remove_path(directory.join(format!("{name}.rkyv")))?; } } Self::FlatIndex => { // We can't know if the flat index includes a package, so we just remove the entire // cache entry. let root = cache.bucket(self); - summary += rm_rf(root)?; + summary += cache.remove_path(root)?; } Self::Git | Self::Interpreter diff --git a/crates/uv-cache/src/removal.rs b/crates/uv-cache/src/removal.rs index d0f9ae0a73a..23a055f7b8a 100644 --- a/crates/uv-cache/src/removal.rs +++ b/crates/uv-cache/src/removal.rs @@ -5,18 +5,25 @@ use std::io; use std::path::Path; +use tracing::debug; + use crate::CleanReporter; -/// Remove a file or directory and all its contents, returning a [`Removal`] with -/// the number of files and directories removed, along with a total byte count. -pub fn rm_rf(path: impl AsRef) -> io::Result { - Remover::default().rm_rf(path, false) +/// The storage accounting used when removing cache entries. +#[derive(Debug, Clone, Copy, Default)] +pub enum RemovalMode { + /// Report the logical size of the removed files. + #[default] + Logical, + /// Report the exclusively owned physical storage reclaimed by the removed files. + Physical, } /// A builder for a [`Remover`] that can remove files and directories. #[derive(Default)] pub(crate) struct Remover { reporter: Option>, + removal_mode: RemovalMode, } impl Remover { @@ -24,9 +31,16 @@ impl Remover { pub(crate) fn new(reporter: Box) -> Self { Self { reporter: Some(reporter), + ..Self::default() } } + /// Set the storage accounting used before each file is removed. + pub(crate) fn with_removal_mode(mut self, removal_mode: RemovalMode) -> Self { + self.removal_mode = removal_mode; + self + } + /// Remove a file or directory and all its contents, returning a [`Removal`] with /// the number of files and directories removed, along with a total byte count. pub(crate) fn rm_rf( @@ -34,7 +48,7 @@ impl Remover { path: impl AsRef, skip_locked_file: bool, ) -> io::Result { - let mut removal = Removal::default(); + let mut removal = Removal::new(self.removal_mode); removal.rm_rf(path.as_ref(), self.reporter.as_deref(), skip_locked_file)?; Ok(removal) } @@ -47,14 +61,49 @@ pub struct Removal { pub num_files: u64, /// The number of directories removed. pub num_dirs: u64, - /// The total number of bytes removed. + /// The logical number of bytes removed. /// /// Note: this will both over-count bytes removed for hard-linked files, and under-count /// bytes in general since it's a measure of the exact byte size (as opposed to the block size). - pub total_bytes: u64, + pub logical_bytes: u64, + /// The exclusively owned physical file data reclaimed by the removal, when available. + pub physical_bytes: Option, + /// Whether any removed entries could not be measured, making the physical count a lower bound. + pub physical_bytes_incomplete: bool, } impl Removal { + /// Create an empty removal summary with the requested storage accounting. + pub(crate) fn new(removal_mode: RemovalMode) -> Self { + Self { + physical_bytes: match removal_mode { + RemovalMode::Logical => None, + RemovalMode::Physical => Some(0), + }, + ..Self::default() + } + } + + /// Account for a file while its current sharing state can still be inspected. + fn add_file(&mut self, path: &Path, metadata: &std::fs::Metadata) { + self.logical_bytes += metadata.len(); + + if let Some(physical_bytes) = self.physical_bytes { + match uv_fs::physical_space(path, metadata) { + Ok(physical) => { + self.physical_bytes = Some(physical_bytes.saturating_add(physical)); + } + Err(error) => { + debug!( + "Failed to measure physical space for {}: {error}", + path.display() + ); + self.physical_bytes_incomplete = true; + } + } + } + } + /// Recursively remove a file or directory and all its contents. fn rm_rf( &mut self, @@ -74,7 +123,7 @@ impl Removal { self.num_files += 1; // Remove the file. - self.total_bytes += metadata.len(); + self.add_file(&path, &metadata); if metadata.is_symlink() { cfg_select! { windows => { @@ -158,8 +207,10 @@ impl Removal { self.num_files += 1; // Remove the file. - if let Ok(meta) = entry.metadata() { - self.total_bytes += meta.len(); + if let Ok(metadata) = entry.metadata() { + self.add_file(entry.path(), &metadata); + } else if self.physical_bytes.is_some() { + self.physical_bytes_incomplete = true; } remove_file(entry.path())?; } @@ -177,7 +228,12 @@ impl std::ops::AddAssign for Removal { fn add_assign(&mut self, other: Self) { self.num_files += other.num_files; self.num_dirs += other.num_dirs; - self.total_bytes += other.total_bytes; + self.logical_bytes += other.logical_bytes; + self.physical_bytes = self + .physical_bytes + .zip(other.physical_bytes) + .map(|(left, right)| left.saturating_add(right)); + self.physical_bytes_incomplete |= other.physical_bytes_incomplete; } } diff --git a/crates/uv-dev/src/generate_preview_features_reference.rs b/crates/uv-dev/src/generate_preview_features_reference.rs index d1dcb927b3e..b8d005b6f82 100644 --- a/crates/uv-dev/src/generate_preview_features_reference.rs +++ b/crates/uv-dev/src/generate_preview_features_reference.rs @@ -111,6 +111,7 @@ mod tests { - `audit-command`: Allows using `uv audit`. - `auth-helper`: Allows using `uv auth helper` as a credential helper for external tools. - `azure-endpoint`: Allows signing requests to Azure Blob Storage endpoints with Azure credentials. + - `cache-physical-space`: Reports the physical disk space reclaimed by cache cleanup, accounting for hardlinks and copy-on-write clones. - `cache-size`: Allows using `uv cache size`. - `centralized-project-envs`: Stores [project virtual environments](./projects/layout.md#centralized-project-environments) in the uv cache. diff --git a/crates/uv-distribution/src/source/mod.rs b/crates/uv-distribution/src/source/mod.rs index ae65795d857..b303c835626 100644 --- a/crates/uv-distribution/src/source/mod.rs +++ b/crates/uv-distribution/src/source/mod.rs @@ -3306,7 +3306,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { /// Prune any unused source distributions from the cache. pub fn prune(cache: &Cache) -> Result { - let mut removal = Removal::default(); + let mut removal = cache.removal(); let bucket = cache.bucket(CacheBucket::SourceDistributions); if bucket.is_dir() { @@ -3332,8 +3332,9 @@ pub fn prune(cache: &Cache) -> Result { "Removing dangling source revision: {}", sibling.path().display() ); - removal += - uv_cache::rm_rf(sibling.path()).map_err(Error::CacheWrite)?; + removal += cache + .remove_path(sibling.path()) + .map_err(Error::CacheWrite)?; } } } @@ -3355,8 +3356,9 @@ pub fn prune(cache: &Cache) -> Result { "Removing dangling source revision: {}", sibling.path().display() ); - removal += - uv_cache::rm_rf(sibling.path()).map_err(Error::CacheWrite)?; + removal += cache + .remove_path(sibling.path()) + .map_err(Error::CacheWrite)?; } } } diff --git a/crates/uv-fs/Cargo.toml b/crates/uv-fs/Cargo.toml index aef75e028d5..4559f47afc7 100644 --- a/crates/uv-fs/Cargo.toml +++ b/crates/uv-fs/Cargo.toml @@ -40,6 +40,9 @@ walkdir = { workspace = true } [target.'cfg(any(unix, target_os = "wasi", target_os = "redox"))'.dependencies] rustix = { workspace = true } +[target.'cfg(any(target_os = "macos", target_os = "ios"))'.dependencies] +libc = { workspace = true } + [target.'cfg(windows)'.dependencies] backon = { workspace = true } junction = { workspace = true } diff --git a/crates/uv-fs/src/lib.rs b/crates/uv-fs/src/lib.rs index c110aba84f1..3c6e0ed62c1 100644 --- a/crates/uv-fs/src/lib.rs +++ b/crates/uv-fs/src/lib.rs @@ -12,12 +12,14 @@ use tracing::{debug, warn}; pub use crate::locked_file::*; pub use crate::path::*; pub use crate::read::ValidatedReader; +pub use crate::space::{physical_space, supports_physical_space}; pub mod cachedir; pub mod link; mod locked_file; mod path; mod read; +mod space; pub mod which; /// Attempt to check if the two paths refer to the same file. diff --git a/crates/uv-fs/src/space.rs b/crates/uv-fs/src/space.rs new file mode 100644 index 00000000000..4f042fe51ba --- /dev/null +++ b/crates/uv-fs/src/space.rs @@ -0,0 +1,222 @@ +use std::io; +use std::path::Path; + +#[cfg(any(target_os = "macos", target_os = "ios"))] +use std::ffi::CString; +#[cfg(any(target_os = "macos", target_os = "ios"))] +use std::os::unix::ffi::OsStrExt; +#[cfg(unix)] +use std::os::unix::fs::MetadataExt; + +/// Return whether the current platform can identify individual files' physical storage. +pub const fn supports_physical_space() -> bool { + cfg!(any( + target_os = "linux", + target_os = "macos", + target_os = "ios" + )) +} + +/// Return the physical file data that would be reclaimed by deleting `path`. +/// +/// The result excludes data retained by another hardlink, copy-on-write clone, or snapshot. +/// Filesystem metadata is not included. +pub fn physical_space(path: &Path, metadata: &std::fs::Metadata) -> io::Result { + if !metadata.is_file() { + #[cfg(unix)] + { + return Ok(metadata.blocks().saturating_mul(512)); + } + + #[cfg(not(unix))] + { + return Ok(0); + } + } + + #[cfg(unix)] + if metadata.nlink() > 1 { + return Ok(0); + } + + #[cfg(any(target_os = "macos", target_os = "ios"))] + { + apple_physical_space(path) + } + + #[cfg(target_os = "linux")] + { + linux_physical_space(path) + } + + #[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "ios")))] + { + let _ = path; + Err(io::Error::new( + io::ErrorKind::Unsupported, + "per-file space measurement is unsupported on this platform", + )) + } +} + +#[cfg(any(target_os = "macos", target_os = "ios"))] +#[expect(unsafe_code)] +fn apple_physical_space(path: &Path) -> io::Result { + let path = CString::new(path.as_os_str().as_bytes()) + .map_err(|error| io::Error::new(io::ErrorKind::InvalidInput, error))?; + let mut attributes = libc::attrlist { + bitmapcount: libc::ATTR_BIT_MAP_COUNT, + reserved: 0, + commonattr: libc::ATTR_CMN_RETURNED_ATTRS, + volattr: 0, + dirattr: 0, + fileattr: 0, + forkattr: libc::ATTR_CMNEXT_PRIVATESIZE, + }; + let mut response = [0_u8; 32]; + + // SAFETY: `path` is null-terminated and valid for the duration of the call. `attributes` is a + // valid attribute request, and `response` has enough space for the length, returned attribute + // set, and requested private-size value. + let result = unsafe { + libc::getattrlist( + path.as_ptr(), + (&raw mut attributes).cast(), + response.as_mut_ptr().cast(), + response.len(), + libc::FSOPT_ATTR_CMN_EXTENDED, + ) + }; + if result != 0 { + return Err(io::Error::last_os_error()); + } + + let returned_fork_attributes = + u32::from_ne_bytes(response[20..24].try_into().map_err(io::Error::other)?); + if returned_fork_attributes & libc::ATTR_CMNEXT_PRIVATESIZE == 0 { + return Err(io::Error::new( + io::ErrorKind::Unsupported, + "the filesystem does not report private file sizes", + )); + } + + let private_size = i64::from_ne_bytes(response[24..32].try_into().map_err(io::Error::other)?); + u64::try_from(private_size).map_err(io::Error::other) +} + +#[cfg(target_os = "linux")] +#[expect(unsafe_code)] +fn linux_physical_space(path: &Path) -> io::Result { + const FIEMAP_EXTENT_LAST: u32 = 0x0000_0001; + const FIEMAP_EXTENT_UNKNOWN: u32 = 0x0000_0002; + const FIEMAP_EXTENT_DELALLOC: u32 = 0x0000_0004; + const FIEMAP_EXTENT_ENCODED: u32 = 0x0000_0008; + const FIEMAP_EXTENT_NOT_ALIGNED: u32 = 0x0000_0100; + const FIEMAP_EXTENT_DATA_INLINE: u32 = 0x0000_0200; + const FIEMAP_EXTENT_SHARED: u32 = 0x0000_2000; + const MAX_EXTENTS: usize = 32; + + #[derive(Default)] + #[repr(C)] + struct Fiemap { + start: u64, + length: u64, + flags: u32, + mapped_extents: u32, + extent_count: u32, + reserved: u32, + } + + #[derive(Clone, Copy, Default)] + #[repr(C)] + struct FiemapExtent { + logical: u64, + physical: u64, + length: u64, + reserved64: [u64; 2], + flags: u32, + reserved: [u32; 3], + } + + #[derive(Default)] + #[repr(C)] + struct FiemapBuffer { + header: Fiemap, + extents: [FiemapExtent; MAX_EXTENTS], + } + + const FS_IOC_FIEMAP: rustix::ioctl::Opcode = + rustix::ioctl::opcode::read_write::(b'f', 11); + + let file = fs_err::File::open(path)?; + let mut physical = 0_u64; + let mut start = 0_u64; + + loop { + let mut request = FiemapBuffer { + header: Fiemap { + start, + length: u64::MAX.saturating_sub(start), + extent_count: u32::try_from(MAX_EXTENTS).map_err(io::Error::other)?, + ..Fiemap::default() + }, + ..FiemapBuffer::default() + }; + + // SAFETY: `FS_IOC_FIEMAP` is the Linux fiemap ioctl opcode, and `request` begins with the + // expected fiemap header followed by enough initialized storage for all requested extents. + unsafe { + rustix::ioctl::ioctl( + &file, + rustix::ioctl::Updater::::new(&mut request), + )?; + } + + let mapped_extents = + usize::try_from(request.header.mapped_extents).map_err(io::Error::other)?; + if mapped_extents > request.extents.len() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "the filesystem returned more extents than requested", + )); + } + if mapped_extents == 0 { + return Ok(physical); + } + + for extent in &request.extents[..mapped_extents] { + if extent.flags + & (FIEMAP_EXTENT_DELALLOC | FIEMAP_EXTENT_DATA_INLINE | FIEMAP_EXTENT_SHARED) + != 0 + { + continue; + } + + if extent.flags + & (FIEMAP_EXTENT_UNKNOWN | FIEMAP_EXTENT_ENCODED | FIEMAP_EXTENT_NOT_ALIGNED) + != 0 + { + return Err(io::Error::new( + io::ErrorKind::Unsupported, + "the filesystem cannot report the physical size of an extent", + )); + } + + physical = physical.saturating_add(extent.length); + } + + let last_extent = &request.extents[mapped_extents - 1]; + if last_extent.flags & FIEMAP_EXTENT_LAST != 0 { + return Ok(physical); + } + + let next = last_extent.logical.saturating_add(last_extent.length); + if next <= start { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "the filesystem returned a non-advancing extent", + )); + } + start = next; + } +} diff --git a/crates/uv-preview/src/lib.rs b/crates/uv-preview/src/lib.rs index 0eda60a7962..ea00c808df1 100644 --- a/crates/uv-preview/src/lib.rs +++ b/crates/uv-preview/src/lib.rs @@ -253,6 +253,8 @@ pub enum PreviewFeature { S3Endpoint, /// Allows using `uv cache size`. CacheSize, + /// Reports the physical disk space reclaimed by cache cleanup, accounting for hardlinks and copy-on-write clones. + CachePhysicalSpace, /// Rejects the deprecated `--project` option in `uv init`. InitProjectFlag, /// Allows using `uv workspace metadata`. diff --git a/crates/uv/src/commands/cache_clean.rs b/crates/uv/src/commands/cache_clean.rs index 583b1fe698f..3166a1089ce 100644 --- a/crates/uv/src/commands/cache_clean.rs +++ b/crates/uv/src/commands/cache_clean.rs @@ -4,9 +4,10 @@ use anyhow::{Context, Result}; use owo_colors::OwoColorize; use tracing::debug; -use uv_cache::{Cache, Removal}; +use uv_cache::{Cache, RemovalMode}; use uv_fs::Simplified; use uv_normalize::PackageName; +use uv_preview::{Preview, PreviewFeature}; use crate::commands::reporters::{CleaningDirectoryReporter, CleaningPackageReporter}; use crate::commands::{ExitStatus, human_readable_bytes}; @@ -18,6 +19,7 @@ pub(crate) async fn cache_clean( force: bool, cache: Cache, printer: Printer, + preview: Preview, ) -> Result { if !cache.root().exists() { writeln!( @@ -43,6 +45,13 @@ pub(crate) async fn cache_clean( } }; + let removal_mode = if preview.is_enabled(PreviewFeature::CachePhysicalSpace) { + RemovalMode::Physical + } else { + RemovalMode::Logical + }; + let cache = cache.with_removal_mode(removal_mode); + let summary = if packages.is_empty() { writeln!( printer.stderr(), @@ -59,7 +68,7 @@ pub(crate) async fn cache_clean( .with_context(|| format!("Failed to clear cache at: {}", root.user_display()))? } else { let reporter = CleaningPackageReporter::new(printer, Some(packages.len())); - let mut summary = Removal::default(); + let mut summary = cache.removal(); for package in packages { let removed = cache.remove(package)?; @@ -90,15 +99,20 @@ pub(crate) async fn cache_clean( } } - // If any, write a summary of the total byte count removed. - if summary.total_bytes > 0 { - let bytes = if summary.total_bytes < 1024 { - format!("{}B", summary.total_bytes) + // If any, report the physical space, falling back to the logical removed size. + let reported_bytes = summary.physical_bytes.unwrap_or(summary.logical_bytes); + if summary.logical_bytes > 0 || reported_bytes > 0 { + let bytes = if reported_bytes < 1024 { + format!("{reported_bytes}B") } else { - let (bytes, unit) = human_readable_bytes(summary.total_bytes); + let (bytes, unit) = human_readable_bytes(reported_bytes); format!("{bytes:.1}{unit}") }; - write!(printer.stderr(), " ({})", bytes.green())?; + if summary.physical_bytes_incomplete { + write!(printer.stderr(), " (at least {})", bytes.green())?; + } else { + write!(printer.stderr(), " ({})", bytes.green())?; + } } writeln!(printer.stderr())?; diff --git a/crates/uv/src/commands/cache_prune.rs b/crates/uv/src/commands/cache_prune.rs index 7c7c371c9f6..7109eddef21 100644 --- a/crates/uv/src/commands/cache_prune.rs +++ b/crates/uv/src/commands/cache_prune.rs @@ -4,8 +4,9 @@ use anyhow::{Context, Result}; use owo_colors::OwoColorize; use tracing::debug; -use uv_cache::{Cache, Removal}; +use uv_cache::{Cache, RemovalMode}; use uv_fs::Simplified; +use uv_preview::{Preview, PreviewFeature}; use crate::commands::{ExitStatus, human_readable_bytes}; use crate::printer::Printer; @@ -16,6 +17,7 @@ pub(crate) async fn cache_prune( force: bool, cache: Cache, printer: Printer, + preview: Preview, ) -> Result { if !cache.root().exists() { writeln!( @@ -41,13 +43,20 @@ pub(crate) async fn cache_prune( } }; + let removal_mode = if preview.is_enabled(PreviewFeature::CachePhysicalSpace) { + RemovalMode::Physical + } else { + RemovalMode::Logical + }; + let cache = cache.with_removal_mode(removal_mode); + writeln!( printer.stderr(), "Pruning cache at: {}", cache.root().user_display().cyan() )?; - let mut summary = Removal::default(); + let mut summary = cache.removal(); // Prune the source distribution cache, which is tightly coupled to the builder crate. summary += uv_distribution::prune(&cache) @@ -77,15 +86,20 @@ pub(crate) async fn cache_prune( } } - // If any, write a summary of the total byte count removed. - if summary.total_bytes > 0 { - let bytes = if summary.total_bytes < 1024 { - format!("{}B", summary.total_bytes) + // If any, report the physical space, falling back to the logical removed size. + let reported_bytes = summary.physical_bytes.unwrap_or(summary.logical_bytes); + if summary.logical_bytes > 0 || reported_bytes > 0 { + let bytes = if reported_bytes < 1024 { + format!("{reported_bytes}B") } else { - let (bytes, unit) = human_readable_bytes(summary.total_bytes); + let (bytes, unit) = human_readable_bytes(reported_bytes); format!("{bytes:.1}{unit}") }; - write!(printer.stderr(), " ({})", bytes.green())?; + if summary.physical_bytes_incomplete { + write!(printer.stderr(), " (at least {})", bytes.green())?; + } else { + write!(printer.stderr(), " ({})", bytes.green())?; + } } writeln!(printer.stderr())?; diff --git a/crates/uv/src/lib.rs b/crates/uv/src/lib.rs index a8371114396..f631342cbfc 100644 --- a/crates/uv/src/lib.rs +++ b/crates/uv/src/lib.rs @@ -1279,13 +1279,13 @@ pub async fn run(cli: Cli, global_initialization: GlobalInitialization) -> Resul }) | Commands::Clean(args) => { show_settings!(args); - commands::cache_clean(&args.package, args.force, cache, printer).await + commands::cache_clean(&args.package, args.force, cache, printer, globals.preview).await } Commands::Cache(CacheNamespace { command: CacheCommand::Prune(args), }) => { show_settings!(args); - commands::cache_prune(args.ci, args.force, cache, printer).await + commands::cache_prune(args.ci, args.force, cache, printer, globals.preview).await } Commands::Cache(CacheNamespace { command: CacheCommand::Dir, diff --git a/crates/uv/tests/build/cache_clean.rs b/crates/uv/tests/build/cache_clean.rs index 9d68c84afde..e2bf23fbfc9 100644 --- a/crates/uv/tests/build/cache_clean.rs +++ b/crates/uv/tests/build/cache_clean.rs @@ -2,7 +2,12 @@ use anyhow::Result; use assert_cmd::prelude::*; use assert_fs::prelude::*; +#[cfg(target_os = "linux")] +use std::process::Command; + use uv_cache::Cache; +#[cfg(unix)] +use uv_fs::link::{LinkMode, LinkOptions, link_dir}; use uv_static::EnvVars; use uv_test::uv_snapshot; @@ -34,6 +39,201 @@ fn clean_all() -> Result<()> { Ok(()) } +/// `cache clean` should report physical space for hardlinks only when the preview is enabled. +#[cfg(unix)] +#[test] +fn clean_all_hardlinked_file() -> Result<()> { + let context = uv_test::test_context!("3.12").with_filtered_counts(); + + // Keep the retained hardlink beside the cache so both entries share a filesystem. + let retained = context.cache_dir.path().with_file_name("retained.bin"); + fs_err::write(&retained, vec![42; 1024 * 1024])?; + fs_err::OpenOptions::new() + .write(true) + .open(&retained)? + .sync_all()?; + + let cached = context.cache_dir.child("hardlinked.bin"); + fs_err::hard_link(&retained, &cached)?; + + let filters = size_filters(&context); + + uv_snapshot!(&filters, context.clean(), @" + exit_code: 0 (success) + ----- stderr ----- + Clearing cache at: [CACHE_DIR]/ + Removed [N] files (1.0MiB) + "); + + context.cache_dir.create_dir_all()?; + fs_err::hard_link(&retained, &cached)?; + + uv_snapshot!(&filters, context.clean().arg("--preview-features").arg("cache-physical-space"), @" + exit_code: 0 (success) + ----- stderr ----- + Clearing cache at: [CACHE_DIR]/ + Removed [N] files (0B) + "); + + assert!(retained.is_file()); + + context.cache_dir.create_dir_all()?; + cached.write_binary(&vec![42; 1024 * 1024])?; + fs_err::OpenOptions::new() + .write(true) + .open(cached.path())? + .sync_all()?; + fs_err::hard_link(&cached, context.cache_dir.child("second-hardlink.bin"))?; + + uv_snapshot!(&filters, context.clean().arg("--preview-features").arg("cache-physical-space"), @" + exit_code: 0 (success) + ----- stderr ----- + Clearing cache at: [CACHE_DIR]/ + Removed [N] files (1.0MiB) + "); + + Ok(()) +} + +/// `cache clean` should report physical space for copy-on-write clones in preview mode. +#[cfg(unix)] +#[test] +fn clean_all_cloned_file() -> Result<()> { + let context = copy_on_write_test_context()?; + let retained = context.cache_dir.path().with_file_name("retained"); + fs_err::create_dir_all(&retained)?; + let original = retained.join("original.bin"); + fs_err::write(&original, vec![42; 1024 * 1024])?; + + // Remove unrelated cache entries so the cloned file is the only allocated data being cleaned. + context.clean().assert().success(); + context.cache_dir.create_dir_all()?; + + let cached = context.cache_dir.child("cloned"); + let link_mode = link_dir(&retained, &cached, &LinkOptions::new(LinkMode::Clone))?; + if link_mode != LinkMode::Clone { + assert!( + std::env::var_os(EnvVars::UV_INTERNAL__TEST_COW_FS).is_none(), + "the configured copy-on-write filesystem did not clone the cached file" + ); + return Ok(()); + } + + let filters = size_filters(&context); + + uv_snapshot!(&filters, context.clean().arg("--preview"), @" + exit_code: 0 (success) + ----- stderr ----- + Clearing cache at: [CACHE_DIR]/ + Removed [N] files (0B) + "); + + assert!(original.is_file()); + + Ok(()) +} + +/// Clones shared only within the cache should be counted once when their final reference is removed. +#[cfg(unix)] +#[test] +fn clean_all_cached_clones() -> Result<()> { + let context = copy_on_write_test_context()?; + let original = context.cache_dir.child("original"); + original.create_dir_all()?; + original + .child("original.bin") + .write_binary(&vec![42; 1024 * 1024])?; + + let cloned = context.cache_dir.child("cloned"); + let link_mode = link_dir(&original, &cloned, &LinkOptions::new(LinkMode::Clone))?; + if link_mode != LinkMode::Clone { + assert!( + std::env::var_os(EnvVars::UV_INTERNAL__TEST_COW_FS).is_none(), + "the configured copy-on-write filesystem did not clone the cached file" + ); + return Ok(()); + } + + let filters = size_filters(&context); + + uv_snapshot!(&filters, context.clean().arg("--preview-features").arg("cache-physical-space"), @" + exit_code: 0 (success) + ----- stderr ----- + Clearing cache at: [CACHE_DIR]/ + Removed [N] files (1.0MiB) + "); + + Ok(()) +} + +/// Unknown compressed extents should not discard measurements for unrelated cache entries. +#[cfg(target_os = "linux")] +#[test] +fn clean_all_compressed_file() -> Result<()> { + if std::env::var_os(EnvVars::UV_INTERNAL__TEST_COW_FS).is_none() { + return Ok(()); + } + + let context = copy_on_write_test_context()?; + let measured = context.cache_dir.child("measured.bin"); + measured.write_binary(&vec![42; 1024 * 1024])?; + fs_err::OpenOptions::new() + .write(true) + .open(measured.path())? + .sync_all()?; + + let compressed = context.cache_dir.child("compressed.bin"); + fs_err::File::create(compressed.path())?; + Command::new("btrfs") + .args(["property", "set"]) + .arg(compressed.path()) + .args(["compression", "zstd"]) + .assert() + .success(); + compressed.write_binary(&vec![42; 1024 * 1024])?; + fs_err::OpenOptions::new() + .write(true) + .open(compressed.path())? + .sync_all()?; + + let filters = size_filters(&context); + + uv_snapshot!(&filters, context.clean().arg("--preview-features").arg("cache-physical-space"), @" + exit_code: 0 (success) + ----- stderr ----- + Clearing cache at: [CACHE_DIR]/ + Removed [N] files (at least 1.0MiB) + "); + + Ok(()) +} + +/// Put the cache and retained files on CI's Btrfs or APFS volume, when configured. +#[cfg(unix)] +fn copy_on_write_test_context() -> Result { + let context = uv_test::test_context!("3.12").with_filtered_counts(); + if std::env::var_os(EnvVars::UV_INTERNAL__TEST_COW_FS).is_none() { + return Ok(context); + } + + let Some(context) = context.with_cache_on_cow_fs()? else { + anyhow::bail!("the configured copy-on-write cache filesystem was unavailable"); + }; + + let cache_dir = context.cache_dir.path().to_path_buf(); + Ok(context.with_filtered_path(&cache_dir, "CACHE_DIR")) +} + +/// Preserve physical sizes while applying the context's other snapshot filters. +#[cfg(unix)] +fn size_filters(context: &uv_test::TestContext) -> Vec<(&str, &str)> { + context + .filters() + .into_iter() + .filter(|(_, replacement)| *replacement != "$1[SIZE]") + .collect() +} + /// `cache clear` should behave as an alias of `cache clean`. #[test] fn clear_all_alias() -> Result<()> { diff --git a/crates/uv/tests/build/cache_prune.rs b/crates/uv/tests/build/cache_prune.rs index ffe2fca906f..4a4c55587ae 100644 --- a/crates/uv/tests/build/cache_prune.rs +++ b/crates/uv/tests/build/cache_prune.rs @@ -40,6 +40,52 @@ fn prune_no_op() -> Result<()> { Ok(()) } +/// `cache prune` should report physical space for hardlinks only when the preview is enabled. +#[cfg(unix)] +#[test] +fn prune_hardlinked_file() -> Result<()> { + let context = uv_test::test_context!("3.12"); + + // Keep both hardlinks on the selected filesystem. + let retained = context.cache_dir.path().with_file_name("retained.bin"); + fs_err::write(&retained, vec![42; 1024 * 1024])?; + fs_err::OpenOptions::new() + .write(true) + .open(&retained)? + .sync_all()?; + + let stale = context.cache_dir.child("stale-v0"); + stale.create_dir_all()?; + fs_err::hard_link(&retained, stale.child("hardlinked.bin"))?; + + let filters: Vec<_> = context + .filters() + .into_iter() + .filter(|(_, replacement)| *replacement != "$1[SIZE]") + .collect(); + + uv_snapshot!(&filters, context.prune(), @" + exit_code: 0 (success) + ----- stderr ----- + Pruning cache at: [CACHE_DIR]/ + Removed 1 file (1.0MiB) + "); + + stale.create_dir_all()?; + fs_err::hard_link(&retained, stale.child("hardlinked.bin"))?; + + uv_snapshot!(&filters, context.prune().arg("--preview-features").arg("cache-physical-space"), @" + exit_code: 0 (success) + ----- stderr ----- + Pruning cache at: [CACHE_DIR]/ + Removed 1 file (0B) + "); + + assert!(retained.is_file()); + + Ok(()) +} + /// `cache prune` should remove any stale top-level directories from the cache. #[test] fn prune_stale_directory() -> Result<()> { diff --git a/crates/uv/tests/sync/show_settings.rs b/crates/uv/tests/sync/show_settings.rs index 4ab2c86542c..947fce21bb3 100644 --- a/crates/uv/tests/sync/show_settings.rs +++ b/crates/uv/tests/sync/show_settings.rs @@ -3386,6 +3386,7 @@ fn preview_features() { + NativeAuth, + S3Endpoint, + CacheSize, + + CachePhysicalSpace, + InitProjectFlag, + WorkspaceMetadata, + WorkspaceDir, diff --git a/docs/concepts/cache.md b/docs/concepts/cache.md index 7f26eff6027..1b93ac39557 100644 --- a/docs/concepts/cache.md +++ b/docs/concepts/cache.md @@ -144,6 +144,18 @@ uv provides a few different mechanisms for removing entries from the cache: longer necessary and can be safely removed. Centralized project environments are recreated as needed. `uv cache prune` is safe to run periodically, to keep the cache directory clean. +By default, cache cleanup reports the logical size of removed entries. Enable the +`cache-physical-space` [preview feature](./preview.md) to instead report the physical disk space +reclaimed, accounting for hardlinks and copy-on-write clones: + +```console +$ uv cache clean --preview-features cache-physical-space +``` + +If an entry's allocated size cannot be measured, such as a compressed extent on Btrfs, uv reports a +lower bound for the space reclaimed from the remaining entries. The preview feature is currently +supported on macOS and Linux; other platforms continue reporting the logical removed size. + uv blocks cache-modifying operations while other uv commands are running. By default, those `uv cache` commands have a 5 min timeout waiting for other uv processes to terminate to avoid deadlocks. This timeout can be changed with diff --git a/uv.schema.json b/uv.schema.json index a46cb9ed7b2..be23640ee9a 100644 --- a/uv.schema.json +++ b/uv.schema.json @@ -1794,6 +1794,7 @@ "native-auth", "s3-endpoint", "cache-size", + "cache-physical-space", "init-project-flag", "workspace-metadata", "workspace-dir",