From 336621712f4ba91a9e2b3b924873d3535a722bdf Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Mon, 3 Aug 2026 14:59:52 -0400 Subject: [PATCH 01/13] Report reclaimed disk space during cache cleanup --- crates/uv-fs/src/lib.rs | 2 + crates/uv-fs/src/space.rs | 47 +++++++++++++++++++ crates/uv/src/commands/cache_clean.rs | 36 ++++++++++++--- crates/uv/src/commands/cache_prune.rs | 33 +++++++++++--- crates/uv/tests/build/cache_clean.rs | 65 ++++++++++++++++++++++++--- crates/uv/tests/build/cache_prune.rs | 32 +++++++++++-- crates/uv/tests/pip/pip_sync.rs | 10 ++--- 7 files changed, 197 insertions(+), 28 deletions(-) create mode 100644 crates/uv-fs/src/space.rs diff --git a/crates/uv-fs/src/lib.rs b/crates/uv-fs/src/lib.rs index c110aba84f1..6d10b1e8bf0 100644 --- a/crates/uv-fs/src/lib.rs +++ b/crates/uv-fs/src/lib.rs @@ -12,12 +12,14 @@ use tracing::{debug, warn}; pub use crate::locked_file::*; pub use crate::path::*; pub use crate::read::ValidatedReader; +pub use crate::space::available_space; pub mod cachedir; pub mod link; mod locked_file; mod path; mod read; +mod space; pub mod which; /// Attempt to check if the two paths refer to the same file. diff --git a/crates/uv-fs/src/space.rs b/crates/uv-fs/src/space.rs new file mode 100644 index 00000000000..bc4eea67a14 --- /dev/null +++ b/crates/uv-fs/src/space.rs @@ -0,0 +1,47 @@ +use std::io; +use std::path::Path; + +#[cfg(windows)] +use std::os::windows::ffi::OsStrExt; + +#[cfg(windows)] +use windows::Win32::Storage::FileSystem::GetDiskFreeSpaceExW; +#[cfg(windows)] +use windows::core::PCWSTR; + +/// Return the available space on the filesystem containing `path`, in bytes. +#[cfg_attr(windows, expect(unsafe_code))] +pub fn available_space(path: &Path) -> io::Result { + #[cfg(unix)] + { + let statistics = rustix::fs::statvfs(path)?; + Ok(statistics.f_bavail.saturating_mul(statistics.f_frsize)) + } + + #[cfg(windows)] + { + let path = path + .as_os_str() + .encode_wide() + .chain(Some(0)) + .collect::>(); + let mut available = 0; + + // SAFETY: `path` is null-terminated and remains valid for the duration of the call, and + // `available` is a valid, writable pointer to the requested output value. + unsafe { + GetDiskFreeSpaceExW(PCWSTR(path.as_ptr()), Some(&raw mut available), None, None)?; + } + + Ok(available) + } + + #[cfg(not(any(unix, windows)))] + { + let _ = path; + Err(io::Error::new( + io::ErrorKind::Unsupported, + "filesystem space measurement is unsupported on this platform", + )) + } +} diff --git a/crates/uv/src/commands/cache_clean.rs b/crates/uv/src/commands/cache_clean.rs index 583b1fe698f..541e2a5eae4 100644 --- a/crates/uv/src/commands/cache_clean.rs +++ b/crates/uv/src/commands/cache_clean.rs @@ -43,6 +43,10 @@ pub(crate) async fn cache_clean( } }; + // The cache root itself is removed when cleaning the entire cache, so measure its parent. + let filesystem_path = cache.root().parent().unwrap_or(cache.root()).to_path_buf(); + let available_before = uv_fs::available_space(&filesystem_path).ok(); + let summary = if packages.is_empty() { writeln!( printer.stderr(), @@ -71,6 +75,12 @@ pub(crate) async fn cache_clean( summary }; + let reclaimed_bytes = available_before.and_then(|before| { + uv_fs::available_space(&filesystem_path) + .ok() + .map(|after| after.saturating_sub(before)) + }); + // Write a summary of the number of files and directories removed. match (summary.num_files, summary.num_dirs) { (0, 0) => { @@ -92,13 +102,27 @@ pub(crate) async fn cache_clean( // If any, write a summary of the total byte count removed. if summary.total_bytes > 0 { - let bytes = if summary.total_bytes < 1024 { - format!("{}B", summary.total_bytes) - } else { - let (bytes, unit) = human_readable_bytes(summary.total_bytes); - format!("{bytes:.1}{unit}") + let format_bytes = |bytes| { + if bytes < 1024 { + format!("{bytes}B") + } else { + let (bytes, unit) = human_readable_bytes(bytes); + format!("{bytes:.1}{unit}") + } }; - write!(printer.stderr(), " ({})", bytes.green())?; + let bytes = format_bytes(summary.total_bytes); + + if let Some(reclaimed_bytes) = reclaimed_bytes { + let reclaimed = format_bytes(reclaimed_bytes); + write!( + printer.stderr(), + " ({}, {} reclaimed)", + bytes.green(), + reclaimed.green() + )?; + } else { + write!(printer.stderr(), " ({})", bytes.green())?; + } } writeln!(printer.stderr())?; diff --git a/crates/uv/src/commands/cache_prune.rs b/crates/uv/src/commands/cache_prune.rs index 7c7c371c9f6..fa716c6c890 100644 --- a/crates/uv/src/commands/cache_prune.rs +++ b/crates/uv/src/commands/cache_prune.rs @@ -47,6 +47,7 @@ pub(crate) async fn cache_prune( cache.root().user_display().cyan() )?; + let available_before = uv_fs::available_space(cache.root()).ok(); let mut summary = Removal::default(); // Prune the source distribution cache, which is tightly coupled to the builder crate. @@ -58,6 +59,12 @@ pub(crate) async fn cache_prune( .prune(ci) .with_context(|| format!("Failed to prune cache at: {}", cache.root().user_display()))?; + let reclaimed_bytes = available_before.and_then(|before| { + uv_fs::available_space(cache.root()) + .ok() + .map(|after| after.saturating_sub(before)) + }); + // Write a summary of the number of files and directories removed. match (summary.num_files, summary.num_dirs) { (0, 0) => { @@ -79,13 +86,27 @@ pub(crate) async fn cache_prune( // If any, write a summary of the total byte count removed. if summary.total_bytes > 0 { - let bytes = if summary.total_bytes < 1024 { - format!("{}B", summary.total_bytes) - } else { - let (bytes, unit) = human_readable_bytes(summary.total_bytes); - format!("{bytes:.1}{unit}") + let format_bytes = |bytes| { + if bytes < 1024 { + format!("{bytes}B") + } else { + let (bytes, unit) = human_readable_bytes(bytes); + format!("{bytes:.1}{unit}") + } }; - write!(printer.stderr(), " ({})", bytes.green())?; + let bytes = format_bytes(summary.total_bytes); + + if let Some(reclaimed_bytes) = reclaimed_bytes { + let reclaimed = format_bytes(reclaimed_bytes); + write!( + printer.stderr(), + " ({}, {} reclaimed)", + bytes.green(), + reclaimed.green() + )?; + } else { + write!(printer.stderr(), " ({})", bytes.green())?; + } } writeln!(printer.stderr())?; diff --git a/crates/uv/tests/build/cache_clean.rs b/crates/uv/tests/build/cache_clean.rs index 9d68c84afde..0d7301cd9e7 100644 --- a/crates/uv/tests/build/cache_clean.rs +++ b/crates/uv/tests/build/cache_clean.rs @@ -3,6 +3,7 @@ use assert_cmd::prelude::*; use assert_fs::prelude::*; use uv_cache::Cache; +use uv_fs::link::{LinkMode, LinkOptions, link_dir}; use uv_static::EnvVars; use uv_test::uv_snapshot; @@ -28,12 +29,62 @@ fn clean_all() -> Result<()> { DEBUG Searching for user configuration in: `[UV_USER_CONFIG_DIR]/uv.toml` DEBUG uv [VERSION] ([COMMIT] DATE) Clearing cache at: [CACHE_DIR]/ - Removed [N] files ([SIZE]) + Removed [N] files ([SIZE], [SIZE] reclaimed) "); Ok(()) } +/// `cache clean` should report reclaimed space separately for files retained by hardlinks. +#[test] +fn clean_all_hardlinked_file() -> Result<()> { + let context = uv_test::test_context!("3.12").with_filtered_counts(); + let retained = context.temp_dir.child("retained.bin"); + retained.write_binary(&vec![42; 1024 * 1024])?; + + let cached = context.cache_dir.child("hardlinked.bin"); + fs_err::hard_link(&retained, &cached)?; + + uv_snapshot!(context.filters(), context.clean(), @" + exit_code: 0 (success) + ----- stderr ----- + Clearing cache at: [CACHE_DIR]/ + Removed [N] files ([SIZE], [SIZE] reclaimed) + "); + + assert!(retained.is_file()); + assert_eq!(fs_err::metadata(retained)?.len(), 1024 * 1024); + + Ok(()) +} + +/// `cache clean` should report reclaimed space separately for copy-on-write clones. +#[test] +fn clean_all_cloned_file() -> Result<()> { + let context = uv_test::test_context!("3.12").with_filtered_counts(); + let retained = context.temp_dir.child("retained"); + retained.create_dir_all()?; + let original = retained.child("original.bin"); + original.write_binary(&vec![42; 1024 * 1024])?; + + let cached = context.cache_dir.child("cloned"); + if link_dir(&retained, &cached, &LinkOptions::new(LinkMode::Clone))? != LinkMode::Clone { + return Ok(()); + } + + uv_snapshot!(context.filters(), context.clean(), @" + exit_code: 0 (success) + ----- stderr ----- + Clearing cache at: [CACHE_DIR]/ + Removed [N] files ([SIZE], [SIZE] reclaimed) + "); + + assert!(original.is_file()); + assert_eq!(fs_err::metadata(original)?.len(), 1024 * 1024); + + Ok(()) +} + /// `cache clear` should behave as an alias of `cache clean`. #[test] fn clear_all_alias() -> Result<()> { @@ -58,7 +109,7 @@ fn clear_all_alias() -> Result<()> { DEBUG Searching for user configuration in: `[UV_USER_CONFIG_DIR]/uv.toml` DEBUG uv [VERSION] ([COMMIT] DATE) Clearing cache at: [CACHE_DIR]/ - Removed [N] files ([SIZE]) + Removed [N] files ([SIZE], [SIZE] reclaimed) "); Ok(()) @@ -85,7 +136,7 @@ async fn clean_force() -> Result<()> { DEBUG Searching for user configuration in: `[UV_USER_CONFIG_DIR]/uv.toml` DEBUG uv [VERSION] ([COMMIT] DATE) Clearing cache at: [CACHE_DIR]/ - Removed [N] files ([SIZE]) + Removed [N] files ([SIZE], [SIZE] reclaimed) "); // Install a requirement, to re-populate the cache. @@ -107,7 +158,7 @@ async fn clean_force() -> Result<()> { DEBUG Lock is busy for `[CACHE_DIR]/` DEBUG Cache is currently in use, proceeding due to `--force` Clearing cache at: [CACHE_DIR]/ - Removed [N] files ([SIZE]) + Removed [N] files ([SIZE], [SIZE] reclaimed) "); Ok(()) @@ -159,7 +210,7 @@ fn clean_package_pypi() -> Result<()> { DEBUG Searching for user configuration in: `[UV_USER_CONFIG_DIR]/uv.toml` DEBUG uv [VERSION] ([COMMIT] DATE) DEBUG Removing dangling cache entry: [CACHE_DIR]/archive-v0/[ENTRY] - Removed [N] files ([SIZE]) + Removed [N] files ([SIZE], [SIZE] reclaimed) "); // Assert that the `.rkyv` file is removed for `iniconfig`. @@ -230,7 +281,7 @@ fn clean_package_index() -> Result<()> { DEBUG Searching for user configuration in: `[UV_USER_CONFIG_DIR]/uv.toml` DEBUG uv [VERSION] ([COMMIT] DATE) DEBUG Removing dangling cache entry: [CACHE_DIR]/archive-v0/[ENTRY] - Removed [N] files ([SIZE]) + Removed [N] files ([SIZE], [SIZE] reclaimed) "); // Assert that the `.rkyv` file is removed for `iniconfig`. @@ -267,7 +318,7 @@ fn clean_package_does_not_follow_symlinks() -> Result<()> { uv_snapshot!(context.filters(), context.clean().arg("demo"), @" exit_code: 0 (success) ----- stderr ----- - Removed 3 files ([SIZE]) + Removed 3 files ([SIZE], [SIZE] reclaimed) "); assert!(victim_dir.is_dir()); diff --git a/crates/uv/tests/build/cache_prune.rs b/crates/uv/tests/build/cache_prune.rs index ffe2fca906f..8daacfab6c5 100644 --- a/crates/uv/tests/build/cache_prune.rs +++ b/crates/uv/tests/build/cache_prune.rs @@ -40,6 +40,30 @@ fn prune_no_op() -> Result<()> { Ok(()) } +/// `cache prune` should report reclaimed space separately for files retained by hardlinks. +#[test] +fn prune_hardlinked_file() -> Result<()> { + let context = uv_test::test_context!("3.12"); + let retained = context.temp_dir.child("retained.bin"); + retained.write_binary(&vec![42; 1024 * 1024])?; + + let stale = context.cache_dir.child("stale-v0"); + stale.create_dir_all()?; + fs_err::hard_link(&retained, stale.child("hardlinked.bin"))?; + + uv_snapshot!(context.filters(), context.prune(), @" + exit_code: 0 (success) + ----- stderr ----- + Pruning cache at: [CACHE_DIR]/ + Removed 1 file ([SIZE], [SIZE] reclaimed) + "); + + assert!(retained.is_file()); + assert_eq!(fs_err::metadata(retained)?.len(), 1024 * 1024); + + Ok(()) +} + /// `cache prune` should remove any stale top-level directories from the cache. #[test] fn prune_stale_directory() -> Result<()> { @@ -152,7 +176,7 @@ fn prune_cached_env() { Pruning cache at: [CACHE_DIR]/ DEBUG Removing cached environment: [CACHE_DIR]/environments-v2/[ENTRY] DEBUG Removing dangling cache archive: [CACHE_DIR]/archive-v0/[ENTRY] - Removed [N] files ([SIZE]) + Removed [N] files ([SIZE], [SIZE] reclaimed) "); } @@ -194,7 +218,7 @@ fn prune_stale_symlink() -> Result<()> { DEBUG uv [VERSION] ([COMMIT] DATE) Pruning cache at: [CACHE_DIR]/ DEBUG Removing dangling cache archive: [CACHE_DIR]/archive-v0/[ENTRY] - Removed 44 files ([SIZE]) + Removed 44 files ([SIZE], [SIZE] reclaimed) "); Ok(()) @@ -294,7 +318,7 @@ fn prune_unzipped() -> Result<()> { exit_code: 0 (success) ----- stderr ----- Pruning cache at: [CACHE_DIR]/ - Removed [N] files ([SIZE]) + Removed [N] files ([SIZE], [SIZE] reclaimed) "); context.venv().arg("--clear").assert().success(); @@ -412,7 +436,7 @@ fn prune_stale_revision() -> Result<()> { Pruning cache at: [CACHE_DIR]/ DEBUG Removing dangling source revision: [CACHE_DIR]/sdists-v9/[ENTRY] DEBUG Removing dangling cache archive: [CACHE_DIR]/archive-v0/[ENTRY] - Removed [N] files ([SIZE]) + Removed [N] files ([SIZE], [SIZE] reclaimed) "); // Uninstall and reinstall the package. We should use the cached version. diff --git a/crates/uv/tests/pip/pip_sync.rs b/crates/uv/tests/pip/pip_sync.rs index 8b60ede4397..721a1545296 100644 --- a/crates/uv/tests/pip/pip_sync.rs +++ b/crates/uv/tests/pip/pip_sync.rs @@ -1454,7 +1454,7 @@ fn install_url_source_dist_cached() -> Result<()> { context.clean().arg("source_distribution"), @" exit_code: 0 (success) ----- stderr ----- - Removed [N] files ([SIZE]) + Removed [N] files ([SIZE], [SIZE] reclaimed) " ); @@ -1613,7 +1613,7 @@ fn install_registry_source_dist_cached() -> Result<()> { .arg("source_distribution"), @" exit_code: 0 (success) ----- stderr ----- - Removed [N] files ([SIZE]) + Removed [N] files ([SIZE], [SIZE] reclaimed) " ); @@ -1701,7 +1701,7 @@ fn install_path_source_dist_cached() -> Result<()> { context.clean().arg("source-distribution"), @" exit_code: 0 (success) ----- stderr ----- - Removed [N] files ([SIZE]) + Removed [N] files ([SIZE], [SIZE] reclaimed) " ); @@ -1783,7 +1783,7 @@ fn install_path_built_dist_cached() -> Result<()> { context.clean().arg("tomli"), @" exit_code: 0 (success) ----- stderr ----- - Removed [N] files ([SIZE]) + Removed [N] files ([SIZE], [SIZE] reclaimed) " ); @@ -1863,7 +1863,7 @@ fn install_url_built_dist_cached() -> Result<()> { context.clean().arg("tqdm"), @" exit_code: 0 (success) ----- stderr ----- - Removed [N] files ([SIZE]) + Removed [N] files ([SIZE], [SIZE] reclaimed) " ); From 778e95ab4b2d95c791d0d3898a985faa6eaf31e1 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Mon, 3 Aug 2026 15:05:57 -0400 Subject: [PATCH 02/13] Show only reclaimed disk space during cache cleanup --- crates/uv/src/commands/cache_clean.rs | 29 ++++++++------------------- crates/uv/src/commands/cache_prune.rs | 29 ++++++++------------------- crates/uv/tests/build/cache_clean.rs | 18 ++++++++--------- crates/uv/tests/build/cache_prune.rs | 10 ++++----- crates/uv/tests/pip/pip_sync.rs | 10 ++++----- 5 files changed, 35 insertions(+), 61 deletions(-) diff --git a/crates/uv/src/commands/cache_clean.rs b/crates/uv/src/commands/cache_clean.rs index 541e2a5eae4..bc17b4b4e4f 100644 --- a/crates/uv/src/commands/cache_clean.rs +++ b/crates/uv/src/commands/cache_clean.rs @@ -100,29 +100,16 @@ pub(crate) async fn cache_clean( } } - // If any, write a summary of the total byte count removed. + // If any, report the reclaimed space, falling back to the apparent removed size. if summary.total_bytes > 0 { - let format_bytes = |bytes| { - if bytes < 1024 { - format!("{bytes}B") - } else { - let (bytes, unit) = human_readable_bytes(bytes); - format!("{bytes:.1}{unit}") - } - }; - let bytes = format_bytes(summary.total_bytes); - - if let Some(reclaimed_bytes) = reclaimed_bytes { - let reclaimed = format_bytes(reclaimed_bytes); - write!( - printer.stderr(), - " ({}, {} reclaimed)", - bytes.green(), - reclaimed.green() - )?; + let total_bytes = reclaimed_bytes.unwrap_or(summary.total_bytes); + let bytes = if total_bytes < 1024 { + format!("{total_bytes}B") } else { - write!(printer.stderr(), " ({})", bytes.green())?; - } + let (bytes, unit) = human_readable_bytes(total_bytes); + format!("{bytes:.1}{unit}") + }; + write!(printer.stderr(), " ({})", bytes.green())?; } writeln!(printer.stderr())?; diff --git a/crates/uv/src/commands/cache_prune.rs b/crates/uv/src/commands/cache_prune.rs index fa716c6c890..32de5387250 100644 --- a/crates/uv/src/commands/cache_prune.rs +++ b/crates/uv/src/commands/cache_prune.rs @@ -84,29 +84,16 @@ pub(crate) async fn cache_prune( } } - // If any, write a summary of the total byte count removed. + // If any, report the reclaimed space, falling back to the apparent removed size. if summary.total_bytes > 0 { - let format_bytes = |bytes| { - if bytes < 1024 { - format!("{bytes}B") - } else { - let (bytes, unit) = human_readable_bytes(bytes); - format!("{bytes:.1}{unit}") - } - }; - let bytes = format_bytes(summary.total_bytes); - - if let Some(reclaimed_bytes) = reclaimed_bytes { - let reclaimed = format_bytes(reclaimed_bytes); - write!( - printer.stderr(), - " ({}, {} reclaimed)", - bytes.green(), - reclaimed.green() - )?; + let total_bytes = reclaimed_bytes.unwrap_or(summary.total_bytes); + let bytes = if total_bytes < 1024 { + format!("{total_bytes}B") } else { - write!(printer.stderr(), " ({})", bytes.green())?; - } + let (bytes, unit) = human_readable_bytes(total_bytes); + format!("{bytes:.1}{unit}") + }; + write!(printer.stderr(), " ({})", bytes.green())?; } writeln!(printer.stderr())?; diff --git a/crates/uv/tests/build/cache_clean.rs b/crates/uv/tests/build/cache_clean.rs index 0d7301cd9e7..e6f14528d4b 100644 --- a/crates/uv/tests/build/cache_clean.rs +++ b/crates/uv/tests/build/cache_clean.rs @@ -29,7 +29,7 @@ fn clean_all() -> Result<()> { DEBUG Searching for user configuration in: `[UV_USER_CONFIG_DIR]/uv.toml` DEBUG uv [VERSION] ([COMMIT] DATE) Clearing cache at: [CACHE_DIR]/ - Removed [N] files ([SIZE], [SIZE] reclaimed) + Removed [N] files ([SIZE]) "); Ok(()) @@ -49,7 +49,7 @@ fn clean_all_hardlinked_file() -> Result<()> { exit_code: 0 (success) ----- stderr ----- Clearing cache at: [CACHE_DIR]/ - Removed [N] files ([SIZE], [SIZE] reclaimed) + Removed [N] files ([SIZE]) "); assert!(retained.is_file()); @@ -76,7 +76,7 @@ fn clean_all_cloned_file() -> Result<()> { exit_code: 0 (success) ----- stderr ----- Clearing cache at: [CACHE_DIR]/ - Removed [N] files ([SIZE], [SIZE] reclaimed) + Removed [N] files ([SIZE]) "); assert!(original.is_file()); @@ -109,7 +109,7 @@ fn clear_all_alias() -> Result<()> { DEBUG Searching for user configuration in: `[UV_USER_CONFIG_DIR]/uv.toml` DEBUG uv [VERSION] ([COMMIT] DATE) Clearing cache at: [CACHE_DIR]/ - Removed [N] files ([SIZE], [SIZE] reclaimed) + Removed [N] files ([SIZE]) "); Ok(()) @@ -136,7 +136,7 @@ async fn clean_force() -> Result<()> { DEBUG Searching for user configuration in: `[UV_USER_CONFIG_DIR]/uv.toml` DEBUG uv [VERSION] ([COMMIT] DATE) Clearing cache at: [CACHE_DIR]/ - Removed [N] files ([SIZE], [SIZE] reclaimed) + Removed [N] files ([SIZE]) "); // Install a requirement, to re-populate the cache. @@ -158,7 +158,7 @@ async fn clean_force() -> Result<()> { DEBUG Lock is busy for `[CACHE_DIR]/` DEBUG Cache is currently in use, proceeding due to `--force` Clearing cache at: [CACHE_DIR]/ - Removed [N] files ([SIZE], [SIZE] reclaimed) + Removed [N] files ([SIZE]) "); Ok(()) @@ -210,7 +210,7 @@ fn clean_package_pypi() -> Result<()> { DEBUG Searching for user configuration in: `[UV_USER_CONFIG_DIR]/uv.toml` DEBUG uv [VERSION] ([COMMIT] DATE) DEBUG Removing dangling cache entry: [CACHE_DIR]/archive-v0/[ENTRY] - Removed [N] files ([SIZE], [SIZE] reclaimed) + Removed [N] files ([SIZE]) "); // Assert that the `.rkyv` file is removed for `iniconfig`. @@ -281,7 +281,7 @@ fn clean_package_index() -> Result<()> { DEBUG Searching for user configuration in: `[UV_USER_CONFIG_DIR]/uv.toml` DEBUG uv [VERSION] ([COMMIT] DATE) DEBUG Removing dangling cache entry: [CACHE_DIR]/archive-v0/[ENTRY] - Removed [N] files ([SIZE], [SIZE] reclaimed) + Removed [N] files ([SIZE]) "); // Assert that the `.rkyv` file is removed for `iniconfig`. @@ -318,7 +318,7 @@ fn clean_package_does_not_follow_symlinks() -> Result<()> { uv_snapshot!(context.filters(), context.clean().arg("demo"), @" exit_code: 0 (success) ----- stderr ----- - Removed 3 files ([SIZE], [SIZE] reclaimed) + Removed 3 files ([SIZE]) "); assert!(victim_dir.is_dir()); diff --git a/crates/uv/tests/build/cache_prune.rs b/crates/uv/tests/build/cache_prune.rs index 8daacfab6c5..31c1ff2d17e 100644 --- a/crates/uv/tests/build/cache_prune.rs +++ b/crates/uv/tests/build/cache_prune.rs @@ -55,7 +55,7 @@ fn prune_hardlinked_file() -> Result<()> { exit_code: 0 (success) ----- stderr ----- Pruning cache at: [CACHE_DIR]/ - Removed 1 file ([SIZE], [SIZE] reclaimed) + Removed 1 file ([SIZE]) "); assert!(retained.is_file()); @@ -176,7 +176,7 @@ fn prune_cached_env() { Pruning cache at: [CACHE_DIR]/ DEBUG Removing cached environment: [CACHE_DIR]/environments-v2/[ENTRY] DEBUG Removing dangling cache archive: [CACHE_DIR]/archive-v0/[ENTRY] - Removed [N] files ([SIZE], [SIZE] reclaimed) + Removed [N] files ([SIZE]) "); } @@ -218,7 +218,7 @@ fn prune_stale_symlink() -> Result<()> { DEBUG uv [VERSION] ([COMMIT] DATE) Pruning cache at: [CACHE_DIR]/ DEBUG Removing dangling cache archive: [CACHE_DIR]/archive-v0/[ENTRY] - Removed 44 files ([SIZE], [SIZE] reclaimed) + Removed 44 files ([SIZE]) "); Ok(()) @@ -318,7 +318,7 @@ fn prune_unzipped() -> Result<()> { exit_code: 0 (success) ----- stderr ----- Pruning cache at: [CACHE_DIR]/ - Removed [N] files ([SIZE], [SIZE] reclaimed) + Removed [N] files ([SIZE]) "); context.venv().arg("--clear").assert().success(); @@ -436,7 +436,7 @@ fn prune_stale_revision() -> Result<()> { Pruning cache at: [CACHE_DIR]/ DEBUG Removing dangling source revision: [CACHE_DIR]/sdists-v9/[ENTRY] DEBUG Removing dangling cache archive: [CACHE_DIR]/archive-v0/[ENTRY] - Removed [N] files ([SIZE], [SIZE] reclaimed) + Removed [N] files ([SIZE]) "); // Uninstall and reinstall the package. We should use the cached version. diff --git a/crates/uv/tests/pip/pip_sync.rs b/crates/uv/tests/pip/pip_sync.rs index 721a1545296..8b60ede4397 100644 --- a/crates/uv/tests/pip/pip_sync.rs +++ b/crates/uv/tests/pip/pip_sync.rs @@ -1454,7 +1454,7 @@ fn install_url_source_dist_cached() -> Result<()> { context.clean().arg("source_distribution"), @" exit_code: 0 (success) ----- stderr ----- - Removed [N] files ([SIZE], [SIZE] reclaimed) + Removed [N] files ([SIZE]) " ); @@ -1613,7 +1613,7 @@ fn install_registry_source_dist_cached() -> Result<()> { .arg("source_distribution"), @" exit_code: 0 (success) ----- stderr ----- - Removed [N] files ([SIZE], [SIZE] reclaimed) + Removed [N] files ([SIZE]) " ); @@ -1701,7 +1701,7 @@ fn install_path_source_dist_cached() -> Result<()> { context.clean().arg("source-distribution"), @" exit_code: 0 (success) ----- stderr ----- - Removed [N] files ([SIZE], [SIZE] reclaimed) + Removed [N] files ([SIZE]) " ); @@ -1783,7 +1783,7 @@ fn install_path_built_dist_cached() -> Result<()> { context.clean().arg("tomli"), @" exit_code: 0 (success) ----- stderr ----- - Removed [N] files ([SIZE], [SIZE] reclaimed) + Removed [N] files ([SIZE]) " ); @@ -1863,7 +1863,7 @@ fn install_url_built_dist_cached() -> Result<()> { context.clean().arg("tqdm"), @" exit_code: 0 (success) ----- stderr ----- - Removed [N] files ([SIZE], [SIZE] reclaimed) + Removed [N] files ([SIZE]) " ); From c0082907dffbffc0606c8ba23b462adae5fe6740 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Mon, 3 Aug 2026 15:26:59 -0400 Subject: [PATCH 03/13] Gate reclaimed cache space behind a preview feature --- .../generate_preview_features_reference.rs | 1 + crates/uv-preview/src/lib.rs | 2 ++ crates/uv/src/commands/cache_clean.rs | 7 +++++- crates/uv/src/commands/cache_prune.rs | 7 +++++- crates/uv/src/lib.rs | 4 ++-- crates/uv/tests/build/cache_clean.rs | 24 +++++++++++++++---- crates/uv/tests/build/cache_prune.rs | 20 ++++++++++++++-- crates/uv/tests/sync/show_settings.rs | 1 + docs/concepts/cache.md | 8 +++++++ uv.schema.json | 1 + 10 files changed, 65 insertions(+), 10 deletions(-) diff --git a/crates/uv-dev/src/generate_preview_features_reference.rs b/crates/uv-dev/src/generate_preview_features_reference.rs index d1dcb927b3e..23b29fcd0ff 100644 --- a/crates/uv-dev/src/generate_preview_features_reference.rs +++ b/crates/uv-dev/src/generate_preview_features_reference.rs @@ -111,6 +111,7 @@ mod tests { - `audit-command`: Allows using `uv audit`. - `auth-helper`: Allows using `uv auth helper` as a credential helper for external tools. - `azure-endpoint`: Allows signing requests to Azure Blob Storage endpoints with Azure credentials. + - `cache-reclaimed-space`: Reports the disk space reclaimed by cache cleanup, accounting for hardlinks and copy-on-write clones. - `cache-size`: Allows using `uv cache size`. - `centralized-project-envs`: Stores [project virtual environments](./projects/layout.md#centralized-project-environments) in the uv cache. diff --git a/crates/uv-preview/src/lib.rs b/crates/uv-preview/src/lib.rs index 0eda60a7962..f353b3b9996 100644 --- a/crates/uv-preview/src/lib.rs +++ b/crates/uv-preview/src/lib.rs @@ -253,6 +253,8 @@ pub enum PreviewFeature { S3Endpoint, /// Allows using `uv cache size`. CacheSize, + /// Reports the disk space reclaimed by cache cleanup, accounting for hardlinks and copy-on-write clones. + CacheReclaimedSpace, /// Rejects the deprecated `--project` option in `uv init`. InitProjectFlag, /// Allows using `uv workspace metadata`. diff --git a/crates/uv/src/commands/cache_clean.rs b/crates/uv/src/commands/cache_clean.rs index bc17b4b4e4f..a7c79422f6e 100644 --- a/crates/uv/src/commands/cache_clean.rs +++ b/crates/uv/src/commands/cache_clean.rs @@ -7,6 +7,7 @@ use tracing::debug; use uv_cache::{Cache, Removal}; use uv_fs::Simplified; use uv_normalize::PackageName; +use uv_preview::{Preview, PreviewFeature}; use crate::commands::reporters::{CleaningDirectoryReporter, CleaningPackageReporter}; use crate::commands::{ExitStatus, human_readable_bytes}; @@ -18,6 +19,7 @@ pub(crate) async fn cache_clean( force: bool, cache: Cache, printer: Printer, + preview: Preview, ) -> Result { if !cache.root().exists() { writeln!( @@ -45,7 +47,10 @@ pub(crate) async fn cache_clean( // The cache root itself is removed when cleaning the entire cache, so measure its parent. let filesystem_path = cache.root().parent().unwrap_or(cache.root()).to_path_buf(); - let available_before = uv_fs::available_space(&filesystem_path).ok(); + let available_before = preview + .is_enabled(PreviewFeature::CacheReclaimedSpace) + .then(|| uv_fs::available_space(&filesystem_path).ok()) + .flatten(); let summary = if packages.is_empty() { writeln!( diff --git a/crates/uv/src/commands/cache_prune.rs b/crates/uv/src/commands/cache_prune.rs index 32de5387250..9b740999121 100644 --- a/crates/uv/src/commands/cache_prune.rs +++ b/crates/uv/src/commands/cache_prune.rs @@ -6,6 +6,7 @@ use tracing::debug; use uv_cache::{Cache, Removal}; use uv_fs::Simplified; +use uv_preview::{Preview, PreviewFeature}; use crate::commands::{ExitStatus, human_readable_bytes}; use crate::printer::Printer; @@ -16,6 +17,7 @@ pub(crate) async fn cache_prune( force: bool, cache: Cache, printer: Printer, + preview: Preview, ) -> Result { if !cache.root().exists() { writeln!( @@ -47,7 +49,10 @@ pub(crate) async fn cache_prune( cache.root().user_display().cyan() )?; - let available_before = uv_fs::available_space(cache.root()).ok(); + let available_before = preview + .is_enabled(PreviewFeature::CacheReclaimedSpace) + .then(|| uv_fs::available_space(cache.root()).ok()) + .flatten(); let mut summary = Removal::default(); // Prune the source distribution cache, which is tightly coupled to the builder crate. diff --git a/crates/uv/src/lib.rs b/crates/uv/src/lib.rs index a8371114396..f631342cbfc 100644 --- a/crates/uv/src/lib.rs +++ b/crates/uv/src/lib.rs @@ -1279,13 +1279,13 @@ pub async fn run(cli: Cli, global_initialization: GlobalInitialization) -> Resul }) | Commands::Clean(args) => { show_settings!(args); - commands::cache_clean(&args.package, args.force, cache, printer).await + commands::cache_clean(&args.package, args.force, cache, printer, globals.preview).await } Commands::Cache(CacheNamespace { command: CacheCommand::Prune(args), }) => { show_settings!(args); - commands::cache_prune(args.ci, args.force, cache, printer).await + commands::cache_prune(args.ci, args.force, cache, printer, globals.preview).await } Commands::Cache(CacheNamespace { command: CacheCommand::Dir, diff --git a/crates/uv/tests/build/cache_clean.rs b/crates/uv/tests/build/cache_clean.rs index e6f14528d4b..9b132fd8c41 100644 --- a/crates/uv/tests/build/cache_clean.rs +++ b/crates/uv/tests/build/cache_clean.rs @@ -35,7 +35,7 @@ fn clean_all() -> Result<()> { Ok(()) } -/// `cache clean` should report reclaimed space separately for files retained by hardlinks. +/// `cache clean` should report reclaimed space for hardlinks only when the preview is enabled. #[test] fn clean_all_hardlinked_file() -> Result<()> { let context = uv_test::test_context!("3.12").with_filtered_counts(); @@ -45,7 +45,23 @@ fn clean_all_hardlinked_file() -> Result<()> { let cached = context.cache_dir.child("hardlinked.bin"); fs_err::hard_link(&retained, &cached)?; - uv_snapshot!(context.filters(), context.clean(), @" + let filters: Vec<_> = context + .filters() + .into_iter() + .filter(|(_, replacement)| *replacement != "$1[SIZE]") + .collect(); + + uv_snapshot!(&filters, context.clean(), @" + exit_code: 0 (success) + ----- stderr ----- + Clearing cache at: [CACHE_DIR]/ + Removed [N] files (1.0MiB) + "); + + context.cache_dir.create_dir_all()?; + fs_err::hard_link(&retained, &cached)?; + + uv_snapshot!(context.filters(), context.clean().arg("--preview-features").arg("cache-reclaimed-space"), @" exit_code: 0 (success) ----- stderr ----- Clearing cache at: [CACHE_DIR]/ @@ -58,7 +74,7 @@ fn clean_all_hardlinked_file() -> Result<()> { Ok(()) } -/// `cache clean` should report reclaimed space separately for copy-on-write clones. +/// `cache clean` should report reclaimed space for copy-on-write clones in preview mode. #[test] fn clean_all_cloned_file() -> Result<()> { let context = uv_test::test_context!("3.12").with_filtered_counts(); @@ -72,7 +88,7 @@ fn clean_all_cloned_file() -> Result<()> { return Ok(()); } - uv_snapshot!(context.filters(), context.clean(), @" + uv_snapshot!(context.filters(), context.clean().arg("--preview"), @" exit_code: 0 (success) ----- stderr ----- Clearing cache at: [CACHE_DIR]/ diff --git a/crates/uv/tests/build/cache_prune.rs b/crates/uv/tests/build/cache_prune.rs index 31c1ff2d17e..fd67fc1ba9d 100644 --- a/crates/uv/tests/build/cache_prune.rs +++ b/crates/uv/tests/build/cache_prune.rs @@ -40,7 +40,7 @@ fn prune_no_op() -> Result<()> { Ok(()) } -/// `cache prune` should report reclaimed space separately for files retained by hardlinks. +/// `cache prune` should report reclaimed space for hardlinks only when the preview is enabled. #[test] fn prune_hardlinked_file() -> Result<()> { let context = uv_test::test_context!("3.12"); @@ -51,7 +51,23 @@ fn prune_hardlinked_file() -> Result<()> { stale.create_dir_all()?; fs_err::hard_link(&retained, stale.child("hardlinked.bin"))?; - uv_snapshot!(context.filters(), context.prune(), @" + let filters: Vec<_> = context + .filters() + .into_iter() + .filter(|(_, replacement)| *replacement != "$1[SIZE]") + .collect(); + + uv_snapshot!(&filters, context.prune(), @" + exit_code: 0 (success) + ----- stderr ----- + Pruning cache at: [CACHE_DIR]/ + Removed 1 file (1.0MiB) + "); + + stale.create_dir_all()?; + fs_err::hard_link(&retained, stale.child("hardlinked.bin"))?; + + uv_snapshot!(context.filters(), context.prune().arg("--preview-features").arg("cache-reclaimed-space"), @" exit_code: 0 (success) ----- stderr ----- Pruning cache at: [CACHE_DIR]/ diff --git a/crates/uv/tests/sync/show_settings.rs b/crates/uv/tests/sync/show_settings.rs index 4ab2c86542c..35851e13e81 100644 --- a/crates/uv/tests/sync/show_settings.rs +++ b/crates/uv/tests/sync/show_settings.rs @@ -3386,6 +3386,7 @@ fn preview_features() { + NativeAuth, + S3Endpoint, + CacheSize, + + CacheReclaimedSpace, + InitProjectFlag, + WorkspaceMetadata, + WorkspaceDir, diff --git a/docs/concepts/cache.md b/docs/concepts/cache.md index 7f26eff6027..d8f9e449d83 100644 --- a/docs/concepts/cache.md +++ b/docs/concepts/cache.md @@ -144,6 +144,14 @@ uv provides a few different mechanisms for removing entries from the cache: longer necessary and can be safely removed. Centralized project environments are recreated as needed. `uv cache prune` is safe to run periodically, to keep the cache directory clean. +By default, cache cleanup reports the apparent size of removed entries. Enable the +`cache-reclaimed-space` [preview feature](./preview.md) to instead report the actual disk space +reclaimed, accounting for hardlinks and copy-on-write clones: + +```console +$ uv cache clean --preview-features cache-reclaimed-space +``` + uv blocks cache-modifying operations while other uv commands are running. By default, those `uv cache` commands have a 5 min timeout waiting for other uv processes to terminate to avoid deadlocks. This timeout can be changed with diff --git a/uv.schema.json b/uv.schema.json index a46cb9ed7b2..3d68859e91e 100644 --- a/uv.schema.json +++ b/uv.schema.json @@ -1794,6 +1794,7 @@ "native-auth", "s3-endpoint", "cache-size", + "cache-reclaimed-space", "init-project-flag", "workspace-metadata", "workspace-dir", From e74c835ab5a9dabae6309fca2c7b25910e623e91 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Mon, 3 Aug 2026 16:22:16 -0400 Subject: [PATCH 04/13] Measure reclaimable cache storage per file --- Cargo.lock | 1 + Cargo.toml | 1 + crates/uv-cache/src/lib.rs | 78 ++++-- crates/uv-cache/src/removal.rs | 43 +++- crates/uv-distribution/src/source/mod.rs | 12 +- crates/uv-fs/Cargo.toml | 3 + crates/uv-fs/src/lib.rs | 2 +- crates/uv-fs/src/space.rs | 293 ++++++++++++++++++++++- crates/uv/src/commands/cache_clean.rs | 20 +- crates/uv/src/commands/cache_prune.rs | 20 +- crates/uv/tests/build/cache_clean.rs | 112 ++++++++- crates/uv/tests/build/cache_prune.rs | 22 +- 12 files changed, 546 insertions(+), 61 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2ba91d96710..41fd9751a62 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6614,6 +6614,7 @@ dependencies = [ "encoding_rs_io", "fs-err", "junction", + "libc", "path-slash", "percent-encoding", "reflink-copy", diff --git a/Cargo.toml b/Cargo.toml index 8feee2bf0be..004eb49e339 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -171,6 +171,7 @@ indoc = { version = "2.0.5" } itertools = { version = "0.14.0" } jiff = { version = "0.2.0", features = ["serde"] } junction = { version = "2.0.0" } +libc = { version = "0.2.0" } mailparse = { version = "0.16.0" } md-5 = { version = "0.11.0" } memchr = { version = "2.7.4" } diff --git a/crates/uv-cache/src/lib.rs b/crates/uv-cache/src/lib.rs index d5eaa50f9d5..416e1b977f6 100644 --- a/crates/uv-cache/src/lib.rs +++ b/crates/uv-cache/src/lib.rs @@ -174,6 +174,8 @@ pub struct Cache { /// Ensure that `uv cache` operations don't remove items from the cache that are used by another /// uv process. lock_file: Option>, + /// Inspect each removed file's actual sharing state when reporting reclaimed cache space. + measure_reclaimed_space: bool, } impl Cache { @@ -184,6 +186,7 @@ impl Cache { refresh: Refresh::None(Timestamp::now()), temp_dir: None, lock_file: None, + measure_reclaimed_space: false, } } @@ -195,6 +198,7 @@ impl Cache { refresh: Refresh::None(Timestamp::now()), temp_dir: Some(Arc::new(temp_dir)), lock_file: None, + measure_reclaimed_space: false, }) } @@ -204,6 +208,22 @@ impl Cache { Self { refresh, ..self } } + /// Enable per-file reclaimed-space accounting when the filesystem can support it. + #[must_use] + pub fn with_reclaimed_space(self, enabled: bool) -> Self { + let measure_reclaimed_space = + enabled && uv_fs::supports_reclaimable_space(&self.root).unwrap_or(false); + Self { + measure_reclaimed_space, + ..self + } + } + + /// Create an empty removal summary using the cache's configured accounting mode. + pub fn removal(&self) -> Removal { + Removal::new(self.measure_reclaimed_space) + } + /// Acquire a lock that allows removing entries from the cache. pub async fn with_exclusive_lock(self) -> Result { let Self { @@ -211,6 +231,7 @@ impl Cache { refresh, temp_dir, lock_file, + measure_reclaimed_space, } = self; // Release the existing lock, avoid deadlocks from a cloned cache. @@ -233,6 +254,7 @@ impl Cache { refresh, temp_dir, lock_file: Some(Arc::new(lock_file)), + measure_reclaimed_space, }) } @@ -245,6 +267,7 @@ impl Cache { refresh, temp_dir, lock_file, + measure_reclaimed_space, } = self; match LockedFile::acquire_no_wait( @@ -257,12 +280,14 @@ impl Cache { refresh, temp_dir, lock_file: Some(Arc::new(lock_file)), + measure_reclaimed_space, }), None => Err(Self { root, refresh, temp_dir, lock_file, + measure_reclaimed_space, }), } } @@ -521,7 +546,9 @@ impl Cache { /// Clear the cache, removing all entries. pub fn clear(self, reporter: Box) -> Result { // Remove everything but `.lock`, Windows does not allow removal of a locked file - let mut removal = Remover::new(reporter).rm_rf(&self.root, true)?; + let mut removal = Remover::new(reporter) + .with_reclaimed_space(self.measure_reclaimed_space) + .rm_rf(&self.root, true)?; let Self { root, lock_file, .. } = self; @@ -557,7 +584,7 @@ impl Cache { let references = self.find_archive_references()?; // Remove any entries for the package from the cache. - let mut summary = Removal::default(); + let mut summary = self.removal(); for bucket in CacheBucket::iter() { summary += bucket.remove(self, name)?; } @@ -574,7 +601,7 @@ impl Cache { for (target, references) in references { if target.starts_with(&archive_root) && references.iter().all(|path| !path.exists()) { debug!("Removing dangling cache entry: {}", target.display()); - summary += rm_rf(target)?; + summary += self.remove_path(target)?; } } @@ -583,7 +610,7 @@ impl Cache { /// Prune dangling cache entries and cached environments. pub fn prune(&self, ci: bool) -> Result { - let mut summary = Removal::default(); + let mut summary = self.removal(); // First, remove any top-level directories that are unused. These typically represent // outdated cache buckets (e.g., `wheels-v0`, when latest is `wheels-v1`). @@ -604,13 +631,13 @@ impl Cache { if CacheBucket::iter().all(|bucket| entry.file_name() != bucket.to_str()) { let path = entry.path(); debug!("Removing dangling cache bucket: {}", path.display()); - summary += rm_rf(path)?; + summary += self.remove_path(path)?; } } else { // If the file is not a marker file, remove it. let path = entry.path(); debug!("Removing dangling cache bucket: {}", path.display()); - summary += rm_rf(path)?; + summary += self.remove_path(path)?; } } @@ -622,7 +649,7 @@ impl Cache { let entry = entry?; let path = entry.path(); debug!("Removing cached environment: {}", path.display()); - summary += rm_rf(path)?; + summary += self.remove_path(path)?; } } Err(err) if err.kind() == io::ErrorKind::NotFound => (), @@ -639,7 +666,7 @@ impl Cache { let path = entry.path(); if path.is_dir() { debug!("Removing unzipped wheel entry: {}", path.display()); - summary += rm_rf(path)?; + summary += self.remove_path(path)?; } } } @@ -683,7 +710,7 @@ impl Cache { } debug!("Removing unzipped built wheel entry: {}", path.display()); - summary += rm_rf(path)?; + summary += self.remove_path(path)?; } } } @@ -700,7 +727,7 @@ impl Cache { let target = fs_err::canonicalize(&path)?; if !references.contains_key(&target) { debug!("Removing dangling cache archive: {}", path.display()); - summary += rm_rf(path)?; + summary += self.remove_path(path)?; } } } @@ -711,6 +738,13 @@ impl Cache { Ok(summary) } + /// Remove a cache path using the cache's configured reclaimed-space accounting. + pub fn remove_path(&self, path: impl AsRef) -> io::Result { + Remover::default() + .with_reclaimed_space(self.measure_reclaimed_space) + .rm_rf(path, false) + } + /// Find all references to entries in the archive bucket. /// /// Archive entries are often referenced by symlinks in other cache buckets. This method @@ -1233,37 +1267,37 @@ impl CacheBucket { metadata.name == *name } - let mut summary = Removal::default(); + let mut summary = cache.removal(); match self { Self::Wheels => { // For `pypi` wheels, we expect a directory per package (indexed by name). let root = cache.bucket(self).join(WheelCacheKind::Pypi); - summary += rm_rf(root.join(name.to_string()))?; + summary += cache.remove_path(root.join(name.to_string()))?; // For alternate indices, we expect a directory for every index (under an `index` // subdirectory), followed by a directory per package (indexed by name). let root = cache.bucket(self).join(WheelCacheKind::Index); for directory in directories(root)? { - summary += rm_rf(directory.join(name.to_string()))?; + summary += cache.remove_path(directory.join(name.to_string()))?; } // For direct URLs, we expect a directory for every URL, followed by a // directory per package (indexed by name). let root = cache.bucket(self).join(WheelCacheKind::Url); for directory in directories(root)? { - summary += rm_rf(directory.join(name.to_string()))?; + summary += cache.remove_path(directory.join(name.to_string()))?; } } Self::SourceDistributions => { // For `pypi` wheels, we expect a directory per package (indexed by name). let root = cache.bucket(self).join(WheelCacheKind::Pypi); - summary += rm_rf(root.join(name.to_string()))?; + summary += cache.remove_path(root.join(name.to_string()))?; // For alternate indices, we expect a directory for every index (under an `index` // subdirectory), followed by a directory per package (indexed by name). let root = cache.bucket(self).join(WheelCacheKind::Index); for directory in directories(root)? { - summary += rm_rf(directory.join(name.to_string()))?; + summary += cache.remove_path(directory.join(name.to_string()))?; } // For direct URLs, we expect a directory for every URL, followed by a @@ -1272,7 +1306,7 @@ impl CacheBucket { let root = cache.bucket(self).join(WheelCacheKind::Url); for url in directories(root)? { if directories(&url)?.any(|version| is_match(&version, name)) { - summary += rm_rf(url)?; + summary += cache.remove_path(url)?; } } @@ -1282,7 +1316,7 @@ impl CacheBucket { let root = cache.bucket(self).join(WheelCacheKind::Path); for path in directories(root)? { if directories(&path)?.any(|version| is_match(&version, name)) { - summary += rm_rf(path)?; + summary += cache.remove_path(path)?; } } @@ -1293,7 +1327,7 @@ impl CacheBucket { for repository in directories(root)? { for sha in directories(repository)? { if is_match(&sha, name) { - summary += rm_rf(sha)?; + summary += cache.remove_path(sha)?; } } } @@ -1301,20 +1335,20 @@ impl CacheBucket { Self::Simple => { // For `pypi` wheels, we expect a rkyv file per package, indexed by name. let root = cache.bucket(self).join(WheelCacheKind::Pypi); - summary += rm_rf(root.join(format!("{name}.rkyv")))?; + summary += cache.remove_path(root.join(format!("{name}.rkyv")))?; // For alternate indices, we expect a directory for every index (under an `index` // subdirectory), followed by a directory per package (indexed by name). let root = cache.bucket(self).join(WheelCacheKind::Index); for directory in directories(root)? { - summary += rm_rf(directory.join(format!("{name}.rkyv")))?; + summary += cache.remove_path(directory.join(format!("{name}.rkyv")))?; } } Self::FlatIndex => { // We can't know if the flat index includes a package, so we just remove the entire // cache entry. let root = cache.bucket(self); - summary += rm_rf(root)?; + summary += cache.remove_path(root)?; } Self::Git | Self::Interpreter diff --git a/crates/uv-cache/src/removal.rs b/crates/uv-cache/src/removal.rs index d0f9ae0a73a..b9583b45c3d 100644 --- a/crates/uv-cache/src/removal.rs +++ b/crates/uv-cache/src/removal.rs @@ -17,6 +17,7 @@ pub fn rm_rf(path: impl AsRef) -> io::Result { #[derive(Default)] pub(crate) struct Remover { reporter: Option>, + measure_reclaimed_space: bool, } impl Remover { @@ -24,9 +25,16 @@ impl Remover { pub(crate) fn new(reporter: Box) -> Self { Self { reporter: Some(reporter), + measure_reclaimed_space: false, } } + /// Enable accounting for exclusively owned storage before each file is removed. + pub(crate) fn with_reclaimed_space(mut self, enabled: bool) -> Self { + self.measure_reclaimed_space = enabled; + self + } + /// Remove a file or directory and all its contents, returning a [`Removal`] with /// the number of files and directories removed, along with a total byte count. pub(crate) fn rm_rf( @@ -34,7 +42,7 @@ impl Remover { path: impl AsRef, skip_locked_file: bool, ) -> io::Result { - let mut removal = Removal::default(); + let mut removal = Removal::new(self.measure_reclaimed_space); removal.rm_rf(path.as_ref(), self.reporter.as_deref(), skip_locked_file)?; Ok(removal) } @@ -52,9 +60,30 @@ pub struct Removal { /// Note: this will both over-count bytes removed for hard-linked files, and under-count /// bytes in general since it's a measure of the exact byte size (as opposed to the block size). pub total_bytes: u64, + /// The exclusively owned allocated file data reclaimed by the removal, when available. + pub reclaimed_bytes: Option, } impl Removal { + /// Create an empty removal summary with optional per-file reclaimed-space accounting. + pub fn new(measure_reclaimed_space: bool) -> Self { + Self { + reclaimed_bytes: measure_reclaimed_space.then_some(0), + ..Self::default() + } + } + + /// Account for a file while its current sharing state can still be inspected. + fn add_file(&mut self, path: &Path, metadata: &std::fs::Metadata) { + self.total_bytes += metadata.len(); + + if let Some(reclaimed_bytes) = self.reclaimed_bytes { + self.reclaimed_bytes = uv_fs::reclaimable_space(path, metadata) + .ok() + .map(|reclaimable| reclaimed_bytes.saturating_add(reclaimable)); + } + } + /// Recursively remove a file or directory and all its contents. fn rm_rf( &mut self, @@ -74,7 +103,7 @@ impl Removal { self.num_files += 1; // Remove the file. - self.total_bytes += metadata.len(); + self.add_file(&path, &metadata); if metadata.is_symlink() { cfg_select! { windows => { @@ -158,8 +187,10 @@ impl Removal { self.num_files += 1; // Remove the file. - if let Ok(meta) = entry.metadata() { - self.total_bytes += meta.len(); + if let Ok(metadata) = entry.metadata() { + self.add_file(entry.path(), &metadata); + } else { + self.reclaimed_bytes = None; } remove_file(entry.path())?; } @@ -178,6 +209,10 @@ impl std::ops::AddAssign for Removal { self.num_files += other.num_files; self.num_dirs += other.num_dirs; self.total_bytes += other.total_bytes; + self.reclaimed_bytes = self + .reclaimed_bytes + .zip(other.reclaimed_bytes) + .map(|(left, right)| left.saturating_add(right)); } } diff --git a/crates/uv-distribution/src/source/mod.rs b/crates/uv-distribution/src/source/mod.rs index ae65795d857..b303c835626 100644 --- a/crates/uv-distribution/src/source/mod.rs +++ b/crates/uv-distribution/src/source/mod.rs @@ -3306,7 +3306,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { /// Prune any unused source distributions from the cache. pub fn prune(cache: &Cache) -> Result { - let mut removal = Removal::default(); + let mut removal = cache.removal(); let bucket = cache.bucket(CacheBucket::SourceDistributions); if bucket.is_dir() { @@ -3332,8 +3332,9 @@ pub fn prune(cache: &Cache) -> Result { "Removing dangling source revision: {}", sibling.path().display() ); - removal += - uv_cache::rm_rf(sibling.path()).map_err(Error::CacheWrite)?; + removal += cache + .remove_path(sibling.path()) + .map_err(Error::CacheWrite)?; } } } @@ -3355,8 +3356,9 @@ pub fn prune(cache: &Cache) -> Result { "Removing dangling source revision: {}", sibling.path().display() ); - removal += - uv_cache::rm_rf(sibling.path()).map_err(Error::CacheWrite)?; + removal += cache + .remove_path(sibling.path()) + .map_err(Error::CacheWrite)?; } } } diff --git a/crates/uv-fs/Cargo.toml b/crates/uv-fs/Cargo.toml index aef75e028d5..4559f47afc7 100644 --- a/crates/uv-fs/Cargo.toml +++ b/crates/uv-fs/Cargo.toml @@ -40,6 +40,9 @@ walkdir = { workspace = true } [target.'cfg(any(unix, target_os = "wasi", target_os = "redox"))'.dependencies] rustix = { workspace = true } +[target.'cfg(any(target_os = "macos", target_os = "ios"))'.dependencies] +libc = { workspace = true } + [target.'cfg(windows)'.dependencies] backon = { workspace = true } junction = { workspace = true } diff --git a/crates/uv-fs/src/lib.rs b/crates/uv-fs/src/lib.rs index 6d10b1e8bf0..07aab33da95 100644 --- a/crates/uv-fs/src/lib.rs +++ b/crates/uv-fs/src/lib.rs @@ -12,7 +12,7 @@ use tracing::{debug, warn}; pub use crate::locked_file::*; pub use crate::path::*; pub use crate::read::ValidatedReader; -pub use crate::space::available_space; +pub use crate::space::{available_space, reclaimable_space, supports_reclaimable_space}; pub mod cachedir; pub mod link; diff --git a/crates/uv-fs/src/space.rs b/crates/uv-fs/src/space.rs index bc4eea67a14..5605d122e9c 100644 --- a/crates/uv-fs/src/space.rs +++ b/crates/uv-fs/src/space.rs @@ -1,11 +1,27 @@ use std::io; use std::path::Path; +#[cfg(any(target_os = "macos", target_os = "ios"))] +use std::ffi::CString; +#[cfg(any(target_os = "macos", target_os = "ios"))] +use std::os::unix::ffi::OsStrExt; +#[cfg(unix)] +use std::os::unix::fs::MetadataExt; + #[cfg(windows)] use std::os::windows::ffi::OsStrExt; +#[cfg(windows)] +use std::os::windows::io::AsRawHandle; #[cfg(windows)] -use windows::Win32::Storage::FileSystem::GetDiskFreeSpaceExW; +use fs_err::os::windows::fs::OpenOptionsExt; +#[cfg(windows)] +use windows::Win32::Foundation::HANDLE; +#[cfg(windows)] +use windows::Win32::Storage::FileSystem::{ + FILE_FLAG_BACKUP_SEMANTICS, FILE_STANDARD_INFO, FileStandardInfo, GetDiskFreeSpaceExW, + GetFileInformationByHandleEx, GetVolumeInformationByHandleW, +}; #[cfg(windows)] use windows::core::PCWSTR; @@ -45,3 +61,278 @@ pub fn available_space(path: &Path) -> io::Result { )) } } + +/// Return whether the filesystem can identify storage reclaimed by individual files. +#[cfg_attr(windows, expect(unsafe_code))] +pub fn supports_reclaimable_space(path: &Path) -> io::Result { + #[cfg(windows)] + { + // ReFS exposes block cloning but does not expose the clone reference counts needed to + // identify exclusively owned extents. + const FILE_SUPPORTS_BLOCK_REFCOUNTING: u32 = 0x0800_0000; + + let directory = fs_err::OpenOptions::new() + .read(true) + .custom_flags(FILE_FLAG_BACKUP_SEMANTICS.0) + .open(path)?; + let mut filesystem_flags = 0; + + // SAFETY: `directory` remains open during the call, and `filesystem_flags` points to a + // valid, writable output value. All other optional output values are omitted. + unsafe { + GetVolumeInformationByHandleW( + HANDLE(directory.as_raw_handle()), + None, + None, + None, + Some(&raw mut filesystem_flags), + None, + )?; + } + + Ok(filesystem_flags & FILE_SUPPORTS_BLOCK_REFCOUNTING == 0) + } + + #[cfg(any(target_os = "linux", target_os = "macos", target_os = "ios"))] + { + let _ = path; + Ok(true) + } + + #[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "ios", windows)))] + { + let _ = path; + Ok(false) + } +} + +/// Return the allocated file data that would be reclaimed by deleting `path` immediately. +/// +/// The result excludes data retained by another hardlink, copy-on-write clone, or snapshot. +/// Filesystem metadata is not included. +pub fn reclaimable_space(path: &Path, metadata: &std::fs::Metadata) -> io::Result { + if !metadata.is_file() { + #[cfg(unix)] + { + return Ok(metadata.blocks().saturating_mul(512)); + } + + #[cfg(not(unix))] + { + return Ok(0); + } + } + + #[cfg(unix)] + if metadata.nlink() > 1 { + return Ok(0); + } + + #[cfg(any(target_os = "macos", target_os = "ios"))] + { + apple_reclaimable_space(path) + } + + #[cfg(target_os = "linux")] + { + linux_reclaimable_space(path) + } + + #[cfg(windows)] + { + windows_reclaimable_space(path) + } + + #[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "ios", windows)))] + { + let _ = path; + Err(io::Error::new( + io::ErrorKind::Unsupported, + "per-file space measurement is unsupported on this platform", + )) + } +} + +#[cfg(any(target_os = "macos", target_os = "ios"))] +#[expect(unsafe_code)] +fn apple_reclaimable_space(path: &Path) -> io::Result { + let path = CString::new(path.as_os_str().as_bytes()) + .map_err(|error| io::Error::new(io::ErrorKind::InvalidInput, error))?; + let mut attributes = libc::attrlist { + bitmapcount: libc::ATTR_BIT_MAP_COUNT, + reserved: 0, + commonattr: libc::ATTR_CMN_RETURNED_ATTRS, + volattr: 0, + dirattr: 0, + fileattr: 0, + forkattr: libc::ATTR_CMNEXT_PRIVATESIZE, + }; + let mut response = [0_u8; 32]; + + // SAFETY: `path` is null-terminated and valid for the duration of the call. `attributes` is a + // valid attribute request, and `response` has enough space for the length, returned attribute + // set, and requested private-size value. + let result = unsafe { + libc::getattrlist( + path.as_ptr(), + (&raw mut attributes).cast(), + response.as_mut_ptr().cast(), + response.len(), + libc::FSOPT_ATTR_CMN_EXTENDED, + ) + }; + if result != 0 { + return Err(io::Error::last_os_error()); + } + + let returned_fork_attributes = + u32::from_ne_bytes(response[20..24].try_into().map_err(io::Error::other)?); + if returned_fork_attributes & libc::ATTR_CMNEXT_PRIVATESIZE == 0 { + return Err(io::Error::new( + io::ErrorKind::Unsupported, + "the filesystem does not report private file sizes", + )); + } + + let private_size = i64::from_ne_bytes(response[24..32].try_into().map_err(io::Error::other)?); + u64::try_from(private_size).map_err(io::Error::other) +} + +#[cfg(target_os = "linux")] +#[expect(unsafe_code)] +fn linux_reclaimable_space(path: &Path) -> io::Result { + const FIEMAP_EXTENT_LAST: u32 = 0x0000_0001; + const FIEMAP_EXTENT_UNKNOWN: u32 = 0x0000_0002; + const FIEMAP_EXTENT_DELALLOC: u32 = 0x0000_0004; + const FIEMAP_EXTENT_ENCODED: u32 = 0x0000_0008; + const FIEMAP_EXTENT_NOT_ALIGNED: u32 = 0x0000_0100; + const FIEMAP_EXTENT_SHARED: u32 = 0x0000_2000; + const MAX_EXTENTS: usize = 32; + + #[derive(Default)] + #[repr(C)] + struct Fiemap { + start: u64, + length: u64, + flags: u32, + mapped_extents: u32, + extent_count: u32, + reserved: u32, + } + + #[derive(Clone, Copy, Default)] + #[repr(C)] + struct FiemapExtent { + logical: u64, + physical: u64, + length: u64, + reserved64: [u64; 2], + flags: u32, + reserved: [u32; 3], + } + + #[derive(Default)] + #[repr(C)] + struct FiemapBuffer { + header: Fiemap, + extents: [FiemapExtent; MAX_EXTENTS], + } + + const FS_IOC_FIEMAP: rustix::ioctl::Opcode = + rustix::ioctl::opcode::read_write::(b'f', 11); + + let file = fs_err::File::open(path)?; + let mut reclaimable = 0_u64; + let mut start = 0_u64; + + loop { + let mut request = FiemapBuffer { + header: Fiemap { + start, + length: u64::MAX.saturating_sub(start), + extent_count: u32::try_from(MAX_EXTENTS).map_err(io::Error::other)?, + ..Fiemap::default() + }, + ..FiemapBuffer::default() + }; + + // SAFETY: `FS_IOC_FIEMAP` is the Linux fiemap ioctl opcode, and `request` begins with the + // expected fiemap header followed by enough initialized storage for all requested extents. + unsafe { + rustix::ioctl::ioctl( + &file, + rustix::ioctl::Updater::::new(&mut request), + )?; + } + + let mapped_extents = + usize::try_from(request.header.mapped_extents).map_err(io::Error::other)?; + if mapped_extents > request.extents.len() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "the filesystem returned more extents than requested", + )); + } + if mapped_extents == 0 { + return Ok(reclaimable); + } + + for extent in &request.extents[..mapped_extents] { + if extent.flags & FIEMAP_EXTENT_DELALLOC != 0 { + continue; + } + + if extent.flags + & (FIEMAP_EXTENT_UNKNOWN | FIEMAP_EXTENT_ENCODED | FIEMAP_EXTENT_NOT_ALIGNED) + != 0 + { + return Err(io::Error::new( + io::ErrorKind::Unsupported, + "the filesystem cannot report the physical size of an extent", + )); + } + + if extent.flags & FIEMAP_EXTENT_SHARED == 0 { + reclaimable = reclaimable.saturating_add(extent.length); + } + } + + let last_extent = &request.extents[mapped_extents - 1]; + if last_extent.flags & FIEMAP_EXTENT_LAST != 0 { + return Ok(reclaimable); + } + + let next = last_extent.logical.saturating_add(last_extent.length); + if next <= start { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "the filesystem returned a non-advancing extent", + )); + } + start = next; + } +} + +#[cfg(windows)] +#[expect(unsafe_code)] +fn windows_reclaimable_space(path: &Path) -> io::Result { + let file = fs_err::File::open(path)?; + let mut information = FILE_STANDARD_INFO::default(); + + // SAFETY: `file` remains open during the call, and `information` is a valid, writable + // `FILE_STANDARD_INFO` buffer whose exact size is passed to Windows. + unsafe { + GetFileInformationByHandleEx( + HANDLE(file.as_raw_handle()), + FileStandardInfo, + (&raw mut information).cast(), + u32::try_from(std::mem::size_of::()).map_err(io::Error::other)?, + )?; + } + + if information.NumberOfLinks > 1 { + return Ok(0); + } + + u64::try_from(information.AllocationSize).map_err(io::Error::other) +} diff --git a/crates/uv/src/commands/cache_clean.rs b/crates/uv/src/commands/cache_clean.rs index a7c79422f6e..2f0395df538 100644 --- a/crates/uv/src/commands/cache_clean.rs +++ b/crates/uv/src/commands/cache_clean.rs @@ -4,7 +4,7 @@ use anyhow::{Context, Result}; use owo_colors::OwoColorize; use tracing::debug; -use uv_cache::{Cache, Removal}; +use uv_cache::Cache; use uv_fs::Simplified; use uv_normalize::PackageName; use uv_preview::{Preview, PreviewFeature}; @@ -45,10 +45,12 @@ pub(crate) async fn cache_clean( } }; + let measure_reclaimed_space = preview.is_enabled(PreviewFeature::CacheReclaimedSpace); + let cache = cache.with_reclaimed_space(measure_reclaimed_space); + // The cache root itself is removed when cleaning the entire cache, so measure its parent. let filesystem_path = cache.root().parent().unwrap_or(cache.root()).to_path_buf(); - let available_before = preview - .is_enabled(PreviewFeature::CacheReclaimedSpace) + let available_before = measure_reclaimed_space .then(|| uv_fs::available_space(&filesystem_path).ok()) .flatten(); @@ -68,7 +70,7 @@ pub(crate) async fn cache_clean( .with_context(|| format!("Failed to clear cache at: {}", root.user_display()))? } else { let reporter = CleaningPackageReporter::new(printer, Some(packages.len())); - let mut summary = Removal::default(); + let mut summary = cache.removal(); for package in packages { let removed = cache.remove(package)?; @@ -80,10 +82,12 @@ pub(crate) async fn cache_clean( summary }; - let reclaimed_bytes = available_before.and_then(|before| { - uv_fs::available_space(&filesystem_path) - .ok() - .map(|after| after.saturating_sub(before)) + let reclaimed_bytes = summary.reclaimed_bytes.or_else(|| { + available_before.and_then(|before| { + uv_fs::available_space(&filesystem_path) + .ok() + .map(|after| after.saturating_sub(before)) + }) }); // Write a summary of the number of files and directories removed. diff --git a/crates/uv/src/commands/cache_prune.rs b/crates/uv/src/commands/cache_prune.rs index 9b740999121..55371d1acc6 100644 --- a/crates/uv/src/commands/cache_prune.rs +++ b/crates/uv/src/commands/cache_prune.rs @@ -4,7 +4,7 @@ use anyhow::{Context, Result}; use owo_colors::OwoColorize; use tracing::debug; -use uv_cache::{Cache, Removal}; +use uv_cache::Cache; use uv_fs::Simplified; use uv_preview::{Preview, PreviewFeature}; @@ -43,17 +43,19 @@ pub(crate) async fn cache_prune( } }; + let measure_reclaimed_space = preview.is_enabled(PreviewFeature::CacheReclaimedSpace); + let cache = cache.with_reclaimed_space(measure_reclaimed_space); + writeln!( printer.stderr(), "Pruning cache at: {}", cache.root().user_display().cyan() )?; - let available_before = preview - .is_enabled(PreviewFeature::CacheReclaimedSpace) + let available_before = measure_reclaimed_space .then(|| uv_fs::available_space(cache.root()).ok()) .flatten(); - let mut summary = Removal::default(); + let mut summary = cache.removal(); // Prune the source distribution cache, which is tightly coupled to the builder crate. summary += uv_distribution::prune(&cache) @@ -64,10 +66,12 @@ pub(crate) async fn cache_prune( .prune(ci) .with_context(|| format!("Failed to prune cache at: {}", cache.root().user_display()))?; - let reclaimed_bytes = available_before.and_then(|before| { - uv_fs::available_space(cache.root()) - .ok() - .map(|after| after.saturating_sub(before)) + let reclaimed_bytes = summary.reclaimed_bytes.or_else(|| { + available_before.and_then(|before| { + uv_fs::available_space(cache.root()) + .ok() + .map(|after| after.saturating_sub(before)) + }) }); // Write a summary of the number of files and directories removed. diff --git a/crates/uv/tests/build/cache_clean.rs b/crates/uv/tests/build/cache_clean.rs index 9b132fd8c41..57b94234523 100644 --- a/crates/uv/tests/build/cache_clean.rs +++ b/crates/uv/tests/build/cache_clean.rs @@ -39,8 +39,23 @@ fn clean_all() -> Result<()> { #[test] fn clean_all_hardlinked_file() -> Result<()> { let context = uv_test::test_context!("3.12").with_filtered_counts(); - let retained = context.temp_dir.child("retained.bin"); - retained.write_binary(&vec![42; 1024 * 1024])?; + + #[cfg(windows)] + let context = if std::env::var_os(EnvVars::UV_INTERNAL__TEST_LOWLINKS_FS).is_some() { + let Some(context) = context.with_cache_on_lowlinks_fs()? else { + return Ok(()); + }; + let cache_dir = context.cache_dir.path().to_path_buf(); + context.with_filtered_path(&cache_dir, "CACHE_DIR") + } else { + context + }; + + // Keep the retained hardlink beside the cache so both entries share a filesystem, even when + // Windows CI explicitly places the cache on its NTFS test volume. + let retained = context.cache_dir.path().with_file_name("retained.bin"); + fs_err::write(&retained, vec![42; 1024 * 1024])?; + fs_err::File::open(&retained)?.sync_all()?; let cached = context.cache_dir.child("hardlinked.bin"); fs_err::hard_link(&retained, &cached)?; @@ -61,38 +76,65 @@ fn clean_all_hardlinked_file() -> Result<()> { context.cache_dir.create_dir_all()?; fs_err::hard_link(&retained, &cached)?; - uv_snapshot!(context.filters(), context.clean().arg("--preview-features").arg("cache-reclaimed-space"), @" + uv_snapshot!(&filters, context.clean().arg("--preview-features").arg("cache-reclaimed-space"), @" exit_code: 0 (success) ----- stderr ----- Clearing cache at: [CACHE_DIR]/ - Removed [N] files ([SIZE]) + Removed [N] files (0B) "); assert!(retained.is_file()); assert_eq!(fs_err::metadata(retained)?.len(), 1024 * 1024); + context.cache_dir.create_dir_all()?; + cached.write_binary(&vec![42; 1024 * 1024])?; + fs_err::File::open(cached.path())?.sync_all()?; + fs_err::hard_link(&cached, context.cache_dir.child("second-hardlink.bin"))?; + + uv_snapshot!(&filters, context.clean().arg("--preview-features").arg("cache-reclaimed-space"), @" + exit_code: 0 (success) + ----- stderr ----- + Clearing cache at: [CACHE_DIR]/ + Removed [N] files (1.0MiB) + "); + Ok(()) } /// `cache clean` should report reclaimed space for copy-on-write clones in preview mode. #[test] fn clean_all_cloned_file() -> Result<()> { - let context = uv_test::test_context!("3.12").with_filtered_counts(); + let context = copy_on_write_test_context()?; let retained = context.temp_dir.child("retained"); retained.create_dir_all()?; let original = retained.child("original.bin"); original.write_binary(&vec![42; 1024 * 1024])?; + // Remove unrelated cache entries so the cloned file is the only allocated data being cleaned. + context.clean().assert().success(); + context.cache_dir.create_dir_all()?; + let cached = context.cache_dir.child("cloned"); - if link_dir(&retained, &cached, &LinkOptions::new(LinkMode::Clone))? != LinkMode::Clone { + let link_mode = link_dir(&retained, &cached, &LinkOptions::new(LinkMode::Clone))?; + if link_mode != LinkMode::Clone { + assert!( + std::env::var_os(EnvVars::UV_INTERNAL__TEST_COW_FS).is_none(), + "the configured copy-on-write filesystem did not clone the cached file" + ); return Ok(()); } - uv_snapshot!(context.filters(), context.clean().arg("--preview"), @" + let filters: Vec<_> = context + .filters() + .into_iter() + .filter(|(_, replacement)| *replacement != "$1[SIZE]") + .collect(); + + uv_snapshot!(&filters, context.clean().arg("--preview"), @" exit_code: 0 (success) ----- stderr ----- Clearing cache at: [CACHE_DIR]/ - Removed [N] files ([SIZE]) + Removed [N] files (0B) "); assert!(original.is_file()); @@ -101,6 +143,60 @@ fn clean_all_cloned_file() -> Result<()> { Ok(()) } +/// Clones shared only within the cache should be counted once when their final reference is removed. +#[test] +fn clean_all_cached_clones() -> Result<()> { + let context = copy_on_write_test_context()?; + let original = context.cache_dir.child("original"); + original.create_dir_all()?; + original + .child("original.bin") + .write_binary(&vec![42; 1024 * 1024])?; + + let cloned = context.cache_dir.child("cloned"); + let link_mode = link_dir(&original, &cloned, &LinkOptions::new(LinkMode::Clone))?; + if link_mode != LinkMode::Clone { + assert!( + std::env::var_os(EnvVars::UV_INTERNAL__TEST_COW_FS).is_none(), + "the configured copy-on-write filesystem did not clone the cached file" + ); + return Ok(()); + } + + let filters: Vec<_> = context + .filters() + .into_iter() + .filter(|(_, replacement)| *replacement != "$1[SIZE]") + .collect(); + + uv_snapshot!(&filters, context.clean().arg("--preview-features").arg("cache-reclaimed-space"), @" + exit_code: 0 (success) + ----- stderr ----- + Clearing cache at: [CACHE_DIR]/ + Removed [N] files (1.0MiB) + "); + + Ok(()) +} + +/// Put the cache and retained files on CI's Btrfs or APFS volume, when one is configured. +fn copy_on_write_test_context() -> Result { + let context = uv_test::test_context!("3.12").with_filtered_counts(); + if std::env::var_os(EnvVars::UV_INTERNAL__TEST_COW_FS).is_none() { + return Ok(context); + } + + let Some(context) = context.with_cache_on_cow_fs()? else { + anyhow::bail!("the configured copy-on-write cache filesystem was unavailable"); + }; + let Some(context) = context.with_working_dir_on_cow_fs()? else { + anyhow::bail!("the configured copy-on-write working filesystem was unavailable"); + }; + + let cache_dir = context.cache_dir.path().to_path_buf(); + Ok(context.with_filtered_path(&cache_dir, "CACHE_DIR")) +} + /// `cache clear` should behave as an alias of `cache clean`. #[test] fn clear_all_alias() -> Result<()> { diff --git a/crates/uv/tests/build/cache_prune.rs b/crates/uv/tests/build/cache_prune.rs index fd67fc1ba9d..ef1239b48ce 100644 --- a/crates/uv/tests/build/cache_prune.rs +++ b/crates/uv/tests/build/cache_prune.rs @@ -44,8 +44,22 @@ fn prune_no_op() -> Result<()> { #[test] fn prune_hardlinked_file() -> Result<()> { let context = uv_test::test_context!("3.12"); - let retained = context.temp_dir.child("retained.bin"); - retained.write_binary(&vec![42; 1024 * 1024])?; + + #[cfg(windows)] + let context = if std::env::var_os(EnvVars::UV_INTERNAL__TEST_LOWLINKS_FS).is_some() { + let Some(context) = context.with_cache_on_lowlinks_fs()? else { + return Ok(()); + }; + let cache_dir = context.cache_dir.path().to_path_buf(); + context.with_filtered_path(&cache_dir, "CACHE_DIR") + } else { + context + }; + + // Keep both hardlinks on the selected filesystem, including Windows CI's NTFS test volume. + let retained = context.cache_dir.path().with_file_name("retained.bin"); + fs_err::write(&retained, vec![42; 1024 * 1024])?; + fs_err::File::open(&retained)?.sync_all()?; let stale = context.cache_dir.child("stale-v0"); stale.create_dir_all()?; @@ -67,11 +81,11 @@ fn prune_hardlinked_file() -> Result<()> { stale.create_dir_all()?; fs_err::hard_link(&retained, stale.child("hardlinked.bin"))?; - uv_snapshot!(context.filters(), context.prune().arg("--preview-features").arg("cache-reclaimed-space"), @" + uv_snapshot!(&filters, context.prune().arg("--preview-features").arg("cache-reclaimed-space"), @" exit_code: 0 (success) ----- stderr ----- Pruning cache at: [CACHE_DIR]/ - Removed 1 file ([SIZE]) + Removed 1 file (0B) "); assert!(retained.is_file()); From 69cad88bcdc1ca0fa87f7c4837742f42394b2130 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Mon, 3 Aug 2026 16:36:11 -0400 Subject: [PATCH 05/13] Fix cache space accounting CI failures --- crates/uv-cache/src/lib.rs | 2 +- crates/uv-cache/src/removal.rs | 8 +---- crates/uv-fs/src/lib.rs | 2 +- crates/uv-fs/src/space.rs | 45 ++------------------------- crates/uv/src/commands/cache_clean.rs | 19 ++--------- crates/uv/src/commands/cache_prune.rs | 16 ++-------- crates/uv/tests/build/cache_clean.rs | 10 ++++-- crates/uv/tests/build/cache_prune.rs | 5 ++- 8 files changed, 21 insertions(+), 86 deletions(-) diff --git a/crates/uv-cache/src/lib.rs b/crates/uv-cache/src/lib.rs index 416e1b977f6..777401e0834 100644 --- a/crates/uv-cache/src/lib.rs +++ b/crates/uv-cache/src/lib.rs @@ -17,8 +17,8 @@ use uv_pypi_types::ResolutionMetadata; pub use crate::by_timestamp::CachedByTimestamp; #[cfg(feature = "clap")] pub use crate::cli::CacheArgs; +pub use crate::removal::Removal; use crate::removal::Remover; -pub use crate::removal::{Removal, rm_rf}; pub use crate::wheel::WheelCache; use crate::wheel::WheelCacheKind; pub use archive::ArchiveId; diff --git a/crates/uv-cache/src/removal.rs b/crates/uv-cache/src/removal.rs index b9583b45c3d..fc27ce8be1d 100644 --- a/crates/uv-cache/src/removal.rs +++ b/crates/uv-cache/src/removal.rs @@ -7,12 +7,6 @@ use std::path::Path; use crate::CleanReporter; -/// Remove a file or directory and all its contents, returning a [`Removal`] with -/// the number of files and directories removed, along with a total byte count. -pub fn rm_rf(path: impl AsRef) -> io::Result { - Remover::default().rm_rf(path, false) -} - /// A builder for a [`Remover`] that can remove files and directories. #[derive(Default)] pub(crate) struct Remover { @@ -66,7 +60,7 @@ pub struct Removal { impl Removal { /// Create an empty removal summary with optional per-file reclaimed-space accounting. - pub fn new(measure_reclaimed_space: bool) -> Self { + pub(crate) fn new(measure_reclaimed_space: bool) -> Self { Self { reclaimed_bytes: measure_reclaimed_space.then_some(0), ..Self::default() diff --git a/crates/uv-fs/src/lib.rs b/crates/uv-fs/src/lib.rs index 07aab33da95..ea43bcbea2f 100644 --- a/crates/uv-fs/src/lib.rs +++ b/crates/uv-fs/src/lib.rs @@ -12,7 +12,7 @@ use tracing::{debug, warn}; pub use crate::locked_file::*; pub use crate::path::*; pub use crate::read::ValidatedReader; -pub use crate::space::{available_space, reclaimable_space, supports_reclaimable_space}; +pub use crate::space::{reclaimable_space, supports_reclaimable_space}; pub mod cachedir; pub mod link; diff --git a/crates/uv-fs/src/space.rs b/crates/uv-fs/src/space.rs index 5605d122e9c..4a7b27cde74 100644 --- a/crates/uv-fs/src/space.rs +++ b/crates/uv-fs/src/space.rs @@ -8,8 +8,6 @@ use std::os::unix::ffi::OsStrExt; #[cfg(unix)] use std::os::unix::fs::MetadataExt; -#[cfg(windows)] -use std::os::windows::ffi::OsStrExt; #[cfg(windows)] use std::os::windows::io::AsRawHandle; @@ -19,48 +17,9 @@ use fs_err::os::windows::fs::OpenOptionsExt; use windows::Win32::Foundation::HANDLE; #[cfg(windows)] use windows::Win32::Storage::FileSystem::{ - FILE_FLAG_BACKUP_SEMANTICS, FILE_STANDARD_INFO, FileStandardInfo, GetDiskFreeSpaceExW, - GetFileInformationByHandleEx, GetVolumeInformationByHandleW, + FILE_FLAG_BACKUP_SEMANTICS, FILE_STANDARD_INFO, FileStandardInfo, GetFileInformationByHandleEx, + GetVolumeInformationByHandleW, }; -#[cfg(windows)] -use windows::core::PCWSTR; - -/// Return the available space on the filesystem containing `path`, in bytes. -#[cfg_attr(windows, expect(unsafe_code))] -pub fn available_space(path: &Path) -> io::Result { - #[cfg(unix)] - { - let statistics = rustix::fs::statvfs(path)?; - Ok(statistics.f_bavail.saturating_mul(statistics.f_frsize)) - } - - #[cfg(windows)] - { - let path = path - .as_os_str() - .encode_wide() - .chain(Some(0)) - .collect::>(); - let mut available = 0; - - // SAFETY: `path` is null-terminated and remains valid for the duration of the call, and - // `available` is a valid, writable pointer to the requested output value. - unsafe { - GetDiskFreeSpaceExW(PCWSTR(path.as_ptr()), Some(&raw mut available), None, None)?; - } - - Ok(available) - } - - #[cfg(not(any(unix, windows)))] - { - let _ = path; - Err(io::Error::new( - io::ErrorKind::Unsupported, - "filesystem space measurement is unsupported on this platform", - )) - } -} /// Return whether the filesystem can identify storage reclaimed by individual files. #[cfg_attr(windows, expect(unsafe_code))] diff --git a/crates/uv/src/commands/cache_clean.rs b/crates/uv/src/commands/cache_clean.rs index 2f0395df538..f6451c4770d 100644 --- a/crates/uv/src/commands/cache_clean.rs +++ b/crates/uv/src/commands/cache_clean.rs @@ -45,14 +45,7 @@ pub(crate) async fn cache_clean( } }; - let measure_reclaimed_space = preview.is_enabled(PreviewFeature::CacheReclaimedSpace); - let cache = cache.with_reclaimed_space(measure_reclaimed_space); - - // The cache root itself is removed when cleaning the entire cache, so measure its parent. - let filesystem_path = cache.root().parent().unwrap_or(cache.root()).to_path_buf(); - let available_before = measure_reclaimed_space - .then(|| uv_fs::available_space(&filesystem_path).ok()) - .flatten(); + let cache = cache.with_reclaimed_space(preview.is_enabled(PreviewFeature::CacheReclaimedSpace)); let summary = if packages.is_empty() { writeln!( @@ -82,14 +75,6 @@ pub(crate) async fn cache_clean( summary }; - let reclaimed_bytes = summary.reclaimed_bytes.or_else(|| { - available_before.and_then(|before| { - uv_fs::available_space(&filesystem_path) - .ok() - .map(|after| after.saturating_sub(before)) - }) - }); - // Write a summary of the number of files and directories removed. match (summary.num_files, summary.num_dirs) { (0, 0) => { @@ -111,7 +96,7 @@ pub(crate) async fn cache_clean( // If any, report the reclaimed space, falling back to the apparent removed size. if summary.total_bytes > 0 { - let total_bytes = reclaimed_bytes.unwrap_or(summary.total_bytes); + let total_bytes = summary.reclaimed_bytes.unwrap_or(summary.total_bytes); let bytes = if total_bytes < 1024 { format!("{total_bytes}B") } else { diff --git a/crates/uv/src/commands/cache_prune.rs b/crates/uv/src/commands/cache_prune.rs index 55371d1acc6..8039ec435a1 100644 --- a/crates/uv/src/commands/cache_prune.rs +++ b/crates/uv/src/commands/cache_prune.rs @@ -43,8 +43,7 @@ pub(crate) async fn cache_prune( } }; - let measure_reclaimed_space = preview.is_enabled(PreviewFeature::CacheReclaimedSpace); - let cache = cache.with_reclaimed_space(measure_reclaimed_space); + let cache = cache.with_reclaimed_space(preview.is_enabled(PreviewFeature::CacheReclaimedSpace)); writeln!( printer.stderr(), @@ -52,9 +51,6 @@ pub(crate) async fn cache_prune( cache.root().user_display().cyan() )?; - let available_before = measure_reclaimed_space - .then(|| uv_fs::available_space(cache.root()).ok()) - .flatten(); let mut summary = cache.removal(); // Prune the source distribution cache, which is tightly coupled to the builder crate. @@ -66,14 +62,6 @@ pub(crate) async fn cache_prune( .prune(ci) .with_context(|| format!("Failed to prune cache at: {}", cache.root().user_display()))?; - let reclaimed_bytes = summary.reclaimed_bytes.or_else(|| { - available_before.and_then(|before| { - uv_fs::available_space(cache.root()) - .ok() - .map(|after| after.saturating_sub(before)) - }) - }); - // Write a summary of the number of files and directories removed. match (summary.num_files, summary.num_dirs) { (0, 0) => { @@ -95,7 +83,7 @@ pub(crate) async fn cache_prune( // If any, report the reclaimed space, falling back to the apparent removed size. if summary.total_bytes > 0 { - let total_bytes = reclaimed_bytes.unwrap_or(summary.total_bytes); + let total_bytes = summary.reclaimed_bytes.unwrap_or(summary.total_bytes); let bytes = if total_bytes < 1024 { format!("{total_bytes}B") } else { diff --git a/crates/uv/tests/build/cache_clean.rs b/crates/uv/tests/build/cache_clean.rs index 57b94234523..b7ad8fb9d26 100644 --- a/crates/uv/tests/build/cache_clean.rs +++ b/crates/uv/tests/build/cache_clean.rs @@ -55,7 +55,10 @@ fn clean_all_hardlinked_file() -> Result<()> { // Windows CI explicitly places the cache on its NTFS test volume. let retained = context.cache_dir.path().with_file_name("retained.bin"); fs_err::write(&retained, vec![42; 1024 * 1024])?; - fs_err::File::open(&retained)?.sync_all()?; + fs_err::OpenOptions::new() + .write(true) + .open(&retained)? + .sync_all()?; let cached = context.cache_dir.child("hardlinked.bin"); fs_err::hard_link(&retained, &cached)?; @@ -88,7 +91,10 @@ fn clean_all_hardlinked_file() -> Result<()> { context.cache_dir.create_dir_all()?; cached.write_binary(&vec![42; 1024 * 1024])?; - fs_err::File::open(cached.path())?.sync_all()?; + fs_err::OpenOptions::new() + .write(true) + .open(cached.path())? + .sync_all()?; fs_err::hard_link(&cached, context.cache_dir.child("second-hardlink.bin"))?; uv_snapshot!(&filters, context.clean().arg("--preview-features").arg("cache-reclaimed-space"), @" diff --git a/crates/uv/tests/build/cache_prune.rs b/crates/uv/tests/build/cache_prune.rs index ef1239b48ce..2d2ff7ee3f5 100644 --- a/crates/uv/tests/build/cache_prune.rs +++ b/crates/uv/tests/build/cache_prune.rs @@ -59,7 +59,10 @@ fn prune_hardlinked_file() -> Result<()> { // Keep both hardlinks on the selected filesystem, including Windows CI's NTFS test volume. let retained = context.cache_dir.path().with_file_name("retained.bin"); fs_err::write(&retained, vec![42; 1024 * 1024])?; - fs_err::File::open(&retained)?.sync_all()?; + fs_err::OpenOptions::new() + .write(true) + .open(&retained)? + .sync_all()?; let stale = context.cache_dir.child("stale-v0"); stale.create_dir_all()?; From 53db2873f5fdbc8e4cb475ff285f8a8192b09e4b Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Mon, 3 Aug 2026 19:06:24 -0400 Subject: [PATCH 06/13] Account for shared ReFS cache extents --- .github/workflows/test.yml | 27 ++++ crates/uv-fs/src/space.rs | 202 ++++++++++++++++++++++----- crates/uv-static/src/env_vars.rs | 5 + crates/uv-test/src/lib.rs | 18 +++ crates/uv/tests/build/cache_clean.rs | 42 ++++-- 5 files changed, 249 insertions(+), 45 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ae135446ecb..8b75df888cf 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -291,6 +291,33 @@ jobs: - name: "Create NTFS test directory (low hardlink limit)" run: New-Item -Path "C:\uv" -ItemType Directory -Force + - name: "Create ReFS test directory (copy-on-write)" + run: | + $devVolume = Get-Volume -DriveLetter $env:DEV_DRIVE.TrimEnd(':') + if ($devVolume.FileSystem -eq "ReFS") { + $refsDrive = $env:DEV_DRIVE + } else { + $vhdPath = Join-Path $env:RUNNER_TEMP "uv-refs-test.vhdx" + @" + create vdisk file="$vhdPath" maximum=1024 type=expandable + attach vdisk + create partition primary + assign letter=R + "@ | diskpart + if ($LASTEXITCODE -ne 0) { + throw "Failed to create the ReFS test volume" + } + format R: /fs:ReFS /q /y + if ($LASTEXITCODE -ne 0) { + throw "Failed to format the ReFS test volume" + } + $refsDrive = "R:" + } + + $refsDirectory = "$refsDrive\uv-cow" + New-Item -Path $refsDirectory -ItemType Directory -Force + "UV_INTERNAL__TEST_REFS_FS=$refsDirectory" >> $env:GITHUB_ENV + - name: "Create SMB test share (alternative filesystem)" run: | New-Item -Path "C:\uv-smb" -ItemType Directory -Force diff --git a/crates/uv-fs/src/space.rs b/crates/uv-fs/src/space.rs index 4a7b27cde74..cbb296a3b46 100644 --- a/crates/uv-fs/src/space.rs +++ b/crates/uv-fs/src/space.rs @@ -12,47 +12,26 @@ use std::os::unix::fs::MetadataExt; use std::os::windows::io::AsRawHandle; #[cfg(windows)] -use fs_err::os::windows::fs::OpenOptionsExt; -#[cfg(windows)] -use windows::Win32::Foundation::HANDLE; +use windows::Win32::Foundation::{ERROR_HANDLE_EOF, ERROR_MORE_DATA, HANDLE}; #[cfg(windows)] use windows::Win32::Storage::FileSystem::{ - FILE_FLAG_BACKUP_SEMANTICS, FILE_STANDARD_INFO, FileStandardInfo, GetFileInformationByHandleEx, + FILE_STANDARD_INFO, FileStandardInfo, GetFileInformationByHandleEx, GetVolumeInformationByHandleW, }; +#[cfg(windows)] +use windows::Win32::System::IO::DeviceIoControl; +#[cfg(windows)] +use windows::Win32::System::Ioctl::{ + FSCTL_GET_INTEGRITY_INFORMATION, FSCTL_GET_INTEGRITY_INFORMATION_BUFFER, + FSCTL_GET_RETRIEVAL_POINTERS_AND_REFCOUNT, RETRIEVAL_POINTERS_AND_REFCOUNT_BUFFER_0, + STARTING_VCN_INPUT_BUFFER, +}; +#[cfg(windows)] +use windows::core::HRESULT; /// Return whether the filesystem can identify storage reclaimed by individual files. -#[cfg_attr(windows, expect(unsafe_code))] pub fn supports_reclaimable_space(path: &Path) -> io::Result { - #[cfg(windows)] - { - // ReFS exposes block cloning but does not expose the clone reference counts needed to - // identify exclusively owned extents. - const FILE_SUPPORTS_BLOCK_REFCOUNTING: u32 = 0x0800_0000; - - let directory = fs_err::OpenOptions::new() - .read(true) - .custom_flags(FILE_FLAG_BACKUP_SEMANTICS.0) - .open(path)?; - let mut filesystem_flags = 0; - - // SAFETY: `directory` remains open during the call, and `filesystem_flags` points to a - // valid, writable output value. All other optional output values are omitted. - unsafe { - GetVolumeInformationByHandleW( - HANDLE(directory.as_raw_handle()), - None, - None, - None, - Some(&raw mut filesystem_flags), - None, - )?; - } - - Ok(filesystem_flags & FILE_SUPPORTS_BLOCK_REFCOUNTING == 0) - } - - #[cfg(any(target_os = "linux", target_os = "macos", target_os = "ios"))] + #[cfg(any(target_os = "linux", target_os = "macos", target_os = "ios", windows))] { let _ = path; Ok(true) @@ -275,14 +254,17 @@ fn linux_reclaimable_space(path: &Path) -> io::Result { #[cfg(windows)] #[expect(unsafe_code)] fn windows_reclaimable_space(path: &Path) -> io::Result { + const FILE_SUPPORTS_BLOCK_REFCOUNTING: u32 = 0x0800_0000; + let file = fs_err::File::open(path)?; + let handle = HANDLE(file.as_raw_handle()); let mut information = FILE_STANDARD_INFO::default(); // SAFETY: `file` remains open during the call, and `information` is a valid, writable // `FILE_STANDARD_INFO` buffer whose exact size is passed to Windows. unsafe { GetFileInformationByHandleEx( - HANDLE(file.as_raw_handle()), + handle, FileStandardInfo, (&raw mut information).cast(), u32::try_from(std::mem::size_of::()).map_err(io::Error::other)?, @@ -293,5 +275,153 @@ fn windows_reclaimable_space(path: &Path) -> io::Result { return Ok(0); } - u64::try_from(information.AllocationSize).map_err(io::Error::other) + let allocated = u64::try_from(information.AllocationSize).map_err(io::Error::other)?; + if allocated == 0 { + return Ok(0); + } + + let mut filesystem_flags = 0; + + // SAFETY: `file` remains open during the call, and `filesystem_flags` points to a valid, + // writable output value. All other optional output values are omitted. + unsafe { + GetVolumeInformationByHandleW( + handle, + None, + None, + None, + Some(&raw mut filesystem_flags), + None, + )?; + } + + if filesystem_flags & FILE_SUPPORTS_BLOCK_REFCOUNTING == 0 { + return Ok(allocated); + } + + refs_reclaimable_space(handle) +} + +/// Sum the exclusively owned allocated extents of a file on ReFS. +#[cfg(windows)] +#[expect(unsafe_code)] +fn refs_reclaimable_space(handle: HANDLE) -> io::Result { + const MAX_EXTENTS: usize = 32; + + #[derive(Default)] + #[repr(C)] + struct RefcountBuffer { + extent_count: u32, + starting_vcn: i64, + extents: [RETRIEVAL_POINTERS_AND_REFCOUNT_BUFFER_0; MAX_EXTENTS], + } + + let mut integrity = FSCTL_GET_INTEGRITY_INFORMATION_BUFFER::default(); + let mut bytes_returned = 0; + + // SAFETY: `handle` is valid for the duration of the call, and `integrity` is a writable + // `FSCTL_GET_INTEGRITY_INFORMATION_BUFFER` whose exact size is passed to Windows. + unsafe { + DeviceIoControl( + handle, + FSCTL_GET_INTEGRITY_INFORMATION, + None, + 0, + Some((&raw mut integrity).cast()), + u32::try_from(std::mem::size_of::()) + .map_err(io::Error::other)?, + Some(&raw mut bytes_returned), + None, + )?; + } + + let cluster_size = u64::from(integrity.ClusterSizeInBytes); + if cluster_size == 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "ReFS returned a zero-byte cluster size", + )); + } + + let mut reclaimable = 0_u64; + let mut starting_vcn = 0_i64; + + loop { + let request = STARTING_VCN_INPUT_BUFFER { + StartingVcn: starting_vcn, + }; + let mut response = RefcountBuffer::default(); + bytes_returned = 0; + + // SAFETY: `handle` remains valid, `request` is an initialized input buffer, and `response` + // has the documented header followed by initialized storage for all requested extents. + let result = unsafe { + DeviceIoControl( + handle, + FSCTL_GET_RETRIEVAL_POINTERS_AND_REFCOUNT, + Some((&raw const request).cast()), + u32::try_from(std::mem::size_of::()) + .map_err(io::Error::other)?, + Some((&raw mut response).cast()), + u32::try_from(std::mem::size_of::()).map_err(io::Error::other)?, + Some(&raw mut bytes_returned), + None, + ) + }; + + let has_more = match result { + Ok(()) => false, + Err(error) if error.code() == HRESULT::from(ERROR_MORE_DATA) => true, + Err(error) if error.code() == HRESULT::from(ERROR_HANDLE_EOF) => { + return Ok(reclaimable); + } + Err(error) => return Err(error.into()), + }; + + let extent_count = usize::try_from(response.extent_count).map_err(io::Error::other)?; + if extent_count > response.extents.len() + || usize::try_from(bytes_returned).map_err(io::Error::other)? + < std::mem::offset_of!(RefcountBuffer, extents) + + extent_count * std::mem::size_of::() + { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "ReFS returned an invalid extent list", + )); + } + + let mut current_vcn = response.starting_vcn; + for extent in &response.extents[..extent_count] { + let clusters = extent.NextVcn.checked_sub(current_vcn).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "ReFS returned unordered extents", + ) + })?; + if clusters <= 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "ReFS returned a non-advancing extent", + )); + } + + if extent.Lcn >= 0 && extent.ReferenceCount == 1 { + let clusters = u64::try_from(clusters).map_err(io::Error::other)?; + reclaimable = reclaimable.saturating_add(clusters.saturating_mul(cluster_size)); + } + + current_vcn = extent.NextVcn; + } + + if !has_more { + return Ok(reclaimable); + } + if current_vcn <= starting_vcn { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "ReFS returned a non-advancing extent list", + )); + } + starting_vcn = current_vcn; + } } diff --git a/crates/uv-static/src/env_vars.rs b/crates/uv-static/src/env_vars.rs index 89b0c77af6e..cbf159953f3 100644 --- a/crates/uv-static/src/env_vars.rs +++ b/crates/uv-static/src/env_vars.rs @@ -685,6 +685,11 @@ impl EnvVars { #[attr_added_in("0.10.5")] pub const UV_INTERNAL__TEST_COW_FS: &'static str = "UV_INTERNAL__TEST_COW_FS"; + /// Path to a directory on a ReFS filesystem for Windows copy-on-write accounting tests. + #[attr_hidden] + #[attr_added_in("0.12.1")] + pub const UV_INTERNAL__TEST_REFS_FS: &'static str = "UV_INTERNAL__TEST_REFS_FS"; + /// Path to a directory on a filesystem that does **not** support copy-on-write. /// /// When populated, uv will run additional tests that verify fallback behavior diff --git a/crates/uv-test/src/lib.rs b/crates/uv-test/src/lib.rs index 680698b36bd..91000b08dc3 100755 --- a/crates/uv-test/src/lib.rs +++ b/crates/uv-test/src/lib.rs @@ -708,6 +708,15 @@ impl TestContext { self.with_cache_on_fs(&dir, "COW_FS").map(Some) } + /// Use a cache directory on the ReFS filesystem configured for Windows accounting tests. + #[cfg(windows)] + pub fn with_cache_on_refs_fs(self) -> anyhow::Result> { + let Some(dir) = env::var(EnvVars::UV_INTERNAL__TEST_REFS_FS).ok() else { + return Ok(None); + }; + self.with_cache_on_fs(&dir, "REFS_FS").map(Some) + } + /// Use a cache directory on the filesystem specified by /// [`EnvVars::UV_INTERNAL__TEST_ALT_FS`]. /// @@ -754,6 +763,15 @@ impl TestContext { self.with_working_dir_on_fs(&dir, "COW_FS").map(Some) } + /// Use a working directory on the ReFS filesystem configured for Windows accounting tests. + #[cfg(windows)] + pub fn with_working_dir_on_refs_fs(self) -> anyhow::Result> { + let Some(dir) = env::var(EnvVars::UV_INTERNAL__TEST_REFS_FS).ok() else { + return Ok(None); + }; + self.with_working_dir_on_fs(&dir, "REFS_FS").map(Some) + } + /// Use a working directory on the filesystem specified by /// [`EnvVars::UV_INTERNAL__TEST_NOCOW_FS`]. /// diff --git a/crates/uv/tests/build/cache_clean.rs b/crates/uv/tests/build/cache_clean.rs index b7ad8fb9d26..b3037b6c24e 100644 --- a/crates/uv/tests/build/cache_clean.rs +++ b/crates/uv/tests/build/cache_clean.rs @@ -124,7 +124,7 @@ fn clean_all_cloned_file() -> Result<()> { let link_mode = link_dir(&retained, &cached, &LinkOptions::new(LinkMode::Clone))?; if link_mode != LinkMode::Clone { assert!( - std::env::var_os(EnvVars::UV_INTERNAL__TEST_COW_FS).is_none(), + !copy_on_write_filesystem_configured(), "the configured copy-on-write filesystem did not clone the cached file" ); return Ok(()); @@ -163,7 +163,7 @@ fn clean_all_cached_clones() -> Result<()> { let link_mode = link_dir(&original, &cloned, &LinkOptions::new(LinkMode::Clone))?; if link_mode != LinkMode::Clone { assert!( - std::env::var_os(EnvVars::UV_INTERNAL__TEST_COW_FS).is_none(), + !copy_on_write_filesystem_configured(), "the configured copy-on-write filesystem did not clone the cached file" ); return Ok(()); @@ -185,24 +185,48 @@ fn clean_all_cached_clones() -> Result<()> { Ok(()) } -/// Put the cache and retained files on CI's Btrfs or APFS volume, when one is configured. +/// Put the cache and retained files on CI's Btrfs, APFS, or ReFS volume, when configured. fn copy_on_write_test_context() -> Result { let context = uv_test::test_context!("3.12").with_filtered_counts(); - if std::env::var_os(EnvVars::UV_INTERNAL__TEST_COW_FS).is_none() { + if !copy_on_write_filesystem_configured() { return Ok(context); } - let Some(context) = context.with_cache_on_cow_fs()? else { - anyhow::bail!("the configured copy-on-write cache filesystem was unavailable"); - }; - let Some(context) = context.with_working_dir_on_cow_fs()? else { - anyhow::bail!("the configured copy-on-write working filesystem was unavailable"); + let context = if std::env::var_os(EnvVars::UV_INTERNAL__TEST_COW_FS).is_some() { + let Some(context) = context.with_cache_on_cow_fs()? else { + anyhow::bail!("the configured copy-on-write cache filesystem was unavailable"); + }; + let Some(context) = context.with_working_dir_on_cow_fs()? else { + anyhow::bail!("the configured copy-on-write working filesystem was unavailable"); + }; + context + } else { + #[cfg(windows)] + { + let Some(context) = context.with_cache_on_refs_fs()? else { + anyhow::bail!("the configured ReFS cache filesystem was unavailable"); + }; + let Some(context) = context.with_working_dir_on_refs_fs()? else { + anyhow::bail!("the configured ReFS working filesystem was unavailable"); + }; + context + } + #[cfg(not(windows))] + { + context + } }; let cache_dir = context.cache_dir.path().to_path_buf(); Ok(context.with_filtered_path(&cache_dir, "CACHE_DIR")) } +/// Return whether CI explicitly configured a copy-on-write filesystem for accounting tests. +fn copy_on_write_filesystem_configured() -> bool { + std::env::var_os(EnvVars::UV_INTERNAL__TEST_COW_FS).is_some() + || cfg!(windows) && std::env::var_os(EnvVars::UV_INTERNAL__TEST_REFS_FS).is_some() +} + /// `cache clear` should behave as an alias of `cache clean`. #[test] fn clear_all_alias() -> Result<()> { From 0ec71835a22584c68433046362f733545efb0006 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Mon, 3 Aug 2026 19:09:01 -0400 Subject: [PATCH 07/13] Preserve measurable cache space when accounting is incomplete --- crates/uv-cache/src/removal.rs | 53 ++++++++++++++++++++++++--- crates/uv-fs/src/space.rs | 10 +++-- crates/uv/src/commands/cache_clean.rs | 6 ++- crates/uv/src/commands/cache_prune.rs | 6 ++- crates/uv/tests/build/cache_clean.rs | 49 +++++++++++++++++++++++++ docs/concepts/cache.md | 3 ++ 6 files changed, 116 insertions(+), 11 deletions(-) diff --git a/crates/uv-cache/src/removal.rs b/crates/uv-cache/src/removal.rs index fc27ce8be1d..85e2824ed4c 100644 --- a/crates/uv-cache/src/removal.rs +++ b/crates/uv-cache/src/removal.rs @@ -5,6 +5,8 @@ use std::io; use std::path::Path; +use tracing::debug; + use crate::CleanReporter; /// A builder for a [`Remover`] that can remove files and directories. @@ -56,6 +58,8 @@ pub struct Removal { pub total_bytes: u64, /// The exclusively owned allocated file data reclaimed by the removal, when available. pub reclaimed_bytes: Option, + /// Whether any removed entries could not be measured, making the reclaimed count a lower bound. + pub reclaimed_bytes_incomplete: bool, } impl Removal { @@ -72,9 +76,18 @@ impl Removal { self.total_bytes += metadata.len(); if let Some(reclaimed_bytes) = self.reclaimed_bytes { - self.reclaimed_bytes = uv_fs::reclaimable_space(path, metadata) - .ok() - .map(|reclaimable| reclaimed_bytes.saturating_add(reclaimable)); + match uv_fs::reclaimable_space(path, metadata) { + Ok(reclaimable) => { + self.reclaimed_bytes = Some(reclaimed_bytes.saturating_add(reclaimable)); + } + Err(error) => { + debug!( + "Failed to measure reclaimed space for {}: {error}", + path.display() + ); + self.reclaimed_bytes_incomplete = true; + } + } } } @@ -183,8 +196,8 @@ impl Removal { // Remove the file. if let Ok(metadata) = entry.metadata() { self.add_file(entry.path(), &metadata); - } else { - self.reclaimed_bytes = None; + } else if self.reclaimed_bytes.is_some() { + self.reclaimed_bytes_incomplete = true; } remove_file(entry.path())?; } @@ -207,6 +220,36 @@ impl std::ops::AddAssign for Removal { .reclaimed_bytes .zip(other.reclaimed_bytes) .map(|(left, right)| left.saturating_add(right)); + self.reclaimed_bytes_incomplete |= other.reclaimed_bytes_incomplete; + } +} + +#[cfg(test)] +mod tests { + use super::Removal; + + #[test] + fn retain_measured_space_when_an_entry_cannot_be_measured() -> std::io::Result<()> { + let directory = tempfile::tempdir()?; + let measured = directory.path().join("measured.bin"); + fs_err::write(&measured, vec![42; 4096])?; + let metadata = fs_err::metadata(&measured)?; + let expected = uv_fs::reclaimable_space(&measured, &metadata)?; + + let mut removal = Removal::new(true); + removal.add_file(&measured, &metadata); + removal.add_file(&directory.path().join("missing.bin"), &metadata); + removal.add_file(&measured, &metadata); + + assert_eq!(removal.reclaimed_bytes, Some(expected.saturating_mul(2))); + assert!(removal.reclaimed_bytes_incomplete); + + let mut combined = Removal::new(true); + combined += removal; + assert_eq!(combined.reclaimed_bytes, Some(expected.saturating_mul(2))); + assert!(combined.reclaimed_bytes_incomplete); + + Ok(()) } } diff --git a/crates/uv-fs/src/space.rs b/crates/uv-fs/src/space.rs index cbb296a3b46..407791629ef 100644 --- a/crates/uv-fs/src/space.rs +++ b/crates/uv-fs/src/space.rs @@ -144,6 +144,7 @@ fn linux_reclaimable_space(path: &Path) -> io::Result { const FIEMAP_EXTENT_DELALLOC: u32 = 0x0000_0004; const FIEMAP_EXTENT_ENCODED: u32 = 0x0000_0008; const FIEMAP_EXTENT_NOT_ALIGNED: u32 = 0x0000_0100; + const FIEMAP_EXTENT_DATA_INLINE: u32 = 0x0000_0200; const FIEMAP_EXTENT_SHARED: u32 = 0x0000_2000; const MAX_EXTENTS: usize = 32; @@ -216,7 +217,10 @@ fn linux_reclaimable_space(path: &Path) -> io::Result { } for extent in &request.extents[..mapped_extents] { - if extent.flags & FIEMAP_EXTENT_DELALLOC != 0 { + if extent.flags + & (FIEMAP_EXTENT_DELALLOC | FIEMAP_EXTENT_DATA_INLINE | FIEMAP_EXTENT_SHARED) + != 0 + { continue; } @@ -230,9 +234,7 @@ fn linux_reclaimable_space(path: &Path) -> io::Result { )); } - if extent.flags & FIEMAP_EXTENT_SHARED == 0 { - reclaimable = reclaimable.saturating_add(extent.length); - } + reclaimable = reclaimable.saturating_add(extent.length); } let last_extent = &request.extents[mapped_extents - 1]; diff --git a/crates/uv/src/commands/cache_clean.rs b/crates/uv/src/commands/cache_clean.rs index f6451c4770d..146820c08f9 100644 --- a/crates/uv/src/commands/cache_clean.rs +++ b/crates/uv/src/commands/cache_clean.rs @@ -103,7 +103,11 @@ pub(crate) async fn cache_clean( let (bytes, unit) = human_readable_bytes(total_bytes); format!("{bytes:.1}{unit}") }; - write!(printer.stderr(), " ({})", bytes.green())?; + if summary.reclaimed_bytes_incomplete { + write!(printer.stderr(), " (at least {})", bytes.green())?; + } else { + write!(printer.stderr(), " ({})", bytes.green())?; + } } writeln!(printer.stderr())?; diff --git a/crates/uv/src/commands/cache_prune.rs b/crates/uv/src/commands/cache_prune.rs index 8039ec435a1..a72c3d0d470 100644 --- a/crates/uv/src/commands/cache_prune.rs +++ b/crates/uv/src/commands/cache_prune.rs @@ -90,7 +90,11 @@ pub(crate) async fn cache_prune( let (bytes, unit) = human_readable_bytes(total_bytes); format!("{bytes:.1}{unit}") }; - write!(printer.stderr(), " ({})", bytes.green())?; + if summary.reclaimed_bytes_incomplete { + write!(printer.stderr(), " (at least {})", bytes.green())?; + } else { + write!(printer.stderr(), " ({})", bytes.green())?; + } } writeln!(printer.stderr())?; diff --git a/crates/uv/tests/build/cache_clean.rs b/crates/uv/tests/build/cache_clean.rs index b3037b6c24e..b4db6b33a57 100644 --- a/crates/uv/tests/build/cache_clean.rs +++ b/crates/uv/tests/build/cache_clean.rs @@ -2,6 +2,9 @@ use anyhow::Result; use assert_cmd::prelude::*; use assert_fs::prelude::*; +#[cfg(target_os = "linux")] +use std::process::Command; + use uv_cache::Cache; use uv_fs::link::{LinkMode, LinkOptions, link_dir}; use uv_static::EnvVars; @@ -185,6 +188,52 @@ fn clean_all_cached_clones() -> Result<()> { Ok(()) } +/// Unknown compressed extents should not discard measurements for unrelated cache entries. +#[cfg(target_os = "linux")] +#[test] +fn clean_all_compressed_file() -> Result<()> { + if std::env::var_os(EnvVars::UV_INTERNAL__TEST_COW_FS).is_none() { + return Ok(()); + } + + let context = copy_on_write_test_context()?; + let measured = context.cache_dir.child("measured.bin"); + measured.write_binary(&vec![42; 1024 * 1024])?; + fs_err::OpenOptions::new() + .write(true) + .open(&measured)? + .sync_all()?; + + let compressed = context.cache_dir.child("compressed.bin"); + fs_err::File::create(&compressed)?; + Command::new("btrfs") + .args(["property", "set"]) + .arg(compressed.path()) + .args(["compression", "zstd"]) + .assert() + .success(); + compressed.write_binary(&vec![42; 1024 * 1024])?; + fs_err::OpenOptions::new() + .write(true) + .open(&compressed)? + .sync_all()?; + + let filters: Vec<_> = context + .filters() + .into_iter() + .filter(|(_, replacement)| *replacement != "$1[SIZE]") + .collect(); + + uv_snapshot!(&filters, context.clean().arg("--preview-features").arg("cache-reclaimed-space"), @" + exit_code: 0 (success) + ----- stderr ----- + Clearing cache at: [CACHE_DIR]/ + Removed [N] files (at least 1.0MiB) + "); + + Ok(()) +} + /// Put the cache and retained files on CI's Btrfs, APFS, or ReFS volume, when configured. fn copy_on_write_test_context() -> Result { let context = uv_test::test_context!("3.12").with_filtered_counts(); diff --git a/docs/concepts/cache.md b/docs/concepts/cache.md index d8f9e449d83..3c393b65b0f 100644 --- a/docs/concepts/cache.md +++ b/docs/concepts/cache.md @@ -152,6 +152,9 @@ reclaimed, accounting for hardlinks and copy-on-write clones: $ uv cache clean --preview-features cache-reclaimed-space ``` +If an entry's allocated size cannot be measured, such as a compressed extent on Btrfs, uv reports a +lower bound for the space reclaimed from the remaining entries. + uv blocks cache-modifying operations while other uv commands are running. By default, those `uv cache` commands have a 5 min timeout waiting for other uv processes to terminate to avoid deadlocks. This timeout can be changed with From ab03a8cb909f2a3aa978b2e9e05864060d58963a Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Mon, 3 Aug 2026 19:27:14 -0400 Subject: [PATCH 08/13] Limit reclaimed cache space reporting to Unix --- .github/workflows/test.yml | 27 ---- crates/uv-cache/src/lib.rs | 3 +- crates/uv-cache/src/removal.rs | 2 +- crates/uv-fs/src/space.rs | 223 +-------------------------- crates/uv-static/src/env_vars.rs | 5 - crates/uv-test/src/lib.rs | 18 --- crates/uv/tests/build/cache_clean.rs | 61 ++------ crates/uv/tests/build/cache_prune.rs | 14 +- docs/concepts/cache.md | 3 +- 9 files changed, 29 insertions(+), 327 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8b75df888cf..ae135446ecb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -291,33 +291,6 @@ jobs: - name: "Create NTFS test directory (low hardlink limit)" run: New-Item -Path "C:\uv" -ItemType Directory -Force - - name: "Create ReFS test directory (copy-on-write)" - run: | - $devVolume = Get-Volume -DriveLetter $env:DEV_DRIVE.TrimEnd(':') - if ($devVolume.FileSystem -eq "ReFS") { - $refsDrive = $env:DEV_DRIVE - } else { - $vhdPath = Join-Path $env:RUNNER_TEMP "uv-refs-test.vhdx" - @" - create vdisk file="$vhdPath" maximum=1024 type=expandable - attach vdisk - create partition primary - assign letter=R - "@ | diskpart - if ($LASTEXITCODE -ne 0) { - throw "Failed to create the ReFS test volume" - } - format R: /fs:ReFS /q /y - if ($LASTEXITCODE -ne 0) { - throw "Failed to format the ReFS test volume" - } - $refsDrive = "R:" - } - - $refsDirectory = "$refsDrive\uv-cow" - New-Item -Path $refsDirectory -ItemType Directory -Force - "UV_INTERNAL__TEST_REFS_FS=$refsDirectory" >> $env:GITHUB_ENV - - name: "Create SMB test share (alternative filesystem)" run: | New-Item -Path "C:\uv-smb" -ItemType Directory -Force diff --git a/crates/uv-cache/src/lib.rs b/crates/uv-cache/src/lib.rs index 777401e0834..0bd911229b8 100644 --- a/crates/uv-cache/src/lib.rs +++ b/crates/uv-cache/src/lib.rs @@ -211,8 +211,7 @@ impl Cache { /// Enable per-file reclaimed-space accounting when the filesystem can support it. #[must_use] pub fn with_reclaimed_space(self, enabled: bool) -> Self { - let measure_reclaimed_space = - enabled && uv_fs::supports_reclaimable_space(&self.root).unwrap_or(false); + let measure_reclaimed_space = enabled && uv_fs::supports_reclaimable_space(); Self { measure_reclaimed_space, ..self diff --git a/crates/uv-cache/src/removal.rs b/crates/uv-cache/src/removal.rs index 85e2824ed4c..fbda477ffa6 100644 --- a/crates/uv-cache/src/removal.rs +++ b/crates/uv-cache/src/removal.rs @@ -224,7 +224,7 @@ impl std::ops::AddAssign for Removal { } } -#[cfg(test)] +#[cfg(all(test, any(target_os = "linux", target_os = "macos", target_os = "ios")))] mod tests { use super::Removal; diff --git a/crates/uv-fs/src/space.rs b/crates/uv-fs/src/space.rs index 407791629ef..135372f6dc2 100644 --- a/crates/uv-fs/src/space.rs +++ b/crates/uv-fs/src/space.rs @@ -8,40 +8,13 @@ use std::os::unix::ffi::OsStrExt; #[cfg(unix)] use std::os::unix::fs::MetadataExt; -#[cfg(windows)] -use std::os::windows::io::AsRawHandle; - -#[cfg(windows)] -use windows::Win32::Foundation::{ERROR_HANDLE_EOF, ERROR_MORE_DATA, HANDLE}; -#[cfg(windows)] -use windows::Win32::Storage::FileSystem::{ - FILE_STANDARD_INFO, FileStandardInfo, GetFileInformationByHandleEx, - GetVolumeInformationByHandleW, -}; -#[cfg(windows)] -use windows::Win32::System::IO::DeviceIoControl; -#[cfg(windows)] -use windows::Win32::System::Ioctl::{ - FSCTL_GET_INTEGRITY_INFORMATION, FSCTL_GET_INTEGRITY_INFORMATION_BUFFER, - FSCTL_GET_RETRIEVAL_POINTERS_AND_REFCOUNT, RETRIEVAL_POINTERS_AND_REFCOUNT_BUFFER_0, - STARTING_VCN_INPUT_BUFFER, -}; -#[cfg(windows)] -use windows::core::HRESULT; - -/// Return whether the filesystem can identify storage reclaimed by individual files. -pub fn supports_reclaimable_space(path: &Path) -> io::Result { - #[cfg(any(target_os = "linux", target_os = "macos", target_os = "ios", windows))] - { - let _ = path; - Ok(true) - } - - #[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "ios", windows)))] - { - let _ = path; - Ok(false) - } +/// Return whether the current platform can identify storage reclaimed by individual files. +pub const fn supports_reclaimable_space() -> bool { + cfg!(any( + target_os = "linux", + target_os = "macos", + target_os = "ios" + )) } /// Return the allocated file data that would be reclaimed by deleting `path` immediately. @@ -76,12 +49,7 @@ pub fn reclaimable_space(path: &Path, metadata: &std::fs::Metadata) -> io::Resul linux_reclaimable_space(path) } - #[cfg(windows)] - { - windows_reclaimable_space(path) - } - - #[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "ios", windows)))] + #[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "ios")))] { let _ = path; Err(io::Error::new( @@ -252,178 +220,3 @@ fn linux_reclaimable_space(path: &Path) -> io::Result { start = next; } } - -#[cfg(windows)] -#[expect(unsafe_code)] -fn windows_reclaimable_space(path: &Path) -> io::Result { - const FILE_SUPPORTS_BLOCK_REFCOUNTING: u32 = 0x0800_0000; - - let file = fs_err::File::open(path)?; - let handle = HANDLE(file.as_raw_handle()); - let mut information = FILE_STANDARD_INFO::default(); - - // SAFETY: `file` remains open during the call, and `information` is a valid, writable - // `FILE_STANDARD_INFO` buffer whose exact size is passed to Windows. - unsafe { - GetFileInformationByHandleEx( - handle, - FileStandardInfo, - (&raw mut information).cast(), - u32::try_from(std::mem::size_of::()).map_err(io::Error::other)?, - )?; - } - - if information.NumberOfLinks > 1 { - return Ok(0); - } - - let allocated = u64::try_from(information.AllocationSize).map_err(io::Error::other)?; - if allocated == 0 { - return Ok(0); - } - - let mut filesystem_flags = 0; - - // SAFETY: `file` remains open during the call, and `filesystem_flags` points to a valid, - // writable output value. All other optional output values are omitted. - unsafe { - GetVolumeInformationByHandleW( - handle, - None, - None, - None, - Some(&raw mut filesystem_flags), - None, - )?; - } - - if filesystem_flags & FILE_SUPPORTS_BLOCK_REFCOUNTING == 0 { - return Ok(allocated); - } - - refs_reclaimable_space(handle) -} - -/// Sum the exclusively owned allocated extents of a file on ReFS. -#[cfg(windows)] -#[expect(unsafe_code)] -fn refs_reclaimable_space(handle: HANDLE) -> io::Result { - const MAX_EXTENTS: usize = 32; - - #[derive(Default)] - #[repr(C)] - struct RefcountBuffer { - extent_count: u32, - starting_vcn: i64, - extents: [RETRIEVAL_POINTERS_AND_REFCOUNT_BUFFER_0; MAX_EXTENTS], - } - - let mut integrity = FSCTL_GET_INTEGRITY_INFORMATION_BUFFER::default(); - let mut bytes_returned = 0; - - // SAFETY: `handle` is valid for the duration of the call, and `integrity` is a writable - // `FSCTL_GET_INTEGRITY_INFORMATION_BUFFER` whose exact size is passed to Windows. - unsafe { - DeviceIoControl( - handle, - FSCTL_GET_INTEGRITY_INFORMATION, - None, - 0, - Some((&raw mut integrity).cast()), - u32::try_from(std::mem::size_of::()) - .map_err(io::Error::other)?, - Some(&raw mut bytes_returned), - None, - )?; - } - - let cluster_size = u64::from(integrity.ClusterSizeInBytes); - if cluster_size == 0 { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "ReFS returned a zero-byte cluster size", - )); - } - - let mut reclaimable = 0_u64; - let mut starting_vcn = 0_i64; - - loop { - let request = STARTING_VCN_INPUT_BUFFER { - StartingVcn: starting_vcn, - }; - let mut response = RefcountBuffer::default(); - bytes_returned = 0; - - // SAFETY: `handle` remains valid, `request` is an initialized input buffer, and `response` - // has the documented header followed by initialized storage for all requested extents. - let result = unsafe { - DeviceIoControl( - handle, - FSCTL_GET_RETRIEVAL_POINTERS_AND_REFCOUNT, - Some((&raw const request).cast()), - u32::try_from(std::mem::size_of::()) - .map_err(io::Error::other)?, - Some((&raw mut response).cast()), - u32::try_from(std::mem::size_of::()).map_err(io::Error::other)?, - Some(&raw mut bytes_returned), - None, - ) - }; - - let has_more = match result { - Ok(()) => false, - Err(error) if error.code() == HRESULT::from(ERROR_MORE_DATA) => true, - Err(error) if error.code() == HRESULT::from(ERROR_HANDLE_EOF) => { - return Ok(reclaimable); - } - Err(error) => return Err(error.into()), - }; - - let extent_count = usize::try_from(response.extent_count).map_err(io::Error::other)?; - if extent_count > response.extents.len() - || usize::try_from(bytes_returned).map_err(io::Error::other)? - < std::mem::offset_of!(RefcountBuffer, extents) - + extent_count * std::mem::size_of::() - { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "ReFS returned an invalid extent list", - )); - } - - let mut current_vcn = response.starting_vcn; - for extent in &response.extents[..extent_count] { - let clusters = extent.NextVcn.checked_sub(current_vcn).ok_or_else(|| { - io::Error::new( - io::ErrorKind::InvalidData, - "ReFS returned unordered extents", - ) - })?; - if clusters <= 0 { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "ReFS returned a non-advancing extent", - )); - } - - if extent.Lcn >= 0 && extent.ReferenceCount == 1 { - let clusters = u64::try_from(clusters).map_err(io::Error::other)?; - reclaimable = reclaimable.saturating_add(clusters.saturating_mul(cluster_size)); - } - - current_vcn = extent.NextVcn; - } - - if !has_more { - return Ok(reclaimable); - } - if current_vcn <= starting_vcn { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "ReFS returned a non-advancing extent list", - )); - } - starting_vcn = current_vcn; - } -} diff --git a/crates/uv-static/src/env_vars.rs b/crates/uv-static/src/env_vars.rs index cbf159953f3..89b0c77af6e 100644 --- a/crates/uv-static/src/env_vars.rs +++ b/crates/uv-static/src/env_vars.rs @@ -685,11 +685,6 @@ impl EnvVars { #[attr_added_in("0.10.5")] pub const UV_INTERNAL__TEST_COW_FS: &'static str = "UV_INTERNAL__TEST_COW_FS"; - /// Path to a directory on a ReFS filesystem for Windows copy-on-write accounting tests. - #[attr_hidden] - #[attr_added_in("0.12.1")] - pub const UV_INTERNAL__TEST_REFS_FS: &'static str = "UV_INTERNAL__TEST_REFS_FS"; - /// Path to a directory on a filesystem that does **not** support copy-on-write. /// /// When populated, uv will run additional tests that verify fallback behavior diff --git a/crates/uv-test/src/lib.rs b/crates/uv-test/src/lib.rs index 91000b08dc3..680698b36bd 100755 --- a/crates/uv-test/src/lib.rs +++ b/crates/uv-test/src/lib.rs @@ -708,15 +708,6 @@ impl TestContext { self.with_cache_on_fs(&dir, "COW_FS").map(Some) } - /// Use a cache directory on the ReFS filesystem configured for Windows accounting tests. - #[cfg(windows)] - pub fn with_cache_on_refs_fs(self) -> anyhow::Result> { - let Some(dir) = env::var(EnvVars::UV_INTERNAL__TEST_REFS_FS).ok() else { - return Ok(None); - }; - self.with_cache_on_fs(&dir, "REFS_FS").map(Some) - } - /// Use a cache directory on the filesystem specified by /// [`EnvVars::UV_INTERNAL__TEST_ALT_FS`]. /// @@ -763,15 +754,6 @@ impl TestContext { self.with_working_dir_on_fs(&dir, "COW_FS").map(Some) } - /// Use a working directory on the ReFS filesystem configured for Windows accounting tests. - #[cfg(windows)] - pub fn with_working_dir_on_refs_fs(self) -> anyhow::Result> { - let Some(dir) = env::var(EnvVars::UV_INTERNAL__TEST_REFS_FS).ok() else { - return Ok(None); - }; - self.with_working_dir_on_fs(&dir, "REFS_FS").map(Some) - } - /// Use a working directory on the filesystem specified by /// [`EnvVars::UV_INTERNAL__TEST_NOCOW_FS`]. /// diff --git a/crates/uv/tests/build/cache_clean.rs b/crates/uv/tests/build/cache_clean.rs index b4db6b33a57..314b0548f45 100644 --- a/crates/uv/tests/build/cache_clean.rs +++ b/crates/uv/tests/build/cache_clean.rs @@ -6,6 +6,7 @@ use assert_fs::prelude::*; use std::process::Command; use uv_cache::Cache; +#[cfg(unix)] use uv_fs::link::{LinkMode, LinkOptions, link_dir}; use uv_static::EnvVars; @@ -39,23 +40,12 @@ fn clean_all() -> Result<()> { } /// `cache clean` should report reclaimed space for hardlinks only when the preview is enabled. +#[cfg(unix)] #[test] fn clean_all_hardlinked_file() -> Result<()> { let context = uv_test::test_context!("3.12").with_filtered_counts(); - #[cfg(windows)] - let context = if std::env::var_os(EnvVars::UV_INTERNAL__TEST_LOWLINKS_FS).is_some() { - let Some(context) = context.with_cache_on_lowlinks_fs()? else { - return Ok(()); - }; - let cache_dir = context.cache_dir.path().to_path_buf(); - context.with_filtered_path(&cache_dir, "CACHE_DIR") - } else { - context - }; - - // Keep the retained hardlink beside the cache so both entries share a filesystem, even when - // Windows CI explicitly places the cache on its NTFS test volume. + // Keep the retained hardlink beside the cache so both entries share a filesystem. let retained = context.cache_dir.path().with_file_name("retained.bin"); fs_err::write(&retained, vec![42; 1024 * 1024])?; fs_err::OpenOptions::new() @@ -111,6 +101,7 @@ fn clean_all_hardlinked_file() -> Result<()> { } /// `cache clean` should report reclaimed space for copy-on-write clones in preview mode. +#[cfg(unix)] #[test] fn clean_all_cloned_file() -> Result<()> { let context = copy_on_write_test_context()?; @@ -127,7 +118,7 @@ fn clean_all_cloned_file() -> Result<()> { let link_mode = link_dir(&retained, &cached, &LinkOptions::new(LinkMode::Clone))?; if link_mode != LinkMode::Clone { assert!( - !copy_on_write_filesystem_configured(), + std::env::var_os(EnvVars::UV_INTERNAL__TEST_COW_FS).is_none(), "the configured copy-on-write filesystem did not clone the cached file" ); return Ok(()); @@ -153,6 +144,7 @@ fn clean_all_cloned_file() -> Result<()> { } /// Clones shared only within the cache should be counted once when their final reference is removed. +#[cfg(unix)] #[test] fn clean_all_cached_clones() -> Result<()> { let context = copy_on_write_test_context()?; @@ -166,7 +158,7 @@ fn clean_all_cached_clones() -> Result<()> { let link_mode = link_dir(&original, &cloned, &LinkOptions::new(LinkMode::Clone))?; if link_mode != LinkMode::Clone { assert!( - !copy_on_write_filesystem_configured(), + std::env::var_os(EnvVars::UV_INTERNAL__TEST_COW_FS).is_none(), "the configured copy-on-write filesystem did not clone the cached file" ); return Ok(()); @@ -234,48 +226,25 @@ fn clean_all_compressed_file() -> Result<()> { Ok(()) } -/// Put the cache and retained files on CI's Btrfs, APFS, or ReFS volume, when configured. +/// Put the cache and retained files on CI's Btrfs or APFS volume, when configured. +#[cfg(unix)] fn copy_on_write_test_context() -> Result { let context = uv_test::test_context!("3.12").with_filtered_counts(); - if !copy_on_write_filesystem_configured() { + if std::env::var_os(EnvVars::UV_INTERNAL__TEST_COW_FS).is_none() { return Ok(context); } - let context = if std::env::var_os(EnvVars::UV_INTERNAL__TEST_COW_FS).is_some() { - let Some(context) = context.with_cache_on_cow_fs()? else { - anyhow::bail!("the configured copy-on-write cache filesystem was unavailable"); - }; - let Some(context) = context.with_working_dir_on_cow_fs()? else { - anyhow::bail!("the configured copy-on-write working filesystem was unavailable"); - }; - context - } else { - #[cfg(windows)] - { - let Some(context) = context.with_cache_on_refs_fs()? else { - anyhow::bail!("the configured ReFS cache filesystem was unavailable"); - }; - let Some(context) = context.with_working_dir_on_refs_fs()? else { - anyhow::bail!("the configured ReFS working filesystem was unavailable"); - }; - context - } - #[cfg(not(windows))] - { - context - } + let Some(context) = context.with_cache_on_cow_fs()? else { + anyhow::bail!("the configured copy-on-write cache filesystem was unavailable"); + }; + let Some(context) = context.with_working_dir_on_cow_fs()? else { + anyhow::bail!("the configured copy-on-write working filesystem was unavailable"); }; let cache_dir = context.cache_dir.path().to_path_buf(); Ok(context.with_filtered_path(&cache_dir, "CACHE_DIR")) } -/// Return whether CI explicitly configured a copy-on-write filesystem for accounting tests. -fn copy_on_write_filesystem_configured() -> bool { - std::env::var_os(EnvVars::UV_INTERNAL__TEST_COW_FS).is_some() - || cfg!(windows) && std::env::var_os(EnvVars::UV_INTERNAL__TEST_REFS_FS).is_some() -} - /// `cache clear` should behave as an alias of `cache clean`. #[test] fn clear_all_alias() -> Result<()> { diff --git a/crates/uv/tests/build/cache_prune.rs b/crates/uv/tests/build/cache_prune.rs index 2d2ff7ee3f5..fda1836784b 100644 --- a/crates/uv/tests/build/cache_prune.rs +++ b/crates/uv/tests/build/cache_prune.rs @@ -41,22 +41,12 @@ fn prune_no_op() -> Result<()> { } /// `cache prune` should report reclaimed space for hardlinks only when the preview is enabled. +#[cfg(unix)] #[test] fn prune_hardlinked_file() -> Result<()> { let context = uv_test::test_context!("3.12"); - #[cfg(windows)] - let context = if std::env::var_os(EnvVars::UV_INTERNAL__TEST_LOWLINKS_FS).is_some() { - let Some(context) = context.with_cache_on_lowlinks_fs()? else { - return Ok(()); - }; - let cache_dir = context.cache_dir.path().to_path_buf(); - context.with_filtered_path(&cache_dir, "CACHE_DIR") - } else { - context - }; - - // Keep both hardlinks on the selected filesystem, including Windows CI's NTFS test volume. + // Keep both hardlinks on the selected filesystem. let retained = context.cache_dir.path().with_file_name("retained.bin"); fs_err::write(&retained, vec![42; 1024 * 1024])?; fs_err::OpenOptions::new() diff --git a/docs/concepts/cache.md b/docs/concepts/cache.md index 3c393b65b0f..ddf6c17b449 100644 --- a/docs/concepts/cache.md +++ b/docs/concepts/cache.md @@ -153,7 +153,8 @@ $ uv cache clean --preview-features cache-reclaimed-space ``` If an entry's allocated size cannot be measured, such as a compressed extent on Btrfs, uv reports a -lower bound for the space reclaimed from the remaining entries. +lower bound for the space reclaimed from the remaining entries. The preview feature is currently +supported on macOS and Linux; other platforms continue reporting the apparent removed size. uv blocks cache-modifying operations while other uv commands are running. By default, those `uv cache` commands have a 5 min timeout waiting for other uv processes to terminate to avoid From da318953681a996781ebe4a27afbbddf284d7b6f Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Mon, 3 Aug 2026 19:40:54 -0400 Subject: [PATCH 09/13] Report reclaimed space for zero-length files --- crates/uv/src/commands/cache_clean.rs | 4 ++-- crates/uv/src/commands/cache_prune.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/uv/src/commands/cache_clean.rs b/crates/uv/src/commands/cache_clean.rs index 146820c08f9..515d722d200 100644 --- a/crates/uv/src/commands/cache_clean.rs +++ b/crates/uv/src/commands/cache_clean.rs @@ -95,8 +95,8 @@ pub(crate) async fn cache_clean( } // If any, report the reclaimed space, falling back to the apparent removed size. - if summary.total_bytes > 0 { - let total_bytes = summary.reclaimed_bytes.unwrap_or(summary.total_bytes); + let total_bytes = summary.reclaimed_bytes.unwrap_or(summary.total_bytes); + if summary.total_bytes > 0 || total_bytes > 0 { let bytes = if total_bytes < 1024 { format!("{total_bytes}B") } else { diff --git a/crates/uv/src/commands/cache_prune.rs b/crates/uv/src/commands/cache_prune.rs index a72c3d0d470..8297147f3a7 100644 --- a/crates/uv/src/commands/cache_prune.rs +++ b/crates/uv/src/commands/cache_prune.rs @@ -82,8 +82,8 @@ pub(crate) async fn cache_prune( } // If any, report the reclaimed space, falling back to the apparent removed size. - if summary.total_bytes > 0 { - let total_bytes = summary.reclaimed_bytes.unwrap_or(summary.total_bytes); + let total_bytes = summary.reclaimed_bytes.unwrap_or(summary.total_bytes); + if summary.total_bytes > 0 || total_bytes > 0 { let bytes = if total_bytes < 1024 { format!("{total_bytes}B") } else { From 27b61002cb479df6f27977df814c9ce3d0efd83f Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Mon, 3 Aug 2026 19:53:32 -0400 Subject: [PATCH 10/13] Use logical and physical cache-space accounting --- crates/uv-cache/src/lib.rs | 40 +++++---- crates/uv-cache/src/removal.rs | 87 +++++++++++-------- .../generate_preview_features_reference.rs | 2 +- crates/uv-fs/src/lib.rs | 2 +- crates/uv-fs/src/space.rs | 24 ++--- crates/uv-preview/src/lib.rs | 4 +- crates/uv/src/commands/cache_clean.rs | 16 ++-- crates/uv/src/commands/cache_prune.rs | 16 ++-- crates/uv/tests/build/cache_clean.rs | 12 +-- crates/uv/tests/build/cache_prune.rs | 4 +- crates/uv/tests/sync/show_settings.rs | 2 +- docs/concepts/cache.md | 8 +- uv.schema.json | 2 +- 13 files changed, 118 insertions(+), 101 deletions(-) diff --git a/crates/uv-cache/src/lib.rs b/crates/uv-cache/src/lib.rs index 0bd911229b8..7ebfb7dc4fa 100644 --- a/crates/uv-cache/src/lib.rs +++ b/crates/uv-cache/src/lib.rs @@ -18,7 +18,7 @@ pub use crate::by_timestamp::CachedByTimestamp; #[cfg(feature = "clap")] pub use crate::cli::CacheArgs; pub use crate::removal::Removal; -use crate::removal::Remover; +use crate::removal::{RemovalMode, Remover}; pub use crate::wheel::WheelCache; use crate::wheel::WheelCacheKind; pub use archive::ArchiveId; @@ -174,8 +174,8 @@ pub struct Cache { /// Ensure that `uv cache` operations don't remove items from the cache that are used by another /// uv process. lock_file: Option>, - /// Inspect each removed file's actual sharing state when reporting reclaimed cache space. - measure_reclaimed_space: bool, + /// The storage accounting used when removing cache entries. + removal_mode: RemovalMode, } impl Cache { @@ -186,7 +186,7 @@ impl Cache { refresh: Refresh::None(Timestamp::now()), temp_dir: None, lock_file: None, - measure_reclaimed_space: false, + removal_mode: RemovalMode::Logical, } } @@ -198,7 +198,7 @@ impl Cache { refresh: Refresh::None(Timestamp::now()), temp_dir: Some(Arc::new(temp_dir)), lock_file: None, - measure_reclaimed_space: false, + removal_mode: RemovalMode::Logical, }) } @@ -208,19 +208,23 @@ impl Cache { Self { refresh, ..self } } - /// Enable per-file reclaimed-space accounting when the filesystem can support it. + /// Enable per-file physical-space accounting when the filesystem can support it. #[must_use] - pub fn with_reclaimed_space(self, enabled: bool) -> Self { - let measure_reclaimed_space = enabled && uv_fs::supports_reclaimable_space(); + pub fn with_physical_space(self, enabled: bool) -> Self { + let removal_mode = if enabled && uv_fs::supports_physical_space() { + RemovalMode::Physical + } else { + RemovalMode::Logical + }; Self { - measure_reclaimed_space, + removal_mode, ..self } } /// Create an empty removal summary using the cache's configured accounting mode. pub fn removal(&self) -> Removal { - Removal::new(self.measure_reclaimed_space) + Removal::new(self.removal_mode) } /// Acquire a lock that allows removing entries from the cache. @@ -230,7 +234,7 @@ impl Cache { refresh, temp_dir, lock_file, - measure_reclaimed_space, + removal_mode, } = self; // Release the existing lock, avoid deadlocks from a cloned cache. @@ -253,7 +257,7 @@ impl Cache { refresh, temp_dir, lock_file: Some(Arc::new(lock_file)), - measure_reclaimed_space, + removal_mode, }) } @@ -266,7 +270,7 @@ impl Cache { refresh, temp_dir, lock_file, - measure_reclaimed_space, + removal_mode, } = self; match LockedFile::acquire_no_wait( @@ -279,14 +283,14 @@ impl Cache { refresh, temp_dir, lock_file: Some(Arc::new(lock_file)), - measure_reclaimed_space, + removal_mode, }), None => Err(Self { root, refresh, temp_dir, lock_file, - measure_reclaimed_space, + removal_mode, }), } } @@ -546,7 +550,7 @@ impl Cache { pub fn clear(self, reporter: Box) -> Result { // Remove everything but `.lock`, Windows does not allow removal of a locked file let mut removal = Remover::new(reporter) - .with_reclaimed_space(self.measure_reclaimed_space) + .with_removal_mode(self.removal_mode) .rm_rf(&self.root, true)?; let Self { root, lock_file, .. @@ -737,10 +741,10 @@ impl Cache { Ok(summary) } - /// Remove a cache path using the cache's configured reclaimed-space accounting. + /// Remove a cache path using the cache's configured storage accounting. pub fn remove_path(&self, path: impl AsRef) -> io::Result { Remover::default() - .with_reclaimed_space(self.measure_reclaimed_space) + .with_removal_mode(self.removal_mode) .rm_rf(path, false) } diff --git a/crates/uv-cache/src/removal.rs b/crates/uv-cache/src/removal.rs index fbda477ffa6..86538f89a6d 100644 --- a/crates/uv-cache/src/removal.rs +++ b/crates/uv-cache/src/removal.rs @@ -9,11 +9,21 @@ use tracing::debug; use crate::CleanReporter; +/// The storage accounting used when removing cache entries. +#[derive(Debug, Clone, Copy, Default)] +pub(crate) enum RemovalMode { + /// Report the logical size of the removed files. + #[default] + Logical, + /// Report the exclusively owned physical storage reclaimed by the removed files. + Physical, +} + /// A builder for a [`Remover`] that can remove files and directories. #[derive(Default)] pub(crate) struct Remover { reporter: Option>, - measure_reclaimed_space: bool, + removal_mode: RemovalMode, } impl Remover { @@ -21,13 +31,13 @@ impl Remover { pub(crate) fn new(reporter: Box) -> Self { Self { reporter: Some(reporter), - measure_reclaimed_space: false, + ..Self::default() } } - /// Enable accounting for exclusively owned storage before each file is removed. - pub(crate) fn with_reclaimed_space(mut self, enabled: bool) -> Self { - self.measure_reclaimed_space = enabled; + /// Set the storage accounting used before each file is removed. + pub(crate) fn with_removal_mode(mut self, removal_mode: RemovalMode) -> Self { + self.removal_mode = removal_mode; self } @@ -38,7 +48,7 @@ impl Remover { path: impl AsRef, skip_locked_file: bool, ) -> io::Result { - let mut removal = Removal::new(self.measure_reclaimed_space); + let mut removal = Removal::new(self.removal_mode); removal.rm_rf(path.as_ref(), self.reporter.as_deref(), skip_locked_file)?; Ok(removal) } @@ -51,41 +61,44 @@ pub struct Removal { pub num_files: u64, /// The number of directories removed. pub num_dirs: u64, - /// The total number of bytes removed. + /// The logical number of bytes removed. /// /// Note: this will both over-count bytes removed for hard-linked files, and under-count /// bytes in general since it's a measure of the exact byte size (as opposed to the block size). - pub total_bytes: u64, - /// The exclusively owned allocated file data reclaimed by the removal, when available. - pub reclaimed_bytes: Option, - /// Whether any removed entries could not be measured, making the reclaimed count a lower bound. - pub reclaimed_bytes_incomplete: bool, + pub logical_bytes: u64, + /// The exclusively owned physical file data reclaimed by the removal, when available. + pub physical_bytes: Option, + /// Whether any removed entries could not be measured, making the physical count a lower bound. + pub physical_bytes_incomplete: bool, } impl Removal { - /// Create an empty removal summary with optional per-file reclaimed-space accounting. - pub(crate) fn new(measure_reclaimed_space: bool) -> Self { + /// Create an empty removal summary with the requested storage accounting. + pub(crate) fn new(removal_mode: RemovalMode) -> Self { Self { - reclaimed_bytes: measure_reclaimed_space.then_some(0), + physical_bytes: match removal_mode { + RemovalMode::Logical => None, + RemovalMode::Physical => Some(0), + }, ..Self::default() } } /// Account for a file while its current sharing state can still be inspected. fn add_file(&mut self, path: &Path, metadata: &std::fs::Metadata) { - self.total_bytes += metadata.len(); + self.logical_bytes += metadata.len(); - if let Some(reclaimed_bytes) = self.reclaimed_bytes { - match uv_fs::reclaimable_space(path, metadata) { - Ok(reclaimable) => { - self.reclaimed_bytes = Some(reclaimed_bytes.saturating_add(reclaimable)); + if let Some(physical_bytes) = self.physical_bytes { + match uv_fs::physical_space(path, metadata) { + Ok(physical) => { + self.physical_bytes = Some(physical_bytes.saturating_add(physical)); } Err(error) => { debug!( - "Failed to measure reclaimed space for {}: {error}", + "Failed to measure physical space for {}: {error}", path.display() ); - self.reclaimed_bytes_incomplete = true; + self.physical_bytes_incomplete = true; } } } @@ -196,8 +209,8 @@ impl Removal { // Remove the file. if let Ok(metadata) = entry.metadata() { self.add_file(entry.path(), &metadata); - } else if self.reclaimed_bytes.is_some() { - self.reclaimed_bytes_incomplete = true; + } else if self.physical_bytes.is_some() { + self.physical_bytes_incomplete = true; } remove_file(entry.path())?; } @@ -215,18 +228,18 @@ impl std::ops::AddAssign for Removal { fn add_assign(&mut self, other: Self) { self.num_files += other.num_files; self.num_dirs += other.num_dirs; - self.total_bytes += other.total_bytes; - self.reclaimed_bytes = self - .reclaimed_bytes - .zip(other.reclaimed_bytes) + self.logical_bytes += other.logical_bytes; + self.physical_bytes = self + .physical_bytes + .zip(other.physical_bytes) .map(|(left, right)| left.saturating_add(right)); - self.reclaimed_bytes_incomplete |= other.reclaimed_bytes_incomplete; + self.physical_bytes_incomplete |= other.physical_bytes_incomplete; } } #[cfg(all(test, any(target_os = "linux", target_os = "macos", target_os = "ios")))] mod tests { - use super::Removal; + use super::{Removal, RemovalMode}; #[test] fn retain_measured_space_when_an_entry_cannot_be_measured() -> std::io::Result<()> { @@ -234,20 +247,20 @@ mod tests { let measured = directory.path().join("measured.bin"); fs_err::write(&measured, vec![42; 4096])?; let metadata = fs_err::metadata(&measured)?; - let expected = uv_fs::reclaimable_space(&measured, &metadata)?; + let expected = uv_fs::physical_space(&measured, &metadata)?; - let mut removal = Removal::new(true); + let mut removal = Removal::new(RemovalMode::Physical); removal.add_file(&measured, &metadata); removal.add_file(&directory.path().join("missing.bin"), &metadata); removal.add_file(&measured, &metadata); - assert_eq!(removal.reclaimed_bytes, Some(expected.saturating_mul(2))); - assert!(removal.reclaimed_bytes_incomplete); + assert_eq!(removal.physical_bytes, Some(expected.saturating_mul(2))); + assert!(removal.physical_bytes_incomplete); - let mut combined = Removal::new(true); + let mut combined = Removal::new(RemovalMode::Physical); combined += removal; - assert_eq!(combined.reclaimed_bytes, Some(expected.saturating_mul(2))); - assert!(combined.reclaimed_bytes_incomplete); + assert_eq!(combined.physical_bytes, Some(expected.saturating_mul(2))); + assert!(combined.physical_bytes_incomplete); Ok(()) } diff --git a/crates/uv-dev/src/generate_preview_features_reference.rs b/crates/uv-dev/src/generate_preview_features_reference.rs index 23b29fcd0ff..b8d005b6f82 100644 --- a/crates/uv-dev/src/generate_preview_features_reference.rs +++ b/crates/uv-dev/src/generate_preview_features_reference.rs @@ -111,7 +111,7 @@ mod tests { - `audit-command`: Allows using `uv audit`. - `auth-helper`: Allows using `uv auth helper` as a credential helper for external tools. - `azure-endpoint`: Allows signing requests to Azure Blob Storage endpoints with Azure credentials. - - `cache-reclaimed-space`: Reports the disk space reclaimed by cache cleanup, accounting for hardlinks and copy-on-write clones. + - `cache-physical-space`: Reports the physical disk space reclaimed by cache cleanup, accounting for hardlinks and copy-on-write clones. - `cache-size`: Allows using `uv cache size`. - `centralized-project-envs`: Stores [project virtual environments](./projects/layout.md#centralized-project-environments) in the uv cache. diff --git a/crates/uv-fs/src/lib.rs b/crates/uv-fs/src/lib.rs index ea43bcbea2f..3c6e0ed62c1 100644 --- a/crates/uv-fs/src/lib.rs +++ b/crates/uv-fs/src/lib.rs @@ -12,7 +12,7 @@ use tracing::{debug, warn}; pub use crate::locked_file::*; pub use crate::path::*; pub use crate::read::ValidatedReader; -pub use crate::space::{reclaimable_space, supports_reclaimable_space}; +pub use crate::space::{physical_space, supports_physical_space}; pub mod cachedir; pub mod link; diff --git a/crates/uv-fs/src/space.rs b/crates/uv-fs/src/space.rs index 135372f6dc2..4f042fe51ba 100644 --- a/crates/uv-fs/src/space.rs +++ b/crates/uv-fs/src/space.rs @@ -8,8 +8,8 @@ use std::os::unix::ffi::OsStrExt; #[cfg(unix)] use std::os::unix::fs::MetadataExt; -/// Return whether the current platform can identify storage reclaimed by individual files. -pub const fn supports_reclaimable_space() -> bool { +/// Return whether the current platform can identify individual files' physical storage. +pub const fn supports_physical_space() -> bool { cfg!(any( target_os = "linux", target_os = "macos", @@ -17,11 +17,11 @@ pub const fn supports_reclaimable_space() -> bool { )) } -/// Return the allocated file data that would be reclaimed by deleting `path` immediately. +/// Return the physical file data that would be reclaimed by deleting `path`. /// /// The result excludes data retained by another hardlink, copy-on-write clone, or snapshot. /// Filesystem metadata is not included. -pub fn reclaimable_space(path: &Path, metadata: &std::fs::Metadata) -> io::Result { +pub fn physical_space(path: &Path, metadata: &std::fs::Metadata) -> io::Result { if !metadata.is_file() { #[cfg(unix)] { @@ -41,12 +41,12 @@ pub fn reclaimable_space(path: &Path, metadata: &std::fs::Metadata) -> io::Resul #[cfg(any(target_os = "macos", target_os = "ios"))] { - apple_reclaimable_space(path) + apple_physical_space(path) } #[cfg(target_os = "linux")] { - linux_reclaimable_space(path) + linux_physical_space(path) } #[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "ios")))] @@ -61,7 +61,7 @@ pub fn reclaimable_space(path: &Path, metadata: &std::fs::Metadata) -> io::Resul #[cfg(any(target_os = "macos", target_os = "ios"))] #[expect(unsafe_code)] -fn apple_reclaimable_space(path: &Path) -> io::Result { +fn apple_physical_space(path: &Path) -> io::Result { let path = CString::new(path.as_os_str().as_bytes()) .map_err(|error| io::Error::new(io::ErrorKind::InvalidInput, error))?; let mut attributes = libc::attrlist { @@ -106,7 +106,7 @@ fn apple_reclaimable_space(path: &Path) -> io::Result { #[cfg(target_os = "linux")] #[expect(unsafe_code)] -fn linux_reclaimable_space(path: &Path) -> io::Result { +fn linux_physical_space(path: &Path) -> io::Result { const FIEMAP_EXTENT_LAST: u32 = 0x0000_0001; const FIEMAP_EXTENT_UNKNOWN: u32 = 0x0000_0002; const FIEMAP_EXTENT_DELALLOC: u32 = 0x0000_0004; @@ -149,7 +149,7 @@ fn linux_reclaimable_space(path: &Path) -> io::Result { rustix::ioctl::opcode::read_write::(b'f', 11); let file = fs_err::File::open(path)?; - let mut reclaimable = 0_u64; + let mut physical = 0_u64; let mut start = 0_u64; loop { @@ -181,7 +181,7 @@ fn linux_reclaimable_space(path: &Path) -> io::Result { )); } if mapped_extents == 0 { - return Ok(reclaimable); + return Ok(physical); } for extent in &request.extents[..mapped_extents] { @@ -202,12 +202,12 @@ fn linux_reclaimable_space(path: &Path) -> io::Result { )); } - reclaimable = reclaimable.saturating_add(extent.length); + physical = physical.saturating_add(extent.length); } let last_extent = &request.extents[mapped_extents - 1]; if last_extent.flags & FIEMAP_EXTENT_LAST != 0 { - return Ok(reclaimable); + return Ok(physical); } let next = last_extent.logical.saturating_add(last_extent.length); diff --git a/crates/uv-preview/src/lib.rs b/crates/uv-preview/src/lib.rs index f353b3b9996..ea00c808df1 100644 --- a/crates/uv-preview/src/lib.rs +++ b/crates/uv-preview/src/lib.rs @@ -253,8 +253,8 @@ pub enum PreviewFeature { S3Endpoint, /// Allows using `uv cache size`. CacheSize, - /// Reports the disk space reclaimed by cache cleanup, accounting for hardlinks and copy-on-write clones. - CacheReclaimedSpace, + /// Reports the physical disk space reclaimed by cache cleanup, accounting for hardlinks and copy-on-write clones. + CachePhysicalSpace, /// Rejects the deprecated `--project` option in `uv init`. InitProjectFlag, /// Allows using `uv workspace metadata`. diff --git a/crates/uv/src/commands/cache_clean.rs b/crates/uv/src/commands/cache_clean.rs index 515d722d200..5ba48f3d470 100644 --- a/crates/uv/src/commands/cache_clean.rs +++ b/crates/uv/src/commands/cache_clean.rs @@ -45,7 +45,7 @@ pub(crate) async fn cache_clean( } }; - let cache = cache.with_reclaimed_space(preview.is_enabled(PreviewFeature::CacheReclaimedSpace)); + let cache = cache.with_physical_space(preview.is_enabled(PreviewFeature::CachePhysicalSpace)); let summary = if packages.is_empty() { writeln!( @@ -94,16 +94,16 @@ pub(crate) async fn cache_clean( } } - // If any, report the reclaimed space, falling back to the apparent removed size. - let total_bytes = summary.reclaimed_bytes.unwrap_or(summary.total_bytes); - if summary.total_bytes > 0 || total_bytes > 0 { - let bytes = if total_bytes < 1024 { - format!("{total_bytes}B") + // If any, report the physical space, falling back to the logical removed size. + let reported_bytes = summary.physical_bytes.unwrap_or(summary.logical_bytes); + if summary.logical_bytes > 0 || reported_bytes > 0 { + let bytes = if reported_bytes < 1024 { + format!("{reported_bytes}B") } else { - let (bytes, unit) = human_readable_bytes(total_bytes); + let (bytes, unit) = human_readable_bytes(reported_bytes); format!("{bytes:.1}{unit}") }; - if summary.reclaimed_bytes_incomplete { + if summary.physical_bytes_incomplete { write!(printer.stderr(), " (at least {})", bytes.green())?; } else { write!(printer.stderr(), " ({})", bytes.green())?; diff --git a/crates/uv/src/commands/cache_prune.rs b/crates/uv/src/commands/cache_prune.rs index 8297147f3a7..dc11ee030fe 100644 --- a/crates/uv/src/commands/cache_prune.rs +++ b/crates/uv/src/commands/cache_prune.rs @@ -43,7 +43,7 @@ pub(crate) async fn cache_prune( } }; - let cache = cache.with_reclaimed_space(preview.is_enabled(PreviewFeature::CacheReclaimedSpace)); + let cache = cache.with_physical_space(preview.is_enabled(PreviewFeature::CachePhysicalSpace)); writeln!( printer.stderr(), @@ -81,16 +81,16 @@ pub(crate) async fn cache_prune( } } - // If any, report the reclaimed space, falling back to the apparent removed size. - let total_bytes = summary.reclaimed_bytes.unwrap_or(summary.total_bytes); - if summary.total_bytes > 0 || total_bytes > 0 { - let bytes = if total_bytes < 1024 { - format!("{total_bytes}B") + // If any, report the physical space, falling back to the logical removed size. + let reported_bytes = summary.physical_bytes.unwrap_or(summary.logical_bytes); + if summary.logical_bytes > 0 || reported_bytes > 0 { + let bytes = if reported_bytes < 1024 { + format!("{reported_bytes}B") } else { - let (bytes, unit) = human_readable_bytes(total_bytes); + let (bytes, unit) = human_readable_bytes(reported_bytes); format!("{bytes:.1}{unit}") }; - if summary.reclaimed_bytes_incomplete { + if summary.physical_bytes_incomplete { write!(printer.stderr(), " (at least {})", bytes.green())?; } else { write!(printer.stderr(), " ({})", bytes.green())?; diff --git a/crates/uv/tests/build/cache_clean.rs b/crates/uv/tests/build/cache_clean.rs index 314b0548f45..85a5ec23534 100644 --- a/crates/uv/tests/build/cache_clean.rs +++ b/crates/uv/tests/build/cache_clean.rs @@ -39,7 +39,7 @@ fn clean_all() -> Result<()> { Ok(()) } -/// `cache clean` should report reclaimed space for hardlinks only when the preview is enabled. +/// `cache clean` should report physical space for hardlinks only when the preview is enabled. #[cfg(unix)] #[test] fn clean_all_hardlinked_file() -> Result<()> { @@ -72,7 +72,7 @@ fn clean_all_hardlinked_file() -> Result<()> { context.cache_dir.create_dir_all()?; fs_err::hard_link(&retained, &cached)?; - uv_snapshot!(&filters, context.clean().arg("--preview-features").arg("cache-reclaimed-space"), @" + uv_snapshot!(&filters, context.clean().arg("--preview-features").arg("cache-physical-space"), @" exit_code: 0 (success) ----- stderr ----- Clearing cache at: [CACHE_DIR]/ @@ -90,7 +90,7 @@ fn clean_all_hardlinked_file() -> Result<()> { .sync_all()?; fs_err::hard_link(&cached, context.cache_dir.child("second-hardlink.bin"))?; - uv_snapshot!(&filters, context.clean().arg("--preview-features").arg("cache-reclaimed-space"), @" + uv_snapshot!(&filters, context.clean().arg("--preview-features").arg("cache-physical-space"), @" exit_code: 0 (success) ----- stderr ----- Clearing cache at: [CACHE_DIR]/ @@ -100,7 +100,7 @@ fn clean_all_hardlinked_file() -> Result<()> { Ok(()) } -/// `cache clean` should report reclaimed space for copy-on-write clones in preview mode. +/// `cache clean` should report physical space for copy-on-write clones in preview mode. #[cfg(unix)] #[test] fn clean_all_cloned_file() -> Result<()> { @@ -170,7 +170,7 @@ fn clean_all_cached_clones() -> Result<()> { .filter(|(_, replacement)| *replacement != "$1[SIZE]") .collect(); - uv_snapshot!(&filters, context.clean().arg("--preview-features").arg("cache-reclaimed-space"), @" + uv_snapshot!(&filters, context.clean().arg("--preview-features").arg("cache-physical-space"), @" exit_code: 0 (success) ----- stderr ----- Clearing cache at: [CACHE_DIR]/ @@ -216,7 +216,7 @@ fn clean_all_compressed_file() -> Result<()> { .filter(|(_, replacement)| *replacement != "$1[SIZE]") .collect(); - uv_snapshot!(&filters, context.clean().arg("--preview-features").arg("cache-reclaimed-space"), @" + uv_snapshot!(&filters, context.clean().arg("--preview-features").arg("cache-physical-space"), @" exit_code: 0 (success) ----- stderr ----- Clearing cache at: [CACHE_DIR]/ diff --git a/crates/uv/tests/build/cache_prune.rs b/crates/uv/tests/build/cache_prune.rs index fda1836784b..bd7aac28598 100644 --- a/crates/uv/tests/build/cache_prune.rs +++ b/crates/uv/tests/build/cache_prune.rs @@ -40,7 +40,7 @@ fn prune_no_op() -> Result<()> { Ok(()) } -/// `cache prune` should report reclaimed space for hardlinks only when the preview is enabled. +/// `cache prune` should report physical space for hardlinks only when the preview is enabled. #[cfg(unix)] #[test] fn prune_hardlinked_file() -> Result<()> { @@ -74,7 +74,7 @@ fn prune_hardlinked_file() -> Result<()> { stale.create_dir_all()?; fs_err::hard_link(&retained, stale.child("hardlinked.bin"))?; - uv_snapshot!(&filters, context.prune().arg("--preview-features").arg("cache-reclaimed-space"), @" + uv_snapshot!(&filters, context.prune().arg("--preview-features").arg("cache-physical-space"), @" exit_code: 0 (success) ----- stderr ----- Pruning cache at: [CACHE_DIR]/ diff --git a/crates/uv/tests/sync/show_settings.rs b/crates/uv/tests/sync/show_settings.rs index 35851e13e81..947fce21bb3 100644 --- a/crates/uv/tests/sync/show_settings.rs +++ b/crates/uv/tests/sync/show_settings.rs @@ -3386,7 +3386,7 @@ fn preview_features() { + NativeAuth, + S3Endpoint, + CacheSize, - + CacheReclaimedSpace, + + CachePhysicalSpace, + InitProjectFlag, + WorkspaceMetadata, + WorkspaceDir, diff --git a/docs/concepts/cache.md b/docs/concepts/cache.md index ddf6c17b449..1b93ac39557 100644 --- a/docs/concepts/cache.md +++ b/docs/concepts/cache.md @@ -144,17 +144,17 @@ uv provides a few different mechanisms for removing entries from the cache: longer necessary and can be safely removed. Centralized project environments are recreated as needed. `uv cache prune` is safe to run periodically, to keep the cache directory clean. -By default, cache cleanup reports the apparent size of removed entries. Enable the -`cache-reclaimed-space` [preview feature](./preview.md) to instead report the actual disk space +By default, cache cleanup reports the logical size of removed entries. Enable the +`cache-physical-space` [preview feature](./preview.md) to instead report the physical disk space reclaimed, accounting for hardlinks and copy-on-write clones: ```console -$ uv cache clean --preview-features cache-reclaimed-space +$ uv cache clean --preview-features cache-physical-space ``` If an entry's allocated size cannot be measured, such as a compressed extent on Btrfs, uv reports a lower bound for the space reclaimed from the remaining entries. The preview feature is currently -supported on macOS and Linux; other platforms continue reporting the apparent removed size. +supported on macOS and Linux; other platforms continue reporting the logical removed size. uv blocks cache-modifying operations while other uv commands are running. By default, those `uv cache` commands have a 5 min timeout waiting for other uv processes to terminate to avoid diff --git a/uv.schema.json b/uv.schema.json index 3d68859e91e..be23640ee9a 100644 --- a/uv.schema.json +++ b/uv.schema.json @@ -1794,7 +1794,7 @@ "native-auth", "s3-endpoint", "cache-size", - "cache-reclaimed-space", + "cache-physical-space", "init-project-flag", "workspace-metadata", "workspace-dir", From 6f258bd0a686bdc7c030a4f19a665ffda56c4ca3 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Mon, 3 Aug 2026 20:05:49 -0400 Subject: [PATCH 11/13] Use explicit cache removal modes --- crates/uv-cache/src/lib.rs | 17 ++++++++------- crates/uv-cache/src/removal.rs | 31 +-------------------------- crates/uv/src/commands/cache_clean.rs | 9 ++++++-- crates/uv/src/commands/cache_prune.rs | 9 ++++++-- 4 files changed, 24 insertions(+), 42 deletions(-) diff --git a/crates/uv-cache/src/lib.rs b/crates/uv-cache/src/lib.rs index 7ebfb7dc4fa..5592c042233 100644 --- a/crates/uv-cache/src/lib.rs +++ b/crates/uv-cache/src/lib.rs @@ -17,8 +17,8 @@ use uv_pypi_types::ResolutionMetadata; pub use crate::by_timestamp::CachedByTimestamp; #[cfg(feature = "clap")] pub use crate::cli::CacheArgs; -pub use crate::removal::Removal; -use crate::removal::{RemovalMode, Remover}; +use crate::removal::Remover; +pub use crate::removal::{Removal, RemovalMode}; pub use crate::wheel::WheelCache; use crate::wheel::WheelCacheKind; pub use archive::ArchiveId; @@ -208,13 +208,14 @@ impl Cache { Self { refresh, ..self } } - /// Enable per-file physical-space accounting when the filesystem can support it. + /// Set the storage accounting used when removing cache entries. + /// + /// Falls back to logical accounting when physical accounting is unsupported. #[must_use] - pub fn with_physical_space(self, enabled: bool) -> Self { - let removal_mode = if enabled && uv_fs::supports_physical_space() { - RemovalMode::Physical - } else { - RemovalMode::Logical + pub fn with_removal_mode(self, removal_mode: RemovalMode) -> Self { + let removal_mode = match removal_mode { + RemovalMode::Physical if !uv_fs::supports_physical_space() => RemovalMode::Logical, + removal_mode => removal_mode, }; Self { removal_mode, diff --git a/crates/uv-cache/src/removal.rs b/crates/uv-cache/src/removal.rs index 86538f89a6d..23a055f7b8a 100644 --- a/crates/uv-cache/src/removal.rs +++ b/crates/uv-cache/src/removal.rs @@ -11,7 +11,7 @@ use crate::CleanReporter; /// The storage accounting used when removing cache entries. #[derive(Debug, Clone, Copy, Default)] -pub(crate) enum RemovalMode { +pub enum RemovalMode { /// Report the logical size of the removed files. #[default] Logical, @@ -237,35 +237,6 @@ impl std::ops::AddAssign for Removal { } } -#[cfg(all(test, any(target_os = "linux", target_os = "macos", target_os = "ios")))] -mod tests { - use super::{Removal, RemovalMode}; - - #[test] - fn retain_measured_space_when_an_entry_cannot_be_measured() -> std::io::Result<()> { - let directory = tempfile::tempdir()?; - let measured = directory.path().join("measured.bin"); - fs_err::write(&measured, vec![42; 4096])?; - let metadata = fs_err::metadata(&measured)?; - let expected = uv_fs::physical_space(&measured, &metadata)?; - - let mut removal = Removal::new(RemovalMode::Physical); - removal.add_file(&measured, &metadata); - removal.add_file(&directory.path().join("missing.bin"), &metadata); - removal.add_file(&measured, &metadata); - - assert_eq!(removal.physical_bytes, Some(expected.saturating_mul(2))); - assert!(removal.physical_bytes_incomplete); - - let mut combined = Removal::new(RemovalMode::Physical); - combined += removal; - assert_eq!(combined.physical_bytes, Some(expected.saturating_mul(2))); - assert!(combined.physical_bytes_incomplete); - - Ok(()) - } -} - /// If the directory isn't readable by the current user, change the permissions to make it readable. #[cfg_attr(windows, allow(unused_variables, clippy::unnecessary_wraps))] fn set_readable(path: &Path) -> io::Result { diff --git a/crates/uv/src/commands/cache_clean.rs b/crates/uv/src/commands/cache_clean.rs index 5ba48f3d470..3166a1089ce 100644 --- a/crates/uv/src/commands/cache_clean.rs +++ b/crates/uv/src/commands/cache_clean.rs @@ -4,7 +4,7 @@ use anyhow::{Context, Result}; use owo_colors::OwoColorize; use tracing::debug; -use uv_cache::Cache; +use uv_cache::{Cache, RemovalMode}; use uv_fs::Simplified; use uv_normalize::PackageName; use uv_preview::{Preview, PreviewFeature}; @@ -45,7 +45,12 @@ pub(crate) async fn cache_clean( } }; - let cache = cache.with_physical_space(preview.is_enabled(PreviewFeature::CachePhysicalSpace)); + let removal_mode = if preview.is_enabled(PreviewFeature::CachePhysicalSpace) { + RemovalMode::Physical + } else { + RemovalMode::Logical + }; + let cache = cache.with_removal_mode(removal_mode); let summary = if packages.is_empty() { writeln!( diff --git a/crates/uv/src/commands/cache_prune.rs b/crates/uv/src/commands/cache_prune.rs index dc11ee030fe..7109eddef21 100644 --- a/crates/uv/src/commands/cache_prune.rs +++ b/crates/uv/src/commands/cache_prune.rs @@ -4,7 +4,7 @@ use anyhow::{Context, Result}; use owo_colors::OwoColorize; use tracing::debug; -use uv_cache::Cache; +use uv_cache::{Cache, RemovalMode}; use uv_fs::Simplified; use uv_preview::{Preview, PreviewFeature}; @@ -43,7 +43,12 @@ pub(crate) async fn cache_prune( } }; - let cache = cache.with_physical_space(preview.is_enabled(PreviewFeature::CachePhysicalSpace)); + let removal_mode = if preview.is_enabled(PreviewFeature::CachePhysicalSpace) { + RemovalMode::Physical + } else { + RemovalMode::Logical + }; + let cache = cache.with_removal_mode(removal_mode); writeln!( printer.stderr(), From 2011df8e8238a747056de3838aefa1ad8def0198 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Mon, 3 Aug 2026 20:19:49 -0400 Subject: [PATCH 12/13] Simplify cache space accounting integration tests --- crates/uv/tests/build/cache_clean.rs | 47 +++++++++++----------------- crates/uv/tests/build/cache_prune.rs | 1 - 2 files changed, 18 insertions(+), 30 deletions(-) diff --git a/crates/uv/tests/build/cache_clean.rs b/crates/uv/tests/build/cache_clean.rs index 85a5ec23534..7ede44543b4 100644 --- a/crates/uv/tests/build/cache_clean.rs +++ b/crates/uv/tests/build/cache_clean.rs @@ -56,11 +56,7 @@ fn clean_all_hardlinked_file() -> Result<()> { let cached = context.cache_dir.child("hardlinked.bin"); fs_err::hard_link(&retained, &cached)?; - let filters: Vec<_> = context - .filters() - .into_iter() - .filter(|(_, replacement)| *replacement != "$1[SIZE]") - .collect(); + let filters = size_filters(&context); uv_snapshot!(&filters, context.clean(), @" exit_code: 0 (success) @@ -80,7 +76,6 @@ fn clean_all_hardlinked_file() -> Result<()> { "); assert!(retained.is_file()); - assert_eq!(fs_err::metadata(retained)?.len(), 1024 * 1024); context.cache_dir.create_dir_all()?; cached.write_binary(&vec![42; 1024 * 1024])?; @@ -105,10 +100,10 @@ fn clean_all_hardlinked_file() -> Result<()> { #[test] fn clean_all_cloned_file() -> Result<()> { let context = copy_on_write_test_context()?; - let retained = context.temp_dir.child("retained"); - retained.create_dir_all()?; - let original = retained.child("original.bin"); - original.write_binary(&vec![42; 1024 * 1024])?; + let retained = context.cache_dir.path().with_file_name("retained"); + fs_err::create_dir_all(&retained)?; + let original = retained.join("original.bin"); + fs_err::write(&original, vec![42; 1024 * 1024])?; // Remove unrelated cache entries so the cloned file is the only allocated data being cleaned. context.clean().assert().success(); @@ -124,11 +119,7 @@ fn clean_all_cloned_file() -> Result<()> { return Ok(()); } - let filters: Vec<_> = context - .filters() - .into_iter() - .filter(|(_, replacement)| *replacement != "$1[SIZE]") - .collect(); + let filters = size_filters(&context); uv_snapshot!(&filters, context.clean().arg("--preview"), @" exit_code: 0 (success) @@ -138,7 +129,6 @@ fn clean_all_cloned_file() -> Result<()> { "); assert!(original.is_file()); - assert_eq!(fs_err::metadata(original)?.len(), 1024 * 1024); Ok(()) } @@ -164,11 +154,7 @@ fn clean_all_cached_clones() -> Result<()> { return Ok(()); } - let filters: Vec<_> = context - .filters() - .into_iter() - .filter(|(_, replacement)| *replacement != "$1[SIZE]") - .collect(); + let filters = size_filters(&context); uv_snapshot!(&filters, context.clean().arg("--preview-features").arg("cache-physical-space"), @" exit_code: 0 (success) @@ -210,11 +196,7 @@ fn clean_all_compressed_file() -> Result<()> { .open(&compressed)? .sync_all()?; - let filters: Vec<_> = context - .filters() - .into_iter() - .filter(|(_, replacement)| *replacement != "$1[SIZE]") - .collect(); + let filters = size_filters(&context); uv_snapshot!(&filters, context.clean().arg("--preview-features").arg("cache-physical-space"), @" exit_code: 0 (success) @@ -237,14 +219,21 @@ fn copy_on_write_test_context() -> Result { let Some(context) = context.with_cache_on_cow_fs()? else { anyhow::bail!("the configured copy-on-write cache filesystem was unavailable"); }; - let Some(context) = context.with_working_dir_on_cow_fs()? else { - anyhow::bail!("the configured copy-on-write working filesystem was unavailable"); - }; let cache_dir = context.cache_dir.path().to_path_buf(); Ok(context.with_filtered_path(&cache_dir, "CACHE_DIR")) } +/// Preserve physical sizes while applying the context's other snapshot filters. +#[cfg(unix)] +fn size_filters(context: &uv_test::TestContext) -> Vec<(&str, &str)> { + context + .filters() + .into_iter() + .filter(|(_, replacement)| *replacement != "$1[SIZE]") + .collect() +} + /// `cache clear` should behave as an alias of `cache clean`. #[test] fn clear_all_alias() -> Result<()> { diff --git a/crates/uv/tests/build/cache_prune.rs b/crates/uv/tests/build/cache_prune.rs index bd7aac28598..4a4c55587ae 100644 --- a/crates/uv/tests/build/cache_prune.rs +++ b/crates/uv/tests/build/cache_prune.rs @@ -82,7 +82,6 @@ fn prune_hardlinked_file() -> Result<()> { "); assert!(retained.is_file()); - assert_eq!(fs_err::metadata(retained)?.len(), 1024 * 1024); Ok(()) } From 68311bf12ba9bb78b5ada7c8ebdd0ee963416222 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Mon, 3 Aug 2026 20:24:56 -0400 Subject: [PATCH 13/13] Fix Linux cache accounting test paths --- crates/uv/tests/build/cache_clean.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/uv/tests/build/cache_clean.rs b/crates/uv/tests/build/cache_clean.rs index 7ede44543b4..e2bf23fbfc9 100644 --- a/crates/uv/tests/build/cache_clean.rs +++ b/crates/uv/tests/build/cache_clean.rs @@ -179,11 +179,11 @@ fn clean_all_compressed_file() -> Result<()> { measured.write_binary(&vec![42; 1024 * 1024])?; fs_err::OpenOptions::new() .write(true) - .open(&measured)? + .open(measured.path())? .sync_all()?; let compressed = context.cache_dir.child("compressed.bin"); - fs_err::File::create(&compressed)?; + fs_err::File::create(compressed.path())?; Command::new("btrfs") .args(["property", "set"]) .arg(compressed.path()) @@ -193,7 +193,7 @@ fn clean_all_compressed_file() -> Result<()> { compressed.write_binary(&vec![42; 1024 * 1024])?; fs_err::OpenOptions::new() .write(true) - .open(&compressed)? + .open(compressed.path())? .sync_all()?; let filters = size_filters(&context);