Deduplicate all files in the wheel cache

crates/uv-bench/benches/uv.rs

@@ -5,7 +5,7 @@ extern crate uv_performance_memory_allocator;
 use std::env;
 use std::fmt::Write;
 use std::hint::black_box;
-use std::path::{Path, PathBuf};
+use std::path::Path;
 use std::str::FromStr;
 
 use async_zip::base::write::ZipFileWriter;
@@ -21,6 +21,7 @@ use uv_cache::Cache;
 use uv_client::{BaseClientBuilder, Connectivity, RegistryClientBuilder};
 use uv_distribution_filename::{SourceDistExtension, WheelFilename};
 use uv_distribution_types::Requirement;
+use uv_extract::dirhash::UnhashedFile;
 use uv_install_wheel::{InstallState, Layout, LinkMode};
 use uv_preview::{MaybePreviewFeature, Preview, PreviewFeature};
 use uv_pypi_types::Scheme;
@@ -264,10 +265,14 @@ fn prepare_wheel(
     archive: fs_err::File,
     extracted_wheel: &Path,
     filename: &WheelFilename,
-) -> Vec<(PathBuf, u64)> {
+) -> Vec<UnhashedFile> {
     let files = uv_extract::unzip(archive, extracted_wheel).expect("Failed to extract wheel");
-    uv_install_wheel::validate_and_heal_record(extracted_wheel, files.iter(), filename)
-        .expect("Failed to validate wheel");
+    uv_install_wheel::validate_and_heal_record(
+        extracted_wheel,
+        files.iter().map(|file| (file.path(), file.size())),
+        filename,
+    )
+    .expect("Failed to validate wheel");
     files
 }
 

crates/uv-cache/src/archive.rs

@@ -1,5 +1,5 @@
 use std::convert::Infallible;
-use std::path::Path;
+use std::path::{Path, PathBuf};
 use std::str::FromStr;
 
 /// A unique identifier for an archive (unzipped wheel) in the cache.
@@ -10,6 +10,10 @@ use std::str::FromStr;
 #[derive(Debug, Clone, Eq, PartialEq, Hash, serde::Serialize, serde::Deserialize)]
 pub struct ArchiveId(String);
 
+/// A unique identifier for a file stored in the archive file bucket.
+#[derive(Debug, Clone, Eq, PartialEq, Hash)]
+pub struct ArchiveFileId(PathBuf);
+
 impl Default for ArchiveId {
     fn default() -> Self {
         Self::new()
@@ -31,6 +35,20 @@ impl ArchiveId {
     }
 }
 
+impl ArchiveFileId {
+    /// Identify a file object by a digest that includes its contents and executable status.
+    pub fn from_digest(digest: &str) -> Self {
+        let shard = digest.get(..2).unwrap_or(digest);
+        Self(PathBuf::from(shard).join(digest))
+    }
+}
+
+impl AsRef<Path> for ArchiveFileId {
+    fn as_ref(&self) -> &Path {
+        &self.0
+    }
+}
+
 impl AsRef<Path> for ArchiveId {
     fn as_ref(&self) -> &Path {
         self.0.as_ref()

crates/uv-cache/src/lib.rs

@@ -21,7 +21,7 @@ use crate::removal::Remover;
 pub use crate::removal::{Removal, RemovalAccounting};
 pub use crate::wheel::WheelCache;
 use crate::wheel::WheelCacheKind;
-pub use archive::ArchiveId;
+pub use archive::{ArchiveFileId, ArchiveId};
 
 mod archive;
 mod by_timestamp;
@@ -328,6 +328,11 @@ impl Cache {
         self.bucket(CacheBucket::Archive).join(id)
     }
 
+    /// Return the path to an archive file in the cache.
+    pub fn archive_file(&self, id: &ArchiveFileId) -> PathBuf {
+        self.bucket(CacheBucket::Files).join(id)
+    }
+
     /// Create a temporary directory to be used as a Python virtual environment.
     pub fn venv_dir(&self) -> io::Result<tempfile::TempDir> {
         fs_err::create_dir_all(self.bucket(CacheBucket::Builds))?;
@@ -609,6 +614,8 @@ impl Cache {
 
     /// Remove a package from the cache.
     ///
+    /// Unreferenced file objects are removed separately by [`Cache::prune_archive_files`].
+    ///
     /// Returns the number of entries removed from the cache.
     pub fn remove(&self, name: &PackageName) -> io::Result<Removal> {
         // Collect the set of referenced archives.
@@ -639,6 +646,48 @@ impl Cache {
         Ok(summary)
     }
 
+    /// Remove file objects with no hardlinks outside the files bucket.
+    ///
+    /// Archives refer to these objects via hardlinks, independently of the installation link mode.
+    /// Installed copies and reflinks remain valid when the cached file is removed, so they do not
+    /// need to keep the file object alive.
+    pub fn prune_archive_files(&self) -> Result<Removal, io::Error> {
+        let root = self.bucket(CacheBucket::Files);
+        if !root.exists() {
+            return Ok(self.removal());
+        }
+
+        let mut summary = self.removal();
+        for entry in walkdir::WalkDir::new(&root)
+            .min_depth(1)
+            .contents_first(true)
+        {
+            let entry = entry?;
+            if entry.file_type().is_file() {
+                match uv_fs::hardlink_count(entry.path()) {
+                    Ok(1) => summary += self.remove_path(entry.path())?,
+                    Ok(_) => {}
+                    Err(err) if err.kind() == io::ErrorKind::NotFound => {}
+                    Err(err) => return Err(err),
+                }
+            } else if entry.file_type().is_dir() {
+                match fs_err::remove_dir(entry.path()) {
+                    Ok(()) => {
+                        summary.num_dirs += 1;
+                    }
+                    Err(err)
+                        if matches!(
+                            err.kind(),
+                            io::ErrorKind::DirectoryNotEmpty | io::ErrorKind::NotFound
+                        ) => {}
+                    Err(err) => return Err(err),
+                }
+            }
+        }
+
+        Ok(summary)
+    }
+
     /// Prune dangling cache entries and cached environments.
     pub fn prune(&self, ci: bool) -> Result<Removal, io::Error> {
         let mut summary = self.removal();
@@ -766,6 +815,8 @@ impl Cache {
             Err(err) => return Err(err),
         }
 
+        summary += self.prune_archive_files()?;
+
         Ok(summary)
     }
 
@@ -1240,6 +1291,8 @@ pub enum CacheBucket {
     /// that cache entries can be atomically replaced and removed, as storing directories in the
     /// other buckets directly would make atomic operations impossible.
     Archive,
+    /// Content-addressed files that are hardlinked into cached archives.
+    Files,
     /// Ephemeral virtual environments used to execute PEP 517 builds and other operations.
     Builds,
     /// Reusable virtual environments for Python tools and projects.
@@ -1275,6 +1328,7 @@ impl CacheBucket {
             // Note that when bumping this, you'll also need to bump
             // `ARCHIVE_VERSION` in `crates/uv-cache/src/lib.rs`.
             Self::Archive => "archive-v0",
+            Self::Files => "files-v0",
             Self::Builds => "builds-v0",
             Self::Environments => "environments-v2",
             Self::Python => "python-v0",
@@ -1384,6 +1438,7 @@ impl CacheBucket {
             Self::Git
             | Self::Interpreter
             | Self::Archive
+            | Self::Files
             | Self::Builds
             | Self::Environments
             | Self::Python
@@ -1405,6 +1460,7 @@ impl CacheBucket {
             Self::Interpreter,
             Self::Simple,
             Self::Archive,
+            Self::Files,
             Self::Builds,
             Self::Environments,
             Self::Python,

crates/uv-distribution/Cargo.toml

@@ -48,6 +48,7 @@ either = { workspace = true }
 fs-err = { workspace = true }
 futures = { workspace = true }
 owo-colors = { workspace = true }
+rayon = { workspace = true }
 reqwest = { workspace = true }
 reqwest-middleware = { workspace = true }
 rmp-serde = { workspace = true }