Share downloaded URL wheels across hash fragments

crates/uv-cache-key/src/canonical_url.rs

@@ -8,14 +8,14 @@ use uv_redacted::{DisplaySafeUrl, DisplaySafeUrlError};
 
 use crate::cache_key::{CacheKey, CacheKeyHasher};
 
-/// A wrapper around `Url` which represents a "canonical" version of an original URL.
+/// A normalized URL for comparing package sources.
 ///
 /// A "canonical" url is only intended for internal comparison purposes. It's to help paper over
 /// mistakes such as depending on `github.com/foo/bar` vs. `github.com/foo/bar.git`.
 ///
-/// This is **only** for internal purposes and provides no means to actually read the underlying
-/// string value of the `Url` it contains. This is intentional, because all fetching should still
-/// happen within the context of the original URL.
+/// Known hash fragments are omitted from source identity. All other fragments are preserved,
+/// including unknown fields that may acquire meaning in the future. Fetching and hash validation
+/// must use the original URL; equal canonical URLs do not imply equal archive contents.
 #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)]
 pub struct CanonicalUrl(DisplaySafeUrl);
 
@@ -26,6 +26,22 @@ impl CanonicalUrl {
             return Self(url);
         }
 
+        // Match the hash fields recognized by `uv_pypi_types::Hashes::parse_fragment`, without
+        // interpreting other fragments. In particular, malformed or unknown fields are retained.
+        if let Some(fragment) = url.fragment() {
+            let fragments = fragment
+                .split('&')
+                .filter(|fragment| {
+                    !fragment.split_once('=').is_some_and(|(name, value)| {
+                        matches!(name, "md5" | "sha256" | "sha384" | "sha512" | "blake2b")
+                            && !value.contains('=')
+                    })
+                })
+                .collect::<Vec<_>>();
+            let fragment = (!fragments.is_empty()).then(|| fragments.join("&"));
+            url.set_fragment(fragment.as_deref());
+        }
+
         // Strip credentials.
         let _ = url.set_password(None);
         let _ = url.set_username("");
@@ -226,6 +242,56 @@ impl std::fmt::Display for RepositoryUrl {
 mod tests {
     use super::*;
 
+    #[test]
+    fn canonical_url_ignores_known_hash_fragments() -> Result<(), DisplaySafeUrlError> {
+        for algorithm in ["md5", "sha256", "sha384", "sha512", "blake2b"] {
+            assert_eq!(
+                CanonicalUrl::parse(&format!("https://example.com/pkg.whl#{algorithm}=abc"))?,
+                CanonicalUrl::parse("https://example.com/pkg.whl")?,
+            );
+            assert_eq!(
+                CanonicalUrl::parse(&format!(
+                    "https://example.com/pkg.tar.gz#subdirectory=src&{algorithm}=abc&unknown=value"
+                ))?,
+                CanonicalUrl::parse(
+                    "https://example.com/pkg.tar.gz#subdirectory=src&unknown=value"
+                )?,
+            );
+        }
+        assert_eq!(
+            CanonicalUrl::parse("https://example.com/pkg.whl#sha256=abc&sha512=def")?,
+            CanonicalUrl::parse("https://example.com/pkg.whl")?,
+        );
+        Ok(())
+    }
+
+    #[test]
+    fn canonical_url_preserves_other_fragments() -> Result<(), DisplaySafeUrlError> {
+        for fragment in [
+            "subdirectory=src",
+            "path=dist/pkg.whl",
+            "lfs=true",
+            "egg=pkg",
+            "unknown=value",
+            "sha3_256=abc",
+            "SHA256=abc",
+            "sha256",
+            "sha256=abc=def",
+            "0123456789012345678901234567890123456789",
+        ] {
+            let url = DisplaySafeUrl::parse(&format!("https://example.com/pkg#{fragment}"))?;
+            assert_eq!(DisplaySafeUrl::from(CanonicalUrl::new(url.clone())), url);
+        }
+        // Preserve duplicate fields and their order, matching the source parsers.
+        assert_eq!(
+            CanonicalUrl::parse(
+                "https://example.com/pkg#subdirectory=first&sha256=abc&subdirectory=second"
+            )?,
+            CanonicalUrl::parse("https://example.com/pkg#subdirectory=first&subdirectory=second")?,
+        );
+        Ok(())
+    }
+
     #[test]
     fn user_credential_does_not_affect_cache_key() -> Result<(), DisplaySafeUrlError> {
         let mut hasher = CacheKeyHasher::new();

crates/uv-cache/src/lib.rs

@@ -421,9 +421,7 @@ impl Cache {
         fs_err::create_dir_all(archive_entry.dir())?;
         uv_fs::rename_with_retry(temp_dir.as_ref(), archive_entry.path()).await?;
 
-        // Create a symlink to the directory store.
-        fs_err::create_dir_all(path.as_ref().parent().expect("Cache entry to have parent"))?;
-        self.create_link(&id, path.as_ref())?;
+        self.link(&id, path)?;
 
         Ok(id)
     }
@@ -454,6 +452,12 @@ impl Cache {
         Ok(id)
     }
 
+    /// Link an existing archive into a cache entry, retaining it when the cache is pruned.
+    pub fn link(&self, id: &ArchiveId, path: impl AsRef<Path>) -> io::Result<()> {
+        fs_err::create_dir_all(path.as_ref().parent().expect("Cache entry to have parent"))?;
+        self.create_link(id, path)
+    }
+
     /// Returns `true` if the [`Cache`] is temporary.
     pub fn is_temporary(&self) -> bool {
         self.temp_dir.is_some()

crates/uv-cache/src/wheel.rs

@@ -2,6 +2,7 @@ use std::path::{Path, PathBuf};
 
 use uv_cache_key::{CanonicalUrl, cache_digest};
 use uv_distribution_types::IndexUrl;
+use uv_pypi_types::HashDigest;
 use uv_redacted::DisplaySafeUrl;
 
 /// Cache wheels and their metadata, both from remote wheels and built from source distributions.
@@ -29,27 +30,47 @@ impl WheelCache<'_> {
             Self::Index(IndexUrl::Pypi(_)) => WheelCacheKind::Pypi.root(),
             Self::Index(url) => WheelCacheKind::Index
                 .root()
-                .join(cache_digest(&CanonicalUrl::new(url.url().clone()))),
-            Self::Url(url) => WheelCacheKind::Url
-                .root()
-                .join(cache_digest(&CanonicalUrl::new((*url).clone()))),
-            Self::Path(url) => WheelCacheKind::Path
-                .root()
-                .join(cache_digest(&CanonicalUrl::new((*url).clone()))),
-            Self::Editable(url) => WheelCacheKind::Editable
-                .root()
-                .join(cache_digest(&CanonicalUrl::new((*url).clone()))),
+                .join(revision_digest(url.url())),
+            Self::Url(url) => WheelCacheKind::Url.root().join(revision_digest(url)),
+            Self::Path(url) => WheelCacheKind::Path.root().join(revision_digest(url)),
+            Self::Editable(url) => WheelCacheKind::Editable.root().join(revision_digest(url)),
             Self::Git(url, sha) => WheelCacheKind::Git
                 .root()
-                .join(cache_digest(&CanonicalUrl::new((*url).clone())))
+                .join(revision_digest(url))
                 .join(sha),
         }
     }
 
-    /// A subdirectory in a bucket for wheels for a specific package.
+    /// A subdirectory for wheels and metadata belonging to a specific package and URL revision.
     pub fn wheel_dir(&self, package_name: impl AsRef<Path>) -> PathBuf {
         self.root().join(package_name)
     }
+
+    /// A shared cache location for a URL wheel with a computed hash.
+    ///
+    /// Unlike HTTP responses, downloaded archives can be shared across hash declarations by
+    /// checking their computed digests. Keep unknown and source-selecting fragments in the key.
+    pub fn url_hash_dir(
+        url: &DisplaySafeUrl,
+        package_name: impl AsRef<Path>,
+        hash: &HashDigest,
+    ) -> PathBuf {
+        WheelCacheKind::Url
+            .root()
+            .join(cache_digest(&CanonicalUrl::new(url.clone())))
+            .join(package_name)
+            .join("hashes")
+            .join(cache_digest(&hash.to_string()))
+    }
+}
+
+/// Identify a URL revision, retaining hash declarations that cannot be checked against metadata.
+fn revision_digest(url: &DisplaySafeUrl) -> String {
+    // Metadata and source builds are not interchangeable across expected hashes. Keep a separate
+    // revision for each declaration, preserving the existing on-disk keys.
+    let mut revision = DisplaySafeUrl::from(CanonicalUrl::new(url.clone()));
+    revision.set_fragment(url.fragment());
+    cache_digest(&revision.as_str())
 }
 
 #[derive(Debug, Clone, Copy)]
@@ -90,3 +111,39 @@ impl AsRef<Path> for WheelCacheKind {
         self.to_str().as_ref()
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use uv_cache_key::cache_digest;
+    use uv_pypi_types::{HashAlgorithm, HashDigest};
+    use uv_redacted::{DisplaySafeUrl, DisplaySafeUrlError};
+
+    use super::{WheelCache, revision_digest};
+
+    #[test]
+    fn archive_and_metadata_url_identity() -> Result<(), DisplaySafeUrlError> {
+        let plain = DisplaySafeUrl::parse("https://example.org/pkg.whl")?;
+        let hashed = DisplaySafeUrl::parse("https://example.org/pkg.whl#sha256=abc")?;
+        assert_ne!(
+            WheelCache::Url(&plain).wheel_dir("pkg"),
+            WheelCache::Url(&hashed).wheel_dir("pkg"),
+        );
+        assert_eq!(revision_digest(&hashed), cache_digest(&hashed.as_str()));
+        let hash = HashDigest {
+            algorithm: HashAlgorithm::Sha256,
+            digest: "abc".into(),
+        };
+        assert_eq!(
+            WheelCache::url_hash_dir(&plain, "pkg", &hash),
+            WheelCache::url_hash_dir(&hashed, "pkg", &hash),
+        );
+
+        let first = DisplaySafeUrl::parse("https://example.org/pkg.tar.gz#subdirectory=first")?;
+        let second = DisplaySafeUrl::parse("https://example.org/pkg.tar.gz#subdirectory=second")?;
+        assert_ne!(
+            WheelCache::Url(&first).root(),
+            WheelCache::Url(&second).root()
+        );
+        Ok(())
+    }
+}

crates/uv-distribution-types/src/id.rs

@@ -204,7 +204,9 @@ impl Display for VersionId {
 /// the URL would also be sufficient).
 #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
 pub enum DistributionId {
-    Url(CanonicalUrl),
+    /// The source location and original fragment, including its expected hashes. In-flight
+    /// downloads must not conflate different content expectations at the same canonical URL.
+    Url(CanonicalUrl, Option<String>),
     PathBuf(PathBuf),
     Digest(HashDigest),
     AbsoluteUrl(String),
@@ -250,6 +252,7 @@ mod tests {
     use fs_err as fs;
 
     use super::VersionId;
+    use crate::Identifier;
     use uv_redacted::DisplaySafeUrl;
 
     #[test]
@@ -264,6 +267,7 @@ mod tests {
         .unwrap();
 
         assert_eq!(VersionId::from_url(&first), VersionId::from_url(&second));
+        assert_ne!(first.distribution_id(), second.distribution_id());
     }
 
     #[test]

crates/uv-distribution-types/src/lib.rs

@@ -49,6 +49,7 @@ use std::str::FromStr;
 use memchr::memchr3;
 use url::Url;
 
+use uv_cache_key::{CanonicalUrl, RepositoryUrl};
 use uv_distribution_filename::{
     DistExtension, SourceDistExtension, SourceDistFilename, WheelFilename,
 };
@@ -1457,11 +1458,14 @@ impl RemoteSource for Dist {
 
 impl Identifier for DisplaySafeUrl {
     fn distribution_id(&self) -> DistributionId {
-        DistributionId::Url(uv_cache_key::CanonicalUrl::new(self.clone()))
+        DistributionId::Url(
+            CanonicalUrl::new(self.clone()),
+            self.fragment().map(str::to_owned),
+        )
     }
 
     fn resource_id(&self) -> ResourceId {
-        ResourceId::Url(uv_cache_key::RepositoryUrl::new(self.clone()))
+        ResourceId::Url(RepositoryUrl::new(self.clone()))
     }
 }
 

crates/uv-distribution/src/archive.rs

@@ -1,7 +1,7 @@
 use serde::ser::SerializeMap;
 use uv_cache::{ARCHIVE_VERSION, ArchiveId, Cache};
 use uv_distribution_filename::WheelFilename;
-use uv_distribution_types::Hashed;
+use uv_distribution_types::{ArchiveHashPolicy, Hashed};
 use uv_pypi_types::{HashDigest, HashDigests};
 
 /// An archive (unzipped wheel) that exists in the local cache.
@@ -58,6 +58,20 @@ impl Archive {
     pub(crate) fn exists(&self, cache: &Cache) -> bool {
         self.version == ARCHIVE_VERSION && cache.archive(&self.id).exists()
     }
+
+    /// Return whether these cached contents satisfy the requested wheel's integrity constraints.
+    pub(crate) fn is_usable(
+        &self,
+        cache: &Cache,
+        filename: &WheelFilename,
+        hashes: ArchiveHashPolicy<'_>,
+        size: Option<u64>,
+    ) -> bool {
+        self.filename == *filename
+            && self.satisfies(hashes)
+            && self.exists(cache)
+            && size.is_none_or(|size| self.size == Some(size))
+    }
 }
 
 impl Hashed for Archive {