diff --git a/crates/uv-types/Cargo.toml b/crates/uv-types/Cargo.toml index c7be8179c8cf5..e519e4235af80 100644 --- a/crates/uv-types/Cargo.toml +++ b/crates/uv-types/Cargo.toml @@ -11,7 +11,6 @@ license = { workspace = true } [lib] doctest = false -test = false [lints] workspace = true diff --git a/crates/uv-types/src/hash.rs b/crates/uv-types/src/hash.rs index 9d48c8221bc0a..8bb01b067ca9e 100644 --- a/crates/uv-types/src/hash.rs +++ b/crates/uv-types/src/hash.rs @@ -1,3 +1,4 @@ +use std::fmt::Display; use std::str::FromStr; use std::sync::Arc; @@ -151,31 +152,24 @@ impl HashStrategy { continue; }; - let digests = if digests.is_empty() { - // If there are no hashes, and the distribution is URL-based, attempt to extract - // it from the fragment. - requirement - .hashes() - .map(HashDigests::from) - .map(|hashes| hashes.to_vec()) - .unwrap_or_default() - } else { - // Parse the hashes. - digests - .iter() - .map(|digest| HashDigest::from_str(digest)) - .collect::, _>>()? - }; + // Parse the hashes provided directly on the requirement, then merge in any hashes from + // the URL fragment. + let mut digests = digests + .iter() + .map(|digest| HashDigest::from_str(digest)) + .collect::, _>>()?; + if let Some(fragment_hashes) = requirement.hashes().map(HashDigests::from) { + merge_digests(&mut digests, fragment_hashes.iter(), requirement)?; + } if digests.is_empty() { continue; } - constraint_hashes.insert(id, digests); + merge_hashes(&mut constraint_hashes, id, digests, requirement)?; } - // For each requirement, map from name to allowed hashes. We use the last entry for each - // package. + // For each requirement, map from hash identity to allowed hashes. let mut requirement_hashes = FxHashMap::>::default(); for (requirement, digests) in requirements { if !requirement @@ -201,30 +195,28 @@ impl HashStrategy { } UnresolvedRequirement::Unnamed(requirement) => { // Direct URLs are always allowed. - VersionId::from_url(&requirement.url.verbatim) + VersionId::from_parsed_url(&requirement.url.parsed_url) } }; - let digests = if digests.is_empty() { - // If there are no hashes, and the distribution is URL-based, attempt to extract - // it from the fragment. - requirement - .hashes() - .map(HashDigests::from) - .map(|hashes| hashes.to_vec()) - .unwrap_or_default() - } else { - // Parse the hashes. - digests - .iter() - .map(|digest| HashDigest::from_str(digest)) - .collect::, _>>()? - }; + // Parse the hashes provided directly on the requirement, then merge in any hashes from + // the URL fragment. + let mut digests = digests + .iter() + .map(|digest| HashDigest::from_str(digest)) + .collect::, _>>()?; + if let Some(fragment_hashes) = requirement.hashes().map(HashDigests::from) { + merge_digests(&mut digests, fragment_hashes.iter(), requirement)?; + } let digests = if let Some(constraint) = constraint_hashes.remove(&id) { if digests.is_empty() { // If there are _only_ hashes on the constraints, use them. constraint + } else if matches!(id, VersionId::ArchiveUrl { .. }) { + let mut merged = digests; + merge_digests(&mut merged, &constraint, requirement)?; + merged } else { // If there are constraint and requirement hashes, take the intersection. let intersection: Vec<_> = digests @@ -254,7 +246,7 @@ impl HashStrategy { continue; } - requirement_hashes.insert(id, digests); + merge_hashes(&mut requirement_hashes, id, digests, requirement)?; } // Merge the hashes, preferring requirements over constraints, since overlapping @@ -296,7 +288,7 @@ impl HashStrategy { } } - /// Pin a [`Requirement`] to a [`PackageId`], if possible. + /// Pin a [`Requirement`] to a [`VersionId`], if possible. fn pin(requirement: &Requirement) -> Option { match &requirement.source { RequirementSource::Registry { specifier, .. } => { @@ -315,18 +307,87 @@ impl HashStrategy { specifier.version().clone(), )) } - RequirementSource::Url { url, .. } - | RequirementSource::Git { url, .. } - | RequirementSource::Path { url, .. } - | RequirementSource::Directory { url, .. } => Some(VersionId::from_url(url)), + RequirementSource::Url { + location, + subdirectory, + .. + } => Some(VersionId::from_archive(location, subdirectory.as_deref())), + RequirementSource::Git { + git, subdirectory, .. + } => Some(VersionId::from_git(git, subdirectory.as_deref())), + RequirementSource::Path { install_path, .. } => { + Some(VersionId::from_path(install_path)) + } + RequirementSource::Directory { install_path, .. } => { + Some(VersionId::from_directory(install_path)) + } } } } +/// Merge repeated hashes for a requirement or constraint into the hash map. +fn merge_hashes( + hashes: &mut FxHashMap>, + id: VersionId, + incoming: Vec, + requirement: impl Display, +) -> Result<(), HashStrategyError> { + if incoming.is_empty() { + return Ok(()); + } + + if !matches!(&id, VersionId::ArchiveUrl { .. }) { + hashes.insert(id, incoming); + return Ok(()); + } + + if let Some(existing) = hashes.get_mut(&id) { + return merge_digests(existing, &incoming, requirement); + } + + let mut merged = Vec::new(); + merge_digests(&mut merged, &incoming, requirement)?; + hashes.insert(id, merged); + Ok(()) +} + +/// Merge `incoming` digests into `existing`. +/// +/// Exact duplicates are ignored. Digests for different algorithms are accumulated. If the +/// same algorithm appears with two different values, returns +/// [`HashStrategyError::ConflictingArchiveUrlHashes`]. +fn merge_digests<'a>( + existing: &mut Vec, + incoming: impl IntoIterator, + requirement: impl Display, +) -> Result<(), HashStrategyError> { + for digest in incoming { + match existing + .iter() + .find(|candidate| candidate.algorithm == digest.algorithm) + { + Some(candidate) if candidate == digest => {} + Some(conflict) => { + return Err(HashStrategyError::ConflictingArchiveUrlHashes( + requirement.to_string(), + conflict.clone(), + digest.clone(), + )); + } + None => existing.push(digest.clone()), + } + } + existing.sort_unstable(); + + Ok(()) +} + #[derive(thiserror::Error, Debug)] pub enum HashStrategyError { #[error(transparent)] Hash(#[from] HashError), + #[error("Conflicting archive URL hashes for `{0}`: `{1}` conflicts with `{2}`")] + ConflictingArchiveUrlHashes(String, HashDigest, HashDigest), #[error( "In `{1}` mode, all requirements must have their versions pinned with `==`, but found: {0}" )] @@ -338,3 +399,77 @@ pub enum HashStrategyError { )] NoIntersection(String, HashCheckingMode), } + +#[cfg(test)] +mod tests { + use std::str::FromStr; + use uv_configuration::HashCheckingMode; + use uv_distribution_filename::DistExtension; + use uv_distribution_types::{ + HashPolicy, Requirement, RequirementSource, UnresolvedRequirement, + }; + use uv_pypi_types::HashDigest; + + use super::HashStrategy; + + fn requirement(url: &str) -> Requirement { + Requirement { + name: "anyio".parse().unwrap(), + extras: Box::default(), + groups: Box::default(), + marker: "python_version >= '3.8'".parse().unwrap(), + source: RequirementSource::Url { + location: "https://files.pythonhosted.org/packages/36/55/ad4de788d84a630656ece71059665e01ca793c04294c463fd84132f40fe6/anyio-4.0.0-py3-none-any.whl" + .parse() + .unwrap(), + subdirectory: None, + ext: DistExtension::Wheel, + url: url.parse().unwrap(), + }, + origin: None, + } + } + + #[test] + fn from_requirements_merges_direct_url_hashes_across_fragments() { + let first = UnresolvedRequirement::Named(requirement( + "https://files.pythonhosted.org/packages/36/55/ad4de788d84a630656ece71059665e01ca793c04294c463fd84132f40fe6/anyio-4.0.0-py3-none-any.whl#sha256=cfdb2b588b9fc25ede96d8db56ed50848b0b649dca3dd1df0b11f683bb9e0b5f", + )); + let second = UnresolvedRequirement::Named(requirement( + "https://files.pythonhosted.org/packages/36/55/ad4de788d84a630656ece71059665e01ca793c04294c463fd84132f40fe6/anyio-4.0.0-py3-none-any.whl#sha512=f30761c1e8725b49c498273b90dba4b05c0fd157811994c806183062cb6647e773364ce45f0e1ff0b10e32fe6d0232ea5ad39476ccf37109d6b49603a09c11c2", + )); + + let hasher = HashStrategy::from_requirements( + [(&first, &[][..]), (&second, &[][..])].into_iter(), + std::iter::empty(), + None, + HashCheckingMode::Require, + ) + .unwrap(); + + let mut expected = vec![ + HashDigest::from_str( + "sha256:cfdb2b588b9fc25ede96d8db56ed50848b0b649dca3dd1df0b11f683bb9e0b5f", + ) + .unwrap(), + HashDigest::from_str( + "sha512:f30761c1e8725b49c498273b90dba4b05c0fd157811994c806183062cb6647e773364ce45f0e1ff0b10e32fe6d0232ea5ad39476ccf37109d6b49603a09c11c2", + ) + .unwrap(), + ]; + expected.sort_unstable(); + + for requirement in [&first, &second] { + let UnresolvedRequirement::Named(requirement) = requirement else { + panic!("expected named requirement"); + }; + let RequirementSource::Url { url, .. } = &requirement.source else { + panic!("expected direct URL requirement"); + }; + let HashPolicy::Validate(digests) = hasher.get_url(url) else { + panic!("expected hash validation policy"); + }; + assert_eq!(digests, expected.as_slice()); + } + } +} diff --git a/crates/uv/tests/it/pip_sync.rs b/crates/uv/tests/it/pip_sync.rs index 16f02f5e8b7bf..bdda02b98a6bc 100644 --- a/crates/uv/tests/it/pip_sync.rs +++ b/crates/uv/tests/it/pip_sync.rs @@ -4461,7 +4461,7 @@ fn require_hashes_repeated_dependency() -> Result<()> { Ok(()) } -/// If a dependency is repeated, use the last hash provided. pip seems to use the _first_ hash. +/// Repeated direct URL requirements merge compatible hashes instead of overwriting them. #[test] fn require_hashes_repeated_hash() -> Result<()> { let context = uv_test::test_context!("3.12"); @@ -4514,12 +4514,12 @@ fn require_hashes_repeated_hash() -> Result<()> { " ); - // Use a different hash. The first hash is wrong, but that's fine, since we use the last hash. + // Use a different hash. The `sha512` is wrong, but the correct `sha256` still allows install. let requirements_txt = context.temp_dir.child("requirements.txt"); requirements_txt .write_str(indoc::indoc! { r" - anyio @ https://files.pythonhosted.org/packages/36/55/ad4de788d84a630656ece71059665e01ca793c04294c463fd84132f40fe6/anyio-4.0.0-py3-none-any.whl --hash=sha256:a7ed51751b2c2add651e5747c891b47e26d2a21be5d32d9311dfe9692f3e5d7a - anyio @ https://files.pythonhosted.org/packages/36/55/ad4de788d84a630656ece71059665e01ca793c04294c463fd84132f40fe6/anyio-4.0.0-py3-none-any.whl --hash=md5:420d85e19168705cdf0223621b18831a + anyio @ https://files.pythonhosted.org/packages/36/55/ad4de788d84a630656ece71059665e01ca793c04294c463fd84132f40fe6/anyio-4.0.0-py3-none-any.whl --hash=sha256:cfdb2b588b9fc25ede96d8db56ed50848b0b649dca3dd1df0b11f683bb9e0b5f + anyio @ https://files.pythonhosted.org/packages/36/55/ad4de788d84a630656ece71059665e01ca793c04294c463fd84132f40fe6/anyio-4.0.0-py3-none-any.whl --hash=sha512:e30761c1e8725b49c498273b90dba4b05c0fd157811994c806183062cb6647e773364ce45f0e1ff0b10e32fe6d0232ea5ad39476ccf37109d6b49603a09c11c2 " })?; uv_snapshot!(context.pip_sync() @@ -4539,12 +4539,13 @@ fn require_hashes_repeated_hash() -> Result<()> { " ); - // Use a different hash. The second hash is wrong. This should fail, since we use the last hash. + // Use different hashes, but both are wrong. This should fail because none of the merged + // hashes match. let requirements_txt = context.temp_dir.child("requirements.txt"); requirements_txt .write_str(indoc::indoc! { r" anyio @ https://files.pythonhosted.org/packages/36/55/ad4de788d84a630656ece71059665e01ca793c04294c463fd84132f40fe6/anyio-4.0.0-py3-none-any.whl --hash=sha256:f7ed51751b2c2add651e5747c891b47e26d2a21be5d32d9311dfe9692f3e5d7a - anyio @ https://files.pythonhosted.org/packages/36/55/ad4de788d84a630656ece71059665e01ca793c04294c463fd84132f40fe6/anyio-4.0.0-py3-none-any.whl --hash=md5:520d85e19168705cdf0223621b18831a + anyio @ https://files.pythonhosted.org/packages/36/55/ad4de788d84a630656ece71059665e01ca793c04294c463fd84132f40fe6/anyio-4.0.0-py3-none-any.whl --hash=sha512:e30761c1e8725b49c498273b90dba4b05c0fd157811994c806183062cb6647e773364ce45f0e1ff0b10e32fe6d0232ea5ad39476ccf37109d6b49603a09c11c2 " })?; uv_snapshot!(context.pip_sync() @@ -4561,10 +4562,50 @@ fn require_hashes_repeated_hash() -> Result<()> { ╰─▶ Hash mismatch for `anyio @ https://files.pythonhosted.org/packages/36/55/ad4de788d84a630656ece71059665e01ca793c04294c463fd84132f40fe6/anyio-4.0.0-py3-none-any.whl` Expected: - md5:520d85e19168705cdf0223621b18831a + sha256:f7ed51751b2c2add651e5747c891b47e26d2a21be5d32d9311dfe9692f3e5d7a + sha512:e30761c1e8725b49c498273b90dba4b05c0fd157811994c806183062cb6647e773364ce45f0e1ff0b10e32fe6d0232ea5ad39476ccf37109d6b49603a09c11c2 Computed: - md5:420d85e19168705cdf0223621b18831a + sha256:cfdb2b588b9fc25ede96d8db56ed50848b0b649dca3dd1df0b11f683bb9e0b5f + " + ); + + Ok(()) +} + +/// Repeated direct URL requirements merge hashes across nested requirements files. +#[test] +fn require_hashes_repeated_hash_multiple_files() -> Result<()> { + let context = uv_test::test_context!("3.12"); + + let requirements_a = context.temp_dir.child("requirements-a.txt"); + requirements_a.write_str(indoc::indoc! { r" + anyio @ https://files.pythonhosted.org/packages/36/55/ad4de788d84a630656ece71059665e01ca793c04294c463fd84132f40fe6/anyio-4.0.0-py3-none-any.whl --hash=sha256:cfdb2b588b9fc25ede96d8db56ed50848b0b649dca3dd1df0b11f683bb9e0b5f + " })?; + + let requirements_b = context.temp_dir.child("requirements-b.txt"); + requirements_b.write_str(indoc::indoc! { r" + anyio @ https://files.pythonhosted.org/packages/36/55/ad4de788d84a630656ece71059665e01ca793c04294c463fd84132f40fe6/anyio-4.0.0-py3-none-any.whl --hash=sha512:f30761c1e8725b49c498273b90dba4b05c0fd157811994c806183062cb6647e773364ce45f0e1ff0b10e32fe6d0232ea5ad39476ccf37109d6b49603a09c11c2 + " })?; + + let requirements_txt = context.temp_dir.child("requirements.txt"); + requirements_txt.write_str(indoc::indoc! { r" + -r requirements-a.txt + -r requirements-b.txt + " })?; + + uv_snapshot!(context.pip_sync() + .arg("requirements.txt") + .arg("--require-hashes"), @" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 1 package in [TIME] + Prepared 1 package in [TIME] + Installed 1 package in [TIME] + + anyio==4.0.0 (from https://files.pythonhosted.org/packages/36/55/ad4de788d84a630656ece71059665e01ca793c04294c463fd84132f40fe6/anyio-4.0.0-py3-none-any.whl) " ); @@ -5220,32 +5261,27 @@ fn require_hashes_url_invalid() -> Result<()> { Ok(()) } -/// Ignore the (valid) hash on the fragment if (invalid) hashes are provided directly. +/// Merge the hash on the fragment with hashes provided directly. #[test] -fn require_hashes_url_ignore() -> Result<()> { +fn require_hashes_url_merge() -> Result<()> { let context = uv_test::test_context!("3.12").with_exclude_newer("2025-01-29T00:00:00Z"); let requirements_txt = context.temp_dir.child("requirements.txt"); requirements_txt - .write_str("iniconfig @ https://files.pythonhosted.org/packages/ef/a6/62565a6e1cf69e10f5727360368e451d4b7f58beeac6173dc9db836a5b46/iniconfig-2.0.0-py3-none-any.whl#sha256=b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374 --hash sha256:c6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374")?; + .write_str("anyio @ https://files.pythonhosted.org/packages/36/55/ad4de788d84a630656ece71059665e01ca793c04294c463fd84132f40fe6/anyio-4.0.0-py3-none-any.whl#sha256=cfdb2b588b9fc25ede96d8db56ed50848b0b649dca3dd1df0b11f683bb9e0b5f --hash sha512:f30761c1e8725b49c498273b90dba4b05c0fd157811994c806183062cb6647e773364ce45f0e1ff0b10e32fe6d0232ea5ad39476ccf37109d6b49603a09c11c2")?; uv_snapshot!(context.pip_sync() .arg("requirements.txt") .arg("--require-hashes"), @" - success: false - exit_code: 1 + success: true + exit_code: 0 ----- stdout ----- ----- stderr ----- Resolved 1 package in [TIME] - × Failed to download `iniconfig @ https://files.pythonhosted.org/packages/ef/a6/62565a6e1cf69e10f5727360368e451d4b7f58beeac6173dc9db836a5b46/iniconfig-2.0.0-py3-none-any.whl#sha256=b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374` - ╰─▶ Hash mismatch for `iniconfig @ https://files.pythonhosted.org/packages/ef/a6/62565a6e1cf69e10f5727360368e451d4b7f58beeac6173dc9db836a5b46/iniconfig-2.0.0-py3-none-any.whl#sha256=b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374` - - Expected: - sha256:c6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374 - - Computed: - sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374 + Prepared 1 package in [TIME] + Installed 1 package in [TIME] + + anyio==4.0.0 (from https://files.pythonhosted.org/packages/36/55/ad4de788d84a630656ece71059665e01ca793c04294c463fd84132f40fe6/anyio-4.0.0-py3-none-any.whl#sha256=cfdb2b588b9fc25ede96d8db56ed50848b0b649dca3dd1df0b11f683bb9e0b5f) " );