From 6d8ba49a435047e2a08016330b03baa86906a8ab Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Wed, 3 Jun 2026 12:42:44 -0400 Subject: [PATCH 1/2] Always compute SHA256 for remote distributions --- .../src/distribution_database.rs | 77 +++++----------- crates/uv-distribution/src/hash.rs | 11 +++ crates/uv-distribution/src/lib.rs | 1 + crates/uv-distribution/src/source/mod.rs | 5 +- crates/uv/tests/it/pip_sync.rs | 1 + crates/uv/tests/it/sync.rs | 89 +++++++++++++++++++ 6 files changed, 127 insertions(+), 57 deletions(-) create mode 100644 crates/uv-distribution/src/hash.rs diff --git a/crates/uv-distribution/src/distribution_database.rs b/crates/uv-distribution/src/distribution_database.rs index d26fde89a153d..5465605b9368f 100644 --- a/crates/uv-distribution/src/distribution_database.rs +++ b/crates/uv-distribution/src/distribution_database.rs @@ -28,14 +28,14 @@ use uv_git::{GIT_LFS, GitError}; use uv_install_wheel::validate_and_heal_record; use uv_platform_tags::Tags; use uv_pypi_types::{HashDigest, HashDigests, PyProjectToml}; +use uv_python::PythonVariant; use uv_redacted::DisplaySafeUrl; use uv_types::{BuildContext, BuildStack}; use uv_warnings::warn_user_once; use crate::archive::Archive; -use uv_python::PythonVariant; - use crate::error::PythonVersion; +use crate::hash::http_hash_algorithms; use crate::metadata::{ArchiveMetadata, Metadata}; use crate::source::SourceDistributionBuilder; use crate::{Error, LocalWheel, Reporter, RequiresDist}; @@ -731,7 +731,7 @@ impl<'a, Context: BuildContext> DistributionDatabase<'a, Context> { .into_async_read(); // Create a hasher for each hash algorithm. - let algorithms = hashes.algorithms(); + let algorithms = http_hash_algorithms(hashes); let mut hashers = algorithms.into_iter().map(Hasher::from).collect::>(); let mut hasher = uv_extract::hash::HashReader::new(reader.compat(), &mut hashers); @@ -768,10 +768,8 @@ impl<'a, Context: BuildContext> DistributionDatabase<'a, Context> { } }, }; - // If necessary, exhaust the reader to compute the hash. - if !hashes.is_none() { - hasher.finish().await.map_err(Error::HashExhaustion)?; - } + // Exhaust the reader to compute the hashes. + hasher.finish().await.map_err(Error::HashExhaustion)?; // Before we make the wheel accessible by persisting it, ensure that the RECORD is // valid. @@ -891,8 +889,6 @@ impl<'a, Context: BuildContext> DistributionDatabase<'a, Context> { // Create an entry for the HTTP cache. let http_entry = wheel_entry.with_file(format!("{}.http", filename.cache_key())); - let query_url = &url.clone(); - let download = |response: reqwest::Response| { async { let size = size.or_else(|| content_length(&response)); @@ -906,6 +902,9 @@ impl<'a, Context: BuildContext> DistributionDatabase<'a, Context> { .bytes_stream() .map_err(|err| self.handle_response_errors(err)) .into_async_read(); + let algorithms = http_hash_algorithms(hashes); + let mut hashers = algorithms.into_iter().map(Hasher::from).collect::>(); + let mut hasher = uv_extract::hash::HashReader::new(reader.compat(), &mut hashers); // Download the wheel to a temporary file. let temp_file = tempfile::tempfile_in(self.build_context.cache().root()) @@ -917,18 +916,16 @@ impl<'a, Context: BuildContext> DistributionDatabase<'a, Context> { match progress { Some((reporter, progress)) => { - // Wrap the reader in a progress reporter. This will report 100% progress - // after the download is complete, even if we still have to unzip and hash - // part of the file. - let mut reader = - ProgressReader::new(reader.compat(), progress, &**reporter); + // Wrap the reader in a progress reporter. This will report 100% progress once + // the download is complete, before the wheel is unzipped. + let mut reader = ProgressReader::new(&mut hasher, progress, &**reporter); tokio::io::copy(&mut reader, &mut writer) .await .map_err(Error::CacheWrite)?; } None => { - tokio::io::copy(&mut reader.compat(), &mut writer) + tokio::io::copy(&mut hasher, &mut writer) .await .map_err(Error::CacheWrite)?; } @@ -942,47 +939,17 @@ impl<'a, Context: BuildContext> DistributionDatabase<'a, Context> { .await .map_err(Error::CacheWrite)?; - // If no hashes are required, extract the wheel without hashing. - let (files, hashes) = if hashes.is_none() { - let target = temp_dir.path().to_owned(); - let files = match extension { - WheelExtension::Whl => { - let file = file.into_std().await; - tokio::task::spawn_blocking(move || uv_extract::unzip(file, &target)) - .await? - } - WheelExtension::WhlZst => { - uv_extract::stream::untar_zst(file, &target).await - } + let target = temp_dir.path().to_owned(); + let files = match extension { + WheelExtension::Whl => { + let file = file.into_std().await; + tokio::task::spawn_blocking(move || uv_extract::unzip(file, &target)) + .await? } - .map_err(|err| Error::Extract(filename.to_string(), err))?; - - (files, HashDigests::empty()) - } else { - // Create a hasher for each hash algorithm. - let algorithms = hashes.algorithms(); - let mut hashers = algorithms.into_iter().map(Hasher::from).collect::>(); - let mut hasher = uv_extract::hash::HashReader::new(file, &mut hashers); - - let files = match extension { - WheelExtension::Whl => { - uv_extract::stream::unzip(query_url, &mut hasher, temp_dir.path()) - .await - .map_err(|err| Error::Extract(filename.to_string(), err))? - } - WheelExtension::WhlZst => { - uv_extract::stream::untar_zst(&mut hasher, temp_dir.path()) - .await - .map_err(|err| Error::Extract(filename.to_string(), err))? - } - }; - - // If necessary, exhaust the reader to compute the hash. - hasher.finish().await.map_err(Error::HashExhaustion)?; - let hashes = hashers.into_iter().map(HashDigest::from).collect(); - - (files, hashes) - }; + WheelExtension::WhlZst => uv_extract::stream::untar_zst(file, &target).await, + } + .map_err(|err| Error::Extract(filename.to_string(), err))?; + let hashes = hashers.into_iter().map(HashDigest::from).collect(); // Before we make the wheel accessible by persisting it, ensure that the RECORD is // valid. diff --git a/crates/uv-distribution/src/hash.rs b/crates/uv-distribution/src/hash.rs new file mode 100644 index 0000000000000..5de3f6fc907d0 --- /dev/null +++ b/crates/uv-distribution/src/hash.rs @@ -0,0 +1,11 @@ +use uv_distribution_types::HashPolicy; +use uv_pypi_types::HashAlgorithm; + +/// Return the algorithms to compute for an HTTP distribution. +pub(crate) fn http_hash_algorithms(hashes: HashPolicy<'_>) -> Vec { + let mut algorithms = hashes.algorithms(); + algorithms.push(HashAlgorithm::Sha256); + algorithms.sort(); + algorithms.dedup(); + algorithms +} diff --git a/crates/uv-distribution/src/lib.rs b/crates/uv-distribution/src/lib.rs index f5c83eeb5f4e7..f185233ab19eb 100644 --- a/crates/uv-distribution/src/lib.rs +++ b/crates/uv-distribution/src/lib.rs @@ -14,6 +14,7 @@ mod archive; mod distribution_database; mod download; mod error; +mod hash; mod index; mod metadata; mod reporter; diff --git a/crates/uv-distribution/src/source/mod.rs b/crates/uv-distribution/src/source/mod.rs index b80adac497dc6..532d6f7b01c55 100644 --- a/crates/uv-distribution/src/source/mod.rs +++ b/crates/uv-distribution/src/source/mod.rs @@ -50,6 +50,7 @@ use uv_workspace::pyproject::ToolUvSources; use crate::distribution_database::ManagedClient; use crate::error::Error; +use crate::hash::http_hash_algorithms; use crate::metadata::{ArchiveMetadata, GitWorkspaceMember, Metadata}; use crate::source::built_wheel_metadata::{BuiltWheelFile, BuiltWheelMetadata}; use crate::source::revision::Revision; @@ -980,7 +981,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { // Download the source distribution. debug!("Downloading source distribution: {source}"); let entry = cache_shard.shard(revision.id()).entry(SOURCE); - let algorithms = hashes.algorithms(); + let algorithms = http_hash_algorithms(hashes); let hashes = self .download_archive(query_url, response, source, ext, entry.path(), &algorithms) .await?; @@ -2718,7 +2719,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { async { // Take the union of the requested and existing hash algorithms. let algorithms = { - let mut algorithms = hashes.algorithms(); + let mut algorithms = http_hash_algorithms(hashes); for digest in revision.hashes() { algorithms.push(digest.algorithm()); } diff --git a/crates/uv/tests/it/pip_sync.rs b/crates/uv/tests/it/pip_sync.rs index 7b3e24c31af5c..4d183533d1a65 100644 --- a/crates/uv/tests/it/pip_sync.rs +++ b/crates/uv/tests/it/pip_sync.rs @@ -3913,6 +3913,7 @@ fn require_hashes_wrong_algorithm() -> Result<()> { sha512:cfdb2b588b9fc25ede96d8db56ed50848b0b649dca3dd1df0b11f683bb9e0b5f Computed: + sha256:cfdb2b588b9fc25ede96d8db56ed50848b0b649dca3dd1df0b11f683bb9e0b5f sha512:f30761c1e8725b49c498273b90dba4b05c0fd157811994c806183062cb6647e773364ce45f0e1ff0b10e32fe6d0232ea5ad39476ccf37109d6b49603a09c11c2 " ); diff --git a/crates/uv/tests/it/sync.rs b/crates/uv/tests/it/sync.rs index f5469a238f311..21905b6281914 100644 --- a/crates/uv/tests/it/sync.rs +++ b/crates/uv/tests/it/sync.rs @@ -49,6 +49,95 @@ fn sync() -> Result<()> { Ok(()) } +/// Ensure that `uv sync` reuses remote wheels cached by `uv pip install`. +#[test] +fn sync_reuses_pip_install_wheel_cache() -> Result<()> { + let context = uv_test::test_context!("3.12"); + + let pyproject_toml = context.temp_dir.child("pyproject.toml"); + pyproject_toml.write_str( + r#" + [project] + name = "project" + version = "0.1.0" + requires-python = ">=3.12" + dependencies = ["iniconfig==2.0.0"] + "#, + )?; + + // Create a lockfile with hashes, but populate the wheel cache via `uv pip install`. + context.lock().assert().success(); + context + .pip_install() + .arg("iniconfig==2.0.0") + .assert() + .success(); + fs_err::remove_dir_all(&context.venv)?; + + uv_snapshot!(context.filters(), context.sync().arg("--offline"), @" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Using CPython 3.12.[X] interpreter at: [PYTHON-3.12] + Creating virtual environment at: .venv + Resolved 2 packages in [TIME] + Installed 1 package in [TIME] + + iniconfig==2.0.0 + "); + + Ok(()) +} + +/// Ensure that `uv sync` reuses remote source distributions cached by `uv pip install`. +#[test] +fn sync_reuses_pip_install_sdist_cache() -> Result<()> { + let context = uv_test::test_context!("3.12"); + + let pyproject_toml = context.temp_dir.child("pyproject.toml"); + pyproject_toml.write_str( + r#" + [project] + name = "project" + version = "0.1.0" + requires-python = ">=3.12" + dependencies = ["iniconfig==2.0.0"] + "#, + )?; + + // Create a lockfile with hashes, but populate the source distribution cache via `uv pip + // install`. + context.lock().assert().success(); + context + .pip_install() + .arg("iniconfig==2.0.0") + .arg("--no-binary") + .arg("iniconfig") + .assert() + .success(); + fs_err::remove_dir_all(&context.venv)?; + + uv_snapshot!(context.filters(), context + .sync() + .arg("--offline") + .arg("--no-binary-package") + .arg("iniconfig"), @" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Using CPython 3.12.[X] interpreter at: [PYTHON-3.12] + Creating virtual environment at: .venv + Resolved 2 packages in [TIME] + Installed 1 package in [TIME] + + iniconfig==2.0.0 + "); + + Ok(()) +} + #[test] fn locked() -> Result<()> { let context = uv_test::test_context!("3.12"); From dd49ab6e7544b61a19df2655eb450b82c4f784fc Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Wed, 3 Jun 2026 14:09:38 -0400 Subject: [PATCH 2/2] Update sync cache reuse snapshots --- crates/uv/tests/it/sync.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/uv/tests/it/sync.rs b/crates/uv/tests/it/sync.rs index 21905b6281914..f383ec72317ef 100644 --- a/crates/uv/tests/it/sync.rs +++ b/crates/uv/tests/it/sync.rs @@ -10021,7 +10021,6 @@ fn sync_all_extras() -> Result<()> { ----- stderr ----- Resolved 8 packages in [TIME] - Prepared 1 package in [TIME] Uninstalled 2 packages in [TIME] Installed 1 package in [TIME] + packaging==24.0 @@ -10320,7 +10319,6 @@ fn sync_all_groups() -> Result<()> { ----- stderr ----- Resolved 8 packages in [TIME] - Prepared 1 package in [TIME] Uninstalled 2 packages in [TIME] Installed 1 package in [TIME] + packaging==24.0 @@ -10706,7 +10704,7 @@ fn sync_stale_egg_info() -> Result<()> { ----- stderr ----- Resolved 4 packages in [TIME] - Prepared 3 packages in [TIME] + Prepared 2 packages in [TIME] Installed 3 packages in [TIME] + member==0.1.dev5+gfea1041 (from git+https://github.com/astral-sh/uv-stale-egg-info-test.git@fea10416b9c479ac88fb217e14e40249b63bfbee#subdirectory=member) + root==0.1.dev5+gfea1041 (from git+https://github.com/astral-sh/uv-stale-egg-info-test.git@fea10416b9c479ac88fb217e14e40249b63bfbee)