diff --git a/Cargo.lock b/Cargo.lock index ed95cf485686f..9fad143ebd1ec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -363,14 +363,12 @@ version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06575e6a9673580f52661c92107baabffbf41e2141373441cbcdc47cb733003c" dependencies = [ - "bzip2", "flate2", "futures-core", "futures-io", "memchr", "pin-project-lite", "tokio", - "xz2", "zstd", "zstd-safe", ] @@ -657,25 +655,6 @@ version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" -[[package]] -name = "bzip2" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47" -dependencies = [ - "bzip2-sys", -] - -[[package]] -name = "bzip2-sys" -version = "0.1.13+1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" -dependencies = [ - "cc", - "pkg-config", -] - [[package]] name = "camino" version = "1.2.2" @@ -2697,17 +2676,6 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" -[[package]] -name = "lzma-sys" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - [[package]] name = "mailparse" version = "0.16.1" @@ -6489,7 +6457,6 @@ dependencies = [ "uv-python", "uv-redacted", "uv-types", - "uv-warnings", "uv-workspace", "walkdir", ] @@ -6592,8 +6559,6 @@ dependencies = [ "uv-distribution-filename", "uv-pypi-types", "uv-static", - "uv-warnings", - "xz2", ] [[package]] @@ -8122,15 +8087,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec7a2a501ed189703dba8b08142f057e887dfc4b2cc4db2d343ac6376ba3e0b9" -[[package]] -name = "xz2" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" -dependencies = [ - "lzma-sys", -] - [[package]] name = "yansi" version = "1.0.1" diff --git a/Cargo.toml b/Cargo.toml index 4f7d7e36888ef..ae0df04a8c035 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -103,19 +103,14 @@ arrayvec = { version = "0.7.6" } astral-tokio-tar = { version = "0.6.3" } async-channel = { version = "2.3.1" } async-compression = { version = "0.4.12", features = [ - "bzip2", "gzip", - "xz", "zstd", ] } async-trait = { version = "0.1.82" } async_http_range_reader = { version = "0.11.0", package = "astral_async_http_range_reader" } async_zip = { version = "0.0.20", package = "astral_async_zip", features = [ - "bzip2", "deflate", - "lzma", "tokio", - "xz", "zstd", ] } axoupdater = { version = "0.10.0", default-features = false } @@ -317,7 +312,6 @@ windows = { version = "0.61.0", features = [ windows-registry = { version = "0.5.0" } windows-version = { version = "0.1.6" } wiremock = { version = "0.6.4" } -xz2 = { version = "0.1.7", features = ["static"] } zeroize = { version = "1.8.1" } # dev-dependencies diff --git a/crates/uv-bench/Cargo.toml b/crates/uv-bench/Cargo.toml index ca8a62227c93f..8cbea182fd035 100644 --- a/crates/uv-bench/Cargo.toml +++ b/crates/uv-bench/Cargo.toml @@ -69,6 +69,3 @@ tempfile = { workspace = true } tokio = { workspace = true } tokio-util = { workspace = true } toml = { workspace = true } - -[features] -static = ["uv-extract/static"] diff --git a/crates/uv-bench/benches/uv.rs b/crates/uv-bench/benches/uv.rs index cb371739797bd..d37339178c6fb 100644 --- a/crates/uv-bench/benches/uv.rs +++ b/crates/uv-bench/benches/uv.rs @@ -27,7 +27,6 @@ use uv_resolver::Manifest; const MANY_FILES_WHEEL_FILENAME: &str = "manyfiles-0.0.0-py3-none-any.whl"; const MANY_FILES_WHEEL_FILE_COUNT: usize = 10_000; -const MANY_FILES_SDIST_FILENAME: &str = "manyfiles-0.0.0.tar.gz"; const MANY_FILES_SDIST_TOP_LEVEL: &str = "manyfiles-0.0.0"; const MANY_FILES_SDIST_FILE_COUNT: usize = 10_000; @@ -126,7 +125,6 @@ fn unpack_sdist_many_files(c: &mut Criterion) { |(archive, extracted_sdist)| { let files = runtime .block_on(uv_extract::stream::archive( - MANY_FILES_SDIST_FILENAME, archive, SourceDistExtension::TarGz, extracted_sdist.path(), diff --git a/crates/uv-bin-install/src/lib.rs b/crates/uv-bin-install/src/lib.rs index 1766ed9f95e7a..0bd35ed8d19e4 100644 --- a/crates/uv-bin-install/src/lib.rs +++ b/crates/uv-bin-install/src/lib.rs @@ -21,6 +21,7 @@ use tokio::io::{AsyncRead, ReadBuf}; use tokio_util::compat::FuturesAsyncReadCompatExt; use url::Url; use uv_client::retryable_on_request_failure; +use uv_distribution_filename::LegacySourceDistExtension; use uv_distribution_filename::SourceDistExtension; use uv_static::{astral_mirror_base_url, astral_mirror_url_from_env, custom_astral_mirror_url}; @@ -230,7 +231,7 @@ impl ArchiveFormat { impl From for SourceDistExtension { fn from(val: ArchiveFormat) -> Self { match val { - ArchiveFormat::Zip => Self::Zip, + ArchiveFormat::Zip => Self::Legacy(LegacySourceDistExtension::Zip), ArchiveFormat::TarGz => Self::TarGz, } } @@ -841,14 +842,9 @@ async fn download_and_unpack( let id = reporter.on_download_start(binary.name(), version, size); let mut progress_reader = ProgressReader::new(reader, id, reporter); - stream::archive( - &download_url, - &mut progress_reader, - format.into(), - temp_dir.path(), - ) - .await - .map_err(|e| Error::Extract { source: e })?; + stream::archive(&mut progress_reader, format.into(), temp_dir.path()) + .await + .map_err(|e| Error::Extract { source: e })?; reporter.on_download_complete(id); // Find the binary in the extracted files diff --git a/crates/uv-dev/Cargo.toml b/crates/uv-dev/Cargo.toml index c7e6b39ad8abf..d7cf399cd753d 100644 --- a/crates/uv-dev/Cargo.toml +++ b/crates/uv-dev/Cargo.toml @@ -79,7 +79,7 @@ name = "uv-dev" required-features = ["dev"] [features] -default = ["performance", "uv-extract/static"] +default = ["performance"] # Actually build the dev CLI. dev = [] performance = ["performance-memory-allocator"] diff --git a/crates/uv-dev/src/validate_zip.rs b/crates/uv-dev/src/validate_zip.rs index e7a44835db7d0..1614254274872 100644 --- a/crates/uv-dev/src/validate_zip.rs +++ b/crates/uv-dev/src/validate_zip.rs @@ -47,7 +47,7 @@ pub(crate) async fn validate_zip( let target = tempfile::TempDir::new()?; - uv_extract::stream::unzip(args.url.into_url(), reader.compat(), target.path()).await?; + uv_extract::stream::unzip(reader.compat(), target.path()).await?; Ok(()) } diff --git a/crates/uv-distribution-filename/src/extension.rs b/crates/uv-distribution-filename/src/extension.rs index 5fb4f033caf6e..4c75caee7b8fa 100644 --- a/crates/uv-distribution-filename/src/extension.rs +++ b/crates/uv-distribution-filename/src/extension.rs @@ -27,9 +27,29 @@ pub enum DistExtension { )] #[rkyv(derive(Debug))] pub enum SourceDistExtension { + TarGz, + Legacy(LegacySourceDistExtension), +} + +#[derive( + Clone, + Copy, + Debug, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + Serialize, + Deserialize, + rkyv::Archive, + rkyv::Deserialize, + rkyv::Serialize, +)] +#[rkyv(derive(Debug))] +pub enum LegacySourceDistExtension { Tar, TarBz2, - TarGz, TarLz, TarLzma, TarXz, @@ -83,20 +103,17 @@ impl SourceDistExtension { match extension { "gz" if is_tar(path.as_ref()) => Ok(Self::TarGz), - // TODO: Remove these other extensions in the future. - // NOTE: These get parsed from network sources like PyPI, so we don't - // necessarily want to hard-fail, just skip in the future. - "zip" => Ok(Self::Zip), - "tar" => Ok(Self::Tar), - "tgz" => Ok(Self::Tgz), - "tbz" => Ok(Self::Tbz), - "txz" => Ok(Self::Txz), - "tlz" => Ok(Self::Tlz), - "bz2" if is_tar(path.as_ref()) => Ok(Self::TarBz2), - "xz" if is_tar(path.as_ref()) => Ok(Self::TarXz), - "lz" if is_tar(path.as_ref()) => Ok(Self::TarLz), - "lzma" if is_tar(path.as_ref()) => Ok(Self::TarLzma), - "zst" if is_tar(path.as_ref()) => Ok(Self::TarZst), + "zip" => Ok(Self::Legacy(LegacySourceDistExtension::Zip)), + "tar" => Ok(Self::Legacy(LegacySourceDistExtension::Tar)), + "tgz" => Ok(Self::Legacy(LegacySourceDistExtension::Tgz)), + "tbz" => Ok(Self::Legacy(LegacySourceDistExtension::Tbz)), + "txz" => Ok(Self::Legacy(LegacySourceDistExtension::Txz)), + "tlz" => Ok(Self::Legacy(LegacySourceDistExtension::Tlz)), + "bz2" if is_tar(path.as_ref()) => Ok(Self::Legacy(LegacySourceDistExtension::TarBz2)), + "xz" if is_tar(path.as_ref()) => Ok(Self::Legacy(LegacySourceDistExtension::TarXz)), + "lz" if is_tar(path.as_ref()) => Ok(Self::Legacy(LegacySourceDistExtension::TarLz)), + "lzma" if is_tar(path.as_ref()) => Ok(Self::Legacy(LegacySourceDistExtension::TarLzma)), + "zst" if is_tar(path.as_ref()) => Ok(Self::Legacy(LegacySourceDistExtension::TarZst)), _ => Err(ExtensionError::SourceDist), } } @@ -104,20 +121,31 @@ impl SourceDistExtension { /// Return the name for the extension. pub(crate) fn name(self) -> &'static str { match self { - Self::Tar => "tar", - Self::TarBz2 => "tar.bz2", Self::TarGz => "tar.gz", - Self::TarLz => "tar.lz", - Self::TarLzma => "tar.lzma", - Self::TarXz => "tar.xz", - Self::TarZst => "tar.zst", - Self::Tbz => "tbz", - Self::Tgz => "tgz", - Self::Tlz => "tlz", - Self::Txz => "txz", - Self::Zip => "zip", + Self::Legacy(LegacySourceDistExtension::Tar) => "tar", + Self::Legacy(LegacySourceDistExtension::TarBz2) => "tar.bz2", + Self::Legacy(LegacySourceDistExtension::TarLz) => "tar.lz", + Self::Legacy(LegacySourceDistExtension::TarLzma) => "tar.lzma", + Self::Legacy(LegacySourceDistExtension::TarXz) => "tar.xz", + Self::Legacy(LegacySourceDistExtension::TarZst) => "tar.zst", + Self::Legacy(LegacySourceDistExtension::Tbz) => "tbz", + Self::Legacy(LegacySourceDistExtension::Tgz) => "tgz", + Self::Legacy(LegacySourceDistExtension::Tlz) => "tlz", + Self::Legacy(LegacySourceDistExtension::Txz) => "txz", + Self::Legacy(LegacySourceDistExtension::Zip) => "zip", } } + + /// Returns `true` if the extension conforms to [PEP 625](https://peps.python.org/pep-0625/)'s + /// naming requirements. + /// + /// PEP 625 mandates `.tar.gz`; `.zip` is also accepted for backwards compatibility. + pub fn is_pep625_compliant(&self) -> bool { + matches!( + self, + Self::TarGz | Self::Legacy(LegacySourceDistExtension::Zip) + ) + } } impl Display for SourceDistExtension { diff --git a/crates/uv-distribution-filename/src/lib.rs b/crates/uv-distribution-filename/src/lib.rs index bb83a50aa3bce..8fa91b804e85d 100644 --- a/crates/uv-distribution-filename/src/lib.rs +++ b/crates/uv-distribution-filename/src/lib.rs @@ -8,7 +8,9 @@ use uv_pep440::Version; pub use build_tag::{BuildTag, BuildTagError}; pub use egg::{EggInfoFilename, EggInfoFilenameError}; pub use expanded_tags::{ExpandedTagError, ExpandedTags}; -pub use extension::{DistExtension, ExtensionError, SourceDistExtension}; +pub use extension::{ + DistExtension, ExtensionError, LegacySourceDistExtension, SourceDistExtension, +}; pub use source_dist::{SourceDistFilename, SourceDistFilenameError}; pub use wheel::{WheelFilename, WheelFilenameError}; diff --git a/crates/uv-distribution-filename/src/source_dist.rs b/crates/uv-distribution-filename/src/source_dist.rs index e58228bf88b48..69252fd6ae568 100644 --- a/crates/uv-distribution-filename/src/source_dist.rs +++ b/crates/uv-distribution-filename/src/source_dist.rs @@ -188,7 +188,7 @@ mod tests { use uv_normalize::PackageName; - use crate::{SourceDistExtension, SourceDistFilename}; + use crate::{SourceDistExtension, SourceDistFilename, extension::LegacySourceDistExtension}; /// Only test already normalized names since the parsing is lossy /// @@ -239,8 +239,14 @@ mod tests { fn malformed_non_ascii() { let package_name = PackageName::from_str("a").unwrap(); for (filename, extension) in [ - ("é-1.2.3.zip", SourceDistExtension::Zip), - ("aé1.2.3.zip", SourceDistExtension::Zip), + ( + "é-1.2.3.zip", + SourceDistExtension::Legacy(LegacySourceDistExtension::Zip), + ), + ( + "aé1.2.3.zip", + SourceDistExtension::Legacy(LegacySourceDistExtension::Zip), + ), ("é-1.zip", SourceDistExtension::TarGz), ] { assert!(SourceDistFilename::parse(filename, extension, &package_name).is_err()); @@ -257,9 +263,13 @@ mod tests { "Foo.Bar-1.2.3.zip", ] { assert_eq!( - SourceDistFilename::parse(filename, SourceDistExtension::Zip, &package_name) - .unwrap() - .name, + SourceDistFilename::parse( + filename, + SourceDistExtension::Legacy(LegacySourceDistExtension::Zip), + &package_name + ) + .unwrap() + .name, package_name ); } @@ -270,7 +280,7 @@ mod tests { assert!( SourceDistFilename::parse( "foo.zip", - SourceDistExtension::Zip, + SourceDistExtension::Legacy(LegacySourceDistExtension::Zip), &PackageName::from_str("foo-lib").unwrap() ) .is_err() diff --git a/crates/uv-distribution-types/src/error.rs b/crates/uv-distribution-types/src/error.rs index f78ec47386037..b18d27361a178 100644 --- a/crates/uv-distribution-types/src/error.rs +++ b/crates/uv-distribution-types/src/error.rs @@ -24,4 +24,9 @@ pub enum Error { #[error("Requested package name `{0}` does not match `{1}` in the distribution filename: {2}")] PackageNameMismatch(PackageName, PackageName, String), + + #[error( + "Source distribution `{0}` has a non-PEP 625-compliant filename; only `.tar.gz` and `.zip` archives are accepted" + )] + NotPep625Filename(String), } diff --git a/crates/uv-distribution-types/src/lib.rs b/crates/uv-distribution-types/src/lib.rs index 0a749d65c8a53..3c9dc6c9c9b06 100644 --- a/crates/uv-distribution-types/src/lib.rs +++ b/crates/uv-distribution-types/src/lib.rs @@ -417,6 +417,9 @@ impl Dist { }))) } DistExtension::Source(ext) => { + if !ext.is_pep625_compliant() { + return Err(Error::NotPep625Filename(url.verbatim().to_string())); + } Ok(Self::Source(SourceDist::DirectUrl(DirectUrlSourceDist { name, location: Box::new(location), @@ -469,6 +472,10 @@ impl Dist { }))) } DistExtension::Source(ext) => { + if !ext.is_pep625_compliant() { + return Err(Error::NotPep625Filename(url.verbatim().to_string())); + } + // If there is a version in the filename, record it. let version = url .filename() @@ -712,17 +719,6 @@ impl BuiltDist { } impl SourceDist { - /// Returns the [`SourceDistExtension`] of the distribution, if it has one. - pub fn extension(&self) -> Option { - match self { - Self::Registry(source_dist) => Some(source_dist.ext), - Self::DirectUrl(source_dist) => Some(source_dist.ext), - Self::GitPath(source_dist) => Some(source_dist.ext), - Self::Path(source_dist) => Some(source_dist.ext), - Self::GitDirectory(_) | Self::Directory(_) => None, - } - } - /// Returns the [`IndexUrl`], if the distribution is from a registry. fn index(&self) -> Option<&IndexUrl> { match self { diff --git a/crates/uv-distribution-types/src/prioritized_distribution.rs b/crates/uv-distribution-types/src/prioritized_distribution.rs index cb66b1a28862c..da385fa8f4547 100644 --- a/crates/uv-distribution-types/src/prioritized_distribution.rs +++ b/crates/uv-distribution-types/src/prioritized_distribution.rs @@ -144,6 +144,7 @@ impl IncompatibleDist { IncompatibleSource::RequiresPython(..) => { format!("requires {self}") } + IncompatibleSource::NotPep625Filename => format!("has {self}"), }, Self::Unavailable => format!("has {self}"), } @@ -172,6 +173,7 @@ impl IncompatibleDist { IncompatibleSource::RequiresPython(..) => { format!("require {self}") } + IncompatibleSource::NotPep625Filename => format!("have {self}"), }, Self::Unavailable => format!("have {self}"), } @@ -277,6 +279,9 @@ impl Display for IncompatibleDist { IncompatibleSource::RequiresPython(python, _) => { write!(f, "Python {python}") } + IncompatibleSource::NotPep625Filename => { + f.write_str("a non-PEP 625-compliant source distribution filename") + } }, Self::Unavailable => f.write_str("no available distributions"), } @@ -327,6 +332,8 @@ pub enum IncompatibleSource { RequiresPython(VersionSpecifiers, PythonRequirementKind), Yanked(Yanked), NoBuild, + /// The source distribution's filename does not confirm to PEP 625. + NotPep625Filename, } #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] @@ -737,21 +744,26 @@ impl IncompatibleSource { Self::ExcludeNewer(timestamp_self) => match other { // Smaller timestamps are closer to the cut-off time Self::ExcludeNewer(timestamp_other) => timestamp_other < timestamp_self, - Self::NoBuild | Self::RequiresPython(_, _) | Self::Yanked(_) => true, + Self::NoBuild + | Self::RequiresPython(_, _) + | Self::Yanked(_) + | Self::NotPep625Filename => true, }, Self::RequiresPython(_, _) => match other { Self::ExcludeNewer(_) => false, // Version specifiers cannot be reasonably compared Self::RequiresPython(_, _) => false, - Self::NoBuild | Self::Yanked(_) => true, + Self::NoBuild | Self::Yanked(_) | Self::NotPep625Filename => true, }, Self::Yanked(_) => match other { - Self::ExcludeNewer(_) | Self::RequiresPython(_, _) => false, + Self::ExcludeNewer(_) | Self::RequiresPython(_, _) | Self::NotPep625Filename => { + false + } // Yanks with a reason are more helpful for errors Self::Yanked(yanked_other) => matches!(yanked_other, Yanked::Reason(_)), Self::NoBuild => true, }, - Self::NoBuild => false, + Self::NoBuild | Self::NotPep625Filename => false, } } } diff --git a/crates/uv-distribution/Cargo.toml b/crates/uv-distribution/Cargo.toml index e961afd2b8f07..0a014b2f5133c 100644 --- a/crates/uv-distribution/Cargo.toml +++ b/crates/uv-distribution/Cargo.toml @@ -40,7 +40,6 @@ uv-pypi-types = { workspace = true } uv-python = { workspace = true } uv-redacted = { workspace = true } uv-types = { workspace = true } -uv-warnings = { workspace = true } uv-workspace = { workspace = true } anyhow = { workspace = true } @@ -68,4 +67,3 @@ insta = { workspace = true } [features] default = [] -static = ["uv-extract/static"] diff --git a/crates/uv-distribution/src/distribution_database.rs b/crates/uv-distribution/src/distribution_database.rs index 96fe70c780b50..fa7261f0e3caa 100644 --- a/crates/uv-distribution/src/distribution_database.rs +++ b/crates/uv-distribution/src/distribution_database.rs @@ -17,7 +17,7 @@ use uv_cache_info::{CacheInfo, Timestamp}; use uv_client::{ CacheControl, CachedClientError, Connectivity, DataWithCachePolicy, RegistryClient, }; -use uv_distribution_filename::{SourceDistExtension, WheelFilename}; +use uv_distribution_filename::WheelFilename; use uv_distribution_types::{ BuildInfo, BuildableSource, BuiltDist, Dist, DistRef, File, HashPolicy, Hashed, IndexUrl, InstalledDist, Name, SourceDist, ToUrlError, @@ -31,7 +31,6 @@ use uv_pypi_types::{HashDigest, HashDigests, PyProjectToml}; use uv_python::PythonVariant; use uv_redacted::DisplaySafeUrl; use uv_types::{BuildContext, BuildStack}; -use uv_warnings::warn_user_once; use crate::archive::Archive; use crate::error::PythonVersion; @@ -437,34 +436,6 @@ impl<'a, Context: BuildContext> DistributionDatabase<'a, Context> { tags: &Tags, hashes: HashPolicy<'_>, ) -> Result { - // Warn if the source distribution isn't PEP 625 compliant. - // We do this here instead of in `SourceDistExtension::from_path` to minimize log volume: - // a non-compliant distribution isn't a huge problem if it's not actually being - // materialized into a wheel. Observe that we also allow no extension, since we expect that - // for directory and Git installs. - // NOTE: Observe that we also allow `.zip` sdists here, which are not PEP 625 compliant. - // This is because they were allowed on PyPI until relatively recently (2020). - if let Some(extension) = dist.extension() - && !matches!( - extension, - SourceDistExtension::TarGz | SourceDistExtension::Zip - ) - { - if matches!(dist, SourceDist::Registry(_)) { - // Observe that we display a slightly different warning when the sdist comes - // from a registry, since that suggests that the user has inadvertently - // (rather than explicitly) depended on a non-compliant sdist. - warn_user_once!( - "{dist} uses a legacy source distribution format ('.{extension}') that is not compliant with PEP 625. A future version of uv will reject this source distribution. Consider upgrading to a newer version of {package}", - package = dist.name(), - ); - } else { - warn_user_once!( - "{dist} is not a standards-compliant source distribution: expected '.tar.gz' but found '.{extension}'. A future version of uv will reject source distributions that do not meet the requirements specified in PEP 625", - ); - } - } - let built_wheel = self .builder .download_and_build(&BuildableSource::Dist(dist), tags, hashes, &self.client) @@ -714,8 +685,6 @@ impl<'a, Context: BuildContext> DistributionDatabase<'a, Context> { // Create an entry for the HTTP cache. let http_entry = wheel_entry.with_file(format!("{}.http", filename.cache_key())); - let query_url = &url.clone(); - let download = |response: reqwest::Response| { async { let size = size.or_else(|| content_length(&response)); @@ -744,7 +713,7 @@ impl<'a, Context: BuildContext> DistributionDatabase<'a, Context> { let mut reader = ProgressReader::new(&mut hasher, progress, &**reporter); match extension { WheelExtension::Whl => { - uv_extract::stream::unzip(query_url, &mut reader, temp_dir.path()) + uv_extract::stream::unzip(&mut reader, temp_dir.path()) .await .map_err(|err| Error::Extract(filename.to_string(), err))? } @@ -757,7 +726,7 @@ impl<'a, Context: BuildContext> DistributionDatabase<'a, Context> { } None => match extension { WheelExtension::Whl => { - uv_extract::stream::unzip(query_url, &mut hasher, temp_dir.path()) + uv_extract::stream::unzip(&mut hasher, temp_dir.path()) .await .map_err(|err| Error::Extract(filename.to_string(), err))? } @@ -1127,11 +1096,9 @@ impl<'a, Context: BuildContext> DistributionDatabase<'a, Context> { // Unzip the wheel to a temporary directory. let files = match extension { - WheelExtension::Whl => { - uv_extract::stream::unzip(path.display(), &mut hasher, temp_dir.path()) - .await - .map_err(|err| Error::Extract(filename.to_string(), err))? - } + WheelExtension::Whl => uv_extract::stream::unzip(&mut hasher, temp_dir.path()) + .await + .map_err(|err| Error::Extract(filename.to_string(), err))?, WheelExtension::WhlZst => { uv_extract::stream::untar_zst(&mut hasher, temp_dir.path()) .await diff --git a/crates/uv-distribution/src/source/mod.rs b/crates/uv-distribution/src/source/mod.rs index 59e78a7c3e52f..1a69c7189b9c7 100644 --- a/crates/uv-distribution/src/source/mod.rs +++ b/crates/uv-distribution/src/source/mod.rs @@ -974,8 +974,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { }; let download = |response| { - let query_url = url.clone(); - async { // At this point, we're seeing a new or updated source distribution. Initialize a // new revision, to collect the source and built artifacts. @@ -986,7 +984,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { let entry = cache_shard.shard(revision.id()).entry(SOURCE); let algorithms = http_hash_algorithms(hashes); let hashes = self - .download_archive(query_url, response, source, ext, entry.path(), &algorithms) + .download_archive(response, source, ext, entry.path(), &algorithms) .await?; Ok(revision.with_hashes(HashDigests::from(hashes))) @@ -2749,8 +2747,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { }; let download = |response| { - let query_url = url.clone(); - async { // Take the union of the requested and existing hash algorithms. let algorithms = { @@ -2764,7 +2760,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { }; let hashes = self - .download_archive(query_url, response, source, ext, entry.path(), &algorithms) + .download_archive(response, source, ext, entry.path(), &algorithms) .await?; for existing in revision.hashes() { if !hashes.contains(existing) { @@ -2798,7 +2794,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { /// Download and unzip a source distribution into the cache from an HTTP response. async fn download_archive( &self, - query_url: DisplaySafeUrl, response: Response, source: &BuildableSource<'_>, ext: SourceDistExtension, @@ -2827,7 +2822,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { // Download and unzip the source distribution into a temporary directory. let span = info_span!("download_source_dist", source_dist = %source); - uv_extract::stream::archive(query_url, &mut hasher, ext, temp_dir.path()) + uv_extract::stream::archive(&mut hasher, ext, temp_dir.path()) .await .map_err(|err| Error::Extract(source.to_string(), err))?; drop(span); @@ -2896,7 +2891,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { let mut hasher = uv_extract::hash::HashReader::new(reader, &mut hashers); // Unzip the archive into a temporary directory. - uv_extract::stream::archive(path.display(), &mut hasher, ext, &temp_dir.path()) + uv_extract::stream::archive(&mut hasher, ext, &temp_dir.path()) .await .map_err(|err| Error::Extract(temp_dir.path().to_string_lossy().into_owned(), err))?; diff --git a/crates/uv-extract/Cargo.toml b/crates/uv-extract/Cargo.toml index 677356a62db67..406105626a54a 100644 --- a/crates/uv-extract/Cargo.toml +++ b/crates/uv-extract/Cargo.toml @@ -20,10 +20,9 @@ uv-configuration = { workspace = true } uv-distribution-filename = { workspace = true } uv-pypi-types = { workspace = true } uv-static = { workspace = true } -uv-warnings = { workspace = true } astral-tokio-tar = { workspace = true } -async-compression = { workspace = true, features = ["bzip2", "gzip", "zstd", "xz"] } +async-compression = { workspace = true, features = ["gzip", "zstd"] } async_zip = { workspace = true } blake2 = { workspace = true } flate2 = { workspace = true } @@ -39,15 +38,12 @@ thiserror = { workspace = true } tokio = { workspace = true } tokio-util = { workspace = true, features = ["compat"] } tracing = { workspace = true } -xz2 = { workspace = true } [features] default = [] -# Avoid a liblzma.so dependency -static = ["xz2/static"] [package.metadata.cargo-shear] # NOTE: `async-compression` owns the actual Deflate codec, but we need to # explicitly declare `flate2` here so the workspace's `zlib-rs` backend stays # selected after removing the synchronous `zip` dependency. -ignored = ["flate2", "xz2"] +ignored = ["flate2"] diff --git a/crates/uv-extract/src/error.rs b/crates/uv-extract/src/error.rs index dc1f1d504cb50..88997d103060a 100644 --- a/crates/uv-extract/src/error.rs +++ b/crates/uv-extract/src/error.rs @@ -93,17 +93,23 @@ pub enum Error { EmptyFilename, #[error("Archive contains unacceptable filename: {filename}")] UnacceptableFilename { filename: String }, + #[error( + "Archive contains a file with an unsupported compression method; files must be compressed with 'stored', 'DEFLATE', or 'zstd'" + )] + UnsupportedCompression, } impl From for Error { fn from(err: async_zip::error::ZipError) -> Self { - let async_zip::error::ZipError::FileNameContainsNul { filename } = err else { - return Self::AsyncZip(err); - }; - - let filename = String::from_utf8_lossy(&filename); - validate_archive_member_name(&filename) - .expect_err("a filename containing an embedded NUL must be rejected") + match err { + async_zip::error::ZipError::CompressionNotSupported(_) => Self::UnsupportedCompression, + async_zip::error::ZipError::FileNameContainsNul { filename } => { + let filename = String::from_utf8_lossy(&filename); + validate_archive_member_name(&filename) + .expect_err("a filename containing an embedded NUL must be rejected") + } + error => Self::AsyncZip(error), + } } } @@ -122,7 +128,7 @@ impl Error { Err(err) => err, }; let err = match err.downcast::() { - Ok(zip_err) => return Self::AsyncZip(zip_err), + Ok(zip_err) => return Self::from(zip_err), Err(err) => err, }; Self::Io(err) diff --git a/crates/uv-extract/src/lib.rs b/crates/uv-extract/src/lib.rs index 6fd5deacdf887..c6a20b3014015 100644 --- a/crates/uv-extract/src/lib.rs +++ b/crates/uv-extract/src/lib.rs @@ -1,5 +1,3 @@ -use std::fmt::Display; - pub use error::Error; use regex::regex; pub use sync::*; @@ -13,50 +11,6 @@ mod vendor; static REPLACEMENT_CHARACTER: &str = "\u{FFFD}"; -/// Compression methods that we consider supported. -/// -/// Our underlying ZIP dependencies may support more. -pub(crate) enum CompressionMethod { - Stored, - Deflated, - Zstd, - // NOTE: This will become `Unsupported(...)` in the future. - Deprecated(&'static str), -} - -impl CompressionMethod { - /// Returns `true` if this is a well-known compression method that we - /// expect other ZIP implementations to support. - fn is_well_known(&self) -> bool { - matches!(self, Self::Stored | Self::Deflated | Self::Zstd) - } -} - -impl Display for CompressionMethod { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Stored => write!(f, "stored"), - Self::Deflated => write!(f, "DEFLATE"), - Self::Zstd => write!(f, "zstd"), - Self::Deprecated(name) => write!(f, "{name}"), - } - } -} - -impl From for CompressionMethod { - fn from(value: async_zip::Compression) -> Self { - match value { - async_zip::Compression::Stored => Self::Stored, - async_zip::Compression::Deflate => Self::Deflated, - async_zip::Compression::Zstd => Self::Zstd, - async_zip::Compression::Bz => Self::Deprecated("bzip2"), - async_zip::Compression::Lzma => Self::Deprecated("lzma"), - async_zip::Compression::Xz => Self::Deprecated("xz"), - _ => Self::Deprecated("unknown"), - } - } -} - /// Validate that a given filename (e.g. reported by a ZIP archive's /// local file entries or central directory entries) is "safe" to use. /// diff --git a/crates/uv-extract/src/stream.rs b/crates/uv-extract/src/stream.rs index 368ae07c8b3cb..3d332c961c0aa 100644 --- a/crates/uv-extract/src/stream.rs +++ b/crates/uv-extract/src/stream.rs @@ -1,4 +1,3 @@ -use std::fmt::Display; use std::path::{Component, Path, PathBuf}; use std::pin::Pin; @@ -9,10 +8,9 @@ use rustc_hash::{FxHashMap, FxHashSet}; use tokio_util::compat::{FuturesAsyncReadCompatExt, TokioAsyncReadCompatExt}; use tracing::{debug, warn}; -use uv_distribution_filename::SourceDistExtension; -use uv_warnings::warn_user_once; +use uv_distribution_filename::{LegacySourceDistExtension, SourceDistExtension}; -use crate::{CompressionMethod, Error, insecure_no_validate, validate_archive_member_name}; +use crate::{Error, insecure_no_validate, validate_archive_member_name}; const DEFAULT_BUF_SIZE: usize = 128 * 1024; @@ -66,12 +64,8 @@ struct ComputedEntry { /// is already fully on disk, consider using `unzip_archive`, which can use multiple /// threads to work faster in that case. /// -/// `source_hint` is used for warning messages, to identify the source of the ZIP archive -/// beneath the reader. It might be a URL, a file path, or something else. -/// /// Returns the list of unpacked files and their sizes. -pub async fn unzip( - source_hint: D, +pub async fn unzip( reader: R, target: impl AsRef, ) -> Result, Error> { @@ -90,18 +84,6 @@ pub async fn unzip( while let Some(mut entry) = zip.next_with_entry().await? { let zip_entry = entry.reader().entry(); - // Check for unexpected compression methods. - // A future version of uv will reject instead of warning about these. - let compression = CompressionMethod::from(zip_entry.compression()); - if !compression.is_well_known() { - warn_user_once!( - "One or more file entries in '{source_hint}' use the '{compression}' compression method, which is not widely supported. A future version of uv will reject ZIP archives containing entries compressed with this method. Entries must be compressed with the '{stored}', '{deflate}', or '{zstd}' compression methods.", - stored = CompressionMethod::Stored, - deflate = CompressionMethod::Deflated, - zstd = CompressionMethod::Zstd, - ); - } - // Construct the (expected) path to the file on-disk. let path = match zip_entry.filename().as_str() { Ok(path) => path, @@ -684,30 +666,6 @@ async fn untar_gz( .map_err(Error::io_or_compression) } -/// Unpack a `.tar.bz2` archive into the target directory, without requiring `Seek`. -/// -/// This is useful for unpacking files as they're being downloaded. -/// -/// Returns the list of unpacked files and their sizes. -async fn untar_bz2( - reader: R, - target: impl AsRef, -) -> Result, Error> { - let reader = tokio::io::BufReader::with_capacity(DEFAULT_BUF_SIZE, reader); - let mut decompressed_bytes = async_compression::tokio::bufread::BzDecoder::new(reader); - - let archive = tokio_tar::ArchiveBuilder::new( - &mut decompressed_bytes as &mut (dyn tokio::io::AsyncRead + Unpin), - ) - .set_preserve_mtime(false) - .set_preserve_permissions(false) - .set_allow_external_symlinks(false) - .build(); - untar_in(archive, target.as_ref()) - .await - .map_err(Error::io_or_compression) -} - /// Unpack a `.tar.zst` archive into the target directory, without requiring `Seek`. /// /// This is useful for unpacking files as they're being downloaded. @@ -732,30 +690,6 @@ pub async fn untar_zst( .map_err(Error::io_or_compression) } -/// Unpack a `.tar.xz` archive into the target directory, without requiring `Seek`. -/// -/// This is useful for unpacking files as they're being downloaded. -/// -/// Returns the list of unpacked files and their sizes. -async fn untar_xz( - reader: R, - target: impl AsRef, -) -> Result, Error> { - let reader = tokio::io::BufReader::with_capacity(DEFAULT_BUF_SIZE, reader); - let mut decompressed_bytes = async_compression::tokio::bufread::XzDecoder::new(reader); - - let archive = tokio_tar::ArchiveBuilder::new( - &mut decompressed_bytes as &mut (dyn tokio::io::AsyncRead + Unpin), - ) - .set_preserve_mtime(false) - .set_preserve_permissions(false) - .set_allow_external_symlinks(false) - .build(); - untar_in(archive, target.as_ref()) - .await - .map_err(Error::io_or_compression) -} - /// Unpack a `.tar` archive into the target directory, without requiring `Seek`. /// /// This is useful for unpacking files as they're being downloaded. @@ -778,29 +712,23 @@ async fn untar( .map_err(Error::io_or_compression) } -/// Unpack a `.zip`, `.tar.gz`, `.tar.bz2`, `.tar.zst`, or `.tar.xz` archive into the target directory, +/// Unpack a `.zip`, `.tar.gz`, or `.tar.zst` archive into the target directory, /// without requiring `Seek`. /// -/// `source_hint` is used for warning messages, to identify the source of the archive -/// beneath the reader. It might be a URL, a file path, or something else. -/// /// Returns the list of unpacked files and their sizes. -pub async fn archive( - source_hint: D, +pub async fn archive( reader: R, ext: SourceDistExtension, target: impl AsRef, ) -> Result, Error> { match ext { - SourceDistExtension::Zip => unzip(source_hint, reader, target).await, - SourceDistExtension::Tar => untar(reader, target).await, - SourceDistExtension::Tgz | SourceDistExtension::TarGz => untar_gz(reader, target).await, - SourceDistExtension::Tbz | SourceDistExtension::TarBz2 => untar_bz2(reader, target).await, - SourceDistExtension::Txz - | SourceDistExtension::TarXz - | SourceDistExtension::Tlz - | SourceDistExtension::TarLz - | SourceDistExtension::TarLzma => untar_xz(reader, target).await, - SourceDistExtension::TarZst => untar_zst(reader, target).await, + SourceDistExtension::Legacy(LegacySourceDistExtension::Zip) => unzip(reader, target).await, + SourceDistExtension::Legacy(LegacySourceDistExtension::Tar) => untar(reader, target).await, + SourceDistExtension::Legacy(LegacySourceDistExtension::Tgz) + | SourceDistExtension::TarGz => untar_gz(reader, target).await, + SourceDistExtension::Legacy(LegacySourceDistExtension::TarZst) => { + untar_zst(reader, target).await + } + SourceDistExtension::Legacy(_) => Err(Error::UnsupportedCompression), } } diff --git a/crates/uv-extract/src/sync.rs b/crates/uv-extract/src/sync.rs index 795db51a828d8..8edb01d482340 100644 --- a/crates/uv-extract/src/sync.rs +++ b/crates/uv-extract/src/sync.rs @@ -2,7 +2,7 @@ use std::path::{Path, PathBuf}; use std::sync::Mutex; use crate::vendor::CloneableSeekableReader; -use crate::{CompressionMethod, Error, insecure_no_validate, validate_archive_member_name}; +use crate::{Error, insecure_no_validate, validate_archive_member_name}; use async_zip::base::read::seek::ZipFileReader; use async_zip::error::ZipError; use futures::executor::block_on; @@ -11,13 +11,12 @@ use rayon::prelude::*; use rustc_hash::FxHashSet; use tracing::warn; use uv_configuration::initialize_rayon_once; -use uv_warnings::warn_user_once; /// Unzip a `.zip` archive into the target directory. /// /// Returns the list of unpacked files and their sizes. pub fn unzip(reader: fs_err::File, target: &Path) -> Result, Error> { - let (reader, filename) = reader.into_parts(); + let (reader, _) = reader.into_parts(); // Parse the central directory once, then clone the archive reader per Rayon worker so // extraction stays parallel for already-downloaded wheels. @@ -43,17 +42,6 @@ pub fn unzip(reader: fs_err::File, target: &Path) -> Result, Err(err) => return Err(err.into()), }; - let compression = CompressionMethod::from(entry.compression()); - if !compression.is_well_known() { - warn_user_once!( - "One or more file entries in '{filename}' use the '{compression}' compression method, which is not widely supported. A future version of uv will reject ZIP archives containing entries compressed with this method. Entries must be compressed with the '{stored}', '{deflate}', or '{zstd}' compression methods.", - filename = filename.display(), - stored = CompressionMethod::Stored, - deflate = CompressionMethod::Deflated, - zstd = CompressionMethod::Zstd, - ); - } - if let Err(e) = validate_archive_member_name(file_name) { if !skip_validation { return Err(e); diff --git a/crates/uv-metadata/src/lib.rs b/crates/uv-metadata/src/lib.rs index a94765322c0e3..0c651e1690ea6 100644 --- a/crates/uv-metadata/src/lib.rs +++ b/crates/uv-metadata/src/lib.rs @@ -40,12 +40,25 @@ pub enum Error { expected: u32, }, #[error("Failed to read from zip file")] - AsyncZip(#[from] async_zip::error::ZipError), + AsyncZip(#[source] async_zip::error::ZipError), + #[error( + "Archive contains a file with an unsupported compression method; files must be compressed with 'stored', 'DEFLATE', or 'zstd'" + )] + UnsupportedCompression, // No `#[from]` to enforce manual review of `io::Error` sources. #[error(transparent)] Io(io::Error), } +impl From for Error { + fn from(err: async_zip::error::ZipError) -> Self { + match err { + async_zip::error::ZipError::CompressionNotSupported(_) => Self::UnsupportedCompression, + o => Self::AsyncZip(o), + } + } +} + /// Find the `.dist-info` directory in a zipped wheel. /// /// Returns the dist info dir prefix without the `.dist-info` extension. diff --git a/crates/uv-pep508/src/lib.rs b/crates/uv-pep508/src/lib.rs index cf506e213a704..e2bd1d289bc26 100644 --- a/crates/uv-pep508/src/lib.rs +++ b/crates/uv-pep508/src/lib.rs @@ -544,7 +544,9 @@ fn looks_like_unnamed_requirement(cursor: &mut Cursor) -> bool { /// Returns `true` if a file looks like an archive. /// /// See -/// for the list of supported archive extensions. +/// for the original list of supported archive extensions. +/// +/// Observe that we're currently much stricter here, as we no longer accept bz2 or lzma variants. fn looks_like_archive(file: impl AsRef) -> bool { let file = file.as_ref(); @@ -560,8 +562,7 @@ fn looks_like_archive(file: impl AsRef) -> bool { matches!( (pre_extension, extension), - (_, "whl" | "tbz" | "txz" | "tlz" | "zip" | "tgz" | "tar") - | (Some("tar"), "bz2" | "xz" | "lz" | "lzma" | "gz") + (_, "whl" | "zip" | "tgz" | "tar") | (Some("tar"), "gz") ) } diff --git a/crates/uv-python/src/downloads.rs b/crates/uv-python/src/downloads.rs index 1975a5a49dd9f..803c88c2248ad 100644 --- a/crates/uv-python/src/downloads.rs +++ b/crates/uv-python/src/downloads.rs @@ -1485,12 +1485,12 @@ impl ManagedPythonDownload { if let Some(reporter) = reporter { let progress_key = reporter.on_request_start(direction, &self.key, size); let mut reader = ProgressReader::new(&mut hasher, progress_key, reporter); - uv_extract::stream::archive(filename, &mut reader, ext, target) + uv_extract::stream::archive(&mut reader, ext, target) .await .map_err(|err| Error::ExtractError(filename.to_owned(), err))?; reporter.on_request_complete(direction, progress_key); } else { - uv_extract::stream::archive(filename, &mut hasher, ext, target) + uv_extract::stream::archive(&mut hasher, ext, target) .await .map_err(|err| Error::ExtractError(filename.to_owned(), err))?; } diff --git a/crates/uv-resolver/src/flat_index.rs b/crates/uv-resolver/src/flat_index.rs index 2115487db3a7a..ecf6e450161f4 100644 --- a/crates/uv-resolver/src/flat_index.rs +++ b/crates/uv-resolver/src/flat_index.rs @@ -164,6 +164,14 @@ impl FlatDistributions { return SourceDistCompatibility::Incompatible(IncompatibleSource::NoBuild); } + // Check if the filename is PEP 625-compliant. + // TODO: Strengthen this check more; right now we allow `.zip` + // (which is not compliant) and we don't strictly + // enforce the formatting rules for the name or version. + if !filename.extension.is_pep625_compliant() { + return SourceDistCompatibility::Incompatible(IncompatibleSource::NotPep625Filename); + } + // Check if hashes line up let hash_policy = hasher.get_package(&filename.name, &filename.version); let hash = if hash_policy.requires_validation() { diff --git a/crates/uv-resolver/src/lock/mod.rs b/crates/uv-resolver/src/lock/mod.rs index fb8e812aa272c..d9bb5e559be9f 100644 --- a/crates/uv-resolver/src/lock/mod.rs +++ b/crates/uv-resolver/src/lock/mod.rs @@ -3159,6 +3159,12 @@ impl Package { else { return Ok(None); }; + if !ext.is_pep625_compliant() { + return Err(LockErrorKind::NotPep625Filename { + id: self.id.clone(), + } + .into()); + } let install_path = absolute_path(workspace_root, path)?; let given = path.to_str().expect("lock file paths must be UTF-8"); let path_dist = PathSourceDist { @@ -3274,6 +3280,12 @@ impl Package { else { return Ok(None); }; + if !ext.is_pep625_compliant() { + return Err(LockErrorKind::NotPep625Filename { + id: self.id.clone(), + } + .into()); + } let location = url.to_url().map_err(LockErrorKind::InvalidUrl)?; let url = DisplaySafeUrl::from(ParsedArchiveUrl { url: location.clone(), @@ -5768,6 +5780,12 @@ impl LockError { LockErrorKind::NoBuild { .. } | LockErrorKind::NoBinaryNoBuild { .. } ) } + + /// Returns true if the [`LockError`] indicates that the lockfile references a + /// non-PEP 625-compliant source distribution. + pub fn is_not_pep625(&self) -> bool { + matches!(&*self.kind, LockErrorKind::NotPep625Filename { .. }) + } } impl From for LockError @@ -6186,6 +6204,16 @@ enum LockErrorKind { /// The list of valid extensions that were expected. err: ExtensionError, }, + /// An error that occurs when a locked source distribution has a + /// non-PEP 625-compliant filename (e.g., `.tar.bz2`). + #[error( + "Source distribution for `{id}` has a non-PEP 625-compliant filename; only `.tar.gz` and `.zip` archives are accepted", + id = id.cyan() + )] + NotPep625Filename { + /// The ID of the package whose source distribution has a non-PEP 625-compliant filename. + id: PackageId, + }, /// Failed to parse a Git source URL. #[error("Failed to parse Git URL")] InvalidGitSourceUrl( diff --git a/crates/uv-resolver/src/version_map.rs b/crates/uv-resolver/src/version_map.rs index 07f004ce18ba3..10b3802c25907 100644 --- a/crates/uv-resolver/src/version_map.rs +++ b/crates/uv-resolver/src/version_map.rs @@ -9,7 +9,7 @@ use tracing::{instrument, trace}; use uv_client::{FlatIndexEntry, OwnedArchive, SimpleDetailMetadata, VersionFiles}; use uv_configuration::BuildOptions; -use uv_distribution_filename::{DistFilename, WheelFilename}; +use uv_distribution_filename::{DistFilename, SourceDistFilename, WheelFilename}; use uv_distribution_types::{ HashComparison, IncompatibleSource, IncompatibleWheel, IndexUrl, PrioritizedDist, RegistryBuiltWheel, RegistrySourceDist, RequiresPython, SourceDistCompatibility, @@ -692,8 +692,7 @@ impl VersionMapLazy { } DistFilename::SourceDistFilename(filename) => { let compatibility = self.source_dist_compatibility( - &filename.name, - &filename.version, + &filename, hashes.as_slice(), yanked, excluded, @@ -722,8 +721,7 @@ impl VersionMapLazy { fn source_dist_compatibility( &self, - name: &PackageName, - version: &Version, + filename: &SourceDistFilename, hashes: &[HashDigest], yanked: Option<&Yanked>, excluded: bool, @@ -743,15 +741,27 @@ impl VersionMapLazy { // Check if yanked if let Some(yanked) = yanked { - if yanked.is_yanked() && !self.allowed_yanks.contains(name, version) { + if yanked.is_yanked() + && !self + .allowed_yanks + .contains(&filename.name, &filename.version) + { return SourceDistCompatibility::Incompatible(IncompatibleSource::Yanked( yanked.clone(), )); } } + // Check if the filename is PEP 625-compliant. + // TODO: Strengthen this check more; right now we allow `.zip` + // (which is not compliant) and we don't strictly + // enforce the formatting rules for the name or version. + if !filename.extension.is_pep625_compliant() { + return SourceDistCompatibility::Incompatible(IncompatibleSource::NotPep625Filename); + } + // Check if hashes line up. If hashes aren't required, they're considered matching. - let hash_policy = self.hasher.get_package(name, version); + let hash_policy = self.hasher.get_package(&filename.name, &filename.version); let required_hashes = hash_policy.digests(); let hash = if required_hashes.is_empty() { HashComparison::Matched diff --git a/crates/uv/Cargo.toml b/crates/uv/Cargo.toml index d4c6ce80bd8d2..5d82635a9e593 100644 --- a/crates/uv/Cargo.toml +++ b/crates/uv/Cargo.toml @@ -160,7 +160,7 @@ nix = { workspace = true } uv-unix = { workspace = true } [features] -default = ["performance", "uv-distribution/static", "test-defaults"] +default = ["performance", "test-defaults"] native-auth = [] # Use better memory allocators, etc. performance = ["performance-memory-allocator"] diff --git a/crates/uv/src/commands/build_frontend.rs b/crates/uv/src/commands/build_frontend.rs index 433e7412a1e71..994e1dc578075 100644 --- a/crates/uv/src/commands/build_frontend.rs +++ b/crates/uv/src/commands/build_frontend.rs @@ -773,7 +773,7 @@ async fn build_package( let ext = SourceDistExtension::from_path(path.as_path()) .map_err(|err| Error::InvalidSourceDistExt(path.user_display().to_string(), err))?; let temp_dir = tempfile::tempdir_in(cache.bucket(CacheBucket::SourceDistributions))?; - uv_extract::stream::archive(path.display(), reader, ext, temp_dir.path()).await?; + uv_extract::stream::archive(reader, ext, temp_dir.path()).await?; // Extract the top-level directory from the archive. let extracted = match uv_extract::strip_component(temp_dir.path()) { @@ -880,8 +880,7 @@ async fn build_package( Error::InvalidSourceDistExt(source.path().user_display().to_string(), err) })?; let temp_dir = tempfile::tempdir_in(&output_dir)?; - uv_extract::stream::archive(source.path().display(), reader, ext, temp_dir.path()) - .await?; + uv_extract::stream::archive(reader, ext, temp_dir.path()).await?; // If the source distribution has a normalized filename, check its identity. let source_dist = source diff --git a/crates/uv/src/commands/project/lock.rs b/crates/uv/src/commands/project/lock.rs index c471aea612fb4..a74fdc1312eda 100644 --- a/crates/uv/src/commands/project/lock.rs +++ b/crates/uv/src/commands/project/lock.rs @@ -958,6 +958,11 @@ async fn do_lock( // metadata that cannot be obtained under `--no-build`. return Err(ProjectError::Lock(err)); } + Err(ProjectError::Lock(err)) if err.is_not_pep625() => { + // A non-PEP 625-compliant sdist in the lockfile will also be rejected by a fresh + // resolve, so short-circuit rather than doing the extra work. + return Err(ProjectError::Lock(err)); + } Err(err) => { warn_user!("Failed to validate existing lockfile: {err}"); None diff --git a/crates/uv/tests/build/extract.rs b/crates/uv/tests/build/extract.rs index 6386780fb69cb..18e9b6c5e0fac 100644 --- a/crates/uv/tests/build/extract.rs +++ b/crates/uv/tests/build/extract.rs @@ -23,7 +23,7 @@ async fn unzip(url: &str) -> anyhow::Result<(), uv_extract::Error> { .into_async_read(); let target = tempfile::TempDir::new().map_err(uv_extract::Error::Io)?; - uv_extract::stream::unzip(url, reader.compat(), target.path()).await?; + uv_extract::stream::unzip(reader.compat(), target.path()).await?; Ok(()) } diff --git a/crates/uv/tests/pip/pip_sync.rs b/crates/uv/tests/pip/pip_sync.rs index f42a4ddf6fb8d..b59f18127abe4 100644 --- a/crates/uv/tests/pip/pip_sync.rs +++ b/crates/uv/tests/pip/pip_sync.rs @@ -789,9 +789,10 @@ fn install_sdist_url() -> Result<()> { Ok(()) } -/// Install a package with source archive format `.tar.bz2`. +/// Attempt to install a direct URL source distribution with a non-PEP 625-compliant +/// archive format (e.g., `.tar.bz2`). This should hard-error. #[test] -fn install_sdist_archive_type_bz2() -> Result<()> { +fn reject_sdist_archive_type_bz2() -> Result<()> { let context = uv_test::test_context!("3.9"); let requirements_txt = context.temp_dir.child("requirements.txt"); @@ -806,13 +807,9 @@ fn install_sdist_archive_type_bz2() -> Result<()> { uv_snapshot!(context.filters(), context.pip_sync() .arg("requirements.txt") .arg("--strict"), @" - exit_code: 0 (success) + exit_code: 2 (failure) ----- stderr ----- - Resolved 1 package in [TIME] - warning: bz2 @ file://[WORKSPACE]/test/links/bz2-1.0.0.tar.bz2 is not a standards-compliant source distribution: expected '.tar.gz' but found '.tar.bz2'. A future version of uv will reject source distributions that do not meet the requirements specified in PEP 625 - Prepared 1 package in [TIME] - Installed 1 package in [TIME] - + bz2==1.0.0 (from file://[WORKSPACE]/test/links/bz2-1.0.0.tar.bz2) + error: Source distribution `[WORKSPACE]/test/links/bz2-1.0.0.tar.bz2` has a non-PEP 625-compliant filename; only `.tar.gz` and `.zip` archives are accepted " ); diff --git a/crates/uv/tests/pip_install/pip_install.rs b/crates/uv/tests/pip_install/pip_install.rs index 6a603e4787625..9fe867d4d0a13 100644 --- a/crates/uv/tests/pip_install/pip_install.rs +++ b/crates/uv/tests/pip_install/pip_install.rs @@ -15359,7 +15359,7 @@ fn abi_compatibility_on_nondebug_python_with_debug_wheel() { } #[test] -fn warn_on_bz2_wheel() { +fn fail_on_bz2_wheel() { let context = uv_test::test_context!("3.14"); let vendor = FindLinksServer::vendor(); @@ -15368,19 +15368,17 @@ fn warn_on_bz2_wheel() { context.pip_install() .arg(format!("futzed_bz2 @ {}/futzed_bz2-0.1.0-py3-none-any.whl", vendor.url())), @" - exit_code: 0 (success) + exit_code: 1 (failure) ----- stderr ----- - Resolved 1 package in [TIME] - warning: One or more file entries in 'http://[LOCALHOST]/futzed_bz2-0.1.0-py3-none-any.whl' use the 'bzip2' compression method, which is not widely supported. A future version of uv will reject ZIP archives containing entries compressed with this method. Entries must be compressed with the 'stored', 'DEFLATE', or 'zstd' compression methods. - Prepared 1 package in [TIME] - Installed 1 package in [TIME] - + futzed-bz2==0.1.0 (from http://[LOCALHOST]/futzed_bz2-0.1.0-py3-none-any.whl) + × Failed to download `futzed-bz2 @ http://[LOCALHOST]/futzed_bz2-0.1.0-py3-none-any.whl` + ├─▶ Failed to read metadata: `http://[LOCALHOST]/futzed_bz2-0.1.0-py3-none-any.whl` + ╰─▶ Archive contains a file with an unsupported compression method; files must be compressed with 'stored', 'DEFLATE', or 'zstd' " ); } #[test] -fn warn_on_lzma_wheel() { +fn fail_on_lzma_wheel() { let context = uv_test::test_context!("3.14"); let vendor = FindLinksServer::vendor(); @@ -15392,11 +15390,8 @@ fn warn_on_lzma_wheel() { exit_code: 1 (failure) ----- stderr ----- × Failed to download `futzed-lzma @ http://[LOCALHOST]/futzed_lzma-0.1.0-py3-none-any.whl` - ├─▶ Request failed after 3 retries in [TIME] ├─▶ Failed to read metadata: `http://[LOCALHOST]/futzed_lzma-0.1.0-py3-none-any.whl` - ├─▶ Failed to read from zip file - ├─▶ an upstream reader returned an error: stream/file format not recognized - ╰─▶ stream/file format not recognized + ╰─▶ Archive contains a file with an unsupported compression method; files must be compressed with 'stored', 'DEFLATE', or 'zstd' " ); } diff --git a/crates/uv/tests/sync/sync.rs b/crates/uv/tests/sync/sync.rs index 37b5cf50290e5..ccd7c8865ed8f 100644 --- a/crates/uv/tests/sync/sync.rs +++ b/crates/uv/tests/sync/sync.rs @@ -14848,6 +14848,237 @@ async fn sync_zstd_wheel() -> Result<()> { Ok(()) } +/// Sync with an index whose only source distribution has a non-PEP 625-compliant +/// extension (e.g., `.tar.bz2`). The resolver should reject it as incompatible. +#[tokio::test] +async fn sync_non_pep625_sdist() -> Result<()> { + use serde_json::json; + use wiremock::{ + Mock, MockServer, ResponseTemplate, + matchers::{method, path}, + }; + + let context = uv_test::test_context!("3.13"); + let server = MockServer::start().await; + + let sdist_url = format!("{}/files/basic_package-0.1.0.tar.bz2", server.uri()); + + let simple_index = json!({ + "meta": { + "api-version": "1.1" + }, + "name": "basic-package", + "files": [{ + "filename": "basic_package-0.1.0.tar.bz2", + "url": sdist_url, + "hashes": { + "sha256": "0000000000000000000000000000000000000000000000000000000000000000" + } + }] + }); + + Mock::given(method("GET")) + .and(path("/simple/basic-package/")) + .respond_with(ResponseTemplate::new(200).set_body_raw( + simple_index.to_string().into_bytes(), + "application/vnd.pyx.simple.v1+json", + )) + .mount(&server) + .await; + + let pyproject_toml = context.temp_dir.child("pyproject.toml"); + pyproject_toml.write_str(&formatdoc! { r#" + [project] + name = "project" + version = "0.1.0" + requires-python = ">=3.13" + dependencies = ["basic-package"] + + [tool.uv.sources] + basic-package = {{ index = "test-registry" }} + + [[tool.uv.index]] + name = "test-registry" + url = "{}/simple" + "#, + server.uri() + })?; + + uv_snapshot!(context.filters(), context.sync().env_remove(EnvVars::UV_EXCLUDE_NEWER), @" + exit_code: 1 (failure) + ----- stderr ----- + × No solution found when resolving dependencies: + ╰─▶ Because only basic-package==0.1.0 is available and basic-package==0.1.0 has a non-PEP 625-compliant source distribution filename, we can conclude that all versions of basic-package cannot be used. + And because your project depends on basic-package, we can conclude that your project's requirements are unsatisfiable. + + hint: `basic-package` was found on http://[LOCALHOST]/simple, but not at the requested version (all of: + basic-package<0.1.0 + basic-package>0.1.0 + ). A compatible version may be available on a subsequent index (e.g., https://pypi.org/simple). By default, uv will only consider versions that are published on the first index that contains a given package, to avoid dependency confusion attacks. If all indexes are equally trusted, use `--index-strategy unsafe-best-match` to consider all versions from all indexes, regardless of the order in which they were defined. + "); + + Ok(()) +} + +/// Sync with an index that serves both a non-PEP 625-compliant sdist and a +/// compatible wheel. The resolver should use the wheel and skip the sdist +/// without surfacing an incompatibility error. +#[tokio::test] +async fn sync_non_pep625_sdist_with_compatible_wheel() -> Result<()> { + use serde_json::json; + use wiremock::{ + Mock, MockServer, ResponseTemplate, + matchers::{method, path}, + }; + + let context = uv_test::test_context!("3.13"); + let server = MockServer::start().await; + + let wheel_path = context + .temp_dir + .child("basic_package-0.1.0-py3-none-any.whl"); + fs_err::copy( + context + .workspace_root + .join("test/links/basic_package-0.1.0-py3-none-any.whl"), + &wheel_path, + )?; + + let wheel_url = format!( + "{}/files/basic_package-0.1.0-py3-none-any.whl", + server.uri() + ); + let sdist_url = format!("{}/files/basic_package-0.1.0.tar.bz2", server.uri()); + + Mock::given(method("GET")) + .and(path("/files/basic_package-0.1.0-py3-none-any.whl")) + .respond_with(ResponseTemplate::new(200).set_body_bytes(fs_err::read(&wheel_path)?)) + .mount(&server) + .await; + + let simple_index = json!({ + "meta": { + "api-version": "1.1" + }, + "name": "basic-package", + "files": [ + { + "filename": "basic_package-0.1.0.tar.bz2", + "url": sdist_url, + "hashes": { + "sha256": "0000000000000000000000000000000000000000000000000000000000000000" + } + }, + { + "filename": "basic_package-0.1.0-py3-none-any.whl", + "url": wheel_url, + "hashes": { + "sha256": "7b6229db79b5800e4e98a351b5628c1c8a944533a2d428aeeaa7275a30d4ea82" + } + } + ] + }); + + Mock::given(method("GET")) + .and(path("/simple/basic-package/")) + .respond_with(ResponseTemplate::new(200).set_body_raw( + simple_index.to_string().into_bytes(), + "application/vnd.pyx.simple.v1+json", + )) + .mount(&server) + .await; + + let pyproject_toml = context.temp_dir.child("pyproject.toml"); + pyproject_toml.write_str(&formatdoc! { r#" + [project] + name = "project" + version = "0.1.0" + requires-python = ">=3.13" + dependencies = ["basic-package"] + + [tool.uv.sources] + basic-package = {{ index = "test-registry" }} + + [[tool.uv.index]] + name = "test-registry" + url = "{}/simple" + "#, + server.uri() + })?; + + uv_snapshot!(context.filters(), context.sync().env_remove(EnvVars::UV_EXCLUDE_NEWER), @" + exit_code: 0 (success) + ----- stderr ----- + Resolved 2 packages in [TIME] + Prepared 1 package in [TIME] + Installed 1 package in [TIME] + + basic-package==0.1.0 + "); + + Ok(()) +} + +/// A pre-existing lockfile may refer to a non-PEP 625-compliant direct URL sdist (e.g. +/// one that was locked by an older uv). Refreshing it via `uv sync` should hard-error +/// rather than silently install. +#[test] +fn sync_non_pep625_sdist_from_lockfile() -> Result<()> { + let context = uv_test::test_context!("3.12"); + + // Stage the `.tar.bz2` sdist alongside the project. + let archive = context.temp_dir.child("bz2-1.0.0.tar.bz2"); + fs_err::copy( + context.workspace_root.join("test/links/bz2-1.0.0.tar.bz2"), + &archive, + )?; + + let pyproject_toml = context.temp_dir.child("pyproject.toml"); + pyproject_toml.write_str(indoc! {r#" + [project] + name = "project" + version = "0.1.0" + requires-python = ">=3.12" + dependencies = ["bz2"] + + [tool.uv.sources] + bz2 = { path = "bz2-1.0.0.tar.bz2" } + "#})?; + + // Pre-write a lockfile as if a previous uv had resolved the `.tar.bz2` direct URL. + context.temp_dir.child("uv.lock").write_str(indoc! {r#" + version = 1 + requires-python = ">=3.12" + + [options] + exclude-newer = "2024-03-25T00:00:00Z" + + [[package]] + name = "bz2" + version = "1.0.0" + source = { path = "bz2-1.0.0.tar.bz2" } + sdist = { hash = "sha256:f792d237bf8d8f1fc0b0ea16848371cbbb06a6b8a43b0da2f50d30bfba834f7b" } + + [[package]] + name = "project" + version = "0.1.0" + source = { virtual = "." } + dependencies = [ + { name = "bz2" }, + ] + + [package.metadata] + requires-dist = [{ name = "bz2", path = "bz2-1.0.0.tar.bz2" }] + "#})?; + + uv_snapshot!(context.filters(), context.sync(), @" + exit_code: 2 (failure) + ----- stderr ----- + error: Source distribution for `bz2==1.0.0 @ path+bz2-1.0.0.tar.bz2` has a non-PEP 625-compliant filename; only `.tar.gz` and `.zip` archives are accepted + "); + + Ok(()) +} + #[test] #[cfg(not(windows))] fn toggle_workspace_editable() -> Result<()> { diff --git a/docs/concepts/resolution.md b/docs/concepts/resolution.md index 84d6949b8734f..90d120841378e 100644 --- a/docs/concepts/resolution.md +++ b/docs/concepts/resolution.md @@ -869,27 +869,13 @@ exclude-newer-package = { setuptools = "30 days" } [PEP 625](https://peps.python.org/pep-0625/) specifies that packages must distribute source distributions as gzip tarball (`.tar.gz`) archives. Prior to this specification, other archive -formats, which need to be supported for backward compatibility, were also allowed. uv supports -reading and extracting archives in the following formats: - -- gzip tarball (`.tar.gz`, `.tgz`) -- bzip2 tarball (`.tar.bz2`, `.tbz`) -- xz tarball (`.tar.xz`, `.txz`) -- zstd tarball (`.tar.zst`) -- lzip tarball (`.tar.lz`) -- lzma tarball (`.tar.lzma`) -- zip (`.zip`) +formats, which need to be supported for backward compatibility, were also allowed. !!! important - Using source distribution extensions other than `.tar.gz` is strongly - discouraged, as these extensions are not widely or consistently - supported across the Python packaging ecosystem. - -!!! warning "Deprecated" - - Support for source distribution extensions other than `.tar.gz` is - deprecated and will be removed in a future release of uv. + As of 0.12, uv rejects source distributions that do not confirm to + [PEP 625]'s extension requirements with the exception of `.zip` archives, + which are still accepted for backward compatibility. ## Lockfile versioning