Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

30 changes: 29 additions & 1 deletion crates/uv-client/src/flat_index.rs
Original file line number Diff line number Diff line change
Expand Up @@ -299,7 +299,9 @@ impl<'a> FlatIndexClient<'a> {
) -> FlatIndexEntries {
let entries = files
.into_iter()
.filter_map(|file| {
.filter_map(|mut file| {
// Ignore deprecated pyx-specific zstd wheel metadata from an older cache entry.
file.zstd = None;
Some(FlatIndexEntry {
filename: DistFilename::try_from_normalized_filename(&file.filename)?,
file,
Expand Down Expand Up @@ -395,6 +397,32 @@ mod tests {
use fs_err::File;
use std::io::Write;
use tempfile::tempdir;
use uv_distribution_types::Zstd;

/// Round-trip a synthetic cache entry containing deprecated pyx-specific zstd wheel metadata
/// and verify that the metadata is discarded before the file is used.
#[test]
fn cached_files_ignore_deprecated_zstd() -> Result<(), Box<dyn std::error::Error>> {
let url = DisplaySafeUrl::parse("https://example.com/flat/")?;
let mut files = FlatIndexClient::parse_html(
r#"<a href="example-1.0.0-py3-none-any.whl">example-1.0.0-py3-none-any.whl</a>"#,
&url,
)?;
assert_eq!(files.len(), 1);
assert!(files[0].zstd.is_none());
files[0].zstd = Some(Box::new(Zstd {
hashes: HashDigests::empty(),
size: Some(42),
}));

let archived = OwnedArchive::from_unarchived(&files)?;
let files = OwnedArchive::deserialize(&archived);
let entries =
FlatIndexClient::entries_from_files(files, &IndexUrl::parse(url.as_str(), None)?);
assert_eq!(entries.entries.len(), 1);
assert!(entries.entries[0].file.zstd.is_none());
Ok(())
}

#[test]
fn read_from_directory_sorts_distributions() {
Expand Down
35 changes: 28 additions & 7 deletions crates/uv-client/src/registry_client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1331,6 +1331,9 @@ pub struct CachedFile {
filename: Option<Box<SmallString>>,
#[rkyv(with = rkyv::with::Niche)]
yanked: Option<Box<Yanked>>,
/// Deprecated pyx-specific zstd wheel metadata, retained only for compatibility with the
/// Simple API cache layout.
// TODO: Remove this field when the Simple API cache format is next bumped.

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Decided not to remove this key as it would require churning all users' caches, even if they weren't pyx users. If cache churn is acceptable, this PR can be simplified a bit.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Avoiding the churn seems fine to me!

#[rkyv(with = rkyv::with::Niche)]
zstd: Option<Box<Zstd>>,
dist_info_metadata: bool,
Expand Down Expand Up @@ -1377,7 +1380,7 @@ impl From<File> for CachedFile {
has_upload_time,
url: file.url,
yanked: file.yanked.filter(|yanked| yanked.is_yanked()),
zstd: file.zstd,
zstd: None,
}
}
}
Expand All @@ -1394,7 +1397,7 @@ impl From<CachedFile> for File {
upload_time_utc_ms: file.has_upload_time.then_some(file.upload_time_utc_ms),
url: file.url,
yanked: file.yanked,
zstd: file.zstd,
zstd: None,
}
}
}
Expand Down Expand Up @@ -1754,7 +1757,7 @@ mod tests {
use tokio::sync::Semaphore;
use url::Url;
use uv_normalize::PackageName;
use uv_pypi_types::PypiSimpleDetail;
use uv_pypi_types::{HashDigests, PypiSimpleDetail};
use uv_redacted::DisplaySafeUrl;
use uv_torch::{TorchBackend, TorchStrategy};

Expand All @@ -1764,8 +1767,8 @@ mod tests {
};
use uv_cache::Cache;
use uv_distribution_types::{
FileLocation, Index, IndexCapabilities, IndexFormat, IndexLocations, IndexMetadataRef,
IndexUrl, ToUrlError,
File, FileLocation, Index, IndexCapabilities, IndexFormat, IndexLocations,
IndexMetadataRef, IndexUrl, ToUrlError, Zstd,
};
use uv_small_str::SmallString;
use wiremock::matchers::{basic_auth, method, path_regex};
Expand Down Expand Up @@ -2128,20 +2131,38 @@ mod tests {
let package_name = PackageName::from_str("example-1")?;
let data: PypiSimpleDetail = serde_json::from_str(response)?;
let base = DisplaySafeUrl::parse("https://pypi.org/simple/example-1/")?;
let simple_metadata = SimpleDetailMetadata::from_pypi_files(
let mut simple_metadata = SimpleDetailMetadata::from_pypi_files(
data.files,
&package_name,
data.project_status,
&base,
);
let cached_wheel = &mut simple_metadata.versions[0].files.wheels[0];
assert!(cached_wheel.zstd.is_none());
// An entry written by an older uv may still contain pyx-specific zstd wheel metadata.
cached_wheel.zstd = Some(Box::new(Zstd {
hashes: HashDigests::empty(),
size: Some(42),
}));
let archived = super::OwnedArchive::from_unarchived(&simple_metadata)?;
let simple_metadata = super::OwnedArchive::deserialize(&archived);

let filenames: Vec<_> = simple_metadata
.versions
.into_iter()
.flat_map(|datum| datum.files.all(&package_name))
.map(|(filename, _)| filename.to_string())
.map(|(filename, mut file)| {
assert!(file.zstd.is_none());
// New cache entries must not preserve pyx-specific zstd wheel metadata either.
file.zstd = Some(Box::new(Zstd {
hashes: HashDigests::empty(),
size: Some(42),
}));
let cached = super::CachedFile::from(file);
assert!(cached.zstd.is_none());
assert!(File::from(cached).zstd.is_none());
filename.to_string()
})
.collect();
assert_eq!(
filenames,
Expand Down
6 changes: 6 additions & 0 deletions crates/uv-distribution-types/src/file.rs
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,9 @@ pub struct File {
pub upload_time_utc_ms: Option<i64>,
pub url: FileLocation,
pub yanked: Option<Box<Yanked>>,
/// Deprecated pyx-specific zstd wheel metadata, retained only for compatibility with the
/// flat-index cache layout.
// TODO: Remove this field when the flat-index cache format is next bumped.

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Decided not to remove this key as it would require churning all users' caches, even if they weren't pyx users. If cache churn is acceptable, this PR can be simplified a bit.

pub zstd: Option<Box<Zstd>>,
}

Expand Down Expand Up @@ -253,6 +256,9 @@ pub enum ToUrlError {
},
}

/// Deprecated pyx-specific zstd wheel metadata, retained only for compatibility with existing
/// cache layouts.
// TODO: Remove this type once the Simple API and flat-index cache formats are both bumped.
#[derive(Debug, Clone, PartialEq, Eq, Hash, rkyv::Archive, rkyv::Deserialize, rkyv::Serialize)]
pub struct Zstd {
pub hashes: HashDigests,
Expand Down
Loading
Loading