Skip to content

Allow registries to pre-provide core metadata #15644

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
charliermarsh merged 1 commit into from
Sep 3, 2025
Merged

Allow registries to pre-provide core metadata #15644

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion crates/uv-cache/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1002,7 +1002,7 @@ impl CacheBucket {
Self::Interpreter => "interpreter-v4",
// Note that when bumping this, you'll also need to bump it
// in `crates/uv/tests/it/cache_clean.rs`.
Self::Simple => "simple-v16",
Self::Simple => "simple-v17",
// Note that when bumping this, you'll also need to bump it
// in `crates/uv/tests/it/cache_prune.rs`.
Self::Wheels => "wheels-v5",
Expand Down
11 changes: 11 additions & 0 deletions crates/uv-client/src/error.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,11 @@ impl Error {
ErrorKind::BadHtml { source: err, url }.into()
}

/// Create a new error from a `MessagePack` parsing error.
pub(crate) fn from_msgpack_err(err: rmp_serde::decode::Error, url: DisplaySafeUrl) -> Self {
ErrorKind::BadMessagePack { source: err, url }.into()
}

/// Returns `true` if this error corresponds to an offline error.
pub(crate) fn is_offline(&self) -> bool {
matches!(&*self.kind, ErrorKind::Offline(_))
Expand Down Expand Up @@ -251,6 +256,12 @@ pub enum ErrorKind {
url: DisplaySafeUrl,
},

#[error("Received some unexpected MessagePack from {}", url)]
BadMessagePack {
source: rmp_serde::decode::Error,
url: DisplaySafeUrl,
},

#[error("Failed to read zip with range requests: `{0}`")]
AsyncHttpRangeReader(DisplaySafeUrl, #[source] AsyncHttpRangeReaderError),

Expand Down
2 changes: 1 addition & 1 deletion crates/uv-client/src/flat_index.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,7 @@ impl<'a> FlatIndexClient<'a> {
let unarchived: Vec<File> = files
.into_iter()
.filter_map(|file| {
match File::try_from(file, &base) {
match File::try_from_pypi(file, &base) {
Ok(file) => Some(file),
Err(err) => {
// Ignore files with unparsable version specifiers.
Expand Down
180 changes: 160 additions & 20 deletions crates/uv-client/src/registry_client.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ use tokio::sync::{Mutex, Semaphore};
use tracing::{Instrument, debug, info_span, instrument, trace, warn};
use url::Url;

use uv_auth::Indexes;
use uv_auth::{Indexes, PyxTokenStore};
use uv_cache::{Cache, CacheBucket, CacheEntry, WheelCache};
use uv_configuration::IndexStrategy;
use uv_configuration::KeyringProviderType;
Expand All @@ -29,7 +29,7 @@ use uv_normalize::PackageName;
use uv_pep440::Version;
use uv_pep508::MarkerEnvironment;
use uv_platform_tags::Platform;
use uv_pypi_types::{PypiSimpleDetail, ResolutionMetadata};
use uv_pypi_types::{PypiSimpleDetail, PyxSimpleDetail, ResolutionMetadata};
use uv_redacted::DisplaySafeUrl;
use uv_small_str::SmallString;
use uv_torch::TorchStrategy;
Expand Down Expand Up @@ -173,6 +173,7 @@ impl<'a> RegistryClientBuilder<'a> {
client,
timeout,
flat_indexes: Arc::default(),
pyx_token_store: PyxTokenStore::from_settings().ok(),
}
}

Expand Down Expand Up @@ -202,6 +203,7 @@ impl<'a> RegistryClientBuilder<'a> {
client,
timeout,
flat_indexes: Arc::default(),
pyx_token_store: PyxTokenStore::from_settings().ok(),
}
}
}
Expand All @@ -225,6 +227,9 @@ pub struct RegistryClient {
timeout: Duration,
/// The flat index entries for each `--find-links`-style index URL.
flat_indexes: Arc<Mutex<FlatIndexCache>>,
/// The pyx token store to use for persistent credentials.
// TODO(charlie): The token store is only needed for `is_known_url`; can we avoid storing it here?
pyx_token_store: Option<PyxTokenStore>,
Comment thread
charliermarsh marked this conversation as resolved.
}

/// The format of the package metadata returned by querying an index.
Expand Down Expand Up @@ -512,7 +517,7 @@ impl RegistryClient {
let result = if matches!(index, IndexUrl::Path(_)) {
self.fetch_local_index(package_name, &url).await
} else {
self.fetch_remote_index(package_name, &url, &cache_entry, cache_control)
self.fetch_remote_index(package_name, &url, index, &cache_entry, cache_control)
.await
};

Expand Down Expand Up @@ -553,14 +558,27 @@ impl RegistryClient {
&self,
package_name: &PackageName,
url: &DisplaySafeUrl,
index: &IndexUrl,
cache_entry: &CacheEntry,
cache_control: CacheControl<'_>,
) -> Result<OwnedArchive<SimpleMetadata>, Error> {
// In theory, we should be able to pass `MediaType::all()` to all registries, and as
// unsupported media types should be ignored by the server. For now, we implement this
// defensively to avoid issues with misconfigured servers.
let accept = if self
.pyx_token_store
.as_ref()
.is_some_and(|token_store| token_store.is_known_url(index.url()))
{
MediaType::all()
} else {
MediaType::pypi()
};
Comment thread
charliermarsh marked this conversation as resolved.
let simple_request = self
.uncached_client(url)
.get(Url::from(url.clone()))
.header("Accept-Encoding", "gzip, deflate, zstd")
.header("Accept", MediaType::accepts())
.header("Accept", accept)
.build()
.map_err(|err| ErrorKind::from_reqwest(url.clone(), err))?;
let parse_simple_response = |response: Response| {
Expand All @@ -585,17 +603,48 @@ impl RegistryClient {
})?;

let unarchived = match media_type {
MediaType::Json => {
MediaType::PyxV1Msgpack => {
let bytes = response
.bytes()
.await
.map_err(|err| ErrorKind::from_reqwest(url.clone(), err))?;
let data: PyxSimpleDetail = rmp_serde::from_slice(bytes.as_ref())
.map_err(|err| Error::from_msgpack_err(err, url.clone()))?;

SimpleMetadata::from_pyx_files(
data.files,
data.core_metadata,
package_name,
&url,
)
}
MediaType::PyxV1Json => {
let bytes = response
.bytes()
.await
.map_err(|err| ErrorKind::from_reqwest(url.clone(), err))?;
let data: PyxSimpleDetail = serde_json::from_slice(bytes.as_ref())
.map_err(|err| Error::from_json_err(err, url.clone()))?;

SimpleMetadata::from_pyx_files(
data.files,
data.core_metadata,
package_name,
&url,
)
}
MediaType::PypiV1Json => {
let bytes = response
.bytes()
.await
.map_err(|err| ErrorKind::from_reqwest(url.clone(), err))?;

let data: PypiSimpleDetail = serde_json::from_slice(bytes.as_ref())
.map_err(|err| Error::from_json_err(err, url.clone()))?;

SimpleMetadata::from_pypi_files(data.files, package_name, &url)
}
MediaType::Html => {
MediaType::PypiV1Html | MediaType::TextHtml => {
let text = response
.text()
.await
Expand Down Expand Up @@ -1089,6 +1138,7 @@ pub struct SimpleMetadata(Vec<SimpleMetadatum>);
pub struct SimpleMetadatum {
pub version: Version,
pub files: VersionFiles,
pub metadata: Option<ResolutionMetadata>,
}

impl SimpleMetadata {
Expand All @@ -1101,7 +1151,7 @@ impl SimpleMetadata {
package_name: &PackageName,
base: &Url,
) -> Self {
let mut map: BTreeMap<Version, VersionFiles> = BTreeMap::default();
let mut version_map: BTreeMap<Version, VersionFiles> = BTreeMap::default();

// Convert to a reference-counted string.
let base = SmallString::from(base.as_str());
Expand All @@ -1113,19 +1163,65 @@ impl SimpleMetadata {
warn!("Skipping file for {package_name}: {}", file.filename);
continue;
};
let version = match filename {
DistFilename::SourceDistFilename(ref inner) => &inner.version,
DistFilename::WheelFilename(ref inner) => &inner.version,
let file = match File::try_from_pypi(file, &base) {
Ok(file) => file,
Err(err) => {
// Ignore files with unparsable version specifiers.
warn!("Skipping file for {package_name}: {err}");
continue;
}
};
let file = match File::try_from(file, &base) {
match version_map.entry(filename.version().clone()) {
std::collections::btree_map::Entry::Occupied(mut entry) => {
entry.get_mut().push(filename, file);
}
std::collections::btree_map::Entry::Vacant(entry) => {
let mut files = VersionFiles::default();
files.push(filename, file);
entry.insert(files);
}
}
}

Self(
version_map
.into_iter()
.map(|(version, files)| SimpleMetadatum {
version,
files,
metadata: None,
})
.collect(),
)
}

fn from_pyx_files(
files: Vec<uv_pypi_types::PyxFile>,
mut core_metadata: FxHashMap<Version, uv_pypi_types::CoreMetadatum>,
package_name: &PackageName,
base: &Url,
) -> Self {
let mut version_map: BTreeMap<Version, VersionFiles> = BTreeMap::default();

// Convert to a reference-counted string.
let base = SmallString::from(base.as_str());

// Group the distributions by version and kind
for file in files {
let file = match File::try_from_pyx(file, &base) {
Ok(file) => file,
Err(err) => {
// Ignore files with unparsable version specifiers.
warn!("Skipping file for {package_name}: {err}");
continue;
}
};
match map.entry(version.clone()) {
let Some(filename) = DistFilename::try_from_filename(&file.filename, package_name)
else {
warn!("Skipping file for {package_name}: {}", file.filename);
Comment thread
charliermarsh marked this conversation as resolved.
continue;
};
match version_map.entry(filename.version().clone()) {
std::collections::btree_map::Entry::Occupied(mut entry) => {
entry.get_mut().push(filename, file);
}
Expand All @@ -1136,9 +1232,28 @@ impl SimpleMetadata {
}
}
}

Self(
map.into_iter()
.map(|(version, files)| SimpleMetadatum { version, files })
version_map
.into_iter()
.map(|(version, files)| {
let metadata =
core_metadata
.remove(&version)
.map(|metadata| ResolutionMetadata {
name: package_name.clone(),
version: version.clone(),
requires_dist: metadata.requires_dist,
requires_python: metadata.requires_python,
provides_extras: metadata.provides_extras,
dynamic: false,
});
SimpleMetadatum {
version,
files,
metadata,
}
})
.collect(),
)
}
Expand Down Expand Up @@ -1177,26 +1292,51 @@ impl ArchivedSimpleMetadata {

#[derive(Debug)]
enum MediaType {
Json,
Html,
PyxV1Msgpack,
PyxV1Json,
PypiV1Json,
PypiV1Html,
TextHtml,
}

impl MediaType {
/// Parse a media type from a string, returning `None` if the media type is not supported.
fn from_str(s: &str) -> Option<Self> {
match s {
"application/vnd.pypi.simple.v1+json" => Some(Self::Json),
"application/vnd.pypi.simple.v1+html" | "text/html" => Some(Self::Html),
"application/vnd.pyx.simple.v1+msgpack" => Some(Self::PyxV1Msgpack),
"application/vnd.pyx.simple.v1+json" => Some(Self::PyxV1Json),
"application/vnd.pypi.simple.v1+json" => Some(Self::PypiV1Json),
"application/vnd.pypi.simple.v1+html" => Some(Self::PypiV1Html),
"text/html" => Some(Self::TextHtml),
_ => None,
}
}

/// Return the `Accept` header value for all supported media types.
/// Return the `Accept` header value for all PyPI media types.
#[inline]
const fn accepts() -> &'static str {
const fn pypi() -> &'static str {
// See: https://peps.python.org/pep-0691/#version-format-selection
"application/vnd.pypi.simple.v1+json, application/vnd.pypi.simple.v1+html;q=0.2, text/html;q=0.01"
}

/// Return the `Accept` header value for all supported media types.
#[inline]
const fn all() -> &'static str {
// See: https://peps.python.org/pep-0691/#version-format-selection
"application/vnd.pyx.simple.v1+msgpack, application/vnd.pyx.simple.v1+json;q=0.9, application/vnd.pypi.simple.v1+json;q=0.8, application/vnd.pypi.simple.v1+html;q=0.2, text/html;q=0.01"
}
}

impl std::fmt::Display for MediaType {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::PyxV1Msgpack => write!(f, "application/vnd.pyx.simple.v1+msgpack"),
Self::PyxV1Json => write!(f, "application/vnd.pyx.simple.v1+json"),
Self::PypiV1Json => write!(f, "application/vnd.pypi.simple.v1+json"),
Self::PypiV1Html => write!(f, "application/vnd.pypi.simple.v1+html"),
Self::TextHtml => write!(f, "text/html"),
}
}
}

#[derive(Debug, Copy, Clone, Eq, PartialEq, Default)]
Expand Down
Loading
Loading