Skip to content

Commit

Permalink
Generate hashes for --find-links entries
Browse files Browse the repository at this point in the history
  • Loading branch information
charliermarsh committed Jul 29, 2024
1 parent 9af0ae2 commit eb5aca4
Show file tree
Hide file tree
Showing 4 changed files with 147 additions and 86 deletions.
10 changes: 10 additions & 0 deletions crates/distribution-types/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -478,6 +478,16 @@ impl BuiltDist {
matches!(self, Self::Path(_))
}

/// Returns `true` if the distribution comes from a registry that implements the Simple API,
/// like PyPI.
pub fn is_simple_registry(&self) -> bool {
let BuiltDist::Registry(wheel) = self else {
return false;
};
let wheel = wheel.best_wheel();
matches!(wheel.index, IndexUrl::Pypi(_) | IndexUrl::Url(_))
}

/// Returns the [`IndexUrl`], if the distribution is from a registry.
pub fn index(&self) -> Option<&IndexUrl> {
match self {
Expand Down
18 changes: 10 additions & 8 deletions crates/uv-distribution/src/distribution_database.rs
Original file line number Diff line number Diff line change
Expand Up @@ -428,14 +428,16 @@ impl<'a, Context: BuildContext> DistributionDatabase<'a, Context> {
// For hash _validation_, callers are expected to enforce the policy when retrieving the
// wheel.
// TODO(charlie): Request the hashes via a separate method, to reduce the coupling in this API.
if hashes.is_generate() && matches!(dist, BuiltDist::DirectUrl(_) | BuiltDist::Path(_)) {
let wheel = self.get_wheel(dist, hashes).await?;
let metadata = wheel.metadata()?;
let hashes = wheel.hashes;
return Ok(ArchiveMetadata {
metadata: Metadata::from_metadata23(metadata),
hashes,
});
if hashes.is_generate() {
if dist.file().map_or(true, |file| file.hashes.is_empty()) {
let wheel = self.get_wheel(dist, hashes).await?;
let metadata = wheel.metadata()?;
let hashes = wheel.hashes;
return Ok(ArchiveMetadata {
metadata: Metadata::from_metadata23(metadata),
hashes,
});
}
}

let result = self
Expand Down
146 changes: 68 additions & 78 deletions crates/uv-resolver/src/resolution/graph.rs
Original file line number Diff line number Diff line change
@@ -1,24 +1,24 @@
use indexmap::IndexSet;
use petgraph::{
graph::{Graph, NodeIndex},
Directed,
};
use rustc_hash::{FxBuildHasher, FxHashMap, FxHashSet};

use distribution_types::{
Dist, DistributionMetadata, Name, ResolutionDiagnostic, ResolvedDist, VersionId,
VersionOrUrlRef,
};
use indexmap::IndexSet;
use pep440_rs::{Version, VersionSpecifier};
use pep508_rs::{MarkerEnvironment, MarkerTree};
use petgraph::{
graph::{Graph, NodeIndex},
Directed,
};
use pypi_types::{HashDigest, ParsedUrlError, Requirement, VerbatimParsedUrl, Yanked};
use rustc_hash::{FxBuildHasher, FxHashMap, FxHashSet};
use uv_configuration::{Constraints, Overrides};
use uv_distribution::Metadata;
use uv_git::GitResolver;
use uv_normalize::{ExtraName, GroupName, PackageName};

use crate::pins::FilePins;
use crate::preferences::Preferences;
use crate::pubgrub::PubGrubDistribution;
use crate::python_requirement::PythonTarget;
use crate::redirect::url_to_precise;
use crate::resolution::AnnotatedDist;
Expand Down Expand Up @@ -274,44 +274,19 @@ impl ResolutionGraph {
// Create the distribution.
let dist = Dist::from_url(name.clone(), url_to_precise(url.clone(), git))?;

// Extract the hashes, preserving those that were already present in the
// lockfile if necessary.
let hashes = if let Some(digests) = preferences
.match_hashes(name, version)
.filter(|digests| !digests.is_empty())
{
digests.to_vec()
} else if let Some(metadata_response) = index.distributions().get(&dist.version_id()) {
if let MetadataResponse::Found(ref archive) = *metadata_response {
let mut digests = archive.hashes.clone();
digests.sort_unstable();
digests
} else {
vec![]
}
} else {
vec![]
};
let version_id = VersionId::from_url(&url.verbatim);

// Extract the hashes.
let hashes = Self::get_hashes(&version_id, name, version, preferences, index);

// Extract the metadata.
let metadata = {
let dist = PubGrubDistribution::from_url(name, url);

let response = index
.distributions()
.get(&dist.version_id())
.unwrap_or_else(|| {
panic!(
"Every package should have metadata: {:?}",
dist.version_id()
)
});
let response = index.distributions().get(&version_id).unwrap_or_else(|| {
panic!("Every package should have metadata: {version_id:?}")
});

let MetadataResponse::Found(archive) = &*response else {
panic!(
"Every package should have metadata: {:?}",
dist.version_id()
)
panic!("Every package should have metadata: {version_id:?}")
};

archive.metadata.clone()
Expand All @@ -324,6 +299,8 @@ impl ResolutionGraph {
.expect("Every package should be pinned")
.clone();

let version_id = dist.version_id();

// Track yanks for any registry distributions.
match dist.yanked() {
None | Some(Yanked::Bool(false)) => {}
Expand All @@ -341,49 +318,17 @@ impl ResolutionGraph {
}
}

// Extract the hashes, preserving those that were already present in the
// lockfile if necessary.
let hashes = if let Some(digests) = preferences
.match_hashes(name, version)
.filter(|digests| !digests.is_empty())
{
digests.to_vec()
} else if let Some(versions_response) = index.packages().get(name) {
if let VersionsResponse::Found(ref version_maps) = *versions_response {
version_maps
.iter()
.find_map(|version_map| version_map.hashes(version))
.map(|mut digests| {
digests.sort_unstable();
digests
})
.unwrap_or_default()
} else {
vec![]
}
} else {
vec![]
};
// Extract the hashes.
let hashes = Self::get_hashes(&version_id, name, version, preferences, index);

// Extract the metadata.
let metadata = {
let dist = PubGrubDistribution::from_registry(name, version);

let response = index
.distributions()
.get(&dist.version_id())
.unwrap_or_else(|| {
panic!(
"Every package should have metadata: {:?}",
dist.version_id()
)
});
let response = index.distributions().get(&version_id).unwrap_or_else(|| {
panic!("Every package should have metadata: {version_id:?}")
});

let MetadataResponse::Found(archive) = &*response else {
panic!(
"Every package should have metadata: {:?}",
dist.version_id()
)
panic!("Every package should have metadata: {version_id:?}")
};

archive.metadata.clone()
Expand All @@ -393,6 +338,51 @@ impl ResolutionGraph {
})
}

/// Identify the hashes for the [`VersionId`], preserving any hashes that were provided by the
/// lockfile.
fn get_hashes(
version_id: &VersionId,
name: &PackageName,
version: &Version,
preferences: &Preferences,
index: &InMemoryIndex,
) -> Vec<HashDigest> {
if let Some(digests) = preferences.match_hashes(name, version) {
if !digests.is_empty() {
return digests.to_vec();
}
}

if let Some(versions_response) = index.packages().get(name) {
if let VersionsResponse::Found(ref version_maps) = *versions_response {
if let Some(digests) = version_maps
.iter()
.find_map(|version_map| version_map.hashes(version))
.map(|mut digests| {
digests.sort_unstable();
digests
})
{
if !digests.is_empty() {
return digests;
}
}
}
}

if let Some(metadata_response) = index.distributions().get(version_id) {
if let MetadataResponse::Found(ref archive) = *metadata_response {
let mut digests = archive.hashes.clone();
digests.sort_unstable();
if !digests.is_empty() {
return digests;
}
}
}

vec![]
}

/// Returns an iterator over the distinct packages in the graph.
fn dists(&self) -> impl Iterator<Item = &AnnotatedDist> {
self.petgraph
Expand Down
59 changes: 59 additions & 0 deletions crates/uv/tests/pip_compile.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4409,6 +4409,65 @@ fn generate_hashes_editable() -> Result<()> {
Ok(())
}

/// Include hashes from a `--find-links` index in the generated output.
#[test]
fn generate_hashes_find_links_directory() -> Result<()> {
let context = TestContext::new("3.12");
let requirements_in = context.temp_dir.child("requirements.in");
requirements_in.write_str("tqdm")?;

uv_snapshot!(context.pip_compile()
.arg("requirements.in")
.arg("--generate-hashes")
.arg("--find-links")
.arg(context.workspace_root.join("scripts").join("links")), @r###"
success: true
exit_code: 0
----- stdout -----
# This file was autogenerated by uv via the following command:
# uv pip compile --cache-dir [CACHE_DIR] requirements.in --generate-hashes
tqdm==1000.0.0 \
--hash=sha256:a34996d4bd5abb2336e14ff0a2d22b92cfd0f0ed344e6883041ce01953276a13
# via -r requirements.in
----- stderr -----
Resolved 1 package in [TIME]
"###
);

Ok(())
}

/// Include hashes from a `--find-links` index in the generated output.
#[test]
fn generate_hashes_find_links_url() -> Result<()> {
let context = TestContext::new("3.12");
let requirements_in = context.temp_dir.child("requirements.in");
requirements_in.write_str("tqdm")?;

uv_snapshot!(context.pip_compile()
.arg("requirements.in")
.arg("--generate-hashes")
.arg("--no-index")
.arg("--find-links")
.arg("https://download.pytorch.org/whl/torch_stable.html"), @r###"
success: true
exit_code: 0
----- stdout -----
# This file was autogenerated by uv via the following command:
# uv pip compile --cache-dir [CACHE_DIR] requirements.in --generate-hashes --no-index
tqdm==4.64.1 \
--hash=sha256:6fee160d6ffcd1b1c68c65f14c829c22832bc401726335ce92c52d395944a6a1
# via -r requirements.in
----- stderr -----
Resolved 1 package in [TIME]
"###
);

Ok(())
}

/// Compile using `--find-links` with a local directory.
#[test]
fn find_links_directory() -> Result<()> {
Expand Down

0 comments on commit eb5aca4

Please sign in to comment.