Skip to content

Commit

Permalink
Implement a --verify-hashes hash-checking mode
Browse files Browse the repository at this point in the history
  • Loading branch information
charliermarsh committed Jul 17, 2024
1 parent e271d1f commit 088b164
Show file tree
Hide file tree
Showing 14 changed files with 478 additions and 81 deletions.
40 changes: 38 additions & 2 deletions crates/uv-cli/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -803,13 +803,33 @@ pub struct PipSyncArgs {
/// - Editable installs are not supported.
/// - Local dependencies are not supported, unless they point to a specific wheel (`.whl`) or
/// source archive (`.zip`, `.tar.gz`), as opposed to a directory.
#[arg(long, env = "UV_REQUIRE_HASHES",
value_parser = clap::builder::BoolishValueParser::new(), overrides_with("no_require_hashes"))]
#[arg(
long,
env = "UV_REQUIRE_HASHES",
value_parser = clap::builder::BoolishValueParser::new(),
overrides_with("no_require_hashes"),
)]
pub require_hashes: bool,

#[arg(long, overrides_with("require_hashes"), hide = true)]
pub no_require_hashes: bool,

/// Validate any hashes provided in the requirements file.
///
/// Unlike `--require-hashes`, `--verify-hashes` does not require that all requirements have
/// hashes; instead, it will limit itself to verifying the hashes of those requirements that do
/// include them.
#[arg(
long,
env = "UV_VERIFY_HASHES",
value_parser = clap::builder::BoolishValueParser::new(),
overrides_with("no_verify_hashes"),
)]
pub verify_hashes: bool,

#[arg(long, overrides_with("verify_hashes"), hide = true)]
pub no_verify_hashes: bool,

/// The Python interpreter into which packages should be installed.
///
/// By default, uv installs into the virtual environment in the current working directory or
Expand Down Expand Up @@ -1084,6 +1104,22 @@ pub struct PipInstallArgs {
#[arg(long, overrides_with("require_hashes"), hide = true)]
pub no_require_hashes: bool,

/// Validate any hashes provided in the requirements file.
///
/// Unlike `--require-hashes`, `--verify-hashes` does not require that all requirements have
/// hashes; instead, it will limit itself to verifying the hashes of those requirements that do
/// include them.
#[arg(
long,
env = "UV_VERIFY_HASHES",
value_parser = clap::builder::BoolishValueParser::new(),
overrides_with("no_verify_hashes"),
)]
pub verify_hashes: bool,

#[arg(long, overrides_with("verify_hashes"), hide = true)]
pub no_verify_hashes: bool,

/// The Python interpreter into which packages should be installed.
///
/// By default, uv installs into the virtual environment in the current working directory or
Expand Down
35 changes: 35 additions & 0 deletions crates/uv-configuration/src/hash.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
#[derive(Debug, Copy, Clone)]
pub enum HashCheckingMode {
/// Hashes should be validated against a pre-defined list of hashes. Every requirement must
/// itself be hashable (e.g., Git dependencies are forbidden) _and_ have a hash in the lockfile.
Require,
/// Hashes should be validated, if present, but ignored if absent.
Verify,
}

impl HashCheckingMode {
/// Return the [`HashCheckingMode`] from the command-line arguments, if any.
pub fn from_args(require_hashes: bool, verify_hashes: bool) -> Option<Self> {
if require_hashes {
Some(Self::Require)
} else if verify_hashes {
Some(Self::Verify)
} else {
None
}
}

/// Returns `true` if the hash checking mode is `Require`.
pub fn is_require(&self) -> bool {
matches!(self, Self::Require)
}
}

impl std::fmt::Display for HashCheckingMode {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Require => write!(f, "--require-hashes"),
Self::Verify => write!(f, "--verify-hashes"),
}
}
}
2 changes: 2 additions & 0 deletions crates/uv-configuration/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ pub use concurrency::*;
pub use config_settings::*;
pub use constraints::*;
pub use extras::*;
pub use hash::*;
pub use name_specifiers::*;
pub use overrides::*;
pub use package_options::*;
Expand All @@ -16,6 +17,7 @@ mod concurrency;
mod config_settings;
mod constraints;
mod extras;
mod hash;
mod name_specifiers;
mod overrides;
mod package_options;
Expand Down
3 changes: 2 additions & 1 deletion crates/uv-requirements/src/source_tree.rs
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,8 @@ impl<'a, Context: BuildContext> SourceTreeResolver<'a, Context> {
let hashes = match self.hasher {
HashStrategy::None => HashPolicy::None,
HashStrategy::Generate => HashPolicy::Generate,
HashStrategy::Validate { .. } => {
HashStrategy::Verify(_) => HashPolicy::Generate,
HashStrategy::Require(_) => {
return Err(anyhow::anyhow!(
"Hash-checking is not supported for local directories: {}",
path.user_display()
Expand Down
13 changes: 13 additions & 0 deletions crates/uv-settings/src/settings.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1018,6 +1018,19 @@ pub struct PipOptions {
"#
)]
pub require_hashes: Option<bool>,
/// Validate any hashes provided in the requirements file.
///
/// Unlike `--require-hashes`, `--verify-hashes` does not require that all requirements have
/// hashes; instead, it will limit itself to verifying the hashes of those requirements that do
/// include them.
#[option(
default = "false",
value_type = "bool",
example = r#"
verify-hashes = true
"#
)]
pub verify_hashes: Option<bool>,
/// Allow package upgrades, ignoring pinned versions in any existing output file.
#[option(
default = "false",
Expand Down
116 changes: 78 additions & 38 deletions crates/uv-types/src/hash.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ use url::Url;
use distribution_types::{DistributionMetadata, HashPolicy, PackageId, UnresolvedRequirement};
use pep508_rs::MarkerEnvironment;
use pypi_types::{HashDigest, HashError, Requirement, RequirementSource};
use uv_configuration::HashCheckingMode;
use uv_normalize::PackageName;

#[derive(Debug, Default, Clone)]
Expand All @@ -15,9 +16,14 @@ pub enum HashStrategy {
None,
/// Hashes should be generated (specifically, a SHA-256 hash), but not validated.
Generate,
/// Hashes should be validated against a pre-defined list of hashes. If necessary, hashes should
/// be generated so as to ensure that the archive is valid.
Validate(FxHashMap<PackageId, Vec<HashDigest>>),
/// Hashes should be validated, if present, but ignored if absent.
///
/// If necessary, hashes should be generated to ensure that the archive is valid.
Verify(FxHashMap<PackageId, Vec<HashDigest>>),
/// Hashes should be validated against a pre-defined list of hashes.
///
/// If necessary, hashes should be generated to ensure that the archive is valid.
Require(FxHashMap<PackageId, Vec<HashDigest>>),
}

impl HashStrategy {
Expand All @@ -26,7 +32,14 @@ impl HashStrategy {
match self {
Self::None => HashPolicy::None,
Self::Generate => HashPolicy::Generate,
Self::Validate(hashes) => HashPolicy::Validate(
Self::Verify(hashes) => {
if let Some(hashes) = hashes.get(&distribution.package_id()) {
HashPolicy::Validate(hashes.as_slice())
} else {
HashPolicy::None
}
}
Self::Require(hashes) => HashPolicy::Validate(
hashes
.get(&distribution.package_id())
.map(Vec::as_slice)
Expand All @@ -40,7 +53,14 @@ impl HashStrategy {
match self {
Self::None => HashPolicy::None,
Self::Generate => HashPolicy::Generate,
Self::Validate(hashes) => HashPolicy::Validate(
Self::Verify(hashes) => {
if let Some(hashes) = hashes.get(&PackageId::from_registry(name.clone())) {
HashPolicy::Validate(hashes.as_slice())
} else {
HashPolicy::None
}
}
Self::Require(hashes) => HashPolicy::Validate(
hashes
.get(&PackageId::from_registry(name.clone()))
.map(Vec::as_slice)
Expand All @@ -54,7 +74,14 @@ impl HashStrategy {
match self {
Self::None => HashPolicy::None,
Self::Generate => HashPolicy::Generate,
Self::Validate(hashes) => HashPolicy::Validate(
Self::Verify(hashes) => {
if let Some(hashes) = hashes.get(&PackageId::from_url(url)) {
HashPolicy::Validate(hashes.as_slice())
} else {
HashPolicy::None
}
}
Self::Require(hashes) => HashPolicy::Validate(
hashes
.get(&PackageId::from_url(url))
.map(Vec::as_slice)
Expand All @@ -68,7 +95,8 @@ impl HashStrategy {
match self {
Self::None => true,
Self::Generate => true,
Self::Validate(hashes) => hashes.contains_key(&PackageId::from_registry(name.clone())),
Self::Verify(_) => true,
Self::Require(hashes) => hashes.contains_key(&PackageId::from_registry(name.clone())),
}
}

Expand All @@ -77,7 +105,8 @@ impl HashStrategy {
match self {
Self::None => true,
Self::Generate => true,
Self::Validate(hashes) => hashes.contains_key(&PackageId::from_url(url)),
Self::Verify(_) => true,
Self::Require(hashes) => hashes.contains_key(&PackageId::from_url(url)),
}
}

Expand All @@ -90,6 +119,7 @@ impl HashStrategy {
pub fn from_requirements<'a>(
requirements: impl Iterator<Item = (&'a UnresolvedRequirement, &'a [String])>,
markers: Option<&MarkerEnvironment>,
mode: HashCheckingMode,
) -> Result<Self, HashStrategyError> {
let mut hashes = FxHashMap::<PackageId, Vec<HashDigest>>::default();

Expand All @@ -103,17 +133,25 @@ impl HashStrategy {
// Every requirement must be either a pinned version or a direct URL.
let id = match &requirement {
UnresolvedRequirement::Named(requirement) => {
uv_requirement_to_package_id(requirement)?
Self::pin(requirement).ok_or_else(|| {
HashStrategyError::UnpinnedRequirement(requirement.to_string(), mode)
})?
}
UnresolvedRequirement::Unnamed(requirement) => {
// Direct URLs are always allowed.
PackageId::from_url(&requirement.url.verbatim)
}
};

// Every requirement must include a hash.
if digests.is_empty() {
return Err(HashStrategyError::MissingHashes(requirement.to_string()));
// Under `--require-hashes`, every requirement must include a hash.
if mode.is_require() {
return Err(HashStrategyError::MissingHashes(
requirement.to_string(),
mode,
));
}
continue;
}

// Parse the hashes.
Expand All @@ -125,42 +163,44 @@ impl HashStrategy {
hashes.insert(id, digests);
}

Ok(Self::Validate(hashes))
match mode {
HashCheckingMode::Verify => Ok(Self::Verify(hashes)),
HashCheckingMode::Require => Ok(Self::Require(hashes)),
}
}
}

fn uv_requirement_to_package_id(requirement: &Requirement) -> Result<PackageId, HashStrategyError> {
Ok(match &requirement.source {
RequirementSource::Registry { specifier, .. } => {
// Must be a single specifier.
let [specifier] = specifier.as_ref() else {
return Err(HashStrategyError::UnpinnedRequirement(
requirement.to_string(),
));
};
/// Pin a [`Requirement`] to a [`PackageId`], if possible.
fn pin(requirement: &Requirement) -> Option<PackageId> {
match &requirement.source {
RequirementSource::Registry { specifier, .. } => {
// Must be a single specifier.
let [specifier] = specifier.as_ref() else {
return None;
};

// Must be pinned to a specific version.
if *specifier.operator() != pep440_rs::Operator::Equal {
return None;
}

// Must be pinned to a specific version.
if *specifier.operator() != pep440_rs::Operator::Equal {
return Err(HashStrategyError::UnpinnedRequirement(
requirement.to_string(),
));
Some(PackageId::from_registry(requirement.name.clone()))
}

PackageId::from_registry(requirement.name.clone())
RequirementSource::Url { url, .. }
| RequirementSource::Git { url, .. }
| RequirementSource::Path { url, .. }
| RequirementSource::Directory { url, .. } => Some(PackageId::from_url(url)),
}
RequirementSource::Url { url, .. }
| RequirementSource::Git { url, .. }
| RequirementSource::Path { url, .. }
| RequirementSource::Directory { url, .. } => PackageId::from_url(url),
})
}
}

#[derive(thiserror::Error, Debug)]
pub enum HashStrategyError {
#[error(transparent)]
Hash(#[from] HashError),
#[error("In `--require-hashes` mode, all requirement must have their versions pinned with `==`, but found: {0}")]
UnpinnedRequirement(String),
#[error("In `--require-hashes` mode, all requirement must have a hash, but none were provided for: {0}")]
MissingHashes(String),
#[error(
"In `{1}` mode, all requirement must have their versions pinned with `==`, but found: {0}"
)]
UnpinnedRequirement(String, HashCheckingMode),
#[error("In `{1}` mode, all requirement must have a hash, but none were provided for: {0}")]
MissingHashes(String, HashCheckingMode),
}
9 changes: 5 additions & 4 deletions crates/uv/src/commands/pip/install.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@ use uv_auth::store_credentials_from_url;
use uv_cache::Cache;
use uv_client::{BaseClientBuilder, Connectivity, FlatIndexClient, RegistryClientBuilder};
use uv_configuration::{
BuildOptions, Concurrency, ConfigSettings, ExtrasSpecification, IndexStrategy, PreviewMode,
Reinstall, SetupPyStrategy, Upgrade,
BuildOptions, Concurrency, ConfigSettings, ExtrasSpecification, HashCheckingMode,
IndexStrategy, PreviewMode, Reinstall, SetupPyStrategy, Upgrade,
};
use uv_configuration::{KeyringProviderType, TargetTriple};
use uv_dispatch::BuildDispatch;
Expand Down Expand Up @@ -52,7 +52,7 @@ pub(crate) async fn pip_install(
reinstall: Reinstall,
link_mode: LinkMode,
compile: bool,
require_hashes: bool,
hash_checking: Option<HashCheckingMode>,
setup_py: SetupPyStrategy,
connectivity: Connectivity,
config_settings: &ConfigSettings,
Expand Down Expand Up @@ -226,13 +226,14 @@ pub(crate) async fn pip_install(
let (tags, markers) = resolution_environment(python_version, python_platform, interpreter)?;

// Collect the set of required hashes.
let hasher = if require_hashes {
let hasher = if let Some(hash_checking) = hash_checking {
HashStrategy::from_requirements(
requirements
.iter()
.chain(overrides.iter())
.map(|entry| (&entry.requirement, entry.hashes.as_slice())),
Some(&markers),
hash_checking,
)?
} else {
HashStrategy::None
Expand Down
Loading

0 comments on commit 088b164

Please sign in to comment.