Skip to content

Commit

Permalink
Add a universal resolution mode to pip compile
Browse files Browse the repository at this point in the history
  • Loading branch information
charliermarsh committed Jun 25, 2024
1 parent 99152c9 commit 7e3b618
Show file tree
Hide file tree
Showing 9 changed files with 169 additions and 27 deletions.
9 changes: 9 additions & 0 deletions crates/uv-cli/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -568,6 +568,15 @@ pub struct PipCompileArgs {
#[arg(long)]
pub python_platform: Option<TargetTriple>,

/// Perform a universal resolution, attempting to generate a single `requirements.txt` output
/// file that is compatible with all operating systems, architectures and supported Python
/// versions.
#[arg(long, overrides_with("no_universal"))]
pub universal: bool,

#[arg(long, overrides_with("universal"), hide = true)]
pub no_universal: bool,

/// Specify a package to omit from the output resolution. Its dependencies will still be
/// included in the resolution. Equivalent to pip-compile's `--unsafe-package` option.
#[arg(long, alias = "unsafe-package")]
Expand Down
34 changes: 28 additions & 6 deletions crates/uv-resolver/src/python_requirement.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
use pep440_rs::VersionSpecifiers;
use pep508_rs::StringVersion;
use pep440_rs::{Operator, Version, VersionSpecifier, VersionSpecifiers};
use pep508_rs::{MarkerExpression, MarkerTree, MarkerValueVersion, StringVersion};
use uv_toolchain::{Interpreter, PythonVersion};

use crate::RequiresPython;
Expand Down Expand Up @@ -59,10 +59,32 @@ impl PythonRequirement {
self.target.as_ref()
}

/// Return the target version of Python as a "requires python" type,
/// if available.
pub(crate) fn requires_python(&self) -> Option<&RequiresPython> {
self.target().and_then(|target| target.as_requires_python())
/// Return a [`MarkerTree`] representing the Python requirement.
///
/// See: [`RequiresPython::to_marker_tree`]
pub fn to_marker_tree(&self) -> MarkerTree {
let version = match &self.target {
None => self.installed.version.clone(),
Some(PythonTarget::Version(version)) => version.version.clone(),
Some(PythonTarget::RequiresPython(requires_python)) => {
return requires_python.to_marker_tree()
}
};

let version_major_minor_only = Version::new(version.release().iter().take(2));
let expr_python_version = MarkerExpression::Version {
key: MarkerValueVersion::PythonVersion,
specifier: VersionSpecifier::from_version(Operator::Equal, version_major_minor_only)
.unwrap(),
};
let expr_python_full_version = MarkerExpression::Version {
key: MarkerValueVersion::PythonFullVersion,
specifier: VersionSpecifier::from_version(Operator::Equal, version).unwrap(),
};
MarkerTree::And(vec![
MarkerTree::Expression(expr_python_version),
MarkerTree::Expression(expr_python_full_version),
])
}
}

Expand Down
8 changes: 1 addition & 7 deletions crates/uv-resolver/src/resolver/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,6 @@ use crate::pubgrub::{
PubGrubPriorities, PubGrubPython, PubGrubSpecifier,
};
use crate::python_requirement::PythonRequirement;
use crate::requires_python::RequiresPython;
use crate::resolution::ResolutionGraph;
pub(crate) use crate::resolver::availability::{
IncompletePackage, ResolverVersion, UnavailablePackage, UnavailableReason, UnavailableVersion,
Expand Down Expand Up @@ -192,12 +191,7 @@ impl<Provider: ResolverProvider, InstalledPackages: InstalledPackagesProvider>
let requires_python = if markers.is_some() {
None
} else {
Some(
python_requirement
.requires_python()
.map(RequiresPython::to_marker_tree)
.unwrap_or_else(|| MarkerTree::And(vec![])),
)
Some(python_requirement.to_marker_tree())
};
let state = ResolverState {
index: index.clone(),
Expand Down
1 change: 1 addition & 0 deletions crates/uv-settings/src/settings.rs
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,7 @@ pub struct PipOptions {
pub config_settings: Option<ConfigSettings>,
pub python_version: Option<PythonVersion>,
pub python_platform: Option<TargetTriple>,
pub universal: Option<bool>,
pub exclude_newer: Option<ExcludeNewer>,
pub no_emit_package: Option<Vec<PackageName>>,
pub emit_index_url: Option<bool>,
Expand Down
37 changes: 23 additions & 14 deletions crates/uv/src/commands/pip/compile.rs
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ pub(crate) async fn pip_compile(
build_options: BuildOptions,
python_version: Option<PythonVersion>,
python_platform: Option<TargetTriple>,
universal: bool,
exclude_newer: Option<ExcludeNewer>,
annotation_style: AnnotationStyle,
link_mode: LinkMode,
Expand Down Expand Up @@ -219,7 +220,13 @@ pub(crate) async fn pip_compile(
};

// Determine the environment for the resolution.
let (tags, markers) = resolution_environment(python_version, python_platform, &interpreter)?;
let (tags, markers) = if universal {
(None, None)
} else {
let (tags, markers) =
resolution_environment(python_version, python_platform, &interpreter)?;
(Some(tags), Some(markers))
};

// Generate, but don't enforce hashes for the requirements.
let hasher = if generate_hashes {
Expand Down Expand Up @@ -247,7 +254,7 @@ pub(crate) async fn pip_compile(
.index_urls(index_locations.index_urls())
.index_strategy(index_strategy)
.keyring(keyring_provider)
.markers(&markers)
.markers(interpreter.markers())
.platform(interpreter.platform())
.build();

Expand All @@ -262,7 +269,7 @@ pub(crate) async fn pip_compile(
let flat_index = {
let client = FlatIndexClient::new(&client, &cache);
let entries = client.fetch(index_locations.flat_index()).await?;
FlatIndex::from_entries(entries, Some(&tags), &hasher, &build_options)
FlatIndex::from_entries(entries, tags.as_deref(), &hasher, &build_options)
};

// Track in-flight downloads, builds, etc., across resolutions.
Expand Down Expand Up @@ -319,8 +326,8 @@ pub(crate) async fn pip_compile(
&hasher,
&Reinstall::None,
&upgrade,
Some(&tags),
Some(&markers),
tags.as_deref(),
markers.as_deref(),
python_requirement,
&client,
&flat_index,
Expand Down Expand Up @@ -368,13 +375,15 @@ pub(crate) async fn pip_compile(
}

if include_marker_expression {
let relevant_markers = resolution.marker_tree(&top_level_index, &markers)?;
writeln!(
writer,
"{}",
"# Pinned dependencies known to be valid for:".green()
)?;
writeln!(writer, "{}", format!("# {relevant_markers}").green())?;
if let Some(markers) = markers.as_deref() {
let relevant_markers = resolution.marker_tree(&top_level_index, markers)?;
writeln!(
writer,
"{}",
"# Pinned dependencies known to be valid for:".green()
)?;
writeln!(writer, "{}", format!("# {relevant_markers}").green())?;
}
}

let mut wrote_preamble = false;
Expand Down Expand Up @@ -439,11 +448,11 @@ pub(crate) async fn pip_compile(
"{}",
DisplayResolutionGraph::new(
&resolution,
Some(&markers),
markers.as_deref(),
&no_emit_packages,
generate_hashes,
include_extras,
include_markers,
include_markers || universal,
include_annotations,
include_index_annotation,
annotation_style,
Expand Down
1 change: 1 addition & 0 deletions crates/uv/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -282,6 +282,7 @@ async fn run() -> Result<ExitStatus> {
args.settings.build_options,
args.settings.python_version,
args.settings.python_platform,
args.settings.universal,
args.settings.exclude_newer,
args.settings.annotation_style,
args.settings.link_mode,
Expand Down
6 changes: 6 additions & 0 deletions crates/uv/src/settings.rs
Original file line number Diff line number Diff line change
Expand Up @@ -525,6 +525,8 @@ impl PipCompileSettings {
only_binary,
python_version,
python_platform,
universal,
no_universal,
no_emit_package,
emit_index_url,
no_emit_index_url,
Expand Down Expand Up @@ -583,6 +585,7 @@ impl PipCompileSettings {
legacy_setup_py: flag(legacy_setup_py, no_legacy_setup_py),
python_version,
python_platform,
universal: flag(universal, no_universal),
no_emit_package,
emit_index_url: flag(emit_index_url, no_emit_index_url),
emit_find_links: flag(emit_find_links, no_emit_find_links),
Expand Down Expand Up @@ -1499,6 +1502,7 @@ pub(crate) struct PipSettings {
pub(crate) config_setting: ConfigSettings,
pub(crate) python_version: Option<PythonVersion>,
pub(crate) python_platform: Option<TargetTriple>,
pub(crate) universal: bool,
pub(crate) exclude_newer: Option<ExcludeNewer>,
pub(crate) no_emit_package: Vec<PackageName>,
pub(crate) emit_index_url: bool,
Expand Down Expand Up @@ -1555,6 +1559,7 @@ impl PipSettings {
config_settings,
python_version,
python_platform,
universal,
exclude_newer,
no_emit_package,
emit_index_url,
Expand Down Expand Up @@ -1686,6 +1691,7 @@ impl PipSettings {
.unwrap_or_default(),
python_version: args.python_version.combine(python_version),
python_platform: args.python_platform.combine(python_platform),
universal: args.universal.combine(universal).unwrap_or_default(),
exclude_newer: args.exclude_newer.combine(exclude_newer),
no_emit_package: args
.no_emit_package
Expand Down
94 changes: 94 additions & 0 deletions crates/uv/tests/pip_compile.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6324,6 +6324,100 @@ fn no_strip_markers_transitive_marker() -> Result<()> {
Ok(())
}

/// Perform a universal resolution with a package that has a marker.
#[test]
fn universal() -> Result<()> {
let context = TestContext::new("3.12");
let requirements_in = context.temp_dir.child("requirements.in");
requirements_in.write_str(indoc::indoc! {r"
trio ; python_version > '3.11'
trio ; sys_platform == 'win32'
"})?;

uv_snapshot!(context.compile()
.arg("requirements.in")
.arg("--universal"), @r###"
success: true
exit_code: 0
----- stdout -----
# This file was autogenerated by uv via the following command:
# uv pip compile --cache-dir [CACHE_DIR] --exclude-newer 2024-03-25T00:00:00Z requirements.in --universal
attrs==23.2.0 ; python_version > '3.11' or sys_platform == 'win32'
# via
# outcome
# trio
cffi==1.16.0 ; implementation_name != 'pypy' and os_name == 'nt' and (python_version > '3.11' or sys_platform == 'win32')
# via trio
idna==3.6 ; python_version > '3.11' or sys_platform == 'win32'
# via trio
outcome==1.3.0.post0 ; python_version > '3.11' or sys_platform == 'win32'
# via trio
pycparser==2.21 ; implementation_name != 'pypy' and os_name == 'nt' and (python_version > '3.11' or sys_platform == 'win32')
# via cffi
sniffio==1.3.1 ; python_version > '3.11' or sys_platform == 'win32'
# via trio
sortedcontainers==2.4.0 ; python_version > '3.11' or sys_platform == 'win32'
# via trio
trio==0.25.0 ; python_version > '3.11' or sys_platform == 'win32'
# via -r requirements.in
----- stderr -----
Resolved 8 packages in [TIME]
"###
);

Ok(())
}

/// Perform a universal resolution with conflicting versions and markers.
#[test]
fn universal_conflicting() -> Result<()> {
let context = TestContext::new("3.12");
let requirements_in = context.temp_dir.child("requirements.in");
requirements_in.write_str(indoc::indoc! {r"
trio==0.25.0 ; sys_platform == 'darwin'
trio==0.10.0 ; sys_platform == 'win32'
"})?;

uv_snapshot!(context.compile()
.arg("requirements.in")
.arg("--universal"), @r###"
success: true
exit_code: 0
----- stdout -----
# This file was autogenerated by uv via the following command:
# uv pip compile --cache-dir [CACHE_DIR] --exclude-newer 2024-03-25T00:00:00Z requirements.in --universal
async-generator==1.10 ; sys_platform == 'win32'
# via trio
attrs==23.2.0 ; sys_platform == 'darwin' or sys_platform == 'win32'
# via
# outcome
# trio
cffi==1.16.0 ; (implementation_name != 'pypy' and os_name == 'nt' and sys_platform == 'darwin') or (os_name == 'nt' and sys_platform == 'win32')
# via trio
idna==3.6 ; sys_platform == 'darwin' or sys_platform == 'win32'
# via trio
outcome==1.3.0.post0 ; sys_platform == 'darwin' or sys_platform == 'win32'
# via trio
pycparser==2.21 ; (implementation_name != 'pypy' and os_name == 'nt' and sys_platform == 'darwin') or (os_name == 'nt' and sys_platform == 'win32')
# via cffi
sniffio==1.3.1 ; sys_platform == 'darwin' or sys_platform == 'win32'
# via trio
sortedcontainers==2.4.0 ; sys_platform == 'darwin' or sys_platform == 'win32'
# via trio
trio==0.10.0 ; sys_platform == 'win32'
# via -r requirements.in
trio==0.25.0 ; sys_platform == 'darwin'
# via -r requirements.in
----- stderr -----
Resolved 10 packages in [TIME]
"###
);

Ok(())
}

/// Resolve a package from a `requirements.in` file, with a `constraints.txt` file pinning one of
/// its transitive dependencies to a specific version.
#[test]
Expand Down
6 changes: 6 additions & 0 deletions uv.schema.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit 7e3b618

Please sign in to comment.