From ae11317cc00ea6203ae04284bc740ab0dc0ef0dc Mon Sep 17 00:00:00 2001 From: Jo <10510431+j178@users.noreply.github.com> Date: Sat, 27 Jul 2024 20:26:46 +0800 Subject: [PATCH 1/6] Make `pip list --editable` conflicts with `--exlcude-editable` (#5506) ## Summary I think it makes no sense to allow `--editable` and `--exclude-editable` at the same time. ## Test Plan ```console $ cargo run -- pip list --editable --exclude-editable error: the argument '--editable' cannot be used with '--exclude-editable' Usage: uv.exe pip list --editable For more information, try '--help'. ``` --- crates/uv-cli/src/lib.rs | 2 +- crates/uv/src/commands/pip/list.rs | 7 ++---- crates/uv/src/lib.rs | 1 - crates/uv/src/settings.rs | 6 ++---- crates/uv/tests/pip_list.rs | 34 ++++++------------------------ 5 files changed, 12 insertions(+), 38 deletions(-) diff --git a/crates/uv-cli/src/lib.rs b/crates/uv-cli/src/lib.rs index 27b680952041..2eb378765e04 100644 --- a/crates/uv-cli/src/lib.rs +++ b/crates/uv-cli/src/lib.rs @@ -1465,7 +1465,7 @@ pub struct PipListArgs { pub editable: bool, /// Exclude any editable packages from output. - #[arg(long)] + #[arg(long, conflicts_with = "editable")] pub exclude_editable: bool, /// Exclude the specified package(s) from the output. diff --git a/crates/uv/src/commands/pip/list.rs b/crates/uv/src/commands/pip/list.rs index 60e4055f57cf..d2ea76787789 100644 --- a/crates/uv/src/commands/pip/list.rs +++ b/crates/uv/src/commands/pip/list.rs @@ -24,8 +24,7 @@ use crate::printer::Printer; /// Enumerate the installed packages in the current environment. #[allow(clippy::fn_params_excessive_bools)] pub(crate) fn pip_list( - editable: bool, - exclude_editable: bool, + editable: Option, exclude: &[PackageName], format: &ListFormat, strict: bool, @@ -54,9 +53,7 @@ pub(crate) fn pip_list( // Filter if `--editable` is specified; always sort by name. let results = site_packages .iter() - .filter(|dist| { - (!dist.is_editable() && !editable) || (dist.is_editable() && !exclude_editable) - }) + .filter(|dist| editable.is_none() || editable == Some(dist.is_editable())) .filter(|dist| !exclude.contains(dist.name())) .sorted_unstable_by(|a, b| a.name().cmp(b.name()).then(a.version().cmp(b.version()))) .collect_vec(); diff --git a/crates/uv/src/lib.rs b/crates/uv/src/lib.rs index c5ca7eef3956..7fd48a83239d 100644 --- a/crates/uv/src/lib.rs +++ b/crates/uv/src/lib.rs @@ -465,7 +465,6 @@ async fn run(cli: Cli) -> Result { commands::pip_list( args.editable, - args.exclude_editable, &args.exclude, &args.format, args.settings.strict, diff --git a/crates/uv/src/settings.rs b/crates/uv/src/settings.rs index cf4e7f510287..a6a7eb28d509 100644 --- a/crates/uv/src/settings.rs +++ b/crates/uv/src/settings.rs @@ -1240,8 +1240,7 @@ impl PipFreezeSettings { #[allow(clippy::struct_excessive_bools)] #[derive(Debug, Clone)] pub(crate) struct PipListSettings { - pub(crate) editable: bool, - pub(crate) exclude_editable: bool, + pub(crate) editable: Option, pub(crate) exclude: Vec, pub(crate) format: ListFormat, pub(crate) settings: PipSettings, @@ -1264,8 +1263,7 @@ impl PipListSettings { } = args; Self { - editable, - exclude_editable, + editable: flag(editable, exclude_editable), exclude, format, settings: PipSettings::combine( diff --git a/crates/uv/tests/pip_list.rs b/crates/uv/tests/pip_list.rs index ebe06b670f3c..03f636754d80 100644 --- a/crates/uv/tests/pip_list.rs +++ b/crates/uv/tests/pip_list.rs @@ -213,11 +213,16 @@ fn list_editable_only() { uv_snapshot!(filters, list_command(&context) .arg("--editable") .arg("--exclude-editable"), @r###" - success: true - exit_code: 0 + success: false + exit_code: 2 ----- stdout ----- ----- stderr ----- + error: the argument '--editable' cannot be used with '--exclude-editable' + + Usage: uv pip list --cache-dir [CACHE_DIR] --editable + + For more information, try '--help'. "### ); } @@ -367,19 +372,6 @@ fn list_format_json() { ----- stderr ----- "### ); - - uv_snapshot!(filters, list_command(&context) - .arg("--format=json") - .arg("--editable") - .arg("--exclude-editable"), @r###" - success: true - exit_code: 0 - ----- stdout ----- - [] - - ----- stderr ----- - "### - ); } #[test] @@ -451,18 +443,6 @@ fn list_format_freeze() { ----- stderr ----- "### ); - - uv_snapshot!(filters, list_command(&context) - .arg("--format=freeze") - .arg("--editable") - .arg("--exclude-editable"), @r###" - success: true - exit_code: 0 - ----- stdout ----- - - ----- stderr ----- - "### - ); } #[test] From 1734c7ed507e5ca7b69d6377da477c711e653974 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Sat, 27 Jul 2024 14:38:16 -0400 Subject: [PATCH 2/6] Use existing METADATA parser in wheel installer (#5508) --- Cargo.lock | 1 - crates/install-wheel-rs/Cargo.toml | 1 - crates/install-wheel-rs/src/linker.rs | 30 ++++++------ crates/install-wheel-rs/src/metadata.rs | 30 ------------ crates/install-wheel-rs/src/wheel.rs | 49 ++------------------ crates/pypi-types/src/metadata.rs | 59 ++++++++++++++++++++++++ crates/uv/src/commands/pip/operations.rs | 5 +- 7 files changed, 77 insertions(+), 98 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c68161995a0b..d92610ebf4df 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1821,7 +1821,6 @@ dependencies = [ "distribution-filename", "fs-err", "indoc", - "mailparse", "pathdiff", "pep440_rs", "platform-info", diff --git a/crates/install-wheel-rs/Cargo.toml b/crates/install-wheel-rs/Cargo.toml index 7ca22b8ed5c6..eec879177f6e 100644 --- a/crates/install-wheel-rs/Cargo.toml +++ b/crates/install-wheel-rs/Cargo.toml @@ -33,7 +33,6 @@ configparser = { workspace = true } csv = { workspace = true } data-encoding = { workspace = true } fs-err = { workspace = true } -mailparse = { workspace = true } pathdiff = { workspace = true } platform-info = { workspace = true } reflink-copy = { workspace = true } diff --git a/crates/install-wheel-rs/src/linker.rs b/crates/install-wheel-rs/src/linker.rs index 5eb48195ce21..1bc70aadf4b9 100644 --- a/crates/install-wheel-rs/src/linker.rs +++ b/crates/install-wheel-rs/src/linker.rs @@ -2,28 +2,25 @@ //! reading from a zip file. use std::path::{Path, PathBuf}; -use std::str::FromStr; use std::sync::{Arc, Mutex}; use std::time::SystemTime; use distribution_filename::WheelFilename; use fs_err as fs; use fs_err::{DirEntry, File}; -use pep440_rs::Version; -use pypi_types::DirectUrl; +use pypi_types::{DirectUrl, Metadata12}; use reflink_copy as reflink; use rustc_hash::FxHashMap; use serde::{Deserialize, Serialize}; use tempfile::tempdir_in; use tracing::{debug, instrument}; -use uv_normalize::PackageName; use uv_warnings::warn_user_once; use walkdir::WalkDir; use crate::script::{scripts_from_ini, Script}; use crate::wheel::{ - extra_dist_info, install_data, parse_metadata, parse_wheel_file, read_record_file, - write_script_entrypoints, LibKind, + extra_dist_info, install_data, parse_wheel_file, read_record_file, write_script_entrypoints, + LibKind, }; use crate::{Error, Layout}; @@ -49,16 +46,15 @@ pub fn install_wheel( ) -> Result<(), Error> { let dist_info_prefix = find_dist_info(&wheel)?; let metadata = dist_info_metadata(&dist_info_prefix, &wheel)?; - let (name, version) = parse_metadata(&dist_info_prefix, &metadata)?; + let Metadata12 { name, version, .. } = Metadata12::parse_metadata(&metadata) + .map_err(|err| Error::InvalidWheel(err.to_string()))?; // Validate the wheel name and version. { - let name = PackageName::from_str(&name)?; if name != filename.name { return Err(Error::MismatchedName(name, filename.name.clone())); } - let version = Version::from_str(&version)?; if version != filename.version && version != filename.version.clone().without_local() { return Err(Error::MismatchedVersion(version, filename.version.clone())); } @@ -76,13 +72,13 @@ pub fn install_wheel( // > 1.c If Root-Is-Purelib == ‘true’, unpack archive into purelib (site-packages). // > 1.d Else unpack archive into platlib (site-packages). - debug!(name, "Extracting file"); + debug!(?name, "Extracting file"); let site_packages = match lib_kind { LibKind::Pure => &layout.scheme.purelib, LibKind::Plat => &layout.scheme.platlib, }; let num_unpacked = link_mode.link_wheel_files(site_packages, &wheel, locks)?; - debug!(name, "Extracted {num_unpacked} files"); + debug!(?name, "Extracted {num_unpacked} files"); // Read the RECORD file. let mut record_file = File::open( @@ -96,9 +92,9 @@ pub fn install_wheel( parse_scripts(&wheel, &dist_info_prefix, None, layout.python_version.1)?; if console_scripts.is_empty() && gui_scripts.is_empty() { - debug!(name, "No entrypoints"); + debug!(?name, "No entrypoints"); } else { - debug!(name, "Writing entrypoints"); + debug!(?name, "Writing entrypoints"); fs_err::create_dir_all(&layout.scheme.scripts)?; write_script_entrypoints(layout, site_packages, &console_scripts, &mut record, false)?; @@ -109,7 +105,7 @@ pub fn install_wheel( // 2.b Move each subtree of distribution-1.0.data/ onto its destination path. Each subdirectory of distribution-1.0.data/ is a key into a dict of destination directories, such as distribution-1.0.data/(purelib|platlib|headers|scripts|data). The initially supported paths are taken from distutils.command.install. let data_dir = site_packages.join(format!("{dist_info_prefix}.data")); if data_dir.is_dir() { - debug!(name, "Installing data"); + debug!(?name, "Installing data"); install_data( layout, site_packages, @@ -124,10 +120,10 @@ pub fn install_wheel( // 2.e Remove empty distribution-1.0.data directory. fs::remove_dir_all(data_dir)?; } else { - debug!(name, "No data"); + debug!(?name, "No data"); } - debug!(name, "Writing extra metadata"); + debug!(?name, "Writing extra metadata"); extra_dist_info( site_packages, &dist_info_prefix, @@ -137,7 +133,7 @@ pub fn install_wheel( &mut record, )?; - debug!(name, "Writing record"); + debug!(?name, "Writing record"); let mut record_writer = csv::WriterBuilder::new() .has_headers(false) .escape(b'"') diff --git a/crates/install-wheel-rs/src/metadata.rs b/crates/install-wheel-rs/src/metadata.rs index d0334c2d5023..da3622b35a55 100644 --- a/crates/install-wheel-rs/src/metadata.rs +++ b/crates/install-wheel-rs/src/metadata.rs @@ -11,36 +11,6 @@ use uv_normalize::PackageName; use crate::Error; -/// Returns `true` if the file is a `METADATA` file in a `.dist-info` directory that matches the -/// wheel filename. -pub fn is_metadata_entry(path: &str, filename: &WheelFilename) -> bool { - let Some((dist_info_dir, file)) = path.split_once('/') else { - return false; - }; - if file != "METADATA" { - return false; - } - let Some(dir_stem) = dist_info_dir.strip_suffix(".dist-info") else { - return false; - }; - let Some((name, version)) = dir_stem.rsplit_once('-') else { - return false; - }; - let Ok(name) = PackageName::from_str(name) else { - return false; - }; - if name != filename.name { - return false; - } - let Ok(version) = Version::from_str(version) else { - return false; - }; - if version != filename.version { - return false; - } - true -} - /// Find the `.dist-info` directory in a zipped wheel. /// /// Returns the dist info dir prefix without the `.dist-info` extension. diff --git a/crates/install-wheel-rs/src/wheel.rs b/crates/install-wheel-rs/src/wheel.rs index f53bf8f6ec3f..a41f4f2fe041 100644 --- a/crates/install-wheel-rs/src/wheel.rs +++ b/crates/install-wheel-rs/src/wheel.rs @@ -6,7 +6,6 @@ use std::{env, io}; use data_encoding::BASE64URL_NOPAD; use fs_err as fs; use fs_err::{DirEntry, File}; -use mailparse::MailHeaderMap; use rustc_hash::FxHashMap; use sha2::{Digest, Sha256}; use tracing::{instrument, warn}; @@ -16,6 +15,7 @@ use zip::ZipWriter; use pypi_types::DirectUrl; use uv_fs::{relative_to, Simplified}; +use uv_normalize::PackageName; use crate::record::RecordEntry; use crate::script::Script; @@ -557,7 +557,7 @@ pub(crate) fn install_data( layout: &Layout, site_packages: &Path, data_dir: &Path, - dist_name: &str, + dist_name: &PackageName, console_scripts: &[Script], gui_scripts: &[Script], record: &mut [RecordEntry], @@ -602,7 +602,7 @@ pub(crate) fn install_data( } } Some("headers") => { - let target_path = layout.scheme.include.join(dist_name); + let target_path = layout.scheme.include.join(dist_name.as_str()); move_folder_recorded(&path, &target_path, site_packages, record)?; } Some("purelib") => { @@ -727,49 +727,6 @@ fn parse_key_value_file( Ok(data) } -/// Parse the distribution name and version from a wheel's `dist-info` metadata. -/// -/// See: -pub(crate) fn parse_metadata( - dist_info_prefix: &str, - content: &[u8], -) -> Result<(String, String), Error> { - // HACK: trick mailparse to parse as UTF-8 instead of ASCII - let mut mail = b"Content-Type: text/plain; charset=utf-8\n".to_vec(); - mail.extend_from_slice(content); - let msg = mailparse::parse_mail(&mail).map_err(|err| { - Error::InvalidWheel(format!( - "Invalid metadata in {dist_info_prefix}.dist-info/METADATA: {err}" - )) - })?; - let headers = msg.get_headers(); - let metadata_version = - headers - .get_first_value("Metadata-Version") - .ok_or(Error::InvalidWheel(format!( - "No `Metadata-Version` field in: {dist_info_prefix}.dist-info/METADATA" - )))?; - // Crude but it should do https://packaging.python.org/en/latest/specifications/core-metadata/#metadata-version - // At time of writing: - // > Version of the file format; legal values are “1.0”, “1.1”, “1.2”, “2.1”, “2.2”, and “2.3”. - if !(metadata_version.starts_with("1.") || metadata_version.starts_with("2.")) { - return Err(Error::InvalidWheel(format!( - "`Metadata-Version` field has unsupported value {metadata_version} in: {dist_info_prefix}.dist-info/METADATA" - ))); - } - let name = headers - .get_first_value("Name") - .ok_or(Error::InvalidWheel(format!( - "No `Name` field in: {dist_info_prefix}.dist-info/METADATA" - )))?; - let version = headers - .get_first_value("Version") - .ok_or(Error::InvalidWheel(format!( - "No `Version` field in: {dist_info_prefix}.dist-info/METADATA" - )))?; - Ok((name, version)) -} - #[cfg(test)] mod test { use std::io::Cursor; diff --git a/crates/pypi-types/src/metadata.rs b/crates/pypi-types/src/metadata.rs index 50a35f6f62f7..3aa24add303b 100644 --- a/crates/pypi-types/src/metadata.rs +++ b/crates/pypi-types/src/metadata.rs @@ -338,6 +338,65 @@ impl Metadata10 { } } +/// Python Package Metadata 1.2 and later as specified in +/// . +/// +/// This is a subset of the full metadata specification, and only includes the +/// fields that have been consistent across all versions of the specification later than 1.2. +#[derive(Deserialize, Debug, Clone)] +#[serde(rename_all = "kebab-case")] +pub struct Metadata12 { + pub name: PackageName, + pub version: Version, + pub requires_python: Option, +} + +impl Metadata12 { + /// Parse the [`Metadata12`] from a `.dist-info` `METADATA` file, as included in a built + /// distribution. + pub fn parse_metadata(content: &[u8]) -> Result { + let headers = Headers::parse(content)?; + + // To rely on a source distribution's `PKG-INFO` file, the `Metadata-Version` field must be + // present and set to a value of at least `2.2`. + let metadata_version = headers + .get_first_value("Metadata-Version") + .ok_or(MetadataError::FieldNotFound("Metadata-Version"))?; + + // Parse the version into (major, minor). + let (major, minor) = parse_version(&metadata_version)?; + + // At time of writing: + // > Version of the file format; legal values are “1.0”, “1.1”, “1.2”, “2.1”, “2.2”, and “2.3”. + if (major, minor) < (1, 0) || (major, minor) >= (3, 0) { + return Err(MetadataError::InvalidMetadataVersion(metadata_version)); + } + + let name = PackageName::new( + headers + .get_first_value("Name") + .ok_or(MetadataError::FieldNotFound("Name"))?, + )?; + let version = Version::from_str( + &headers + .get_first_value("Version") + .ok_or(MetadataError::FieldNotFound("Version"))?, + ) + .map_err(MetadataError::Pep440VersionError)?; + let requires_python = headers + .get_first_value("Requires-Python") + .map(|requires_python| LenientVersionSpecifiers::from_str(&requires_python)) + .transpose()? + .map(VersionSpecifiers::from); + + Ok(Self { + name, + version, + requires_python, + }) + } +} + /// Parse a `Metadata-Version` field into a (major, minor) tuple. fn parse_version(metadata_version: &str) -> Result<(u8, u8), MetadataError> { let (major, minor) = diff --git a/crates/uv/src/commands/pip/operations.rs b/crates/uv/src/commands/pip/operations.rs index 483aa5020661..5b57c600868d 100644 --- a/crates/uv/src/commands/pip/operations.rs +++ b/crates/uv/src/commands/pip/operations.rs @@ -2,7 +2,6 @@ use std::fmt::{self, Write}; use std::path::PathBuf; -use std::time::Instant; use anyhow::{anyhow, Context}; use itertools::Itertools; @@ -99,7 +98,7 @@ pub(crate) async fn resolve( preview: PreviewMode, quiet: bool, ) -> Result { - let start = Instant::now(); + let start = std::time::Instant::now(); // Resolve the requirements from the provided sources. let requirements = { @@ -261,7 +260,7 @@ pub(crate) async fn resolve( // Prints a success message after completing resolution. pub(crate) fn resolution_success( resolution: &ResolutionGraph, - start: Instant, + start: std::time::Instant, printer: Printer, ) -> fmt::Result { let s = if resolution.len() == 1 { "" } else { "s" }; From 88340fbd0dab868a215565b905070105632c3a33 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Sun, 28 Jul 2024 13:20:12 -0400 Subject: [PATCH 3/6] Remove some unused methods (#5512) --- crates/pep508-rs/src/lib.rs | 5 --- crates/uv-cli/src/options.rs | 57 +----------------------- crates/uv-git/src/resolver.rs | 9 ---- crates/uv-installer/src/site_packages.rs | 7 --- crates/uv-python/src/interpreter.rs | 34 -------------- crates/uv-resolver/src/yanks.rs | 6 --- 6 files changed, 1 insertion(+), 117 deletions(-) diff --git a/crates/pep508-rs/src/lib.rs b/crates/pep508-rs/src/lib.rs index 5f8f17dfb48d..79135804cf0c 100644 --- a/crates/pep508-rs/src/lib.rs +++ b/crates/pep508-rs/src/lib.rs @@ -551,11 +551,6 @@ impl Extras { pub fn parse(input: &str) -> Result> { Ok(Self(parse_extras_cursor(&mut Cursor::new(input))?)) } - - /// Convert the [`Extras`] into a [`Vec`] of [`ExtraName`]. - pub fn into_vec(self) -> Vec { - self.0 - } } /// The actual version specifier or URL to install. diff --git a/crates/uv-cli/src/options.rs b/crates/uv-cli/src/options.rs index 77cdacf46378..bed0537948ba 100644 --- a/crates/uv-cli/src/options.rs +++ b/crates/uv-cli/src/options.rs @@ -1,7 +1,7 @@ use uv_cache::Refresh; use uv_configuration::ConfigSettings; use uv_resolver::PreReleaseMode; -use uv_settings::{InstallerOptions, PipOptions, ResolverInstallerOptions, ResolverOptions}; +use uv_settings::{PipOptions, ResolverInstallerOptions, ResolverOptions}; use crate::{ BuildArgs, IndexArgs, InstallerArgs, Maybe, RefreshArgs, ResolverArgs, ResolverInstallerArgs, @@ -166,61 +166,6 @@ impl From for PipOptions { } } -/// Construct the [`InstallerOptions`] from the [`InstallerArgs`] and [`BuildArgs`]. -pub fn installer_options(installer_args: InstallerArgs, build_args: BuildArgs) -> InstallerOptions { - let InstallerArgs { - index_args, - reinstall, - no_reinstall, - reinstall_package, - index_strategy, - keyring_provider, - config_setting, - exclude_newer, - link_mode, - compile_bytecode, - no_compile_bytecode, - } = installer_args; - - let BuildArgs { - no_build, - build, - no_build_package, - no_binary, - binary, - no_binary_package, - } = build_args; - - InstallerOptions { - index_url: index_args.index_url.and_then(Maybe::into_option), - extra_index_url: index_args.extra_index_url.map(|extra_index_urls| { - extra_index_urls - .into_iter() - .filter_map(Maybe::into_option) - .collect() - }), - no_index: if index_args.no_index { - Some(true) - } else { - None - }, - find_links: index_args.find_links, - reinstall: flag(reinstall, no_reinstall), - reinstall_package: Some(reinstall_package), - index_strategy, - keyring_provider, - config_settings: config_setting - .map(|config_settings| config_settings.into_iter().collect::()), - exclude_newer, - link_mode, - compile_bytecode: flag(compile_bytecode, no_compile_bytecode), - no_build: flag(no_build, build), - no_build_package: Some(no_build_package), - no_binary: flag(no_binary, binary), - no_binary_package: Some(no_binary_package), - } -} - /// Construct the [`ResolverOptions`] from the [`ResolverArgs`] and [`BuildArgs`]. pub fn resolver_options(resolver_args: ResolverArgs, build_args: BuildArgs) -> ResolverOptions { let ResolverArgs { diff --git a/crates/uv-git/src/resolver.rs b/crates/uv-git/src/resolver.rs index 82357c67d51d..93a7ead98942 100644 --- a/crates/uv-git/src/resolver.rs +++ b/crates/uv-git/src/resolver.rs @@ -27,15 +27,6 @@ pub enum GitResolverError { pub struct GitResolver(Arc>); impl GitResolver { - /// Initialize a [`GitResolver`] with a set of resolved references. - pub fn from_refs(refs: Vec) -> Self { - Self(Arc::new( - refs.into_iter() - .map(|ResolvedRepositoryReference { reference, sha }| (reference, sha)) - .collect(), - )) - } - /// Inserts a new [`GitSha`] for the given [`RepositoryReference`]. pub fn insert(&self, reference: RepositoryReference, sha: GitSha) { self.0.insert(reference, sha); diff --git a/crates/uv-installer/src/site_packages.rs b/crates/uv-installer/src/site_packages.rs index 6883bcebe9fc..500d36fe1c1e 100644 --- a/crates/uv-installer/src/site_packages.rs +++ b/crates/uv-installer/src/site_packages.rs @@ -130,13 +130,6 @@ impl SitePackages { .collect() } - /// Returns `true` if there are any installed distributions with the given package name. - pub fn contains_package(&self, name: &PackageName) -> bool { - self.by_name - .get(name) - .is_some_and(|packages| !packages.is_empty()) - } - /// Remove the given packages from the index, returning all installed versions, if any. pub fn remove_packages(&mut self, name: &PackageName) -> Vec { let Some(indexes) = self.by_name.get(name) else { diff --git a/crates/uv-python/src/interpreter.rs b/crates/uv-python/src/interpreter.rs index 55e3ba4a04ca..f98b0040aa04 100644 --- a/crates/uv-python/src/interpreter.rs +++ b/crates/uv-python/src/interpreter.rs @@ -78,40 +78,6 @@ impl Interpreter { }) } - // TODO(konstin): Find a better way mocking the fields - pub fn artificial(platform: Platform, markers: MarkerEnvironment) -> Self { - Self { - platform, - markers: Box::new(markers), - scheme: Scheme { - purelib: PathBuf::from("/dev/null"), - platlib: PathBuf::from("/dev/null"), - include: PathBuf::from("/dev/null"), - scripts: PathBuf::from("/dev/null"), - data: PathBuf::from("/dev/null"), - }, - virtualenv: Scheme { - purelib: PathBuf::from("/dev/null"), - platlib: PathBuf::from("/dev/null"), - include: PathBuf::from("/dev/null"), - scripts: PathBuf::from("/dev/null"), - data: PathBuf::from("/dev/null"), - }, - sys_prefix: PathBuf::from("/dev/null"), - sys_base_exec_prefix: PathBuf::from("/dev/null"), - sys_base_prefix: PathBuf::from("/dev/null"), - sys_base_executable: None, - sys_executable: PathBuf::from("/dev/null"), - sys_path: vec![], - stdlib: PathBuf::from("/dev/null"), - tags: OnceLock::new(), - target: None, - prefix: None, - pointer_size: PointerSize::_64, - gil_disabled: false, - } - } - /// Return a new [`Interpreter`] with the given virtual environment root. #[must_use] pub fn with_virtualenv(self, virtualenv: VirtualEnvironment) -> Self { diff --git a/crates/uv-resolver/src/yanks.rs b/crates/uv-resolver/src/yanks.rs index 6525b48bd308..c4a3d0986438 100644 --- a/crates/uv-resolver/src/yanks.rs +++ b/crates/uv-resolver/src/yanks.rs @@ -57,10 +57,4 @@ impl AllowedYanks { .get(package_name) .map_or(false, |versions| versions.contains(version)) } - - /// Returns versions for the given package which are allowed even if marked as yanked by the - /// relevant index. - pub fn allowed_versions(&self, package_name: &PackageName) -> Option<&FxHashSet> { - self.0.get(package_name) - } } From caf01735faad7288347f15d7683a5e324205ba5c Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Sun, 28 Jul 2024 14:35:18 -0400 Subject: [PATCH 4/6] Avoid warning users for missing self-extra lower bounds (#5518) ## Summary Closes https://github.com/astral-sh/uv/issues/5227. --- .../uv-resolver/src/pubgrub/dependencies.rs | 2 +- crates/uv-resolver/src/resolver/mod.rs | 2 +- crates/uv/tests/lock.rs | 34 +++++++++++++++++++ 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/crates/uv-resolver/src/pubgrub/dependencies.rs b/crates/uv-resolver/src/pubgrub/dependencies.rs index ab92432138ba..3af4458bf0a0 100644 --- a/crates/uv-resolver/src/pubgrub/dependencies.rs +++ b/crates/uv-resolver/src/pubgrub/dependencies.rs @@ -79,7 +79,7 @@ impl PubGrubDependency { package: package.clone(), version: version.clone(), specifier, - url: None, + url, }) } _ => None, diff --git a/crates/uv-resolver/src/resolver/mod.rs b/crates/uv-resolver/src/resolver/mod.rs index 4738f41e92e2..333f6f91c5aa 100644 --- a/crates/uv-resolver/src/resolver/mod.rs +++ b/crates/uv-resolver/src/resolver/mod.rs @@ -2135,7 +2135,7 @@ impl ForkState { // A dependency from the root package or requirements.txt. debug!("Adding direct dependency: {package}{version}"); - // Warn the user if the direct dependency lacks a lower bound in lowest resolution. + // Warn the user if a direct dependency lacks a lower bound in `--lowest` resolution. let missing_lower_bound = version .bounding_range() .map(|(lowest, _highest)| lowest == Bound::Unbounded) diff --git a/crates/uv/tests/lock.rs b/crates/uv/tests/lock.rs index fecc6d634cce..dd9127fa3c08 100644 --- a/crates/uv/tests/lock.rs +++ b/crates/uv/tests/lock.rs @@ -4105,3 +4105,37 @@ fn lock_same_version_multiple_urls() -> Result<()> { Ok(()) } + +/// When locking with `--resolution-mode=lowest`, we shouldn't warn on unbounded direct +/// dependencies. +#[test] +fn lock_unsafe_lowest() -> Result<()> { + let context = TestContext::new("3.12"); + + let pyproject_toml = context.temp_dir.child("pyproject.toml"); + pyproject_toml.write_str( + r#" + [project] + name = "project" + version = "0.1.0" + requires-python = ">=3.12" + dependencies = [] + + [project.optional-dependencies] + dev = ["iniconfig"] + all = ["project[dev]"] + "#, + )?; + + uv_snapshot!(context.filters(), context.lock().arg("--resolution").arg("lowest-direct"), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + warning: `uv lock` is experimental and may change without warning + Resolved 2 packages in [TIME] + "###); + + Ok(()) +} From 4b4128446da9ade3368e103dc6cd2a28fa324cf8 Mon Sep 17 00:00:00 2001 From: Krishnan Chandra <1229365+krishnan-chandra@users.noreply.github.com> Date: Sun, 28 Jul 2024 14:37:48 -0400 Subject: [PATCH 5/6] Support xz compressed packages (#5513) ## Summary Closes #2187. The [xz backdoor](https://gist.github.com/thesamesam/223949d5a074ebc3dce9ee78baad9e27) is still fairly recent, but luckily the [Rust `xz2` crate bundles version 5.2.5 of the C `xz` package](https://github.com/alexcrichton/xz2-rs/tree/main/lzma-sys), which is before the backdoor was introduced. It's worth noting that a security risk still exists if you have a compromised version of `xz` installed on your system, but that risk is not introduced by `uv` or the Rust packages in general. ## Test Plan Tried installing the package mentioned in the linked issue: `python-apt @ https://launchpad.net/ubuntu/+archive/primary/+sourcefiles/python-apt/2.7.6/python-apt_2.7.6.tar.xz` (Note that this will only work on Ubuntu - I tried on a Mac and while the archive was extracted properly, the package did not install because of some missing files) --------- Co-authored-by: Charlie Marsh --- Cargo.lock | 23 +++++++++++++++++++++- crates/uv-extract/Cargo.toml | 2 +- crates/uv-extract/src/stream.rs | 35 ++++++++++++++++++++++++++++++++- 3 files changed, 57 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d92610ebf4df..5a9dce6c2799 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -212,6 +212,7 @@ dependencies = [ "memchr", "pin-project-lite", "tokio", + "xz2", "zstd", "zstd-safe", ] @@ -2039,6 +2040,17 @@ version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +[[package]] +name = "lzma-sys" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "mailparse" version = "0.15.0" @@ -2713,7 +2725,7 @@ dependencies = [ "indoc", "libc", "memoffset 0.9.1", - "parking_lot 0.11.2", + "parking_lot 0.12.3", "portable-atomic", "pyo3-build-config", "pyo3-ffi", @@ -5750,6 +5762,15 @@ version = "0.13.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" +[[package]] +name = "xz2" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" +dependencies = [ + "lzma-sys", +] + [[package]] name = "yansi" version = "0.5.1" diff --git a/crates/uv-extract/Cargo.toml b/crates/uv-extract/Cargo.toml index 234eee8f6748..d866cd43598b 100644 --- a/crates/uv-extract/Cargo.toml +++ b/crates/uv-extract/Cargo.toml @@ -15,7 +15,7 @@ workspace = true [dependencies] pypi-types = { workspace = true } -async-compression = { workspace = true, features = ["bzip2", "gzip", "zstd"] } +async-compression = { workspace = true, features = ["bzip2", "gzip", "zstd", "xz"] } async_zip = { workspace = true, features = ["tokio"] } fs-err = { workspace = true, features = ["tokio"] } futures = { workspace = true } diff --git a/crates/uv-extract/src/stream.rs b/crates/uv-extract/src/stream.rs index f73839a295fc..b3a2ae824ee3 100644 --- a/crates/uv-extract/src/stream.rs +++ b/crates/uv-extract/src/stream.rs @@ -198,7 +198,25 @@ pub async fn untar_zst( Ok(untar_in(&mut archive, target.as_ref()).await?) } -/// Unzip a `.zip`, `.tar.gz`, or `.tar.bz2` archive into the target directory, without requiring `Seek`. +/// Unzip a `.tar.xz` archive into the target directory, without requiring `Seek`. +/// +/// This is useful for unpacking files as they're being downloaded. +pub async fn untar_xz( + reader: R, + target: impl AsRef, +) -> Result<(), Error> { + let reader = tokio::io::BufReader::new(reader); + let decompressed_bytes = async_compression::tokio::bufread::XzDecoder::new(reader); + + let mut archive = tokio_tar::ArchiveBuilder::new(decompressed_bytes) + .set_preserve_mtime(false) + .build(); + untar_in(&mut archive, target.as_ref()).await?; + Ok(()) +} + +/// Unzip a `.zip`, `.tar.gz`, `.tar.bz2`, `.tar.zst`, or `.tar.xz` archive into the target directory, +/// without requiring `Seek`. pub async fn archive( reader: R, source: impl AsRef, @@ -258,5 +276,20 @@ pub async fn archive( return Ok(()); } + // `.tar.xz` + if source + .as_ref() + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("xz")) + && source.as_ref().file_stem().is_some_and(|stem| { + Path::new(stem) + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("tar")) + }) + { + untar_xz(reader, target).await?; + return Ok(()); + } + Err(Error::UnsupportedArchive(source.as_ref().to_path_buf())) } From b0c841ee3b5ef75ad9143e883f8af17d0c6510c9 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Sun, 28 Jul 2024 15:01:17 -0400 Subject: [PATCH 6/6] Ban `--no-cache` with `--link-mode=symlink` (#5519) ## Summary Closes https://github.com/astral-sh/uv/issues/5360. --- crates/install-wheel-rs/src/linker.rs | 5 ++++ crates/uv-cache/src/lib.rs | 13 ++++++--- crates/uv-dispatch/src/lib.rs | 1 + crates/uv-installer/src/installer.rs | 35 ++++++++++++++++++++++-- crates/uv/src/commands/pip/operations.rs | 1 + crates/uv/tests/pip_sync.rs | 28 +++++++++++++++++++ 6 files changed, 76 insertions(+), 7 deletions(-) diff --git a/crates/install-wheel-rs/src/linker.rs b/crates/install-wheel-rs/src/linker.rs index 1bc70aadf4b9..7d71f198ddde 100644 --- a/crates/install-wheel-rs/src/linker.rs +++ b/crates/install-wheel-rs/src/linker.rs @@ -258,6 +258,11 @@ impl LinkMode { Self::Symlink => symlink_wheel_files(site_packages, wheel, locks), } } + + /// Returns `true` if the link mode is [`LinkMode::Symlink`]. + pub fn is_symlink(&self) -> bool { + matches!(self, Self::Symlink) + } } /// Extract a wheel by cloning all of its files into site packages. The files will be cloned diff --git a/crates/uv-cache/src/lib.rs b/crates/uv-cache/src/lib.rs index 457a0513ee13..6c92df98aa3b 100644 --- a/crates/uv-cache/src/lib.rs +++ b/crates/uv-cache/src/lib.rs @@ -120,7 +120,7 @@ pub struct Cache { /// /// Included to ensure that the temporary directory exists for the length of the operation, but /// is dropped at the end as appropriate. - _temp_dir_drop: Option>, + temp_dir: Option>, } impl Cache { @@ -129,7 +129,7 @@ impl Cache { Self { root: root.into(), refresh: Refresh::None(Timestamp::now()), - _temp_dir_drop: None, + temp_dir: None, } } @@ -139,7 +139,7 @@ impl Cache { Ok(Self { root: temp_dir.path().to_path_buf(), refresh: Refresh::None(Timestamp::now()), - _temp_dir_drop: Some(Arc::new(temp_dir)), + temp_dir: Some(Arc::new(temp_dir)), }) } @@ -271,7 +271,12 @@ impl Cache { Ok(id) } - /// Initialize the cache. + /// Returns `true` if the [`Cache`] is temporary. + pub fn is_temporary(&self) -> bool { + self.temp_dir.is_some() + } + + /// Initialize the [`Cache`]. pub fn init(self) -> Result { let root = &self.root; diff --git a/crates/uv-dispatch/src/lib.rs b/crates/uv-dispatch/src/lib.rs index 3928e63b9666..77c7039206ac 100644 --- a/crates/uv-dispatch/src/lib.rs +++ b/crates/uv-dispatch/src/lib.rs @@ -288,6 +288,7 @@ impl<'a> BuildContext for BuildDispatch<'a> { ); wheels = Installer::new(venv) .with_link_mode(self.link_mode) + .with_cache(self.cache) .install(wheels) .await .context("Failed to install build dependencies")?; diff --git a/crates/uv-installer/src/installer.rs b/crates/uv-installer/src/installer.rs index 008b8d32543b..1e3b7a02a559 100644 --- a/crates/uv-installer/src/installer.rs +++ b/crates/uv-installer/src/installer.rs @@ -7,11 +7,13 @@ use tokio::sync::oneshot; use tracing::instrument; use distribution_types::CachedDist; +use uv_cache::Cache; use uv_python::PythonEnvironment; pub struct Installer<'a> { venv: &'a PythonEnvironment, link_mode: LinkMode, + cache: Option<&'a Cache>, reporter: Option>, installer_name: Option, } @@ -22,6 +24,7 @@ impl<'a> Installer<'a> { Self { venv, link_mode: LinkMode::default(), + cache: None, reporter: None, installer_name: Some("uv".to_string()), } @@ -33,6 +36,15 @@ impl<'a> Installer<'a> { Self { link_mode, ..self } } + /// Set the [`Cache`] to use for this installer. + #[must_use] + pub fn with_cache(self, cache: &'a Cache) -> Self { + Self { + cache: Some(cache), + ..self + } + } + /// Set the [`Reporter`] to use for this installer. #[must_use] pub fn with_reporter(self, reporter: impl Reporter + 'static) -> Self { @@ -54,16 +66,25 @@ impl<'a> Installer<'a> { /// Install a set of wheels into a Python virtual environment. #[instrument(skip_all, fields(num_wheels = %wheels.len()))] pub async fn install(self, wheels: Vec) -> Result> { - let (tx, rx) = oneshot::channel(); - let Self { venv, + cache, link_mode, reporter, installer_name, } = self; - let layout = venv.interpreter().layout(); + if cache.is_some_and(Cache::is_temporary) { + if link_mode.is_symlink() { + return Err(anyhow::anyhow!( + "Symlink-based installation is not supported with `--no-cache`. The created environment will be rendered unusable by the removal of the cache." + )); + } + } + + let (tx, rx) = oneshot::channel(); + + let layout = venv.interpreter().layout(); rayon::spawn(move || { let result = install(wheels, layout, installer_name, link_mode, reporter); tx.send(result).unwrap(); @@ -77,6 +98,14 @@ impl<'a> Installer<'a> { /// Install a set of wheels into a Python virtual environment synchronously. #[instrument(skip_all, fields(num_wheels = %wheels.len()))] pub fn install_blocking(self, wheels: Vec) -> Result> { + if self.cache.is_some_and(Cache::is_temporary) { + if self.link_mode.is_symlink() { + return Err(anyhow::anyhow!( + "Symlink-based installation is not supported with `--no-cache`. The created environment will be rendered unusable by the removal of the cache." + )); + } + } + install( wheels, self.venv.interpreter().layout(), diff --git a/crates/uv/src/commands/pip/operations.rs b/crates/uv/src/commands/pip/operations.rs index 5b57c600868d..6c7418161f90 100644 --- a/crates/uv/src/commands/pip/operations.rs +++ b/crates/uv/src/commands/pip/operations.rs @@ -463,6 +463,7 @@ pub(crate) async fn install( let start = std::time::Instant::now(); wheels = uv_installer::Installer::new(venv) .with_link_mode(link_mode) + .with_cache(cache) .with_reporter(InstallReporter::from(printer).with_length(wheels.len() as u64)) // This technically can block the runtime, but we are on the main thread and // have no other running tasks at this point, so this lets us avoid spawning a blocking diff --git a/crates/uv/tests/pip_sync.rs b/crates/uv/tests/pip_sync.rs index 7361a170c733..622937a0652d 100644 --- a/crates/uv/tests/pip_sync.rs +++ b/crates/uv/tests/pip_sync.rs @@ -236,6 +236,34 @@ fn install_symlink() -> Result<()> { Ok(()) } +/// Reject attempts to use symlink semantics with `--no-cache`. +#[test] +fn install_symlink_no_cache() -> Result<()> { + let context = TestContext::new("3.12"); + + let requirements_txt = context.temp_dir.child("requirements.txt"); + requirements_txt.write_str("MarkupSafe==2.1.3")?; + + uv_snapshot!(context.pip_sync() + .arg("requirements.txt") + .arg("--link-mode") + .arg("symlink") + .arg("--no-cache") + .arg("--strict"), @r###" + success: false + exit_code: 2 + ----- stdout ----- + + ----- stderr ----- + Resolved 1 package in [TIME] + Prepared 1 package in [TIME] + error: Symlink-based installation is not supported with `--no-cache`. The created environment will be rendered unusable by the removal of the cache. + "### + ); + + Ok(()) +} + /// Install multiple packages into a virtual environment. #[test] fn install_many() -> Result<()> {