From cded83f4d8c3557ece49dcdab3dcc5e35ca1919d Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Thu, 18 Jun 2026 12:29:37 -0500 Subject: [PATCH 1/7] Compile bytecode only for installed distributions --- crates/uv-install-wheel/src/install.rs | 50 +++--- crates/uv-install-wheel/src/lib.rs | 2 +- crates/uv-installer/src/compile.rs | 182 +++++++++++++++------ crates/uv-installer/src/lib.rs | 2 +- crates/uv/src/commands/mod.rs | 35 +++- crates/uv/src/commands/pip/install.rs | 2 +- crates/uv/src/commands/pip/operations.rs | 81 ++++++++- crates/uv/src/commands/pip/sync.rs | 2 +- crates/uv/src/commands/project/mod.rs | 4 +- crates/uv/src/commands/project/sync.rs | 2 +- crates/uv/tests/pip_install/pip_install.rs | 124 ++++++++++++++ 11 files changed, 399 insertions(+), 87 deletions(-) diff --git a/crates/uv-install-wheel/src/install.rs b/crates/uv-install-wheel/src/install.rs index 64b20246fb5..ea7d7194047 100644 --- a/crates/uv-install-wheel/src/install.rs +++ b/crates/uv-install-wheel/src/install.rs @@ -1,7 +1,7 @@ //! Like `wheel.rs`, but for installing wheels that have already been unzipped, rather than //! reading from a zip file. -use std::path::Path; +use std::path::{Path, PathBuf}; use std::str::FromStr; use fs_err::File; @@ -18,6 +18,29 @@ use crate::wheel::{ }; use crate::{Error, Layout}; +/// Return the path at which the wheel's `.dist-info` directory will be installed. +pub fn installed_dist_info_path( + layout: &Layout, + wheel: impl AsRef, +) -> Result { + let (dist_info_prefix, site_packages) = wheel_destination(layout, wheel.as_ref())?; + Ok(site_packages.join(format!("{dist_info_prefix}.dist-info"))) +} + +fn wheel_destination<'layout>( + layout: &'layout Layout, + wheel: &Path, +) -> Result<(String, &'layout Path), Error> { + let dist_info_prefix = find_dist_info(wheel)?; + let wheel_file_path = wheel.join(format!("{dist_info_prefix}.dist-info/WHEEL")); + let wheel_text = fs_err::read_to_string(wheel_file_path)?; + let site_packages = match WheelFile::parse(&wheel_text)?.lib_kind() { + LibKind::Pure => &layout.scheme.purelib, + LibKind::Plat => &layout.scheme.platlib, + }; + Ok((dist_info_prefix, site_packages)) +} + /// Install the given wheel to the given venv /// /// The caller must ensure that the wheel is compatible to the environment. @@ -39,8 +62,9 @@ pub fn install_wheel( link_mode: LinkMode, state: &InstallState, ) -> Result<(), Error> { - let dist_info_prefix = find_dist_info(&wheel)?; - let metadata = dist_info_metadata(&dist_info_prefix, &wheel)?; + let wheel = wheel.as_ref(); + let (dist_info_prefix, site_packages) = wheel_destination(layout, wheel)?; + let metadata = dist_info_metadata(&dist_info_prefix, wheel)?; let Metadata10 { name, version } = Metadata10::parse_pkg_info(&metadata) .map_err(|err| Error::InvalidWheel(err.to_string()))?; @@ -61,32 +85,18 @@ pub fn install_wheel( // https://packaging.python.org/en/latest/specifications/binary-distribution-format/#installing-a-wheel-distribution-1-0-py32-none-any-whl // > 1.a Parse distribution-1.0.dist-info/WHEEL. // > 1.b Check that installer is compatible with Wheel-Version. Warn if minor version is greater, abort if major version is greater. - let wheel_file_path = wheel - .as_ref() - .join(format!("{dist_info_prefix}.dist-info/WHEEL")); - let wheel_text = fs_err::read_to_string(wheel_file_path)?; - let lib_kind = WheelFile::parse(&wheel_text)?.lib_kind(); - // > 1.c If Root-Is-Purelib == ‘true’, unpack archive into purelib (site-packages). // > 1.d Else unpack archive into platlib (site-packages). trace!(?name, "Extracting wheel files"); - let site_packages = match lib_kind { - LibKind::Pure => &layout.scheme.purelib, - LibKind::Plat => &layout.scheme.platlib, - }; - link_wheel_files(link_mode, site_packages, &wheel, state, filename)?; + link_wheel_files(link_mode, site_packages, wheel, state, filename)?; trace!(?name, "Extracted wheel files"); // Read the RECORD file. - let mut record_file = File::open( - wheel - .as_ref() - .join(format!("{dist_info_prefix}.dist-info/RECORD")), - )?; + let mut record_file = File::open(wheel.join(format!("{dist_info_prefix}.dist-info/RECORD")))?; let mut record = read_record(&mut record_file)?; let (console_scripts, gui_scripts) = - parse_scripts(&wheel, &dist_info_prefix, None, layout.python_version.1)?; + parse_scripts(wheel, &dist_info_prefix, None, layout.python_version.1)?; if console_scripts.is_empty() && gui_scripts.is_empty() { trace!(?name, "No entrypoints"); diff --git a/crates/uv-install-wheel/src/lib.rs b/crates/uv-install-wheel/src/lib.rs index 4803a10e66e..23c2424b7d5 100644 --- a/crates/uv-install-wheel/src/lib.rs +++ b/crates/uv-install-wheel/src/lib.rs @@ -11,7 +11,7 @@ use uv_normalize::PackageName; use uv_pep440::Version; use uv_pypi_types::Scheme; -pub use install::install_wheel; +pub use install::{install_wheel, installed_dist_info_path}; pub use linker::{InstallState, LinkMode}; pub use record::RecordEntry; pub use uninstall::{Uninstall, uninstall_egg, uninstall_legacy_editable, uninstall_wheel}; diff --git a/crates/uv-installer/src/compile.rs b/crates/uv-installer/src/compile.rs index e77c2dc9c15..4cb9131172f 100644 --- a/crates/uv-installer/src/compile.rs +++ b/crates/uv-installer/src/compile.rs @@ -22,6 +22,9 @@ const COMPILEALL_SCRIPT: &str = include_str!("pip_compileall.py"); /// This is longer than any compilation should ever take. const DEFAULT_COMPILE_TIMEOUT: Duration = Duration::from_mins(1); +type WorkerOutcome = std::thread::Result>; +type WorkerHandle = oneshot::Receiver; + #[derive(Debug, Error)] pub enum CompileError { #[error("Failed to list files in `site-packages`")] @@ -59,38 +62,8 @@ pub enum CompileError { EnvironmentError { var: &'static str, message: String }, } -/// Bytecode compile all file in `dir` using a pool of Python interpreters running a Python script -/// that calls `compileall.compile_file`. -/// -/// All compilation errors are muted (like pip). There is a 60s timeout for each file to handle -/// a broken `python`. -/// -/// We only compile all files, but we don't update the RECORD, relying on PEP 491: -/// > Uninstallers should be smart enough to remove .pyc even if it is not mentioned in RECORD. -/// -/// We've confirmed that both uv and pip (as of 24.0.0) remove the `__pycache__` directory. -#[instrument(skip(python_executable))] -pub async fn compile_tree( - dir: &Path, - python_executable: &Path, - concurrency: &Concurrency, - cache: &Path, -) -> Result { - debug_assert!( - dir.is_absolute(), - "compileall doesn't work with relative paths: `{}`", - dir.display() - ); - let worker_count = concurrency.installs; - - // A larger buffer is significantly faster than just 1 or the worker count. - let (sender, receiver) = async_channel::bounded::(worker_count * 10); - - // Running Python with an actual file will produce better error messages. - let tempdir = tempdir_in(cache).map_err(CompileError::TempFile)?; - let pip_compileall_py = tempdir.path().join("pip_compileall.py"); - - let timeout: Option = match env::var(EnvVars::UV_COMPILE_BYTECODE_TIMEOUT) { +fn compile_timeout() -> Result, CompileError> { + let timeout = match env::var(EnvVars::UV_COMPILE_BYTECODE_TIMEOUT) { Ok(value) => match value.as_str() { "0" => None, _ => match value.parse::().map(Duration::from_secs) { @@ -113,16 +86,26 @@ pub async fn compile_tree( } else { debug!("Disabling bytecode compilation timeout"); } + Ok(timeout) +} +fn spawn_workers( + dir: &Path, + python_executable: &Path, + pip_compileall_py: &Path, + receiver: &Receiver, + worker_count: usize, + timeout: Option, +) -> Vec { debug!("Starting {} bytecode compilation workers", worker_count); - let mut worker_handles = Vec::new(); + let mut worker_handles = Vec::with_capacity(worker_count); for _ in 0..worker_count { let (tx, rx) = oneshot::channel(); let worker = worker( dir.to_path_buf(), python_executable.to_path_buf(), - pip_compileall_py.clone(), + pip_compileall_py.to_path_buf(), receiver.clone(), timeout, ); @@ -145,8 +128,73 @@ pub async fn compile_tree( }) .expect("Failed to start compilation worker"); - worker_handles.push(async { rx.await.unwrap() }); + worker_handles.push(rx); + } + worker_handles +} + +async fn wait_for_workers( + worker_handles: Vec, + send_error: Option>, +) -> Result<(), CompileError> { + // Make sure all workers exit regularly, avoid hiding errors. + for result in futures::future::join_all(worker_handles).await { + match result { + // A worker thread panicked or exited without reporting its result. + Err(_) | Ok(Err(_)) => return Err(CompileError::Join), + Ok(Ok(Err(compile_error))) => return Err(compile_error), + Ok(Ok(Ok(()))) => {} + } } + + if let Some(send_error) = send_error { + // This is suspicious: Why did the channel stop working, but all workers exited + // successfully? + return Err(CompileError::WorkerDisappeared(send_error)); + } + + Ok(()) +} + +/// Bytecode compile all file in `dir` using a pool of Python interpreters running a Python script +/// that calls `compileall.compile_file`. +/// +/// All compilation errors are muted (like pip). There is a 60s timeout for each file to handle +/// a broken `python`. +/// +/// We only compile all files, but we don't update the RECORD, relying on PEP 491: +/// > Uninstallers should be smart enough to remove .pyc even if it is not mentioned in RECORD. +/// +/// We've confirmed that both uv and pip (as of 24.0.0) remove the `__pycache__` directory. +#[instrument(skip(python_executable))] +pub async fn compile_tree( + dir: &Path, + python_executable: &Path, + concurrency: &Concurrency, + cache: &Path, +) -> Result { + debug_assert!( + dir.is_absolute(), + "compileall doesn't work with relative paths: `{}`", + dir.display() + ); + let worker_count = concurrency.installs; + + // A larger buffer is significantly faster than just 1 or the worker count. + let (sender, receiver) = async_channel::bounded::(worker_count * 10); + + // Running Python with an actual file will produce better error messages. + let tempdir = tempdir_in(cache).map_err(CompileError::TempFile)?; + let pip_compileall_py = tempdir.path().join("pip_compileall.py"); + let timeout = compile_timeout()?; + let worker_handles = spawn_workers( + dir, + python_executable, + &pip_compileall_py, + &receiver, + worker_count, + timeout, + ); // Make sure the channel gets closed when all workers exit. drop(receiver); @@ -191,24 +239,60 @@ pub async fn compile_tree( // up to worker_count * 10 items in the queue. drop(sender); - // Make sure all workers exit regularly, avoid hiding errors. - for result in futures::future::join_all(worker_handles).await { - match result { - // There spawning earlier errored due to a panic in a task. - Err(_) => return Err(CompileError::Join), - // The worker reports an error. - Ok(Err(compile_error)) => return Err(compile_error), - Ok(Ok(())) => {} - } + wait_for_workers(worker_handles, send_error).await?; + + Ok(source_files) +} + +/// Bytecode compile the given Python source files using a pool of Python interpreters. +/// +/// All paths must be absolute. Compilation errors are muted (like pip), while failures to launch +/// or communicate with the Python workers are returned. +#[instrument(skip(files, python_executable))] +pub async fn compile_files( + files: &[PathBuf], + python_executable: &Path, + concurrency: &Concurrency, + cache: &Path, +) -> Result { + if files.is_empty() { + return Ok(0); } - if let Some(send_error) = send_error { - // This is suspicious: Why did the channel stop working, but all workers exited - // successfully? - return Err(CompileError::WorkerDisappeared(send_error)); + let worker_count = concurrency.installs.min(files.len()); + let (sender, receiver) = async_channel::bounded::(worker_count * 10); + + // Running Python with an actual file will produce better error messages. + let tempdir = tempdir_in(cache).map_err(CompileError::TempFile)?; + let pip_compileall_py = tempdir.path().join("pip_compileall.py"); + let timeout = compile_timeout()?; + let worker_handles = spawn_workers( + cache, + python_executable, + &pip_compileall_py, + &receiver, + worker_count, + timeout, + ); + drop(receiver); + + let mut send_error = None; + for file in files { + debug_assert!( + file.is_absolute(), + "compileall doesn't work with relative paths: `{}`", + file.display() + ); + if let Err(err) = sender.send(file.clone()).await { + send_error = Some(err); + break; + } } + drop(sender); - Ok(source_files) + wait_for_workers(worker_handles, send_error).await?; + + Ok(files.len()) } async fn worker( diff --git a/crates/uv-installer/src/lib.rs b/crates/uv-installer/src/lib.rs index a8bd1ca778b..2ddfcd71db4 100644 --- a/crates/uv-installer/src/lib.rs +++ b/crates/uv-installer/src/lib.rs @@ -1,4 +1,4 @@ -pub use compile::{CompileError, compile_tree}; +pub use compile::{CompileError, compile_files, compile_tree}; pub use installer::{Installer, Reporter as InstallReporter}; pub use plan::{IncompatibleWheelError, Plan, Planner}; pub use preparer::{Error as PrepareError, Preparer, Reporter as PrepareReporter}; diff --git a/crates/uv/src/commands/mod.rs b/crates/uv/src/commands/mod.rs index 5e4a9fb43f5..6c475958ee4 100644 --- a/crates/uv/src/commands/mod.rs +++ b/crates/uv/src/commands/mod.rs @@ -67,7 +67,7 @@ use uv_cache::Cache; use uv_configuration::Concurrency; pub(crate) use uv_console::human_readable_bytes; use uv_fs::{CWD, Simplified}; -use uv_installer::compile_tree; +use uv_installer::{compile_files, compile_tree}; use uv_python::PythonEnvironment; use uv_scripts::Pep723Script; pub(crate) use venv::venv; @@ -281,6 +281,36 @@ pub(super) async fn compile_bytecode( ) })?; } + write_bytecode_summary(files, start, printer)?; + Ok(()) +} + +/// Compile the given Python source files to bytecode. +pub(super) async fn compile_bytecode_files( + files: &[PathBuf], + venv: &PythonEnvironment, + concurrency: &Concurrency, + cache: &Cache, + printer: Printer, +) -> anyhow::Result<()> { + if files.is_empty() { + return Ok(()); + } + + let start = std::time::Instant::now(); + let files = compile_files(files, venv.python_executable(), concurrency, cache.root()) + .await + .context("Failed to bytecode-compile installed packages")?; + + write_bytecode_summary(files, start, printer)?; + Ok(()) +} + +fn write_bytecode_summary( + files: usize, + start: std::time::Instant, + printer: Printer, +) -> std::fmt::Result { let s = if files == 1 { "" } else { "s" }; writeln!( printer.stderr(), @@ -291,8 +321,7 @@ pub(super) async fn compile_bytecode( format!("in {}", elapsed(start.elapsed())).dimmed() ) .dimmed() - )?; - Ok(()) + ) } /// A multicasting writer that writes to both the standard output and an output file, if present. diff --git a/crates/uv/src/commands/pip/install.rs b/crates/uv/src/commands/pip/install.rs index 5d809ec54ab..64bc349b4e7 100644 --- a/crates/uv/src/commands/pip/install.rs +++ b/crates/uv/src/commands/pip/install.rs @@ -639,7 +639,7 @@ pub(crate) async fn pip_install( &reinstall, &build_options, link_mode, - compile, + compile.then_some(operations::BytecodeCompilation::Installed), &hasher, &tags, &client, diff --git a/crates/uv/src/commands/pip/operations.rs b/crates/uv/src/commands/pip/operations.rs index 061b60c6456..78b7b40ab9d 100644 --- a/crates/uv/src/commands/pip/operations.rs +++ b/crates/uv/src/commands/pip/operations.rs @@ -2,7 +2,7 @@ use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::fmt::Write; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::sync::Arc; use anyhow::{Context, anyhow}; @@ -24,8 +24,8 @@ use uv_distribution_types::{ UnresolvedRequirementSpecification, VersionOrUrlRef, }; use uv_distribution_types::{DistributionMetadata, InstalledMetadata, Name, Resolution}; -use uv_fs::Simplified; -use uv_install_wheel::LinkMode; +use uv_fs::{CWD, Simplified, normalize_path_under}; +use uv_install_wheel::{LinkMode, installed_dist_info_path, read_record}; use uv_installer::{InstallationStrategy, Plan, Planner, Preparer, SitePackages}; use uv_normalize::PackageName; use uv_pep440::Version; @@ -47,9 +47,9 @@ use uv_tool::InstalledTools; use uv_types::{BuildContext, HashStrategy, InFlight, InstalledPackagesProvider}; use uv_warnings::warn_user; -use crate::commands::compile_bytecode; use crate::commands::pip::loggers::{InstallLogger, ResolveLogger}; use crate::commands::reporters::{InstallReporter, PrepareReporter, ResolverReporter}; +use crate::commands::{compile_bytecode, compile_bytecode_files}; use crate::printer::Printer; /// Consolidate the requirements for an installation. @@ -547,6 +547,14 @@ impl Changelog { } } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum BytecodeCompilation { + /// Compile all Python source files in the environment. + All, + /// Compile Python source files installed by this operation. + Installed, +} + /// Install a set of requirements into the current environment. /// /// Returns a [`Changelog`] summarizing the changes made to the environment. @@ -558,7 +566,7 @@ pub(crate) async fn install( reinstall: &Reinstall, build_options: &BuildOptions, link_mode: LinkMode, - compile: bool, + compile: Option, hasher: &HashStrategy, tags: &Tags, client: &RegistryClient, @@ -625,7 +633,7 @@ pub(crate) async fn install( && cached.is_empty() && reinstalls.is_empty() && extraneous.is_empty() - && !compile + && compile.is_none() { logger.on_check(resolution.len(), start, printer, dry_run)?; return Ok(Changelog::default()); @@ -703,8 +711,16 @@ pub(crate) async fn install( uninstalls.extend(shared_uninstalls); } - if compile { - compile_bytecode(venv, concurrency, cache, printer).await?; + if let Some(compile) = compile { + match compile { + BytecodeCompilation::All => { + compile_bytecode(venv, concurrency, cache, printer).await?; + } + BytecodeCompilation::Installed => { + let files = python_source_files_for_installs(venv, &installs)?; + compile_bytecode_files(&files, venv, concurrency, cache, printer).await?; + } + } } // Construct a summary of the changes made to the environment. @@ -716,6 +732,55 @@ pub(crate) async fn install( Ok(changelog) } +/// Return the Python source files owned by the distributions installed by this operation. +fn python_source_files_for_installs( + venv: &PythonEnvironment, + installs: &[CachedDist], +) -> anyhow::Result> { + let layout = venv.interpreter().layout(); + let site_packages = [ + CWD.join(&layout.scheme.purelib), + CWD.join(&layout.scheme.platlib), + ]; + let mut files = BTreeSet::new(); + + for install in installs { + let dist_info = installed_dist_info_path(&layout, install.path()).with_context(|| { + format!("Failed to locate installed distribution for bytecode compilation: `{install}`") + })?; + let record_root = dist_info.parent().with_context(|| { + format!( + "Invalid installed distribution path: `{}`", + dist_info.user_display() + ) + })?; + let record_path = dist_info.join("RECORD"); + let record = read_record(fs_err::File::open(&record_path)?) + .with_context(|| format!("Failed to read `{}`", record_path.user_display()))?; + + for entry in record { + let path = Path::new(&entry.path); + if path.extension().is_none_or(|extension| extension != "py") { + continue; + } + + let path = record_root.join(path); + let Some(path) = site_packages + .iter() + .find_map(|site_packages| normalize_path_under(&path, site_packages)) + else { + continue; + }; + if !path.is_file() { + continue; + } + files.insert(path); + } + } + + Ok(files.into_iter().collect()) +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum InstallPhase { /// A dedicated phase for building and installing packages with build-isolation disabled. diff --git a/crates/uv/src/commands/pip/sync.rs b/crates/uv/src/commands/pip/sync.rs index 58a7b135453..56fe12b977a 100644 --- a/crates/uv/src/commands/pip/sync.rs +++ b/crates/uv/src/commands/pip/sync.rs @@ -534,7 +534,7 @@ pub(crate) async fn pip_sync( &reinstall, &build_options, link_mode, - compile, + compile.then_some(operations::BytecodeCompilation::All), &hasher, &tags, &client, diff --git a/crates/uv/src/commands/project/mod.rs b/crates/uv/src/commands/project/mod.rs index 13dadea0268..c8c6911db4e 100644 --- a/crates/uv/src/commands/project/mod.rs +++ b/crates/uv/src/commands/project/mod.rs @@ -2371,7 +2371,7 @@ pub(crate) async fn sync_environment( reinstall, build_options, link_mode, - compile_bytecode, + compile_bytecode.then_some(pip::operations::BytecodeCompilation::All), &hasher, tags, &client, @@ -2668,7 +2668,7 @@ pub(crate) async fn update_environment( reinstall, build_options, *link_mode, - *compile_bytecode, + (*compile_bytecode).then_some(pip::operations::BytecodeCompilation::All), &hasher, &tags, &client, diff --git a/crates/uv/src/commands/project/sync.rs b/crates/uv/src/commands/project/sync.rs index 777a93bc177..b31b70bb592 100644 --- a/crates/uv/src/commands/project/sync.rs +++ b/crates/uv/src/commands/project/sync.rs @@ -891,7 +891,7 @@ pub(crate) async fn do_sync( reinstall, build_options, link_mode, - compile_bytecode, + compile_bytecode.then_some(operations::BytecodeCompilation::All), &hasher, &tags, &client, diff --git a/crates/uv/tests/pip_install/pip_install.rs b/crates/uv/tests/pip_install/pip_install.rs index e65aa268a5a..c3f08ffa14d 100644 --- a/crates/uv/tests/pip_install/pip_install.rs +++ b/crates/uv/tests/pip_install/pip_install.rs @@ -99,6 +99,130 @@ fn empty_requirements_txt() -> Result<()> { Ok(()) } +/// Compile only distributions installed by the current operation. +#[test] +fn compile_bytecode_for_installed_distributions() { + let context = uv_test::test_context!("3.12"); + + uv_snapshot!(context.pip_install() + .arg("sniffio==1.3.1"), @" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 1 package in [TIME] + Prepared 1 package in [TIME] + Installed 1 package in [TIME] + + sniffio==1.3.1 + " + ); + + uv_snapshot!(context.pip_install() + .arg("anyio==3.7.1") + .arg("--compile-bytecode"), @" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 3 packages in [TIME] + Prepared 2 packages in [TIME] + Installed 2 packages in [TIME] + Bytecode compiled 45 files in [TIME] + + anyio==3.7.1 + + idna==3.6 + " + ); + + assert!( + context + .site_packages() + .join("anyio") + .join("__pycache__") + .join("__init__.cpython-312.pyc") + .exists() + ); + assert!( + context + .site_packages() + .join("idna") + .join("__pycache__") + .join("__init__.cpython-312.pyc") + .exists() + ); + assert!( + !context + .site_packages() + .join("sniffio") + .join("__pycache__") + .join("__init__.cpython-312.pyc") + .exists() + ); + + uv_snapshot!(context.pip_install() + .arg("sniffio==1.3.1") + .arg("--reinstall-package") + .arg("sniffio") + .arg("--compile-bytecode"), @" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 1 package in [TIME] + Prepared 1 package in [TIME] + Uninstalled 1 package in [TIME] + Installed 1 package in [TIME] + Bytecode compiled 5 files in [TIME] + ~ sniffio==1.3.1 + " + ); + + assert!( + context + .site_packages() + .join("sniffio") + .join("__pycache__") + .join("__init__.cpython-312.pyc") + .exists() + ); +} + +/// Compile symlinked source files installed by the current operation. +#[test] +#[cfg(unix)] +fn compile_bytecode_with_symlink_link_mode() { + let context = uv_test::test_context!("3.12"); + + uv_snapshot!(context.pip_install() + .arg("sniffio==1.3.1") + .arg("--compile-bytecode") + .arg("--link-mode") + .arg("symlink"), @" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 1 package in [TIME] + Prepared 1 package in [TIME] + Installed 1 package in [TIME] + Bytecode compiled 5 files in [TIME] + + sniffio==1.3.1 + " + ); + + assert!( + context + .site_packages() + .join("sniffio") + .join("__pycache__") + .join("__init__.cpython-312.pyc") + .exists() + ); +} + #[test] fn missing_pyproject_toml() { let context = uv_test::test_context!("3.12"); From 05a1b51c1ba09b102d44c5e57205bcbfc89f8380 Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Thu, 18 Jun 2026 12:58:50 -0500 Subject: [PATCH 2/7] Add targeted bytecode compilation regressions --- crates/uv/src/commands/pip/operations.rs | 65 +++++++++++++++++++--- crates/uv/tests/pip_install/pip_install.rs | 3 +- 2 files changed, 58 insertions(+), 10 deletions(-) diff --git a/crates/uv/src/commands/pip/operations.rs b/crates/uv/src/commands/pip/operations.rs index 78b7b40ab9d..08c909c9bed 100644 --- a/crates/uv/src/commands/pip/operations.rs +++ b/crates/uv/src/commands/pip/operations.rs @@ -759,15 +759,8 @@ fn python_source_files_for_installs( .with_context(|| format!("Failed to read `{}`", record_path.user_display()))?; for entry in record { - let path = Path::new(&entry.path); - if path.extension().is_none_or(|extension| extension != "py") { - continue; - } - - let path = record_root.join(path); - let Some(path) = site_packages - .iter() - .find_map(|site_packages| normalize_path_under(&path, site_packages)) + let Some(path) = + python_source_path_from_record(record_root, &entry.path, &site_packages) else { continue; }; @@ -781,6 +774,60 @@ fn python_source_files_for_installs( Ok(files.into_iter().collect()) } +/// Resolve a Python source path from an installed `RECORD` entry. +fn python_source_path_from_record( + record_root: &Path, + entry: &str, + site_packages: &[PathBuf], +) -> Option { + let path = Path::new(entry); + if path.extension().is_none_or(|extension| extension != "py") { + return None; + } + + let path = record_root.join(path); + site_packages + .iter() + .find_map(|site_packages| normalize_path_under(&path, site_packages)) +} + +#[cfg(test)] +mod tests { + use super::python_source_path_from_record; + use std::path::{Path, PathBuf}; + + #[test] + fn record_python_sources_stay_in_site_packages() { + let record_root = Path::new("venv/purelib"); + let site_packages = [PathBuf::from("venv/purelib"), PathBuf::from("venv/platlib")]; + + assert_eq!( + python_source_path_from_record(record_root, "package/__init__.py", &site_packages,), + Some(PathBuf::from("venv/purelib/package/__init__.py")) + ); + assert_eq!( + python_source_path_from_record( + record_root, + "../platlib/package/module.py", + &site_packages, + ), + Some(PathBuf::from("venv/platlib/package/module.py")) + ); + assert_eq!( + python_source_path_from_record(record_root, "../scripts/tool.py", &site_packages), + None + ); + assert_eq!( + python_source_path_from_record(record_root, "/outside.py", &site_packages), + None + ); + assert_eq!( + python_source_path_from_record(record_root, "package/data.txt", &site_packages), + None + ); + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum InstallPhase { /// A dedicated phase for building and installing packages with build-isolation disabled. diff --git a/crates/uv/tests/pip_install/pip_install.rs b/crates/uv/tests/pip_install/pip_install.rs index c3f08ffa14d..b45c7f13ff3 100644 --- a/crates/uv/tests/pip_install/pip_install.rs +++ b/crates/uv/tests/pip_install/pip_install.rs @@ -120,7 +120,8 @@ fn compile_bytecode_for_installed_distributions() { uv_snapshot!(context.pip_install() .arg("anyio==3.7.1") - .arg("--compile-bytecode"), @" + .arg("--compile-bytecode") + .env(EnvVars::UV_CONCURRENT_INSTALLS, "1"), @" success: true exit_code: 0 ----- stdout ----- From 7ddda297737d786bb1235d1753d8823634db1c40 Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Thu, 18 Jun 2026 18:07:10 -0500 Subject: [PATCH 3/7] Stream installed bytecode targets from RECORD --- crates/uv-install-wheel/src/lib.rs | 2 +- crates/uv-install-wheel/src/wheel.rs | 9 ++- crates/uv-installer/src/compile.rs | 32 ++++++-- crates/uv/src/commands/mod.rs | 9 +-- crates/uv/src/commands/pip/operations.rs | 71 +++++++++-------- crates/uv/tests/pip_install/pip_install.rs | 89 +++++++++++++++++++++- 6 files changed, 163 insertions(+), 49 deletions(-) diff --git a/crates/uv-install-wheel/src/lib.rs b/crates/uv-install-wheel/src/lib.rs index 23c2424b7d5..eb3a58a9328 100644 --- a/crates/uv-install-wheel/src/lib.rs +++ b/crates/uv-install-wheel/src/lib.rs @@ -15,7 +15,7 @@ pub use install::{install_wheel, installed_dist_info_path}; pub use linker::{InstallState, LinkMode}; pub use record::RecordEntry; pub use uninstall::{Uninstall, uninstall_egg, uninstall_legacy_editable, uninstall_wheel}; -pub use wheel::{WheelFile, read_record, validate_and_heal_record}; +pub use wheel::{WheelFile, read_record, read_record_iter, validate_and_heal_record}; mod install; mod linker; diff --git a/crates/uv-install-wheel/src/wheel.rs b/crates/uv-install-wheel/src/wheel.rs index ab1b5a42e5a..1db022b5386 100644 --- a/crates/uv-install-wheel/src/wheel.rs +++ b/crates/uv-install-wheel/src/wheel.rs @@ -841,12 +841,12 @@ fn get_relocatable_executable( /// Reads the record file /// -pub fn read_record(record: impl Read) -> Result, Error> { +pub fn read_record_iter(record: impl Read) -> impl Iterator> { csv::ReaderBuilder::new() .has_headers(false) .escape(Some(b'"')) .from_reader(record) - .deserialize() + .into_deserialize() .map(|entry| { let entry: RecordEntry = entry?; Ok(RecordEntry { @@ -855,7 +855,10 @@ pub fn read_record(record: impl Read) -> Result, Error> { ..entry }) }) - .collect() +} + +pub fn read_record(record: impl Read) -> Result, Error> { + read_record_iter(record).collect() } pub(crate) fn write_record( diff --git a/crates/uv-installer/src/compile.rs b/crates/uv-installer/src/compile.rs index 4cb9131172f..f3bbe4305b6 100644 --- a/crates/uv-installer/src/compile.rs +++ b/crates/uv-installer/src/compile.rs @@ -31,6 +31,8 @@ pub enum CompileError { Walkdir(#[from] walkdir::Error), #[error("Failed to send task to worker")] WorkerDisappeared(SendError), + #[error("Failed to identify Python source files")] + SourceFiles(#[source] anyhow::Error), #[error("The task executor is broken, did some other task panic?")] Join, #[error("Failed to start Python interpreter to run compile script")] @@ -250,16 +252,21 @@ pub async fn compile_tree( /// or communicate with the Python workers are returned. #[instrument(skip(files, python_executable))] pub async fn compile_files( - files: &[PathBuf], + files: impl IntoIterator>, python_executable: &Path, concurrency: &Concurrency, cache: &Path, ) -> Result { - if files.is_empty() { + let mut files = files.into_iter(); + let mut initial_files = Vec::with_capacity(concurrency.installs); + for file in files.by_ref().take(concurrency.installs) { + initial_files.push(file.map_err(CompileError::SourceFiles)?); + } + if initial_files.is_empty() { return Ok(0); } - let worker_count = concurrency.installs.min(files.len()); + let worker_count = initial_files.len(); let (sender, receiver) = async_channel::bounded::(worker_count * 10); // Running Python with an actual file will produce better error messages. @@ -277,13 +284,23 @@ pub async fn compile_files( drop(receiver); let mut send_error = None; - for file in files { + let mut source_error = None; + let mut source_files = 0; + for file in initial_files.into_iter().map(Ok).chain(files) { + let file = match file { + Ok(file) => file, + Err(err) => { + source_error = Some(err); + break; + } + }; debug_assert!( file.is_absolute(), "compileall doesn't work with relative paths: `{}`", file.display() ); - if let Err(err) = sender.send(file.clone()).await { + source_files += 1; + if let Err(err) = sender.send(file).await { send_error = Some(err); break; } @@ -291,8 +308,11 @@ pub async fn compile_files( drop(sender); wait_for_workers(worker_handles, send_error).await?; + if let Some(source_error) = source_error { + return Err(CompileError::SourceFiles(source_error)); + } - Ok(files.len()) + Ok(source_files) } async fn worker( diff --git a/crates/uv/src/commands/mod.rs b/crates/uv/src/commands/mod.rs index 6c475958ee4..f19e1aa2b1a 100644 --- a/crates/uv/src/commands/mod.rs +++ b/crates/uv/src/commands/mod.rs @@ -287,20 +287,19 @@ pub(super) async fn compile_bytecode( /// Compile the given Python source files to bytecode. pub(super) async fn compile_bytecode_files( - files: &[PathBuf], + files: impl IntoIterator>, venv: &PythonEnvironment, concurrency: &Concurrency, cache: &Cache, printer: Printer, ) -> anyhow::Result<()> { - if files.is_empty() { - return Ok(()); - } - let start = std::time::Instant::now(); let files = compile_files(files, venv.python_executable(), concurrency, cache.root()) .await .context("Failed to bytecode-compile installed packages")?; + if files == 0 { + return Ok(()); + } write_bytecode_summary(files, start, printer)?; Ok(()) diff --git a/crates/uv/src/commands/pip/operations.rs b/crates/uv/src/commands/pip/operations.rs index 08c909c9bed..f08038b1f31 100644 --- a/crates/uv/src/commands/pip/operations.rs +++ b/crates/uv/src/commands/pip/operations.rs @@ -25,7 +25,7 @@ use uv_distribution_types::{ }; use uv_distribution_types::{DistributionMetadata, InstalledMetadata, Name, Resolution}; use uv_fs::{CWD, Simplified, normalize_path_under}; -use uv_install_wheel::{LinkMode, installed_dist_info_path, read_record}; +use uv_install_wheel::{LinkMode, installed_dist_info_path, read_record_iter}; use uv_installer::{InstallationStrategy, Plan, Planner, Preparer, SitePackages}; use uv_normalize::PackageName; use uv_pep440::Version; @@ -717,8 +717,8 @@ pub(crate) async fn install( compile_bytecode(venv, concurrency, cache, printer).await?; } BytecodeCompilation::Installed => { - let files = python_source_files_for_installs(venv, &installs)?; - compile_bytecode_files(&files, venv, concurrency, cache, printer).await?; + let files = python_source_files_for_installs(venv, &installs); + compile_bytecode_files(files, venv, concurrency, cache, printer).await?; } } } @@ -732,46 +732,53 @@ pub(crate) async fn install( Ok(changelog) } +type PythonSourceFileIterator = Box>>; + /// Return the Python source files owned by the distributions installed by this operation. -fn python_source_files_for_installs( - venv: &PythonEnvironment, - installs: &[CachedDist], -) -> anyhow::Result> { +fn python_source_files_for_installs<'a>( + venv: &'a PythonEnvironment, + installs: &'a [CachedDist], +) -> impl Iterator> + 'a { let layout = venv.interpreter().layout(); let site_packages = [ CWD.join(&layout.scheme.purelib), CWD.join(&layout.scheme.platlib), ]; - let mut files = BTreeSet::new(); - - for install in installs { - let dist_info = installed_dist_info_path(&layout, install.path()).with_context(|| { + installs.iter().flat_map(move |install| { + let dist_info = match installed_dist_info_path(&layout, install.path()).with_context(|| { format!("Failed to locate installed distribution for bytecode compilation: `{install}`") - })?; - let record_root = dist_info.parent().with_context(|| { - format!( + }) { + Ok(dist_info) => dist_info, + Err(err) => return Box::new(std::iter::once(Err(err))) as PythonSourceFileIterator, + }; + let Some(record_root) = dist_info.parent().map(Path::to_path_buf) else { + return Box::new(std::iter::once(Err(anyhow!( "Invalid installed distribution path: `{}`", dist_info.user_display() - ) - })?; + )))); + }; let record_path = dist_info.join("RECORD"); - let record = read_record(fs_err::File::open(&record_path)?) - .with_context(|| format!("Failed to read `{}`", record_path.user_display()))?; - - for entry in record { - let Some(path) = - python_source_path_from_record(record_root, &entry.path, &site_packages) - else { - continue; + let record_file = match fs_err::File::open(&record_path) + .with_context(|| format!("Failed to read `{}`", record_path.user_display())) + { + Ok(record_file) => record_file, + Err(err) => return Box::new(std::iter::once(Err(err))), + }; + let site_packages = site_packages.clone(); + + Box::new(read_record_iter(record_file).filter_map(move |entry| { + let entry = match entry { + Ok(entry) => entry, + Err(err) => { + return Some(Err(err).with_context(|| { + format!("Failed to read `{}`", record_path.user_display()) + })); + } }; - if !path.is_file() { - continue; - } - files.insert(path); - } - } - - Ok(files.into_iter().collect()) + let path = python_source_path_from_record(&record_root, &entry.path, &site_packages)?; + path.is_file().then_some(Ok(path)) + })) + }) } /// Resolve a Python source path from an installed `RECORD` entry. diff --git a/crates/uv/tests/pip_install/pip_install.rs b/crates/uv/tests/pip_install/pip_install.rs index b45c7f13ff3..5b1d4dead85 100644 --- a/crates/uv/tests/pip_install/pip_install.rs +++ b/crates/uv/tests/pip_install/pip_install.rs @@ -1,5 +1,6 @@ +use std::fmt::Write; use std::io::Cursor; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::process::Command; use anyhow::{Result, anyhow}; @@ -55,6 +56,47 @@ fn write_tar_gz(file: File, entries: &[(&str, &str)]) -> Result<()> { Ok(()) } +fn write_many_files_wheel(path: &Path, source_files: usize) -> Result<()> { + let mut writer = ZipFileWriter::new(Vec::new()); + let mut record = String::new(); + + for index in 0..source_files { + let name = format!("large_wheel/module_{index:05}.py"); + let entry = ZipEntryBuilder::new(name.clone().into(), Compression::Stored); + block_on(writer.write_entry_whole(entry, b"VALUE = 1\n"))?; + writeln!(record, "{name},,")?; + } + + let metadata = indoc! {" + Metadata-Version: 2.1 + Name: large-wheel + Version: 1.0.0 + "}; + let wheel = indoc! {" + Wheel-Version: 1.0 + Generator: uv-test + Root-Is-Purelib: true + Tag: py3-none-any + "}; + for (name, contents) in [ + ("large_wheel-1.0.0.dist-info/METADATA", metadata), + ("large_wheel-1.0.0.dist-info/WHEEL", wheel), + ] { + let entry = ZipEntryBuilder::new(name.into(), Compression::Stored); + block_on(writer.write_entry_whole(entry, contents.as_bytes()))?; + writeln!(record, "{name},,")?; + } + record.push_str("large_wheel-1.0.0.dist-info/RECORD,,\n"); + let entry = ZipEntryBuilder::new( + "large_wheel-1.0.0.dist-info/RECORD".into(), + Compression::Stored, + ); + block_on(writer.write_entry_whole(entry, record.as_bytes()))?; + + fs_err::write(path, block_on(writer.close())?)?; + Ok(()) +} + #[test] fn missing_requirements_txt() { let context = uv_test::test_context!("3.12"); @@ -101,8 +143,13 @@ fn empty_requirements_txt() -> Result<()> { /// Compile only distributions installed by the current operation. #[test] -fn compile_bytecode_for_installed_distributions() { +fn compile_bytecode_for_installed_distributions() -> Result<()> { + const SOURCE_FILES: usize = 16; + let context = uv_test::test_context!("3.12"); + let wheel = context.temp_dir.join("large_wheel-1.0.0-py3-none-any.whl"); + // This exceeds the one-worker compilation queue capacity, exercising producer backpressure. + write_many_files_wheel(&wheel, SOURCE_FILES)?; uv_snapshot!(context.pip_install() .arg("sniffio==1.3.1"), @" @@ -161,6 +208,42 @@ fn compile_bytecode_for_installed_distributions() { .exists() ); + uv_snapshot!(context.filters(), context.pip_install() + .arg(&wheel) + .arg("--compile-bytecode") + .env(EnvVars::UV_CONCURRENT_INSTALLS, "1"), @" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 1 package in [TIME] + Prepared 1 package in [TIME] + Installed 1 package in [TIME] + Bytecode compiled 16 files in [TIME] + + large-wheel==1.0.0 (from file://[TEMP_DIR]/large_wheel-1.0.0-py3-none-any.whl) + " + ); + + let compiled = WalkDir::new(context.site_packages().join("large_wheel")) + .into_iter() + .filter_map(Result::ok) + .filter(|entry| { + entry + .path() + .extension() + .is_some_and(|extension| extension == "pyc") + }) + .count(); + assert_eq!(compiled, SOURCE_FILES); + assert!( + !context + .site_packages() + .join("sniffio") + .join("__pycache__") + .exists() + ); + uv_snapshot!(context.pip_install() .arg("sniffio==1.3.1") .arg("--reinstall-package") @@ -188,6 +271,8 @@ fn compile_bytecode_for_installed_distributions() { .join("__init__.cpython-312.pyc") .exists() ); + + Ok(()) } /// Compile symlinked source files installed by the current operation. From 1c6ece2f1f92fdbb580d5556abff14fe6e728dd1 Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Mon, 6 Jul 2026 14:10:58 -0500 Subject: [PATCH 4/7] Document wheel destination helper --- crates/uv-install-wheel/src/install.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/uv-install-wheel/src/install.rs b/crates/uv-install-wheel/src/install.rs index ea7d7194047..6a4ee177d19 100644 --- a/crates/uv-install-wheel/src/install.rs +++ b/crates/uv-install-wheel/src/install.rs @@ -27,6 +27,7 @@ pub fn installed_dist_info_path( Ok(site_packages.join(format!("{dist_info_prefix}.dist-info"))) } +/// Return the wheel's `.dist-info` prefix and target `site-packages` directory. fn wheel_destination<'layout>( layout: &'layout Layout, wheel: &Path, From 424bc90de654bdf0b517eb830c8484e3da9dfff2 Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Mon, 6 Jul 2026 14:59:25 -0500 Subject: [PATCH 5/7] Handle bytecode compilation edge cases --- crates/uv-cli/src/lib.rs | 21 ++++--- crates/uv/src/commands/pip/operations.rs | 16 +++-- crates/uv/tests/pip_install/pip_install.rs | 68 ++++++++++++++++++++++ 3 files changed, 91 insertions(+), 14 deletions(-) diff --git a/crates/uv-cli/src/lib.rs b/crates/uv-cli/src/lib.rs index bffce38d89c..35f71f8d5d0 100644 --- a/crates/uv-cli/src/lib.rs +++ b/crates/uv-cli/src/lib.rs @@ -6397,9 +6397,10 @@ pub struct ToolUpgradeArgs { /// in which start time is critical, such as CLI applications and Docker containers, this option /// can be enabled to trade longer installation times for faster start times. /// - /// When enabled, uv will process the entire site-packages directory (including packages that - /// are not being modified by the current operation) for consistency. Like pip, it will also - /// ignore errors. + /// When enabled, uv will compile the installed Python files required by the command. `uv pip + /// install` limits compilation to packages installed by the current operation, while project + /// and environment sync commands process the entire site-packages directory. Like pip, + /// compilation errors are ignored. #[arg( long, alias = "compile", @@ -7468,9 +7469,10 @@ pub struct InstallerArgs { /// in which start time is critical, such as CLI applications and Docker containers, this option /// can be enabled to trade longer installation times for faster start times. /// - /// When enabled, uv will process the entire site-packages directory (including packages that - /// are not being modified by the current operation) for consistency. Like pip, it will also - /// ignore errors. + /// When enabled, uv will compile the installed Python files required by the command. `uv pip + /// install` limits compilation to packages installed by the current operation, while project + /// and environment sync commands process the entire site-packages directory. Like pip, + /// compilation errors are ignored. #[arg( long, alias = "compile", @@ -7965,9 +7967,10 @@ pub struct ResolverInstallerArgs { /// in which start time is critical, such as CLI applications and Docker containers, this option /// can be enabled to trade longer installation times for faster start times. /// - /// When enabled, uv will process the entire site-packages directory (including packages that - /// are not being modified by the current operation) for consistency. Like pip, it will also - /// ignore errors. + /// When enabled, uv will compile the installed Python files required by the command. `uv pip + /// install` limits compilation to packages installed by the current operation, while project + /// and environment sync commands process the entire site-packages directory. Like pip, + /// compilation errors are ignored. #[arg( long, alias = "compile", diff --git a/crates/uv/src/commands/pip/operations.rs b/crates/uv/src/commands/pip/operations.rs index f08038b1f31..2ad0ef56416 100644 --- a/crates/uv/src/commands/pip/operations.rs +++ b/crates/uv/src/commands/pip/operations.rs @@ -751,18 +751,24 @@ fn python_source_files_for_installs<'a>( Ok(dist_info) => dist_info, Err(err) => return Box::new(std::iter::once(Err(err))) as PythonSourceFileIterator, }; - let Some(record_root) = dist_info.parent().map(Path::to_path_buf) else { + let Some(record_root) = dist_info.parent().map(|path| CWD.join(path)) else { return Box::new(std::iter::once(Err(anyhow!( "Invalid installed distribution path: `{}`", dist_info.user_display() )))); }; let record_path = dist_info.join("RECORD"); - let record_file = match fs_err::File::open(&record_path) - .with_context(|| format!("Failed to read `{}`", record_path.user_display())) - { + let record_file = match fs_err::File::open(&record_path) { Ok(record_file) => record_file, - Err(err) => return Box::new(std::iter::once(Err(err))), + // Another process may have removed the installed distribution. + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + return Box::new(std::iter::empty()); + } + Err(err) => { + return Box::new(std::iter::once(Err(err).with_context(|| { + format!("Failed to read `{}`", record_path.user_display()) + }))); + } }; let site_packages = site_packages.clone(); diff --git a/crates/uv/tests/pip_install/pip_install.rs b/crates/uv/tests/pip_install/pip_install.rs index 5b1d4dead85..e9d8dde9cb3 100644 --- a/crates/uv/tests/pip_install/pip_install.rs +++ b/crates/uv/tests/pip_install/pip_install.rs @@ -309,6 +309,74 @@ fn compile_bytecode_with_symlink_link_mode() { ); } +/// Compile bytecode when installing into a relative `--target` or `--prefix` path. +#[test] +fn compile_bytecode_for_relative_install_root() { + let context = uv_test::test_context!("3.12") + .with_filtered_python_names() + .with_filtered_virtualenv_bin() + .with_filtered_exe_suffix(); + + uv_snapshot!(context.filters(), context.pip_install() + .arg("sniffio==1.3.1") + .arg("--target") + .arg("target") + .arg("--compile-bytecode"), @" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Using CPython 3.12.[X] interpreter at: .venv/[BIN]/[PYTHON] + Resolved 1 package in [TIME] + Prepared 1 package in [TIME] + Installed 1 package in [TIME] + Bytecode compiled 5 files in [TIME] + + sniffio==1.3.1 + " + ); + + assert!( + context + .temp_dir + .join("target") + .join("sniffio") + .join("__pycache__") + .join("__init__.cpython-312.pyc") + .exists() + ); + + uv_snapshot!(context.filters(), context.pip_install() + .arg("sniffio==1.3.1") + .arg("--prefix") + .arg("prefix") + .arg("--compile-bytecode"), @" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Using CPython 3.12.[X] interpreter at: .venv/[BIN]/[PYTHON] + Resolved 1 package in [TIME] + Installed 1 package in [TIME] + Bytecode compiled 5 files in [TIME] + + sniffio==1.3.1 + " + ); + + let compiled = WalkDir::new(context.temp_dir.join("prefix")) + .into_iter() + .filter_map(Result::ok) + .filter(|entry| { + entry + .path() + .extension() + .is_some_and(|extension| extension == "pyc") + }) + .count(); + assert_eq!(compiled, 5); +} + #[test] fn missing_pyproject_toml() { let context = uv_test::test_context!("3.12"); From 8676bd4fc9f87c6bdbfb7c32ac04addac22290db Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Tue, 7 Jul 2026 08:41:45 -0500 Subject: [PATCH 6/7] Clarify bytecode compilation help --- crates/uv-cli/src/lib.rs | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/crates/uv-cli/src/lib.rs b/crates/uv-cli/src/lib.rs index 35f71f8d5d0..32bb010e538 100644 --- a/crates/uv-cli/src/lib.rs +++ b/crates/uv-cli/src/lib.rs @@ -6397,10 +6397,10 @@ pub struct ToolUpgradeArgs { /// in which start time is critical, such as CLI applications and Docker containers, this option /// can be enabled to trade longer installation times for faster start times. /// - /// When enabled, uv will compile the installed Python files required by the command. `uv pip - /// install` limits compilation to packages installed by the current operation, while project - /// and environment sync commands process the entire site-packages directory. Like pip, - /// compilation errors are ignored. + /// When enabled, install operations (e.g., `uv pip install`) will compile installed or + /// reinstalled Python files. Commands that perform a sync operation (e.g., `uv sync` or `uv + /// run`) will process the entire site-packages directory including packages that are not being + /// modified. #[arg( long, alias = "compile", @@ -7469,10 +7469,10 @@ pub struct InstallerArgs { /// in which start time is critical, such as CLI applications and Docker containers, this option /// can be enabled to trade longer installation times for faster start times. /// - /// When enabled, uv will compile the installed Python files required by the command. `uv pip - /// install` limits compilation to packages installed by the current operation, while project - /// and environment sync commands process the entire site-packages directory. Like pip, - /// compilation errors are ignored. + /// When enabled, install operations (e.g., `uv pip install`) will compile installed or + /// reinstalled Python files. Commands that perform a sync operation (e.g., `uv sync` or `uv + /// run`) will process the entire site-packages directory including packages that are not being + /// modified. #[arg( long, alias = "compile", @@ -7967,10 +7967,10 @@ pub struct ResolverInstallerArgs { /// in which start time is critical, such as CLI applications and Docker containers, this option /// can be enabled to trade longer installation times for faster start times. /// - /// When enabled, uv will compile the installed Python files required by the command. `uv pip - /// install` limits compilation to packages installed by the current operation, while project - /// and environment sync commands process the entire site-packages directory. Like pip, - /// compilation errors are ignored. + /// When enabled, install operations (e.g., `uv pip install`) will compile installed or + /// reinstalled Python files. Commands that perform a sync operation (e.g., `uv sync` or `uv + /// run`) will process the entire site-packages directory including packages that are not being + /// modified. #[arg( long, alias = "compile", From cf63fb24d2941510b35b760ec1b053ddb3d1d5d1 Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Tue, 7 Jul 2026 09:44:18 -0500 Subject: [PATCH 7/7] Address bytecode compilation review feedback --- crates/uv-install-wheel/src/lib.rs | 2 +- crates/uv-install-wheel/src/wheel.rs | 6 ++++-- crates/uv-installer/src/compile.rs | 2 +- crates/uv/src/commands/pip/operations.rs | 4 ++-- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/crates/uv-install-wheel/src/lib.rs b/crates/uv-install-wheel/src/lib.rs index eb3a58a9328..fe3942e59c5 100644 --- a/crates/uv-install-wheel/src/lib.rs +++ b/crates/uv-install-wheel/src/lib.rs @@ -15,7 +15,7 @@ pub use install::{install_wheel, installed_dist_info_path}; pub use linker::{InstallState, LinkMode}; pub use record::RecordEntry; pub use uninstall::{Uninstall, uninstall_egg, uninstall_legacy_editable, uninstall_wheel}; -pub use wheel::{WheelFile, read_record, read_record_iter, validate_and_heal_record}; +pub use wheel::{WheelFile, read_record, read_record_into_iter, validate_and_heal_record}; mod install; mod linker; diff --git a/crates/uv-install-wheel/src/wheel.rs b/crates/uv-install-wheel/src/wheel.rs index 1db022b5386..8fb4690391d 100644 --- a/crates/uv-install-wheel/src/wheel.rs +++ b/crates/uv-install-wheel/src/wheel.rs @@ -841,7 +841,9 @@ fn get_relocatable_executable( /// Reads the record file /// -pub fn read_record_iter(record: impl Read) -> impl Iterator> { +pub fn read_record_into_iter( + record: impl Read, +) -> impl Iterator> { csv::ReaderBuilder::new() .has_headers(false) .escape(Some(b'"')) @@ -858,7 +860,7 @@ pub fn read_record_iter(record: impl Read) -> impl Iterator Result, Error> { - read_record_iter(record).collect() + read_record_into_iter(record).collect() } pub(crate) fn write_record( diff --git a/crates/uv-installer/src/compile.rs b/crates/uv-installer/src/compile.rs index f3bbe4305b6..7904f977b8c 100644 --- a/crates/uv-installer/src/compile.rs +++ b/crates/uv-installer/src/compile.rs @@ -135,11 +135,11 @@ fn spawn_workers( worker_handles } +/// Wait for all workers to exit so worker failures are not hidden by channel send errors. async fn wait_for_workers( worker_handles: Vec, send_error: Option>, ) -> Result<(), CompileError> { - // Make sure all workers exit regularly, avoid hiding errors. for result in futures::future::join_all(worker_handles).await { match result { // A worker thread panicked or exited without reporting its result. diff --git a/crates/uv/src/commands/pip/operations.rs b/crates/uv/src/commands/pip/operations.rs index 2ad0ef56416..68f98cfd6fd 100644 --- a/crates/uv/src/commands/pip/operations.rs +++ b/crates/uv/src/commands/pip/operations.rs @@ -25,7 +25,7 @@ use uv_distribution_types::{ }; use uv_distribution_types::{DistributionMetadata, InstalledMetadata, Name, Resolution}; use uv_fs::{CWD, Simplified, normalize_path_under}; -use uv_install_wheel::{LinkMode, installed_dist_info_path, read_record_iter}; +use uv_install_wheel::{LinkMode, installed_dist_info_path, read_record_into_iter}; use uv_installer::{InstallationStrategy, Plan, Planner, Preparer, SitePackages}; use uv_normalize::PackageName; use uv_pep440::Version; @@ -772,7 +772,7 @@ fn python_source_files_for_installs<'a>( }; let site_packages = site_packages.clone(); - Box::new(read_record_iter(record_file).filter_map(move |entry| { + Box::new(read_record_into_iter(record_file).filter_map(move |entry| { let entry = match entry { Ok(entry) => entry, Err(err) => {