diff --git a/crates/uv-python/src/discovery.rs b/crates/uv-python/src/discovery.rs index 29659116ccdda..2fe6609493776 100644 --- a/crates/uv-python/src/discovery.rs +++ b/crates/uv-python/src/discovery.rs @@ -1,5 +1,7 @@ use itertools::{Either, Itertools}; +use owo_colors::AnsiColors; use regex::Regex; +use reqwest_retry::policies::ExponentialBackoff; use rustc_hash::{FxBuildHasher, FxHashSet}; use same_file::is_same_file; use std::borrow::Cow; @@ -10,6 +12,7 @@ use std::{path::Path, path::PathBuf, str::FromStr}; use thiserror::Error; use tracing::{debug, instrument, trace}; use uv_cache::Cache; +use uv_client::BaseClient; use uv_fs::Simplified; use uv_fs::which::is_executable; use uv_pep440::{ @@ -18,10 +21,11 @@ use uv_pep440::{ }; use uv_preview::Preview; use uv_static::EnvVars; +use uv_warnings::anstream; use uv_warnings::warn_user_once; use which::{which, which_all}; -use crate::downloads::{PlatformRequest, PythonDownloadRequest}; +use crate::downloads::{ManagedPythonDownloadList, PlatformRequest, PythonDownloadRequest}; use crate::implementation::ImplementationName; use crate::installation::PythonInstallation; use crate::interpreter::Error as InterpreterError; @@ -1443,34 +1447,31 @@ pub(crate) fn find_python_installation( /// without comparing the patch version number. If that cannot be found, we fall back to /// the first available version. /// +/// At all points, if the specified version cannot be found, we will attempt to +/// download it if downloads are enabled. +/// /// See [`find_python_installation`] for more details on installation discovery. #[instrument(skip_all, fields(request))] -pub(crate) fn find_best_python_installation( +pub(crate) async fn find_best_python_installation( request: &PythonRequest, environments: EnvironmentPreference, preference: PythonPreference, + downloads_enabled: bool, + download_list: &ManagedPythonDownloadList, + client: &BaseClient, + retry_policy: &ExponentialBackoff, cache: &Cache, + reporter: Option<&dyn crate::downloads::Reporter>, + python_install_mirror: Option<&str>, + pypy_install_mirror: Option<&str>, preview: Preview, -) -> Result { - debug!("Starting Python discovery for {}", request); +) -> Result { + debug!("Starting Python discovery for {request}"); + let original_request = request; - // First, check for an exact match (or the first available version if no Python version was provided) - debug!("Looking for exact match for request {request}"); - let result = find_python_installation(request, environments, preference, cache, preview); - match result { - Ok(Ok(installation)) => { - warn_on_unsupported_python(installation.interpreter()); - return Ok(Ok(installation)); - } - // Continue if we can't find a matching Python and ignore non-critical discovery errors - Ok(Err(_)) => {} - Err(ref err) if !err.is_critical() => {} - _ => return result, - } + let mut previous_fetch_failed = false; - // If that fails, and a specific patch version was requested try again allowing a - // different patch version - if let Some(request) = match request { + let request_without_patch = match request { PythonRequest::Version(version) => { if version.has_patch() { Some(PythonRequest::Version(version.clone().without_patch())) @@ -1482,36 +1483,119 @@ pub(crate) fn find_best_python_installation( PythonRequest::ImplementationVersion(*implementation, version.clone().without_patch()), ), _ => None, - } { - debug!("Looking for relaxed patch version {request}"); - let result = find_python_installation(&request, environments, preference, cache, preview); - match result { + }; + + for (attempt, request) in iter::once(original_request) + .chain(request_without_patch.iter()) + .chain(iter::once(&PythonRequest::Default)) + .enumerate() + { + debug!( + "Looking for {request}{}", + if request != original_request { + format!(" attempt {attempt} (fallback after failing to find: {original_request})") + } else { + String::new() + } + ); + let result = find_python_installation(request, environments, preference, cache, preview); + let error = match result { Ok(Ok(installation)) => { warn_on_unsupported_python(installation.interpreter()); - return Ok(Ok(installation)); + return Ok(installation); } // Continue if we can't find a matching Python and ignore non-critical discovery errors - Ok(Err(_)) => {} - Err(ref err) if !err.is_critical() => {} - _ => return result, + Ok(Err(error)) => error.into(), + Err(error) if !error.is_critical() => error.into(), + Err(error) => return Err(error.into()), + }; + + // Attempt to download the version if downloads are enabled + if downloads_enabled + && !previous_fetch_failed + && let Some(download_request) = PythonDownloadRequest::from_request(request) + { + let download = download_request + .clone() + .fill() + .map(|request| download_list.find(&request)); + + let result = match download { + Ok(Ok(download)) => PythonInstallation::fetch( + download, + client, + retry_policy, + cache, + reporter, + python_install_mirror, + pypy_install_mirror, + preview, + ) + .await + .map(Some), + Ok(Err(crate::downloads::Error::NoDownloadFound(_))) => Ok(None), + Ok(Err(error)) => Err(error.into()), + Err(error) => Err(error.into()), + }; + if let Ok(Some(installation)) = result { + return Ok(installation); + } + // Emit a warning instead of failing since we may find a suitable + // interpreter on the system after relaxing the request further. + // Additionally, uv did not previously attempt downloads in this + // code path and we want to minimize the fatal cases for + // backwards compatibility. + // Errors encountered here are either network errors or quirky + // configuration problems. + if let Err(error) = result { + // This is a hack to get `write_error_chain` to format things the way we want. + #[derive(Debug, thiserror::Error)] + #[error( + "A managed Python download is available for {0}, but an error occurred when attempting to download it." + )] + struct WrappedError<'a>(&'a PythonRequest, #[source] crate::Error); + + // If the request was for the default or any version, propagate + // the error as nothing else we are about to do will help the + // situation. + if matches!(request, PythonRequest::Default | PythonRequest::Any) { + return Err(error); + } + + let mut error_chain = String::new(); + // Writing to a string can't fail with errors (panics on allocation failure) + uv_warnings::write_error_chain( + &WrappedError(request, error), + &mut error_chain, + "warning", + AnsiColors::Yellow, + ) + .unwrap(); + anstream::eprint!("{}", error_chain); + previous_fetch_failed = true; + } } - } - // If a Python version was requested but cannot be fulfilled, just take any version - debug!("Looking for a default Python installation"); - let request = PythonRequest::Default; - Ok( - find_python_installation(&request, environments, preference, cache, preview)?.map_err( - |err| { - // Use a more general error in this case since we looked for multiple versions - PythonNotFound { - request, + // If this was a request for the Default or Any version, this means that + // either that's what we were called with, or we're on the last + // iteration. + // + // The most recent find error therefore becomes a fatal one. + if matches!(request, PythonRequest::Default | PythonRequest::Any) { + return Err(match error { + crate::Error::MissingPython(err, _) => PythonNotFound { + // Use a more general error in this case since we looked for multiple versions + request: original_request.clone(), python_preference: err.python_preference, environment_preference: err.environment_preference, } - }, - ), - ) + .into(), + other => other, + }); + } + } + + unreachable!("The loop should have terminated when it reached PythonRequest::Default"); } /// Display a warning if the Python version of the [`Interpreter`] is unsupported by uv. diff --git a/crates/uv-python/src/installation.rs b/crates/uv-python/src/installation.rs index 2cc751bdae756..07506b5e1861c 100644 --- a/crates/uv-python/src/installation.rs +++ b/crates/uv-python/src/installation.rs @@ -73,19 +73,44 @@ impl PythonInstallation { Ok(installation) } - /// Find an installed [`PythonInstallation`] that satisfies a requested version, if the request cannot - /// be satisfied, fallback to the best available Python installation. - pub fn find_best( + /// Find or download a [`PythonInstallation`] that satisfies a requested version, if the request + /// cannot be satisfied, fallback to the best available Python installation. + pub async fn find_best( request: &PythonRequest, environments: EnvironmentPreference, preference: PythonPreference, - download_list: &ManagedPythonDownloadList, + python_downloads: PythonDownloads, + client_builder: &BaseClientBuilder<'_>, cache: &Cache, + reporter: Option<&dyn Reporter>, + python_install_mirror: Option<&str>, + pypy_install_mirror: Option<&str>, + python_downloads_json_url: Option<&str>, preview: Preview, ) -> Result { - let installation = - find_best_python_installation(request, environments, preference, cache, preview)??; - installation.warn_if_outdated_prerelease(request, download_list); + let retry_policy = client_builder.retry_policy(); + let client = client_builder.clone().retries(0).build(); + let download_list = + ManagedPythonDownloadList::new(&client, python_downloads_json_url).await?; + let downloads_enabled = preference.allows_managed() + && python_downloads.is_automatic() + && client_builder.connectivity.is_online(); + let installation = find_best_python_installation( + request, + environments, + preference, + downloads_enabled, + &download_list, + &client, + &retry_policy, + cache, + reporter, + python_install_mirror, + pypy_install_mirror, + preview, + ) + .await?; + installation.warn_if_outdated_prerelease(request, &download_list); Ok(installation) } diff --git a/crates/uv-python/src/lib.rs b/crates/uv-python/src/lib.rs index 5dbdd3dc26bf3..e71aac0c1997a 100644 --- a/crates/uv-python/src/lib.rs +++ b/crates/uv-python/src/lib.rs @@ -135,6 +135,7 @@ mod tests { use indoc::{formatdoc, indoc}; use temp_env::with_vars; use test_log::test; + use uv_client::BaseClientBuilder; use uv_preview::Preview; use uv_static::EnvVars; @@ -142,8 +143,9 @@ mod tests { use crate::{ PythonNotFound, PythonRequest, PythonSource, PythonVersion, - implementation::ImplementationName, installation::PythonInstallation, - managed::ManagedPythonInstallations, virtualenv::virtualenv_python_executable, + downloads::ManagedPythonDownloadList, implementation::ImplementationName, + installation::PythonInstallation, managed::ManagedPythonInstallations, + virtualenv::virtualenv_python_executable, }; use crate::{ PythonPreference, @@ -989,20 +991,49 @@ mod tests { Ok(()) } + fn find_best_python_installation_no_download( + request: &PythonRequest, + environments: EnvironmentPreference, + preference: PythonPreference, + cache: &Cache, + preview: Preview, + ) -> Result { + let client_builder = BaseClientBuilder::default(); + let download_list = ManagedPythonDownloadList::new_only_embedded()?; + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("Failed to build runtime") + .block_on(find_best_python_installation( + request, + environments, + preference, + false, + &download_list, + &client_builder.clone().retries(0).build(), + &client_builder.retry_policy(), + cache, + None, + None, + None, + preview, + )) + } + #[test] fn find_best_python_version_patch_exact() -> Result<()> { let mut context = TestContext::new()?; context.add_python_versions(&["3.10.1", "3.11.2", "3.11.4", "3.11.3", "3.12.5"])?; let python = context.run(|| { - find_best_python_installation( + find_best_python_installation_no_download( &PythonRequest::parse("3.11.3"), EnvironmentPreference::Any, PythonPreference::OnlySystem, &context.cache, Preview::default(), ) - })??; + })?; assert!( matches!( @@ -1029,14 +1060,14 @@ mod tests { context.add_python_versions(&["3.10.1", "3.11.2", "3.11.4", "3.11.3", "3.12.5"])?; let python = context.run(|| { - find_best_python_installation( + find_best_python_installation_no_download( &PythonRequest::parse("3.11.11"), EnvironmentPreference::Any, PythonPreference::OnlySystem, &context.cache, Preview::default(), ) - })??; + })?; assert!( matches!( @@ -1066,14 +1097,14 @@ mod tests { let python = context.run_with_vars(&[(EnvVars::VIRTUAL_ENV, Some(venv.as_os_str()))], || { - find_best_python_installation( + find_best_python_installation_no_download( &PythonRequest::parse("3.10"), EnvironmentPreference::Any, PythonPreference::OnlySystem, &context.cache, Preview::default(), ) - })??; + })?; assert!( matches!( python, @@ -1097,14 +1128,14 @@ mod tests { let python = context.run_with_vars(&[(EnvVars::VIRTUAL_ENV, Some(venv.as_os_str()))], || { - find_best_python_installation( + find_best_python_installation_no_download( &PythonRequest::parse("3.10.2"), EnvironmentPreference::Any, PythonPreference::OnlySystem, &context.cache, Preview::default(), ) - })??; + })?; assert!( matches!( python, diff --git a/crates/uv-warnings/src/lib.rs b/crates/uv-warnings/src/lib.rs index deb1df9e68b1b..2e6fbcc091346 100644 --- a/crates/uv-warnings/src/lib.rs +++ b/crates/uv-warnings/src/lib.rs @@ -94,7 +94,7 @@ pub fn write_error_chain( "{}{} {}", level.as_ref().color(color).bold(), ":".bold(), - err.to_string().trim() + err.to_string().trim().bold() )?; for source in iter::successors(err.source(), |&err| err.source()) { let msg = source.to_string(); diff --git a/crates/uv/src/commands/pip/compile.rs b/crates/uv/src/commands/pip/compile.rs index 9ca44540ffcc3..08d8d32d269e4 100644 --- a/crates/uv/src/commands/pip/compile.rs +++ b/crates/uv/src/commands/pip/compile.rs @@ -11,8 +11,6 @@ use owo_colors::OwoColorize; use rustc_hash::FxHashSet; use tracing::debug; -use uv_python::downloads::ManagedPythonDownloadList; - use uv_cache::Cache; use uv_client::{BaseClientBuilder, FlatIndexClient, RegistryClientBuilder}; use uv_configuration::{ @@ -297,15 +295,9 @@ pub(crate) async fn pip_compile( // Find an interpreter to use for building distributions let environment_preference = EnvironmentPreference::from_system_flag(system, false); let python_preference = python_preference.with_system_flag(system); - let client = client_builder.clone().retries(0).build(); - let download_list = ManagedPythonDownloadList::new( - &client, - install_mirrors.python_downloads_json_url.as_deref(), - ) - .await?; + let reporter = PythonDownloadReporter::single(printer); let interpreter = if let Some(python) = python.as_ref() { let request = PythonRequest::parse(python); - let reporter = PythonDownloadReporter::single(printer); PythonInstallation::find_or_download( Some(&request), environment_preference, @@ -333,10 +325,16 @@ pub(crate) async fn pip_compile( &request, environment_preference, python_preference, - &download_list, + python_downloads, + &client_builder, &cache, + Some(&reporter), + install_mirrors.python_install_mirror.as_deref(), + install_mirrors.pypy_install_mirror.as_deref(), + install_mirrors.python_downloads_json_url.as_deref(), preview, ) + .await }? .into_interpreter(); diff --git a/crates/uv/tests/it/pip_compile.rs b/crates/uv/tests/it/pip_compile.rs index 51ab8739b2f94..a695185837053 100644 --- a/crates/uv/tests/it/pip_compile.rs +++ b/crates/uv/tests/it/pip_compile.rs @@ -8,6 +8,7 @@ use anyhow::Result; use assert_fs::prelude::*; use flate2::write::GzEncoder; use fs_err::File; +use http::StatusCode; use indoc::indoc; use url::Url; use wiremock::matchers::{method, path}; @@ -18110,3 +18111,214 @@ fn compile_missing_python() -> Result<()> { Ok(()) } + +#[cfg(feature = "python-managed")] +#[test] +fn compile_missing_python_version() -> Result<()> { + let context = TestContext::new("3.12") + .with_python_download_cache() + .with_managed_python_dirs(); + + let requirements_in = context.temp_dir.child("requirements.in"); + requirements_in.write_str("anyio==3.7.0")?; + + uv_snapshot!(context + .pip_compile() + .arg("--python-version").arg("3.13") + .arg("requirements.in"), @r" + success: true + exit_code: 0 + ----- stdout ----- + # This file was autogenerated by uv via the following command: + # uv pip compile --cache-dir [CACHE_DIR] --python-version 3.13 requirements.in + anyio==3.7.0 + # via -r requirements.in + idna==3.6 + # via anyio + sniffio==1.3.1 + # via anyio + + ----- stderr ----- + Resolved 3 packages in [TIME] + "); + + Ok(()) +} + +#[cfg(feature = "python-managed")] +#[test] +fn compile_missing_python_version_patch_fallback() -> Result<()> { + let context = TestContext::new("3.12") + .with_python_download_cache() + .with_managed_python_dirs() + // Filter the patch of the version which will get downloaded + .with_filter(( + r"(3\.13)\.\d+ will be used", + "$1.[X] will be used".to_string(), + )); + + let requirements_in = context.temp_dir.child("requirements.in"); + requirements_in.write_str("anyio==3.7.0")?; + + uv_snapshot!(context.filters(), context + .pip_compile() + .arg("--python-version").arg("3.13.99") + .arg("requirements.in"), @r" + success: true + exit_code: 0 + ----- stdout ----- + # This file was autogenerated by uv via the following command: + # uv pip compile --cache-dir [CACHE_DIR] --python-version 3.13.99 requirements.in + anyio==3.7.0 + # via -r requirements.in + idna==3.6 + # via anyio + sniffio==1.3.1 + # via anyio + + ----- stderr ----- + warning: The requested Python version 3.13.99 is not available; 3.13.[X] will be used to build dependencies instead. + Resolved 3 packages in [TIME] + "); + + Ok(()) +} + +#[cfg(feature = "python-managed")] +#[test] +fn compile_missing_python_version_default_fallback() -> Result<()> { + let context = TestContext::new_with_versions(&[]) + .with_python_download_cache() + .with_managed_python_dirs() + // Filter the version which will get downloaded + .with_filter(( + r" \d\+\.\d\+(\.\d+)? will be used", + " [DEFAULT] will be used".to_string(), + )); + + let requirements_in = context.temp_dir.child("requirements.in"); + requirements_in.write_str("anyio==3.7.0")?; + + uv_snapshot!(context.filters(), context + .pip_compile() + .env(EnvVars::UV_MANAGED_PYTHON, "1") + .arg("--python-version").arg("3.99.99") + .arg("requirements.in"), @r" + success: true + exit_code: 0 + ----- stdout ----- + # This file was autogenerated by uv via the following command: + # uv pip compile --cache-dir [CACHE_DIR] --python-version 3.99.99 requirements.in + anyio==3.7.0 + # via -r requirements.in + idna==3.6 + # via anyio + sniffio==1.3.1 + # via anyio + + ----- stderr ----- + warning: The requested Python version 3.99.99 is not available; 3.14.2 will be used to build dependencies instead. + Resolved 3 packages in [TIME] + "); + + Ok(()) +} + +/// Test that pip compile warns on download errors +#[cfg(feature = "python-managed")] +#[tokio::test] +async fn compile_missing_python_download_error_warning() { + let context = TestContext::new("3.12") + .with_managed_python_dirs() + .with_filter(( + r"(https://github\.com/astral-sh/python-build-standalone/releases/download/).*" + .to_string(), + "$1[FILE-PATH]".to_string(), + )); + + let server = MockServer::start().await; + + Mock::given(method("GET")) + .respond_with(ResponseTemplate::new(StatusCode::INTERNAL_SERVER_ERROR)) + .mount(&server) + .await; + + let requirements_in = context.temp_dir.child("requirements.in"); + requirements_in.write_str("anyio==3.7.0").unwrap(); + + // This produces an error in the end because we've just broken ALL network + // traffic. But the goal here is to check the warning. + uv_snapshot!(context.filters(), context + .pip_compile() + .arg("--python-version").arg("3.10") + .env("ALL_PROXY", server.uri()) + .env(EnvVars::UV_HTTP_RETRIES, "0") + .env(EnvVars::UV_TEST_NO_HTTP_RETRY_DELAY, "true") + .arg("requirements.in"), @r" + success: false + exit_code: 2 + ----- stdout ----- + + ----- stderr ----- + warning: A managed Python download is available for Python 3.10, but an error occurred when attempting to download it. + Caused by: Failed to download https://github.com/astral-sh/python-build-standalone/releases/download/[FILE-PATH] + Caused by: error sending request for url (https://github.com/astral-sh/python-build-standalone/releases/download/[FILE-PATH] + Caused by: client error (Connect) + Caused by: tunnel error: unsuccessful + warning: The requested Python version 3.10 is not available; 3.12.[X] will be used to build dependencies instead. + error: Failed to fetch: `https://pypi.org/simple/anyio/` + Caused by: error sending request for url (https://pypi.org/simple/anyio/) + Caused by: client error (Connect) + Caused by: tunnel error: unsuccessful + "); + + // Also check for the patch fallback + uv_snapshot!(context.filters(), context + .pip_compile() + .arg("--python-version").arg("3.10.99") + .env("ALL_PROXY", server.uri()) + .env(EnvVars::UV_HTTP_RETRIES, "0") + .env(EnvVars::UV_TEST_NO_HTTP_RETRY_DELAY, "true") + .arg("requirements.in"), @r" + success: false + exit_code: 2 + ----- stdout ----- + + ----- stderr ----- + warning: A managed Python download is available for Python 3.10, but an error occurred when attempting to download it. + Caused by: Failed to download https://github.com/astral-sh/python-build-standalone/releases/download/[FILE-PATH] + Caused by: error sending request for url (https://github.com/astral-sh/python-build-standalone/releases/download/[FILE-PATH] + Caused by: client error (Connect) + Caused by: tunnel error: unsuccessful + warning: The requested Python version 3.10.99 is not available; 3.12.[X] will be used to build dependencies instead. + error: Failed to fetch: `https://pypi.org/simple/anyio/` + Caused by: error sending request for url (https://pypi.org/simple/anyio/) + Caused by: client error (Connect) + Caused by: tunnel error: unsuccessful + "); + + // Check that looking up a valid patch version only warns once + uv_snapshot!(context.filters(), context + .pip_compile() + .arg("--python-version").arg("3.10.19") + .env("ALL_PROXY", server.uri()) + .env(EnvVars::UV_HTTP_RETRIES, "0") + .env(EnvVars::UV_TEST_NO_HTTP_RETRY_DELAY, "true") + .arg("requirements.in"), @r" + success: false + exit_code: 2 + ----- stdout ----- + + ----- stderr ----- + warning: A managed Python download is available for Python 3.10.19, but an error occurred when attempting to download it. + Caused by: Failed to download https://github.com/astral-sh/python-build-standalone/releases/download/[FILE-PATH] + Caused by: error sending request for url (https://github.com/astral-sh/python-build-standalone/releases/download/[FILE-PATH] + Caused by: client error (Connect) + Caused by: tunnel error: unsuccessful + warning: The requested Python version 3.10.19 is not available; 3.12.[X] will be used to build dependencies instead. + error: Failed to fetch: `https://pypi.org/simple/anyio/` + Caused by: error sending request for url (https://pypi.org/simple/anyio/) + Caused by: client error (Connect) + Caused by: tunnel error: unsuccessful + "); +}