diff --git a/docs/reference/pixi_manifest.md b/docs/reference/pixi_manifest.md index fc8b06239f..433029c5ce 100644 --- a/docs/reference/pixi_manifest.md +++ b/docs/reference/pixi_manifest.md @@ -371,6 +371,12 @@ no-build-isolation = ["detectron2"] detectron2 = { git = "https://github.com/facebookresearch/detectron2.git", rev = "5b72c27ae39f99db75d43f18fd1312e1ea934e60"} ``` +Setting `no-build-isolation` also affects the order in which PyPI packages are installed. +Packages are installed in that order: +- conda packages in one go +- packages with build isolation in one go +- packages without build isolation installed in the order they are added to `no-build-isolation` + It is also possible to remove all packages from build isolation by setting the `no-build-isolation` to `true`. ```toml diff --git a/src/install_pypi/mod.rs b/src/install_pypi/mod.rs index b15727d312..22848c5650 100644 --- a/src/install_pypi/mod.rs +++ b/src/install_pypi/mod.rs @@ -96,6 +96,13 @@ pub struct PyPIEnvironmentUpdater<'a> { type PyPIRecords = (PypiPackageData, PypiPackageEnvironmentData); +/// Struct holding (regular distributions, no-build-isolation distributions) +#[derive(Debug, Clone)] +pub struct SeparatedDistributions { + pub regular_dists: Vec<(uv_distribution_types::Dist, InstallReason)>, + pub no_build_isolation_dists: Vec<(uv_distribution_types::Dist, InstallReason)>, +} + impl<'a> PyPIEnvironmentUpdater<'a> { /// Create a new PyPI environment updater with the given configurations pub fn new( @@ -328,6 +335,39 @@ impl<'a> PyPIEnvironmentUpdater<'a> { Ok(installation_plan) } + /// Separates distributions into those that require build isolation and those that don't + fn separate_distributions_by_build_isolation( + &self, + dists: &[(uv_distribution_types::Dist, InstallReason)], + ) -> SeparatedDistributions { + let dists = dists.to_owned(); + match self.build_config.no_build_isolation { + NoBuildIsolation::All => SeparatedDistributions { + regular_dists: Vec::new(), + no_build_isolation_dists: dists, + }, + NoBuildIsolation::Packages(no_build_isolation_packages) => { + let mut dist_map = dists + .into_iter() + .map(|(dist, reason)| (dist.name().to_string(), (dist, reason))) + .collect::>(); + let mut no_build_isolation_dists = Vec::new(); + for no_build_isolation in no_build_isolation_packages { + if let Some(dist) = dist_map.remove(&no_build_isolation.to_string()) { + no_build_isolation_dists.push(dist.clone()); + } + } + + let regular_dists = dist_map.into_values().collect(); + + SeparatedDistributions { + regular_dists, + no_build_isolation_dists, + } + } + } + } + /// Execute the installation plan - this is the main installation logic async fn execute_installation_plan( &self, @@ -368,32 +408,54 @@ impl<'a> PyPIEnvironmentUpdater<'a> { // Log installation details for debugging self.log_installation_details(cached, remote, reinstalls, extraneous, duplicates); - // Download, build, and unzip any missing distributions. - let remote_dists = if remote.is_empty() { - Vec::new() - } else { - self.prepare_remote_distributions(remote, setup).await? - }; + // Separate distributions by build isolation requirements + let SeparatedDistributions { + regular_dists, + no_build_isolation_dists, + } = self.separate_distributions_by_build_isolation(remote); - // Remove any duplicate metadata for packages that are now owned by conda self.remove_duplicate_metadata(duplicates) .into_diagnostic() .wrap_err("while removing duplicate metadata")?; - - // Remove any unnecessary packages. self.remove_packages(extraneous, reinstalls).await?; - // Install the resolved distributions. + // Install regular PyPI packages (with build isolation) as a batch + let regular_dists = if regular_dists.is_empty() { + Vec::new() + } else { + self.prepare_remote_distributions(®ular_dists, setup) + .await? + }; + + // Install the resolved distributions let cached_dists = cached.iter().map(|(d, _)| d.clone()); - let all_dists = remote_dists + let dists_build_isolation = regular_dists .into_iter() .chain(cached_dists) .collect::>(); - self.check_and_warn_about_conflicts(&all_dists, reinstalls, setup) + self.check_and_warn_about_conflicts(&dists_build_isolation, reinstalls, setup) + .await?; + + self.install_distributions(dists_build_isolation, setup) + .await?; + + // Install no-build-isolation PyPI packages one by one + let mut prepared_no_build_isolation_dists = + Vec::with_capacity(no_build_isolation_dists.len()); + for no_build_isolation_dist in no_build_isolation_dists { + let no_build_isolation_dist = self + .prepare_remote_distributions(&Vec::from([no_build_isolation_dist]), setup) + .await?; + + prepared_no_build_isolation_dists.extend(no_build_isolation_dist.clone()); + + self.install_distributions(no_build_isolation_dist, setup) + .await?; + } + self.check_and_warn_about_conflicts(&prepared_no_build_isolation_dists, reinstalls, setup) .await?; - self.install_distributions(all_dists, setup).await?; tracing::info!("{}", format!("finished in {}", elapsed(start.elapsed()))); Ok(()) diff --git a/src/install_pypi/plan/reasons.rs b/src/install_pypi/plan/reasons.rs index 8e2b1e1667..c64b14492b 100644 --- a/src/install_pypi/plan/reasons.rs +++ b/src/install_pypi/plan/reasons.rs @@ -1,4 +1,4 @@ -#[derive(Debug)] +#[derive(Debug, Clone)] pub enum InstallReason { /// Reinstall a package from the local cache, will link from the cache ReinstallCached, diff --git a/tests/integration_rust/install_tests.rs b/tests/integration_rust/install_tests.rs index fd93dcd59b..697bc72490 100644 --- a/tests/integration_rust/install_tests.rs +++ b/tests/integration_rust/install_tests.rs @@ -683,6 +683,118 @@ setup( pixi.install().await.expect("cannot install project"); } +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +#[cfg_attr(not(feature = "slow_integration_tests"), ignore)] +async fn test_no_build_isolation_with_dependencies() { + let current_platform = Platform::current(); + + // Create pyproject.toml for package-tdjager (will be installed with build isolation) + let pyproject_toml_a = r#" +[build-system] +requires = ["setuptools>=45", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "package-tdjager" +version = "0.1.0" +description = "Package Tim de Jager for testing build isolation" +authors = [{name = "Test Author"}] +requires-python = ">=3.6" +dependencies = [] + "#; + + // Create pyproject.toml for package-b (will be installed WITHOUT build isolation) + // package-b depends on package-tdjager at build time + let pyproject_toml_b = r#" +[build-system] +requires = ["setuptools>=45", "wheel", "package-tdjager"] +build-backend = "setuptools.build_meta" + +[project] +name = "package-b" +version = "0.1.0" +description = "Package B that depends on Package Tim de Jager during build" +authors = [{name = "Test Author"}] +requires-python = ">=3.6" +dependencies = [] + "#; + + let manifest = format!( + r#" + [project] + name = "no-build-isolation-deps" + channels = ["https://prefix.dev/conda-forge"] + platforms = ["{platform}"] + + [pypi-options] + no-build-isolation = ["package-b"] + + [dependencies] + python = "3.12.*" + setuptools = ">=72,<73" + + [pypi-dependencies.package-b] + path = "./package-b" + + [pypi-dependencies.package-tdjager] + path = "./package-tdjager" + + "#, + platform = current_platform, + ); + + let pixi = PixiControl::from_manifest(&manifest).expect("cannot instantiate pixi project"); + + let project_path = pixi.workspace_path(); + + // Create package-tdjager directory and pyproject.toml + let package_a_dir = project_path.join("package-tdjager"); + fs_err::create_dir_all(&package_a_dir).unwrap(); + fs_err::write(package_a_dir.join("pyproject.toml"), pyproject_toml_a).unwrap(); + + // Create empty __init__.py for package-tdjager + let package_a_pkg_dir = package_a_dir.join("package_tdjager"); + fs_err::create_dir_all(&package_a_pkg_dir).unwrap(); + fs_err::write(package_a_pkg_dir.join("__init__.py"), "").unwrap(); + + // Create setup.py for package-b that imports package-tdjager at build time + let setup_py_b = r#" +from setuptools import setup + +import package_tdjager + +setup( + name="package-b", + version="0.1.0", + description="Package B that depends on Package Tim de Jager during build", + author="Test Author", + python_requires=">=3.6", + install_requires=[], + setup_requires=["package-tdjager"], +) + "#; + + // Create package-b directory, setup.py and pyproject.toml + let package_b_dir = project_path.join("package-b"); + fs_err::create_dir_all(&package_b_dir).unwrap(); + fs_err::write(package_b_dir.join("pyproject.toml"), pyproject_toml_b).unwrap(); + fs_err::write(package_b_dir.join("setup.py"), setup_py_b).unwrap(); + + // Create empty __init__.py for package-b + let package_b_pkg_dir = package_b_dir.join("package_b"); + fs_err::create_dir_all(&package_b_pkg_dir).unwrap(); + fs_err::write(package_b_pkg_dir.join("__init__.py"), "").unwrap(); + + // This should succeed with the 3-step installation: + // 1. Conda packages (python, setuptools) + // 2. PyPI packages with build isolation (package-tdjager) - batch + // 3. PyPI packages without build isolation (package-b) - one by one + // The key test: package-b should be able to import package-tdjager during its build + pixi.install() + .await + .expect("cannot install project with no-build-isolation dependencies"); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] #[cfg_attr(not(feature = "slow_integration_tests"), ignore)] async fn test_setuptools_override_failure() {