diff --git a/crates/pixi_cli/src/install.rs b/crates/pixi_cli/src/install.rs index e1226bbb01..52f1e7ebce 100644 --- a/crates/pixi_cli/src/install.rs +++ b/crates/pixi_cli/src/install.rs @@ -4,7 +4,7 @@ use itertools::Itertools; use pixi_config::ConfigCli; use pixi_core::{ UpdateLockFileOptions, WorkspaceLocator, - environment::get_update_lock_file_and_prefixes, + environment::{InstallFilter, get_update_lock_file_and_prefixes}, lock_file::{ReinstallPackages, UpdateMode}, }; use std::fmt::Write; @@ -49,12 +49,21 @@ pub struct Args { #[arg(long, short, conflicts_with = "environment")] pub all: bool, - /// Skip installation of specific packages present in the lockfile. Requires --frozen. - /// This can be useful for instance in a Dockerfile to skip local source dependencies when installing dependencies. - #[arg(long, requires = "frozen")] + /// Skip installation of specific packages present in the lockfile. This uses a soft exclusion: the package will be skipped but its dependencies are installed. + #[arg(long)] pub skip: Option>, + + /// Skip a package and its entire dependency subtree. This performs a hard exclusion: the package and its dependencies are not installed unless reachable from another non-skipped root. + #[arg(long)] + pub skip_with_deps: Option>, + + /// Install and build only this package and its dependencies + #[arg(long)] + pub only: Option, } +const SKIP_CUTOFF: usize = 5; + pub async fn execute(args: Args) -> miette::Result<()> { let workspace = WorkspaceLocator::for_cli() .with_search_start(args.project_config.workspace_locator_start()) @@ -84,6 +93,12 @@ pub async fn execute(args: Args) -> miette::Result<()> { .map(|env| workspace.environment_from_name_or_env_var(Some(env))) .collect::, _>>()?; + // Build the install filter from CLI args + let filter = InstallFilter::new() + .skip_direct(args.skip.clone().unwrap_or_default()) + .skip_with_deps(args.skip_with_deps.clone().unwrap_or_default()) + .target_package(args.only.clone()); + // Update the prefixes by installing all packages let (lock_file, _) = get_update_lock_file_and_prefixes( &environments, @@ -94,7 +109,7 @@ pub async fn execute(args: Args) -> miette::Result<()> { max_concurrent_solves: workspace.config().max_concurrent_solves(), }, ReinstallPackages::default(), - &args.skip.clone().unwrap_or_default(), + &filter, ) .await?; @@ -106,6 +121,8 @@ pub async fn execute(args: Args) -> miette::Result<()> { // Message what's installed let mut message = console::style(console::Emoji("✔ ", "")).green().to_string(); + let skip_opts = args.skip.is_some() || args.skip_with_deps.is_some() || args.only.is_some(); + if installed_envs.len() == 1 { write!( &mut message, @@ -113,6 +130,70 @@ pub async fn execute(args: Args) -> miette::Result<()> { installed_envs[0].fancy_display(), ) .unwrap(); + + if skip_opts { + let names = lock_file.get_filtered_package_names( + environments + .first() + .expect("at least one environment should be available"), + &filter, + )?; + let num_skipped = names.ignored.len(); + let num_retained = names.retained.len(); + + // When only is set, also print the number of packages that will be installed + if args.only.is_some() { + write!(&mut message, ", including {} packages", num_retained).unwrap(); + } + + // Create set of unmatched packages, that matches the skip filter + let (matched, unmatched): (Vec<_>, Vec<_>) = args + .skip + .iter() + .flatten() + .chain(args.skip_with_deps.iter().flatten()) + .partition(|name| names.ignored.contains(*name)); + + if !unmatched.is_empty() { + tracing::warn!( + "The skipped arg(s) '{}' did not match any packages in the lock file", + unmatched.into_iter().join(", ") + ); + } + + if !num_skipped > 0 { + if num_skipped > 0 && num_skipped < SKIP_CUTOFF { + let mut skipped_packages_vec: Vec<_> = names.ignored.into_iter().collect(); + skipped_packages_vec.sort(); + + write!( + &mut message, + " excluding '{}'", + skipped_packages_vec.join("', '") + ) + .unwrap(); + } else if num_skipped > 0 { + let num_matched = matched.len(); + if num_matched > 0 { + write!( + &mut message, + " excluding '{}' and {} other packages", + matched.into_iter().join("', '"), + num_skipped + ) + .unwrap() + } else { + write!(&mut message, " excluding {} other packages", num_skipped).unwrap() + } + } else { + write!( + &mut message, + " no packages were skipped (check if cli args were correct)" + ) + .unwrap(); + } + } + } } else { write!( &mut message, @@ -131,30 +212,6 @@ pub async fn execute(args: Args) -> miette::Result<()> { .unwrap() } - if let Some(skip) = &args.skip { - let mut all_skipped_packages = std::collections::HashSet::new(); - for env in &environments { - let skipped_packages = lock_file.get_skipped_package_names(env, skip)?; - all_skipped_packages.extend(skipped_packages); - } - - if !all_skipped_packages.is_empty() { - let mut skipped_packages_vec: Vec<_> = all_skipped_packages.into_iter().collect(); - skipped_packages_vec.sort(); - write!( - &mut message, - " excluding '{}'", - skipped_packages_vec.join("', '") - ) - .unwrap(); - } else { - tracing::warn!( - "No packages were skipped. '{}' did not match any packages in the lockfile.", - skip.join("', '") - ); - } - } - eprintln!("{}.", message); Ok(()) diff --git a/crates/pixi_cli/src/reinstall.rs b/crates/pixi_cli/src/reinstall.rs index b42eec74bb..0573351461 100644 --- a/crates/pixi_cli/src/reinstall.rs +++ b/crates/pixi_cli/src/reinstall.rs @@ -2,7 +2,7 @@ use clap::Parser; use fancy_display::FancyDisplay; use itertools::Itertools; use pixi_config::ConfigCli; -use pixi_core::environment::get_update_lock_file_and_prefix; +use pixi_core::environment::{InstallFilter, get_update_lock_file_and_prefix}; use pixi_core::lock_file::{ReinstallPackages, UpdateMode}; use pixi_core::{UpdateLockFileOptions, WorkspaceLocator}; @@ -87,7 +87,7 @@ pub async fn execute(args: Args) -> miette::Result<()> { max_concurrent_solves: workspace.config().max_concurrent_solves(), }, reinstall_packages.clone(), - &[], + &InstallFilter::default(), ) .await?; diff --git a/crates/pixi_cli/src/remove.rs b/crates/pixi_cli/src/remove.rs index 08e8e2d1d4..af1fb0a1a1 100644 --- a/crates/pixi_cli/src/remove.rs +++ b/crates/pixi_cli/src/remove.rs @@ -3,7 +3,7 @@ use miette::{Context, IntoDiagnostic}; use pixi_config::ConfigCli; use pixi_core::{ DependencyType, UpdateLockFileOptions, WorkspaceLocator, - environment::get_update_lock_file_and_prefix, + environment::{InstallFilter, get_update_lock_file_and_prefix}, lock_file::{ReinstallPackages, UpdateMode}, }; use pixi_manifest::FeaturesExt; @@ -125,7 +125,7 @@ pub async fn execute(args: Args) -> miette::Result<()> { max_concurrent_solves: workspace.config().max_concurrent_solves(), }, ReinstallPackages::default(), - &[], + &InstallFilter::default(), ) .await?; } diff --git a/crates/pixi_cli/src/run.rs b/crates/pixi_cli/src/run.rs index 5877be1b69..31dc7ab19f 100644 --- a/crates/pixi_cli/src/run.rs +++ b/crates/pixi_cli/src/run.rs @@ -258,7 +258,7 @@ pub async fn execute(args: Args) -> miette::Result<()> { &executable_task.run_environment, UpdateMode::QuickValidate, &ReinstallPackages::default(), - &[], + &pixi_core::environment::InstallFilter::default(), ) .await?; } diff --git a/crates/pixi_cli/src/shell.rs b/crates/pixi_cli/src/shell.rs index 6213898379..812e439abb 100644 --- a/crates/pixi_cli/src/shell.rs +++ b/crates/pixi_cli/src/shell.rs @@ -12,7 +12,7 @@ use pixi_config::{ConfigCli, ConfigCliActivation, ConfigCliPrompt}; use pixi_core::{ UpdateLockFileOptions, WorkspaceLocator, activation::CurrentEnvVarBehavior, - environment::get_update_lock_file_and_prefix, + environment::{InstallFilter, get_update_lock_file_and_prefix}, lock_file::{ReinstallPackages, UpdateMode}, prompt, workspace::get_activated_environment_variables, @@ -344,7 +344,7 @@ pub async fn execute(args: Args) -> miette::Result<()> { max_concurrent_solves: workspace.config().max_concurrent_solves(), }, ReinstallPackages::default(), - &[], + &InstallFilter::default(), ) .await?; let lock_file = lock_file_data.into_lock_file(); diff --git a/crates/pixi_cli/src/shell_hook.rs b/crates/pixi_cli/src/shell_hook.rs index dd565ea999..b9c3c3ef27 100644 --- a/crates/pixi_cli/src/shell_hook.rs +++ b/crates/pixi_cli/src/shell_hook.rs @@ -163,7 +163,7 @@ pub async fn execute(args: Args) -> miette::Result<()> { max_concurrent_solves: workspace.config().max_concurrent_solves(), }, ReinstallPackages::default(), - &[], + &pixi_core::environment::InstallFilter::default(), ) .await?; diff --git a/crates/pixi_cli/src/workspace/channel/add.rs b/crates/pixi_cli/src/workspace/channel/add.rs index 37d5cddce0..7e699899e4 100644 --- a/crates/pixi_cli/src/workspace/channel/add.rs +++ b/crates/pixi_cli/src/workspace/channel/add.rs @@ -1,7 +1,7 @@ use miette::IntoDiagnostic; use pixi_core::{ UpdateLockFileOptions, WorkspaceLocator, - environment::get_update_lock_file_and_prefix, + environment::{InstallFilter, get_update_lock_file_and_prefix}, lock_file::{ReinstallPackages, UpdateMode}, }; @@ -31,7 +31,7 @@ pub async fn execute(args: AddRemoveArgs) -> miette::Result<()> { max_concurrent_solves: workspace.workspace().config().max_concurrent_solves(), }, ReinstallPackages::default(), - &[], + &InstallFilter::default(), ) .await?; diff --git a/crates/pixi_cli/src/workspace/channel/remove.rs b/crates/pixi_cli/src/workspace/channel/remove.rs index 79331118fe..b443e2cebb 100644 --- a/crates/pixi_cli/src/workspace/channel/remove.rs +++ b/crates/pixi_cli/src/workspace/channel/remove.rs @@ -1,7 +1,7 @@ use miette::IntoDiagnostic; use pixi_core::{ UpdateLockFileOptions, WorkspaceLocator, - environment::get_update_lock_file_and_prefix, + environment::{InstallFilter, get_update_lock_file_and_prefix}, lock_file::{ReinstallPackages, UpdateMode}, }; @@ -29,7 +29,7 @@ pub async fn execute(args: AddRemoveArgs) -> miette::Result<()> { max_concurrent_solves: workspace.workspace().config().max_concurrent_solves(), }, ReinstallPackages::default(), - &[], + &InstallFilter::default(), ) .await?; let workspace = workspace.save().await.into_diagnostic()?; diff --git a/crates/pixi_cli/src/workspace/platform/add.rs b/crates/pixi_cli/src/workspace/platform/add.rs index 71191f70f5..fc10e86ff9 100644 --- a/crates/pixi_cli/src/workspace/platform/add.rs +++ b/crates/pixi_cli/src/workspace/platform/add.rs @@ -7,7 +7,7 @@ use rattler_conda_types::Platform; use pixi_core::{ UpdateLockFileOptions, Workspace, - environment::{LockFileUsage, get_update_lock_file_and_prefix}, + environment::{InstallFilter, LockFileUsage, get_update_lock_file_and_prefix}, lock_file::{ReinstallPackages, UpdateMode}, }; @@ -57,7 +57,7 @@ pub async fn execute(workspace: Workspace, args: Args) -> miette::Result<()> { max_concurrent_solves: workspace.workspace().config().max_concurrent_solves(), }, ReinstallPackages::default(), - &[], + &InstallFilter::default(), ) .await?; workspace.save().await.into_diagnostic()?; diff --git a/crates/pixi_cli/src/workspace/platform/remove.rs b/crates/pixi_cli/src/workspace/platform/remove.rs index 1419f7d9a0..aceac2b1bc 100644 --- a/crates/pixi_cli/src/workspace/platform/remove.rs +++ b/crates/pixi_cli/src/workspace/platform/remove.rs @@ -5,7 +5,7 @@ use rattler_conda_types::Platform; use pixi_core::{ UpdateLockFileOptions, Workspace, - environment::{LockFileUsage, get_update_lock_file_and_prefix}, + environment::{InstallFilter, LockFileUsage, get_update_lock_file_and_prefix}, lock_file::{ReinstallPackages, UpdateMode}, }; @@ -46,7 +46,7 @@ pub async fn execute(workspace: Workspace, args: Args) -> miette::Result<()> { max_concurrent_solves: workspace.workspace().config().max_concurrent_solves(), }, ReinstallPackages::default(), - &[], + &InstallFilter::default(), ) .await?; workspace.save().await.into_diagnostic()?; diff --git a/crates/pixi_command_dispatcher/src/install_pixi/mod.rs b/crates/pixi_command_dispatcher/src/install_pixi/mod.rs index e907ad59ef..7c15d00604 100644 --- a/crates/pixi_command_dispatcher/src/install_pixi/mod.rs +++ b/crates/pixi_command_dispatcher/src/install_pixi/mod.rs @@ -37,6 +37,10 @@ pub struct InstallPixiEnvironmentSpec { #[serde(skip)] pub records: Vec, + /// The packages to ignore, meaning dont remove if not present in records + /// do not update when also present in PixiRecord + pub ignore_packages: Option>, + /// The location to create the prefix at. #[serde(skip)] pub prefix: Prefix, @@ -98,6 +102,7 @@ impl InstallPixiEnvironmentSpec { records, prefix, installed: None, + ignore_packages: None, build_environment: BuildEnvironment::default(), force_reinstall: HashSet::new(), channels: Vec::new(), @@ -132,6 +137,14 @@ impl InstallPixiEnvironmentSpec { binary_records.reserve(source_records.len()); let mut build_futures = ExecutorFutures::new(command_dispatcher.executor()); for source_record in source_records { + // Do not build if package is explicitly ignored + if self + .ignore_packages + .as_ref() + .is_some_and(|ignore| ignore.contains(&source_record.package_record.name)) + { + continue; + } build_futures.push(async { self.build_from_source(&command_dispatcher, &source_record) .await @@ -161,6 +174,7 @@ impl InstallPixiEnvironmentSpec { .with_download_client(command_dispatcher.download_client().clone()) .with_package_cache(command_dispatcher.package_cache().clone()) .with_reinstall_packages(self.force_reinstall) + .with_ignored_packages(self.ignore_packages.unwrap_or_default()) .with_execute_link_scripts(command_dispatcher.allow_execute_link_scripts()) .with_installed_packages(installed_packages); diff --git a/crates/pixi_command_dispatcher/src/instantiate_tool_env/mod.rs b/crates/pixi_command_dispatcher/src/instantiate_tool_env/mod.rs index f08bdba719..f6a71bc54a 100644 --- a/crates/pixi_command_dispatcher/src/instantiate_tool_env/mod.rs +++ b/crates/pixi_command_dispatcher/src/instantiate_tool_env/mod.rs @@ -264,6 +264,7 @@ impl InstantiateToolEnvironmentSpec { prefix: prefix.clone(), installed: None, build_environment: self.build_environment, + ignore_packages: None, force_reinstall: Default::default(), channels: self.channels, channel_config: self.channel_config, diff --git a/crates/pixi_command_dispatcher/src/source_build/mod.rs b/crates/pixi_command_dispatcher/src/source_build/mod.rs index 305d6f0ce4..3f12a4ab0c 100644 --- a/crates/pixi_command_dispatcher/src/source_build/mod.rs +++ b/crates/pixi_command_dispatcher/src/source_build/mod.rs @@ -466,6 +466,7 @@ impl SourceBuildSpec { .map_err(SourceBuildError::CreateBuildEnvironmentDirectory) .map_err(CommandDispatcherError::Failed)?, installed: None, + ignore_packages: None, build_environment: self.build_environment.to_build_from_build(), force_reinstall: Default::default(), channels: self.channels.clone(), @@ -492,6 +493,7 @@ impl SourceBuildSpec { .map_err(SourceBuildError::CreateBuildEnvironmentDirectory) .map_err(CommandDispatcherError::Failed)?, installed: None, + ignore_packages: None, build_environment: self.build_environment.to_build_from_build(), force_reinstall: Default::default(), channels: self.channels.clone(), diff --git a/crates/pixi_command_dispatcher/tests/integration/main.rs b/crates/pixi_command_dispatcher/tests/integration/main.rs index 3477521b19..69d6b0bc19 100644 --- a/crates/pixi_command_dispatcher/tests/integration/main.rs +++ b/crates/pixi_command_dispatcher/tests/integration/main.rs @@ -117,6 +117,7 @@ pub async fn simple_test() { prefix: Prefix::create(&prefix_dir).unwrap(), installed: None, build_environment: build_env, + ignore_packages: None, force_reinstall: Default::default(), channels: vec![ Url::from_str("https://prefix.dev/conda-forge") diff --git a/crates/pixi_core/src/environment/conda_prefix.rs b/crates/pixi_core/src/environment/conda_prefix.rs index 1eb8b796b5..8147ab6340 100644 --- a/crates/pixi_core/src/environment/conda_prefix.rs +++ b/crates/pixi_core/src/environment/conda_prefix.rs @@ -135,6 +135,7 @@ impl CondaPrefixUpdater { &self, pixi_records: Vec, reinstall_packages: Option>, + ignore_packages: Option>, ) -> miette::Result<&CondaPrefixUpdated> { self.inner .created @@ -156,6 +157,7 @@ impl CondaPrefixUpdater { self.inner.variant_config.clone(), self.inner.command_dispatcher.clone(), reinstall_packages, + ignore_packages, ) .await?; @@ -186,6 +188,7 @@ pub async fn update_prefix_conda( variant_config: VariantConfig, command_dispatcher: CommandDispatcher, reinstall_packages: Option>, + ignore_packages: Option>, ) -> miette::Result { // Try to increase the rlimit to a sensible value for installation. try_increase_rlimit_to_sensible(); @@ -199,6 +202,7 @@ pub async fn update_prefix_conda( prefix: rattler_conda_types::prefix::Prefix::create(prefix.root()).into_diagnostic()?, installed: None, force_reinstall: reinstall_packages.unwrap_or_default(), + ignore_packages, build_environment, channels, channel_config, diff --git a/crates/pixi_core/src/environment/mod.rs b/crates/pixi_core/src/environment/mod.rs index cfa6e5394c..c8e36f53cc 100644 --- a/crates/pixi_core/src/environment/mod.rs +++ b/crates/pixi_core/src/environment/mod.rs @@ -172,28 +172,30 @@ impl LockedEnvironmentHash { pub(crate) fn from_environment( environment: rattler_lock::Environment, platform: Platform, - skipped: &[String], ) -> Self { let mut hasher = Xxh3::new(); - for package in - LockFileDerivedData::filter_skipped_packages(environment.packages(platform), skipped) - { - // Always has the url or path - package.location().to_owned().to_string().hash(&mut hasher); - - match package { - // A select set of fields are used to hash the package - LockedPackageRef::Conda(pack) => { - if let Some(sha) = pack.record().sha256 { - sha.hash(&mut hasher); - } else if let Some(md5) = pack.record().md5 { - md5.hash(&mut hasher); + // Intentionally ignore `skipped` here: the quick-validate cache is only + // used during runs, and should not vary based on transient install + // filters. + if let Some(packages) = environment.packages(platform) { + for package in packages { + // Always has the url or path + package.location().to_owned().to_string().hash(&mut hasher); + + match package { + // A select set of fields are used to hash the package + LockedPackageRef::Conda(pack) => { + if let Some(sha) = pack.record().sha256 { + sha.hash(&mut hasher); + } else if let Some(md5) = pack.record().md5 { + md5.hash(&mut hasher); + } + } + LockedPackageRef::Pypi(pack, env) => { + pack.editable.hash(&mut hasher); + env.extras.hash(&mut hasher); } - } - LockedPackageRef::Pypi(pack, env) => { - pack.editable.hash(&mut hasher); - env.extras.hash(&mut hasher); } } } @@ -469,6 +471,38 @@ impl LockFileUsage { } } +/// Options to select a subset of packages to install or skip. +#[derive(Debug, Default, Clone)] +pub struct InstallFilter { + /// Packages to skip directly but still traverse through their dependencies + pub skip_direct: Vec, + /// Packages to skip together with their dependencies (hard stop) + pub skip_with_deps: Vec, + /// Target a single package (and its deps) to install + pub target_package: Option, +} + +impl InstallFilter { + pub fn new() -> Self { + Self::default() + } + + pub fn skip_direct(mut self, packages: impl Into>) -> Self { + self.skip_direct = packages.into(); + self + } + + pub fn skip_with_deps(mut self, packages: impl Into>) -> Self { + self.skip_with_deps = packages.into(); + self + } + + pub fn target_package(mut self, package: Option) -> Self { + self.target_package = package; + self + } +} + /// Update the prefix if it doesn't exist or if it is not up-to-date. /// /// To updated multiple prefixes at once, use [`get_update_lock_file_and_prefixes`]. @@ -477,14 +511,14 @@ pub async fn get_update_lock_file_and_prefix<'env>( update_mode: UpdateMode, update_lock_file_options: UpdateLockFileOptions, reinstall_packages: ReinstallPackages, - skipped: &[String], + filter: &InstallFilter, ) -> miette::Result<(LockFileDerivedData<'env>, Prefix)> { let (lock_file, prefixes) = get_update_lock_file_and_prefixes( &[environment.clone()], update_mode, update_lock_file_options, reinstall_packages, - skipped, + filter, ) .await?; Ok(( @@ -503,7 +537,7 @@ pub async fn get_update_lock_file_and_prefixes<'env>( update_mode: UpdateMode, update_lock_file_options: UpdateLockFileOptions, reinstall_packages: ReinstallPackages, - skipped: &[String], + filter: &InstallFilter, ) -> miette::Result<(LockFileDerivedData<'env>, Vec)> { if environments.is_empty() { return Err(miette::miette!("No environments provided to install.")); @@ -550,7 +584,7 @@ pub async fn get_update_lock_file_and_prefixes<'env>( std::future::ready(Ok(Prefix::new(env.dir()))).left_future() } else { lock_file_ref - .prefix(env, update_mode, reinstall_packages, skipped) + .prefix(env, update_mode, reinstall_packages, filter) .right_future() } }) diff --git a/crates/pixi_core/src/lib.rs b/crates/pixi_core/src/lib.rs index fae2657107..8f3ea67382 100644 --- a/crates/pixi_core/src/lib.rs +++ b/crates/pixi_core/src/lib.rs @@ -11,5 +11,6 @@ pub mod workspace; pub mod signals; +pub use environment::InstallFilter; pub use lock_file::UpdateLockFileOptions; pub use workspace::{DependencyType, Workspace, WorkspaceLocator}; diff --git a/crates/pixi_core/src/lock_file/install_subset.rs b/crates/pixi_core/src/lock_file/install_subset.rs new file mode 100644 index 0000000000..b76b0cad15 --- /dev/null +++ b/crates/pixi_core/src/lock_file/install_subset.rs @@ -0,0 +1,596 @@ +use rattler_lock::LockedPackageRef; +use std::collections::{HashMap, HashSet, VecDeque}; + +/// A simplified representation of a package and its dependencies for efficient filtering. +/// +/// Note: dependency names here are already normalized package names. We only +/// capture names (not version constraints) because the reachability algorithms +/// work purely on the structure of the dependency graph. +#[derive(Clone, Debug, PartialEq, Eq)] +enum PackageSource { + Conda, + Pypi, +} + +/// A simplified representation of a package and its dependencies for efficient filtering. +/// +/// Note: dependency names here are already normalized package names. We only +/// capture names (not version constraints) because the reachability algorithms +/// work purely on the structure of the dependency graph. +#[derive(Clone, Debug)] +struct PackageNode { + /// Name of the package + pub name: String, + /// The list of dependencies + pub dependencies: Vec, + /// Source of the package (Conda or PyPI) + pub source: PackageSource, +} + +impl<'a> From> for PackageNode { + /// Convert a LockedPackageRef to a PackageNode for efficient processing. + fn from(package_ref: LockedPackageRef<'a>) -> Self { + let name = package_ref.name().to_string(); + + let dependency_names: Vec = match package_ref { + LockedPackageRef::Conda(conda_data) => { + // Extract dependencies from conda data and parse as MatchSpec + let depends = match conda_data { + rattler_lock::CondaPackageData::Binary(binary_data) => { + &binary_data.package_record.depends + } + rattler_lock::CondaPackageData::Source(source_data) => { + &source_data.package_record.depends + } + }; + + depends + .iter() + .filter_map(|dep_spec| { + // Parse as MatchSpec to get the package name + dep_spec + .parse::() + .ok() + .and_then(|spec| spec.name.map(|name| name.as_normalized().to_string())) + }) + .collect() + } + LockedPackageRef::Pypi(pypi_data, _env_data) => { + // For PyPI, use the requirement directly to get the name + pypi_data + .requires_dist + .iter() + .map(|req| req.name.to_string()) + .collect() + } + }; + + PackageNode { + name, + dependencies: dependency_names, + source: match package_ref { + LockedPackageRef::Conda(_) => PackageSource::Conda, + LockedPackageRef::Pypi(_, _) => PackageSource::Pypi, + }, + } + } +} + +/// Filters packages using two skip modes and optional target selection. +/// +/// - `skip_with_deps` (hard stop): drop the node and do not traverse through it. +/// Its subtree is only kept if reachable from another non-skipped root. +/// - `skip_direct` (passthrough): drop only the node; continue traversal through +/// its dependencies so they can still be kept if reachable. +/// +/// When `target_package` is set, the target acts as the sole root of traversal. +/// The result returns both the packages to install/process and to ignore, +/// preserving the original package order. +/// +/// ## Note for implementers +/// One thing that I had is that when creating this code, it was tripping me up that `skip_with_deps` maps into the +/// `stop_set` while the `skip_direct` maps into the passthrough set. Intuitively it feels like the opposite. +/// To make sense of this, first consider that the algorithms is interested in finding what can still be reached under new constraints. +/// +/// So while the user is interested in what to *skip*, we are interested in what we can still *reach*. Thats why things are the inverse. +/// 1. Hence, think of the `skip_with_deps` as pruning parts of the the tree so the place for it is the `stop_set`. +/// 2. And think of `skip_direct` as edge joining two nodes, basically ignoring the skipped node, so the place for it is the `passthrough_set`. +/// 3. Finally, think of the `target` node as zooming into the tree selecting that nodes and its dependencies and basically ignoring the rest. +pub struct InstallSubset<'a> { + /// Packages to skip together with their dependencies (hard stop) + skip_with_deps: &'a [String], + /// Packages to skip directly but traverse through (passthrough) + skip_direct: &'a [String], + /// What package should be targeted directly (zooming in) + target_package: Option<&'a str>, +} + +impl<'a> InstallSubset<'a> { + /// Create a new package filter. + pub fn new( + skip_with_deps: &'a [String], + skip_direct: &'a [String], + target_package: Option<&'a str>, + ) -> Self { + Self { + skip_with_deps, + skip_direct, + target_package, + } + } + + /// Filter packages based on skip and target settings with proper dependency handling. + /// + /// Both traversals run in O(V+E) time on the constructed graph. + /// Algorithm overview: + /// - Convert the input packages to a compact graph representation. + /// - If a `target_package` is provided: run a BFS starting at that target, + /// short-circuiting at `skip_with_deps` and not including nodes in `skip_direct`. + /// - Else (skip-mode): find original graph roots (indegree 0) and run a BFS + /// from those roots, again not traversing into `skip_with_deps`, and exclude + /// nodes in `skip_direct` from the final result. + pub fn filter<'lock>( + &self, + packages: Option> + 'lock>, + ) -> FilteredPackages<'lock> { + // Handle None packages + let Some(packages) = packages else { + return FilteredPackages::new(Vec::new(), Vec::new()); + }; + + let all_packages: Vec<_> = packages.into_iter().collect(); + + let filtered_packages = match self.target_package { + Some(target) => { + // Target mode: Collect target + dependencies with skip short-circuiting + let reach = Self::build_reachability(&all_packages); + let required = reach.collect_target_dependencies( + target, + // This is the stop set, because we just short-circuit getting dependencies + self.skip_with_deps, + // This is the pasthrough set, because we are basically edge-joining + self.skip_direct, + ); + + // Map what we get back + let to_process: Vec<_> = all_packages + .iter() + .filter(|pkg| required.contains(pkg.name())) + .copied() + .collect(); + let to_ignore: Vec<_> = all_packages + .iter() + .filter(|pkg| !required.contains(pkg.name())) + .copied() + .collect(); + FilteredPackages::new(to_process, to_ignore) + } + None => { + // Skip mode: Apply stop/passthrough rules from original roots + self.filter_with_skips(&all_packages) + } + }; + + filtered_packages + } + + /// Filter out skip packages and only those dependencies that are no longer + /// required by any remaining (non-skipped) package. + fn filter_with_skips<'lock>( + &self, + all_packages: &[LockedPackageRef<'lock>], + ) -> FilteredPackages<'lock> { + if self.skip_with_deps.is_empty() && self.skip_direct.is_empty() { + return FilteredPackages::new(all_packages.to_vec(), Vec::new()); + } + + // Compute the set of package names that remain required when the skip + // packages are removed. We do this by walking the dependency graph + // starting from every non-skipped package and never traversing through + // skipped packages. + let reach = Self::build_reachability(all_packages); + let kept = reach.collect_reachable_from_non_skipped(self.skip_with_deps, self.skip_direct); + let to_process: Vec<_> = all_packages + .iter() + .filter(|pkg| kept.contains(pkg.name())) + .copied() + .collect(); + let to_ignore: Vec<_> = all_packages + .iter() + .filter(|pkg| !kept.contains(pkg.name())) + .copied() + .collect(); + FilteredPackages::new(to_process, to_ignore) + } + + /// Build a reachability analyzer for a set of packages. + fn build_reachability(all_packages: &[LockedPackageRef<'_>]) -> PackageReachability { + let nodes: Vec = all_packages.iter().copied().map(Into::into).collect(); + PackageReachability::new(nodes) + } +} + +/// Result of applying an InstallSubset over a package set. +#[derive(Default)] +pub struct FilteredPackages<'lock> { + pub install: Vec>, + pub ignore: Vec>, +} + +impl<'lock> FilteredPackages<'lock> { + pub fn new( + install: Vec>, + ignore: Vec>, + ) -> Self { + FilteredPackages { install, ignore } + } +} + +/// Collects reachability over the package graph. +/// +/// Traversal rules use two skip sets: +/// - stop_set: do not include node and do not traverse its dependencies. +/// - passthrough_set: do not include node but DO traverse its dependencies. +struct PackageReachability { + /// Flattened nodes for fast traversal. + nodes: Vec, + /// Map package name -> index into `nodes`. + name_to_index: HashMap, + /// Adjacency list of dependency indices for fast traversal. + edges: Vec>, +} + +impl PackageReachability { + /// Build a collector from a list of nodes. + /// + /// This constructs: + /// - `name_to_index` for O(1) name→index lookups. + /// - `edges` adjacency lists (indices only) for tight traversal loops without + /// repeated string hashing or allocation. + pub(crate) fn new(nodes: Vec) -> Self { + let name_to_index: HashMap = nodes + .iter() + .enumerate() + .map(|(idx, node)| (node.name.clone(), idx)) + .collect(); + + // Build compact adjacency list by resolving dependency names to indices + let mut edges: Vec> = vec![Vec::new(); nodes.len()]; + for (idx, node) in nodes.iter().enumerate() { + let deps = node + .dependencies + .iter() + .filter_map(|name| name_to_index.get(name).copied()) + .collect(); + edges[idx] = deps; + } + + Self { + nodes, + name_to_index, + edges, + } + } + + /// If the current required set contains any PyPI package, ensure a Conda + /// `python` package is also included when one exists in the graph. + fn augment_with_python_if_pypi(&self, required: &mut HashSet) { + let has_pypi_included = self + .nodes + .iter() + .any(|n| matches!(n.source, PackageSource::Pypi) && required.contains(&n.name)); + + if has_pypi_included { + let has_conda_python = self + .nodes + .iter() + .any(|n| matches!(n.source, PackageSource::Conda) && n.name.as_str() == "python"); + if has_conda_python { + required.insert("python".to_string()); + } + } + } + + /// Collect target package and all its dependencies, excluding specified packages. + /// Collect all packages reachable from `target` under skip rules. + /// + /// Semantics: + /// - `stop_set` (skip-with-deps): do not include the node and do not traverse + /// into its dependencies. + /// - `passthrough_set` (skip-direct): do not include the node, but continue + /// traversal into its dependencies. + /// + /// Implementation details: + /// - Uses index-based BFS over `edges` with boolean bitsets for membership tests + pub(crate) fn collect_target_dependencies( + &self, + target: &str, + stop_set: &[String], + passthrough_set: &[String], + ) -> HashSet { + // Resolve sets to boolean index maps for fast membership checks + let mut stop_idx = vec![false; self.nodes.len()]; + for name in stop_set { + if let Some(&i) = self.name_to_index.get(name) { + stop_idx[i] = true; + } + } + // Do the same for the passthrough set + let mut pass_idx = vec![false; self.nodes.len()]; + for name in passthrough_set { + if let Some(&i) = self.name_to_index.get(name) { + pass_idx[i] = true; + } + } + + // BFS over target's dependency tree with exclusions + let mut included = vec![false; self.nodes.len()]; + let mut seen = vec![false; self.nodes.len()]; + let mut queue = VecDeque::new(); + + // Start from the target if it exists; otherwise nothing is required. + if let Some(&start) = self.name_to_index.get(target) { + queue.push_back(start); + } else { + return HashSet::new(); + } + + while let Some(idx) = queue.pop_front() { + // Do not include or traverse nodes in the stop set. + if stop_idx[idx] { + continue; + } + if std::mem::replace(&mut seen[idx], true) { + continue; + } + // Include current node unless it is marked passthrough. + if !pass_idx[idx] { + included[idx] = true; + } + for &dep_idx in &self.edges[idx] { + // Always traverse into children unless they are stopped. + if !stop_idx[dep_idx] { + queue.push_back(dep_idx); + } + } + } + + // Materialize result names, then augment for python if needed + let mut result = HashSet::with_capacity(self.nodes.len()); + for (i, inc) in included.iter().enumerate() { + if *inc { + result.insert(self.nodes[i].name.clone()); + } + } + self.augment_with_python_if_pypi(&mut result); + result + } + + /// Compute the set of packages that should be kept when skipping a set of + /// packages. This keeps any package that is reachable from at least one + /// non-skipped package without traversing through a skipped package. + /// Compute the set of packages that remain required when skipping packages. + /// + /// Approach: + /// - Determine original roots as nodes with indegree 0. These represent the + /// starting points of the environment before skips. + /// - BFS from those roots, never traversing into `stop_set` nodes. + /// - Mark nodes as kept unless they are in `passthrough_set` (skip-direct). + /// - Complexity: O(V+E) for the traversal and indegree computation. + pub(crate) fn collect_reachable_from_non_skipped( + &self, + stop_set: &[String], + passthrough_set: &[String], + ) -> HashSet { + // Resolve sets to boolean index maps + let mut stop_idx = vec![false; self.nodes.len()]; + for name in stop_set { + if let Some(&i) = self.name_to_index.get(name) { + stop_idx[i] = true; + } + } + let mut pass_idx = vec![false; self.nodes.len()]; + for name in passthrough_set { + if let Some(&i) = self.name_to_index.get(name) { + pass_idx[i] = true; + } + } + + // Compute indegree to determine original roots (nodes with indegree 0). + // We do this on the full graph: skip rules affect traversal/inclusion, + // not what counts as a structural root. + let mut indegree = vec![0usize; self.nodes.len()]; + for deps in &self.edges { + for &dep in deps { + indegree[dep] = indegree[dep].saturating_add(1); + } + } + let roots: Vec = indegree + .iter() + .enumerate() + .filter_map(|(i, °)| if deg == 0 { Some(i) } else { None }) + .collect(); + + let mut kept = vec![false; self.nodes.len()]; + let mut seen = vec![false; self.nodes.len()]; + let mut queue = VecDeque::new(); + + // Initialize the queue with all non-skipped original roots. + for &root in &roots { + if !stop_idx[root] { + queue.push_back(root); + } + } + + while let Some(idx) = queue.pop_front() { + // Never include or traverse stopped nodes. + if stop_idx[idx] { + continue; + } + if std::mem::replace(&mut seen[idx], true) { + continue; + } + + // Include unless marked passthrough. + if !pass_idx[idx] { + kept[idx] = true; + } + + for &dep in &self.edges[idx] { + // Do not traverse into stop_set; passthrough happens by excluding at the end + if !stop_idx[dep] { + queue.push_back(dep); + } + } + } + + // Remove passthrough nodes from the kept set, but leave traversal effect + let mut result = HashSet::with_capacity(self.nodes.len()); + for (i, &is_kept) in kept.iter().enumerate() { + if is_kept && !pass_idx[i] { + result.insert(self.nodes[i].name.clone()); + } + } + self.augment_with_python_if_pypi(&mut result); + result + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn node(name: &str, deps: &[&str]) -> PackageNode { + // Default test helper creates Conda nodes + PackageNode { + name: name.to_string(), + dependencies: deps.iter().map(|s| s.to_string()).collect(), + source: PackageSource::Conda, + } + } + + fn node_pypi(name: &str, deps: &[&str]) -> PackageNode { + PackageNode { + name: name.to_string(), + dependencies: deps.iter().map(|s| s.to_string()).collect(), + source: PackageSource::Pypi, + } + } + + // Graph: A -> B -> C <- F + fn graph_a_b_c_f() -> PackageReachability { + let nodes = vec![ + node("A", &["B"]), + node("B", &["C"]), + node("C", &[]), + node("F", &["C"]), + ]; + PackageReachability::new(nodes) + } + + #[test] + fn reachable_with_skip_a_keeps_c_via_f() { + let dc = graph_a_b_c_f(); + let kept = dc.collect_reachable_from_non_skipped(&["A".to_string()], &[]); + // Roots: A, F. With A skipped, traversal from F reaches F and C. + assert!(kept.contains("F")); + assert!(kept.contains("C")); + assert!(!kept.contains("B")); + assert!(!kept.contains("A")); + } + + #[test] + fn reachable_with_skip_a_and_f_drops_c() { + let dc = graph_a_b_c_f(); + let kept = dc.collect_reachable_from_non_skipped(&["A".to_string(), "F".to_string()], &[]); + // Both roots skipped, nothing should remain reachable. + assert!(kept.is_empty()); + } + + #[test] + fn target_with_skip_short_circuits_dependencies() { + // A -> B -> C <- D + let nodes = vec![ + node("A", &["B"]), + node("B", &["C"]), + node("C", &[]), + node("D", &["C"]), + ]; + let dc = PackageReachability::new(nodes); + let required = dc.collect_target_dependencies("A", &["B".to_string()], &[]); + + assert!(required.contains("A")); + assert!(!required.contains("B")); + assert!(!required.contains("C")); + assert!(!required.contains("D")); + } + + #[test] + fn reachable_with_skip_direct_passthrough() { + // A -> B -> C + let nodes = vec![node("A", &["B"]), node("B", &["C"]), node("C", &[])]; + let dc = PackageReachability::new(nodes); + let kept = dc.collect_reachable_from_non_skipped(&[], &["B".to_string()]); + assert!(kept.contains("A")); + assert!(kept.contains("C")); + assert!(!kept.contains("B")); + } + + #[test] + fn target_with_skip_direct_passthrough() { + // A -> B -> C <- D, target=A, skip_direct=B keeps A and C + let nodes = vec![ + node("A", &["B"]), + node("B", &["C"]), + node("C", &[]), + node("D", &["C"]), + ]; + let dc = PackageReachability::new(nodes); + let required = dc.collect_target_dependencies("A", &[], &["B".to_string()]); + assert!(required.contains("A")); + assert!(!required.contains("B")); + assert!(required.contains("C")); + } + + #[test] + fn diamond_graph_retains_shared_dep() { + // A -> B, A -> C, B -> D, C -> D + let nodes = vec![ + node("A", &["B", "C"]), + node("B", &["D"]), + node("C", &["D"]), + node("D", &[]), + ]; + let dc = PackageReachability::new(nodes); + let kept = dc.collect_reachable_from_non_skipped(&["B".to_string()], &[]); + assert!( + kept.contains("A") && !kept.contains("B") && kept.contains("C") && kept.contains("D") + ); + } + + #[test] + fn with_deps_overrides_direct_when_both_present() { + // A -> B -> C; if B in both stop and passthrough, stop wins and C + // should only be kept if reachable from another root (it is not). + let nodes = vec![node("A", &["B"]), node("B", &["C"]), node("C", &[])]; + let dc = PackageReachability::new(nodes); + let kept = dc.collect_reachable_from_non_skipped(&["B".to_string()], &["B".to_string()]); + assert!(kept.contains("A")); + assert!(!kept.contains("B")); + assert!(!kept.contains("C")); + } + + #[test] + fn adds_python_when_pypi_is_included() { + // Graph with no edges, but includes a PyPI package and a Conda python. + let nodes = vec![node("python", &[]), node_pypi("requests", &[])]; + let dc = PackageReachability::new(nodes); + + // Simulate result set that includes a PyPI package + let mut required: HashSet = ["requests".to_string()].into_iter().collect(); + dc.augment_with_python_if_pypi(&mut required); + + assert!(required.contains("python")); + assert!(required.contains("requests")); + } +} diff --git a/crates/pixi_core/src/lock_file/mod.rs b/crates/pixi_core/src/lock_file/mod.rs index b1583c9926..66db8c80ff 100644 --- a/crates/pixi_core/src/lock_file/mod.rs +++ b/crates/pixi_core/src/lock_file/mod.rs @@ -1,3 +1,4 @@ +mod install_subset; mod outdated; mod package_identifier; mod records_by_name; @@ -9,6 +10,7 @@ mod utils; pub mod virtual_packages; pub use crate::environment::CondaPrefixUpdater; +pub use install_subset::{FilteredPackages, InstallSubset}; pub use package_identifier::PypiPackageIdentifier; use pixi_record::PixiRecord; use rattler_lock::{PypiPackageData, PypiPackageEnvironmentData}; diff --git a/crates/pixi_core/src/lock_file/resolve/build_dispatch.rs b/crates/pixi_core/src/lock_file/resolve/build_dispatch.rs index 85c9617bd3..c6adbdd015 100644 --- a/crates/pixi_core/src/lock_file/resolve/build_dispatch.rs +++ b/crates/pixi_core/src/lock_file/resolve/build_dispatch.rs @@ -17,6 +17,7 @@ //! and holds struct that is used to instantiate the conda prefix when its //! needed. use std::cell::Cell; +use std::collections::HashSet; use std::{collections::HashMap, path::Path}; use crate::environment::{CondaPrefixUpdated, CondaPrefixUpdater}; @@ -214,6 +215,8 @@ pub struct LazyBuildDispatch<'a> { pub disallow_install_conda_prefix: bool, workspace_cache: WorkspaceCache, + + pub ignore_packages: Option>, } /// These are resources for the [`BuildDispatch`] that need to be lazily @@ -285,6 +288,7 @@ impl<'a> LazyBuildDispatch<'a> { repodata_records: miette::Result>, no_build_isolation: NoBuildIsolation, lazy_deps: &'a LazyBuildDispatchDependencies, + ignore_packages: Option>, disallow_install_conda_prefix: bool, ) -> Self { Self { @@ -299,6 +303,7 @@ impl<'a> LazyBuildDispatch<'a> { lazy_deps, disallow_install_conda_prefix, workspace_cache: WorkspaceCache::default(), + ignore_packages, } } @@ -323,7 +328,11 @@ impl<'a> LazyBuildDispatch<'a> { let prefix = self .prefix_updater - .update(repodata_records.to_vec(), None) + .update( + repodata_records.to_vec(), + None, + self.ignore_packages.clone(), + ) .await .map_err(|err| LazyBuildDispatchError::InitializationError(err.into()))?; diff --git a/crates/pixi_core/src/lock_file/resolve/pypi.rs b/crates/pixi_core/src/lock_file/resolve/pypi.rs index c3077f9cd1..02e43fbaba 100644 --- a/crates/pixi_core/src/lock_file/resolve/pypi.rs +++ b/crates/pixi_core/src/lock_file/resolve/pypi.rs @@ -501,6 +501,7 @@ pub async fn resolve_pypi( repodata_building_records.map(|r| r.records.clone()), pypi_options.no_build_isolation.clone(), &lazy_build_dispatch_dependencies, + None, disallow_install_conda_prefix, ); diff --git a/crates/pixi_core/src/lock_file/update.rs b/crates/pixi_core/src/lock_file/update.rs index fe71989c9a..7ea9239762 100644 --- a/crates/pixi_core/src/lock_file/update.rs +++ b/crates/pixi_core/src/lock_file/update.rs @@ -42,14 +42,14 @@ use tracing::Instrument; use uv_normalize::ExtraName; use super::{ - CondaPrefixUpdater, PixiRecordsByName, PypiRecordsByName, UvResolutionContext, + CondaPrefixUpdater, InstallSubset, PixiRecordsByName, PypiRecordsByName, UvResolutionContext, outdated::OutdatedEnvironments, utils::IoConcurrencyLimit, }; use crate::{ Workspace, activation::CurrentEnvVarBehavior, environment::{ - CondaPrefixUpdated, EnvironmentFile, LockFileUsage, LockedEnvironmentHash, + CondaPrefixUpdated, EnvironmentFile, InstallFilter, LockFileUsage, LockedEnvironmentHash, PerEnvironmentAndPlatform, PerGroup, PerGroupAndPlatform, PythonStatus, read_environment_file, write_environment_file, }, @@ -328,7 +328,6 @@ impl<'p> LockFileDerivedData<'p> { fn locked_environment_hash( &self, environment: &Environment<'p>, - skipped: &[String], ) -> miette::Result { let locked_environment = self .lock_file @@ -337,7 +336,6 @@ impl<'p> LockFileDerivedData<'p> { Ok(LockedEnvironmentHash::from_environment( locked_environment, environment.best_platform(), - skipped, )) } @@ -347,11 +345,11 @@ impl<'p> LockFileDerivedData<'p> { environment: &Environment<'p>, update_mode: UpdateMode, reinstall_packages: &ReinstallPackages, - skipped: &[String], + filter: &InstallFilter, ) -> miette::Result { // Check if the prefix is already up-to-date by validating the hash with the // environment file - let hash = self.locked_environment_hash(environment, skipped)?; + let hash = self.locked_environment_hash(environment)?; if update_mode == UpdateMode::QuickValidate { if let Some(prefix) = self.cached_prefix(environment, &hash) { return prefix; @@ -360,7 +358,7 @@ impl<'p> LockFileDerivedData<'p> { // Get the up-to-date prefix let prefix = self - .update_prefix(environment, reinstall_packages, skipped) + .update_prefix(environment, reinstall_packages, filter) .await?; // Save an environment file to the environment directory after the update. @@ -431,7 +429,7 @@ impl<'p> LockFileDerivedData<'p> { &self, environment: &Environment<'p>, reinstall_packages: &ReinstallPackages, - skipped: &[String], + filter: &InstallFilter, ) -> miette::Result { let prefix_once_cell = self .updated_pypi_prefixes @@ -456,14 +454,26 @@ impl<'p> LockFileDerivedData<'p> { let platform = environment.best_platform(); let locked_env = self.locked_env(environment)?; - let packages = - Self::filter_skipped_packages(locked_env.packages(platform), skipped); + let subset = InstallSubset::new( + &filter.skip_with_deps, + &filter.skip_direct, + filter.target_package.as_deref(), + ); + let result = subset.filter(locked_env.packages(platform)); + let packages = result.install; + let ignored = result.ignore; // Separate the packages into conda and pypi packages let (conda_packages, pypi_packages) = packages .into_iter() .partition::, _>(|p| p.as_conda().is_some()); + let (ignored_conda, ignored_pypi): (HashSet<_>, HashSet<_>) = + ignored.into_iter().partition_map(|p| match p { + LockedPackageRef::Conda(data) => Either::Left(data.record().name.clone()), + LockedPackageRef::Pypi(data, _) => Either::Right(data.name.clone()), + }); + let pixi_records = locked_packages_to_pixi_records(conda_packages)?; let pypi_records = pypi_packages @@ -487,7 +497,7 @@ impl<'p> LockFileDerivedData<'p> { // Get the prefix with the conda packages installed. let (prefix, python_status) = self - .conda_prefix(environment, conda_reinstall_packages, skipped) + .conda_prefix(environment, conda_reinstall_packages, Some(ignored_conda)) .await?; // No `uv` support for WASM right now @@ -572,7 +582,14 @@ impl<'p> LockFileDerivedData<'p> { environment_variables: env_variables, }; + // Ignored pypi records + let names = ignored_pypi + .iter() + .map(to_uv_normalize) + .collect::, _>>() + .into_diagnostic()?; PyPIEnvironmentUpdater::new(config, build_config, context_config) + .with_ignored_extraneous(names) .update(&pixi_records, &pypi_records, &python_status) .await } @@ -604,34 +621,11 @@ impl<'p> LockFileDerivedData<'p> { .ok_or_else(|| UpdateError::LockFileMissingEnv(environment.name().clone())) } - /// Filters out packages that are in the `skipped` list. - /// so it this will return the packages that should *not* be skipped - pub fn filter_skipped_packages<'lock>( - packages: Option> + 'lock>, - skipped: &[String], - ) -> Vec> { - // No packages to skip - let Some(packages) = packages else { - return Vec::new(); - }; - - // Skip list is empty - if skipped.is_empty() { - return packages.into_iter().collect(); - } - - // Otherwise, lets filter out - packages - .into_iter() - .filter(|package| !skipped.contains(&package.name().to_string())) - .collect() - } - async fn conda_prefix( &self, environment: &Environment<'p>, reinstall_packages: Option>, - skipped: &[String], + ignore_packages: Option>, ) -> miette::Result<(Prefix, PythonStatus)> { // If we previously updated this environment, early out. let prefix_once_cell = self @@ -659,8 +653,12 @@ impl<'p> LockFileDerivedData<'p> { // Get the locked environment from the lock-file. let locked_env = self.locked_env(environment)?; - let packages = - Self::filter_skipped_packages(locked_env.packages(platform), skipped); + let packages = locked_env.packages(platform); + let packages = if let Some(iter) = packages { + iter.collect_vec() + } else { + Vec::new() + }; let records = locked_packages_to_pixi_records(packages)?; // Update the conda prefix @@ -669,7 +667,7 @@ impl<'p> LockFileDerivedData<'p> { python_status, .. } = conda_prefix_updater - .update(records, reinstall_packages) + .update(records, reinstall_packages, ignore_packages) .await?; Ok((prefix.clone(), *python_status.clone())) @@ -678,38 +676,51 @@ impl<'p> LockFileDerivedData<'p> { .map(|(prefix, python_status)| (prefix.clone(), python_status.clone())) } - /// Returns a list of matching pypi or conda package names in the lock file - /// that are also present in the `skipped` list. - pub fn get_skipped_package_names( + /// Returns the retained and ignored package names after applying the + /// install filter (skips and/or target). + pub fn get_filtered_package_names( &self, environment: &Environment<'p>, - skipped: &[String], - ) -> miette::Result> { + filter: &InstallFilter, + ) -> miette::Result { let platform = environment.best_platform(); let locked_env = self.locked_env(environment)?; - // Get all package names - let all_package_names: HashSet = locked_env - .packages(platform) + // Determine kept/ignored packages using the full install filter + let subset = InstallSubset::new( + &filter.skip_with_deps, + &filter.skip_direct, + filter.target_package.as_deref(), + ); + let filtered = subset.filter(locked_env.packages(platform)); + + // Map to names, dedupe and sort for stable output. + let retained = filtered + .install .into_iter() - .flat_map(|p| p.map(|p| p.name().to_string())) + .map(|p| p.name().to_string()) + .unique() + .sorted() .collect(); - - // Get kept package names - let kept_package_names: HashSet = - Self::filter_skipped_packages(locked_env.packages(platform), skipped) - .into_iter() - .map(|p| p.name().to_string()) - .collect(); - - Ok(all_package_names - .difference(&kept_package_names) - .cloned() + let ignored = filtered + .ignore + .into_iter() + .map(|p| p.name().to_string()) + .unique() .sorted() - .collect()) + .collect(); + + Ok(PackageFilterNames { retained, ignored }) } } +/// The result of applying an InstallFilter over the lockfile for a given environment, +/// expressed as just package names. +pub struct PackageFilterNames { + pub retained: Vec, + pub ignored: Vec, +} + fn locked_packages_to_pixi_records( conda_packages: Vec>, ) -> Result, Report> { diff --git a/crates/pixi_core/src/workspace/workspace_mut.rs b/crates/pixi_core/src/workspace/workspace_mut.rs index 8d0a7570ee..0204c78878 100644 --- a/crates/pixi_core/src/workspace/workspace_mut.rs +++ b/crates/pixi_core/src/workspace/workspace_mut.rs @@ -422,7 +422,7 @@ impl WorkspaceMut { &self.workspace().default_environment(), UpdateMode::Revalidate, &ReinstallPackages::default(), - &[], + &crate::environment::InstallFilter::default(), ) .await?; } diff --git a/crates/pixi_global/src/project/mod.rs b/crates/pixi_global/src/project/mod.rs index fabec55c99..20e4185e2c 100644 --- a/crates/pixi_global/src/project/mod.rs +++ b/crates/pixi_global/src/project/mod.rs @@ -616,6 +616,7 @@ impl Project { channel_config: self.config.global_channel_config().clone(), enabled_protocols: EnabledProtocols::default(), installed: None, + ignore_packages: None, force_reinstall: Default::default(), variants: None, }) diff --git a/docs/reference/cli/pixi/install.md b/docs/reference/cli/pixi/install.md index c23aacdf75..ae912da8b1 100644 --- a/docs/reference/cli/pixi/install.md +++ b/docs/reference/cli/pixi/install.md @@ -18,8 +18,13 @@ pixi install [OPTIONS] - `--all (-a)` : Install all environments - `--skip ` -: Skip installation of specific packages present in the lockfile. Requires --frozen. This can be useful for instance in a Dockerfile to skip local source dependencies when installing dependencies +: Skip installation of specific packages present in the lockfile. This uses a soft exclusion: the package will be skipped but its dependencies are installed
May be provided more than once. +- `--skip-with-deps ` +: Skip a package and its entire dependency subtree. This performs a hard exclusion: the package and its dependencies are not installed unless reachable from another non-skipped root +
May be provided more than once. +- `--only ` +: Install and build only this package and its dependencies ## Config Options - `--auth-file ` diff --git a/tests/integration_rust/common/builders.rs b/tests/integration_rust/common/builders.rs index 926945fff9..52732eb10a 100644 --- a/tests/integration_rust/common/builders.rs +++ b/tests/integration_rust/common/builders.rs @@ -465,6 +465,14 @@ impl InstallBuilder { self.args.skip = Some(names); self } + pub fn with_skipped_with_deps(mut self, names: Vec) -> Self { + self.args.skip_with_deps = Some(names); + self + } + pub fn with_only_package(mut self, pkg: impl Into) -> Self { + self.args.only = Some(pkg.into()); + self + } } impl IntoFuture for InstallBuilder { diff --git a/tests/integration_rust/common/mod.rs b/tests/integration_rust/common/mod.rs index 113c9c77c0..d95a4265c4 100644 --- a/tests/integration_rust/common/mod.rs +++ b/tests/integration_rust/common/mod.rs @@ -29,7 +29,7 @@ use pixi_cli::{ }; use pixi_consts::consts; use pixi_core::{ - UpdateLockFileOptions, Workspace, + InstallFilter, UpdateLockFileOptions, Workspace, lock_file::{ReinstallPackages, UpdateMode}, }; use pixi_manifest::{EnvironmentName, FeatureName}; @@ -523,7 +523,7 @@ impl PixiControl { &task.run_environment, UpdateMode::Revalidate, &ReinstallPackages::default(), - &[], + &InstallFilter::default(), ) .await?; let env = @@ -567,6 +567,8 @@ impl PixiControl { config: Default::default(), all: false, skip: None, + skip_with_deps: None, + only: None, }, } } diff --git a/tests/integration_rust/install_filter_tests.rs b/tests/integration_rust/install_filter_tests.rs new file mode 100644 index 0000000000..91c84700f4 --- /dev/null +++ b/tests/integration_rust/install_filter_tests.rs @@ -0,0 +1,205 @@ +use pixi_core::{InstallFilter, UpdateLockFileOptions}; +use rattler_conda_types::Platform; + +use crate::common::{ + PixiControl, + package_database::{Package, PackageDatabase}, +}; +use pixi_utils::prefix::Prefix as CondaPrefix; +use rattler_conda_types::PackageName; + +/// Helper to check if a conda package is installed in a prefix +async fn is_conda_package_installed(prefix_path: &std::path::Path, package_name: &str) -> bool { + let conda_prefix = CondaPrefix::new(prefix_path.to_path_buf()); + conda_prefix + .find_designated_package(&PackageName::try_from(package_name).unwrap()) + .await + .is_ok() +} + +// Build a simple package graph for tests: +// a -> {b, c}; c -> {d}; e (independent) +async fn setup_simple_graph_project() -> (PixiControl, crate::common::package_database::LocalChannel) +{ + let mut db = PackageDatabase::default(); + + // Leaf nodes + db.add_package(Package::build("b", "1").finish()); + db.add_package(Package::build("d", "1").finish()); + db.add_package(Package::build("e", "1").finish()); + + // c depends on d + db.add_package(Package::build("c", "1").with_dependency("d >=1").finish()); + + // a depends on b and c + db.add_package( + Package::build("a", "1") + .with_dependency("b >=1") + .with_dependency("c >=1") + .finish(), + ); + + let channel = db.into_channel().await.unwrap(); + + let platform = Platform::current(); + let manifest = format!( + r#" + [project] + name = "install-subset" + channels = ["{channel}"] + platforms = ["{platform}"] + + [dependencies] + a = "*" + e = "*" + "#, + channel = channel.url(), + ); + + ( + PixiControl::from_manifest(&manifest).expect("cannot instantiate pixi project"), + channel, + ) +} + +#[tokio::test] +async fn install_filter_skip_direct_soft_exclusion() { + let (pixi, _channel) = setup_simple_graph_project().await; + + // Ensure lockfile exists + pixi.update_lock_file().await.unwrap(); + + // Build derived data and workspace env + let workspace = pixi.workspace().unwrap(); + let (derived, _) = workspace + .update_lock_file(UpdateLockFileOptions::default()) + .await + .unwrap(); + let env = workspace.environment("default").unwrap(); + + // Skip only the node `a` but traverse through its deps + let filter = InstallFilter::new().skip_direct(vec!["a".to_string()]); + let skipped = derived + .get_filtered_package_names(&env, &filter) + .unwrap() + .ignored; + + // Only `a` should be skipped; b, c, d remain required via passthrough; e remains + assert_eq!(skipped, vec!["a".to_string()]); +} + +#[tokio::test] +async fn install_filter_skip_with_deps_hard_exclusion() { + let (pixi, _channel) = setup_simple_graph_project().await; + pixi.update_lock_file().await.unwrap(); + + let workspace = pixi.workspace().unwrap(); + let (derived, _) = workspace + .update_lock_file(UpdateLockFileOptions::default()) + .await + .unwrap(); + let env = workspace.environment("default").unwrap(); + + // Hard skip `a` including its dependency subtree + let filter = InstallFilter::new().skip_with_deps(vec!["a".to_string()]); + let skipped = derived + .get_filtered_package_names(&env, &filter) + .unwrap() + .ignored; + + // a, b, c, d are excluded; e remains as an independent root + assert_eq!( + skipped, + vec![ + "a".to_string(), + "b".to_string(), + "c".to_string(), + "d".to_string(), + ] + ); +} + +#[tokio::test] +async fn install_filter_target_package_zoom_in() { + let (pixi, _channel) = setup_simple_graph_project().await; + pixi.update_lock_file().await.unwrap(); + + let workspace = pixi.workspace().unwrap(); + let env = workspace.environment("default").unwrap(); + + // Use derived.get_skipped_package_names with target mode + let (derived, _) = workspace + .update_lock_file(UpdateLockFileOptions::default()) + .await + .unwrap(); + let filter = InstallFilter::new().target_package(Some("a".to_string())); + let skipped = derived + .get_filtered_package_names(&env, &filter) + .unwrap() + .ignored; + assert_eq!(skipped, vec!["e".to_string()]); +} + +#[tokio::test] +async fn install_filter_target_with_skip_with_deps_stop() { + let (pixi, _channel) = setup_simple_graph_project().await; + pixi.update_lock_file().await.unwrap(); + + let workspace = pixi.workspace().unwrap(); + let env = workspace.environment("default").unwrap(); + + // Target a, but hard-skip c subtree: expect skipped c,d,e + let (derived, _) = workspace + .update_lock_file(UpdateLockFileOptions::default()) + .await + .unwrap(); + let filter = InstallFilter::new() + .target_package(Some("a".to_string())) + .skip_with_deps(vec!["c".to_string()]); + let skipped = derived + .get_filtered_package_names(&env, &filter) + .unwrap() + .ignored; + assert_eq!(skipped, vec!["c", "d", "e"]); +} + +// Test to test the actual installation and if this makes sense +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn install_subset_e2e_skip_with_deps() { + use std::path::{Path, PathBuf}; + use url::Url; + + // manifest with dependent packages: dummy-g depends on dummy-b + let platform = Platform::current(); + let channel_path: PathBuf = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../tests/data/channels/channels/dummy_channel_1"); + let channel_path = fs_err::canonicalize(channel_path).expect("canonicalize channel path"); + let channel_url = Url::from_directory_path(&channel_path).expect("valid file url"); + let manifest = format!( + r#" + [project] + name = "e2e-install-filter-hard-skip" + channels = ["{channel}"] + platforms = ["{platform}"] + + [dependencies] + dummy-g = "*" + dummy-a = "*" + "#, + channel = channel_url, + ); + + let pixi = PixiControl::from_manifest(&manifest).expect("cannot instantiate pixi project"); + pixi.update_lock_file().await.unwrap(); + + // Hard-skip dummy-g subtree: expect dummy-g absent, and since dummy-g depends on dummy-b, dummy-b is also absent + pixi.install() + .with_frozen() + .with_skipped_with_deps(vec!["dummy-g".into()]) + .await + .unwrap(); + let prefix = pixi.default_env_path().unwrap(); + assert!(!is_conda_package_installed(&prefix, "dummy-g").await); + assert!(!is_conda_package_installed(&prefix, "dummy-b").await); + assert!(is_conda_package_installed(&prefix, "dummy-a").await); +} diff --git a/tests/integration_rust/install_tests.rs b/tests/integration_rust/install_tests.rs index 479a31be39..c16731e0c7 100644 --- a/tests/integration_rust/install_tests.rs +++ b/tests/integration_rust/install_tests.rs @@ -14,6 +14,7 @@ use pixi_cli::{ }; use pixi_config::{Config, DetachedEnvironments}; use pixi_consts::consts; +use pixi_core::InstallFilter; use pixi_core::{ UpdateLockFileOptions, Workspace, environment::LockFileUsage, @@ -36,7 +37,6 @@ use crate::common::{ builders::{ HasDependencyConfig, HasLockFileUpdateConfig, HasNoInstallConfig, string_from_iter, }, - logging::try_init_test_subscriber, package_database::{Package, PackageDatabase}, }; use crate::setup_tracing; @@ -424,30 +424,6 @@ async fn install_frozen_skip() { assert!(is_conda_package_installed(&prefix_path, "python_rich").await); } -/// Test `pixi install --frozen --skip` functionality with a non existing package -#[tokio::test] -#[cfg_attr(not(feature = "slow_integration_tests"), ignore)] -async fn install_skip_non_existent_package_warning() { - let pixi = PixiControl::new().unwrap(); - pixi.init().await.unwrap(); - // Add a dependency to create a lock file - pixi.add("python").await.unwrap(); - - let log_buffer = try_init_test_subscriber(); - - // Install with a skipped package that doesn't exist in the lock file - pixi.install() - .with_frozen() - .with_skipped(vec!["non-existent-package".to_string()]) - .await - .unwrap(); - - let output = log_buffer.get_output(); - assert!(output.contains( - "No packages were skipped. 'non-existent-package' did not match any packages in the lockfile." - )); -} - #[tokio::test(flavor = "multi_thread", worker_threads = 1)] #[cfg_attr(not(feature = "slow_integration_tests"), ignore)] async fn pypi_reinstall_python() { @@ -744,7 +720,7 @@ async fn test_old_lock_install() { ..Default::default() }, ReinstallPackages::default(), - &[], + &InstallFilter::default(), ) .await .unwrap(); @@ -1269,7 +1245,7 @@ async fn test_multiple_prefix_update() { let pixi_records = pixi_records.clone(); // tasks.push(conda_prefix_updater.update(pixi_records)); let updater = conda_prefix_updater.clone(); - sets.spawn(async move { updater.update(pixi_records, None).await.cloned() }); + sets.spawn(async move { updater.update(pixi_records, None, None).await.cloned() }); } let mut first_modified = None; diff --git a/tests/integration_rust/main.rs b/tests/integration_rust/main.rs index b5e5011492..e988ad5918 100644 --- a/tests/integration_rust/main.rs +++ b/tests/integration_rust/main.rs @@ -4,6 +4,7 @@ mod add_tests; mod build_tests; mod common; mod init_tests; +mod install_filter_tests; mod install_tests; mod project_tests; mod pypi_tests;