diff --git a/Cargo.lock b/Cargo.lock index c1766733ee..ffc48e44a8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1362,6 +1362,16 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" +[[package]] +name = "coalesced_map" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0367ea8c35eadd742b12b06e914fbbc12c00a84e8416c70d166e2c12255c785" +dependencies = [ + "dashmap", + "tokio", +] + [[package]] name = "colorchoice" version = "1.0.4" @@ -2569,9 +2579,9 @@ dependencies = [ [[package]] name = "google-cloud-auth" -version = "0.21.0" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a65fb515e1e726bc58b925fc876e8f02b626ab574b15c8f3fa53cb08177a7815" +checksum = "cb930bdeb70c93de0a8c635a23cfb2447e8da167ffe90eab1330b0a8b6091d2e" dependencies = [ "async-trait", "base64 0.22.1", @@ -2579,6 +2589,7 @@ dependencies = [ "google-cloud-gax", "http 1.3.1", "reqwest", + "rustc_version", "rustls 0.23.31", "rustls-pemfile 2.2.0", "serde", @@ -2590,9 +2601,9 @@ dependencies = [ [[package]] name = "google-cloud-gax" -version = "0.23.2" +version = "0.23.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5560b3cedeea7041862717175d98733ac81f7bedfdd0c7949bf06d761bf4094f" +checksum = "876a08b972e05565d562f2575e110092492835bf5f7b76aec4e90f792b581960" dependencies = [ "base64 0.22.1", "bytes", @@ -2610,9 +2621,9 @@ dependencies = [ [[package]] name = "google-cloud-rpc" -version = "0.4.2" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "949f5858642f68efbd836a82400d04f2572e6d18a612c1144d0868832f40996e" +checksum = "07b375f1a978a69d4e4ebbf673766dc3be4bec01c8f95048896370205f2fe22c" dependencies = [ "bytes", "google-cloud-wkt", @@ -2623,9 +2634,9 @@ dependencies = [ [[package]] name = "google-cloud-wkt" -version = "0.5.2" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2916ee3f086766d6989abd8a0b1c3910322af60d72352f6cdf5cb8eb951464" +checksum = "c2c101cb6257433b87908b91b9d16df9288c7dd0fb8c700f2c8e53cfc23ca13e" dependencies = [ "base64 0.22.1", "bytes", @@ -4549,16 +4560,6 @@ dependencies = [ "sha2", ] -[[package]] -name = "petgraph" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" -dependencies = [ - "fixedbitset", - "indexmap 2.10.0", -] - [[package]] name = "petgraph" version = "0.8.2" @@ -5867,9 +5868,9 @@ dependencies = [ [[package]] name = "rattler" -version = "0.34.11" +version = "0.34.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fd2078ed6a0a58eddfc7e3277aab81f5694c2daa83ccd2a674ae957ebba2205" +checksum = "987684ffe53f6c05649596147566eee012027ca617999cf70b5d30ffe880be97" dependencies = [ "anyhow", "clap", @@ -5912,9 +5913,9 @@ dependencies = [ [[package]] name = "rattler_cache" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4397ef3a8ea358170984a98462ea567d0757e45c49dbb01f68ed8ace3569d1eb" +checksum = "e0a7088529f9ef7ab668e724237900b58c6173517b45c401074b28c2c5cd319d" dependencies = [ "anyhow", "dashmap", @@ -5930,6 +5931,7 @@ dependencies = [ "rattler_digest", "rattler_networking", "rattler_package_streaming", + "rattler_redaction", "rayon", "reqwest", "reqwest-middleware", @@ -5944,9 +5946,9 @@ dependencies = [ [[package]] name = "rattler_conda_types" -version = "0.37.0" +version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82f1e12c2384b3f2d42fa3ee5848fb804091df6a2afbbd9e15323cead8e83afc" +checksum = "bc4c101b12003d7b805aa0237a98a8205508ebf0c4bf04ba036036a29ad93910" dependencies = [ "chrono", "core-foundation 0.10.1", @@ -6003,9 +6005,9 @@ dependencies = [ [[package]] name = "rattler_lock" -version = "0.23.13" +version = "0.23.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e00a569e219a13e86fcd79f14f8824289d2af5e29d19a933eaac1009625c1f1a" +checksum = "9afe057fc7df24f8add9f50f055a6966922048dd2a8df32277f3bc1e39ae53ec" dependencies = [ "chrono", "file_url", @@ -6039,9 +6041,9 @@ dependencies = [ [[package]] name = "rattler_menuinst" -version = "0.2.20" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c33f5b3e00bf20202bd35f2ba2f037c2549626c0754b83a7c2d80f14a52371a0" +checksum = "c61a0622d207559cff498243862636ef5e70e74a407ae58766339005542fca20" dependencies = [ "chrono", "configparser", @@ -6069,9 +6071,9 @@ dependencies = [ [[package]] name = "rattler_networking" -version = "0.25.8" +version = "0.25.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e928835e448cfecf657d088ce725f08b26256b9765e4608c479796f9685e450c" +checksum = "f8344bdb1203730b149535f3287a482b229ca7757dfaf1d13b85229e758b1634" dependencies = [ "anyhow", "async-trait", @@ -6099,9 +6101,9 @@ dependencies = [ [[package]] name = "rattler_package_streaming" -version = "0.22.48" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16b516be73df911ed5149e9034822d0b4ca4167dac1a3ae09cc0f0e1c627df0c" +checksum = "e74db41f0f8a688a42190d7bc4a882f3a84e5633cd92f7f0e14e915bc6de4805" dependencies = [ "bzip2 0.6.0", "chrono", @@ -6151,9 +6153,9 @@ dependencies = [ [[package]] name = "rattler_repodata_gateway" -version = "0.23.10" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ba542cbf46c319f4c6c003c7dd841e249c78c4895b25a5431854b834affbc15" +checksum = "e9d38dfeea12d58ccaf55a5f1c7223697ef9479ec6acb767cabe651617b2f854" dependencies = [ "anyhow", "async-compression", @@ -6164,6 +6166,7 @@ dependencies = [ "cache_control", "cfg-if", "chrono", + "coalesced_map", "dashmap", "dirs", "file_url", @@ -6184,6 +6187,7 @@ dependencies = [ "rattler_conda_types", "rattler_digest", "rattler_networking", + "rattler_package_streaming", "rattler_redaction", "reqwest", "reqwest-middleware", @@ -6209,9 +6213,9 @@ dependencies = [ [[package]] name = "rattler_shell" -version = "0.24.7" +version = "0.24.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c72a98e4d684d3b5679a901b413d52b6fd43d631691e44b696339dda5602964" +checksum = "6995e7fef3aa2a16cfcda21d4744060240bf954366d9c164ab5f4ac13f9e94c8" dependencies = [ "anyhow", "enum_dispatch", @@ -6230,9 +6234,9 @@ dependencies = [ [[package]] name = "rattler_solve" -version = "2.1.8" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11647afb45ffde9874b174fbaef99471369f3c92a9f9d42529503a5c2c20f076" +checksum = "219053881a02e81e5218604f7f7d67b704c9178632155ee06a7e7317e221754f" dependencies = [ "chrono", "futures", @@ -6248,9 +6252,9 @@ dependencies = [ [[package]] name = "rattler_virtual_packages" -version = "2.1.1" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c0931655086b9e7db0b684d03db42641d1dbb11a170b5245d7a3499c6eddd78" +checksum = "e45deaea5dbc4564cf8f9b6aab789b3caabba79d11625b9fd00ba7f3fa4ba31d" dependencies = [ "archspec", "libloading", @@ -6487,9 +6491,9 @@ dependencies = [ [[package]] name = "resolvo" -version = "0.9.1" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dba027c8e5dd4b5e5a690cfcfb3900d5ffe6985adb048cbd111d5aa596a6c0c8" +checksum = "670175f9a825ad2419bea0e14bfe74e5dcc0227ec7a652a655b1c11e2b911754" dependencies = [ "ahash", "bitvec", @@ -6498,7 +6502,7 @@ dependencies = [ "futures", "indexmap 2.10.0", "itertools 0.14.0", - "petgraph 0.7.1", + "petgraph", "tracing", ] @@ -8735,7 +8739,7 @@ dependencies = [ "jiff", "owo-colors", "percent-encoding", - "petgraph 0.8.2", + "petgraph", "rkyv", "rustc-hash", "serde", @@ -9075,7 +9079,7 @@ dependencies = [ "itertools 0.14.0", "jiff", "mailparse", - "petgraph 0.8.2", + "petgraph", "regex", "rkyv", "rustc-hash", @@ -9239,7 +9243,7 @@ dependencies = [ "itertools 0.14.0", "jiff", "owo-colors", - "petgraph 0.8.2", + "petgraph", "pubgrub", "rkyv", "rustc-hash", diff --git a/Cargo.toml b/Cargo.toml index 194066b20b..bc2482540c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -125,23 +125,23 @@ which = "8.0.0" # Rattler crates file_url = "0.2.6" -rattler = { version = "0.34.10", default-features = false } -rattler_cache = { version = "0.3.28", default-features = false } -rattler_conda_types = { version = "0.37.0", default-features = false, features = [ +rattler = { version = "0.34.12", default-features = false } +rattler_cache = { version = "0.3.30", default-features = false } +rattler_conda_types = { version = "0.38.0", default-features = false, features = [ "rayon", ] } rattler_digest = { version = "1.1.5", default-features = false } -rattler_lock = { version = "0.23.13", default-features = false } -rattler_menuinst = { version = "0.2.20", default-features = false } -rattler_networking = { version = "0.25.8", default-features = false, features = [ +rattler_lock = { version = "0.23.14", default-features = false } +rattler_menuinst = { version = "0.2.21", default-features = false } +rattler_networking = { version = "0.25.9", default-features = false, features = [ "google-cloud-auth", "dirs", ] } -rattler_package_streaming = { version = "0.22.48", default-features = false } -rattler_repodata_gateway = { version = "0.23.9", default-features = false } -rattler_shell = { version = "0.24.7", default-features = false } -rattler_solve = { version = "2.1.8", default-features = false } -rattler_virtual_packages = { version = "2.1.1", default-features = false } +rattler_package_streaming = { version = "0.23.0", default-features = false } +rattler_repodata_gateway = { version = "0.24.0", default-features = false } +rattler_shell = { version = "0.24.8", default-features = false } +rattler_solve = { version = "3.0.0", default-features = false } +rattler_virtual_packages = { version = "2.1.2", default-features = false } simple_spawn_blocking = { version = "1.1.0", default-features = false } # Bumping this to a higher version breaks the Windows path handling. diff --git a/crates/pixi_command_dispatcher/src/build/dependencies.rs b/crates/pixi_command_dispatcher/src/build/dependencies.rs index eb8bf41d04..421fcd10d2 100644 --- a/crates/pixi_command_dispatcher/src/build/dependencies.rs +++ b/crates/pixi_command_dispatcher/src/build/dependencies.rs @@ -1,4 +1,4 @@ -use std::str::FromStr; +use std::{str::FromStr, sync::Arc}; use itertools::Either; use pixi_build_types::{ @@ -14,6 +14,7 @@ use rattler_conda_types::{ InvalidPackageNameError, MatchSpec, NamedChannelOrUrl, NamelessMatchSpec, PackageName, ParseStrictness, Platform, VersionSpec, }; +use rattler_repodata_gateway::{Gateway, RunExportExtractorError, RunExportsReporter}; use super::conversion; use crate::SourceMetadataError; @@ -119,105 +120,41 @@ impl Dependencies { } /// Extract run exports from the solved environments. - pub fn extract_run_exports( + pub async fn extract_run_exports( &self, - records: &[PixiRecord], + records: &mut [PixiRecord], ignore: &CondaOutputIgnoreRunExports, - ) -> PixiRunExports { + gateway: &Gateway, + reporter: Option>, + ) -> Result { let mut filter_run_exports = PixiRunExports::default(); - fn filter_match_specs>( - specs: &[String], - ignore: &CondaOutputIgnoreRunExports, - ) -> Vec<(PackageName, T)> { - specs - .iter() - .filter_map(move |spec| { - let (Some(name), spec) = MatchSpec::from_str(spec, ParseStrictness::Lenient) - .ok()? - .into_nameless() - else { - return None; - }; - if ignore.by_name.contains(&name) { - return None; - } - - let binary_spec = match spec { - NamelessMatchSpec { - url: Some(url), - sha256, - md5, - .. - } => BinarySpec::Url(UrlBinarySpec { url, sha256, md5 }), - NamelessMatchSpec { - version, - build: None, - build_number: None, - file_name: None, - extras: None, - channel: None, - subdir: None, - namespace: None, - md5: None, - sha256: None, - url: _, - license: None, - } => BinarySpec::Version(version.unwrap_or(VersionSpec::Any)), - NamelessMatchSpec { - version, - build, - build_number, - file_name, - channel, - subdir, - md5, - sha256, - license, - - // Caught in the above case - url: _, - - // Explicitly ignored - namespace: _, - extras: _, - } => BinarySpec::DetailedVersion(Box::new(DetailedSpec { - version, - build, - build_number, - file_name, - channel: channel - .map(|c| NamedChannelOrUrl::Url(c.base_url.clone().into())), - subdir, - md5, - sha256, - license, - })), - }; - - Some((name, binary_spec.into())) - }) - .collect() - } - - for record in records { + // Find all the records that are relevant for run exports. + let mut relevant_records = records + .iter_mut() // Only record run exports for packages that are direct dependencies. - if !self - .dependencies - .contains_key(&record.package_record().name) - { - continue; - } - + .filter(|r| self.dependencies.contains_key(&r.package_record().name)) // Filter based on whether we want to ignore run exports for a particular // package. - if ignore.from_package.contains(&record.package_record().name) { - continue; - } + .filter(|r| !ignore.from_package.contains(&r.package_record().name)) + .collect::>(); - // Make sure we have valid run exports. + // Determine the records that have missing run exports. + let records_missing_run_exports = relevant_records + .iter_mut() + .flat_map(|r| match *r { + PixiRecord::Binary(repo_data_record) => Some(repo_data_record), + PixiRecord::Source(_source_record) => None, + }) + .filter(|r| r.package_record.run_exports.is_none()); + gateway + .ensure_run_exports(records_missing_run_exports.into_iter(), reporter) + .await?; + + // Iterate over all relevant records and add the run exports to the corresponding sets. + for record in &relevant_records { let Some(run_exports) = &record.package_record().run_exports else { - unimplemented!("Extracting run exports from other places is not implemented yet"); + continue; }; filter_run_exports @@ -237,10 +174,83 @@ impl Dependencies { .extend(filter_match_specs(&run_exports.weak_constrains, ignore)); } - filter_run_exports + Ok(filter_run_exports) } } +fn filter_match_specs>( + specs: &[String], + ignore: &CondaOutputIgnoreRunExports, +) -> Vec<(PackageName, T)> { + specs + .iter() + .filter_map(move |spec| { + let (Some(name), spec) = MatchSpec::from_str(spec, ParseStrictness::Lenient) + .ok()? + .into_nameless() + else { + return None; + }; + if ignore.by_name.contains(&name) { + return None; + } + + let binary_spec = match spec { + NamelessMatchSpec { + url: Some(url), + sha256, + md5, + .. + } => BinarySpec::Url(UrlBinarySpec { url, sha256, md5 }), + NamelessMatchSpec { + version, + build: None, + build_number: None, + file_name: None, + extras: None, + channel: None, + subdir: None, + namespace: None, + md5: None, + sha256: None, + url: _, + license: None, + } => BinarySpec::Version(version.unwrap_or(VersionSpec::Any)), + NamelessMatchSpec { + version, + build, + build_number, + file_name, + channel, + subdir, + md5, + sha256, + license, + + // Caught in the above case + url: _, + + // Explicitly ignored + namespace: _, + extras: _, + } => BinarySpec::DetailedVersion(Box::new(DetailedSpec { + version, + build, + build_number, + file_name, + channel: channel.map(|c| NamedChannelOrUrl::Url(c.base_url.clone().into())), + subdir, + md5, + sha256, + license, + })), + }; + + Some((name, binary_spec.into())) + }) + .collect() +} + /// A variant of [`rattler_conda_types::package::RunExportsJson`] but with pixi /// data types. #[derive(Debug, Default, Clone)] diff --git a/crates/pixi_command_dispatcher/src/command_dispatcher_processor/source_build.rs b/crates/pixi_command_dispatcher/src/command_dispatcher_processor/source_build.rs index 9a7bcda988..a48450a046 100644 --- a/crates/pixi_command_dispatcher/src/command_dispatcher_processor/source_build.rs +++ b/crates/pixi_command_dispatcher/src/command_dispatcher_processor/source_build.rs @@ -78,8 +78,16 @@ impl CommandDispatcherProcessor { fn queue_source_build_task(&mut self, source_build_id: SourceBuildId, spec: SourceBuildSpec) { let dispatcher = self .create_task_command_dispatcher(CommandDispatcherContext::SourceBuild(source_build_id)); + + let dispatcher_context = CommandDispatcherContext::SourceBuild(source_build_id); + let reporter_context = self.reporter_context(dispatcher_context); + let run_exports_reporter = self + .reporter + .as_mut() + .and_then(|reporter| reporter.create_run_exports_reporter(reporter_context)); + self.pending_futures.push( - spec.build(dispatcher) + spec.build(dispatcher, run_exports_reporter) .map(move |result| TaskResult::SourceBuild(source_build_id, result)) .boxed_local(), ); diff --git a/crates/pixi_command_dispatcher/src/command_dispatcher_processor/source_metadata.rs b/crates/pixi_command_dispatcher/src/command_dispatcher_processor/source_metadata.rs index 400c20ce7c..e090f6c315 100644 --- a/crates/pixi_command_dispatcher/src/command_dispatcher_processor/source_metadata.rs +++ b/crates/pixi_command_dispatcher/src/command_dispatcher_processor/source_metadata.rs @@ -95,8 +95,16 @@ impl CommandDispatcherProcessor { let dispatcher = self.create_task_command_dispatcher( CommandDispatcherContext::SourceMetadata(source_metadata_id), ); + + let dispatcher_context = CommandDispatcherContext::SourceMetadata(source_metadata_id); + let reporter_context = self.reporter_context(dispatcher_context); + let run_exports_reporter = self + .reporter + .as_mut() + .and_then(|reporter| reporter.create_run_exports_reporter(reporter_context)); + self.pending_futures.push( - spec.request(dispatcher) + spec.request(dispatcher, run_exports_reporter) .map(move |result| { TaskResult::SourceMetadata(source_metadata_id, result.map(Arc::new)) }) diff --git a/crates/pixi_command_dispatcher/src/reporter.rs b/crates/pixi_command_dispatcher/src/reporter.rs index 632d1686bc..cea2d504de 100644 --- a/crates/pixi_command_dispatcher/src/reporter.rs +++ b/crates/pixi_command_dispatcher/src/reporter.rs @@ -1,5 +1,8 @@ +use std::sync::Arc; + use futures::Stream; use pixi_git::resolver::RepositoryReference; +use rattler_repodata_gateway::RunExportsReporter; use serde::Serialize; use crate::{ @@ -277,6 +280,14 @@ pub trait Reporter: Send { None } + /// Returns a reporter that run exports fetching progress. + fn create_run_exports_reporter( + &mut self, + _reason: Option, + ) -> Option> { + None + } + /// Returns a reporter that reports installation progress. fn create_install_reporter( &mut self, diff --git a/crates/pixi_command_dispatcher/src/solve_pixi/reporter.rs b/crates/pixi_command_dispatcher/src/solve_pixi/reporter.rs index 9dca77b8b4..74b0ae4287 100644 --- a/crates/pixi_command_dispatcher/src/solve_pixi/reporter.rs +++ b/crates/pixi_command_dispatcher/src/solve_pixi/reporter.rs @@ -1,46 +1,13 @@ -use url::Url; +use rattler_repodata_gateway::{DownloadReporter, JLAPReporter}; pub struct WrappingGatewayReporter(pub Box); impl rattler_repodata_gateway::Reporter for WrappingGatewayReporter { - fn on_download_start(&self, url: &Url) -> usize { - self.0.on_download_start(url) + fn download_reporter(&self) -> Option<&dyn DownloadReporter> { + self.0.download_reporter() } - fn on_download_progress( - &self, - url: &Url, - index: usize, - bytes_downloaded: usize, - total_bytes: Option, - ) { - self.0 - .on_download_progress(url, index, bytes_downloaded, total_bytes) - } - fn on_download_complete(&self, url: &Url, index: usize) { - self.0.on_download_complete(url, index) - } - fn on_jlap_start(&self) -> usize { - self.0.on_jlap_start() - } - fn on_jlap_decode_start(&self, index: usize) { - self.0.on_jlap_decode_start(index) - } - fn on_jlap_decode_completed(&self, index: usize) { - self.0.on_jlap_decode_completed(index) - } - fn on_jlap_apply_patch(&self, index: usize, patch_index: usize, total: usize) { - self.0.on_jlap_apply_patch(index, patch_index, total) - } - fn on_jlap_apply_patches_completed(&self, index: usize) { - self.0.on_jlap_apply_patches_completed(index) - } - fn on_jlap_encode_start(&self, index: usize) { - self.0.on_jlap_encode_start(index) - } - fn on_jlap_encode_completed(&self, index: usize) { - self.0.on_jlap_encode_completed(index) - } - fn on_jlap_completed(&self, index: usize) { - self.0.on_jlap_completed(index) + + fn jlap_reporter(&self) -> Option<&dyn JLAPReporter> { + self.0.jlap_reporter() } } diff --git a/crates/pixi_command_dispatcher/src/solve_pixi/source_metadata_collector.rs b/crates/pixi_command_dispatcher/src/solve_pixi/source_metadata_collector.rs index 12a0ce1de8..eafd613343 100644 --- a/crates/pixi_command_dispatcher/src/solve_pixi/source_metadata_collector.rs +++ b/crates/pixi_command_dispatcher/src/solve_pixi/source_metadata_collector.rs @@ -154,6 +154,8 @@ impl SourceMetadataCollector { (Arc, Vec), CommandDispatcherError, > { + tracing::trace!("Collecting source metadata for {name:#?}"); + // Get the source for the particular package. let source = self .command_queue diff --git a/crates/pixi_command_dispatcher/src/source_build/mod.rs b/crates/pixi_command_dispatcher/src/source_build/mod.rs index 77f4279b22..31c581f1f0 100644 --- a/crates/pixi_command_dispatcher/src/source_build/mod.rs +++ b/crates/pixi_command_dispatcher/src/source_build/mod.rs @@ -1,6 +1,7 @@ use std::{ collections::BTreeMap, path::{Path, PathBuf}, + sync::Arc, }; use miette::Diagnostic; @@ -14,6 +15,7 @@ use rattler_conda_types::{ Platform, RepoDataRecord, prefix::Prefix, }; use rattler_digest::Sha256Hash; +use rattler_repodata_gateway::{RunExportExtractorError, RunExportsReporter}; use serde::Serialize; use thiserror::Error; use tracing::instrument; @@ -118,6 +120,7 @@ impl SourceBuildSpec { pub(crate) async fn build( mut self, command_dispatcher: CommandDispatcher, + reporter: Option>, ) -> Result> { // If the output directory is not set, we want to use the build cache. Read the // build cache in that case. @@ -202,7 +205,7 @@ impl SourceBuildSpec { // Build the package based on the support backend capabilities. let mut built_source = if backend.capabilities().provides_conda_build_v1() { - self.build_v1(command_dispatcher, backend, work_directory) + self.build_v1(command_dispatcher, backend, work_directory, reporter) .await? } else { self.build_v0(command_dispatcher, backend, work_directory) @@ -343,6 +346,7 @@ impl SourceBuildSpec { command_dispatcher: CommandDispatcher, backend: Backend, work_directory: PathBuf, + reporter: Option>, ) -> Result> { let source_anchor = SourceAnchor::from(SourceSpec::from(self.source.clone())); let host_platform = self.build_environment.host_platform; @@ -394,7 +398,7 @@ impl SourceBuildSpec { .map_err(SourceBuildError::from) .map_err(CommandDispatcherError::Failed)? .unwrap_or_default(); - let build_records = self + let mut build_records = self .solve_dependencies( format!("{} (build)", self.package.name.as_source()), &command_dispatcher, @@ -404,8 +408,18 @@ impl SourceBuildSpec { .await .map_err_with(Box::new) .map_err_with(SourceBuildError::SolveBuildEnvironment)?; - let build_run_exports = - build_dependencies.extract_run_exports(&build_records, &output.ignore_run_exports); + + let gateway = command_dispatcher.gateway(); + let build_run_exports = build_dependencies + .extract_run_exports( + &mut build_records, + &output.ignore_run_exports, + gateway, + reporter, + ) + .await + .map_err(SourceBuildError::from) + .map_err(CommandDispatcherError::Failed)?; // Solve the host environment for the output. let host_dependencies = output @@ -659,6 +673,9 @@ pub enum SourceBuildError { #[error(transparent)] BuildCache(#[from] BuildCacheError), + #[error("failed to amend run exports: {0}")] + RunExportsExtraction(#[from] RunExportExtractorError), + #[error(transparent)] CreateWorkDirectory(std::io::Error), diff --git a/crates/pixi_command_dispatcher/src/source_metadata/mod.rs b/crates/pixi_command_dispatcher/src/source_metadata/mod.rs index 1a66dfd148..d393736e29 100644 --- a/crates/pixi_command_dispatcher/src/source_metadata/mod.rs +++ b/crates/pixi_command_dispatcher/src/source_metadata/mod.rs @@ -1,6 +1,6 @@ mod cycle; -use std::collections::HashMap; +use std::{collections::HashMap, sync::Arc}; pub use cycle::{Cycle, CycleEnvironment}; use futures::TryStreamExt; @@ -14,6 +14,7 @@ use rattler_conda_types::{ ChannelConfig, InvalidPackageNameError, MatchSpec, PackageName, PackageRecord, package::RunExportsJson, }; +use rattler_repodata_gateway::{RunExportExtractorError, RunExportsReporter}; use thiserror::Error; use tracing::instrument; @@ -61,12 +62,15 @@ impl SourceMetadataSpec { pub(crate) async fn request( self, command_dispatcher: CommandDispatcher, + reporter: Option>, ) -> Result> { // Get the metadata from the build backend. let build_backend_metadata = command_dispatcher .build_backend_metadata(self.backend_metadata.clone()) .await - .map_err_with(SourceMetadataError::BuildBackendMetadata)?; + .map_err_with(SourceMetadataError::BuildBackendMetadata); + + let build_backend_metadata = build_backend_metadata?; match &build_backend_metadata.metadata.metadata { MetadataKind::GetMetadata { packages } => { @@ -94,6 +98,7 @@ impl SourceMetadataSpec { output, build_backend_metadata.metadata.input_hash.clone(), build_backend_metadata.source.clone(), + reporter.clone(), )); } @@ -111,6 +116,7 @@ impl SourceMetadataSpec { output: &CondaOutput, input_hash: Option, source: PinnedSourceSpec, + reporter: Option>, ) -> Result> { let source_anchor = SourceAnchor::from(SourceSpec::from(source.clone())); @@ -123,7 +129,7 @@ impl SourceMetadataSpec { .map_err(SourceMetadataError::from) .map_err(CommandDispatcherError::Failed)? .unwrap_or_default(); - let build_records = self + let mut build_records = self .solve_dependencies( self.package.clone(), CycleEnvironment::Build, @@ -134,8 +140,18 @@ impl SourceMetadataSpec { .to_build_from_build(), ) .await?; - let build_run_exports = - build_dependencies.extract_run_exports(&build_records, &output.ignore_run_exports); + + let gateway = command_dispatcher.gateway(); + let build_run_exports = build_dependencies + .extract_run_exports( + &mut build_records, + &output.ignore_run_exports, + gateway, + reporter.clone(), + ) + .await + .map_err(SourceMetadataError::from) + .map_err(CommandDispatcherError::Failed)?; // Solve the host environment for the output. let host_dependencies = output @@ -148,7 +164,7 @@ impl SourceMetadataSpec { .unwrap_or_default() // Extend with the run exports from the build environment. .extend_with_run_exports_from_build(&build_run_exports); - let host_records = self + let mut host_records = self .solve_dependencies( self.package.clone(), CycleEnvironment::Host, @@ -157,8 +173,16 @@ impl SourceMetadataSpec { self.backend_metadata.build_environment.clone(), ) .await?; - let host_run_exports = - host_dependencies.extract_run_exports(&host_records, &output.ignore_run_exports); + let host_run_exports = host_dependencies + .extract_run_exports( + &mut host_records, + &output.ignore_run_exports, + gateway, + reporter, + ) + .await + .map_err(SourceMetadataError::from) + .map_err(CommandDispatcherError::Failed)?; // Gather the dependencies for the output. let run_dependencies = Dependencies::new(&output.run_dependencies, None) @@ -421,6 +445,9 @@ pub enum SourceMetadataError { #[diagnostic(transparent)] BuildBackendMetadata(#[from] BuildBackendMetadataError), + #[error("failed to amend run exports: {0}")] + RunExportsExtraction(#[from] RunExportExtractorError), + #[error("while trying to solve the build environment for the package")] SolveBuildEnvironment( #[diagnostic_source] diff --git a/crates/pixi_reporters/src/git.rs b/crates/pixi_reporters/src/git.rs index f0b3837d39..073aef9940 100644 --- a/crates/pixi_reporters/src/git.rs +++ b/crates/pixi_reporters/src/git.rs @@ -24,7 +24,7 @@ pub struct GitCheckoutProgress { impl GitCheckoutProgress { /// Creates a new source checkout reporter. - pub fn new(anchor: ProgressBar, multi_progress: MultiProgress) -> Self { + pub fn new(multi_progress: MultiProgress, anchor: ProgressBar) -> Self { Self { multi_progress, anchor, diff --git a/crates/pixi_reporters/src/lib.rs b/crates/pixi_reporters/src/lib.rs index f90349f28e..9196c08d5d 100644 --- a/crates/pixi_reporters/src/lib.rs +++ b/crates/pixi_reporters/src/lib.rs @@ -1,15 +1,17 @@ mod download_verify_reporter; mod git; -mod install_reporter; mod main_progress_bar; mod release_notes; mod repodata_reporter; +mod run_exports; +mod sync_reporter; pub mod uv_reporter; -use std::sync::LazyLock; +use std::sync::{Arc, LazyLock}; use git::GitCheckoutProgress; use indicatif::{MultiProgress, ProgressBar}; +use main_progress_bar::MainProgressBar; use pixi_command_dispatcher::{ InstallPixiEnvironmentSpec, PixiEnvironmentSpec, ReporterContext, SolveCondaEnvironmentSpec, reporter::{ @@ -17,14 +19,11 @@ use pixi_command_dispatcher::{ }, }; use pixi_spec::PixiSpec; -use rattler_repodata_gateway::Reporter; +use rattler_repodata_gateway::{Reporter, RunExportsReporter}; pub use release_notes::format_release_notes; -use uv_configuration::RAYON_INITIALIZE; - -use install_reporter::SyncReporter; -use main_progress_bar::MainProgressBar; use repodata_reporter::RepodataReporter; - +use sync_reporter::SyncReporter; +use uv_configuration::RAYON_INITIALIZE; // Re-export the uv_reporter types for external use pub use uv_reporter::{UvReporter, UvReporterOptions}; @@ -56,7 +55,8 @@ impl TopLevelProgress { multi_progress.clone(), pixi_progress::ProgressBarPlacement::Before(anchor_pb.clone()), ); - let source_checkout_reporter = GitCheckoutProgress::new(anchor_pb, multi_progress.clone()); + let source_checkout_reporter = + GitCheckoutProgress::new(multi_progress.clone(), anchor_pb.clone()); Self { source_checkout_reporter, conda_solve_reporter, @@ -124,6 +124,16 @@ impl pixi_command_dispatcher::Reporter for TopLevelProgress { ) -> Option> { Some(Box::new(self.sync_reporter.create_reporter())) } + + fn create_run_exports_reporter( + &mut self, + _reason: Option, + ) -> Option> { + Some(Arc::new(run_exports::RunExportsReporter::new( + self.repodata_reporter.clone(), + self.sync_reporter.clone(), + ))) + } } impl pixi_command_dispatcher::PixiInstallReporter for TopLevelProgress { diff --git a/crates/pixi_reporters/src/repodata_reporter.rs b/crates/pixi_reporters/src/repodata_reporter.rs index 86de9eae4f..c8aff6b407 100644 --- a/crates/pixi_reporters/src/repodata_reporter.rs +++ b/crates/pixi_reporters/src/repodata_reporter.rs @@ -7,6 +7,7 @@ use std::{ use indicatif::{MultiProgress, ProgressBar, ProgressState, ProgressStyle, style::ProgressTracker}; use parking_lot::RwLock; use pixi_progress::ProgressBarPlacement; +use rattler_repodata_gateway::{DownloadReporter, JLAPReporter}; use url::Url; #[derive(Clone)] @@ -15,22 +16,13 @@ pub struct RepodataReporter { } impl rattler_repodata_gateway::Reporter for RepodataReporter { - fn on_download_start(&self, url: &Url) -> usize { - self.inner.write().on_download_start(url) - } - fn on_download_progress( - &self, - url: &Url, - index: usize, - bytes_downloaded: usize, - total_bytes: Option, - ) { - self.inner - .write() - .on_download_progress(url, index, bytes_downloaded, total_bytes); + fn download_reporter(&self) -> Option<&dyn DownloadReporter> { + Some(self) } - fn on_download_complete(&self, url: &Url, index: usize) { - self.inner.write().on_download_complete(url, index); + + fn jlap_reporter(&self) -> Option<&dyn JLAPReporter> { + // TODO: Implement JLAPReporter for RepodataReporter in the future + None } } @@ -250,3 +242,26 @@ impl ProgressTracker for DurationTracker { } } } + +impl DownloadReporter for RepodataReporter { + fn on_download_complete(&self, url: &Url, index: usize) { + let mut inner = self.inner.write(); + inner.on_download_complete(url, index); + } + + fn on_download_progress( + &self, + url: &Url, + index: usize, + bytes_downloaded: usize, + total_bytes: Option, + ) { + let mut inner = self.inner.write(); + inner.on_download_progress(url, index, bytes_downloaded, total_bytes); + } + + fn on_download_start(&self, url: &Url) -> usize { + let mut inner = self.inner.write(); + inner.on_download_start(url) + } +} diff --git a/crates/pixi_reporters/src/run_exports.rs b/crates/pixi_reporters/src/run_exports.rs new file mode 100644 index 0000000000..e071856167 --- /dev/null +++ b/crates/pixi_reporters/src/run_exports.rs @@ -0,0 +1,36 @@ +use rattler::package_cache::CacheReporter; +use rattler_conda_types::RepoDataRecord; +use rattler_repodata_gateway::DownloadReporter; + +use crate::{repodata_reporter::RepodataReporter, sync_reporter::SyncReporter}; + +pub struct RunExportsReporter { + /// The reporter for repodata fetching (used for reporting progress on + /// run_exports.json fetching). + repodata_reporter: RepodataReporter, + + /// The reporter to use when downloading a package. + sync_reporter: SyncReporter, +} + +impl RunExportsReporter { + pub fn new(repodata_reporter: RepodataReporter, sync_reporter: SyncReporter) -> Self { + Self { + repodata_reporter, + sync_reporter, + } + } +} + +impl rattler_repodata_gateway::RunExportsReporter for RunExportsReporter { + fn download_reporter(&self) -> Option<&dyn DownloadReporter> { + Some(&self.repodata_reporter) + } + + fn create_package_download_reporter( + &self, + repo_data_record: &RepoDataRecord, + ) -> Option> { + Some(self.sync_reporter.create_cache_reporter(repo_data_record)) + } +} diff --git a/crates/pixi_reporters/src/install_reporter.rs b/crates/pixi_reporters/src/sync_reporter.rs similarity index 78% rename from crates/pixi_reporters/src/install_reporter.rs rename to crates/pixi_reporters/src/sync_reporter.rs index c75aec4cda..95aa8ae738 100644 --- a/crates/pixi_reporters/src/install_reporter.rs +++ b/crates/pixi_reporters/src/sync_reporter.rs @@ -10,7 +10,7 @@ use pixi_command_dispatcher::{ }, }; use pixi_progress::ProgressBarPlacement; -use rattler::install::Transaction; +use rattler::{install::Transaction, package_cache::CacheReporter}; use rattler_conda_types::{PrefixRecord, RepoDataRecord}; use tokio::sync::mpsc::UnboundedReceiver; @@ -19,10 +19,10 @@ use crate::{ main_progress_bar::{MainProgressBar, Tracker}, }; +#[derive(Clone)] pub struct SyncReporter { multi_progress: MultiProgress, combined_inner: Arc>, - build_output_receiver: Option>, } impl SyncReporter { @@ -37,7 +37,6 @@ impl SyncReporter { Self { multi_progress, combined_inner, - build_output_receiver: None, } } @@ -59,6 +58,18 @@ impl SyncReporter { combined: Arc::clone(&self.combined_inner), } } + + /// Creates a new reporter instance that can used to report the progress of + /// fetching a single `RepoDataRecord` to the cache. + pub fn create_cache_reporter( + &self, + repo_data_record: &RepoDataRecord, + ) -> Box { + Box::new(SyncCacheReporter::new( + repo_data_record, + Arc::clone(&self.combined_inner), + )) + } } impl BackendSourceBuildReporter for SyncReporter { @@ -84,9 +95,13 @@ impl BackendSourceBuildReporter for SyncReporter { let print_backend_output = tracing::event_enabled!(tracing::Level::INFO); // Stream the progress of the output to the screen. let progress_bar = self.multi_progress.clone(); + + // Create a sender to buffer the output lines so we can output them later if + // needed. let (tx, rx) = tokio::sync::mpsc::unbounded_channel::(); if !print_backend_output { - self.build_output_receiver = Some(rx); + let mut inner = self.combined_inner.lock(); + inner.build_output_receiver = Some(rx); } tokio::spawn(async move { @@ -106,17 +121,24 @@ impl BackendSourceBuildReporter for SyncReporter { } fn on_finished(&mut self, _id: BackendSourceBuildId, failed: bool) { - let Some(mut build_output_receiver) = self.build_output_receiver.take() else { - return; + // Take the stream that receives the output from the backend so we can drop the + // memory. + let build_output_receiver = { + let mut inner = self.combined_inner.lock(); + inner.build_output_receiver.take() }; + + // If the build failed, we want to print the output from the backend. let progress_bar = self.multi_progress.clone(); if failed { - tokio::spawn(async move { - while let Some(line) = build_output_receiver.recv().await { - // Suspend the main progress bar while we print the line. - progress_bar.suspend(|| eprintln!("{}", line)); - } - }); + if let Some(mut build_output_receiver) = build_output_receiver { + tokio::spawn(async move { + while let Some(line) = build_output_receiver.recv().await { + // Suspend the main progress bar while we print the line. + progress_bar.suspend(|| eprintln!("{}", line)); + } + }); + } } } } @@ -152,6 +174,8 @@ pub struct CombinedInstallReporterInner { preparing_progress_bar: BuildDownloadVerifyReporter, install_progress_bar: MainProgressBar, + + build_output_receiver: Option>, } #[derive(PartialEq, Eq)] @@ -204,6 +228,7 @@ impl CombinedInstallReporterInner { install_progress_bar: link_progress_bar, operation_link_id: HashMap::new(), cache_entry_id: HashMap::new(), + build_output_receiver: None, } } @@ -234,6 +259,21 @@ impl CombinedInstallReporterInner { } } + /// Called to start progress on populating the cache for a single + /// `RepoDataRecord`. + fn start_populate_single_cache_entry(&mut self, record: &RepoDataRecord) -> TransactionId { + let transaction_id = TransactionId( + self.next_id + .fetch_add(1, std::sync::atomic::Ordering::SeqCst), + ); + let operation_id = 0; + self.cache_entry_id.insert( + (transaction_id, operation_id), + self.preparing_progress_bar.on_entry_start(record), + ); + transaction_id + } + fn on_transaction_operation_start(&mut self, _id: TransactionId, _operation: usize) {} fn on_populate_cache_start( @@ -416,3 +456,51 @@ impl TransactionId { TransactionId(id) } } + +struct SyncCacheReporter { + combined_inner: Arc>, + transaction_id: TransactionId, +} + +impl SyncCacheReporter { + pub fn new( + record: &RepoDataRecord, + combined_inner: Arc>, + ) -> Self { + let transaction_id = { + let mut inner = combined_inner.lock(); + inner.start_populate_single_cache_entry(record) + }; + Self { + combined_inner, + transaction_id, + } + } +} + +impl CacheReporter for SyncCacheReporter { + fn on_validate_start(&self) -> usize { + let mut inner = self.combined_inner.lock(); + inner.on_validate_start(self.transaction_id, 0) + } + + fn on_validate_complete(&self, index: usize) { + let mut inner = self.combined_inner.lock(); + inner.on_validate_complete(self.transaction_id, index); + } + + fn on_download_start(&self) -> usize { + let mut inner = self.combined_inner.lock(); + inner.on_download_start(self.transaction_id, 0) + } + + fn on_download_progress(&self, index: usize, progress: u64, total: Option) { + let mut inner = self.combined_inner.lock(); + inner.on_download_progress(self.transaction_id, index, progress, total); + } + + fn on_download_completed(&self, index: usize) { + let mut inner = self.combined_inner.lock(); + inner.on_download_completed(self.transaction_id, index); + } +}