From 34105dfd67a865118289c76fba9adb2c6cf22b80 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Mon, 6 May 2024 11:36:30 +0200 Subject: [PATCH] feat: use `best_platform` in a few places for cross-platform running (#1020) Co-authored-by: Tim de Jager Co-authored-by: Bas Zalmstra Co-authored-by: Bas Zalmstra Co-authored-by: Ruben Arts Co-authored-by: Olivier Lacroix --- schema/model.py | 2 ++ schema/schema.json | 6 ++++ src/cli/global/common.rs | 11 +++---- src/cli/global/install.rs | 17 ++++++++--- src/cli/global/upgrade.rs | 19 +++++++++--- src/cli/global/upgrade_all.rs | 8 +++-- src/cli/info.rs | 4 +-- src/cli/list.rs | 4 +-- src/cli/run.rs | 5 ++- src/cli/tree.rs | 2 +- src/consts.rs | 2 ++ src/environment.rs | 4 ++- src/install_pypi.rs | 7 ++--- src/lock_file/satisfiability.rs | 15 ++++++++- src/lock_file/update.rs | 19 +++++++++--- src/project/environment.rs | 54 ++++++++++++++++++++++++++++++--- src/project/virtual_packages.rs | 2 +- src/task/task_hash.rs | 6 ++-- tests/common/mod.rs | 7 +++-- 19 files changed, 149 insertions(+), 45 deletions(-) diff --git a/schema/model.py b/schema/model.py index a3e3e0310..9f45383d0 100644 --- a/schema/model.py +++ b/schema/model.py @@ -47,6 +47,8 @@ | Literal["win-32"] | Literal["win-64"] | Literal["win-arm64"] + | Literal["emscripten-wasm32"] + | Literal["wasi-wasm32"] ) diff --git a/schema/schema.json b/schema/schema.json index 4dd3dee7a..39769e567 100644 --- a/schema/schema.json +++ b/schema/schema.json @@ -671,6 +671,12 @@ }, { "const": "win-arm64" + }, + { + "const": "emscripten-wasm32" + }, + { + "const": "wasi-wasm32" } ] } diff --git a/src/cli/global/common.rs b/src/cli/global/common.rs index 3e3777ef9..4834c4e9a 100644 --- a/src/cli/global/common.rs +++ b/src/cli/global/common.rs @@ -194,19 +194,16 @@ pub fn load_package_records( /// The network client and the fetched sparse repodata pub(super) async fn get_client_and_sparse_repodata( channels: impl IntoIterator, + platform: Platform, config: &Config, ) -> miette::Result<( ClientWithMiddleware, IndexMap<(Channel, Platform), SparseRepoData>, )> { let authenticated_client = build_reqwest_clients(Some(config)).1; - let platform_sparse_repodata = repodata::fetch_sparse_repodata( - channels, - [Platform::current()], - &authenticated_client, - Some(config), - ) - .await?; + let platform_sparse_repodata = + repodata::fetch_sparse_repodata(channels, [platform], &authenticated_client, Some(config)) + .await?; Ok((authenticated_client, platform_sparse_repodata)) } diff --git a/src/cli/global/install.rs b/src/cli/global/install.rs index 00eb343ce..799333c08 100644 --- a/src/cli/global/install.rs +++ b/src/cli/global/install.rs @@ -42,6 +42,9 @@ pub struct Args { #[clap(short, long)] channel: Vec, + #[clap(short, long, default_value_t = Platform::current())] + platform: Platform, + #[clap(flatten)] config: ConfigCli, } @@ -250,15 +253,20 @@ pub async fn execute(args: Args) -> miette::Result<()> { // Fetch sparse repodata let (authenticated_client, sparse_repodata) = - get_client_and_sparse_repodata(&channels, &config).await?; + get_client_and_sparse_repodata(&channels, args.platform, &config).await?; // Install the package(s) let mut executables = vec![]; for (package_name, package_matchspec) in args.specs()? { let records = load_package_records(package_matchspec, &sparse_repodata)?; - let (prefix_package, scripts, _) = - globally_install_package(&package_name, records, authenticated_client.clone()).await?; + let (prefix_package, scripts, _) = globally_install_package( + &package_name, + records, + authenticated_client.clone(), + &args.platform, + ) + .await?; let channel_name = channel_name_from_prefix(&prefix_package, config.channel_config()); let record = &prefix_package.repodata_record.package_record; @@ -323,6 +331,7 @@ pub(super) async fn globally_install_package( package_name: &PackageName, records: Vec, authenticated_client: ClientWithMiddleware, + platform: &Platform, ) -> miette::Result<(PrefixRecord, Vec, bool)> { // Create the binary environment prefix where we install or update the package let BinEnvDir(bin_prefix) = BinEnvDir::create(package_name).await?; @@ -331,7 +340,7 @@ pub(super) async fn globally_install_package( // Create the transaction that we need let transaction = - Transaction::from_current_and_desired(prefix_records.clone(), records, Platform::current()) + Transaction::from_current_and_desired(prefix_records.clone(), records, *platform) .into_diagnostic()?; let has_transactions = !transaction.operations.is_empty(); diff --git a/src/cli/global/upgrade.rs b/src/cli/global/upgrade.rs index d10967b31..bdb7af451 100644 --- a/src/cli/global/upgrade.rs +++ b/src/cli/global/upgrade.rs @@ -6,7 +6,7 @@ use indexmap::IndexMap; use indicatif::ProgressBar; use itertools::Itertools; use miette::IntoDiagnostic; -use rattler_conda_types::{Channel, MatchSpec, PackageName}; +use rattler_conda_types::{Channel, MatchSpec, PackageName, Platform}; use crate::config::Config; use crate::progress::{global_multi_progress, long_running_progress_style}; @@ -35,6 +35,10 @@ pub struct Args { /// the package was installed from will always be used. #[clap(short, long)] channel: Vec, + + /// The platform to install the package for. + #[clap(long, default_value_t = Platform::current())] + platform: Platform, } impl HasSpecs for Args { @@ -46,13 +50,14 @@ impl HasSpecs for Args { pub async fn execute(args: Args) -> miette::Result<()> { let config = Config::load_global(); let specs = args.specs()?; - upgrade_packages(specs, config, &args.channel).await + upgrade_packages(specs, config, &args.channel, &args.platform).await } pub(super) async fn upgrade_packages( specs: IndexMap, config: Config, cli_channels: &[String], + platform: &Platform, ) -> miette::Result<()> { // Get channels and versions of globally installed packages let mut installed_versions = HashMap::with_capacity(specs.len()); @@ -79,7 +84,7 @@ pub(super) async fn upgrade_packages( // Fetch sparse repodata let (authenticated_client, sparse_repodata) = - get_client_and_sparse_repodata(&channels, &config).await?; + get_client_and_sparse_repodata(&channels, *platform, &config).await?; // Upgrade each package when relevant let mut upgraded = false; @@ -119,7 +124,13 @@ pub(super) async fn upgrade_packages( console::style("Updating").green(), message )); - globally_install_package(&package_name, records, authenticated_client.clone()).await?; + globally_install_package( + &package_name, + records, + authenticated_client.clone(), + platform, + ) + .await?; pb.finish_with_message(format!("{} {}", console::style("Updated").green(), message)); upgraded = true; } diff --git a/src/cli/global/upgrade_all.rs b/src/cli/global/upgrade_all.rs index c46ebb0fb..1968da3b4 100644 --- a/src/cli/global/upgrade_all.rs +++ b/src/cli/global/upgrade_all.rs @@ -1,7 +1,7 @@ use clap::Parser; use indexmap::IndexMap; -use rattler_conda_types::MatchSpec; +use rattler_conda_types::{MatchSpec, Platform}; use crate::config::{Config, ConfigCli}; @@ -24,6 +24,10 @@ pub struct Args { #[clap(flatten)] config: ConfigCli, + + /// The platform to install the package for. + #[clap(long, default_value_t = Platform::current())] + platform: Platform, } pub async fn execute(args: Args) -> miette::Result<()> { @@ -41,5 +45,5 @@ pub async fn execute(args: Args) -> miette::Result<()> { ); } - upgrade_packages(specs, config, &args.channel).await + upgrade_packages(specs, config, &args.channel, &args.platform).await } diff --git a/src/cli/info.rs b/src/cli/info.rs index d00266505..93d15565e 100644 --- a/src/cli/info.rs +++ b/src/cli/info.rs @@ -331,12 +331,12 @@ pub async fn execute(args: Args) -> miette::Result<()> { .map(|solve_group| solve_group.name().to_string()), environment_size: None, dependencies: env - .dependencies(None, Some(Platform::current())) + .dependencies(None, Some(env.best_platform())) .names() .map(|p| p.as_source().to_string()) .collect(), pypi_dependencies: env - .pypi_dependencies(Some(Platform::current())) + .pypi_dependencies(Some(env.best_platform())) .into_iter() .map(|(name, _p)| name.as_source().to_string()) .collect(), diff --git a/src/cli/list.rs b/src/cli/list.rs index 8a3e0719b..74a3846e8 100644 --- a/src/cli/list.rs +++ b/src/cli/list.rs @@ -119,7 +119,7 @@ pub async fn execute(args: Args) -> miette::Result<()> { .await?; // Load the platform - let platform = args.platform.unwrap_or_else(Platform::current); + let platform = args.platform.unwrap_or_else(|| environment.best_platform()); // Get all the packages in the environment. let locked_deps = lock_file @@ -140,7 +140,7 @@ pub async fn execute(args: Args) -> miette::Result<()> { uv_context = UvResolutionContext::from_project(&project)?; index_locations = environment.pypi_options().to_index_locations(); tags = get_pypi_tags( - Platform::current(), + platform, &project.system_requirements(), python_record.package_record(), )?; diff --git a/src/cli/run.rs b/src/cli/run.rs index 22eb70026..27cf9726b 100644 --- a/src/cli/run.rs +++ b/src/cli/run.rs @@ -103,7 +103,10 @@ pub async fn execute(args: Args) -> miette::Result<()> { let search_environment = SearchEnvironments::from_opt_env( &project, explicit_environment.clone(), - Some(Platform::current()), + explicit_environment + .as_ref() + .map(|e| e.best_platform()) + .or(Some(Platform::current())), ) .with_disambiguate_fn(disambiguate_task_interactive); diff --git a/src/cli/tree.rs b/src/cli/tree.rs index a5eeff475..653f5c076 100644 --- a/src/cli/tree.rs +++ b/src/cli/tree.rs @@ -78,7 +78,7 @@ pub async fn execute(args: Args) -> miette::Result<()> { ..UpdateLockFileOptions::default() }) .await?; - let platform = args.platform.unwrap_or_else(Platform::current); + let platform = args.platform.unwrap_or_else(|| environment.best_platform()); let locked_deps = lock_file .lock_file .environment(environment.name().as_str()) diff --git a/src/consts.rs b/src/consts.rs index 139445642..045517a18 100644 --- a/src/consts.rs +++ b/src/consts.rs @@ -13,6 +13,8 @@ pub const SOLVE_GROUP_ENVIRONMENTS_DIR: &str = "solve-group-envs"; pub const PYPI_DEPENDENCIES: &str = "pypi-dependencies"; pub const TASK_CACHE_DIR: &str = "task-cache-v0"; +pub const ONE_TIME_MESSAGES_DIR: &str = "one-time-messages"; + pub const DEFAULT_ENVIRONMENT_NAME: &str = "default"; /// The default channels to use for a new project. diff --git a/src/environment.rs b/src/environment.rs index 98c783c67..271578d8b 100644 --- a/src/environment.rs +++ b/src/environment.rs @@ -211,7 +211,7 @@ pub async fn get_up_to_date_prefix( mut no_install: bool, existing_repo_data: IndexMap<(Channel, Platform), SparseRepoData>, ) -> miette::Result { - let current_platform = Platform::current(); + let current_platform = environment.best_platform(); let project = environment.project(); // Do not install if the platform is not supported @@ -255,6 +255,7 @@ pub async fn update_prefix_pypi( pypi_options: &PypiOptions, environment_variables: &HashMap, lock_file_dir: &Path, + platform: Platform, ) -> miette::Result<()> { // Remove python packages from a previous python distribution if the python version changed. @@ -275,6 +276,7 @@ pub async fn update_prefix_pypi( uv_context, pypi_options, environment_variables, + platform, ) }, ) diff --git a/src/install_pypi.rs b/src/install_pypi.rs index 138303f3c..08d55886d 100644 --- a/src/install_pypi.rs +++ b/src/install_pypi.rs @@ -696,6 +696,7 @@ pub async fn update_python_distributions( uv_context: UvResolutionContext, pypi_options: &PypiOptions, environment_variables: &HashMap, + platform: Platform, ) -> miette::Result<()> { let start = std::time::Instant::now(); let Some(python_info) = status.current_info() else { @@ -716,11 +717,7 @@ pub async fn update_python_distributions( .iter() .find(|r| is_python_record(r)) .ok_or_else(|| miette::miette!("could not resolve pypi dependencies because no python interpreter is added to the dependencies of the project.\nMake sure to add a python interpreter to the [dependencies] section of the {PROJECT_MANIFEST}, or run:\n\n\tpixi add python"))?; - let tags = get_pypi_tags( - Platform::current(), - system_requirements, - &python_record.package_record, - )?; + let tags = get_pypi_tags(platform, system_requirements, &python_record.package_record)?; let index_locations = pypi_options.to_index_locations(); let registry_client = Arc::new( diff --git a/src/lock_file/satisfiability.rs b/src/lock_file/satisfiability.rs index 556587cf3..cc27d1129 100644 --- a/src/lock_file/satisfiability.rs +++ b/src/lock_file/satisfiability.rs @@ -412,7 +412,20 @@ pub fn verify_package_platform_satisfiability( let marker_environment = python_interpreter_record .map(|interpreter| determine_marker_environment(platform, &interpreter.package_record)) .transpose() - .map_err(|err| PlatformUnsat::FailedToDetermineMarkerEnvironment(err.into()))?; + .map_err(|err| PlatformUnsat::FailedToDetermineMarkerEnvironment(err.into())); + + // We cannot determine the marker environment, for example if installing `wasm32` dependencies. + // However, it also doesn't really matter if we don't have any pypi requirements. + let marker_environment = match marker_environment { + Err(err) => { + if !pypi_requirements.is_empty() { + return Err(err); + } else { + None + } + } + Ok(marker_environment) => marker_environment, + }; // Determine the pypi packages provided by the locked conda packages. let locked_conda_pypi_packages = locked_conda_packages.by_pypi_name(); diff --git a/src/lock_file/update.rs b/src/lock_file/update.rs index 41e9527f6..daa080c6d 100644 --- a/src/lock_file/update.rs +++ b/src/lock_file/update.rs @@ -24,7 +24,7 @@ use indicatif::ProgressBar; use itertools::Itertools; use miette::{IntoDiagnostic, LabeledSpan, MietteDiagnostic, WrapErr}; use rattler::package_cache::PackageCache; -use rattler_conda_types::{Channel, MatchSpec, PackageName, Platform, RepoDataRecord}; +use rattler_conda_types::{Arch, Channel, MatchSpec, PackageName, Platform, RepoDataRecord}; use rattler_lock::{LockFile, PypiPackageData, PypiPackageEnvironmentData}; use rattler_repodata_gateway::sparse::SparseRepoData; use std::path::PathBuf; @@ -104,13 +104,19 @@ impl<'p> LockFileDerivedData<'p> { } // Get the prefix with the conda packages installed. - let platform = Platform::current(); + let platform = environment.best_platform(); let (prefix, python_status) = self.conda_prefix(environment).await?; let repodata_records = self .repodata_records(environment, platform) .unwrap_or_default(); let pypi_records = self.pypi_records(environment, platform).unwrap_or_default(); + // No `uv` support for WASM right now + // TODO - figure out if we can create the `uv` context more lazily + if platform.arch() == Some(Arch::Wasm32) { + return Ok(prefix); + } + let uv_context = match &self.uv_context { None => { let context = UvResolutionContext::from_project(self.project)?; @@ -134,6 +140,7 @@ impl<'p> LockFileDerivedData<'p> { &environment.pypi_options(), env_variables, self.project.root(), + environment.best_platform(), ) .await?; @@ -178,7 +185,7 @@ impl<'p> LockFileDerivedData<'p> { } let prefix = Prefix::new(environment.dir()); - let platform = Platform::current(); + let platform = environment.best_platform(); // Determine the currently installed packages. let installed_packages = prefix @@ -648,7 +655,9 @@ pub async fn ensure_up_to_date_lock_file( // we solve the platforms. We want to solve the current platform first, so we can start // instantiating prefixes if we have to. let mut ordered_platforms = platforms.into_iter().collect::>(); - if let Some(current_platform_index) = ordered_platforms.get_index_of(¤t_platform) { + if let Some(current_platform_index) = + ordered_platforms.get_index_of(&environment.best_platform()) + { ordered_platforms.move_index(current_platform_index, 0); } @@ -734,7 +743,7 @@ pub async fn ensure_up_to_date_lock_file( // Construct a future that will resolve when we have the repodata available for the current // platform for this group. let records_future = context - .get_latest_group_repodata_records(&group, current_platform) + .get_latest_group_repodata_records(&group, environment.best_platform()) .ok_or_else(|| make_unsupported_pypi_platform_error(environment, current_platform))?; // Spawn a task to instantiate the environment diff --git a/src/project/environment.rs b/src/project/environment.rs index d5d70117f..cfceca09a 100644 --- a/src/project/environment.rs +++ b/src/project/environment.rs @@ -5,13 +5,18 @@ use super::{ }; use crate::project::has_features::HasFeatures; +use crate::consts; use crate::task::TaskName; use crate::{task::Task, Project}; use itertools::Either; -use rattler_conda_types::Platform; -use std::collections::HashSet; -use std::hash::{Hash, Hasher}; -use std::{collections::HashMap, fmt::Debug}; +use rattler_conda_types::{Arch, Platform}; +use std::{ + collections::{HashMap, HashSet}, + fmt::Debug, + fs, + hash::{Hash, Hasher}, + sync::Once, +}; /// Describes a single environment from a project manifest. This is used to describe environments /// that can be installed and activated. @@ -97,6 +102,47 @@ impl<'p> Environment<'p> { .join(self.environment.name.as_str()) } + /// Returns the best platform for the current platform & environment. + pub fn best_platform(&self) -> Platform { + let current = Platform::current(); + + // If the current platform is supported, return it. + if self.platforms().contains(¤t) { + return current; + } + + static WARN_ONCE: Once = Once::new(); + + // If the current platform is osx-arm64 and the environment supports osx-64, return osx-64. + if current.is_osx() && self.platforms().contains(&Platform::Osx64) { + WARN_ONCE.call_once(|| { + let warn_folder = self.project.pixi_dir().join(consts::ONE_TIME_MESSAGES_DIR); + let emulation_warn = warn_folder.join("macos-emulation-warn"); + if !emulation_warn.exists() { + tracing::warn!( + "osx-arm64 (Apple Silicon) is not supported by the pixi.toml, falling back to osx-64 (emulated with Rosetta)" + ); + // Create a file to prevent the warning from showing up multiple times. Also ignore the result. + fs::create_dir_all(warn_folder).and_then(|_| { + std::fs::File::create(emulation_warn) + }).ok(); + } + }); + return Platform::Osx64; + } + + if self.platforms().len() == 1 { + // Take the first platform and see if it is a WASM one. + if let Some(platform) = self.platforms().iter().next() { + if platform.arch() == Some(Arch::Wasm32) { + return *platform; + } + } + } + + current + } + /// Returns the tasks defined for this environment. /// /// Tasks are defined on a per-target per-feature per-environment basis. diff --git a/src/project/virtual_packages.rs b/src/project/virtual_packages.rs index 48f55cd6a..3ab5222e8 100644 --- a/src/project/virtual_packages.rs +++ b/src/project/virtual_packages.rs @@ -138,7 +138,7 @@ pub enum VerifyCurrentPlatformError { pub fn verify_current_platform_has_required_virtual_packages( environment: &Environment<'_>, ) -> Result<(), VerifyCurrentPlatformError> { - let current_platform = Platform::current(); + let current_platform = environment.best_platform(); // Is the current platform in the list of supported platforms? if !environment.platforms().contains(¤t_platform) { diff --git a/src/task/task_hash.rs b/src/task/task_hash.rs index 7f4691355..e88815aea 100644 --- a/src/task/task_hash.rs +++ b/src/task/task_hash.rs @@ -1,7 +1,6 @@ use crate::project; use crate::task::{ExecutableTask, FileHashes, FileHashesError, InvalidWorkingDirectory}; use miette::Diagnostic; -use rattler_conda_types::Platform; use rattler_lock::LockFile; use serde::{Deserialize, Serialize}; use std::fmt::{Display, Formatter}; @@ -46,7 +45,8 @@ pub struct EnvironmentHash(String); impl EnvironmentHash { fn from_environment(run_environment: &project::Environment<'_>, lock_file: &LockFile) -> Self { let mut hasher = Xxh3::new(); - let activation_scripts = run_environment.activation_scripts(Some(Platform::current())); + let activation_scripts = + run_environment.activation_scripts(Some(run_environment.best_platform())); for script in activation_scripts { script.hash(&mut hasher); @@ -55,7 +55,7 @@ impl EnvironmentHash { let mut urls = Vec::new(); if let Some(env) = lock_file.environment(run_environment.name().as_str()) { - if let Some(packages) = env.packages(Platform::current()) { + if let Some(packages) = env.packages(run_environment.best_platform()) { for package in packages { urls.push(package.url_or_path().into_owned().to_string()) } diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 110c20d71..4aa0b4184 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -313,8 +313,11 @@ impl PixiControl { // Create a task graph from the command line arguments. let search_env = SearchEnvironments::from_opt_env( &project, - explicit_environment, - Some(Platform::current()), + explicit_environment.clone(), + explicit_environment + .as_ref() + .map(|e| e.best_platform()) + .or(Some(Platform::current())), ); let task_graph = TaskGraph::from_cmd_args(&project, &search_env, args.task) .map_err(RunError::TaskGraphError)?;