feat: use best_platform in a few places for cross-platform running (#…

…1020) Co-authored-by: Tim de Jager <[email protected]> Co-authored-by: Bas Zalmstra <[email protected]> Co-authored-by: Bas Zalmstra <[email protected]> Co-authored-by: Ruben Arts <[email protected]> Co-authored-by: Olivier Lacroix <[email protected]>
prefix-dev · May 6, 2024 · 34105df · 34105df
1 parent 3523ea3
commit 34105df
Show file tree

Hide file tree

Showing 19 changed files with 149 additions and 45 deletions.
diff --git a/schema/model.py b/schema/model.py
@@ -47,6 +47,8 @@
     | Literal["win-32"]
     | Literal["win-64"]
     | Literal["win-arm64"]
+    | Literal["emscripten-wasm32"]
+    | Literal["wasi-wasm32"]
 )
 
 

diff --git a/schema/schema.json b/schema/schema.json
@@ -671,6 +671,12 @@
               },
               {
                 "const": "win-arm64"
+              },
+              {
+                "const": "emscripten-wasm32"
+              },
+              {
+                "const": "wasi-wasm32"
               }
             ]
           }

diff --git a/src/cli/global/common.rs b/src/cli/global/common.rs
@@ -194,19 +194,16 @@ pub fn load_package_records(
 /// The network client and the fetched sparse repodata
 pub(super) async fn get_client_and_sparse_repodata(
     channels: impl IntoIterator<Item = &'_ Channel>,
+    platform: Platform,
     config: &Config,
 ) -> miette::Result<(
     ClientWithMiddleware,
     IndexMap<(Channel, Platform), SparseRepoData>,
 )> {
     let authenticated_client = build_reqwest_clients(Some(config)).1;
-    let platform_sparse_repodata = repodata::fetch_sparse_repodata(
-        channels,
-        [Platform::current()],
-        &authenticated_client,
-        Some(config),
-    )
-    .await?;
+    let platform_sparse_repodata =
+        repodata::fetch_sparse_repodata(channels, [platform], &authenticated_client, Some(config))
+            .await?;
     Ok((authenticated_client, platform_sparse_repodata))
 }
 

diff --git a/src/cli/global/install.rs b/src/cli/global/install.rs
@@ -42,6 +42,9 @@ pub struct Args {
     #[clap(short, long)]
     channel: Vec<String>,
 
+    #[clap(short, long, default_value_t = Platform::current())]
+    platform: Platform,
+
     #[clap(flatten)]
     config: ConfigCli,
 }
@@ -250,15 +253,20 @@ pub async fn execute(args: Args) -> miette::Result<()> {
 
     // Fetch sparse repodata
     let (authenticated_client, sparse_repodata) =
-        get_client_and_sparse_repodata(&channels, &config).await?;
+        get_client_and_sparse_repodata(&channels, args.platform, &config).await?;
 
     // Install the package(s)
     let mut executables = vec![];
     for (package_name, package_matchspec) in args.specs()? {
         let records = load_package_records(package_matchspec, &sparse_repodata)?;
 
-        let (prefix_package, scripts, _) =
-            globally_install_package(&package_name, records, authenticated_client.clone()).await?;
+        let (prefix_package, scripts, _) = globally_install_package(
+            &package_name,
+            records,
+            authenticated_client.clone(),
+            &args.platform,
+        )
+        .await?;
         let channel_name = channel_name_from_prefix(&prefix_package, config.channel_config());
         let record = &prefix_package.repodata_record.package_record;
 
@@ -323,6 +331,7 @@ pub(super) async fn globally_install_package(
     package_name: &PackageName,
     records: Vec<RepoDataRecord>,
     authenticated_client: ClientWithMiddleware,
+    platform: &Platform,
 ) -> miette::Result<(PrefixRecord, Vec<PathBuf>, bool)> {
     // Create the binary environment prefix where we install or update the package
     let BinEnvDir(bin_prefix) = BinEnvDir::create(package_name).await?;
@@ -331,7 +340,7 @@ pub(super) async fn globally_install_package(
 
     // Create the transaction that we need
     let transaction =
-        Transaction::from_current_and_desired(prefix_records.clone(), records, Platform::current())
+        Transaction::from_current_and_desired(prefix_records.clone(), records, *platform)
             .into_diagnostic()?;
 
     let has_transactions = !transaction.operations.is_empty();

diff --git a/src/cli/global/upgrade.rs b/src/cli/global/upgrade.rs
@@ -6,7 +6,7 @@ use indexmap::IndexMap;
 use indicatif::ProgressBar;
 use itertools::Itertools;
 use miette::IntoDiagnostic;
-use rattler_conda_types::{Channel, MatchSpec, PackageName};
+use rattler_conda_types::{Channel, MatchSpec, PackageName, Platform};
 
 use crate::config::Config;
 use crate::progress::{global_multi_progress, long_running_progress_style};
@@ -35,6 +35,10 @@ pub struct Args {
     /// the package was installed from will always be used.
     #[clap(short, long)]
     channel: Vec<String>,
+
+    /// The platform to install the package for.
+    #[clap(long, default_value_t = Platform::current())]
+    platform: Platform,
 }
 
 impl HasSpecs for Args {
@@ -46,13 +50,14 @@ impl HasSpecs for Args {
 pub async fn execute(args: Args) -> miette::Result<()> {
     let config = Config::load_global();
     let specs = args.specs()?;
-    upgrade_packages(specs, config, &args.channel).await
+    upgrade_packages(specs, config, &args.channel, &args.platform).await
 }
 
 pub(super) async fn upgrade_packages(
     specs: IndexMap<PackageName, MatchSpec>,
     config: Config,
     cli_channels: &[String],
+    platform: &Platform,
 ) -> miette::Result<()> {
     // Get channels and versions of globally installed packages
     let mut installed_versions = HashMap::with_capacity(specs.len());
@@ -79,7 +84,7 @@ pub(super) async fn upgrade_packages(
 
     // Fetch sparse repodata
     let (authenticated_client, sparse_repodata) =
-        get_client_and_sparse_repodata(&channels, &config).await?;
+        get_client_and_sparse_repodata(&channels, *platform, &config).await?;
 
     // Upgrade each package when relevant
     let mut upgraded = false;
@@ -119,7 +124,13 @@ pub(super) async fn upgrade_packages(
                 console::style("Updating").green(),
                 message
             ));
-            globally_install_package(&package_name, records, authenticated_client.clone()).await?;
+            globally_install_package(
+                &package_name,
+                records,
+                authenticated_client.clone(),
+                platform,
+            )
+            .await?;
             pb.finish_with_message(format!("{} {}", console::style("Updated").green(), message));
             upgraded = true;
         }

diff --git a/src/cli/global/upgrade_all.rs b/src/cli/global/upgrade_all.rs
@@ -1,7 +1,7 @@
 use clap::Parser;
 use indexmap::IndexMap;
 
-use rattler_conda_types::MatchSpec;
+use rattler_conda_types::{MatchSpec, Platform};
 
 use crate::config::{Config, ConfigCli};
 
@@ -24,6 +24,10 @@ pub struct Args {
 
     #[clap(flatten)]
     config: ConfigCli,
+
+    /// The platform to install the package for.
+    #[clap(long, default_value_t = Platform::current())]
+    platform: Platform,
 }
 
 pub async fn execute(args: Args) -> miette::Result<()> {
@@ -41,5 +45,5 @@ pub async fn execute(args: Args) -> miette::Result<()> {
         );
     }
 
-    upgrade_packages(specs, config, &args.channel).await
+    upgrade_packages(specs, config, &args.channel, &args.platform).await
 }
diff --git a/src/cli/info.rs b/src/cli/info.rs
@@ -331,12 +331,12 @@ pub async fn execute(args: Args) -> miette::Result<()> {
                             .map(|solve_group| solve_group.name().to_string()),
                         environment_size: None,
                         dependencies: env
-                            .dependencies(None, Some(Platform::current()))
+                            .dependencies(None, Some(env.best_platform()))
                             .names()
                             .map(|p| p.as_source().to_string())
                             .collect(),
                         pypi_dependencies: env
-                            .pypi_dependencies(Some(Platform::current()))
+                            .pypi_dependencies(Some(env.best_platform()))
                             .into_iter()
                             .map(|(name, _p)| name.as_source().to_string())
                             .collect(),

diff --git a/src/cli/list.rs b/src/cli/list.rs
@@ -119,7 +119,7 @@ pub async fn execute(args: Args) -> miette::Result<()> {
         .await?;
 
     // Load the platform
-    let platform = args.platform.unwrap_or_else(Platform::current);
+    let platform = args.platform.unwrap_or_else(|| environment.best_platform());
 
     // Get all the packages in the environment.
     let locked_deps = lock_file
@@ -140,7 +140,7 @@ pub async fn execute(args: Args) -> miette::Result<()> {
         uv_context = UvResolutionContext::from_project(&project)?;
         index_locations = environment.pypi_options().to_index_locations();
         tags = get_pypi_tags(
-            Platform::current(),
+            platform,
             &project.system_requirements(),
             python_record.package_record(),
         )?;

diff --git a/src/cli/run.rs b/src/cli/run.rs
@@ -103,7 +103,10 @@ pub async fn execute(args: Args) -> miette::Result<()> {
     let search_environment = SearchEnvironments::from_opt_env(
         &project,
         explicit_environment.clone(),
-        Some(Platform::current()),
+        explicit_environment
+            .as_ref()
+            .map(|e| e.best_platform())
+            .or(Some(Platform::current())),
     )
     .with_disambiguate_fn(disambiguate_task_interactive);
 

diff --git a/src/cli/tree.rs b/src/cli/tree.rs
@@ -78,7 +78,7 @@ pub async fn execute(args: Args) -> miette::Result<()> {
             ..UpdateLockFileOptions::default()
         })
         .await?;
-    let platform = args.platform.unwrap_or_else(Platform::current);
+    let platform = args.platform.unwrap_or_else(|| environment.best_platform());
     let locked_deps = lock_file
         .lock_file
         .environment(environment.name().as_str())

diff --git a/src/consts.rs b/src/consts.rs
@@ -13,6 +13,8 @@ pub const SOLVE_GROUP_ENVIRONMENTS_DIR: &str = "solve-group-envs";
 pub const PYPI_DEPENDENCIES: &str = "pypi-dependencies";
 pub const TASK_CACHE_DIR: &str = "task-cache-v0";
 
+pub const ONE_TIME_MESSAGES_DIR: &str = "one-time-messages";
+
 pub const DEFAULT_ENVIRONMENT_NAME: &str = "default";
 
 /// The default channels to use for a new project.

diff --git a/src/environment.rs b/src/environment.rs
@@ -211,7 +211,7 @@ pub async fn get_up_to_date_prefix(
     mut no_install: bool,
     existing_repo_data: IndexMap<(Channel, Platform), SparseRepoData>,
 ) -> miette::Result<Prefix> {
-    let current_platform = Platform::current();
+    let current_platform = environment.best_platform();
     let project = environment.project();
 
     // Do not install if the platform is not supported
@@ -255,6 +255,7 @@ pub async fn update_prefix_pypi(
     pypi_options: &PypiOptions,
     environment_variables: &HashMap<String, String>,
     lock_file_dir: &Path,
+    platform: Platform,
 ) -> miette::Result<()> {
     // Remove python packages from a previous python distribution if the python version changed.
 
@@ -275,6 +276,7 @@ pub async fn update_prefix_pypi(
                 uv_context,
                 pypi_options,
                 environment_variables,
+                platform,
             )
         },
     )

diff --git a/src/install_pypi.rs b/src/install_pypi.rs
@@ -696,6 +696,7 @@ pub async fn update_python_distributions(
     uv_context: UvResolutionContext,
     pypi_options: &PypiOptions,
     environment_variables: &HashMap<String, String>,
+    platform: Platform,
 ) -> miette::Result<()> {
     let start = std::time::Instant::now();
     let Some(python_info) = status.current_info() else {
@@ -716,11 +717,7 @@ pub async fn update_python_distributions(
         .iter()
         .find(|r| is_python_record(r))
         .ok_or_else(|| miette::miette!("could not resolve pypi dependencies because no python interpreter is added to the dependencies of the project.\nMake sure to add a python interpreter to the [dependencies] section of the {PROJECT_MANIFEST}, or run:\n\n\tpixi add python"))?;
-    let tags = get_pypi_tags(
-        Platform::current(),
-        system_requirements,
-        &python_record.package_record,
-    )?;
+    let tags = get_pypi_tags(platform, system_requirements, &python_record.package_record)?;
 
     let index_locations = pypi_options.to_index_locations();
     let registry_client = Arc::new(

diff --git a/src/lock_file/satisfiability.rs b/src/lock_file/satisfiability.rs
@@ -412,7 +412,20 @@ pub fn verify_package_platform_satisfiability(
     let marker_environment = python_interpreter_record
         .map(|interpreter| determine_marker_environment(platform, &interpreter.package_record))
         .transpose()
-        .map_err(|err| PlatformUnsat::FailedToDetermineMarkerEnvironment(err.into()))?;
+        .map_err(|err| PlatformUnsat::FailedToDetermineMarkerEnvironment(err.into()));
+
+    // We cannot determine the marker environment, for example if installing `wasm32` dependencies.
+    // However, it also doesn't really matter if we don't have any pypi requirements.
+    let marker_environment = match marker_environment {
+        Err(err) => {
+            if !pypi_requirements.is_empty() {
+                return Err(err);
+            } else {
+                None
+            }
+        }
+        Ok(marker_environment) => marker_environment,
+    };
 
     // Determine the pypi packages provided by the locked conda packages.
     let locked_conda_pypi_packages = locked_conda_packages.by_pypi_name();

diff --git a/src/lock_file/update.rs b/src/lock_file/update.rs
@@ -24,7 +24,7 @@ use indicatif::ProgressBar;
 use itertools::Itertools;
 use miette::{IntoDiagnostic, LabeledSpan, MietteDiagnostic, WrapErr};
 use rattler::package_cache::PackageCache;
-use rattler_conda_types::{Channel, MatchSpec, PackageName, Platform, RepoDataRecord};
+use rattler_conda_types::{Arch, Channel, MatchSpec, PackageName, Platform, RepoDataRecord};
 use rattler_lock::{LockFile, PypiPackageData, PypiPackageEnvironmentData};
 use rattler_repodata_gateway::sparse::SparseRepoData;
 use std::path::PathBuf;
@@ -104,13 +104,19 @@ impl<'p> LockFileDerivedData<'p> {
         }
 
         // Get the prefix with the conda packages installed.
-        let platform = Platform::current();
+        let platform = environment.best_platform();
         let (prefix, python_status) = self.conda_prefix(environment).await?;
         let repodata_records = self
             .repodata_records(environment, platform)
             .unwrap_or_default();
         let pypi_records = self.pypi_records(environment, platform).unwrap_or_default();
 
+        // No `uv` support for WASM right now
+        // TODO - figure out if we can create the `uv` context more lazily
+        if platform.arch() == Some(Arch::Wasm32) {
+            return Ok(prefix);
+        }
+
         let uv_context = match &self.uv_context {
             None => {
                 let context = UvResolutionContext::from_project(self.project)?;
@@ -134,6 +140,7 @@ impl<'p> LockFileDerivedData<'p> {
             &environment.pypi_options(),
             env_variables,
             self.project.root(),
+            environment.best_platform(),
         )
         .await?;
 
@@ -178,7 +185,7 @@ impl<'p> LockFileDerivedData<'p> {
         }
 
         let prefix = Prefix::new(environment.dir());
-        let platform = Platform::current();
+        let platform = environment.best_platform();
 
         // Determine the currently installed packages.
         let installed_packages = prefix
@@ -648,7 +655,9 @@ pub async fn ensure_up_to_date_lock_file(
         // we solve the platforms. We want to solve the current platform first, so we can start
         // instantiating prefixes if we have to.
         let mut ordered_platforms = platforms.into_iter().collect::<IndexSet<_>>();
-        if let Some(current_platform_index) = ordered_platforms.get_index_of(&current_platform) {
+        if let Some(current_platform_index) =
+            ordered_platforms.get_index_of(&environment.best_platform())
+        {
             ordered_platforms.move_index(current_platform_index, 0);
         }
 
@@ -734,7 +743,7 @@ pub async fn ensure_up_to_date_lock_file(
         // Construct a future that will resolve when we have the repodata available for the current
         // platform for this group.
         let records_future = context
-            .get_latest_group_repodata_records(&group, current_platform)
+            .get_latest_group_repodata_records(&group, environment.best_platform())
             .ok_or_else(|| make_unsupported_pypi_platform_error(environment, current_platform))?;
 
         // Spawn a task to instantiate the environment