Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: use best_platform in a few places for cross-platform running #1020

Merged
merged 18 commits into from
May 6, 2024
Merged
Show file tree
Hide file tree
Changes from 9 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions schema/model.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,8 @@
| Literal["win-32"]
| Literal["win-64"]
| Literal["win-arm64"]
| Literal["emscripten-wasm32"]
| Literal["wasi-wasm32"]
)


Expand Down
6 changes: 6 additions & 0 deletions schema/schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -629,6 +629,12 @@
},
{
"const": "win-arm64"
},
{
"const": "emscripten-wasm32"
},
{
"const": "wasi-wasm32"
}
]
}
Expand Down
15 changes: 12 additions & 3 deletions src/cli/global/install.rs
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,9 @@ pub struct Args {
#[clap(short, long)]
channel: Vec<String>,

#[clap(short, long, default_value_t = Platform::current())]
platform: Platform,
ruben-arts marked this conversation as resolved.
Show resolved Hide resolved

#[clap(flatten)]
config: ConfigCli,
}
Expand Down Expand Up @@ -262,8 +265,13 @@ pub async fn execute(args: Args) -> miette::Result<()> {
let package_name = package_name(&package_matchspec)?;
let records = load_package_records(package_matchspec, &sparse_repodata)?;

let (prefix_package, scripts, _) =
globally_install_package(&package_name, records, authenticated_client.clone()).await?;
let (prefix_package, scripts, _) = globally_install_package(
&package_name,
records,
authenticated_client.clone(),
&args.platform,
)
.await?;
let channel_name = channel_name_from_prefix(&prefix_package, config.channel_config());
let record = &prefix_package.repodata_record.package_record;

Expand Down Expand Up @@ -328,6 +336,7 @@ pub(super) async fn globally_install_package(
package_name: &PackageName,
records: Vec<RepoDataRecord>,
authenticated_client: ClientWithMiddleware,
platform: &Platform,
) -> miette::Result<(PrefixRecord, Vec<PathBuf>, bool)> {
// Create the binary environment prefix where we install or update the package
let BinEnvDir(bin_prefix) = BinEnvDir::create(package_name).await?;
Expand All @@ -336,7 +345,7 @@ pub(super) async fn globally_install_package(

// Create the transaction that we need
let transaction =
Transaction::from_current_and_desired(prefix_records.clone(), records, Platform::current())
Transaction::from_current_and_desired(prefix_records.clone(), records, *platform)
.into_diagnostic()?;

let has_transactions = !transaction.operations.is_empty();
Expand Down
10 changes: 8 additions & 2 deletions src/cli/global/upgrade.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ use clap::Parser;
use indicatif::ProgressBar;
use itertools::Itertools;
use miette::IntoDiagnostic;
use rattler_conda_types::{Channel, MatchSpec, PackageName, Version};
use rattler_conda_types::{Channel, MatchSpec, PackageName, Platform, Version};
use rattler_conda_types::{ParseStrictness, RepoDataRecord};
use reqwest_middleware::ClientWithMiddleware;

Expand Down Expand Up @@ -35,6 +35,10 @@ pub struct Args {
/// the package was installed from will always be used.
#[clap(short, long)]
channel: Vec<String>,

/// The platform to install the package for.
#[clap(long, default_value_t = Platform::current())]
platform: Platform,
}

pub async fn execute(args: Args) -> miette::Result<()> {
Expand Down Expand Up @@ -115,6 +119,7 @@ pub async fn execute(args: Args) -> miette::Result<()> {
toinstall_version,
records,
authenticated_client,
&args.platform,
)
.await
}
Expand All @@ -125,6 +130,7 @@ pub(super) async fn upgrade_package(
toinstall_version: Version,
records: Vec<RepoDataRecord>,
authenticated_client: ClientWithMiddleware,
platform: &Platform,
) -> miette::Result<()> {
let message = format!(
"{} v{} -> v{}",
Expand All @@ -141,7 +147,7 @@ pub(super) async fn upgrade_package(
console::style("Updating").green(),
message
));
globally_install_package(package_name, records, authenticated_client).await?;
globally_install_package(package_name, records, authenticated_client, platform).await?;
pb.finish_with_message(format!("{} {}", console::style("Updated").green(), message));
Ok(())
}
7 changes: 6 additions & 1 deletion src/cli/global/upgrade_all.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ use std::collections::HashMap;
use clap::Parser;
use itertools::Itertools;
use miette::IntoDiagnostic;
use rattler_conda_types::{Channel, MatchSpec, ParseStrictness};
use rattler_conda_types::{Channel, MatchSpec, ParseStrictness, Platform};

use crate::config::{Config, ConfigCli};

Expand All @@ -30,6 +30,10 @@ pub struct Args {

#[clap(flatten)]
config: ConfigCli,

/// The platform to install the package for.
#[clap(long, default_value_t = Platform::current())]
platform: Platform,
ruben-arts marked this conversation as resolved.
Show resolved Hide resolved
}

pub async fn execute(args: Args) -> miette::Result<()> {
Expand Down Expand Up @@ -93,6 +97,7 @@ pub async fn execute(args: Args) -> miette::Result<()> {
toinstall_version,
records,
authenticated_client.clone(),
&args.platform,
)
.await?;
upgraded = true;
Expand Down
4 changes: 2 additions & 2 deletions src/cli/info.rs
Original file line number Diff line number Diff line change
Expand Up @@ -321,12 +321,12 @@ pub async fn execute(args: Args) -> miette::Result<()> {
.map(|solve_group| solve_group.name().to_string()),
environment_size: None,
dependencies: env
.dependencies(None, Some(Platform::current()))
.dependencies(None, Some(env.best_platform()))
.names()
.map(|p| p.as_source().to_string())
.collect(),
pypi_dependencies: env
.pypi_dependencies(Some(Platform::current()))
.pypi_dependencies(Some(env.best_platform()))
.into_iter()
.map(|(name, _p)| name.as_source().to_string())
.collect(),
Expand Down
4 changes: 2 additions & 2 deletions src/cli/list.rs
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ pub async fn execute(args: Args) -> miette::Result<()> {
.await?;

// Load the platform
let platform = args.platform.unwrap_or_else(Platform::current);
let platform = args.platform.unwrap_or_else(|| environment.best_platform());

// Get all the packages in the environment.
let locked_deps = lock_file
Expand All @@ -137,7 +137,7 @@ pub async fn execute(args: Args) -> miette::Result<()> {
let mut registry_index = if let Some(python_record) = python_record {
uv_context = UvResolutionContext::from_project(&project)?;
tags = get_pypi_tags(
Platform::current(),
platform,
&project.system_requirements(),
python_record.package_record(),
)?;
Expand Down
9 changes: 6 additions & 3 deletions src/cli/run.rs
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,10 @@ pub async fn execute(args: Args) -> miette::Result<()> {
let search_environment = SearchEnvironments::from_opt_env(
&project,
explicit_environment.clone(),
Some(Platform::current()),
explicit_environment
.as_ref()
.map(|e| e.best_platform())
.or(Some(Platform::current())),
)
.with_disambiguate_fn(disambiguate_task_interactive);

Expand Down Expand Up @@ -205,7 +208,7 @@ fn command_not_found<'p>(project: &'p Project, explicit_environment: Option<Envi
let available_tasks: HashSet<TaskName> =
if let Some(explicit_environment) = explicit_environment {
explicit_environment
.tasks(Some(Platform::current()))
.tasks(Some(explicit_environment.best_platform()))
.into_iter()
.flat_map(|tasks| tasks.into_keys())
.map(ToOwned::to_owned)
Expand All @@ -216,7 +219,7 @@ fn command_not_found<'p>(project: &'p Project, explicit_environment: Option<Envi
.into_iter()
.filter(|env| verify_current_platform_has_required_virtual_packages(env).is_ok())
.flat_map(|env| {
env.tasks(Some(Platform::current()))
env.tasks(Some(env.best_platform()))
.into_iter()
.flat_map(|tasks| tasks.into_keys())
.map(ToOwned::to_owned)
Expand Down
2 changes: 1 addition & 1 deletion src/cli/task.rs
Original file line number Diff line number Diff line change
Expand Up @@ -275,7 +275,7 @@ pub fn execute(args: Args) -> miette::Result<()> {
Operation::List(args) => {
let environment = project.environment_from_name_or_env_var(args.environment)?;
let tasks = environment
.tasks(Some(Platform::current()))?
.tasks(Some(environment.best_platform()))?
.into_keys()
.collect_vec();
if tasks.is_empty() {
Expand Down
2 changes: 1 addition & 1 deletion src/cli/tree.rs
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ pub async fn execute(args: Args) -> miette::Result<()> {
..UpdateLockFileOptions::default()
})
.await?;
let platform = args.platform.unwrap_or_else(Platform::current);
let platform = args.platform.unwrap_or_else(|| environment.best_platform());
let locked_deps = lock_file
.lock_file
.environment(environment.name().as_str())
Expand Down
1 change: 1 addition & 0 deletions src/consts.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ pub const ENVIRONMENTS_DIR: &str = "envs";
pub const SOLVE_GROUP_ENVIRONMENTS_DIR: &str = "solve-group-envs";
pub const PYPI_DEPENDENCIES: &str = "pypi-dependencies";
pub const TASK_CACHE_DIR: &str = "task-cache-v0";
pub const MACOS_EMULATION_WARN: &str = "macos-emulation-warn";

pub const DEFAULT_ENVIRONMENT_NAME: &str = "default";

Expand Down
4 changes: 3 additions & 1 deletion src/environment.rs
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,7 @@ pub async fn get_up_to_date_prefix(
mut no_install: bool,
existing_repo_data: IndexMap<(Channel, Platform), SparseRepoData>,
) -> miette::Result<Prefix> {
let current_platform = Platform::current();
let current_platform = environment.best_platform();
let project = environment.project();

// Do not install if the platform is not supported
Expand Down Expand Up @@ -253,6 +253,7 @@ pub async fn update_prefix_pypi(
uv_context: UvResolutionContext,
environment_variables: &HashMap<String, String>,
lock_file_dir: &Path,
platform: Platform,
) -> miette::Result<()> {
// Remove python packages from a previous python distribution if the python version changed.

Expand All @@ -272,6 +273,7 @@ pub async fn update_prefix_pypi(
system_requirements,
uv_context,
environment_variables,
platform,
)
},
)
Expand Down
7 changes: 2 additions & 5 deletions src/install_pypi.rs
Original file line number Diff line number Diff line change
Expand Up @@ -692,6 +692,7 @@ pub async fn update_python_distributions(
system_requirements: &SystemRequirements,
uv_context: UvResolutionContext,
environment_variables: &HashMap<String, String>,
platform: Platform,
) -> miette::Result<()> {
let start = std::time::Instant::now();
let Some(python_info) = status.current_info() else {
Expand All @@ -712,11 +713,7 @@ pub async fn update_python_distributions(
.iter()
.find(|r| is_python_record(r))
.ok_or_else(|| miette::miette!("could not resolve pypi dependencies because no python interpreter is added to the dependencies of the project.\nMake sure to add a python interpreter to the [dependencies] section of the {PROJECT_MANIFEST}, or run:\n\n\tpixi add python"))?;
let tags = get_pypi_tags(
Platform::current(),
system_requirements,
&python_record.package_record,
)?;
let tags = get_pypi_tags(platform, system_requirements, &python_record.package_record)?;

// Resolve the flat indexes from `--find-links`.
let flat_index = {
Expand Down
15 changes: 14 additions & 1 deletion src/lock_file/satisfiability.rs
Original file line number Diff line number Diff line change
Expand Up @@ -385,7 +385,20 @@ pub fn verify_package_platform_satisfiability(
let marker_environment = python_interpreter_record
.map(|interpreter| determine_marker_environment(platform, &interpreter.package_record))
.transpose()
.map_err(|err| PlatformUnsat::FailedToDetermineMarkerEnvironment(err.into()))?;
.map_err(|err| PlatformUnsat::FailedToDetermineMarkerEnvironment(err.into()));

// We cannot determine the marker environment, for example if installing `wasm32` dependencies.
// However, it also doesn't really matter if we don't have any pypi requirements.
let marker_environment = match marker_environment {
Err(err) => {
if !pypi_requirements.is_empty() {
return Err(err);
} else {
None
}
}
Ok(marker_environment) => marker_environment,
};

// Determine the pypi packages provided by the locked conda packages.
let locked_conda_pypi_packages = locked_conda_packages.by_pypi_name();
Expand Down
19 changes: 14 additions & 5 deletions src/lock_file/update.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ use indicatif::ProgressBar;
use itertools::Itertools;
use miette::{IntoDiagnostic, LabeledSpan, MietteDiagnostic, WrapErr};
use rattler::package_cache::PackageCache;
use rattler_conda_types::{Channel, MatchSpec, PackageName, Platform, RepoDataRecord};
use rattler_conda_types::{Arch, Channel, MatchSpec, PackageName, Platform, RepoDataRecord};
use rattler_lock::{LockFile, PypiPackageData, PypiPackageEnvironmentData};
use rattler_repodata_gateway::sparse::SparseRepoData;
use std::path::PathBuf;
Expand Down Expand Up @@ -104,13 +104,19 @@ impl<'p> LockFileDerivedData<'p> {
}

// Get the prefix with the conda packages installed.
let platform = Platform::current();
let platform = environment.best_platform();
let (prefix, python_status) = self.conda_prefix(environment).await?;
let repodata_records = self
.repodata_records(environment, platform)
.unwrap_or_default();
let pypi_records = self.pypi_records(environment, platform).unwrap_or_default();

// No `uv` support for WASM right now
// TODO - figure out if we can create the `uv` context more lazily
if platform.arch() == Some(Arch::Wasm32) {
return Ok(prefix);
}

let uv_context = match &self.uv_context {
None => {
let context = UvResolutionContext::from_project(self.project)?;
Expand All @@ -133,6 +139,7 @@ impl<'p> LockFileDerivedData<'p> {
uv_context,
env_variables,
self.project.root(),
environment.best_platform(),
)
.await?;

Expand Down Expand Up @@ -177,7 +184,7 @@ impl<'p> LockFileDerivedData<'p> {
}

let prefix = Prefix::new(environment.dir());
let platform = Platform::current();
let platform = environment.best_platform();

// Determine the currently installed packages.
let installed_packages = prefix
Expand Down Expand Up @@ -647,7 +654,9 @@ pub async fn ensure_up_to_date_lock_file(
// we solve the platforms. We want to solve the current platform first, so we can start
// instantiating prefixes if we have to.
let mut ordered_platforms = platforms.into_iter().collect::<IndexSet<_>>();
if let Some(current_platform_index) = ordered_platforms.get_index_of(&current_platform) {
if let Some(current_platform_index) =
ordered_platforms.get_index_of(&environment.best_platform())
{
ordered_platforms.move_index(current_platform_index, 0);
}

Expand Down Expand Up @@ -733,7 +742,7 @@ pub async fn ensure_up_to_date_lock_file(
// Construct a future that will resolve when we have the repodata available for the current
// platform for this group.
let records_future = context
.get_latest_group_repodata_records(&group, current_platform)
.get_latest_group_repodata_records(&group, environment.best_platform())
.ok_or_else(|| make_unsupported_pypi_platform_error(environment, current_platform))?;

// Spawn a task to instantiate the environment
Expand Down
Loading
Loading