diff --git a/docs/reference/cli/pixi/import.md b/docs/reference/cli/pixi/import.md index a17891b948..764598ccea 100644 --- a/docs/reference/cli/pixi/import.md +++ b/docs/reference/cli/pixi/import.md @@ -19,7 +19,7 @@ pixi import [OPTIONS] ## Options - `--format ` : Which format to interpret the file as -
**options**: `conda-env` +
**options**: `conda-env`, `pypi-txt` - `--platform (-p) ` : The platforms for the imported environment
May be provided more than once. @@ -69,7 +69,7 @@ pixi import [OPTIONS] ## Description Imports a file into an environment in an existing workspace. -If `--format` isn't provided, `import` will try to guess the format based on the file extension. +If `--format` isn't provided, `import` will try each format in turn --8<-- "docs/reference/cli/pixi/import_extender:example" diff --git a/src/cli/import.rs b/src/cli/import.rs index 75592bbfff..47023188ad 100644 --- a/src/cli/import.rs +++ b/src/cli/import.rs @@ -5,8 +5,13 @@ use clap::{Parser, ValueEnum}; use pixi_config::{Config, ConfigCli}; use pixi_manifest::{EnvironmentName, FeatureName, HasFeaturesIter, PrioritizedChannel}; use pixi_utils::conda_environment_file::CondaEnvFile; +use pixi_uv_conversions::convert_uv_requirements_to_pep508; use rattler_conda_types::Platform; +use tracing::warn; +use uv_client::BaseClientBuilder; +use uv_requirements_txt::RequirementsTxt; + use miette::{Diagnostic, IntoDiagnostic, Result}; use thiserror::Error; @@ -19,14 +24,15 @@ use crate::{ #[derive(Parser, Debug, Clone, PartialEq, ValueEnum)] pub enum ImportFileFormat { - // TODO: implement conda-lock, conda-txt, pypi-txt + // TODO: implement conda-lock, conda-txt CondaEnv, + PypiTxt, } /// Imports a file into an environment in an existing workspace. /// -/// If `--format` isn't provided, `import` will try to guess the format based on the file extension. -#[derive(Parser, Debug, Default)] +/// If `--format` isn't provided, `import` will try each format in turn +#[derive(Parser, Debug, Default, Clone)] #[clap(arg_required_else_help = true, verbatim_doc_comment)] pub struct Args { #[clap(flatten)] @@ -64,32 +70,29 @@ pub struct Args { pub async fn execute(args: Args) -> miette::Result<()> { if let Some(format) = &args.format { - if *format != ImportFileFormat::CondaEnv { - miette::bail!( - "Only the conda environment.yml format is supported currently. Please pass `conda-env` to `format`." - ); - } - import_conda_env(args).await + import(args.clone(), format).await + } else if let Ok(result) = import(args.clone(), &ImportFileFormat::CondaEnv).await { + return Ok(result); + } else if let Ok(result) = import(args, &ImportFileFormat::PypiTxt).await { + return Ok(result); } else { - import_conda_env(args).await // .or_else(...) + miette::bail!( + "Tried all formats for input file, but none were successful. Pass a `format` argument to see the specific error for that format." + ) } } #[derive(Debug, Error, Diagnostic)] -#[error("Missing name: provide --feature or --environment, or set `name:` in input file.")] +#[error( + "Missing name: provide --feature or --environment, or set `name:` in input file for the conda-env format." +)] struct MissingEnvironmentName; fn get_feature_and_environment( feature_arg: &Option, environment_arg: &Option, - file: &CondaEnvFile, -) -> Result<(String, String), miette::Report> { - let fallback = || { - file.name() - .map(|s| s.to_string()) - .ok_or(MissingEnvironmentName) - }; - + fallback: impl Fn() -> Result, +) -> Result<(FeatureName, EnvironmentName), miette::Report> { let feature_string = match (feature_arg, environment_arg) { (Some(f), _) => f.clone(), (_, Some(e)) => e.clone(), @@ -102,11 +105,40 @@ fn get_feature_and_environment( _ => fallback()?, }; - Ok((feature_string, environment_string)) + Ok(( + FeatureName::from(feature_string), + EnvironmentName::from_str(&environment_string)?, + )) +} + +fn convert_uv_requirements_txt_to_pep508( + reqs_txt: uv_requirements_txt::RequirementsTxt, +) -> Result, miette::Error> { + let uv_requirements: Vec> = reqs_txt + .requirements + .into_iter() + .map(|r| match r.requirement { + uv_requirements_txt::RequirementsTxtRequirement::Named(req) => Ok(req), + uv_requirements_txt::RequirementsTxtRequirement::Unnamed(_) => Err(miette::miette!( + "Error parsing input file: unnamed requirements are currently unsupported." + )), + }) + .collect::>()?; + if !reqs_txt.constraints.is_empty() { + warn!( + "Constraints detected in input file, but these are currently unsupported. Continuing without applying constraints..." + ) + } + + let requirements = + convert_uv_requirements_to_pep508(uv_requirements.iter()).into_diagnostic()?; + + Ok(requirements) } -async fn import_conda_env(args: Args) -> miette::Result<()> { - let (file, platforms, workspace_config) = (args.file, args.platforms, args.workspace_config); +async fn import(args: Args, format: &ImportFileFormat) -> miette::Result<()> { + let (input_file, platforms, workspace_config) = + (args.file, args.platforms, args.workspace_config); let config = Config::from(args.config); let workspace = WorkspaceLocator::for_cli() @@ -120,10 +152,37 @@ async fn import_conda_env(args: Args) -> miette::Result<()> { // TODO: add dry_run logic to import - let file = CondaEnvFile::from_path(&file)?; - let (feature_string, environment_string) = - get_feature_and_environment(&args.feature, &args.environment, &file)?; - let feature_name = FeatureName::from(feature_string.clone()); + enum ProcessedInput { + CondaEnv(CondaEnvFile), + PypiTxt, + } + + let (processed_input, feature_name, environment_name) = match format { + ImportFileFormat::CondaEnv => { + let env_file = CondaEnvFile::from_path(&input_file)?; + let fallback = || { + env_file + .name() + .map(|s| s.to_string()) + .ok_or(MissingEnvironmentName) + }; + let (feature_name, environment_name) = + get_feature_and_environment(&args.feature, &args.environment, fallback)?; + + ( + ProcessedInput::CondaEnv(env_file), + feature_name, + environment_name, + ) + } + ImportFileFormat::PypiTxt => { + let (feature_name, environment_name) = + get_feature_and_environment(&args.feature, &args.environment, || { + Err(MissingEnvironmentName) + })?; + (ProcessedInput::PypiTxt, feature_name, environment_name) + } + }; // Add the platforms if they are not already present if !platforms.is_empty() { @@ -132,29 +191,44 @@ async fn import_conda_env(args: Args) -> miette::Result<()> { .add_platforms(platforms.iter(), &feature_name)?; } - // TODO: handle `variables` section - // let env_vars = file.variables(); + let (conda_deps, pypi_deps) = match processed_input { + ProcessedInput::CondaEnv(env_file) => { + // TODO: handle `variables` section + // let env_vars = file.variables(); + + // TODO: Improve this: + // - Use .condarc as channel config + let (conda_deps, pypi_deps, channels) = env_file.to_manifest(&config.clone())?; + workspace.manifest().add_channels( + channels.iter().map(|c| PrioritizedChannel::from(c.clone())), + &feature_name, + false, + )?; - // TODO: Improve this: - // - Use .condarc as channel config - let (conda_deps, pypi_deps, channels) = file.to_manifest(&config.clone())?; - workspace.manifest().add_channels( - channels.iter().map(|c| PrioritizedChannel::from(c.clone())), - &feature_name, - false, - )?; + (conda_deps, pypi_deps) + } + ProcessedInput::PypiTxt => { + let reqs_txt = RequirementsTxt::parse( + &input_file, + workspace.workspace().root(), + &BaseClientBuilder::new(), + ) + .await + .into_diagnostic()?; + let pypi_deps = convert_uv_requirements_txt_to_pep508(reqs_txt)?; + + (vec![], pypi_deps) + } + }; workspace.add_specs(conda_deps, pypi_deps, &platforms, &feature_name)?; - match workspace - .workspace() - .environment(&EnvironmentName::from_str(&environment_string)?) - { + match workspace.workspace().environment(&environment_name) { None => { // add environment if it does not already exist workspace.manifest().add_environment( - environment_string.clone(), - Some(vec![feature_string.clone()]), + environment_name.to_string(), + Some(vec![feature_name.to_string()]), None, true, )?; @@ -167,7 +241,7 @@ async fn import_conda_env(args: Args) -> miette::Result<()> { let features = env .features() .map(|f| f.name.as_str().to_string()) - .chain(std::iter::once(feature_string)) + .chain(std::iter::once(feature_name.to_string())) .collect(); Some(features) }; diff --git a/tests/integration_python/test_import.py b/tests/integration_python/test_import.py index 1c4f8962b7..e3edf20253 100644 --- a/tests/integration_python/test_import.py +++ b/tests/integration_python/test_import.py @@ -13,36 +13,13 @@ ) -class TestCondaEnv: +class TestImport: simple_env_yaml = { "name": "simple-env", "channels": ["conda-forge"], "dependencies": ["python"], } - cowpy_env_yaml = { - "name": "cowpy", - "channels": ["conda-forge"], - "dependencies": ["cowpy"], - } - - noname_env_yaml = { - "channels": ["conda-forge"], - "dependencies": ["python"], - } - - xpx_env_yaml = { - "name": "array-api-extra", - "channels": ["conda-forge"], - "dependencies": ["array-api-extra"], - } - - complex_env_yaml = { - "name": "complex-env", - "channels": ["conda-forge", "bioconda"], - "dependencies": ["cowpy=1.1.4", "libblas=*=*openblas", "snakemake-minimal"], - } - def test_import_invalid_format(self, pixi: Path, tmp_pixi_workspace: Path) -> None: manifest_path = tmp_pixi_workspace / "pixi.toml" @@ -67,6 +44,37 @@ def test_import_invalid_format(self, pixi: Path, tmp_pixi_workspace: Path) -> No stderr_contains="invalid value 'foobar' for '--format '", ) + +class TestCondaEnv: + simple_env_yaml = { + "name": "simple-env", + "channels": ["conda-forge"], + "dependencies": ["python"], + } + + cowpy_env_yaml = { + "name": "cowpy", + "channels": ["conda-forge"], + "dependencies": ["cowpy"], + } + + noname_env_yaml = { + "channels": ["conda-forge"], + "dependencies": ["python"], + } + + xpx_env_yaml = { + "name": "array-api-extra", + "channels": ["conda-forge"], + "dependencies": ["array-api-extra"], + } + + complex_env_yaml = { + "name": "complex-env", + "channels": ["conda-forge", "bioconda"], + "dependencies": ["cowpy=1.1.4", "libblas=*=*openblas", "snakemake-minimal"], + } + def test_import_conda_env(self, pixi: Path, tmp_pixi_workspace: Path) -> None: manifest_path = tmp_pixi_workspace / "pixi.toml" @@ -113,6 +121,7 @@ def test_import_conda_env(self, pixi: Path, tmp_pixi_workspace: Path) -> None: ) def test_import_no_format(self, pixi: Path, tmp_pixi_workspace: Path) -> None: + # should default to CondaEnv manifest_path = tmp_pixi_workspace / "pixi.toml" import_file_path = tmp_pixi_workspace / "simple_environment.yml" @@ -168,6 +177,7 @@ def test_import_no_name(self, pixi: Path, tmp_pixi_workspace: Path) -> None: [ pixi, "import", + "--format=conda-env", "--manifest-path", manifest_path, import_file_path, @@ -409,7 +419,7 @@ def test_import_feature_environment(self, pixi: Path, tmp_pixi_workspace: Path) "--manifest-path", manifest_path, import_file_path, - "--feature=data", + "--environment=data", ], ) parsed_manifest = tomllib.loads(manifest_path.read_text()) @@ -485,3 +495,330 @@ def test_import_channels_and_versions(self, pixi: Path, tmp_pixi_workspace: Path }, } ) + + +class TestPypiTxt: + simple_txt = "cowpy" + xpx_txt = "array-api-extra" + numpy_txt = "numpy<2" + complex_txt = """ +-c numpy_requirements.txt +cowpy==1.1.4 +-r xpx_requirements.txt +""" + + def test_pypi_txt(self, pixi: Path, tmp_pixi_workspace: Path) -> None: + manifest_path = tmp_pixi_workspace / "pixi.toml" + + import_file_path = tmp_pixi_workspace / "simple_requirements.txt" + with open(import_file_path, "w") as file: + file.write(self.simple_txt) + + # Create a new project + verify_cli_command([pixi, "init", tmp_pixi_workspace]) + + # Import a simple environment + verify_cli_command( + [ + pixi, + "import", + "--manifest-path", + manifest_path, + import_file_path, + "--format=pypi-txt", + "--feature=simple-env", + ], + ) + + # check that no environments are installed + assert not os.path.isdir(tmp_pixi_workspace / ".pixi/envs") + + parsed_manifest = tomllib.loads(manifest_path.read_text()) + assert "cowpy" in parsed_manifest["feature"]["simple-env"]["pypi-dependencies"] + assert parsed_manifest == snapshot( + { + # these keys are irrelevant and some are machine-dependent + "workspace": IsPartialDict, + "tasks": {}, + "dependencies": {}, + "feature": {"simple-env": {"pypi-dependencies": {"cowpy": "*"}}}, + "environments": { + "simple-env": {"features": ["simple-env"], "no-default-feature": True} + }, + } + ) + + def test_no_name(self, pixi: Path, tmp_pixi_workspace: Path) -> None: + manifest_path = tmp_pixi_workspace / "pixi.toml" + + import_file_path = tmp_pixi_workspace / "simple_requirements.txt" + with open(import_file_path, "w") as file: + file.write(self.simple_txt) + + # Create a new project + verify_cli_command([pixi, "init", tmp_pixi_workspace]) + + verify_cli_command( + [ + pixi, + "import", + "--manifest-path", + manifest_path, + import_file_path, + "--format=pypi-txt", + ], + ExitCode.FAILURE, + stderr_contains="Missing name: provide --feature or --environment", + ) + + # Providing a feature name succeeds + verify_cli_command( + [ + pixi, + "import", + "--manifest-path", + manifest_path, + import_file_path, + "--feature=foobar", + ], + ) + + def test_platforms(self, pixi: Path, tmp_pixi_workspace: Path) -> None: + manifest_path = tmp_pixi_workspace / "pixi.toml" + + import_file_path = tmp_pixi_workspace / "simple_requirements.txt" + with open(import_file_path, "w") as file: + file.write(self.simple_txt) + + # Create a new project + verify_cli_command([pixi, "init", tmp_pixi_workspace]) + + # Import an environment for linux-64 only + verify_cli_command( + [ + pixi, + "import", + "--manifest-path", + manifest_path, + import_file_path, + "--format=pypi-txt", + "--feature=simple-env", + "--platform=linux-64", + ], + ) + + parsed_manifest = tomllib.loads(manifest_path.read_text()) + assert ( + "cowpy" + in parsed_manifest["feature"]["simple-env"]["target"]["linux-64"]["pypi-dependencies"] + ) + assert "osx-arm64" not in parsed_manifest["feature"]["simple-env"]["target"] + assert parsed_manifest == snapshot( + { + # these keys are irrelevant and some are machine-dependent + "workspace": IsPartialDict, + "tasks": {}, + "dependencies": {}, + "feature": { + "simple-env": { + "platforms": ["linux-64"], + "target": {"linux-64": {"pypi-dependencies": {"cowpy": "*"}}}, + } + }, + "environments": { + "simple-env": {"features": ["simple-env"], "no-default-feature": True} + }, + } + ) + + def test_feature_environment(self, pixi: Path, tmp_pixi_workspace: Path) -> None: + manifest_path = tmp_pixi_workspace / "pixi.toml" + + import_file_path = tmp_pixi_workspace / "simple_requirements.txt" + with open(import_file_path, "w") as file: + file.write(self.simple_txt) + + # Create a new project + verify_cli_command([pixi, "init", tmp_pixi_workspace]) + + # by default, a new env and feature are created with the same name when one of the flags + # is provided. The env has no-default-feature. + verify_cli_command( + [ + pixi, + "import", + "--manifest-path", + manifest_path, + import_file_path, + "--format=pypi-txt", + "--feature=simple-env", + ], + ) + parsed_manifest = tomllib.loads(manifest_path.read_text()) + assert "simple-env" in parsed_manifest["environments"]["simple-env"]["features"] + assert parsed_manifest["environments"]["simple-env"]["no-default-feature"] is True + assert parsed_manifest == snapshot( + { + # these keys are irrelevant and some are machine-dependent + "workspace": IsPartialDict, + "tasks": {}, + "dependencies": {}, + "feature": {"simple-env": {"pypi-dependencies": {"cowpy": "*"}}}, + "environments": { + "simple-env": {"features": ["simple-env"], "no-default-feature": True} + }, + } + ) + + # we can import into an existing feature + import_file_path = tmp_pixi_workspace / "xpx_requirements.txt" + with open(import_file_path, "w") as file: + file.write(self.xpx_txt) + + verify_cli_command( + [ + pixi, + "import", + "--manifest-path", + manifest_path, + import_file_path, + "--format=pypi-txt", + "--feature=simple-env", + ], + ) + parsed_manifest = tomllib.loads(manifest_path.read_text()) + assert "array-api-extra" in parsed_manifest["feature"]["simple-env"]["pypi-dependencies"] + assert parsed_manifest == snapshot( + { + # these keys are irrelevant and some are machine-dependent + "workspace": IsPartialDict, + "tasks": {}, + "dependencies": {}, + "feature": { + "simple-env": {"pypi-dependencies": {"cowpy": "*", "array-api-extra": "*"}} + }, + "environments": { + "simple-env": {"features": ["simple-env"], "no-default-feature": True} + }, + } + ) + + # we can create a new feature and add it to an existing environment + import_file_path = tmp_pixi_workspace / "numpy_requirements.txt" + with open(import_file_path, "w") as file: + file.write(self.numpy_txt) + + verify_cli_command( + [ + pixi, + "import", + "--manifest-path", + manifest_path, + import_file_path, + "--format=pypi-txt", + "--environment=simple-env", + "--feature=numpy", + ], + ) + parsed_manifest = tomllib.loads(manifest_path.read_text()) + assert "numpy" in parsed_manifest["feature"]["numpy"]["pypi-dependencies"] + assert "numpy" in parsed_manifest["environments"]["simple-env"]["features"] + # no new environment should be created + assert "numpy" not in parsed_manifest["environments"] + assert parsed_manifest == snapshot( + { + # these keys are irrelevant and some are machine-dependent + "workspace": IsPartialDict, + "tasks": {}, + "dependencies": {}, + "feature": { + "simple-env": {"pypi-dependencies": {"cowpy": "*", "array-api-extra": "*"}}, + "numpy": {"pypi-dependencies": {"numpy": "<2"}}, + }, + "environments": { + "simple-env": {"features": ["simple-env", "numpy"], "no-default-feature": True} + }, + } + ) + + # we can create a new env (and a matching feature by default) + import_file_path = tmp_pixi_workspace / "xpx_requirements.txt" + verify_cli_command( + [ + pixi, + "import", + "--manifest-path", + manifest_path, + import_file_path, + "--environment=data", + ], + ) + parsed_manifest = tomllib.loads(manifest_path.read_text()) + assert "data" in parsed_manifest["environments"]["data"]["features"] + assert parsed_manifest == snapshot( + { + # these keys are irrelevant and some are machine-dependent + "workspace": IsPartialDict, + "tasks": {}, + "dependencies": {}, + "feature": { + "simple-env": {"pypi-dependencies": {"cowpy": "*", "array-api-extra": "*"}}, + "numpy": {"pypi-dependencies": {"numpy": "<2"}}, + "data": {"pypi-dependencies": {"array-api-extra": "*"}}, + }, + "environments": { + "simple-env": {"features": ["simple-env", "numpy"], "no-default-feature": True}, + "data": {"features": ["data"], "no-default-feature": True}, + }, + } + ) + + def test_versions_include_constraints(self, pixi: Path, tmp_pixi_workspace: Path) -> None: + manifest_path = tmp_pixi_workspace / "pixi.toml" + + import_file_path = tmp_pixi_workspace / "numpy_requirements.txt" + with open(import_file_path, "w") as file: + file.write(self.numpy_txt) + + import_file_path = tmp_pixi_workspace / "xpx_requirements.txt" + with open(import_file_path, "w") as file: + file.write(self.xpx_txt) + + import_file_path = tmp_pixi_workspace / "complex_requirements.txt" + with open(import_file_path, "w") as file: + file.write(self.complex_txt) + + # Create a new project + verify_cli_command([pixi, "init", tmp_pixi_workspace]) + + # Import an environment which pins versions and uses -c and -r + verify_cli_command( + [ + pixi, + "import", + "--manifest-path", + manifest_path, + import_file_path, + "--format=pypi-txt", + "--feature=complex-env", + ], + # `-c` constraints should be warned about and ignored + stderr_contains=["Constraints detected"], + ) + parsed_manifest = tomllib.loads(manifest_path.read_text()) + assert parsed_manifest == snapshot( + { + # these keys are irrelevant and some are machine-dependent + "workspace": IsPartialDict, + "tasks": {}, + "dependencies": {}, + "feature": { + "complex-env": { + "pypi-dependencies": {"cowpy": "==1.1.4", "array-api-extra": "*"} + } + }, + "environments": { + "complex-env": {"features": ["complex-env"], "no-default-feature": True} + }, + } + )