fix: channels creation on migrate (#2215)

Now the migration script will result in: ```toml [envs.package2] channels = ["file:///C:/Users/Ruben/development/pixi/tests/integration/test_data/global_update_channel_1/output"] dependencies = { package2 = "*" } exposed = { package2 = "package2" } [envs.pixi-pack] channels = ["conda-forge"] dependencies = { pixi-pack= "*" } exposed = { pixi-pack= "pixi-pack" } ```` Instead of ```toml [envs.package2] channels = ["file:///C:/Users/Ruben/development/pixi/tests/integration/test_data/global_update_channel_1/output"] dependencies = { package2 = "*" } exposed = { package2 = "package2" } [envs.pixi-pack] channels = ["https://conda.anaconda.orge/conda-forge/"] dependencies = { pixi-pack= "*" } exposed = { pixi-pack= "pixi-pack" } ```` This PR also fixes the migration from multi channel environments, as it was not checking all channels used in the initial install. Before: ```toml [envs.lsd] channels = ["https://prefix.dev/rust-forge"] dependencies = { lsd = "*" } exposed = { lsd = "lsd" } ``` after this PR: ```toml [envs.lsd] channels = [ "https://prefix.dev/rust-forge", "conda-forge", ] dependencies = { lsd = "*" } exposed = { lsd = "lsd" } ``` --------- Co-authored-by: Julian Hofer <[email protected]>
prefix-dev · Oct 8, 2024 · c5eb033 · c5eb033
1 parent 5fc47fc
commit c5eb033
Show file tree

Hide file tree

Showing 25 changed files with 327 additions and 66 deletions.
diff --git a/src/global/common.rs b/src/global/common.rs
@@ -3,12 +3,15 @@ use fs_err as fs;
 use fs_err::tokio as tokio_fs;
 use miette::{Context, IntoDiagnostic};
 use pixi_config::home_path;
-use rattler_conda_types::PrefixRecord;
+use pixi_manifest::PrioritizedChannel;
+use rattler_conda_types::{Channel, ChannelConfig, NamedChannelOrUrl, PrefixRecord};
 use std::ffi::OsStr;
+use std::str::FromStr;
 use std::{
     io::Read,
     path::{Path, PathBuf},
 };
+use url::Url;
 
 /// Global binaries directory, default to `$HOME/.pixi/bin`
 #[derive(Debug, Clone)]
@@ -185,6 +188,27 @@ pub(crate) async fn find_package_records(conda_meta: &Path) -> miette::Result<Ve
     Ok(records)
 }
 
+/// converts a channel url string to a PrioritizedChannel
+pub(crate) fn channel_url_to_prioritized_channel(
+    channel: &str,
+    channel_config: &ChannelConfig,
+) -> miette::Result<PrioritizedChannel> {
+    // If channel url contains channel config alias as a substring, don't use it as a URL
+    if channel.contains(channel_config.channel_alias.as_str()) {
+        // Create channel from URL for parsing
+        let channel = Channel::from_url(Url::from_str(channel).expect("channel should be url"));
+        // If it has a name return as named channel
+        if let Some(name) = channel.name {
+            // If the channel has a name, use it as the channel
+            return Ok(NamedChannelOrUrl::from_str(&name).into_diagnostic()?.into());
+        }
+    }
+    // If channel doesn't contain the alias or has no name, use it as a URL
+    Ok(NamedChannelOrUrl::from_str(channel)
+        .into_diagnostic()?
+        .into())
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -230,6 +254,44 @@ mod tests {
             .any(|rec| rec.repodata_record.package_record.name.as_normalized() == "python"));
     }
 
+    #[test]
+    fn test_channel_url_to_prioritized_channel() {
+        let channel_config = ChannelConfig {
+            channel_alias: Url::from_str("https://conda.anaconda.org").unwrap(),
+            root_dir: PathBuf::from("/tmp"),
+        };
+        // Same host as alias
+        let channel = "https://conda.anaconda.org/conda-forge";
+        let prioritized_channel =
+            channel_url_to_prioritized_channel(channel, &channel_config).unwrap();
+        assert_eq!(
+            PrioritizedChannel::from(NamedChannelOrUrl::from_str("conda-forge").unwrap()),
+            prioritized_channel
+        );
+
+        // Different host
+        let channel = "https://prefix.dev/conda-forge";
+        let prioritized_channel =
+            channel_url_to_prioritized_channel(channel, &channel_config).unwrap();
+        assert_eq!(
+            PrioritizedChannel::from(
+                NamedChannelOrUrl::from_str("https://prefix.dev/conda-forge").unwrap()
+            ),
+            prioritized_channel
+        );
+
+        // File URL
+        let channel = "file:///C:/Users/user/channel/output";
+        let prioritized_channel =
+            channel_url_to_prioritized_channel(channel, &channel_config).unwrap();
+        assert_eq!(
+            PrioritizedChannel::from(
+                NamedChannelOrUrl::from_str("file:///C:/Users/user/channel/output").unwrap()
+            ),
+            prioritized_channel
+        );
+    }
+
     #[rstest]
     #[case("python3.9.1")]
     #[case("python3.9")]

diff --git a/src/global/project/mod.rs b/src/global/project/mod.rs
@@ -1,4 +1,5 @@
 use super::{extract_executable_from_script, BinDir, EnvRoot};
+use crate::global::common::{channel_url_to_prioritized_channel, find_package_records};
 use crate::global::install::{
     create_activation_script, create_executable_scripts, script_exec_mapping,
 };
@@ -30,7 +31,7 @@ use pixi_utils::reqwest::build_reqwest_clients;
 use rattler::install::{DefaultProgressFormatter, IndicatifReporter, Installer};
 use rattler::package_cache::PackageCache;
 use rattler_conda_types::{
-    GenericVirtualPackage, MatchSpec, NamedChannelOrUrl, PackageName, Platform, PrefixRecord,
+    ChannelConfig, GenericVirtualPackage, MatchSpec, PackageName, Platform, PrefixRecord,
 };
 use rattler_repodata_gateway::Gateway;
 use rattler_shell::shell::ShellEnum;
@@ -92,7 +93,7 @@ impl Debug for Project {
 struct ExposedData {
     env_name: EnvironmentName,
     platform: Option<Platform>,
-    channel: PrioritizedChannel,
+    channels: Vec<PrioritizedChannel>,
     package: PackageName,
     exposed: ExposedName,
     executable_name: String,
@@ -104,7 +105,11 @@ impl ExposedData {
     /// This function extracts metadata from the exposed script path, including the
     /// environment name, platform, channel, and package information, by reading
     /// the associated `conda-meta` directory.
-    pub async fn from_exposed_path(path: &Path, env_root: &EnvRoot) -> miette::Result<Self> {
+    pub async fn from_exposed_path(
+        path: &Path,
+        env_root: &EnvRoot,
+        channel_config: &ChannelConfig,
+    ) -> miette::Result<Self> {
         let exposed = ExposedName::from_str(executable_from_path(path).as_str())?;
         let executable_path = extract_executable_from_script(path).await?;
 
@@ -126,12 +131,29 @@ impl ExposedData {
         let prefix = Prefix::new(env_dir.path());
 
         let (platform, channel, package) =
-            package_from_conda_meta(&conda_meta, &executable, &prefix).await?;
+            package_from_conda_meta(&conda_meta, &executable, &prefix, channel_config).await?;
+
+        let mut channels = vec![channel];
+
+        // Find all channels used to create the environment
+        let all_channels = prefix
+            .find_installed_packages(None)
+            .await?
+            .iter()
+            .map(|prefix_record| prefix_record.repodata_record.channel.clone())
+            .collect::<HashSet<_>>();
+        for channel in all_channels {
+            tracing::debug!("Channel: {} found in environment: {}", channel, env_name);
+            channels.push(channel_url_to_prioritized_channel(
+                &channel,
+                channel_config,
+            )?);
+        }
 
         Ok(ExposedData {
             env_name,
             platform,
-            channel,
+            channels,
             package,
             executable_name: executable,
             exposed,
@@ -159,6 +181,7 @@ fn determine_env_path(executable_path: &Path, env_root: &Path) -> miette::Result
 /// Converts a `PrefixRecord` into package metadata, including platform, channel, and package name.
 fn convert_record_to_metadata(
     prefix_record: &PrefixRecord,
+    channel_config: &ChannelConfig,
 ) -> miette::Result<(Option<Platform>, PrioritizedChannel, PackageName)> {
     let platform = match Platform::from_str(&prefix_record.repodata_record.package_record.subdir) {
         Ok(Platform::NoArch) => None,
@@ -167,14 +190,12 @@ fn convert_record_to_metadata(
         Ok(p) => Some(p),
     };
 
-    let channel: PrioritizedChannel =
-        NamedChannelOrUrl::from_str(&prefix_record.repodata_record.channel)
-            .into_diagnostic()?
-            .into();
+    let package_name = prefix_record.repodata_record.package_record.name.clone();
 
-    let name = prefix_record.repodata_record.package_record.name.clone();
+    let channel =
+        channel_url_to_prioritized_channel(&prefix_record.repodata_record.channel, channel_config)?;
 
-    Ok((platform, channel, name))
+    Ok((platform, channel, package_name))
 }
 
 /// Extracts package metadata from the `conda-meta` directory for a given executable.
@@ -186,15 +207,16 @@ async fn package_from_conda_meta(
     conda_meta: &Path,
     executable: &str,
     prefix: &Prefix,
+    channel_config: &ChannelConfig,
 ) -> miette::Result<(Option<Platform>, PrioritizedChannel, PackageName)> {
-    let records = crate::global::common::find_package_records(conda_meta).await?;
+    let records = find_package_records(conda_meta).await?;
 
     for prefix_record in records {
         if find_executables(prefix, &prefix_record)
             .iter()
             .any(|exe_path| executable_from_path(exe_path) == executable)
         {
-            return convert_record_to_metadata(&prefix_record);
+            return convert_record_to_metadata(&prefix_record, channel_config);
         }
     }
 
@@ -288,6 +310,8 @@ impl Project {
         env_root: EnvRoot,
         bin_dir: BinDir,
     ) -> miette::Result<Self> {
+        let config = Config::load(env_root.path());
+
         let futures = bin_dir
             .files()
             .await?
@@ -299,7 +323,14 @@ impl Project {
             })
             .map(|result| async {
                 match result {
-                    Ok(path) => ExposedData::from_exposed_path(&path, &env_root).await,
+                    Ok(path) => {
+                        ExposedData::from_exposed_path(
+                            &path,
+                            &env_root,
+                            config.global_channel_config(),
+                        )
+                        .await
+                    }
                     Err(e) => Err(e),
                 }
             });
@@ -757,7 +788,11 @@ mod tests {
     use super::*;
     use fake::{faker::filesystem::zh_tw::FilePath, Fake};
     use itertools::Itertools;
+    use rattler_conda_types::{
+        NamedChannelOrUrl, PackageRecord, Platform, RepoDataRecord, VersionWithSource,
+    };
     use tempfile::tempdir;
+    use url::Url;
 
     const SIMPLE_MANIFEST: &str = r#"
         [envs.python]
@@ -944,4 +979,69 @@ mod tests {
 
         assert_eq!(remaining_dirs, vec!["env1", "env3", "non-conda-env-dir"]);
     }
+
+    #[test]
+    fn test_convert_repodata_to_exposed_data() {
+        let temp_dir = tempdir().unwrap();
+        let channel_config = ChannelConfig::default_with_root_dir(temp_dir.path().to_owned());
+        let mut package_record = PackageRecord::new(
+            "python".parse().unwrap(),
+            VersionWithSource::from_str("3.9.7").unwrap(),
+            "build_string".to_string(),
+        );
+
+        // Set platform to something different than current
+        package_record.subdir = Platform::LinuxRiscv32.to_string();
+
+        let repodata_record = RepoDataRecord {
+            package_record: package_record.clone(),
+            file_name: "doesnt_matter.conda".to_string(),
+            url: Url::from_str("https://also_doesnt_matter").unwrap(),
+            channel: format!("{}{}", channel_config.channel_alias.clone(), "test-channel"),
+        };
+        let prefix_record = PrefixRecord::from_repodata_record(
+            repodata_record,
+            None,
+            None,
+            vec![],
+            Default::default(),
+            None,
+        );
+
+        // Test with default channel alias
+        let (platform, channel, package) =
+            convert_record_to_metadata(&prefix_record, &channel_config).unwrap();
+        assert_eq!(
+            channel,
+            NamedChannelOrUrl::from_str("test-channel").unwrap().into()
+        );
+        assert_eq!(package, "python".parse().unwrap());
+        assert_eq!(platform, Some(Platform::LinuxRiscv32));
+
+        // Test with different from default channel alias
+        let repodata_record = RepoDataRecord {
+            package_record: package_record.clone(),
+            file_name: "doesnt_matter.conda".to_string(),
+            url: Url::from_str("https://also_doesnt_matter").unwrap(),
+            channel: "https://test-channel.com/idk".to_string(),
+        };
+        let prefix_record = PrefixRecord::from_repodata_record(
+            repodata_record,
+            None,
+            None,
+            vec![],
+            Default::default(),
+            None,
+        );
+
+        let (_platform, channel, package) =
+            convert_record_to_metadata(&prefix_record, &channel_config).unwrap();
+        assert_eq!(
+            channel,
+            NamedChannelOrUrl::from_str("https://test-channel.com/idk")
+                .unwrap()
+                .into()
+        );
+        assert_eq!(package, "python".parse().unwrap());
+    }
 }
diff --git a/src/global/project/parsed_manifest.rs b/src/global/project/parsed_manifest.rs
@@ -36,13 +36,13 @@ where
             let ExposedData {
                 env_name,
                 platform,
-                channel,
+                channels,
                 package,
                 executable_name,
                 exposed,
             } = data;
             let parsed_environment = envs.entry(env_name).or_default();
-            parsed_environment.channels.insert(channel);
+            parsed_environment.channels.extend(channels);
             parsed_environment.platform = platform;
             parsed_environment
                 .dependencies

diff --git a/tests/integration/test_data/dummy_channel_1/output/linux-64/dummy-a-0.1.0-hb0f4dca_0.conda b/tests/integration/test_data/dummy_channel_1/output/linux-64/dummy-a-0.1.0-hb0f4dca_0.conda
diff --git a/tests/integration/test_data/dummy_channel_1/output/linux-64/dummy-b-0.1.0-hb0f4dca_0.conda b/tests/integration/test_data/dummy_channel_1/output/linux-64/dummy-b-0.1.0-hb0f4dca_0.conda
diff --git a/tests/integration/test_data/dummy_channel_1/output/linux-64/dummy-c-0.1.0-hb0f4dca_0.conda b/tests/integration/test_data/dummy_channel_1/output/linux-64/dummy-c-0.1.0-hb0f4dca_0.conda
diff --git a/tests/integration/test_data/dummy_channel_1/output/linux-64/dummy-d-0.1.0-hb0f4dca_0.conda b/tests/integration/test_data/dummy_channel_1/output/linux-64/dummy-d-0.1.0-hb0f4dca_0.conda
diff --git a/tests/integration/test_data/dummy_channel_1/output/linux-64/repodata.json b/tests/integration/test_data/dummy_channel_1/output/linux-64/repodata.json
@@ -11,41 +11,57 @@
       "depends": [
         "dummy-c"
       ],
-      "md5": "2661f5c52b80e15feb390e50afe6fce8",
+      "md5": "7d6a50e4f9f3b8dca5fa3396571138fc",
       "name": "dummy-a",
       "platform": "linux",
-      "sha256": "f57c7bc0f5d24c82059c34821bfd5fe6bfdf69a1d9a706ad60f16a24cc1f6860",
-      "size": 17880,
+      "sha256": "d64f17f38d7b90470ebaccf95da13a6705bd54d55fcaefb072417e19ee0496fe",
+      "size": 32916,
       "subdir": "linux-64",
-      "timestamp": 1727337509978,
+      "timestamp": 1728313938368,
       "version": "0.1.0"
     },
     "dummy-b-0.1.0-hb0f4dca_0.conda": {
       "arch": "x86_64",
       "build": "hb0f4dca_0",
       "build_number": 0,
       "depends": [],
-      "md5": "bef3b368338d1b11d9693149c31bda9f",
+      "md5": "c8448b433bae5da486e027979f82a1ee",
       "name": "dummy-b",
       "platform": "linux",
-      "sha256": "b5bf447f260382d868ce724f574b85ea531330e367eb1b25f88b5e7fda7f5bf1",
-      "size": 22446,
+      "sha256": "a4bff8ed2df233c6838812689b6b8946d6f6712280e6c7ba1d07fda353163808",
+      "size": 27942,
       "subdir": "linux-64",
-      "timestamp": 1727337509978,
+      "timestamp": 1728313938368,
       "version": "0.1.0"
     },
     "dummy-c-0.1.0-hb0f4dca_0.conda": {
       "arch": "x86_64",
       "build": "hb0f4dca_0",
       "build_number": 0,
       "depends": [],
-      "md5": "ce9bfc58b059ab39ddf5d2996ee99855",
+      "md5": "865c16495b0d51cddb97242a6590291b",
       "name": "dummy-c",
       "platform": "linux",
-      "sha256": "d550467137577c5adf9fdacbdba89858d767ab4cba9152cbb131a58d95fbc9ae",
-      "size": 14679,
+      "sha256": "10c0761ffcf82f67a0110e3f8a99a56739fc8b696c66173179472beabb8e39c7",
+      "size": 22579,
       "subdir": "linux-64",
-      "timestamp": 1727337509978,
+      "timestamp": 1728313938368,
+      "version": "0.1.0"
+    },
+    "dummy-d-0.1.0-hb0f4dca_0.conda": {
+      "arch": "x86_64",
+      "build": "hb0f4dca_0",
+      "build_number": 0,
+      "depends": [
+        "dummy-x"
+      ],
+      "md5": "3285d38a1350e7dbe5675fae5a81de8c",
+      "name": "dummy-d",
+      "platform": "linux",
+      "sha256": "b6018f49e15f8cee06b4076b6e6995541fbc6f363a4d6a96ff210379f10c424a",
+      "size": 18191,
+      "subdir": "linux-64",
+      "timestamp": 1728313938368,
       "version": "0.1.0"
     }
   },

diff --git a/tests/integration/test_data/dummy_channel_1/output/osx-64/dummy-a-0.1.0-h0dc7051_0.conda b/tests/integration/test_data/dummy_channel_1/output/osx-64/dummy-a-0.1.0-h0dc7051_0.conda
diff --git a/tests/integration/test_data/dummy_channel_1/output/osx-64/dummy-b-0.1.0-h0dc7051_0.conda b/tests/integration/test_data/dummy_channel_1/output/osx-64/dummy-b-0.1.0-h0dc7051_0.conda
diff --git a/tests/integration/test_data/dummy_channel_1/output/osx-64/dummy-c-0.1.0-h0dc7051_0.conda b/tests/integration/test_data/dummy_channel_1/output/osx-64/dummy-c-0.1.0-h0dc7051_0.conda
diff --git a/tests/integration/test_data/dummy_channel_1/output/osx-64/dummy-d-0.1.0-h0dc7051_0.conda b/tests/integration/test_data/dummy_channel_1/output/osx-64/dummy-d-0.1.0-h0dc7051_0.conda