feat: add purls to PackageRecord and lockfile (#414)

Implementation of conda/ceps#63
conda · Nov 24, 2023 · d7ecd1f · d7ecd1f
1 parent bdc0e59
commit d7ecd1f
Show file tree

Hide file tree

Showing 9 changed files with 150 additions and 6 deletions.
diff --git a/crates/rattler_conda_types/Cargo.toml b/crates/rattler_conda_types/Cargo.toml
@@ -32,11 +32,12 @@ url = { version = "2.4.1", features = ["serde"] }
 rattler_digest = { version = "0.12.3", path = "../rattler_digest", features = ["serde"] }
 rattler_macros = { version = "0.12.3", path = "../rattler_macros" }
 glob = "0.3.1"
+purl = { version = "0.1.2", features = ["serde"] }
 
 [dev-dependencies]
 rand = "0.8.5"
 insta = { version = "1.33.0", features = ["yaml", "redactions", "toml"] }
-rattler_package_streaming = { path = "../rattler_package_streaming", default-features = false, features=["rustls-tls"] }
+rattler_package_streaming = { path = "../rattler_package_streaming", default-features = false, features = ["rustls-tls"] }
 tempfile = "3.8.0"
 rstest = "0.18.2"
 assert_matches = "1.5.0"

diff --git a/crates/rattler_conda_types/src/lib.rs b/crates/rattler_conda_types/src/lib.rs
@@ -50,6 +50,9 @@ pub use version_spec::VersionSpec;
 #[cfg(test)]
 use std::path::{Path, PathBuf};
 
+/// An package identifier that can be used to identify packages across package ecosystems.
+pub type PackageUrl = purl::GenericPurl<String>;
+
 #[cfg(test)]
 pub(crate) fn get_test_data_dir() -> PathBuf {
     Path::new(env!("CARGO_MANIFEST_DIR")).join("../../test-data")

diff --git a/crates/rattler_conda_types/src/repo_data/mod.rs b/crates/rattler_conda_types/src/repo_data/mod.rs
@@ -21,7 +21,7 @@ use rattler_macros::sorted;
 
 use crate::{
     build_spec::BuildNumber, package::IndexJson, utils::serde::DeserializeFromStrUnchecked,
-    Channel, NoArchType, PackageName, Platform, RepoDataRecord, VersionWithSource,
+    Channel, NoArchType, PackageName, PackageUrl, Platform, RepoDataRecord, VersionWithSource,
 };
 
 /// [`RepoData`] is an index of package binaries available on in a subdirectory of a Conda channel.
@@ -125,6 +125,11 @@ pub struct PackageRecord {
     /// Optionally the platform the package supports
     pub platform: Option<String>, // Note that this does not match the [`Platform`] enum..
 
+    /// Package identifiers of packages that are equivalent to this package but from other
+    /// ecosystems.
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub purls: Vec<PackageUrl>,
+
     /// Optionally a SHA256 hash of the package archive
     #[serde_as(as = "Option<SerializableHash::<rattler_digest::Sha256>>")]
     pub sha256: Option<Sha256Hash>,
@@ -279,6 +284,7 @@ impl PackageRecord {
             timestamp: None,
             track_features: vec![],
             version: version.into(),
+            purls: vec![],
         }
     }
 
@@ -393,6 +399,7 @@ impl PackageRecord {
             timestamp: index.timestamp,
             track_features: index.track_features,
             version: index.version,
+            purls: vec![],
         })
     }
 }

diff --git a/crates/rattler_conda_types/src/repo_data/patches.rs b/crates/rattler_conda_types/src/repo_data/patches.rs
@@ -4,7 +4,7 @@ use serde_with::{serde_as, skip_serializing_none, OneOrMany};
 use std::io;
 use std::path::Path;
 
-use crate::{package::ArchiveType, PackageRecord, RepoData};
+use crate::{package::ArchiveType, PackageRecord, PackageUrl, RepoData};
 
 /// Represents a Conda repodata patch.
 ///
@@ -92,6 +92,10 @@ pub struct PackageRecordPatch {
         with = "::serde_with::rust::double_option"
     )]
     pub license_family: Option<Option<String>>,
+
+    /// Package identifiers of packages that are equivalent to this package but from other
+    /// ecosystems.
+    pub purls: Option<Vec<PackageUrl>>,
 }
 
 /// Repodata patch instructions for a single subdirectory. See [`RepoDataPatch`] for more
@@ -136,6 +140,9 @@ impl PackageRecord {
         if let Some(license_family) = &patch.license_family {
             self.license_family = license_family.clone();
         }
+        if let Some(package_urls) = &patch.purls {
+            self.purls = package_urls.clone();
+        }
     }
 }
 
@@ -260,4 +267,17 @@ mod test {
         // check result
         insta::assert_yaml_snapshot!(repodata);
     }
+
+    #[test]
+    fn test_patch_purl() {
+        // test data
+        let mut repodata = load_test_repodata();
+        let patch_instructions = load_patch_instructions("patch_instructions_4.json");
+
+        // apply patch
+        repodata.apply_patches(&patch_instructions);
+
+        // check result
+        insta::assert_yaml_snapshot!(repodata);
+    }
 }
diff --git a/...es/src/repo_data/snapshots/rattler_conda_types__repo_data__patches__test__patch_purl.snap b/...es/src/repo_data/snapshots/rattler_conda_types__repo_data__patches__test__patch_purl.snap
@@ -0,0 +1,84 @@
+---
+source: crates/rattler_conda_types/src/repo_data/patches.rs
+assertion_line: 281
+expression: repodata
+---
+info:
+  subdir: linux-64
+packages:
+  cross-python_emscripten-32-3.10.1-h60d57d3_8.tar.bz2:
+    build: h60d57d3_8
+    build_number: 8
+    depends:
+      - coreutils
+      - crossenv >=1.2
+      - emscripten_emscripten-32
+      - pip
+      - python 3.10.*
+      - rsync
+      - sed
+      - setuptools
+    md5: 07b8bdf69566cd63e6e49759033830ad
+    name: cross-python_emscripten-32
+    purls:
+      - "pkg:pypi/[email protected]"
+    sha256: 50064d294599734090b9a33a2194a94f5d3743325d5bae61fab3fa5151acf0f9
+    size: 7069
+    subdir: linux-64
+    timestamp: 1679045249592
+    version: 3.10.1
+  emscripten_emscripten-32-3.1.27-h60d57d3_5.tar.bz2:
+    build: h60d57d3_5
+    build_number: 5
+    depends:
+      - emsdk
+    md5: 16f2daa96cb33b21c78f3dbc6d83546f
+    name: emscripten_emscripten-32
+    sha256: ab5117990980e5243446c7e83a501cedfc9863c56bbf896e45c77155c84cc3ed
+    size: 6820
+    subdir: linux-64
+    timestamp: 1678986155405
+    version: 3.1.27
+packages.conda:
+  cross-python_emscripten-32-3.10.1-h60d57d3_8.conda:
+    build: h60d57d3_8
+    build_number: 8
+    depends:
+      - coreutils
+      - crossenv >=1.2
+      - emscripten_emscripten-32
+      - pip
+      - python 3.10.*
+      - rsync
+      - sed
+      - setuptools
+    md5: 07b8bdf69566cd63e6e49759033830ad
+    name: cross-python_emscripten-32
+    purls:
+      - "pkg:pypi/[email protected]"
+    sha256: 12064d294599734090b9a33a2194a94f5d3743325d5bae61fab3fa5151acf0f9
+    size: 7069
+    subdir: linux-64
+    timestamp: 1679045249592
+    version: 3.10.1
+  cross-python_emscripten-32-3.12.1-h60d57d3_8.conda:
+    build: h60d57d3_8
+    build_number: 8
+    depends:
+      - coreutils
+      - crossenv >=1.2
+      - emscripten_emscripten-32
+      - pip
+      - python 3.12.*
+      - rsync
+      - sed
+      - setuptools
+    md5: 07b8bdf69566cd63e6e49759033830ad
+    name: cross-python_emscripten-32
+    sha256: 12064d294599734090b9a33a2194a94f5d3743325d5bae61fab3fa5151acf0f9
+    size: 7069
+    subdir: linux-64
+    timestamp: 1679045249592
+    version: 3.10.1
+repodata_version: 1
+
diff --git a/crates/rattler_lock/src/builder.rs b/crates/rattler_lock/src/builder.rs
@@ -7,7 +7,7 @@ use crate::{
     PackageHashes, PackageName, PipLockedDependency, Platform, RepoDataRecord, TimeMeta,
 };
 use fxhash::{FxHashMap, FxHashSet};
-use rattler_conda_types::NamelessMatchSpec;
+use rattler_conda_types::{NamelessMatchSpec, PackageUrl};
 use std::collections::HashSet;
 use url::Url;
 
@@ -174,6 +174,7 @@ impl LockedPackagesBuilder {
                         noarch: locked_package.noarch,
                         size: locked_package.size,
                         timestamp: locked_package.timestamp,
+                        purls: locked_package.purls,
                     }
                     .into(),
                 },
@@ -248,6 +249,9 @@ pub struct CondaLockedDependencyBuilder {
 
     /// Experimental: The date this entry was created.
     pub timestamp: Option<chrono::DateTime<chrono::Utc>>,
+
+    /// Experimental: Defines that the package is an alias for a package from another ecosystem.
+    pub purls: Vec<PackageUrl>,
 }
 
 impl TryFrom<&RepoDataRecord> for CondaLockedDependencyBuilder {
@@ -286,6 +290,7 @@ impl TryFrom<RepoDataRecord> for CondaLockedDependencyBuilder {
             noarch: record.package_record.noarch,
             size: record.package_record.size,
             timestamp: record.package_record.timestamp,
+            purls: record.package_record.purls,
         })
     }
 }
@@ -401,6 +406,12 @@ impl CondaLockedDependencyBuilder {
         self.timestamp = Some(timestamp);
         self
     }
+
+    /// Adds a PackageUrl to the package
+    pub fn add_purl(mut self, purl: PackageUrl) -> Self {
+        self.purls.push(purl);
+        self
+    }
 }
 
 pub struct PipLockedDependencyBuilder {
@@ -472,6 +483,9 @@ mod tests {
                     noarch: NoArchType::python(),
                     size: Some(12000),
                     timestamp: Some(Utc::now()),
+                    purls: vec![
+                        "pkg:deb/debian/[email protected]?arch=x86_64".parse().unwrap(),
+                    ]
                 }))
             .build().unwrap();
 

diff --git a/crates/rattler_lock/src/conda.rs b/crates/rattler_lock/src/conda.rs
@@ -3,8 +3,8 @@ use crate::{
     PackageHashes::{Md5, Md5Sha256, Sha256},
 };
 use rattler_conda_types::{
-    InvalidPackageNameError, NoArchType, PackageName, PackageRecord, ParseMatchSpecError,
-    ParseVersionError, RepoDataRecord,
+    InvalidPackageNameError, NoArchType, PackageName, PackageRecord, PackageUrl,
+    ParseMatchSpecError, ParseVersionError, RepoDataRecord,
 };
 use serde::{Deserialize, Serialize};
 use serde_with::{serde_as, skip_serializing_none, OneOrMany};
@@ -67,6 +67,10 @@ pub struct CondaLockedDependency {
     /// Experimental: The date this entry was created.
     #[serde_as(as = "Option<crate::utils::serde::Timestamp>")]
     pub timestamp: Option<chrono::DateTime<chrono::Utc>>,
+
+    /// Experimental: Defines that the package is an alias for a package from another ecosystem.
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub purls: Vec<PackageUrl>,
 }
 
 impl TryFrom<&LockedDependency> for RepoDataRecord {
@@ -136,6 +140,7 @@ impl TryFrom<LockedDependency> for RepoDataRecord {
                 timestamp: value.timestamp,
                 track_features: value.track_features,
                 version,
+                purls: value.purls,
             },
             file_name,
             url: value.url,

diff --git a/crates/rattler_solve/tests/backends.rs b/crates/rattler_solve/tests/backends.rs
@@ -104,6 +104,7 @@ fn installed_package(
             timestamp: None,
             legacy_bz2_size: None,
             legacy_bz2_md5: None,
+            purls: Vec::new(),
         },
     }
 }

diff --git a/test-data/channels/patch/linux-64/patch_instructions_4.json b/test-data/channels/patch/linux-64/patch_instructions_4.json
@@ -0,0 +1,9 @@
+{
+    "packages": {
+        "cross-python_emscripten-32-3.10.1-h60d57d3_8.tar.bz2": {
+            "purls": [
+                "pkg:pypi/[email protected]"
+            ]
+        }
+    }
+}