rechunker: Duplicate xattrs when more than 256 links

This is to avoid creating (10s of) thousands of links to a single file
which will cause the rechunker to fail after passing the filesystem's
link limit. This will create a duplicate xattrs file to link to for each
chunk of 256 links.

Assisted-by: Claude Code
Signed-off-by: ckyrouac <[email protected]>

Author: ckyrouac

crates/ostree-ext/src/tar/export.rs

@@ -126,6 +126,8 @@ struct OstreeTarWriter<'a, W: std::io::Write> {
     wrote_dirmeta: HashSet<String>,
     wrote_content: HashSet<String>,
     wrote_xattrs: HashSet<String>,
+    /// Tracks the number of links to each xattrs object (checksum -> (link count, suffix))
+    xattrs_link_counts: std::collections::HashMap<String, (u32, u32)>,
 }
 
 pub(crate) fn object_path(objtype: ostree::ObjectType, checksum: &str) -> Utf8PathBuf {
@@ -141,9 +143,13 @@ pub(crate) fn object_path(objtype: ostree::ObjectType, checksum: &str) -> Utf8Pa
     format!("{OSTREEDIR}/repo/objects/{first}/{rest}.{suffix}").into()
 }
 
-fn v1_xattrs_object_path(checksum: &str) -> Utf8PathBuf {
+fn v1_xattrs_object_path_with_suffix(checksum: &str, suffix: u32) -> Utf8PathBuf {
     let (first, rest) = checksum.split_at(2);
-    format!("{OSTREEDIR}/repo/objects/{first}/{rest}.file-xattrs").into()
+    if suffix == 0 {
+        format!("{OSTREEDIR}/repo/objects/{first}/{rest}.file-xattrs").into()
+    } else {
+        format!("{OSTREEDIR}/repo/objects/{first}/{rest}.file-xattrs.{suffix}").into()
+    }
 }
 
 fn v1_xattrs_link_object_path(checksum: &str) -> Utf8PathBuf {
@@ -197,6 +203,7 @@ impl<'a, W: std::io::Write> OstreeTarWriter<'a, W> {
             wrote_dirtree: HashSet::new(),
             wrote_content: HashSet::new(),
             wrote_xattrs: HashSet::new(),
+            xattrs_link_counts: std::collections::HashMap::new(),
         };
         Ok(r)
     }
@@ -383,6 +390,8 @@ impl<'a, W: std::io::Write> OstreeTarWriter<'a, W> {
     /// Return whether content was written.
     #[context("Writing xattrs")]
     fn append_ostree_xattrs(&mut self, checksum: &str, xattrs: &glib::Variant) -> Result<bool> {
+        const XATTRS_LINK_LIMIT: u32 = 256;
+
         let xattrs_data = xattrs.data_as_bytes();
         let xattrs_data = xattrs_data.as_ref();
 
@@ -391,13 +400,40 @@ impl<'a, W: std::io::Write> OstreeTarWriter<'a, W> {
             hex::encode(digest)
         };
 
-        let path = v1_xattrs_object_path(&xattrs_checksum);
-        // Write xattrs content into a separate `.file-xattrs` object.
-        if !self.wrote_xattrs.contains(&xattrs_checksum) {
-            let inserted = self.wrote_xattrs.insert(xattrs_checksum);
-            debug_assert!(inserted);
-            self.append_default_data(&path, xattrs_data)?;
+        // Check if we've already written this xattrs object or need to create a new one
+        let (link_count, suffix) = self
+            .xattrs_link_counts
+            .get(&xattrs_checksum)
+            .copied()
+            .unwrap_or((0, 0));
+
+        // Determine which suffix to use based on link count
+        let (current_suffix, should_create_new) = if link_count >= XATTRS_LINK_LIMIT {
+            (suffix + 1, true)
+        } else {
+            (suffix, link_count == 0)
+        };
+
+        let path = v1_xattrs_object_path_with_suffix(&xattrs_checksum, current_suffix);
+
+        // Write xattrs content into a separate `.file-xattrs` object if needed
+        if should_create_new {
+            // For the first object (suffix 0) or when we need a new suffixed object
+            let key = format!("{}.{}", xattrs_checksum, current_suffix);
+            if !self.wrote_xattrs.contains(&key) {
+                let inserted = self.wrote_xattrs.insert(key);
+                debug_assert!(inserted);
+                self.append_default_data(&path, xattrs_data)?;
+                // Reset link count for new suffix
+                self.xattrs_link_counts
+                    .insert(xattrs_checksum.clone(), (1, current_suffix));
+            }
+        } else {
+            // Increment link count for existing xattrs object
+            self.xattrs_link_counts
+                .insert(xattrs_checksum.clone(), (link_count + 1, current_suffix));
         }
+
         // Write a `.file-xattrs-link` which links the file object to
         // the corresponding detached xattrs.
         {
@@ -936,8 +972,13 @@ mod tests {
     fn test_v1_xattrs_object_path() {
         let checksum = "b8627e3ef0f255a322d2bd9610cfaaacc8f122b7f8d17c0e7e3caafa160f9fc7";
         let expected = "sysroot/ostree/repo/objects/b8/627e3ef0f255a322d2bd9610cfaaacc8f122b7f8d17c0e7e3caafa160f9fc7.file-xattrs";
-        let output = v1_xattrs_object_path(checksum);
+        let output = v1_xattrs_object_path_with_suffix(checksum, 0);
         assert_eq!(&output, expected);
+
+        // Test with suffix
+        let expected_with_suffix = "sysroot/ostree/repo/objects/b8/627e3ef0f255a322d2bd9610cfaaacc8f122b7f8d17c0e7e3caafa160f9fc7.file-xattrs.1";
+        let output_with_suffix = v1_xattrs_object_path_with_suffix(checksum, 1);
+        assert_eq!(&output_with_suffix, expected_with_suffix);
     }
 
     #[test]
@@ -947,4 +988,86 @@ mod tests {
         let output = v1_xattrs_link_object_path(checksum);
         assert_eq!(&output, expected);
     }
+
+    #[test]
+    fn test_append_ostree_xattrs() {
+        // Create a temporary in-memory tar builder
+        let mut tar_data = Vec::new();
+        let mut tar_builder = tar::Builder::new(&mut tar_data);
+
+        // Create a minimal OstreeTarWriter for testing
+        // We bypass the complex commit creation and just create the struct directly
+        let tmp_dir = tempfile::tempdir().unwrap();
+        let repo_path = tmp_dir.path().join("repo");
+        let repo = ostree::Repo::new_for_path(&repo_path);
+        repo.create(ostree::RepoMode::Archive, gio::Cancellable::NONE)
+            .unwrap();
+
+        // Create a dummy commit - we just need something valid
+        let empty_variant = glib::Variant::from(());
+
+        // Manually create an OstreeTarWriter instance
+        let mut writer = OstreeTarWriter {
+            repo: &repo,
+            commit_checksum: "dummy",
+            commit_object: empty_variant,
+            out: &mut tar_builder,
+            options: ExportOptions::default(),
+            wrote_initdirs: false,
+            structure_only: false,
+            wrote_vartmp: false,
+            wrote_dirtree: HashSet::new(),
+            wrote_dirmeta: HashSet::new(),
+            wrote_content: HashSet::new(),
+            wrote_xattrs: HashSet::new(),
+            xattrs_link_counts: std::collections::HashMap::new(),
+        };
+
+        // Create xattrs in the expected format for ostree (a(ayay))
+        // Format: array of (name bytes with null terminator, value bytes)
+        let xattrs_data: Vec<(Vec<u8>, Vec<u8>)> =
+            vec![(b"user.test\0".to_vec(), b"test_value".to_vec())];
+        let xattrs_variant = glib::Variant::from(xattrs_data);
+
+        // Test append_ostree_xattrs with a test checksum
+        let test_checksum = "abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890";
+        let result = writer
+            .append_ostree_xattrs(test_checksum, &xattrs_variant)
+            .unwrap();
+        assert!(result);
+
+        // Verify that xattrs were written
+        let xattrs_checksum = {
+            let xattrs_data = xattrs_variant.data_as_bytes();
+            let digest =
+                openssl::hash::hash(openssl::hash::MessageDigest::sha256(), xattrs_data.as_ref())
+                    .unwrap();
+            hex::encode(digest)
+        };
+        assert!(writer
+            .wrote_xattrs
+            .contains(&format!("{}.0", xattrs_checksum)));
+
+        // Test with multiple calls to check link count behavior
+        for i in 1..300 {
+            let checksum = format!("test{:064}", i);
+            writer
+                .append_ostree_xattrs(&checksum, &xattrs_variant)
+                .unwrap();
+        }
+
+        // Verify that multiple xattrs objects were created when limit was exceeded
+        // Should have created suffix 0 and suffix 1 objects
+        assert!(writer
+            .wrote_xattrs
+            .contains(&format!("{}.0", xattrs_checksum)));
+        assert!(writer
+            .wrote_xattrs
+            .contains(&format!("{}.1", xattrs_checksum)));
+
+        // Verify the link counts
+        let (link_count, suffix) = writer.xattrs_link_counts.get(&xattrs_checksum).unwrap();
+        assert_eq!(*suffix, 1);
+        assert!(*link_count == 44);
+    }
 }