rechunker: Add --format-version=2

Cargo.toml

@@ -73,7 +73,7 @@ openssl = "0.10.73"
 once_cell = "1.21.3"
 os-release = "0.1.0"
 # We pull this one from git, as the project is no longer published as an external crate.
-ostree-ext = { git = "https://github.com/containers/bootc", rev = "v1.3.0" }
+ostree-ext = { git = "https://github.com/containers/bootc", rev = "v1.4.0" }
 paste = "1.0"
 phf = { version = "0.11", features = ["macros"] }
 rand = "0.9.1"

docs/build-chunked-oci.md

@@ -29,9 +29,9 @@ you are targeting bootc systems with this, please follow
 ## Running
 
 Note that the `--from` and `--rootfs` options are mutually-exclusive;
-exactly one is required.  Currently both `--bootc` and
-`--format-version=1` are required options.  Additional format versions
-may be added in the future.
+exactly one is required.  Currently the `--bootc` option is required.
+The option `--format-version` can be either `1` or `2`, and if
+omitted, defaults to `1`.
 
 ### Using `--from`
 
@@ -58,3 +58,31 @@ rpm-ostree compose build-chunked-oci --bootc --format-version=1 --rootfs=/path/t
            --output containers-storage:quay.io/exampleos/exampleos:latest
 podman push quay.io/exampleos/exampleos:latest
 ```
+
+### Using `--format-version`
+
+The value of `--format-version` must be either `1` or `2`.  Additional
+format versions may be added in the future.
+
+A value of `1` will create an image with "sparse" layers.  A sparse
+layer will contain information for a changed file `/path/to/foo` in
+the tar stream, but may not contain information for the parent
+directories `/path` and `/path/to`.  This has the advantage of
+minimally reducing the size of the image since the tar stream is
+smaller, but has the disadvantage that the directories must be
+implicitly created when unpacking the layer.  This implicit creation
+results in directories with unpredictable metadata and breaks
+reproducible builds.
+
+A value of `2` will ensure that for each layer, any parent directories
+are explicitly defined in the tar stream for that layer.  This
+increases the layer size, but removes ambiguity about the expected
+metadata for the parent directories.
+
+The default value is `--format-version=1` for backwards-compatibility
+to ensure that images previously built with `--format-version=1` can
+be updated while also reusing existing layers from the previous
+version of an image.
+
+If reproducible builds are desirable, it is recommended to use
+`--format-version=2`.

rust/src/compose.rs

@@ -198,7 +198,11 @@ pub(crate) struct BuildChunkedOCIOpts {
     #[clap(long, required = true)]
     bootc: bool,
 
-    /// The format version. At the current time there is only version `1`.
+    /// The format version.  Version `1` creates OCI (tar) layers
+    /// sparsely, meaning parent directories may be omitted from the
+    /// tar stream.  Version `2` ensures that all parent directories
+    /// in all layers are present in the tar stream.  Default value is
+    /// `1` for backward compatibility.
     #[clap(long, default_value_t = 1)]
     format_version: u32,
 
@@ -293,7 +297,7 @@ impl BuildChunkedOCIOpts {
             .with_context(|| format!("Opening {}", rootfs))?;
         // These must be set to exactly this; the CLI parser requires it.
         assert!(self.bootc);
-        assert_eq!(self.format_version, 1);
+        assert!(self.format_version == 1 || self.format_version == 2);
 
         // If we're deriving from an existing image, be sure to preserve its metadata (labels, creation time, etc.)
         // by default.
@@ -367,6 +371,7 @@ impl BuildChunkedOCIOpts {
             .args(label_arg)
             .args(self.max_layers.map(|l| format!("--max-layers={l}")))
             .arg(format!("--arch={arch}"))
+            .arg(format!("--format-version={}", self.format_version))
             .args(
                 config_data
                     .iter()

rust/src/container.rs

@@ -77,7 +77,7 @@ struct ContainerEncapsulateOpts {
     #[clap(long)]
     max_layers: Option<NonZeroU32>,
 
-    /// The encapsulated container format version; must be 0 or 1.
+    /// The encapsulated container format version; must be 1 or 2.
     #[clap(long, default_value = "1")]
     format_version: u32,
 
@@ -519,6 +519,9 @@ pub fn container_encapsulate(args: Vec<String>) -> CxxResult<()> {
             .unwrap();
         opts.platform = Some(platform);
     }
+    if opt.format_version >= 2 {
+        opts.tar_create_parent_dirs = true;
+    }
     let handle = tokio::runtime::Handle::current();
     println!("Generating container image");
     let digest = handle.block_on(async {

tests/build-chunked-oci/test.sh

@@ -8,7 +8,7 @@ podman pull --arch=ppc64le ${testimg_base}
 podman run --rm --privileged --security-opt=label=disable \
   -v /var/lib/containers:/var/lib/containers \
   -v /var/tmp:/var/tmp \
-  -v $(pwd):/output \
+  -v "$(pwd)":/output \
   localhost/builder rpm-ostree compose build-chunked-oci --bootc --from ${testimg_base} --output containers-storage:${chunked_output}
 podman rmi ${testimg_base}
 podman inspect containers-storage:${chunked_output} | jq '.[0]' > config.json
@@ -22,7 +22,7 @@ orig_created=$(podman inspect containers-storage:localhost/base | jq -r '.[0].Cr
 podman run --rm --privileged --security-opt=label=disable \
   -v /var/lib/containers:/var/lib/containers \
   -v /var/tmp:/var/tmp \
-  -v $(pwd):/output \
+  -v "$(pwd)":/output \
   localhost/builder rpm-ostree compose build-chunked-oci --bootc --format-version=1 --max-layers 99 --from localhost/base --output containers-storage:localhost/chunked
 podman inspect containers-storage:localhost/chunked | jq '.[0]' > new-config.json
 # Verify we propagated the creation date
@@ -32,3 +32,18 @@ test "$(date --date="${orig_created}" --rfc-3339=seconds)" = "$(date --date="${n
 # Verify we propagated labels
 test $(jq -r .Labels.testlabel < new-config.json) = "1"
 echo "ok rechunking with labels"
+
+# Verify directory metadata for --format-version=1 image
+# This will have nondeterministic mtimes creep in
+test "$(podman run --rm containers-storage:localhost/chunked find /usr -newermt @0 | wc -l)" -gt 0
+
+# Build a rechunked image with --format-version=2
+podman run --rm --privileged --security-opt=label=disable \
+  -v /var/lib/containers:/var/lib/containers \
+  -v /var/tmp:/var/tmp \
+  -v "$(pwd)":/output \
+  localhost/builder rpm-ostree compose build-chunked-oci --bootc --format-version=2 --max-layers 99 --from localhost/base --output containers-storage:localhost/chunked
+
+# Verify directory metadata for --format-version=2 image
+# This will have deterministic mtimes
+test "$(podman run --rm containers-storage:localhost/chunked find /usr -newermt @0 | wc -l)" -eq 0