diff --git a/Cargo.lock b/Cargo.lock index 967878161f..3ab5d09b34 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -215,7 +215,7 @@ dependencies = [ [[package]] name = "bootc-utils" version = "0.0.0" -source = "git+https://github.com/containers/bootc?rev=v1.3.0#b06d75fed73f26a974423a9db41e19413987fca3" +source = "git+https://github.com/containers/bootc?rev=v1.4.0#58ee28cbab24cccb53cf7c2d975fe7eb5773b137" dependencies = [ "anyhow", "chrono", @@ -2226,7 +2226,7 @@ dependencies = [ [[package]] name = "ostree-ext" version = "0.15.3" -source = "git+https://github.com/containers/bootc?rev=v1.3.0#b06d75fed73f26a974423a9db41e19413987fca3" +source = "git+https://github.com/containers/bootc?rev=v1.4.0#58ee28cbab24cccb53cf7c2d975fe7eb5773b137" dependencies = [ "anyhow", "bootc-utils", @@ -3869,7 +3869,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.48.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index f75d0e4067..ae55542a89 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -73,7 +73,7 @@ openssl = "0.10.73" once_cell = "1.21.3" os-release = "0.1.0" # We pull this one from git, as the project is no longer published as an external crate. -ostree-ext = { git = "https://github.com/containers/bootc", rev = "v1.3.0" } +ostree-ext = { git = "https://github.com/containers/bootc", rev = "v1.4.0" } paste = "1.0" phf = { version = "0.11", features = ["macros"] } rand = "0.9.1" diff --git a/docs/build-chunked-oci.md b/docs/build-chunked-oci.md index 368f31a018..0317e8fccc 100644 --- a/docs/build-chunked-oci.md +++ b/docs/build-chunked-oci.md @@ -29,9 +29,9 @@ you are targeting bootc systems with this, please follow ## Running Note that the `--from` and `--rootfs` options are mutually-exclusive; -exactly one is required. Currently both `--bootc` and -`--format-version=1` are required options. Additional format versions -may be added in the future. +exactly one is required. Currently the `--bootc` option is required. +The option `--format-version` can be either `1` or `2`, and if +omitted, defaults to `1`. ### Using `--from` @@ -58,3 +58,31 @@ rpm-ostree compose build-chunked-oci --bootc --format-version=1 --rootfs=/path/t --output containers-storage:quay.io/exampleos/exampleos:latest podman push quay.io/exampleos/exampleos:latest ``` + +### Using `--format-version` + +The value of `--format-version` must be either `1` or `2`. Additional +format versions may be added in the future. + +A value of `1` will create an image with "sparse" layers. A sparse +layer will contain information for a changed file `/path/to/foo` in +the tar stream, but may not contain information for the parent +directories `/path` and `/path/to`. This has the advantage of +minimally reducing the size of the image since the tar stream is +smaller, but has the disadvantage that the directories must be +implicitly created when unpacking the layer. This implicit creation +results in directories with unpredictable metadata and breaks +reproducible builds. + +A value of `2` will ensure that for each layer, any parent directories +are explicitly defined in the tar stream for that layer. This +increases the layer size, but removes ambiguity about the expected +metadata for the parent directories. + +The default value is `--format-version=1` for backwards-compatibility +to ensure that images previously built with `--format-version=1` can +be updated while also reusing existing layers from the previous +version of an image. + +If reproducible builds are desirable, it is recommended to use +`--format-version=2`. diff --git a/rust/src/compose.rs b/rust/src/compose.rs index 4ca6f3dfc5..adf971b09d 100644 --- a/rust/src/compose.rs +++ b/rust/src/compose.rs @@ -198,7 +198,11 @@ pub(crate) struct BuildChunkedOCIOpts { #[clap(long, required = true)] bootc: bool, - /// The format version. At the current time there is only version `1`. + /// The format version. Version `1` creates OCI (tar) layers + /// sparsely, meaning parent directories may be omitted from the + /// tar stream. Version `2` ensures that all parent directories + /// in all layers are present in the tar stream. Default value is + /// `1` for backward compatibility. #[clap(long, default_value_t = 1)] format_version: u32, @@ -293,7 +297,7 @@ impl BuildChunkedOCIOpts { .with_context(|| format!("Opening {}", rootfs))?; // These must be set to exactly this; the CLI parser requires it. assert!(self.bootc); - assert_eq!(self.format_version, 1); + assert!(self.format_version == 1 || self.format_version == 2); // If we're deriving from an existing image, be sure to preserve its metadata (labels, creation time, etc.) // by default. @@ -367,6 +371,7 @@ impl BuildChunkedOCIOpts { .args(label_arg) .args(self.max_layers.map(|l| format!("--max-layers={l}"))) .arg(format!("--arch={arch}")) + .arg(format!("--format-version={}", self.format_version)) .args( config_data .iter() diff --git a/rust/src/container.rs b/rust/src/container.rs index 50f167cac7..bf28cf2c29 100644 --- a/rust/src/container.rs +++ b/rust/src/container.rs @@ -77,7 +77,7 @@ struct ContainerEncapsulateOpts { #[clap(long)] max_layers: Option, - /// The encapsulated container format version; must be 0 or 1. + /// The encapsulated container format version; must be 1 or 2. #[clap(long, default_value = "1")] format_version: u32, @@ -519,6 +519,9 @@ pub fn container_encapsulate(args: Vec) -> CxxResult<()> { .unwrap(); opts.platform = Some(platform); } + if opt.format_version >= 2 { + opts.tar_create_parent_dirs = true; + } let handle = tokio::runtime::Handle::current(); println!("Generating container image"); let digest = handle.block_on(async { diff --git a/tests/build-chunked-oci/test.sh b/tests/build-chunked-oci/test.sh index 052e6d7086..05dcd8fd7d 100755 --- a/tests/build-chunked-oci/test.sh +++ b/tests/build-chunked-oci/test.sh @@ -8,7 +8,7 @@ podman pull --arch=ppc64le ${testimg_base} podman run --rm --privileged --security-opt=label=disable \ -v /var/lib/containers:/var/lib/containers \ -v /var/tmp:/var/tmp \ - -v $(pwd):/output \ + -v "$(pwd)":/output \ localhost/builder rpm-ostree compose build-chunked-oci --bootc --from ${testimg_base} --output containers-storage:${chunked_output} podman rmi ${testimg_base} podman inspect containers-storage:${chunked_output} | jq '.[0]' > config.json @@ -22,7 +22,7 @@ orig_created=$(podman inspect containers-storage:localhost/base | jq -r '.[0].Cr podman run --rm --privileged --security-opt=label=disable \ -v /var/lib/containers:/var/lib/containers \ -v /var/tmp:/var/tmp \ - -v $(pwd):/output \ + -v "$(pwd)":/output \ localhost/builder rpm-ostree compose build-chunked-oci --bootc --format-version=1 --max-layers 99 --from localhost/base --output containers-storage:localhost/chunked podman inspect containers-storage:localhost/chunked | jq '.[0]' > new-config.json # Verify we propagated the creation date @@ -32,3 +32,18 @@ test "$(date --date="${orig_created}" --rfc-3339=seconds)" = "$(date --date="${n # Verify we propagated labels test $(jq -r .Labels.testlabel < new-config.json) = "1" echo "ok rechunking with labels" + +# Verify directory metadata for --format-version=1 image +# This will have nondeterministic mtimes creep in +test "$(podman run --rm containers-storage:localhost/chunked find /usr -newermt @0 | wc -l)" -gt 0 + +# Build a rechunked image with --format-version=2 +podman run --rm --privileged --security-opt=label=disable \ + -v /var/lib/containers:/var/lib/containers \ + -v /var/tmp:/var/tmp \ + -v "$(pwd)":/output \ + localhost/builder rpm-ostree compose build-chunked-oci --bootc --format-version=2 --max-layers 99 --from localhost/base --output containers-storage:localhost/chunked + +# Verify directory metadata for --format-version=2 image +# This will have deterministic mtimes +test "$(podman run --rm containers-storage:localhost/chunked find /usr -newermt @0 | wc -l)" -eq 0