diff --git a/.packit.yaml b/.packit.yaml index 0f826bb0c..441919721 100644 --- a/.packit.yaml +++ b/.packit.yaml @@ -51,8 +51,9 @@ jobs: # rawhide is basically nil. - fedora-rawhide-x86_64 - fedora-rawhide-aarch64 - - rhel-9-x86_64 - - rhel-9-aarch64 + # Temporarily disabled due to too old Rust...reenable post 9.6 + # - rhel-9-x86_64 + # - rhel-9-aarch64 - job: tests trigger: pull_request diff --git a/Cargo.lock b/Cargo.lock index a9cc578e7..77fe74dba 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -102,6 +102,19 @@ version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" +[[package]] +name = "async-compression" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df895a515f70646414f4b45c0b79082783b80552b373a68283012928df56f522" +dependencies = [ + "flate2", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", +] + [[package]] name = "autocfg" version = "1.4.0" @@ -193,6 +206,7 @@ dependencies = [ "clap", "clap_mangen", "comfy-table", + "composefs", "fn-error-context", "hex", "indicatif", @@ -440,6 +454,29 @@ dependencies = [ "unicode-width 0.2.0", ] +[[package]] +name = "composefs" +version = "0.2.0" +source = "git+https://github.com/containers/composefs-rs?rev=55ae2e9ba72f6afda4887d746e6b98f0a1875ac4#55ae2e9ba72f6afda4887d746e6b98f0a1875ac4" +dependencies = [ + "anyhow", + "async-compression", + "clap", + "containers-image-proxy", + "hex", + "indicatif", + "oci-spec", + "regex-automata 0.4.9", + "rustix", + "sha2", + "tar", + "tempfile", + "thiserror 2.0.11", + "tokio", + "zerocopy 0.8.14", + "zstd", +] + [[package]] name = "console" version = "0.15.8" @@ -1042,6 +1079,7 @@ dependencies = [ "console", "number_prefix", "portable-atomic", + "tokio", "unicode-width 0.2.0", "web-time", ] @@ -1569,7 +1607,7 @@ version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" dependencies = [ - "zerocopy", + "zerocopy 0.7.35", ] [[package]] @@ -2697,7 +2735,9 @@ dependencies = [ "chrono", "fn-error-context", "mandown", + "tar", "tempfile", + "toml", "xshell", ] @@ -2708,7 +2748,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ "byteorder", - "zerocopy-derive", + "zerocopy-derive 0.7.35", +] + +[[package]] +name = "zerocopy" +version = "0.8.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a367f292d93d4eab890745e75a778da40909cab4d6ff8173693812f79c4a2468" +dependencies = [ + "zerocopy-derive 0.8.14", ] [[package]] @@ -2722,6 +2771,17 @@ dependencies = [ "syn 2.0.87", ] +[[package]] +name = "zerocopy-derive" +version = "0.8.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3931cb58c62c13adec22e38686b559c86a30565e16ad6e8510a337cedc611e1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "zstd" version = "0.13.2" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index d9ae82b93..71f830927 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -6,8 +6,6 @@ license = "MIT OR Apache-2.0" repository = "https://github.com/containers/bootc" readme = "README.md" publish = false -# For now don't bump this above what is currently shipped in RHEL9. -rust-version = "1.75.0" default-run = "bootc" # See https://github.com/coreos/cargo-vendor-filterer diff --git a/contrib/packaging/bootc.spec b/contrib/packaging/bootc.spec index 412370bd4..8b19a1af5 100644 --- a/contrib/packaging/bootc.spec +++ b/contrib/packaging/bootc.spec @@ -64,7 +64,11 @@ Provides: ostree-cli(ostree-container) %prep %autosetup -p1 -a1 -%cargo_prep -v vendor +# Default -v vendor config doesn't support non-crates.io deps (i.e. git) +cp .cargo/vendor-config.toml . +%cargo_prep -N +cat vendor-config.toml >> .cargo/config.toml +rm vendor-config.toml %build %if 0%{?fedora} || 0%{?rhel} >= 10 @@ -74,6 +78,8 @@ Provides: ostree-cli(ostree-container) %endif %cargo_vendor_manifest +# https://pagure.io/fedora-rust/rust-packaging/issue/33 +sed -i -e '/https:\/\//d' cargo-vendor.txt %cargo_license_summary %{cargo_license} > LICENSE.dependencies diff --git a/deny.toml b/deny.toml index 56970b87d..bd8269dcd 100644 --- a/deny.toml +++ b/deny.toml @@ -12,4 +12,4 @@ name = "ring" [sources] unknown-registry = "deny" unknown-git = "deny" -allow-git = [] +allow-git = ["https://github.com/containers/composefs-rs"] diff --git a/hack/build.sh b/hack/build.sh index fc8988124..87cf28b31 100755 --- a/hack/build.sh +++ b/hack/build.sh @@ -3,7 +3,7 @@ set -xeu . /usr/lib/os-release case $ID in centos|rhel) dnf config-manager --set-enabled crb;; - fedora) dnf -y install dnf-utils ;; + fedora) dnf -y install dnf-utils 'dnf5-command(builddep)';; esac dnf -y builddep ./contrib/packaging/bootc.spec # Extra dependencies diff --git a/lib/Cargo.toml b/lib/Cargo.toml index a8eb87986..9ceb599bd 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -6,9 +6,9 @@ name = "bootc-lib" readme = "README.md" repository = "https://github.com/containers/bootc" version = "1.1.4" -# For now don't bump this above what is currently shipped in RHEL9; -# also keep in sync with the version in cli. -rust-version = "1.75.0" +# In general we try to keep this pinned to what's in the latest RHEL9. +# However right now, we bumped to 1.82 as that's what composefs-rs uses. +rust-version = "1.82.0" include = ["/src", "LICENSE-APACHE", "LICENSE-MIT"] @@ -23,6 +23,8 @@ ostree-ext = { path = "../ostree-ext", features = ["bootc"] } chrono = { workspace = true, features = ["serde"] } clap = { workspace = true, features = ["derive","cargo"] } clap_mangen = { workspace = true, optional = true } +#composefs = "0.2.0" +composefs = { git = "https://github.com/containers/composefs-rs", rev = "55ae2e9ba72f6afda4887d746e6b98f0a1875ac4" } cap-std-ext = { workspace = true, features = ["fs_utf8"] } hex = { workspace = true } fn-error-context = { workspace = true } diff --git a/lib/src/cli.rs b/lib/src/cli.rs index d693be6d0..15b56bdb1 100644 --- a/lib/src/cli.rs +++ b/lib/src/cli.rs @@ -13,6 +13,7 @@ use cap_std_ext::cap_std; use cap_std_ext::cap_std::fs::Dir; use clap::Parser; use clap::ValueEnum; +use composefs::fsverity; use fn_error_context::context; use ostree::gio; use ostree_container::store::PrepareResult; @@ -376,6 +377,21 @@ pub(crate) enum SchemaType { Progress, } +/// Options for consistency checking +#[derive(Debug, clap::Subcommand, PartialEq, Eq)] +pub(crate) enum FsverityOpts { + /// Measure the fsverity digest of the target file. + Measure { + /// Path to file + path: Utf8PathBuf, + }, + /// Enable fsverity on the target file. + Enable { + /// Ptah to file + path: Utf8PathBuf, + }, +} + /// Hidden, internal only options #[derive(Debug, clap::Subcommand, PartialEq, Eq)] pub(crate) enum InternalsOpts { @@ -392,6 +408,8 @@ pub(crate) enum InternalsOpts { #[clap(long)] of: SchemaType, }, + #[clap(subcommand)] + Fsverity(FsverityOpts), /// Perform cleanup actions Cleanup, /// Proxy frontend for the `ostree-ext` CLI. @@ -1113,6 +1131,24 @@ async fn run_from_opt(opt: Opt) -> Result<()> { ) .await } + // We don't depend on fsverity-utils today, so re-expose some helpful CLI tools. + InternalsOpts::Fsverity(args) => match args { + FsverityOpts::Measure { path } => { + let fd = + std::fs::File::open(&path).with_context(|| format!("Reading {path}"))?; + let digest = + fsverity::measure_verity_digest::<_, fsverity::Sha256HashValue>(&fd)?; + let digest = hex::encode(digest); + println!("{digest}"); + Ok(()) + } + FsverityOpts::Enable { path } => { + let fd = + std::fs::File::open(&path).with_context(|| format!("Reading {path}"))?; + fsverity::ioctl::fs_ioc_enable_verity::<_, fsverity::Sha256HashValue>(&fd)?; + Ok(()) + } + }, InternalsOpts::FixupEtcFstab => crate::deploy::fixup_etc_fstab(&root), InternalsOpts::PrintJsonSchema { of } => { let schema = match of { diff --git a/lib/src/install.rs b/lib/src/install.rs index 98112a4e5..7b8982961 100644 --- a/lib/src/install.rs +++ b/lib/src/install.rs @@ -13,6 +13,7 @@ pub(crate) mod config; mod osbuild; pub(crate) mod osconfig; +use std::collections::HashMap; use std::io::Write; use std::os::fd::{AsFd, AsRawFd}; use std::os::unix::process::CommandExt; @@ -37,11 +38,11 @@ use chrono::prelude::*; use clap::ValueEnum; use fn_error_context::context; use ostree::gio; -use ostree_ext::container as ostree_container; use ostree_ext::oci_spec; use ostree_ext::ostree; use ostree_ext::prelude::Cast; use ostree_ext::sysroot::SysrootLock; +use ostree_ext::{container as ostree_container, ostree_prepareroot}; #[cfg(feature = "install-to-disk")] use rustix::fs::FileTypeExt; use rustix::fs::MetadataExt as _; @@ -349,6 +350,8 @@ pub(crate) struct State { #[allow(dead_code)] pub(crate) config_opts: InstallConfigOpts, pub(crate) target_imgref: ostree_container::OstreeImageReference, + #[allow(dead_code)] + pub(crate) prepareroot_config: HashMap, pub(crate) install_config: Option, /// The parsed contents of the authorized_keys (not the file path) pub(crate) root_ssh_authorized_keys: Option, @@ -1267,6 +1270,20 @@ async fn prepare_install( tracing::debug!("No install configuration found"); } + // Convert the keyfile to a hashmap because GKeyFile isnt Send for probably bad reasons. + let prepareroot_config = { + let kf = ostree_prepareroot::require_config_from_root(&rootfs)?; + let mut r = HashMap::new(); + for grp in kf.groups() { + for key in kf.keys(&grp)? { + let key = key.as_str(); + let value = kf.value(&grp, key)?; + r.insert(format!("{grp}.{key}"), value.to_string()); + } + } + r + }; + // Eagerly read the file now to ensure we error out early if e.g. it doesn't exist, // instead of much later after we're 80% of the way through an install. let root_ssh_authorized_keys = config_opts @@ -1284,6 +1301,7 @@ async fn prepare_install( config_opts, target_imgref, install_config, + prepareroot_config, root_ssh_authorized_keys, container_root: rootfs, tempdir, diff --git a/lib/src/lints.rs b/lib/src/lints.rs index 8bc58dcd6..26e6423f3 100644 --- a/lib/src/lints.rs +++ b/lib/src/lints.rs @@ -6,7 +6,7 @@ use std::collections::BTreeSet; use std::env::consts::ARCH; use std::os::unix::ffi::OsStrExt; -use anyhow::{Context, Result}; +use anyhow::Result; use camino::{Utf8Path, Utf8PathBuf}; use cap_std::fs::Dir; use cap_std_ext::cap_std; @@ -14,6 +14,7 @@ use cap_std_ext::cap_std::fs::MetadataExt; use cap_std_ext::dirext::CapStdExtDirExt as _; use fn_error_context::context; use indoc::indoc; +use ostree_ext::ostree_prepareroot; use serde::Serialize; /// Reference to embedded default baseimage content that should exist. @@ -286,15 +287,8 @@ fn check_baseimage_root_norecurse(dir: &Dir) -> LintResult { return lint_err("Expected /ostree -> {expected}, not {link:?}"); } - // Check the prepare-root config - let prepareroot_path = "usr/lib/ostree/prepare-root.conf"; - let config_data = dir - .read_to_string(prepareroot_path) - .context(prepareroot_path)?; - let config = ostree_ext::glib::KeyFile::new(); - config.load_from_data(&config_data, ostree_ext::glib::KeyFileFlags::empty())?; - - if !ostree_ext::ostree_prepareroot::overlayfs_enabled_in_config(&config)? { + let config = ostree_prepareroot::require_config_from_root(dir)?; + if !ostree_prepareroot::overlayfs_enabled_in_config(&config)? { return lint_err("{prepareroot_path} does not have composefs enabled"); } diff --git a/ostree-ext/Cargo.toml b/ostree-ext/Cargo.toml index a5da3a0a4..097e16e2d 100644 --- a/ostree-ext/Cargo.toml +++ b/ostree-ext/Cargo.toml @@ -7,7 +7,6 @@ name = "ostree-ext" readme = "../README.md" repository = "https://github.com/ostreedev/ostree-rs-ext" version = "0.15.3" -rust-version = "1.74.0" [dependencies] # Note that we re-export the oci-spec types diff --git a/ostree-ext/src/ostree_prepareroot.rs b/ostree-ext/src/ostree_prepareroot.rs index e922213ca..b9fd5aeb9 100644 --- a/ostree-ext/src/ostree_prepareroot.rs +++ b/ostree-ext/src/ostree_prepareroot.rs @@ -3,11 +3,14 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT +use std::io::Read; use std::str::FromStr; use anyhow::{Context, Result}; use camino::Utf8Path; +use cap_std_ext::dirext::CapStdExtDirExt; use glib::Cast; +use ocidir::cap_std::fs::Dir; use ostree::prelude::FileExt; use ostree::{gio, glib}; @@ -37,6 +40,29 @@ pub(crate) fn load_config(root: &ostree::RepoFile) -> Result Result> { + for path in ["etc", "usr/lib"].into_iter().map(Utf8Path::new) { + let path = path.join(CONF_PATH); + let Some(mut f) = root.open_optional(&path)? else { + continue; + }; + let mut contents = String::new(); + f.read_to_string(&mut contents)?; + let kf = glib::KeyFile::new(); + kf.load_from_data(&contents, glib::KeyFileFlags::NONE) + .with_context(|| format!("Parsing {path}"))?; + return Ok(Some(kf)); + } + Ok(None) +} + +/// Require the configuration in the target root. +pub fn require_config_from_root(root: &Dir) -> Result { + load_config_from_root(root)? + .ok_or_else(|| anyhow::anyhow!("Failed to find {CONF_PATH} in /usr/lib or /etc")) +} + /// Query whether the target root has the `root.transient` key /// which sets up a transient overlayfs. pub(crate) fn overlayfs_root_enabled(root: &ostree::RepoFile) -> Result { diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 724e17d15..d1c451d2a 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -16,6 +16,8 @@ anyhow = { workspace = true } camino = { workspace = true } chrono = { workspace = true, features = ["std"] } fn-error-context = { workspace = true } +tar = "0.4" +toml = "0.8" tempfile = { workspace = true } mandown = "0.1.3" xshell = { version = "0.2.6" } diff --git a/xtask/src/xtask.rs b/xtask/src/xtask.rs index 9c6d8081a..d7c0d457c 100644 --- a/xtask/src/xtask.rs +++ b/xtask/src/xtask.rs @@ -3,7 +3,7 @@ use std::fs::File; use std::io::{BufRead, BufReader, BufWriter, Write}; -use std::process::{Command, Stdio}; +use std::process::Command; use anyhow::{anyhow, Context, Result}; use camino::{Utf8Path, Utf8PathBuf}; @@ -16,6 +16,13 @@ const TEST_IMAGES: &[&str] = &[ "quay.io/curl/curl:latest", "registry.access.redhat.com/ubi9/podman:latest", ]; +const TAR_REPRODUCIBLE_OPTS: &[&str] = &[ + "--sort=name", + "--owner=0", + "--group=0", + "--numeric-owner", + "--pax-option=exthdr.name=%d/PaxHeaders/%f,delete=atime,delete=ctime", +]; fn main() { if let Err(e) = try_main() { @@ -261,6 +268,23 @@ fn git_source_date_epoch(dir: &Utf8Path) -> Result { Ok(r) } +/// When using cargo-vendor-filterer --format=tar, the config generated has a bogus source +/// directory. This edits it to refer to vendor/ as a stable relative reference. +#[context("Editing vendor config")] +fn edit_vendor_config(config: &str) -> Result { + let mut config: toml::Value = toml::from_str(config)?; + let config = config.as_table_mut().unwrap(); + let source_table = config.get_mut("source").unwrap(); + let source_table = source_table.as_table_mut().unwrap(); + let vendored_sources = source_table.get_mut("vendored-sources").unwrap(); + let vendored_sources = vendored_sources.as_table_mut().unwrap(); + let previous = + vendored_sources.insert("directory".into(), toml::Value::String("vendor".into())); + assert!(previous.is_some()); + + Ok(config.to_string()) +} + #[context("Packaging")] fn impl_package(sh: &Shell) -> Result { let source_date_epoch = git_source_date_epoch(".".into())?; @@ -269,48 +293,40 @@ fn impl_package(sh: &Shell) -> Result { let namev = format!("{NAME}-{v}"); let p = Utf8Path::new("target").join(format!("{namev}.tar")); - let o = File::create(&p)?; let prefix = format!("{namev}/"); - let st = Command::new("git") - .args([ - "archive", - "--format=tar", - "--prefix", - prefix.as_str(), - "HEAD", - ]) - .stdout(Stdio::from(o)) - .status()?; - if !st.success() { - anyhow::bail!("Failed to run {st:?}"); - } - let st = Command::new("tar") - .args([ - "-r", - "-C", - "target", - "--sort=name", - "--owner=0", - "--group=0", - "--numeric-owner", - "--pax-option=exthdr.name=%d/PaxHeaders/%f,delete=atime,delete=ctime", - ]) - .arg(format!("--transform=s,^,{prefix},")) - .arg(format!("--mtime=@{source_date_epoch}")) - .args(["-f", p.as_str(), "man"]) - .status() - .context("Failed to execute tar")?; - if !st.success() { - anyhow::bail!("Failed to run {st:?}"); - } - let srcpath: Utf8PathBuf = format!("{p}.zstd").into(); - cmd!(sh, "zstd --rm -f {p} -o {srcpath}").run()?; + cmd!(sh, "git archive --format=tar --prefix={prefix} -o {p} HEAD").run()?; + // Generate the vendor directory now, as we want to embed the generated config to use + // it in our source. let vendorpath = Utf8Path::new("target").join(format!("{namev}-vendor.tar.zstd")); - cmd!( + let vendor_config = cmd!( sh, "cargo vendor-filterer --prefix=vendor --format=tar.zstd {vendorpath}" ) + .read()?; + let vendor_config = edit_vendor_config(&vendor_config)?; + // Append .cargo/vendor-config.toml (a made up filename) into the tar archive. + { + let tmpdir = tempfile::tempdir_in("target")?; + let tmpdir_path = tmpdir.path(); + let path = tmpdir_path.join("vendor-config.toml"); + std::fs::write(&path, vendor_config)?; + let source_date_epoch = format!("{source_date_epoch}"); + cmd!( + sh, + "tar -r -C {tmpdir_path} {TAR_REPRODUCIBLE_OPTS...} --mtime=@{source_date_epoch} --transform=s,^,{prefix}.cargo/, -f {p} vendor-config.toml" + ) + .run()?; + } + // Append our generated man pages. + cmd!( + sh, + "tar -r -C target {TAR_REPRODUCIBLE_OPTS...} --transform=s,^,{prefix}, -f {p} man" + ) .run()?; + // Compress with zstd + let srcpath: Utf8PathBuf = format!("{p}.zstd").into(); + cmd!(sh, "zstd --rm -f {p} -o {srcpath}").run()?; + Ok(Package { version: v, srcpath,