From 7660711a3ed16f34dd9bb4136fcb7a53cbc20f5d Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 5 Oct 2021 10:54:56 -0400 Subject: [PATCH 1/2] tests: Use --no-bindings for base commit Since we're not meaning to fetch this via libostree, using bindings inhibits native pulls for forthcoming `container copy` work. --- lib/tests/it/main.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/tests/it/main.rs b/lib/tests/it/main.rs index 4f9c8b9d..e19c2cee 100644 --- a/lib/tests/it/main.rs +++ b/lib/tests/it/main.rs @@ -58,7 +58,7 @@ fn generate_test_repo(dir: &Utf8Path) -> Result { indoc! {" cd {dir} ostree --repo=repo init --mode=archive - ostree --repo=repo commit -b {testref} --bootable --add-metadata-string=version=42.0 --gpg-homedir={gpghome} --gpg-sign={keyid} \ + ostree --repo=repo commit -b {testref} --bootable --no-bindings --add-metadata-string=version=42.0 --gpg-homedir={gpghome} --gpg-sign={keyid} \ --add-detached-metadata-string=my-detached-key=my-detached-value --tree=tar=exampleos.tar.zst ostree --repo=repo show {testref} "}, @@ -79,7 +79,7 @@ fn update_repo(repopath: &Utf8Path) -> Result<()> { let repopath = repopath.as_str(); let testref = TESTREF; bash!( - "ostree --repo={repopath} commit -b {testref} --tree=tar={srcpath}", + "ostree --repo={repopath} commit -b {testref} --no-bindings --tree=tar={srcpath}", testref, repopath, srcpath From 944cf763a7d8d9102a6fd052c66c58121ac5dccc Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 16 Sep 2021 20:34:47 -0400 Subject: [PATCH 2/2] Add a new container/store module The initial scope of this project was just "encapsulating" ostree commits in containers. However, when doing that a very, very natural question arises: Why not support *deriving* from that base image container, and have the tooling natively support importing it? This initial prototype code implements that. Here, we still use the `tar::import` path for the base image - we expect it to have a pre-generated ostree commit. This new `container::store` module processes layered images and generates (client side) ostree commits from the tar layers. There's a whole lot of new infrastructure we need around mapping ostree refs to blobs and images, etc. --- lib/src/cli.rs | 150 ++++++- lib/src/container/deploy.rs | 53 +++ lib/src/container/mod.rs | 2 + lib/src/container/store.rs | 383 ++++++++++++++++++ lib/src/tar/write.rs | 4 + .../fixtures/exampleos-derive-v2.ociarchive | Bin 0 -> 14336 bytes lib/tests/it/main.rs | 123 +++++- 7 files changed, 712 insertions(+), 3 deletions(-) create mode 100644 lib/src/container/deploy.rs create mode 100644 lib/src/container/store.rs create mode 100644 lib/tests/it/fixtures/exampleos-derive-v2.ociarchive diff --git a/lib/src/cli.rs b/lib/src/cli.rs index c834967b..a12098ad 100644 --- a/lib/src/cli.rs +++ b/lib/src/cli.rs @@ -8,11 +8,12 @@ use anyhow::Result; use ostree::gio; use std::collections::BTreeMap; -use std::convert::TryInto; +use std::convert::{TryFrom, TryInto}; use std::ffi::OsString; use structopt::StructOpt; -use crate::container::{Config, ImportOptions}; +use crate::container::store::{LayeredImageImporter, PrepareResult}; +use crate::container::{Config, ImportOptions, OstreeImageReference}; #[derive(Debug, StructOpt)] struct BuildOpts { @@ -107,6 +108,63 @@ enum ContainerOpts { #[structopt(long)] cmd: Option>, }, + + /// Commands for working with (possibly layered, non-encapsulated) container images. + Image(ContainerImageOpts), +} + +/// Options for import/export to tar archives. +#[derive(Debug, StructOpt)] +enum ContainerImageOpts { + /// List container images + List { + /// Path to the repository + #[structopt(long)] + repo: String, + }, + + /// Pull (or update) a container image. + Pull { + /// Path to the repository + #[structopt(long)] + repo: String, + + /// Image reference, e.g. ostree-remote-image:someremote:registry:quay.io/exampleos/exampleos:latest + imgref: String, + }, + + /// Copy a pulled container image from one repo to another. + Copy { + /// Path to the source repository + #[structopt(long)] + src_repo: String, + + /// Path to the destination repository + #[structopt(long)] + dest_repo: String, + + /// Image reference, e.g. ostree-remote-image:someremote:registry:quay.io/exampleos/exampleos:latest + imgref: String, + }, + + /// Perform initial deployment for a container image + Deploy { + /// Path to the system root + #[structopt(long)] + sysroot: String, + + /// Name for the state directory, also known as "osname". + #[structopt(long)] + stateroot: String, + + /// Image reference, e.g. ostree-remote-image:someremote:registry:quay.io/exampleos/exampleos:latest + #[structopt(long)] + imgref: String, + + #[structopt(long)] + /// Add a kernel argument + karg: Option>, + }, } /// Options for the Integrity Measurement Architecture (IMA). @@ -251,6 +309,52 @@ async fn container_info(imgref: &str) -> Result<()> { Ok(()) } +/// Write a layered container image into an OSTree commit. +async fn container_store(repo: &str, imgref: &str) -> Result<()> { + let repo = &ostree::Repo::open_at(libc::AT_FDCWD, repo, gio::NONE_CANCELLABLE)?; + let imgref = imgref.try_into()?; + let mut imp = LayeredImageImporter::new(&repo, &imgref).await?; + let prep = match imp.prepare().await? { + PrepareResult::AlreadyPresent(c) => { + println!("No changes in {} => {}", imgref, c); + return Ok(()); + } + PrepareResult::Ready(r) => r, + }; + if prep.base_layer.commit.is_none() { + let size = crate::glib::format_size(prep.base_layer.size()); + println!( + "Downloading base layer: {} ({})", + prep.base_layer.digest(), + size + ); + } else { + println!("Using base: {}", prep.base_layer.digest()); + } + for layer in prep.layers.iter() { + if layer.commit.is_some() { + println!("Using layer: {}", layer.digest()); + } else { + let size = crate::glib::format_size(layer.size()); + println!("Downloading layer: {} ({})", layer.digest(), size); + } + } + let import = imp.import(prep).await?; + if !import.layer_filtered_content.is_empty() { + for (layerid, filtered) in import.layer_filtered_content { + eprintln!("Unsupported paths filtered from {}:", layerid); + for (prefix, count) in filtered { + eprintln!(" {}: {}", prefix, count); + } + } + } + println!( + "Wrote: {} => {} => {}", + imgref, import.ostree_ref, import.commit + ); + Ok(()) +} + /// Add IMA signatures to an ostree commit, generating a new commit. fn ima_sign(cmdopts: &ImaSignOpts) -> Result<()> { let repo = @@ -309,6 +413,48 @@ where .collect(); container_export(&repo, &rev, &imgref, labels?, cmd).await } + ContainerOpts::Image(opts) => match opts { + ContainerImageOpts::List { repo } => { + let repo = + &ostree::Repo::open_at(libc::AT_FDCWD, &repo, gio::NONE_CANCELLABLE)?; + for image in crate::container::store::list_images(&repo)? { + println!("{}", image); + } + Ok(()) + } + ContainerImageOpts::Pull { repo, imgref } => container_store(&repo, &imgref).await, + ContainerImageOpts::Copy { + src_repo, + dest_repo, + imgref, + } => { + let src_repo = + &ostree::Repo::open_at(libc::AT_FDCWD, &src_repo, gio::NONE_CANCELLABLE)?; + let dest_repo = + &ostree::Repo::open_at(libc::AT_FDCWD, &dest_repo, gio::NONE_CANCELLABLE)?; + let imgref = OstreeImageReference::try_from(imgref.as_str())?; + crate::container::store::copy(src_repo, dest_repo, &imgref).await + } + ContainerImageOpts::Deploy { + sysroot, + stateroot, + imgref, + karg, + } => { + let sysroot = &ostree::Sysroot::new(Some(&gio::File::for_path(&sysroot))); + let imgref = OstreeImageReference::try_from(imgref.as_str())?; + let kargs = karg.as_deref(); + let kargs = kargs.map(|v| { + let r: Vec<_> = v.iter().map(|s| s.as_str()).collect(); + r + }); + let options = crate::container::deploy::DeployOpts { + kargs: kargs.as_deref(), + }; + crate::container::deploy::deploy(sysroot, &stateroot, &imgref, Some(options)) + .await + } + }, }, Opt::ImaSign(ref opts) => ima_sign(opts), } diff --git a/lib/src/container/deploy.rs b/lib/src/container/deploy.rs new file mode 100644 index 00000000..9d638d06 --- /dev/null +++ b/lib/src/container/deploy.rs @@ -0,0 +1,53 @@ +//! Perform initial setup for a container image based system root + +use super::OstreeImageReference; +use crate::container::store::PrepareResult; +use anyhow::Result; +use ostree::glib; + +/// The key in the OSTree origin which holds a serialized [`super::OstreeImageReference`]. +pub const ORIGIN_CONTAINER: &str = "container"; + +async fn pull_idempotent(repo: &ostree::Repo, imgref: &OstreeImageReference) -> Result { + let mut imp = super::store::LayeredImageImporter::new(repo, imgref).await?; + match imp.prepare().await? { + PrepareResult::AlreadyPresent(r) => Ok(r), + PrepareResult::Ready(prep) => Ok(imp.import(prep).await?.commit), + } +} + +/// Options configuring deployment. +#[derive(Debug, Default)] +pub struct DeployOpts<'a> { + /// Kernel arguments to use. + pub kargs: Option<&'a [&'a str]>, +} + +/// Write a container image to an OSTree deployment. +/// +/// This API is currently intended for only an initial deployment. +pub async fn deploy<'opts>( + sysroot: &ostree::Sysroot, + stateroot: &str, + imgref: &OstreeImageReference, + options: Option>, +) -> Result<()> { + let cancellable = ostree::gio::NONE_CANCELLABLE; + let options = options.unwrap_or_default(); + let repo = &sysroot.repo().unwrap(); + let commit = &pull_idempotent(repo, imgref).await?; + let origin = glib::KeyFile::new(); + origin.set_string("ostree", ORIGIN_CONTAINER, &imgref.to_string()); + let deployment = &sysroot.deploy_tree( + Some(stateroot), + commit, + Some(&origin), + None, + options.kargs.unwrap_or_default(), + cancellable, + )?; + let flags = ostree::SysrootSimpleWriteDeploymentFlags::NONE; + sysroot.simple_write_deployment(Some(stateroot), deployment, None, flags, cancellable)?; + sysroot.cleanup(cancellable)?; + Ok(()) +} diff --git a/lib/src/container/mod.rs b/lib/src/container/mod.rs index 1628f405..2612a47f 100644 --- a/lib/src/container/mod.rs +++ b/lib/src/container/mod.rs @@ -223,6 +223,7 @@ impl std::fmt::Display for OstreeImageReference { } } +pub mod deploy; mod export; pub use export::*; mod import; @@ -230,6 +231,7 @@ pub use import::*; mod imageproxy; mod oci; mod skopeo; +pub mod store; #[cfg(test)] mod tests { diff --git a/lib/src/container/store.rs b/lib/src/container/store.rs new file mode 100644 index 00000000..1c9b282f --- /dev/null +++ b/lib/src/container/store.rs @@ -0,0 +1,383 @@ +//! APIs for storing (layered) container images as OSTree commits +//! +//! # Extension of import support +//! +//! This code supports ingesting arbitrary layered container images from an ostree-exported +//! base. See [`super::import`] for more information on encaspulation of images. + +use super::imageproxy::ImageProxy; +use super::oci::ManifestLayer; +use super::*; +use crate::refescape; +use anyhow::{anyhow, Context}; +use fn_error_context::context; +use ostree::prelude::{Cast, ToVariant}; +use ostree::{gio, glib}; +use std::collections::{BTreeMap, HashMap}; + +/// The ostree ref prefix for blobs. +const LAYER_PREFIX: &str = "ostree/container/blob"; +/// The ostree ref prefix for image references. +const IMAGE_PREFIX: &str = "ostree/container/image"; + +/// The key injected into the merge commit for the manifest digest. +const META_MANIFEST_DIGEST: &str = "ostree.manifest-digest"; +/// The key injected into the merge commit with the manifest serialized as JSON. +const META_MANIFEST: &str = "ostree.manifest"; + +/// Convert e.g. sha256:12345... into `/ostree/container/blob/sha256_2B12345...`. +fn ref_for_blob_digest(d: &str) -> Result { + refescape::prefix_escape_for_ref(LAYER_PREFIX, d) +} + +/// Convert e.g. sha256:12345... into `/ostree/container/blob/sha256_2B12345...`. +fn ref_for_layer(l: &oci::ManifestLayer) -> Result { + ref_for_blob_digest(l.digest.as_str()) +} + +/// Convert e.g. sha256:12345... into `/ostree/container/blob/sha256_2B12345...`. +fn ref_for_image(l: &ImageReference) -> Result { + refescape::prefix_escape_for_ref(IMAGE_PREFIX, &l.to_string()) +} + +/// Context for importing a container image. +pub struct LayeredImageImporter { + repo: ostree::Repo, + proxy: ImageProxy, + imgref: OstreeImageReference, + ostree_ref: String, +} + +/// Result of invoking [`LayeredImageImporter::prepare`]. +pub enum PrepareResult { + /// The image reference is already present; the contained string is the OSTree commit. + AlreadyPresent(String), + /// The image needs to be downloaded + Ready(PreparedImport), +} + +/// A container image layer with associated downloaded-or-not state. +#[derive(Debug)] +pub struct ManifestLayerState { + layer: oci::ManifestLayer, + /// The ostree ref name for this layer. + pub ostree_ref: String, + /// The ostree commit that caches this layer, if present. + pub commit: Option, +} + +impl ManifestLayerState { + /// The cryptographic checksum. + pub fn digest(&self) -> &str { + self.layer.digest.as_str() + } + + /// The (possibly compressed) size. + pub fn size(&self) -> u64 { + self.layer.size + } +} + +/// Information about which layers need to be downloaded. +#[derive(Debug)] +pub struct PreparedImport { + /// The manifest digest that was found + pub manifest_digest: String, + /// The previously stored manifest digest. + pub previous_manifest_digest: Option, + /// The previously stored image ID. + pub previous_imageid: Option, + /// The required base layer. + pub base_layer: ManifestLayerState, + /// Any further layers. + pub layers: Vec, + /// TODO: serialize this into the commit object + manifest: oci::Manifest, +} + +/// A successful import of a container image. +#[derive(Debug, PartialEq, Eq)] +pub struct CompletedImport { + /// The ostree ref used for the container image. + pub ostree_ref: String, + /// The current commit. + pub commit: String, + /// A mapping from layer blob IDs to a count of content filtered out + /// by toplevel path. + pub layer_filtered_content: BTreeMap>, +} + +// Given a manifest, compute its ostree ref name and cached ostree commit +fn query_layer(repo: &ostree::Repo, layer: ManifestLayer) -> Result { + let ostree_ref = ref_for_layer(&layer)?; + let commit = repo.resolve_rev(&ostree_ref, true)?.map(|s| s.to_string()); + Ok(ManifestLayerState { + layer, + ostree_ref, + commit, + }) +} + +fn manifest_from_commitmeta(commit_meta: &glib::VariantDict) -> Result { + let manifest_bytes: String = commit_meta + .lookup::(META_MANIFEST)? + .ok_or_else(|| anyhow!("Failed to find {} metadata key", META_MANIFEST))?; + let manifest: oci::Manifest = serde_json::from_str(&manifest_bytes)?; + Ok(manifest) +} + +fn manifest_from_commit(commit: &glib::Variant) -> Result { + let commit_meta = &commit.child_value(0); + let commit_meta = &ostree::glib::VariantDict::new(Some(commit_meta)); + manifest_from_commitmeta(commit_meta) +} + +impl LayeredImageImporter { + /// Create a new importer. + pub async fn new(repo: &ostree::Repo, imgref: &OstreeImageReference) -> Result { + let proxy = ImageProxy::new(&imgref.imgref).await?; + let repo = repo.clone(); + let ostree_ref = ref_for_image(&imgref.imgref)?; + Ok(LayeredImageImporter { + repo, + proxy, + ostree_ref, + imgref: imgref.clone(), + }) + } + + /// Determine if there is a new manifest, and if so return its digest. + #[context("Fetching manifest")] + pub async fn prepare(&mut self) -> Result { + match &self.imgref.sigverify { + SignatureSource::ContainerPolicy if skopeo::container_policy_is_default_insecure()? => { + return Err(anyhow!("containers-policy.json specifies a default of `insecureAcceptAnything`; refusing usage")); + } + SignatureSource::OstreeRemote(_) => { + return Err(anyhow!( + "Cannot currently verify layered containers via ostree remote" + )); + } + _ => {} + } + + let (manifest_digest, manifest_bytes) = self.proxy.fetch_manifest().await?; + let manifest: oci::Manifest = serde_json::from_slice(&manifest_bytes)?; + let new_imageid = manifest.imageid(); + + // Query for previous stored state + let (previous_manifest_digest, previous_imageid) = + if let Some(merge_commit) = self.repo.resolve_rev(&self.ostree_ref, true)? { + let (merge_commit_obj, _) = self.repo.load_commit(merge_commit.as_str())?; + let commit_meta = &merge_commit_obj.child_value(0); + let commit_meta = ostree::glib::VariantDict::new(Some(commit_meta)); + let previous_digest: String = + commit_meta.lookup(META_MANIFEST_DIGEST)?.ok_or_else(|| { + anyhow!("Missing {} metadata on merge commit", META_MANIFEST_DIGEST) + })?; + // If the manifest digests match, we're done. + if previous_digest == manifest_digest { + return Ok(PrepareResult::AlreadyPresent(merge_commit.to_string())); + } + // Failing that, if they have the same imageID, we're also done. + let previous_manifest = manifest_from_commitmeta(&commit_meta)?; + if previous_manifest.imageid() == new_imageid { + return Ok(PrepareResult::AlreadyPresent(merge_commit.to_string())); + } + ( + Some(previous_digest), + Some(previous_manifest.imageid().to_string()), + ) + } else { + (None, None) + }; + + let mut layers = manifest.layers.iter().cloned(); + // We require a base layer. + let base_layer = layers.next().ok_or_else(|| anyhow!("No layers found"))?; + let base_layer = query_layer(&self.repo, base_layer)?; + + let layers: Result> = layers + .map(|layer| -> Result<_> { query_layer(&self.repo, layer) }) + .collect(); + let layers = layers?; + + let imp = PreparedImport { + manifest, + manifest_digest, + previous_manifest_digest, + previous_imageid, + base_layer, + layers, + }; + Ok(PrepareResult::Ready(imp)) + } + + /// Import a layered container image + pub async fn import(mut self, import: PreparedImport) -> Result { + // First download the base image (if necessary) - we need the SELinux policy + // there to label all following layers. + let base_layer = import.base_layer; + let base_commit = if let Some(c) = base_layer.commit { + c + } else { + let blob = self.proxy.fetch_layer_decompress(&base_layer.layer).await?; + let commit = crate::tar::import_tar(&self.repo, blob, None) + .await + .with_context(|| format!("Parsing blob {}", &base_layer.digest()))?; + // TODO support ref writing in tar import + self.repo.set_ref_immediate( + None, + base_layer.ostree_ref.as_str(), + Some(commit.as_str()), + gio::NONE_CANCELLABLE, + )?; + commit + }; + + let mut layer_commits = Vec::new(); + let mut layer_filtered_content = BTreeMap::new(); + for layer in import.layers { + if let Some(c) = layer.commit { + layer_commits.push(c.to_string()); + } else { + let blob = self.proxy.fetch_layer_decompress(&layer.layer).await?; + // An important aspect of this is that we SELinux label the derived layers using + // the base policy. + let opts = crate::tar::WriteTarOptions { + base: Some(base_commit.clone()), + selinux: true, + }; + let r = + crate::tar::write_tar(&self.repo, blob, layer.ostree_ref.as_str(), Some(opts)) + .await + .with_context(|| format!("Parsing layer blob {}", layer.digest()))?; + layer_commits.push(r.commit); + if !r.filtered.is_empty() { + layer_filtered_content.insert(layer.digest().to_string(), r.filtered); + } + } + } + + // We're done with the proxy, make sure it didn't have any errors. + self.proxy.finalize().await?; + + let serialized_manifest = serde_json::to_string(&import.manifest)?; + let mut metadata = HashMap::new(); + metadata.insert(META_MANIFEST_DIGEST, import.manifest_digest.to_variant()); + metadata.insert(META_MANIFEST, serialized_manifest.to_variant()); + metadata.insert( + "ostree.importer.version", + env!("CARGO_PKG_VERSION").to_variant(), + ); + let metadata = metadata.to_variant(); + + // Destructure to transfer ownership to thread + let repo = self.repo; + let target_ref = self.ostree_ref; + let (ostree_ref, commit) = crate::tokio_util::spawn_blocking_cancellable( + move |cancellable| -> Result<(String, String)> { + let cancellable = Some(cancellable); + let repo = &repo; + let txn = repo.auto_transaction(cancellable)?; + let (base_commit_tree, _) = repo.read_commit(&base_commit, cancellable)?; + let base_commit_tree = base_commit_tree.downcast::().unwrap(); + let base_contents_obj = base_commit_tree.tree_get_contents_checksum().unwrap(); + let base_metadata_obj = base_commit_tree.tree_get_metadata_checksum().unwrap(); + let mt = ostree::MutableTree::from_checksum( + &repo, + &base_contents_obj, + &base_metadata_obj, + ); + // Layer all subsequent commits + for commit in layer_commits { + let (layer_tree, _) = repo.read_commit(&commit, cancellable)?; + repo.write_directory_to_mtree(&layer_tree, &mt, None, cancellable)?; + } + + let merged_root = repo.write_mtree(&mt, cancellable)?; + let merged_root = merged_root.downcast::().unwrap(); + let merged_commit = repo.write_commit( + None, + None, + None, + Some(&metadata), + &merged_root, + cancellable, + )?; + repo.transaction_set_ref(None, &target_ref, Some(merged_commit.as_str())); + txn.commit(cancellable)?; + Ok((target_ref, merged_commit.to_string())) + }, + ) + .await??; + Ok(CompletedImport { + ostree_ref, + commit, + layer_filtered_content, + }) + } +} + +/// List all images stored +pub fn list_images(repo: &ostree::Repo) -> Result> { + let cancellable = gio::NONE_CANCELLABLE; + let refs = repo.list_refs_ext( + Some(IMAGE_PREFIX), + ostree::RepoListRefsExtFlags::empty(), + cancellable, + )?; + let r: Result> = refs + .keys() + .map(|imgname| refescape::unprefix_unescape_ref(IMAGE_PREFIX, imgname)) + .collect(); + Ok(r?) +} + +/// Copy a downloaded image from one repository to another. +pub async fn copy( + src_repo: &ostree::Repo, + dest_repo: &ostree::Repo, + imgref: &OstreeImageReference, +) -> Result<()> { + let ostree_ref = ref_for_image(&imgref.imgref)?; + let rev = src_repo.resolve_rev(&ostree_ref, false)?.unwrap(); + let (commit_obj, _) = src_repo.load_commit(rev.as_str())?; + let manifest: oci::Manifest = manifest_from_commit(&commit_obj)?; + // Create a task to copy each layer, plus the final ref + let layer_refs = manifest + .layers + .iter() + .map(|layer| ref_for_layer(&layer)) + .chain(std::iter::once(Ok(ostree_ref))); + for ostree_ref in layer_refs { + let ostree_ref = ostree_ref?; + let src_repo = src_repo.clone(); + let dest_repo = dest_repo.clone(); + crate::tokio_util::spawn_blocking_cancellable(move |cancellable| -> Result<_> { + let cancellable = Some(cancellable); + let srcfd = &format!("file:///proc/self/fd/{}", src_repo.dfd()); + let flags = ostree::RepoPullFlags::MIRROR; + let opts = glib::VariantDict::new(None); + let refs = [ostree_ref.as_str()]; + // Some older archives may have bindings, we don't need to verify them. + opts.insert("disable-verify-bindings", &true); + opts.insert("refs", &&refs[..]); + opts.insert("flags", &(flags.bits() as i32)); + let options = opts.to_variant(); + dest_repo.pull_with_options(&srcfd, &options, None, cancellable)?; + Ok(()) + }) + .await??; + } + Ok(()) +} + +/// Remove the specified images and their corresponding blobs. +pub fn prune_images(_repo: &ostree::Repo, _imgs: &[&str]) -> Result<()> { + // Most robust approach is to iterate over all known images, load the + // manifest and build the set of reachable blobs, then compute the set + // Set(unreachable) = Set(all) - Set(reachable) + // And remove the unreachable ones. + unimplemented!() +} diff --git a/lib/src/tar/write.rs b/lib/src/tar/write.rs index b4ae95bd..f156e06c 100644 --- a/lib/src/tar/write.rs +++ b/lib/src/tar/write.rs @@ -208,6 +208,10 @@ pub async fn write_tar( c.arg("--selinux-policy"); c.arg(sepolicy.path()); } + c.arg(&format!( + "--add-metadata-string=ostree.importer.version={}", + env!("CARGO_PKG_VERSION") + )); c.args(&[ "--no-bindings", "--tar-autocreate-parents", diff --git a/lib/tests/it/fixtures/exampleos-derive-v2.ociarchive b/lib/tests/it/fixtures/exampleos-derive-v2.ociarchive new file mode 100644 index 0000000000000000000000000000000000000000..42b91b1187a133f27dac1a11a4c29d757ee0ab20 GIT binary patch literal 14336 zcmeHNcU%)$*N&h9mc@o*4F(hugh`+HsHBxr|9Ep>lJ!668eijonKzHdWPK|d{73$0AkhDGDvS2r|Nk8{`JMmiA{w71;)uj- zI*){O-y475s{G35HZDU0Ry44sX`8w$rK9_l*XcqIZU2V#G;EBH1=l8iDcv#MFN#~otLgddUSps+T15I z0xP!|*YAKd<<`KC)6yBf)AwZ**qu7rIfS=xfhc~Jo&FEq5-{=f^n;r^Y@EOB@};bz zsi)@N*s^JazgmHfFttuBRRo}YSrB(`=9uCJ)!VCGr3<9 zc+>Vo+M>kE60c>Q>tJMBU2*sL$XhRmVWw+@z6p7@Q|7j+x$@i#hUArXkz1}lRzFH3 z*H1i}lr<}8F9-_o=;EBbaNE-UbIxBjIpM!P%m+YP0H9CW%N2DXb^Av-@_$&^AL5@c zWHVU|geK&(1Z*A^Md@@gDyE5rECHRx=JDtZ8i&E6iTMJU75FSFhbI)UiGO@5Di(YO zr+*XwR4SA8G5(4CPh(K&-{b!?X#TUqVHBCbTja}|FA9`Nq<)mqG#iRQ>L-?XQAUSR zd__VDZ*Hi+h%%bO^Y{0W2zWtomfdncp{-OPv6cApyhOGi=G!i(S}&DB5jGT|#7iU# zf@?nPD@HTai=O+!)I1+*wMv=;b0!4%ueSIZ_)93;i*ldK);xL%74hx`v zn0x`1C1mqOe1QOEh&e1iL%?FQQ7VggvpAeZFsylj0&hu>NDvepsHq6PLKZ_4?QpS= z#4mUS?0hU94t~pFj;73>Fn8)$yI@(MosU$&^RbihC4Qqn>HpUKctJDtp$nT7n6f7M zLId3>nq_dvj?a^M6XP>@e36ePi%F!m{9uWXkmqgtG3`)oX*65<$At7x=D3eqw8_ne zLcAjoYpe3q{#XRLJ5`g$NB5(U}Y} z6{R9fo`A+vzQhXD z7NJuaOa_y#VTE?8Cm$}R*b&~5d7F(An3<38llohkO|Z8&6H9zVqfsHkpu;-Cp@~q0 zPerJFzEC6(u=pG*3XKu+MI1yx6$t4<_?!v12q?TQjsSt0pv~}wkeS5KOhW*8+kqlG zo}Va$kV`Yuu+G<4Q;w1VgB?<11RM*BdP!%a!};_p@HlMM2^ z{QtFkpiks}+Wd0s|D(HCpH9zHeZ!W8 zBzgKidGKcW%jJ)E9`Sv0)^}h!{L8eP<>wJNE3na^x2kOBa>dxveg1QXCSNz9XHKLh z-xN1KS{As|y{hWbQU1`1r?sfaYEVJIZnT@_*wLHm6%{rYUbUx_x37;*7VJ%|>yjU{ zDm1il*R~?}>iXgJi(-B8^j4Ksb^Ug~$FUMT!h9SS)22+>)hyU0wU|pk^8|B?KCBMGhVYVZ(b8zp99Kju~X5(HK?58z~bFtgVHV%WQ_;?6rfLAX{UDi3j9zZu)PKHo45{m$ejKW zLy0*XG;>-!yMW*b-0qFM4_>FI{$OPQPL8PugOuDNkgtxvf-T+ypq4>uV<>|#$)FaL ztBt1s{dZUZsCw(ExeSenZ2HhQhS%GYDIiQ|gud2@AzC9^T>3}cBx?OlM_=3l*mw4Q%wT*&#E;@b%2ryFi2_$(B4{X6||)Wi&53e$<+Vf)?nZ7~Ha>R@G3QDrkHXuxi8%wM3bu1&Vx- zruXv)&itgsp}Uq319g1(n>c9Ll80$vn*EO3Q>!`_8mCd+QLEZqr@Bn#`4r=**wd@z zdwx#`DV$5tq15p2;qY85N$(GI;OkUr=>SZ%J}E#6pZ@?h>2TS7&CrLilha60O&xz3 z6Tr1fyY=8IgryX4nlaedLa-z zMp{r>bRG+X4%tPJc(V$6TLS{IoCwftOQ>2G6gqaa$!?IfF%t$Irb!bzIzVk7nc7FK z@X)EZ)2Z*IQ%@jAxtz=pgilU^-63Ln5u~sZRBM4g9iQ~L;RVq zFic0z#>wc3R@^Baxb9JL{l_$@WBK6N_%`hMigG16A9syt1Q#3fTDURlG`%ViQ4w8_ z^-+2{j5J4DKbeCyw1Df4IA%Q?x5Jy07nb0%PS-%bCy1|luNizJx53>yG-1U(H9#bQ z?9G~daZyzooTst08k{iKgf#*}25XFsXp`4PDY@%`YxxISo$IA#jVlCL6WY_Evxzk0 zsdJw-OsgL$+**gt$-zU0K{Pz^UFB7BYb`)-#Qgsto;3~}(Vjd~r%&gvW4fR+fjWr1 zo~Co)ydxVgRuehT96-`P#jI54x@h?^4~qK|lfUG=NwDDP0@zNctDlZ68?@#C2Q9UK*yME7 z@w%g47h1wg{$+dY{ECIcYLA{=G^*7@bUCy31}CuhK)VbN>QH{(z(vTz>5h31SyNM3 zlQ+7YalZRFZ_SQ-j`!vcxIKUWO@o>TvL)62jfpQd*4HO{`HxiMO1=f<;PiJ3Wzr}Q!t!eYz5$5x@>@bYzayIwOnTK(m zw$FJyxB`!jT2j)~l+>z!#U_4l!ON%|y#80^M91`0*@WfmO^}gG9(6M`={t2?4yT90Y=jl#sa{JqgZ0W*1JEUVn_ALy5 zn-P3Sk>h_s6+ak+D6rK5Z(A)rA2mFPos#Oq*_y#x9A2FXnr~DT6yA;oUQjZxZb| zdU#2Xf(JbYjtQO>c3^vXU;Uv^hg$XMG%^737OT{@cWHq zqdRsWqQgzmi-p1VNgy|&-TBdzBA0XlZ||#?+K9%>fsxY0yPQx()WzL*Dqm&O4haJL z51Z0wrAL3xfB|t$Sf$EyC(u8z+v=bdbrwZJ(tEFWUO2<5D)+sIv(iOTY zIQIVW){6_~gVRr%!><%NOAkA!j=jX5x^>M`V`uTS;ZbA2zpkg`dCC-Lr>v$o^a6Jn zk{!>AUrhcvuD5tou5W=gJN3QisW{`4O25)9{?l!#1!|vSvfpA_MR|xgQMERBPV$P1 zGOu&K58pq1I?Y`1Mh2l=615wc*MMuPG@t1SckV1WW_mDh;fBPxh%m}_|9kZEh=5y* zB#!BqHcN!*T|4WY+jhNBjqO*0oN~`+;0g)vhsLKIy5BPtuRjFpZQ`?DVKPNm^C<%J zKsheP@ z)yr*dPAN``q>Pu>1$dU~w$*+X*1qR_Q%Iy*a8&oomTE^2*op3tlChr4{$$&&DR-Yr8)HQ%rIj zaLI_y%w$^xA__zpjHQUf5r6xnU4gIgMQLmT2;fZOLxcs&n^JpN)0Iw1856m?f)~MuG zXC2-E^A-T{IIQBHK5}-+@vElmFK#kfheVEa-{XcsUxYg39WEqpcfA+%%A!(0+Ar z2zPAY3ERvaFxbS2HUnB(eYx#^$|=^xxsO&=Sq|h%lR>l5E=%{O!g|-4I~%S9Wsf^h zlXC%t$vQsb>FKAZ) zuXgg36QFK9*frPtRtI-dqy&Mr?;6l66QtDwSv0oQ%smxc84n^er!L(`GLOZo z!Q}TiJuok_DA0Y@@NfKx@e2K+ZxKlFts8Zb?^W!^Fj8A`CTRJQaq@J zMTXrHNEKU`7G-wrV!f=V>2br;)6q%WTnkADXSj5J{Jy#V@HKE8v~o%I7r>$UO007k zFi&n$R^M#Q0QUQz-`A0Y1dpxaZZOC*vD%W+jbu>ql~WxYY6+OrZ* zI|cN&E`2*4y*Ml2K<&Oe!!8%{bo)cyi~`gSo^ zZ32e2a&IiGY~ltZ;AbZl9gjS|C#2v!s3^IL_4l%gD0AP|H&k7J0z7M%qTtq~C~iL? zxh=URp5EA6ydK}6QYEh(3kqtt@t2Lcob9x_tH+$8KCUHQL`DxTIY$rfJ9%Xpqk9)O z{>;MmyhRZ`_ME`&;*!DLlQp2y1o)|ox0k8OEhQknSon_Jt}0bDVT)phH{#s577(2S zoaU;r5soPxtcMN1BpggW%$qoR*4imoJJxp?dTjz{%Vq0fMuwZuT)gJ^!w~<<8o*Gs zaW#omFvef)^`Q3wqL#QlS?smJ&eHte`P*sX%G_$>Az5{&calMi<*|Zfe-P>RXz;SG zcBLzuxPzJLWBl&Dvg|up-jIrotK)I8#-qk(w=T~D`(U$d3e zgVF*vy#Y-z*gU+{5g1ISaKWK*3h-iUS6No)j?LXV6<#BqGJnaUEI$+O> zIc|O<)Tn4#mB?@nwLaQ-L2sXk5f!Ot*1h&Kc$FGI-NgdYk9rE&SwLZp&ub&Eto|u` zweiO2LIpT-yN!GNN9PXL#p4g39N);+ z8iid@?Tch@S`E(D9P8d`OFCVdGokWuh`YZ$EMT{{!i1l84Xo?-9Qe!}KeI> zqR(t4l~a^E!sZ<&&U zV`JcxUhq9&*GQMkL%miPC*rX=0LyI#t~BQS2&Bu&Z8sK7-(G1N@ixY&kN=V%Z4j&6 zjuo|U7ArTqjIs&o)eWYlW4BwdDid&2RqEC|>!n`4`zryEmXup8$H~nqMKhS&j2X71 zf$>(CAh{3E1hGRvzfk=x3kLouR07EQ*S|b;GAJ@QfN|}Sj&pB<4)2p^U!2#@t(-LL zcAY%S6vVoN;$wLHaHdBu`yi52M!&+a0-v{cjGkYbZhdJ@gxMsLNsi(8r~`ec9qq%5 zus;C?E(SF)!fUY}Cu8w6m|YgV1gpxqhU(bPC%~TFupTVi4{F+6c1fe%RRtq)gF_&@ zM)lLip)bJF8mwyGZb}-6GY_=iTo_j-g>Pqxvqwh3M(Lysy>Ilc_FIUR_z6WTY>9Il z-$MEanO}bYkHB*mpPc{uaQ+ivQoq0d|8^evDEY6>|M>F!Bx3ljjQCplch9lt)_ma< z5MA_jr^|>Bo|;oYEE?_mSGF-(j=*;Wz9aCz4uSsyDUIH* literal 0 HcmV?d00001 diff --git a/lib/tests/it/main.rs b/lib/tests/it/main.rs index e19c2cee..da773892 100644 --- a/lib/tests/it/main.rs +++ b/lib/tests/it/main.rs @@ -2,6 +2,7 @@ use anyhow::{Context, Result}; use camino::{Utf8Path, Utf8PathBuf}; use fn_error_context::context; use indoc::indoc; +use ostree_ext::container::store::PrepareResult; use ostree_ext::container::{ Config, ImageReference, OstreeImageReference, SignatureSource, Transport, }; @@ -24,6 +25,7 @@ const TEST_REGISTRY_DEFAULT: &str = "localhost:5000"; /// Image that contains a base exported layer, then a `podman build` of an added file on top. const EXAMPLEOS_DERIVED_OCI: &[u8] = include_bytes!("fixtures/exampleos-derive.ociarchive"); +const EXAMPLEOS_DERIVED_V2_OCI: &[u8] = include_bytes!("fixtures/exampleos-derive-v2.ociarchive"); fn assert_err_contains(r: Result, s: impl AsRef) { let s = s.as_ref(); @@ -386,7 +388,7 @@ async fn test_container_import_export() -> Result<()> { Ok(()) } -/// We should currently reject an image with multiple layers. +/// We should reject an image with multiple layers when doing an "import" - i.e. a direct un-encapsulation. #[tokio::test] async fn test_container_import_derive() -> Result<()> { let fixture = Fixture::new()?; @@ -404,6 +406,125 @@ async fn test_container_import_derive() -> Result<()> { Ok(()) } +/// But layers work via the container::write module. +#[tokio::test] +async fn test_container_write_derive() -> Result<()> { + let fixture = Fixture::new()?; + let exampleos_path = &fixture.path.join("exampleos-derive.ociarchive"); + std::fs::write(exampleos_path, EXAMPLEOS_DERIVED_OCI)?; + let exampleos_ref = OstreeImageReference { + sigverify: SignatureSource::ContainerPolicyAllowInsecure, + imgref: ImageReference { + transport: Transport::OciArchive, + name: exampleos_path.to_string(), + }, + }; + + // There shouldn't be any container images stored yet. + let images = ostree_ext::container::store::list_images(&fixture.destrepo)?; + assert!(images.is_empty()); + + // Pull a derived image - two layers, new base plus one layer. + let mut imp = + ostree_ext::container::store::LayeredImageImporter::new(&fixture.destrepo, &exampleos_ref) + .await?; + let prep = match imp.prepare().await? { + PrepareResult::AlreadyPresent(_) => panic!("should not be already imported"), + PrepareResult::Ready(r) => r, + }; + assert!(prep.base_layer.commit.is_none()); + for layer in prep.layers.iter() { + assert!(layer.commit.is_none()); + } + let import = imp.import(prep).await?; + // We should have exactly one image stored. + let images = ostree_ext::container::store::list_images(&fixture.destrepo)?; + assert_eq!(images.len(), 1); + assert_eq!(images[0], exampleos_ref.imgref.to_string()); + + // Parse the commit and verify we pulled the derived content. + bash!( + "ostree --repo={repo} ls {r} /usr/share/anewfile", + repo = fixture.destrepo_path.as_str(), + r = import.ostree_ref.as_str() + )?; + + // Import again, but there should be no changes. + let mut imp = + ostree_ext::container::store::LayeredImageImporter::new(&fixture.destrepo, &exampleos_ref) + .await?; + let already_present = match imp.prepare().await? { + PrepareResult::AlreadyPresent(c) => c, + PrepareResult::Ready(_) => { + panic!("Should have already imported {}", import.ostree_ref) + } + }; + assert_eq!(import.commit, already_present); + + // Test upgrades; replace the oci-archive with new content. + std::fs::write(exampleos_path, EXAMPLEOS_DERIVED_V2_OCI)?; + let mut imp = + ostree_ext::container::store::LayeredImageImporter::new(&fixture.destrepo, &exampleos_ref) + .await?; + let prep = match imp.prepare().await? { + PrepareResult::AlreadyPresent(_) => panic!("should not be already imported"), + PrepareResult::Ready(r) => r, + }; + // We *should* already have the base layer. + assert!(prep.base_layer.commit.is_some()); + // One new layer + assert_eq!(prep.layers.len(), 1); + for layer in prep.layers.iter() { + assert!(layer.commit.is_none()); + } + let import = imp.import(prep).await?; + // New commit. + assert_ne!(import.commit, already_present); + // We should still have exactly one image stored. + let images = ostree_ext::container::store::list_images(&fixture.destrepo)?; + assert_eq!(images.len(), 1); + assert_eq!(images[0], exampleos_ref.imgref.to_string()); + + // Verify we have the new file and *not* the old one + bash!( + "ostree --repo={repo} ls {r} /usr/share/anewfile2 >/dev/null + if ostree --repo={repo} ls {r} /usr/share/anewfile 2>/dev/null; then + echo oops; exit 1 + fi + ", + repo = fixture.destrepo_path.as_str(), + r = import.ostree_ref.as_str() + )?; + + // And there should be no changes on upgrade again. + let mut imp = + ostree_ext::container::store::LayeredImageImporter::new(&fixture.destrepo, &exampleos_ref) + .await?; + let already_present = match imp.prepare().await? { + PrepareResult::AlreadyPresent(c) => c, + PrepareResult::Ready(_) => { + panic!("Should have already imported {}", import.ostree_ref) + } + }; + assert_eq!(import.commit, already_present); + + // Create a new repo, and copy to it + let destrepo2 = ostree::Repo::create_at( + ostree::AT_FDCWD, + fixture.path.join("destrepo2").as_str(), + ostree::RepoMode::Archive, + None, + gio::NONE_CANCELLABLE, + )?; + ostree_ext::container::store::copy(&fixture.destrepo, &destrepo2, &exampleos_ref).await?; + + let images = ostree_ext::container::store::list_images(&destrepo2)?; + assert_eq!(images.len(), 1); + assert_eq!(images[0], exampleos_ref.imgref.to_string()); + + Ok(()) +} + #[ignore] #[tokio::test] // Verify that we can push and pull to a registry, not just oci-archive:.