diff --git a/lib/src/cli.rs b/lib/src/cli.rs index c834967b..a12098ad 100644 --- a/lib/src/cli.rs +++ b/lib/src/cli.rs @@ -8,11 +8,12 @@ use anyhow::Result; use ostree::gio; use std::collections::BTreeMap; -use std::convert::TryInto; +use std::convert::{TryFrom, TryInto}; use std::ffi::OsString; use structopt::StructOpt; -use crate::container::{Config, ImportOptions}; +use crate::container::store::{LayeredImageImporter, PrepareResult}; +use crate::container::{Config, ImportOptions, OstreeImageReference}; #[derive(Debug, StructOpt)] struct BuildOpts { @@ -107,6 +108,63 @@ enum ContainerOpts { #[structopt(long)] cmd: Option>, }, + + /// Commands for working with (possibly layered, non-encapsulated) container images. + Image(ContainerImageOpts), +} + +/// Options for import/export to tar archives. +#[derive(Debug, StructOpt)] +enum ContainerImageOpts { + /// List container images + List { + /// Path to the repository + #[structopt(long)] + repo: String, + }, + + /// Pull (or update) a container image. + Pull { + /// Path to the repository + #[structopt(long)] + repo: String, + + /// Image reference, e.g. ostree-remote-image:someremote:registry:quay.io/exampleos/exampleos:latest + imgref: String, + }, + + /// Copy a pulled container image from one repo to another. + Copy { + /// Path to the source repository + #[structopt(long)] + src_repo: String, + + /// Path to the destination repository + #[structopt(long)] + dest_repo: String, + + /// Image reference, e.g. ostree-remote-image:someremote:registry:quay.io/exampleos/exampleos:latest + imgref: String, + }, + + /// Perform initial deployment for a container image + Deploy { + /// Path to the system root + #[structopt(long)] + sysroot: String, + + /// Name for the state directory, also known as "osname". + #[structopt(long)] + stateroot: String, + + /// Image reference, e.g. ostree-remote-image:someremote:registry:quay.io/exampleos/exampleos:latest + #[structopt(long)] + imgref: String, + + #[structopt(long)] + /// Add a kernel argument + karg: Option>, + }, } /// Options for the Integrity Measurement Architecture (IMA). @@ -251,6 +309,52 @@ async fn container_info(imgref: &str) -> Result<()> { Ok(()) } +/// Write a layered container image into an OSTree commit. +async fn container_store(repo: &str, imgref: &str) -> Result<()> { + let repo = &ostree::Repo::open_at(libc::AT_FDCWD, repo, gio::NONE_CANCELLABLE)?; + let imgref = imgref.try_into()?; + let mut imp = LayeredImageImporter::new(&repo, &imgref).await?; + let prep = match imp.prepare().await? { + PrepareResult::AlreadyPresent(c) => { + println!("No changes in {} => {}", imgref, c); + return Ok(()); + } + PrepareResult::Ready(r) => r, + }; + if prep.base_layer.commit.is_none() { + let size = crate::glib::format_size(prep.base_layer.size()); + println!( + "Downloading base layer: {} ({})", + prep.base_layer.digest(), + size + ); + } else { + println!("Using base: {}", prep.base_layer.digest()); + } + for layer in prep.layers.iter() { + if layer.commit.is_some() { + println!("Using layer: {}", layer.digest()); + } else { + let size = crate::glib::format_size(layer.size()); + println!("Downloading layer: {} ({})", layer.digest(), size); + } + } + let import = imp.import(prep).await?; + if !import.layer_filtered_content.is_empty() { + for (layerid, filtered) in import.layer_filtered_content { + eprintln!("Unsupported paths filtered from {}:", layerid); + for (prefix, count) in filtered { + eprintln!(" {}: {}", prefix, count); + } + } + } + println!( + "Wrote: {} => {} => {}", + imgref, import.ostree_ref, import.commit + ); + Ok(()) +} + /// Add IMA signatures to an ostree commit, generating a new commit. fn ima_sign(cmdopts: &ImaSignOpts) -> Result<()> { let repo = @@ -309,6 +413,48 @@ where .collect(); container_export(&repo, &rev, &imgref, labels?, cmd).await } + ContainerOpts::Image(opts) => match opts { + ContainerImageOpts::List { repo } => { + let repo = + &ostree::Repo::open_at(libc::AT_FDCWD, &repo, gio::NONE_CANCELLABLE)?; + for image in crate::container::store::list_images(&repo)? { + println!("{}", image); + } + Ok(()) + } + ContainerImageOpts::Pull { repo, imgref } => container_store(&repo, &imgref).await, + ContainerImageOpts::Copy { + src_repo, + dest_repo, + imgref, + } => { + let src_repo = + &ostree::Repo::open_at(libc::AT_FDCWD, &src_repo, gio::NONE_CANCELLABLE)?; + let dest_repo = + &ostree::Repo::open_at(libc::AT_FDCWD, &dest_repo, gio::NONE_CANCELLABLE)?; + let imgref = OstreeImageReference::try_from(imgref.as_str())?; + crate::container::store::copy(src_repo, dest_repo, &imgref).await + } + ContainerImageOpts::Deploy { + sysroot, + stateroot, + imgref, + karg, + } => { + let sysroot = &ostree::Sysroot::new(Some(&gio::File::for_path(&sysroot))); + let imgref = OstreeImageReference::try_from(imgref.as_str())?; + let kargs = karg.as_deref(); + let kargs = kargs.map(|v| { + let r: Vec<_> = v.iter().map(|s| s.as_str()).collect(); + r + }); + let options = crate::container::deploy::DeployOpts { + kargs: kargs.as_deref(), + }; + crate::container::deploy::deploy(sysroot, &stateroot, &imgref, Some(options)) + .await + } + }, }, Opt::ImaSign(ref opts) => ima_sign(opts), } diff --git a/lib/src/container/deploy.rs b/lib/src/container/deploy.rs new file mode 100644 index 00000000..9d638d06 --- /dev/null +++ b/lib/src/container/deploy.rs @@ -0,0 +1,53 @@ +//! Perform initial setup for a container image based system root + +use super::OstreeImageReference; +use crate::container::store::PrepareResult; +use anyhow::Result; +use ostree::glib; + +/// The key in the OSTree origin which holds a serialized [`super::OstreeImageReference`]. +pub const ORIGIN_CONTAINER: &str = "container"; + +async fn pull_idempotent(repo: &ostree::Repo, imgref: &OstreeImageReference) -> Result { + let mut imp = super::store::LayeredImageImporter::new(repo, imgref).await?; + match imp.prepare().await? { + PrepareResult::AlreadyPresent(r) => Ok(r), + PrepareResult::Ready(prep) => Ok(imp.import(prep).await?.commit), + } +} + +/// Options configuring deployment. +#[derive(Debug, Default)] +pub struct DeployOpts<'a> { + /// Kernel arguments to use. + pub kargs: Option<&'a [&'a str]>, +} + +/// Write a container image to an OSTree deployment. +/// +/// This API is currently intended for only an initial deployment. +pub async fn deploy<'opts>( + sysroot: &ostree::Sysroot, + stateroot: &str, + imgref: &OstreeImageReference, + options: Option>, +) -> Result<()> { + let cancellable = ostree::gio::NONE_CANCELLABLE; + let options = options.unwrap_or_default(); + let repo = &sysroot.repo().unwrap(); + let commit = &pull_idempotent(repo, imgref).await?; + let origin = glib::KeyFile::new(); + origin.set_string("ostree", ORIGIN_CONTAINER, &imgref.to_string()); + let deployment = &sysroot.deploy_tree( + Some(stateroot), + commit, + Some(&origin), + None, + options.kargs.unwrap_or_default(), + cancellable, + )?; + let flags = ostree::SysrootSimpleWriteDeploymentFlags::NONE; + sysroot.simple_write_deployment(Some(stateroot), deployment, None, flags, cancellable)?; + sysroot.cleanup(cancellable)?; + Ok(()) +} diff --git a/lib/src/container/mod.rs b/lib/src/container/mod.rs index 1628f405..2612a47f 100644 --- a/lib/src/container/mod.rs +++ b/lib/src/container/mod.rs @@ -223,6 +223,7 @@ impl std::fmt::Display for OstreeImageReference { } } +pub mod deploy; mod export; pub use export::*; mod import; @@ -230,6 +231,7 @@ pub use import::*; mod imageproxy; mod oci; mod skopeo; +pub mod store; #[cfg(test)] mod tests { diff --git a/lib/src/container/store.rs b/lib/src/container/store.rs new file mode 100644 index 00000000..1c9b282f --- /dev/null +++ b/lib/src/container/store.rs @@ -0,0 +1,383 @@ +//! APIs for storing (layered) container images as OSTree commits +//! +//! # Extension of import support +//! +//! This code supports ingesting arbitrary layered container images from an ostree-exported +//! base. See [`super::import`] for more information on encaspulation of images. + +use super::imageproxy::ImageProxy; +use super::oci::ManifestLayer; +use super::*; +use crate::refescape; +use anyhow::{anyhow, Context}; +use fn_error_context::context; +use ostree::prelude::{Cast, ToVariant}; +use ostree::{gio, glib}; +use std::collections::{BTreeMap, HashMap}; + +/// The ostree ref prefix for blobs. +const LAYER_PREFIX: &str = "ostree/container/blob"; +/// The ostree ref prefix for image references. +const IMAGE_PREFIX: &str = "ostree/container/image"; + +/// The key injected into the merge commit for the manifest digest. +const META_MANIFEST_DIGEST: &str = "ostree.manifest-digest"; +/// The key injected into the merge commit with the manifest serialized as JSON. +const META_MANIFEST: &str = "ostree.manifest"; + +/// Convert e.g. sha256:12345... into `/ostree/container/blob/sha256_2B12345...`. +fn ref_for_blob_digest(d: &str) -> Result { + refescape::prefix_escape_for_ref(LAYER_PREFIX, d) +} + +/// Convert e.g. sha256:12345... into `/ostree/container/blob/sha256_2B12345...`. +fn ref_for_layer(l: &oci::ManifestLayer) -> Result { + ref_for_blob_digest(l.digest.as_str()) +} + +/// Convert e.g. sha256:12345... into `/ostree/container/blob/sha256_2B12345...`. +fn ref_for_image(l: &ImageReference) -> Result { + refescape::prefix_escape_for_ref(IMAGE_PREFIX, &l.to_string()) +} + +/// Context for importing a container image. +pub struct LayeredImageImporter { + repo: ostree::Repo, + proxy: ImageProxy, + imgref: OstreeImageReference, + ostree_ref: String, +} + +/// Result of invoking [`LayeredImageImporter::prepare`]. +pub enum PrepareResult { + /// The image reference is already present; the contained string is the OSTree commit. + AlreadyPresent(String), + /// The image needs to be downloaded + Ready(PreparedImport), +} + +/// A container image layer with associated downloaded-or-not state. +#[derive(Debug)] +pub struct ManifestLayerState { + layer: oci::ManifestLayer, + /// The ostree ref name for this layer. + pub ostree_ref: String, + /// The ostree commit that caches this layer, if present. + pub commit: Option, +} + +impl ManifestLayerState { + /// The cryptographic checksum. + pub fn digest(&self) -> &str { + self.layer.digest.as_str() + } + + /// The (possibly compressed) size. + pub fn size(&self) -> u64 { + self.layer.size + } +} + +/// Information about which layers need to be downloaded. +#[derive(Debug)] +pub struct PreparedImport { + /// The manifest digest that was found + pub manifest_digest: String, + /// The previously stored manifest digest. + pub previous_manifest_digest: Option, + /// The previously stored image ID. + pub previous_imageid: Option, + /// The required base layer. + pub base_layer: ManifestLayerState, + /// Any further layers. + pub layers: Vec, + /// TODO: serialize this into the commit object + manifest: oci::Manifest, +} + +/// A successful import of a container image. +#[derive(Debug, PartialEq, Eq)] +pub struct CompletedImport { + /// The ostree ref used for the container image. + pub ostree_ref: String, + /// The current commit. + pub commit: String, + /// A mapping from layer blob IDs to a count of content filtered out + /// by toplevel path. + pub layer_filtered_content: BTreeMap>, +} + +// Given a manifest, compute its ostree ref name and cached ostree commit +fn query_layer(repo: &ostree::Repo, layer: ManifestLayer) -> Result { + let ostree_ref = ref_for_layer(&layer)?; + let commit = repo.resolve_rev(&ostree_ref, true)?.map(|s| s.to_string()); + Ok(ManifestLayerState { + layer, + ostree_ref, + commit, + }) +} + +fn manifest_from_commitmeta(commit_meta: &glib::VariantDict) -> Result { + let manifest_bytes: String = commit_meta + .lookup::(META_MANIFEST)? + .ok_or_else(|| anyhow!("Failed to find {} metadata key", META_MANIFEST))?; + let manifest: oci::Manifest = serde_json::from_str(&manifest_bytes)?; + Ok(manifest) +} + +fn manifest_from_commit(commit: &glib::Variant) -> Result { + let commit_meta = &commit.child_value(0); + let commit_meta = &ostree::glib::VariantDict::new(Some(commit_meta)); + manifest_from_commitmeta(commit_meta) +} + +impl LayeredImageImporter { + /// Create a new importer. + pub async fn new(repo: &ostree::Repo, imgref: &OstreeImageReference) -> Result { + let proxy = ImageProxy::new(&imgref.imgref).await?; + let repo = repo.clone(); + let ostree_ref = ref_for_image(&imgref.imgref)?; + Ok(LayeredImageImporter { + repo, + proxy, + ostree_ref, + imgref: imgref.clone(), + }) + } + + /// Determine if there is a new manifest, and if so return its digest. + #[context("Fetching manifest")] + pub async fn prepare(&mut self) -> Result { + match &self.imgref.sigverify { + SignatureSource::ContainerPolicy if skopeo::container_policy_is_default_insecure()? => { + return Err(anyhow!("containers-policy.json specifies a default of `insecureAcceptAnything`; refusing usage")); + } + SignatureSource::OstreeRemote(_) => { + return Err(anyhow!( + "Cannot currently verify layered containers via ostree remote" + )); + } + _ => {} + } + + let (manifest_digest, manifest_bytes) = self.proxy.fetch_manifest().await?; + let manifest: oci::Manifest = serde_json::from_slice(&manifest_bytes)?; + let new_imageid = manifest.imageid(); + + // Query for previous stored state + let (previous_manifest_digest, previous_imageid) = + if let Some(merge_commit) = self.repo.resolve_rev(&self.ostree_ref, true)? { + let (merge_commit_obj, _) = self.repo.load_commit(merge_commit.as_str())?; + let commit_meta = &merge_commit_obj.child_value(0); + let commit_meta = ostree::glib::VariantDict::new(Some(commit_meta)); + let previous_digest: String = + commit_meta.lookup(META_MANIFEST_DIGEST)?.ok_or_else(|| { + anyhow!("Missing {} metadata on merge commit", META_MANIFEST_DIGEST) + })?; + // If the manifest digests match, we're done. + if previous_digest == manifest_digest { + return Ok(PrepareResult::AlreadyPresent(merge_commit.to_string())); + } + // Failing that, if they have the same imageID, we're also done. + let previous_manifest = manifest_from_commitmeta(&commit_meta)?; + if previous_manifest.imageid() == new_imageid { + return Ok(PrepareResult::AlreadyPresent(merge_commit.to_string())); + } + ( + Some(previous_digest), + Some(previous_manifest.imageid().to_string()), + ) + } else { + (None, None) + }; + + let mut layers = manifest.layers.iter().cloned(); + // We require a base layer. + let base_layer = layers.next().ok_or_else(|| anyhow!("No layers found"))?; + let base_layer = query_layer(&self.repo, base_layer)?; + + let layers: Result> = layers + .map(|layer| -> Result<_> { query_layer(&self.repo, layer) }) + .collect(); + let layers = layers?; + + let imp = PreparedImport { + manifest, + manifest_digest, + previous_manifest_digest, + previous_imageid, + base_layer, + layers, + }; + Ok(PrepareResult::Ready(imp)) + } + + /// Import a layered container image + pub async fn import(mut self, import: PreparedImport) -> Result { + // First download the base image (if necessary) - we need the SELinux policy + // there to label all following layers. + let base_layer = import.base_layer; + let base_commit = if let Some(c) = base_layer.commit { + c + } else { + let blob = self.proxy.fetch_layer_decompress(&base_layer.layer).await?; + let commit = crate::tar::import_tar(&self.repo, blob, None) + .await + .with_context(|| format!("Parsing blob {}", &base_layer.digest()))?; + // TODO support ref writing in tar import + self.repo.set_ref_immediate( + None, + base_layer.ostree_ref.as_str(), + Some(commit.as_str()), + gio::NONE_CANCELLABLE, + )?; + commit + }; + + let mut layer_commits = Vec::new(); + let mut layer_filtered_content = BTreeMap::new(); + for layer in import.layers { + if let Some(c) = layer.commit { + layer_commits.push(c.to_string()); + } else { + let blob = self.proxy.fetch_layer_decompress(&layer.layer).await?; + // An important aspect of this is that we SELinux label the derived layers using + // the base policy. + let opts = crate::tar::WriteTarOptions { + base: Some(base_commit.clone()), + selinux: true, + }; + let r = + crate::tar::write_tar(&self.repo, blob, layer.ostree_ref.as_str(), Some(opts)) + .await + .with_context(|| format!("Parsing layer blob {}", layer.digest()))?; + layer_commits.push(r.commit); + if !r.filtered.is_empty() { + layer_filtered_content.insert(layer.digest().to_string(), r.filtered); + } + } + } + + // We're done with the proxy, make sure it didn't have any errors. + self.proxy.finalize().await?; + + let serialized_manifest = serde_json::to_string(&import.manifest)?; + let mut metadata = HashMap::new(); + metadata.insert(META_MANIFEST_DIGEST, import.manifest_digest.to_variant()); + metadata.insert(META_MANIFEST, serialized_manifest.to_variant()); + metadata.insert( + "ostree.importer.version", + env!("CARGO_PKG_VERSION").to_variant(), + ); + let metadata = metadata.to_variant(); + + // Destructure to transfer ownership to thread + let repo = self.repo; + let target_ref = self.ostree_ref; + let (ostree_ref, commit) = crate::tokio_util::spawn_blocking_cancellable( + move |cancellable| -> Result<(String, String)> { + let cancellable = Some(cancellable); + let repo = &repo; + let txn = repo.auto_transaction(cancellable)?; + let (base_commit_tree, _) = repo.read_commit(&base_commit, cancellable)?; + let base_commit_tree = base_commit_tree.downcast::().unwrap(); + let base_contents_obj = base_commit_tree.tree_get_contents_checksum().unwrap(); + let base_metadata_obj = base_commit_tree.tree_get_metadata_checksum().unwrap(); + let mt = ostree::MutableTree::from_checksum( + &repo, + &base_contents_obj, + &base_metadata_obj, + ); + // Layer all subsequent commits + for commit in layer_commits { + let (layer_tree, _) = repo.read_commit(&commit, cancellable)?; + repo.write_directory_to_mtree(&layer_tree, &mt, None, cancellable)?; + } + + let merged_root = repo.write_mtree(&mt, cancellable)?; + let merged_root = merged_root.downcast::().unwrap(); + let merged_commit = repo.write_commit( + None, + None, + None, + Some(&metadata), + &merged_root, + cancellable, + )?; + repo.transaction_set_ref(None, &target_ref, Some(merged_commit.as_str())); + txn.commit(cancellable)?; + Ok((target_ref, merged_commit.to_string())) + }, + ) + .await??; + Ok(CompletedImport { + ostree_ref, + commit, + layer_filtered_content, + }) + } +} + +/// List all images stored +pub fn list_images(repo: &ostree::Repo) -> Result> { + let cancellable = gio::NONE_CANCELLABLE; + let refs = repo.list_refs_ext( + Some(IMAGE_PREFIX), + ostree::RepoListRefsExtFlags::empty(), + cancellable, + )?; + let r: Result> = refs + .keys() + .map(|imgname| refescape::unprefix_unescape_ref(IMAGE_PREFIX, imgname)) + .collect(); + Ok(r?) +} + +/// Copy a downloaded image from one repository to another. +pub async fn copy( + src_repo: &ostree::Repo, + dest_repo: &ostree::Repo, + imgref: &OstreeImageReference, +) -> Result<()> { + let ostree_ref = ref_for_image(&imgref.imgref)?; + let rev = src_repo.resolve_rev(&ostree_ref, false)?.unwrap(); + let (commit_obj, _) = src_repo.load_commit(rev.as_str())?; + let manifest: oci::Manifest = manifest_from_commit(&commit_obj)?; + // Create a task to copy each layer, plus the final ref + let layer_refs = manifest + .layers + .iter() + .map(|layer| ref_for_layer(&layer)) + .chain(std::iter::once(Ok(ostree_ref))); + for ostree_ref in layer_refs { + let ostree_ref = ostree_ref?; + let src_repo = src_repo.clone(); + let dest_repo = dest_repo.clone(); + crate::tokio_util::spawn_blocking_cancellable(move |cancellable| -> Result<_> { + let cancellable = Some(cancellable); + let srcfd = &format!("file:///proc/self/fd/{}", src_repo.dfd()); + let flags = ostree::RepoPullFlags::MIRROR; + let opts = glib::VariantDict::new(None); + let refs = [ostree_ref.as_str()]; + // Some older archives may have bindings, we don't need to verify them. + opts.insert("disable-verify-bindings", &true); + opts.insert("refs", &&refs[..]); + opts.insert("flags", &(flags.bits() as i32)); + let options = opts.to_variant(); + dest_repo.pull_with_options(&srcfd, &options, None, cancellable)?; + Ok(()) + }) + .await??; + } + Ok(()) +} + +/// Remove the specified images and their corresponding blobs. +pub fn prune_images(_repo: &ostree::Repo, _imgs: &[&str]) -> Result<()> { + // Most robust approach is to iterate over all known images, load the + // manifest and build the set of reachable blobs, then compute the set + // Set(unreachable) = Set(all) - Set(reachable) + // And remove the unreachable ones. + unimplemented!() +} diff --git a/lib/src/tar/write.rs b/lib/src/tar/write.rs index b4ae95bd..f156e06c 100644 --- a/lib/src/tar/write.rs +++ b/lib/src/tar/write.rs @@ -208,6 +208,10 @@ pub async fn write_tar( c.arg("--selinux-policy"); c.arg(sepolicy.path()); } + c.arg(&format!( + "--add-metadata-string=ostree.importer.version={}", + env!("CARGO_PKG_VERSION") + )); c.args(&[ "--no-bindings", "--tar-autocreate-parents", diff --git a/lib/tests/it/fixtures/exampleos-derive-v2.ociarchive b/lib/tests/it/fixtures/exampleos-derive-v2.ociarchive new file mode 100644 index 00000000..42b91b11 Binary files /dev/null and b/lib/tests/it/fixtures/exampleos-derive-v2.ociarchive differ diff --git a/lib/tests/it/main.rs b/lib/tests/it/main.rs index 4f9c8b9d..da773892 100644 --- a/lib/tests/it/main.rs +++ b/lib/tests/it/main.rs @@ -2,6 +2,7 @@ use anyhow::{Context, Result}; use camino::{Utf8Path, Utf8PathBuf}; use fn_error_context::context; use indoc::indoc; +use ostree_ext::container::store::PrepareResult; use ostree_ext::container::{ Config, ImageReference, OstreeImageReference, SignatureSource, Transport, }; @@ -24,6 +25,7 @@ const TEST_REGISTRY_DEFAULT: &str = "localhost:5000"; /// Image that contains a base exported layer, then a `podman build` of an added file on top. const EXAMPLEOS_DERIVED_OCI: &[u8] = include_bytes!("fixtures/exampleos-derive.ociarchive"); +const EXAMPLEOS_DERIVED_V2_OCI: &[u8] = include_bytes!("fixtures/exampleos-derive-v2.ociarchive"); fn assert_err_contains(r: Result, s: impl AsRef) { let s = s.as_ref(); @@ -58,7 +60,7 @@ fn generate_test_repo(dir: &Utf8Path) -> Result { indoc! {" cd {dir} ostree --repo=repo init --mode=archive - ostree --repo=repo commit -b {testref} --bootable --add-metadata-string=version=42.0 --gpg-homedir={gpghome} --gpg-sign={keyid} \ + ostree --repo=repo commit -b {testref} --bootable --no-bindings --add-metadata-string=version=42.0 --gpg-homedir={gpghome} --gpg-sign={keyid} \ --add-detached-metadata-string=my-detached-key=my-detached-value --tree=tar=exampleos.tar.zst ostree --repo=repo show {testref} "}, @@ -79,7 +81,7 @@ fn update_repo(repopath: &Utf8Path) -> Result<()> { let repopath = repopath.as_str(); let testref = TESTREF; bash!( - "ostree --repo={repopath} commit -b {testref} --tree=tar={srcpath}", + "ostree --repo={repopath} commit -b {testref} --no-bindings --tree=tar={srcpath}", testref, repopath, srcpath @@ -386,7 +388,7 @@ async fn test_container_import_export() -> Result<()> { Ok(()) } -/// We should currently reject an image with multiple layers. +/// We should reject an image with multiple layers when doing an "import" - i.e. a direct un-encapsulation. #[tokio::test] async fn test_container_import_derive() -> Result<()> { let fixture = Fixture::new()?; @@ -404,6 +406,125 @@ async fn test_container_import_derive() -> Result<()> { Ok(()) } +/// But layers work via the container::write module. +#[tokio::test] +async fn test_container_write_derive() -> Result<()> { + let fixture = Fixture::new()?; + let exampleos_path = &fixture.path.join("exampleos-derive.ociarchive"); + std::fs::write(exampleos_path, EXAMPLEOS_DERIVED_OCI)?; + let exampleos_ref = OstreeImageReference { + sigverify: SignatureSource::ContainerPolicyAllowInsecure, + imgref: ImageReference { + transport: Transport::OciArchive, + name: exampleos_path.to_string(), + }, + }; + + // There shouldn't be any container images stored yet. + let images = ostree_ext::container::store::list_images(&fixture.destrepo)?; + assert!(images.is_empty()); + + // Pull a derived image - two layers, new base plus one layer. + let mut imp = + ostree_ext::container::store::LayeredImageImporter::new(&fixture.destrepo, &exampleos_ref) + .await?; + let prep = match imp.prepare().await? { + PrepareResult::AlreadyPresent(_) => panic!("should not be already imported"), + PrepareResult::Ready(r) => r, + }; + assert!(prep.base_layer.commit.is_none()); + for layer in prep.layers.iter() { + assert!(layer.commit.is_none()); + } + let import = imp.import(prep).await?; + // We should have exactly one image stored. + let images = ostree_ext::container::store::list_images(&fixture.destrepo)?; + assert_eq!(images.len(), 1); + assert_eq!(images[0], exampleos_ref.imgref.to_string()); + + // Parse the commit and verify we pulled the derived content. + bash!( + "ostree --repo={repo} ls {r} /usr/share/anewfile", + repo = fixture.destrepo_path.as_str(), + r = import.ostree_ref.as_str() + )?; + + // Import again, but there should be no changes. + let mut imp = + ostree_ext::container::store::LayeredImageImporter::new(&fixture.destrepo, &exampleos_ref) + .await?; + let already_present = match imp.prepare().await? { + PrepareResult::AlreadyPresent(c) => c, + PrepareResult::Ready(_) => { + panic!("Should have already imported {}", import.ostree_ref) + } + }; + assert_eq!(import.commit, already_present); + + // Test upgrades; replace the oci-archive with new content. + std::fs::write(exampleos_path, EXAMPLEOS_DERIVED_V2_OCI)?; + let mut imp = + ostree_ext::container::store::LayeredImageImporter::new(&fixture.destrepo, &exampleos_ref) + .await?; + let prep = match imp.prepare().await? { + PrepareResult::AlreadyPresent(_) => panic!("should not be already imported"), + PrepareResult::Ready(r) => r, + }; + // We *should* already have the base layer. + assert!(prep.base_layer.commit.is_some()); + // One new layer + assert_eq!(prep.layers.len(), 1); + for layer in prep.layers.iter() { + assert!(layer.commit.is_none()); + } + let import = imp.import(prep).await?; + // New commit. + assert_ne!(import.commit, already_present); + // We should still have exactly one image stored. + let images = ostree_ext::container::store::list_images(&fixture.destrepo)?; + assert_eq!(images.len(), 1); + assert_eq!(images[0], exampleos_ref.imgref.to_string()); + + // Verify we have the new file and *not* the old one + bash!( + "ostree --repo={repo} ls {r} /usr/share/anewfile2 >/dev/null + if ostree --repo={repo} ls {r} /usr/share/anewfile 2>/dev/null; then + echo oops; exit 1 + fi + ", + repo = fixture.destrepo_path.as_str(), + r = import.ostree_ref.as_str() + )?; + + // And there should be no changes on upgrade again. + let mut imp = + ostree_ext::container::store::LayeredImageImporter::new(&fixture.destrepo, &exampleos_ref) + .await?; + let already_present = match imp.prepare().await? { + PrepareResult::AlreadyPresent(c) => c, + PrepareResult::Ready(_) => { + panic!("Should have already imported {}", import.ostree_ref) + } + }; + assert_eq!(import.commit, already_present); + + // Create a new repo, and copy to it + let destrepo2 = ostree::Repo::create_at( + ostree::AT_FDCWD, + fixture.path.join("destrepo2").as_str(), + ostree::RepoMode::Archive, + None, + gio::NONE_CANCELLABLE, + )?; + ostree_ext::container::store::copy(&fixture.destrepo, &destrepo2, &exampleos_ref).await?; + + let images = ostree_ext::container::store::list_images(&destrepo2)?; + assert_eq!(images.len(), 1); + assert_eq!(images[0], exampleos_ref.imgref.to_string()); + + Ok(()) +} + #[ignore] #[tokio::test] // Verify that we can push and pull to a registry, not just oci-archive:.