From 9f1781c21170e6abbb0da039b8e4bcf167637cc1 Mon Sep 17 00:00:00 2001 From: Allison Karlitskaya Date: Wed, 16 Apr 2025 16:05:38 +0200 Subject: [PATCH 1/9] src: use `#[cfg(test)] mod test { }` consistently There are a few cases left over from before I learned how to do this properly. Fix them. Signed-off-by: Allison Karlitskaya --- src/bin/composefs-setup-root.rs | 31 ++++--- src/oci/image.rs | 144 ++++++++++++++++---------------- 2 files changed, 90 insertions(+), 85 deletions(-) diff --git a/src/bin/composefs-setup-root.rs b/src/bin/composefs-setup-root.rs index e356dabc..aed88059 100644 --- a/src/bin/composefs-setup-root.rs +++ b/src/bin/composefs-setup-root.rs @@ -284,18 +284,23 @@ fn main() -> Result<()> { setup_root(args) } -#[test] -fn test_parse() { - let failing = ["", "foo", "composefs", "composefs=foo"]; - for case in failing { - assert!(parse_composefs_cmdline(case.as_bytes()).is_err()); +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_parse() { + let failing = ["", "foo", "composefs", "composefs=foo"]; + for case in failing { + assert!(parse_composefs_cmdline(case.as_bytes()).is_err()); + } + let digest = "8b7df143d91c716ecfa5fc1730022f6b421b05cedee8fd52b1fc65a96030ad52"; + let digest_bytes = hex::decode(digest).unwrap(); + similar_asserts::assert_eq!( + parse_composefs_cmdline(format!("composefs={digest}").as_bytes()) + .unwrap() + .as_slice(), + &digest_bytes + ); } - let digest = "8b7df143d91c716ecfa5fc1730022f6b421b05cedee8fd52b1fc65a96030ad52"; - let digest_bytes = hex::decode(digest).unwrap(); - similar_asserts::assert_eq!( - parse_composefs_cmdline(format!("composefs={digest}").as_bytes()) - .unwrap() - .as_slice(), - &digest_bytes - ); } diff --git a/src/oci/image.rs b/src/oci/image.rs index bd5baecd..4401b681 100644 --- a/src/oci/image.rs +++ b/src/oci/image.rs @@ -98,82 +98,82 @@ pub fn create_image( } #[cfg(test)] -use crate::image::{LeafContent, RegularFile, Stat}; -#[cfg(test)] -use std::{cell::RefCell, collections::BTreeMap, io::BufRead, path::PathBuf}; - -#[cfg(test)] -fn file_entry(path: &str) -> oci::tar::TarEntry { - oci::tar::TarEntry { - path: PathBuf::from(path), - stat: Stat { - st_mode: 0o644, - st_uid: 0, - st_gid: 0, - st_mtim_sec: 0, - xattrs: RefCell::new(BTreeMap::new()), - }, - item: oci::tar::TarItem::Leaf(LeafContent::Regular(RegularFile::Inline([].into()))), +mod test { + use crate::image::{LeafContent, RegularFile, Stat}; + use std::{cell::RefCell, collections::BTreeMap, io::BufRead, path::PathBuf}; + + use super::*; + + fn file_entry(path: &str) -> oci::tar::TarEntry { + oci::tar::TarEntry { + path: PathBuf::from(path), + stat: Stat { + st_mode: 0o644, + st_uid: 0, + st_gid: 0, + st_mtim_sec: 0, + xattrs: RefCell::new(BTreeMap::new()), + }, + item: oci::tar::TarItem::Leaf(LeafContent::Regular(RegularFile::Inline([].into()))), + } } -} -#[cfg(test)] -fn dir_entry(path: &str) -> oci::tar::TarEntry { - oci::tar::TarEntry { - path: PathBuf::from(path), - stat: Stat { - st_mode: 0o755, - st_uid: 0, - st_gid: 0, - st_mtim_sec: 0, - xattrs: RefCell::new(BTreeMap::new()), - }, - item: oci::tar::TarItem::Directory, + fn dir_entry(path: &str) -> oci::tar::TarEntry { + oci::tar::TarEntry { + path: PathBuf::from(path), + stat: Stat { + st_mode: 0o755, + st_uid: 0, + st_gid: 0, + st_mtim_sec: 0, + xattrs: RefCell::new(BTreeMap::new()), + }, + item: oci::tar::TarItem::Directory, + } } -} -#[cfg(test)] -fn assert_files(fs: &FileSystem, expected: &[&str]) -> Result<()> { - let mut out = vec![]; - write_dumpfile(&mut out, fs)?; - let actual: Vec = out - .lines() - .map(|line| line.unwrap().split_once(' ').unwrap().0.into()) - .collect(); - - similar_asserts::assert_eq!(actual, expected); - Ok(()) -} + fn assert_files(fs: &FileSystem, expected: &[&str]) -> Result<()> { + let mut out = vec![]; + write_dumpfile(&mut out, fs)?; + let actual: Vec = out + .lines() + .map(|line| line.unwrap().split_once(' ').unwrap().0.into()) + .collect(); -#[test] -fn test_process_entry() -> Result<()> { - let mut fs = FileSystem::new(); - - // both with and without leading slash should be supported - process_entry(&mut fs, dir_entry("/a"))?; - process_entry(&mut fs, dir_entry("b"))?; - process_entry(&mut fs, dir_entry("c"))?; - assert_files(&fs, &["/", "/a", "/b", "/c"])?; - - // add some files - process_entry(&mut fs, file_entry("/a/b"))?; - process_entry(&mut fs, file_entry("/a/c"))?; - process_entry(&mut fs, file_entry("/b/a"))?; - process_entry(&mut fs, file_entry("/b/c"))?; - process_entry(&mut fs, file_entry("/c/a"))?; - process_entry(&mut fs, file_entry("/c/c"))?; - assert_files( - &fs, - &[ - "/", "/a", "/a/b", "/a/c", "/b", "/b/a", "/b/c", "/c", "/c/a", "/c/c", - ], - )?; - - // try some whiteouts - process_entry(&mut fs, file_entry(".wh.a"))?; // entire dir - process_entry(&mut fs, file_entry("/b/.wh..wh.opq"))?; // opaque dir - process_entry(&mut fs, file_entry("/c/.wh.c"))?; // single file - assert_files(&fs, &["/", "/b", "/c", "/c/a"])?; + similar_asserts::assert_eq!(actual, expected); + Ok(()) + } - Ok(()) + #[test] + fn test_process_entry() -> Result<()> { + let mut fs = FileSystem::new(); + + // both with and without leading slash should be supported + process_entry(&mut fs, dir_entry("/a"))?; + process_entry(&mut fs, dir_entry("b"))?; + process_entry(&mut fs, dir_entry("c"))?; + assert_files(&fs, &["/", "/a", "/b", "/c"])?; + + // add some files + process_entry(&mut fs, file_entry("/a/b"))?; + process_entry(&mut fs, file_entry("/a/c"))?; + process_entry(&mut fs, file_entry("/b/a"))?; + process_entry(&mut fs, file_entry("/b/c"))?; + process_entry(&mut fs, file_entry("/c/a"))?; + process_entry(&mut fs, file_entry("/c/c"))?; + assert_files( + &fs, + &[ + "/", "/a", "/a/b", "/a/c", "/b", "/b/a", "/b/c", "/c", "/c/a", "/c/c", + ], + )?; + + // try some whiteouts + process_entry(&mut fs, file_entry(".wh.a"))?; // entire dir + process_entry(&mut fs, file_entry("/b/.wh..wh.opq"))?; // opaque dir + process_entry(&mut fs, file_entry("/c/.wh.c"))?; // single file + assert_files(&fs, &["/", "/b", "/c", "/c/a"])?; + + Ok(()) + } } From b2d8b400e94e0c8422a0b4459de0fd0723b94b86 Mon Sep 17 00:00:00 2001 From: Allison Karlitskaya Date: Wed, 16 Apr 2025 10:22:30 +0200 Subject: [PATCH 2/9] src: split up different uses of SHA-256 src/fsverity.rs defines Sha256HashValue as an implementation of FsVerityHashValue but we also use it for situations where we handle SHA-256 digests that aren't fs-verity hash values. This happens a lot in the OCI code (to refer to images and layers) and correspondingly in the splitstream code (which stores those things). Introduce a util::Sha256Digest type, returned by util::parse_sha256() and use it consistently for all of the cases that have nothing to do with fs-verity. parse_sha256() is still used in some places in the code for values that are intended as fs-verity digests. We'll start fixing that up in the following commits when we add some helpers directly on the types themselves. Signed-off-by: Allison Karlitskaya --- src/oci/mod.rs | 14 +++++++------- src/repository.rs | 12 ++++++------ src/splitstream.rs | 18 +++++++++--------- src/util.rs | 11 ++++++----- 4 files changed, 28 insertions(+), 27 deletions(-) diff --git a/src/oci/mod.rs b/src/oci/mod.rs index 087a067f..4abe546c 100644 --- a/src/oci/mod.rs +++ b/src/oci/mod.rs @@ -19,12 +19,12 @@ use crate::{ oci::tar::{get_entry, split_async}, repository::Repository, splitstream::DigestMap, - util::parse_sha256, + util::{parse_sha256, Sha256Digest}, }; pub fn import_layer( repo: &Repository, - sha256: &Sha256HashValue, + sha256: &Sha256Digest, name: Option<&str>, tar_stream: &mut impl Read, ) -> Result { @@ -48,21 +48,21 @@ struct ImageOp<'repo> { progress: MultiProgress, } -fn sha256_from_descriptor(descriptor: &Descriptor) -> Result { +fn sha256_from_descriptor(descriptor: &Descriptor) -> Result { let Some(digest) = descriptor.as_digest_sha256() else { bail!("Descriptor in oci config is not sha256"); }; Ok(parse_sha256(digest)?) } -fn sha256_from_digest(digest: &str) -> Result { +fn sha256_from_digest(digest: &str) -> Result { match digest.strip_prefix("sha256:") { Some(rest) => Ok(parse_sha256(rest)?), None => bail!("Manifest has non-sha256 digest"), } } -type ContentAndVerity = (Sha256HashValue, Sha256HashValue); +type ContentAndVerity = (Sha256Digest, Sha256HashValue); impl<'repo> ImageOp<'repo> { async fn new(repo: &'repo Repository, imgref: &str) -> Result { @@ -93,7 +93,7 @@ impl<'repo> ImageOp<'repo> { pub async fn ensure_layer( &self, - layer_sha256: &Sha256HashValue, + layer_sha256: &Sha256Digest, descriptor: &Descriptor, ) -> Result { // We need to use the per_manifest descriptor to download the compressed layer but it gets @@ -245,7 +245,7 @@ pub fn open_config( Ok((config, stream.refs)) } -fn hash(bytes: &[u8]) -> Sha256HashValue { +fn hash(bytes: &[u8]) -> Sha256Digest { let mut context = Sha256::new(); context.update(bytes); context.finalize().into() diff --git a/src/repository.rs b/src/repository.rs index ac24f8fe..9f2031a2 100644 --- a/src/repository.rs +++ b/src/repository.rs @@ -24,7 +24,7 @@ use crate::{ }, mount::mount_composefs_at, splitstream::{DigestMap, SplitStreamReader, SplitStreamWriter}, - util::{parse_sha256, proc_self_fd}, + util::{parse_sha256, proc_self_fd, Sha256Digest}, }; #[derive(Debug)] @@ -136,7 +136,7 @@ impl Repository { /// store the result. pub fn create_stream( &self, - sha256: Option, + sha256: Option, maps: Option, ) -> SplitStreamWriter { SplitStreamWriter::new(self, maps, sha256) @@ -166,7 +166,7 @@ impl Repository { format!("objects/{:02x}/{}", id[0], hex::encode(&id[1..])) } - pub fn has_stream(&self, sha256: &Sha256HashValue) -> Result> { + pub fn has_stream(&self, sha256: &Sha256Digest) -> Result> { let stream_path = format!("streams/{}", hex::encode(sha256)); match readlinkat(&self.repository, &stream_path, []) { @@ -191,7 +191,7 @@ impl Repository { } /// Basically the same as has_stream() except that it performs expensive verification - pub fn check_stream(&self, sha256: &Sha256HashValue) -> Result> { + pub fn check_stream(&self, sha256: &Sha256Digest) -> Result> { match self.openat(&format!("streams/{}", hex::encode(sha256)), OFlags::RDONLY) { Ok(stream) => { let measured_verity: Sha256HashValue = measure_verity(&stream)?; @@ -245,7 +245,7 @@ impl Repository { /// Assign the given name to a stream. The stream must already exist. After this operation it /// will be possible to refer to the stream by its new name 'refs/{name}'. - pub fn name_stream(&self, sha256: Sha256HashValue, name: &str) -> Result<()> { + pub fn name_stream(&self, sha256: Sha256Digest, name: &str) -> Result<()> { let stream_path = format!("streams/{}", hex::encode(sha256)); let reference_path = format!("streams/refs/{name}"); self.symlink(&reference_path, &stream_path)?; @@ -268,7 +268,7 @@ impl Repository { /// ID will be used when referring to the stream from other linked streams. pub fn ensure_stream( &self, - sha256: &Sha256HashValue, + sha256: &Sha256Digest, callback: impl FnOnce(&mut SplitStreamWriter) -> Result<()>, reference: Option<&str>, ) -> Result { diff --git a/src/splitstream.rs b/src/splitstream.rs index 319ffe09..f6bc8770 100644 --- a/src/splitstream.rs +++ b/src/splitstream.rs @@ -12,12 +12,12 @@ use zstd::stream::{read::Decoder, write::Encoder}; use crate::{ fsverity::{FsVerityHashValue, Sha256HashValue}, repository::Repository, - util::read_exactish, + util::{read_exactish, Sha256Digest}, }; #[derive(Debug)] pub struct DigestMapEntry { - pub body: Sha256HashValue, + pub body: Sha256Digest, pub verity: Sha256HashValue, } @@ -37,14 +37,14 @@ impl DigestMap { DigestMap { map: vec![] } } - pub fn lookup(&self, body: &Sha256HashValue) -> Option<&Sha256HashValue> { + pub fn lookup(&self, body: &Sha256Digest) -> Option<&Sha256HashValue> { match self.map.binary_search_by_key(body, |e| e.body) { Ok(idx) => Some(&self.map[idx].verity), Err(..) => None, } } - pub fn insert(&mut self, body: &Sha256HashValue, verity: &Sha256HashValue) { + pub fn insert(&mut self, body: &Sha256Digest, verity: &Sha256HashValue) { match self.map.binary_search_by_key(body, |e| e.body) { Ok(idx) => assert_eq!(self.map[idx].verity, *verity), // or else, bad things... Err(idx) => self.map.insert( @@ -62,7 +62,7 @@ pub struct SplitStreamWriter<'a> { repo: &'a Repository, inline_content: Vec, writer: Encoder<'a, Vec>, - pub sha256: Option<(Sha256, Sha256HashValue)>, + pub sha256: Option<(Sha256, Sha256Digest)>, } impl std::fmt::Debug for SplitStreamWriter<'_> { @@ -80,7 +80,7 @@ impl SplitStreamWriter<'_> { pub fn new( repo: &Repository, refs: Option, - sha256: Option, + sha256: Option, ) -> SplitStreamWriter { // SAFETY: we surely can't get an error writing the header to a Vec let mut writer = Encoder::new(vec![], 0).unwrap(); @@ -157,7 +157,7 @@ impl SplitStreamWriter<'_> { self.flush_inline(vec![])?; if let Some((context, expected)) = self.sha256 { - if Into::::into(context.finalize()) != expected { + if Into::::into(context.finalize()) != expected { bail!("Content doesn't have expected SHA256 hash value!"); } } @@ -362,13 +362,13 @@ impl SplitStreamReader { } } - pub fn get_stream_refs(&mut self, mut callback: impl FnMut(&Sha256HashValue)) { + pub fn get_stream_refs(&mut self, mut callback: impl FnMut(&Sha256Digest)) { for entry in &self.refs.map { callback(&entry.body); } } - pub fn lookup(&self, body: &Sha256HashValue) -> Result<&Sha256HashValue> { + pub fn lookup(&self, body: &Sha256Digest) -> Result<&Sha256HashValue> { match self.refs.lookup(body) { Some(id) => Ok(id), None => bail!("Reference is not found in splitstream"), diff --git a/src/util.rs b/src/util.rs index bafa58e9..1e3006eb 100644 --- a/src/util.rs +++ b/src/util.rs @@ -5,8 +5,6 @@ use std::{ use tokio::io::{AsyncRead, AsyncReadExt}; -use crate::fsverity::{FsVerityHashValue, Sha256HashValue}; - /// Formats a string like "/proc/self/fd/3" for the given fd. This can be used to work with kernel /// APIs that don't directly accept file descriptors. /// @@ -83,14 +81,17 @@ pub(crate) async fn read_exactish_async( Ok(true) } -/// Parse a string containing a SHA256 digest in hexidecimal form into a Sha256HashValue. +/// A utility type representing a SHA-256 digest in binary. +pub type Sha256Digest = [u8; 32]; + +/// Parse a string containing a SHA256 digest in hexidecimal form into a Sha256Digest. /// /// The string must contain exactly 64 characters and consist entirely of [0-9a-f], case /// insensitive. /// /// In case of a failure to parse the string, this function returns ErrorKind::InvalidInput. -pub fn parse_sha256(string: impl AsRef) -> Result { - let mut value = Sha256HashValue::EMPTY; +pub fn parse_sha256(string: impl AsRef) -> Result { + let mut value = [0u8; 32]; hex::decode_to_slice(string.as_ref(), &mut value) .map_err(|source| Error::new(ErrorKind::InvalidInput, source))?; Ok(value) From ca79210286e6dee8144348366652578e660904ae Mon Sep 17 00:00:00 2001 From: Allison Karlitskaya Date: Wed, 16 Apr 2025 16:12:28 +0200 Subject: [PATCH 3/9] src/fsverity: split out FsVerityHashValue and impls Split FsVerityHashValue and its two implementations out to a separate file. Right now it's not a lot of code, but it's going to get substantially larger in the next commit. Signed-off-by: Allison Karlitskaya --- src/fsverity/hashvalue.rs | 27 +++++++++++++++++++++++++++ src/fsverity/mod.rs | 28 ++-------------------------- 2 files changed, 29 insertions(+), 26 deletions(-) create mode 100644 src/fsverity/hashvalue.rs diff --git a/src/fsverity/hashvalue.rs b/src/fsverity/hashvalue.rs new file mode 100644 index 00000000..3d23367d --- /dev/null +++ b/src/fsverity/hashvalue.rs @@ -0,0 +1,27 @@ +use sha2::{digest::FixedOutputReset, digest::Output, Digest, Sha256, Sha512}; + +pub trait FsVerityHashValue +where + Self: Eq + AsRef<[u8]> + Clone, + Self: From>, +{ + type Digest: Digest + FixedOutputReset + std::fmt::Debug; + const ALGORITHM: u8; + const EMPTY: Self; +} + +pub type Sha256HashValue = [u8; 32]; + +impl FsVerityHashValue for Sha256HashValue { + type Digest = Sha256; + const ALGORITHM: u8 = 1; + const EMPTY: Self = [0; 32]; +} + +pub type Sha512HashValue = [u8; 64]; + +impl FsVerityHashValue for Sha512HashValue { + type Digest = Sha512; + const ALGORITHM: u8 = 2; + const EMPTY: Self = [0; 64]; +} diff --git a/src/fsverity/mod.rs b/src/fsverity/mod.rs index f8eb4a64..da2e24e1 100644 --- a/src/fsverity/mod.rs +++ b/src/fsverity/mod.rs @@ -1,36 +1,12 @@ mod digest; +mod hashvalue; mod ioctl; use std::{io::Error, os::fd::AsFd}; -use sha2::{digest::FixedOutputReset, digest::Output, Digest, Sha256, Sha512}; use thiserror::Error; -pub trait FsVerityHashValue -where - Self: Eq + AsRef<[u8]> + Clone, - Self: From>, -{ - type Digest: Digest + FixedOutputReset + std::fmt::Debug; - const ALGORITHM: u8; - const EMPTY: Self; -} - -pub type Sha256HashValue = [u8; 32]; - -impl FsVerityHashValue for Sha256HashValue { - type Digest = Sha256; - const ALGORITHM: u8 = 1; - const EMPTY: Self = [0; 32]; -} - -pub type Sha512HashValue = [u8; 64]; - -impl FsVerityHashValue for Sha512HashValue { - type Digest = Sha512; - const ALGORITHM: u8 = 2; - const EMPTY: Self = [0; 64]; -} +pub use hashvalue::{FsVerityHashValue, Sha256HashValue, Sha512HashValue}; /// Measuring fsverity failed. #[derive(Error, Debug)] // can't derive PartialEq because of std::io::Error From 766ee7093c6d103253a6c8edfd51db0be183f5a1 Mon Sep 17 00:00:00 2001 From: Allison Karlitskaya Date: Wed, 16 Apr 2025 11:27:46 +0200 Subject: [PATCH 4/9] src/fsverity: up our FsVerityHashValue trait game Right now we define pub type Sha256HashValue = [u8; 32]; pub type Sha512HashValue = [u8; 64]; which is simple enough but also painful: we can implement our own traits for these types (and indeed, we do it for FsVerityHashValue) but we can't implement things like fmt::Debug because there's a default impl (which rather unhelpfully prints the hashes out as an array of bytes). Turn these into proper types using a trivial struct wrapper: pub struct Sha256HashValue([u8; 32]); pub struct Sha512HashValue([u8; 64]); which gives us a lot more control. We can start by implementing meaningful fmt::Debug traits. Doing this means that we implicitly lose our implementation of AsRef<[u8]> and AsMut<[u8]> which we've been mostly using in order to do various ad-hoc encoding to/from various forms of hex values all over the codebase. Move all of those ad-hoc forms of encoding into proper methods on the trait itself by adding functions like ::to_hex() and ::from_hex() which we can use instead. We also drop our last instances of using parse_sha256() for fs-verity digests. There are a couple of other places where we need the actual bytes, though: when reading/writing hash values to splitstreams and when reading/writing the metacopy xattr in erofs images. Deal with those by deriving the usual zerocopy FromBytes/IntoBytes traits for our new hash types and moving the code in the splitstream and erofs modules to directly include the data inside of the relevant structure types, mark those types as FromBytes/IntoBytes, and use the zerocopy APIs directly. In the case of the metacopy attr in the erofs code this provides an opportunity for a substantial cleanup in an otherwise scary part of the code. Signed-off-by: Allison Karlitskaya --- Cargo.toml | 2 +- src/bin/cfsctl.rs | 26 +++++--- src/bin/composefs-setup-root.rs | 15 ++--- src/dumpfile.rs | 6 +- src/erofs/composefs.rs | 33 ++++++++++ src/erofs/mod.rs | 1 + src/erofs/reader.rs | 20 +++--- src/erofs/writer.rs | 11 ++-- src/fsverity/digest.rs | 12 ++-- src/fsverity/hashvalue.rs | 112 ++++++++++++++++++++++++++++++-- src/fsverity/mod.rs | 28 ++++---- src/oci/mod.rs | 14 ++-- src/oci/tar.rs | 6 +- src/repository.rs | 91 +++++++++----------------- src/splitstream.rs | 25 +++---- tests/mkfs.rs | 7 +- 16 files changed, 261 insertions(+), 148 deletions(-) create mode 100644 src/erofs/composefs.rs diff --git a/Cargo.toml b/Cargo.toml index 0461c9d9..8c45cfe2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,7 +36,7 @@ thiserror = "2.0.0" tokio = "1.24.2" toml = "0.8.0" xxhash-rust = { version = "0.8.2", features = ["xxh32"] } -zerocopy = { version = "0.8.0", features = ["derive"] } +zerocopy = { version = "0.8.0", features = ["derive", "std"] } zstd = "0.13.0" [dev-dependencies] diff --git a/src/bin/cfsctl.rs b/src/bin/cfsctl.rs index 09cd7372..0f5c5064 100644 --- a/src/bin/cfsctl.rs +++ b/src/bin/cfsctl.rs @@ -5,7 +5,12 @@ use clap::{Parser, Subcommand}; use rustix::fs::CWD; -use composefs::{oci, repository::Repository, util::parse_sha256}; +use composefs::{ + fsverity::{FsVerityHashValue, Sha256HashValue}, + oci, + repository::Repository, + util::parse_sha256, +}; /// cfsctl #[derive(Debug, Parser)] @@ -130,7 +135,7 @@ fn main() -> Result<()> { } Command::ImportImage { reference } => { let image_id = repo.import_image(&reference, &mut std::io::stdin())?; - println!("{}", hex::encode(image_id)); + println!("{}", image_id.to_hex()); } Command::Oci { cmd: oci_cmd } => match oci_cmd { OciCommand::ImportLayer { name, sha256 } => { @@ -140,7 +145,7 @@ fn main() -> Result<()> { name.as_deref(), &mut std::io::stdin(), )?; - println!("{}", hex::encode(object_id)); + println!("{}", object_id.to_hex()); } OciCommand::LsLayer { name } => { oci::ls_layer(&repo, &name)?; @@ -150,7 +155,7 @@ fn main() -> Result<()> { } OciCommand::CreateImage { config, name } => { let image_id = oci::image::create_image(&repo, &config, name.as_deref(), None)?; - println!("{}", hex::encode(image_id)); + println!("{}", image_id.to_hex()); } OciCommand::Pull { ref image, name } => { let runtime = tokio::runtime::Builder::new_current_thread() @@ -161,10 +166,13 @@ fn main() -> Result<()> { runtime.block_on(async move { oci::pull(&repo, image, name.as_deref()).await })?; } OciCommand::Seal { verity, ref name } => { - let (sha256, verity) = - oci::seal(&repo, name, verity.map(parse_sha256).transpose()?.as_ref())?; + let (sha256, verity) = oci::seal( + &repo, + name, + verity.map(Sha256HashValue::from_hex).transpose()?.as_ref(), + )?; println!("sha256 {}", hex::encode(sha256)); - println!("verity {}", hex::encode(verity)); + println!("verity {}", verity.to_hex()); } OciCommand::Mount { ref name, @@ -182,7 +190,7 @@ fn main() -> Result<()> { }, Command::CreateImage { ref path } => { let image_id = composefs::fs::create_image(path, Some(&repo))?; - println!("{}", hex::encode(image_id)); + println!("{}", image_id.to_hex()); } Command::CreateDumpfile { ref path } => { composefs::fs::create_dumpfile(path)?; @@ -193,7 +201,7 @@ fn main() -> Result<()> { Command::ImageObjects { name } => { let objects = repo.objects_for_image(&name)?; for object in objects { - println!("{}", hex::encode(object)); + println!("{}", object.to_hex()); } } Command::GC => { diff --git a/src/bin/composefs-setup-root.rs b/src/bin/composefs-setup-root.rs index aed88059..ebd1c14e 100644 --- a/src/bin/composefs-setup-root.rs +++ b/src/bin/composefs-setup-root.rs @@ -21,7 +21,7 @@ use rustix::{ use serde::Deserialize; use composefs::{ - fsverity::Sha256HashValue, + fsverity::{FsVerityHashValue, Sha256HashValue}, mount::{composefs_fsmount, mount_at, FsHandle}, mountcompat::{overlayfs_set_fd, overlayfs_set_lower_and_data_fds, prepare_mount}, repository::Repository, @@ -204,9 +204,7 @@ fn parse_composefs_cmdline(cmdline: &[u8]) -> Result { // TODO?: officially we need to understand quoting with double-quotes... for part in cmdline.split(|c| c.is_ascii_whitespace()) { if let Some(digest) = part.strip_prefix(b"composefs=") { - let mut value = [0; 32]; - hex::decode_to_slice(digest, &mut value).context("Parsing composefs=")?; - return Ok(value); + return Sha256HashValue::from_hex(digest).context("Parsing composefs="); } } bail!("Unable to find composefs= cmdline parameter"); @@ -238,7 +236,7 @@ fn setup_root(args: Args) -> Result<()> { Some(cmdline) => cmdline.as_bytes(), None => &std::fs::read("/proc/cmdline")?, }; - let image = hex::encode(parse_composefs_cmdline(cmdline)?); + let image = parse_composefs_cmdline(cmdline)?.to_hex(); let new_root = match args.root_fs { Some(path) => open_root_fs(&path).context("Failed to clone specified root fs")?, @@ -295,12 +293,9 @@ mod test { assert!(parse_composefs_cmdline(case.as_bytes()).is_err()); } let digest = "8b7df143d91c716ecfa5fc1730022f6b421b05cedee8fd52b1fc65a96030ad52"; - let digest_bytes = hex::decode(digest).unwrap(); similar_asserts::assert_eq!( - parse_composefs_cmdline(format!("composefs={digest}").as_bytes()) - .unwrap() - .as_slice(), - &digest_bytes + parse_composefs_cmdline(format!("composefs={digest}").as_bytes()).unwrap(), + Sha256HashValue::from_hex(digest).unwrap() ); } } diff --git a/src/dumpfile.rs b/src/dumpfile.rs index d2baa331..4bbc59f4 100644 --- a/src/dumpfile.rs +++ b/src/dumpfile.rs @@ -12,7 +12,7 @@ use anyhow::Result; use rustix::fs::FileType; use crate::{ - fsverity::Sha256HashValue, + fsverity::{FsVerityHashValue, Sha256HashValue}, image::{Directory, FileSystem, Inode, Leaf, LeafContent, RegularFile, Stat}, }; @@ -66,7 +66,7 @@ fn write_entry( write_escaped(writer, content)?; write!(writer, " ")?; if let Some(id) = digest { - write!(writer, "{}", hex::encode(id))?; + write!(writer, "{}", id.to_hex())?; } else { write_empty(writer)?; } @@ -129,7 +129,7 @@ pub fn write_leaf( *size, nlink, 0, - format!("{:02x}/{}", id[0], hex::encode(&id[1..])), + id.to_object_pathname(), &[], Some(id), ), diff --git a/src/erofs/composefs.rs b/src/erofs/composefs.rs new file mode 100644 index 00000000..b293bf60 --- /dev/null +++ b/src/erofs/composefs.rs @@ -0,0 +1,33 @@ +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; + +use crate::fsverity::FsVerityHashValue; + +/* From linux/fs/overlayfs/overlayfs.h struct ovl_metacopy */ +#[derive(Debug, FromBytes, Immutable, KnownLayout, IntoBytes)] +#[repr(C)] +pub(super) struct OverlayMetacopy { + version: u8, + len: u8, + flags: u8, + digest_algo: u8, + pub(super) digest: H, +} + +impl OverlayMetacopy { + pub(super) fn new(digest: &H) -> Self { + Self { + version: 0, + len: size_of::() as u8, + flags: 0, + digest_algo: H::ALGORITHM, + digest: digest.clone(), + } + } + + pub(super) fn valid(&self) -> bool { + self.version == 0 + && self.len == size_of::() as u8 + && self.flags == 0 + && self.digest_algo == H::ALGORITHM + } +} diff --git a/src/erofs/mod.rs b/src/erofs/mod.rs index 8ef9c1b0..fac88cd5 100644 --- a/src/erofs/mod.rs +++ b/src/erofs/mod.rs @@ -1,3 +1,4 @@ +pub mod composefs; pub mod debug; pub mod format; pub mod reader; diff --git a/src/erofs/reader.rs b/src/erofs/reader.rs index 76dc77fb..6b668a76 100644 --- a/src/erofs/reader.rs +++ b/src/erofs/reader.rs @@ -5,9 +5,12 @@ use std::ops::Range; use thiserror::Error; use zerocopy::{little_endian::U32, FromBytes, Immutable, KnownLayout}; -use super::format::{ - CompactInodeHeader, ComposefsHeader, DataLayout, DirectoryEntryHeader, ExtendedInodeHeader, - InodeXAttrHeader, ModeField, Superblock, XAttrHeader, +use super::{ + composefs::OverlayMetacopy, + format::{ + CompactInodeHeader, ComposefsHeader, DataLayout, DirectoryEntryHeader, ExtendedInodeHeader, + InodeXAttrHeader, ModeField, Superblock, XAttrHeader, + }, }; use crate::fsverity::Sha256HashValue; @@ -491,18 +494,17 @@ pub struct ObjectCollector { impl ObjectCollector { fn visit_xattr(&mut self, attr: &XAttr) { - // TODO: "4" is a bit magic, isn't it? + // This is the index of "trusted". See XATTR_PREFIXES in format.rs. if attr.header.name_index != 4 { return; } if attr.suffix() != b"overlay.metacopy" { return; } - let value = attr.value(); - // TODO: oh look, more magic values... - if value.len() == 36 && value[..4] == [0, 36, 0, 1] { - // SAFETY: We already checked that the length is 4 + 32 - self.objects.insert(value[4..].try_into().unwrap()); + if let Ok(value) = OverlayMetacopy::read_from_bytes(attr.value()) { + if value.valid() { + self.objects.insert(value.digest); + } } } diff --git a/src/erofs/writer.rs b/src/erofs/writer.rs index 19d06224..89a19e78 100644 --- a/src/erofs/writer.rs +++ b/src/erofs/writer.rs @@ -11,7 +11,8 @@ use xxhash_rust::xxh32::xxh32; use zerocopy::{Immutable, IntoBytes}; use crate::{ - erofs::{format, reader::round_up}, + erofs::{composefs::OverlayMetacopy, format, reader::round_up}, + fsverity::FsVerityHashValue, image, }; @@ -409,10 +410,12 @@ impl<'a> InodeCollector<'a> { .. }) = content { - let metacopy = [&[0, 36, 0, 1], &id[..]].concat(); - xattrs.add(b"trusted.overlay.metacopy", &metacopy); + xattrs.add( + b"trusted.overlay.metacopy", + OverlayMetacopy::new(id).as_bytes(), + ); - let redirect = format!("/{:02x}/{}", id[0], hex::encode(&id[1..])); + let redirect = format!("/{}", id.to_object_pathname()); xattrs.add(b"trusted.overlay.redirect", redirect.as_bytes()); } diff --git a/src/fsverity/digest.rs b/src/fsverity/digest.rs index f085744b..861fc92c 100644 --- a/src/fsverity/digest.rs +++ b/src/fsverity/digest.rs @@ -64,7 +64,7 @@ impl FsVerityHasher { // We had a complete value, but now we're adding new data. // This means that we need to add a new hash layer... let mut new_layer = FsVerityLayer::new(); - new_layer.add_data(value.as_ref()); + new_layer.add_data(value.as_bytes()); self.layers.push(new_layer); } @@ -76,7 +76,7 @@ impl FsVerityHasher { for layer in self.layers.iter_mut() { // We have a layer we need to hash this value into - layer.add_data(value.as_ref()); + layer.add_data(value.as_bytes()); if layer.remaining != 0 { return; } @@ -97,7 +97,7 @@ impl FsVerityHasher { for layer in self.layers.iter_mut() { // We have a layer we need to hash this value into if value != H::EMPTY { - layer.add_data(value.as_ref()); + layer.add_data(value.as_bytes()); } if layer.remaining != (1 << LG_BLKSZ) { // ...but now this layer itself is complete, so get the value of *it*. @@ -143,7 +143,7 @@ impl FsVerityHasher { context.update(0u8.to_le_bytes()); /* salt_size */ context.update([0; 4]); /* reserved */ context.update(self.n_bytes.to_le_bytes()); - context.update(self.root_hash()); + context.update(self.root_hash().as_bytes()); context.update([0].repeat(64 - size_of::())); context.update([0; 32]); /* salt */ context.update([0; 144]); /* reserved */ @@ -162,12 +162,12 @@ mod tests { #[test] fn test_digest() { assert_eq!( - hex::encode(FsVerityHasher::::hash(b"hello world")), + FsVerityHasher::::hash(b"hello world").to_hex(), "1e2eaa4202d750a41174ee454970b92c1bc2f925b1e35076d8c7d5f56362ba64" ); assert_eq!( - hex::encode(FsVerityHasher::::hash(b"hello world")), + FsVerityHasher::::hash(b"hello world").to_hex(), "18430270729d162d4e469daca123ae61893db4b0583d8f7081e3bf4f92b88ba514e7982f10733fb6aa895195c5ae8fd2eb2c47a8be05513ce5a0c51a6f570409" ); } diff --git a/src/fsverity/hashvalue.rs b/src/fsverity/hashvalue.rs index 3d23367d..c5677999 100644 --- a/src/fsverity/hashvalue.rs +++ b/src/fsverity/hashvalue.rs @@ -1,27 +1,127 @@ +use core::{fmt, hash::Hash}; + +use hex::FromHexError; use sha2::{digest::FixedOutputReset, digest::Output, Digest, Sha256, Sha512}; +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout, Unaligned}; pub trait FsVerityHashValue where - Self: Eq + AsRef<[u8]> + Clone, + Self: Clone, Self: From>, + Self: FromBytes + Immutable + IntoBytes + KnownLayout + Unaligned, + Self: Hash + Eq, + Self: fmt::Debug, { - type Digest: Digest + FixedOutputReset + std::fmt::Debug; + type Digest: Digest + FixedOutputReset + fmt::Debug; const ALGORITHM: u8; const EMPTY: Self; + const ID: &str; + + fn from_hex(hex: impl AsRef<[u8]>) -> Result { + let mut value = Self::EMPTY; + hex::decode_to_slice(hex.as_ref(), value.as_mut_bytes())?; + Ok(value) + } + + fn from_object_dir_and_basename( + dirnum: u8, + basename: impl AsRef<[u8]>, + ) -> Result { + let expected_size = 2 * (size_of::() - 1); + let bytes = basename.as_ref(); + if bytes.len() != expected_size { + return Err(FromHexError::InvalidStringLength); + } + let mut result = Self::EMPTY; + result.as_mut_bytes()[0] = dirnum; + hex::decode_to_slice(bytes, &mut result.as_mut_bytes()[1..])?; + Ok(result) + } + + fn from_object_pathname(pathname: impl AsRef<[u8]>) -> Result { + // We want to the trailing part of "....../xx/yyyyyy" where xxyyyyyy is our hex length + let min_size = 2 * size_of::() + 1; + let bytes = pathname.as_ref(); + if bytes.len() < min_size { + return Err(FromHexError::InvalidStringLength); + } + + let trailing = &bytes[bytes.len() - min_size..]; + let mut result = Self::EMPTY; + hex::decode_to_slice(&trailing[0..2], &mut result.as_mut_bytes()[0..1])?; + if trailing[2] != b'/' { + return Err(FromHexError::InvalidHexCharacter { + c: trailing[2] as char, + index: 2, + }); + } + hex::decode_to_slice(&trailing[3..], &mut result.as_mut_bytes()[1..])?; + Ok(result) + } + + fn to_object_pathname(&self) -> String { + format!("{:02x}/{}", self.as_bytes()[0], self.to_object_basename()) + } + + fn to_object_dir(&self) -> String { + format!("{:02x}", self.as_bytes()[0]) + } + + fn to_object_basename(&self) -> String { + hex::encode(&self.as_bytes()[1..]) + } + + fn to_hex(&self) -> String { + hex::encode(self.as_bytes()) + } + + fn to_id(&self) -> String { + format!("{}:{}", Self::ID, self.to_hex()) + } +} + +impl fmt::Debug for Sha256HashValue { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "sha256:{}", self.to_hex()) + } +} + +impl fmt::Debug for Sha512HashValue { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "sha512:{}", self.to_hex()) + } } -pub type Sha256HashValue = [u8; 32]; +#[derive(Clone, Eq, FromBytes, Hash, Immutable, IntoBytes, KnownLayout, PartialEq, Unaligned)] +#[repr(C)] +pub struct Sha256HashValue([u8; 32]); + +impl From> for Sha256HashValue { + fn from(value: Output) -> Self { + Self(value.into()) + } +} impl FsVerityHashValue for Sha256HashValue { type Digest = Sha256; const ALGORITHM: u8 = 1; - const EMPTY: Self = [0; 32]; + const EMPTY: Self = Self([0; 32]); + const ID: &str = "sha256"; } -pub type Sha512HashValue = [u8; 64]; +#[derive(Clone, Eq, FromBytes, Hash, Immutable, IntoBytes, KnownLayout, PartialEq, Unaligned)] +#[repr(C)] +pub struct Sha512HashValue([u8; 64]); + +impl From> for Sha512HashValue { + fn from(value: Output) -> Self { + Self(value.into()) + } +} impl FsVerityHashValue for Sha512HashValue { type Digest = Sha512; const ALGORITHM: u8 = 2; - const EMPTY: Self = [0; 64]; + const EMPTY: Self = Self([0; 64]); + const ID: &str = "sha512"; } diff --git a/src/fsverity/mod.rs b/src/fsverity/mod.rs index da2e24e1..880f29a3 100644 --- a/src/fsverity/mod.rs +++ b/src/fsverity/mod.rs @@ -132,8 +132,8 @@ pub fn ensure_verity_equal( Ok(()) } else { Err(CompareVerityError::DigestMismatch { - expected: hex::encode(expected), - found: hex::encode(found.as_ref()), + expected: expected.to_hex(), + found: found.to_hex(), }) } } @@ -142,10 +142,7 @@ pub fn ensure_verity_equal( mod tests { use std::{collections::BTreeSet, io::Write}; - use crate::{ - test::tempfile, - util::{parse_sha256, proc_self_fd}, - }; + use crate::{test::tempfile, util::proc_self_fd}; use rustix::{ fd::OwnedFd, fs::{open, Mode, OFlags}, @@ -200,26 +197,33 @@ mod tests { )); assert_eq!( - hex::encode(measure_verity::(&tf).unwrap()), + measure_verity::(&tf).unwrap().to_hex(), "1e2eaa4202d750a41174ee454970b92c1bc2f925b1e35076d8c7d5f56362ba64" ); assert_eq!( - hex::encode(measure_verity_opt::(&tf).unwrap().unwrap()), + measure_verity_opt::(&tf) + .unwrap() + .unwrap() + .to_hex(), "1e2eaa4202d750a41174ee454970b92c1bc2f925b1e35076d8c7d5f56362ba64" ); ensure_verity_equal( &tf, - &parse_sha256("1e2eaa4202d750a41174ee454970b92c1bc2f925b1e35076d8c7d5f56362ba64") - .unwrap(), + &Sha256HashValue::from_hex( + "1e2eaa4202d750a41174ee454970b92c1bc2f925b1e35076d8c7d5f56362ba64", + ) + .unwrap(), ) .unwrap(); let Err(CompareVerityError::DigestMismatch { expected, found }) = ensure_verity_equal( &tf, - &parse_sha256("1e2eaa4202d750a41174ee454970b92c1bc2f925b1e35076d8c7000000000000") - .unwrap(), + &Sha256HashValue::from_hex( + "1e2eaa4202d750a41174ee454970b92c1bc2f925b1e35076d8c7000000000000", + ) + .unwrap(), ) else { panic!("Didn't fail with expected error"); }; diff --git a/src/oci/mod.rs b/src/oci/mod.rs index 4abe546c..8093fc45 100644 --- a/src/oci/mod.rs +++ b/src/oci/mod.rs @@ -15,7 +15,7 @@ use tokio::io::AsyncReadExt; use crate::{ fs::write_to_path, - fsverity::Sha256HashValue, + fsverity::{FsVerityHashValue, Sha256HashValue}, oci::tar::{get_entry, split_async}, repository::Repository, splitstream::DigestMap, @@ -102,7 +102,7 @@ impl<'repo> ImageOp<'repo> { if let Some(layer_id) = self.repo.check_stream(layer_sha256)? { self.progress - .println(format!("Already have layer {}", hex::encode(layer_sha256)))?; + .println(format!("Already have layer {layer_sha256:?}"))?; Ok(layer_id) } else { // Otherwise, we need to fetch it... @@ -189,7 +189,7 @@ impl<'repo> ImageOp<'repo> { } } - pub async fn pull(&self) -> Result<(Sha256HashValue, Sha256HashValue)> { + pub async fn pull(&self) -> Result { let (_manifest_digest, raw_manifest) = self .proxy .fetch_manifest_raw_oci(&self.img) @@ -220,7 +220,7 @@ pub async fn pull(repo: &Repository, imgref: &str, reference: Option<&str>) -> R repo.name_stream(sha256, name)?; } println!("sha256 {}", hex::encode(sha256)); - println!("verity {}", hex::encode(id)); + println!("verity {}", id.to_hex()); Ok(()) } @@ -276,7 +276,7 @@ pub fn write_config( repo: &Repository, config: &ImageConfiguration, refs: DigestMap, -) -> Result<(Sha256HashValue, Sha256HashValue)> { +) -> Result { let json = config.to_string()?; let json_bytes = json.as_bytes(); let sha256 = hash(json_bytes); @@ -290,12 +290,12 @@ pub fn seal( repo: &Repository, name: &str, verity: Option<&Sha256HashValue>, -) -> Result<(Sha256HashValue, Sha256HashValue)> { +) -> Result { let (mut config, refs) = open_config(repo, name, verity)?; let mut myconfig = config.config().clone().context("no config!")?; let labels = myconfig.labels_mut().get_or_insert_with(HashMap::new); let id = crate::oci::image::create_image(repo, name, None, verity)?; - labels.insert("containers.composefs.fsverity".to_string(), hex::encode(id)); + labels.insert("containers.composefs.fsverity".to_string(), id.to_hex()); config.set_config(Some(myconfig)); write_config(repo, &config, refs) } diff --git a/src/oci/tar.rs b/src/oci/tar.rs index 718a1670..2db4818f 100644 --- a/src/oci/tar.rs +++ b/src/oci/tar.rs @@ -183,11 +183,7 @@ pub fn get_entry(reader: &mut SplitStreamReader) -> Result bail!( - "Unsupported external-chunked entry {:?} {}", - header, - hex::encode(id) - ), + _ => bail!("Unsupported external-chunked entry {header:?} {id:?}"), }, SplitStreamData::Inline(content) => match header.entry_type() { EntryType::GNULongLink => { diff --git a/src/repository.rs b/src/repository.rs index 9f2031a2..79ee1f18 100644 --- a/src/repository.rs +++ b/src/repository.rs @@ -24,7 +24,7 @@ use crate::{ }, mount::mount_composefs_at, splitstream::{DigestMap, SplitStreamReader, SplitStreamWriter}, - util::{parse_sha256, proc_self_fd, Sha256Digest}, + util::{proc_self_fd, Sha256Digest}, }; #[derive(Debug)] @@ -78,8 +78,8 @@ impl Repository { pub fn ensure_object(&self, data: &[u8]) -> Result { let digest: Sha256HashValue = compute_verity(data); - let dir = PathBuf::from(format!("objects/{:02x}", digest[0])); - let file = dir.join(hex::encode(&digest[1..])); + let dir = PathBuf::from(format!("objects/{}", digest.to_object_dir())); + let file = dir.join(digest.to_object_basename()); // fairly common... if accessat(&self.repository, &file, Access::READ_OK, AtFlags::empty()) == Ok(()) { @@ -142,28 +142,8 @@ impl Repository { SplitStreamWriter::new(self, maps, sha256) } - fn parse_object_path(path: impl AsRef<[u8]>) -> Result { - // "objects/0c/9513d99b120ee9a709c4d6554d938f6b2b7e213cf5b26f2e255c0b77e40379" - let bytes = path.as_ref(); - ensure!(bytes.len() == 73, "stream symlink has incorrect length"); - ensure!( - bytes.starts_with(b"objects/"), - "stream symlink has incorrect prefix" - ); - ensure!( - bytes[10] == b'/', - "stream symlink has incorrect path separator" - ); - let mut result = Sha256HashValue::EMPTY; - hex::decode_to_slice(&bytes[8..10], &mut result[..1]) - .context("stream symlink has incorrect format")?; - hex::decode_to_slice(&bytes[11..], &mut result[1..]) - .context("stream symlink has incorrect format")?; - Ok(result) - } - fn format_object_path(id: &Sha256HashValue) -> String { - format!("objects/{:02x}/{}", id[0], hex::encode(&id[1..])) + format!("objects/{}", id.to_object_pathname()) } pub fn has_stream(&self, sha256: &Sha256Digest) -> Result> { @@ -183,7 +163,7 @@ impl Repository { bytes.starts_with(b"../"), "stream symlink has incorrect prefix" ); - Ok(Some(Repository::parse_object_path(&bytes[3..])?)) + Ok(Some(Sha256HashValue::from_object_pathname(bytes)?)) } Err(Errno::NOENT) => Ok(None), Err(err) => Err(err)?, @@ -200,7 +180,7 @@ impl Repository { // check the verity of all linked streams for entry in &split_stream.refs.map { - if self.check_stream(&entry.body)? != Some(entry.verity) { + if self.check_stream(&entry.body)?.as_ref() != Some(&entry.verity) { bail!("reference mismatch"); } } @@ -312,10 +292,7 @@ impl Repository { } pub fn open_object(&self, id: &Sha256HashValue) -> Result { - self.open_with_verity( - &format!("objects/{:02x}/{}", id[0], hex::encode(&id[1..])), - id, - ) + self.open_with_verity(&format!("objects/{}", id.to_object_pathname()), id) } pub fn merge_splitstream( @@ -338,12 +315,8 @@ impl Repository { pub fn write_image(&self, name: Option<&str>, data: &[u8]) -> Result { let object_id = self.ensure_object(data)?; - let object_path = format!( - "objects/{:02x}/{}", - object_id[0], - hex::encode(&object_id[1..]) - ); - let image_path = format!("images/{}", hex::encode(object_id)); + let object_path = Self::format_object_path(&object_id); + let image_path = format!("images/{}", object_id.to_hex()); self.ensure_symlink(&image_path, &object_path)?; @@ -367,7 +340,7 @@ impl Repository { if !name.contains("/") { // A name with no slashes in it is taken to be a sha256 fs-verity digest - ensure_verity_equal(&image, &parse_sha256(name)?)?; + ensure_verity_equal(&image, &Sha256HashValue::from_hex(name)?)?; } Ok(image) @@ -423,17 +396,9 @@ impl Repository { fn read_symlink_hashvalue(dirfd: &OwnedFd, name: &CStr) -> Result { let link_content = readlinkat(dirfd, name, [])?; - let link_bytes = link_content.to_bytes(); - let link_size = link_bytes.len(); - // XXX: check correctness of leading ../? - // XXX: or is that something for fsck? - if link_size > 64 { - let mut value = Sha256HashValue::EMPTY; - hex::decode_to_slice(&link_bytes[link_size - 64..link_size], &mut value)?; - Ok(value) - } else { - bail!("symlink has wrong format") - } + Ok(Sha256HashValue::from_object_pathname( + link_content.to_bytes(), + )?) } fn walk_symlinkdir(fd: OwnedFd, objects: &mut HashSet) -> Result<()> { @@ -491,12 +456,19 @@ impl Repository { if entry.file_type() != FileType::Symlink { bail!("category directory contains non-symlink"); } + + // TODO: we need to sort this out. the symlink itself might be a sha256 content ID + // (as for splitstreams), not an object/ to be preserved. + continue; + + /* let mut value = Sha256HashValue::EMPTY; hex::decode_to_slice(filename.to_bytes(), &mut value)?; if !objects.contains(&value) { println!("rm {}/{:?}", category, filename); } + */ } } @@ -516,19 +488,19 @@ impl Repository { let mut objects = HashSet::new(); for ref object in self.gc_category("images")? { - println!("{} lives as an image", hex::encode(object)); - objects.insert(*object); - objects.extend(self.objects_for_image(&hex::encode(object))?); + println!("{object:?} lives as an image"); + objects.insert(object.clone()); + objects.extend(self.objects_for_image(&object.to_hex())?); } for object in self.gc_category("streams")? { - println!("{} lives as a stream", hex::encode(object)); - objects.insert(object); + println!("{object:?} lives as a stream"); + objects.insert(object.clone()); - let mut split_stream = self.open_stream(&hex::encode(object), None)?; + let mut split_stream = self.open_stream(&object.to_hex(), None)?; split_stream.get_object_refs(|id| { - println!(" with {}", hex::encode(*id)); - objects.insert(*id); + println!(" with {id:?}"); + objects.insert(id.clone()); })?; } @@ -545,9 +517,10 @@ impl Repository { let entry = item?; let filename = entry.file_name(); if filename != c"." && filename != c".." { - let mut value = Sha256HashValue::EMPTY; - value[0] = first_byte; - hex::decode_to_slice(filename.to_bytes(), &mut value[1..])?; + let value = Sha256HashValue::from_object_dir_and_basename( + first_byte, + filename.to_bytes(), + )?; if !objects.contains(&value) { println!("rm objects/{first_byte:02x}/{filename:?}"); } else { diff --git a/src/splitstream.rs b/src/splitstream.rs index f6bc8770..ed342c1f 100644 --- a/src/splitstream.rs +++ b/src/splitstream.rs @@ -7,15 +7,17 @@ use std::io::{BufReader, Read, Write}; use anyhow::{bail, Result}; use sha2::{Digest, Sha256}; +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; use zstd::stream::{read::Decoder, write::Encoder}; use crate::{ - fsverity::{FsVerityHashValue, Sha256HashValue}, + fsverity::Sha256HashValue, repository::Repository, util::{read_exactish, Sha256Digest}, }; -#[derive(Debug)] +#[derive(Debug, FromBytes, Immutable, IntoBytes, KnownLayout)] +#[repr(C)] pub struct DigestMapEntry { pub body: Sha256Digest, pub verity: Sha256HashValue, @@ -51,7 +53,7 @@ impl DigestMap { idx, DigestMapEntry { body: *body, - verity: *verity, + verity: verity.clone(), }, ), } @@ -88,10 +90,7 @@ impl SplitStreamWriter<'_> { match refs { Some(DigestMap { map }) => { writer.write_all(&(map.len() as u64).to_le_bytes()).unwrap(); - for ref entry in map { - writer.write_all(&entry.body).unwrap(); - writer.write_all(&entry.verity).unwrap(); - } + writer.write_all(map.as_bytes()).unwrap(); } None => { writer.write_all(&0u64.to_le_bytes()).unwrap(); @@ -141,7 +140,7 @@ impl SplitStreamWriter<'_> { // external data becomes the start of a new inline block. self.flush_inline(padding)?; - SplitStreamWriter::write_fragment(&mut self.writer, 0, &reference) + SplitStreamWriter::write_fragment(&mut self.writer, 0, reference.as_bytes()) } pub fn write_external(&mut self, data: &[u8], padding: Vec) -> Result<()> { @@ -228,12 +227,7 @@ impl SplitStreamReader { map: Vec::with_capacity(n_map_entries), }; for _ in 0..n_map_entries { - let mut body = [0u8; 32]; - let mut verity = [0u8; 32]; - - decoder.read_exact(&mut body)?; - decoder.read_exact(&mut verity)?; - refs.map.push(DigestMapEntry { body, verity }); + refs.map.push(DigestMapEntry::read_from_io(&mut decoder)?); } Ok(SplitStreamReader { @@ -261,8 +255,7 @@ impl SplitStreamReader { if !ext_ok { bail!("Unexpected external reference when parsing splitstream"); } - let mut id = Sha256HashValue::EMPTY; - self.decoder.read_exact(&mut id)?; + let id = Sha256HashValue::read_from_io(&mut self.decoder)?; return Ok(ChunkType::External(id)); } Some(size) => { diff --git a/tests/mkfs.rs b/tests/mkfs.rs index 61992e3a..c53ed85b 100644 --- a/tests/mkfs.rs +++ b/tests/mkfs.rs @@ -13,6 +13,7 @@ use tempfile::NamedTempFile; use composefs::{ dumpfile::write_dumpfile, erofs::{debug::debug_img, writer::mkfs_erofs}, + fsverity::{FsVerityHashValue, Sha256HashValue}, image::{Directory, FileSystem, Inode, Leaf, LeafContent, RegularFile, Stat}, }; @@ -50,6 +51,10 @@ fn add_leaf(dir: &mut Directory, name: impl AsRef, content: LeafContent) } fn simple(fs: &mut FileSystem) { + let ext_id = Sha256HashValue::from_hex( + "5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a", + ) + .unwrap(); add_leaf(&mut fs.root, "fifo", LeafContent::Fifo); add_leaf( &mut fs.root, @@ -59,7 +64,7 @@ fn simple(fs: &mut FileSystem) { add_leaf( &mut fs.root, "regular-external", - LeafContent::Regular(RegularFile::External([0x5a; 32], 1234)), + LeafContent::Regular(RegularFile::External(ext_id, 1234)), ); add_leaf(&mut fs.root, "chrdev", LeafContent::CharacterDevice(123)); add_leaf(&mut fs.root, "blkdev", LeafContent::BlockDevice(123)); From 116cd755de5116e1925ec468707b5de30188d4d4 Mon Sep 17 00:00:00 2001 From: Allison Karlitskaya Date: Thu, 17 Apr 2025 17:31:11 +0200 Subject: [PATCH 5/9] src/fsverity: add tests for trait code This tests out all of the various conversions we can do. Signed-off-by: Allison Karlitskaya --- src/fsverity/hashvalue.rs | 79 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/src/fsverity/hashvalue.rs b/src/fsverity/hashvalue.rs index c5677999..e75f1800 100644 --- a/src/fsverity/hashvalue.rs +++ b/src/fsverity/hashvalue.rs @@ -125,3 +125,82 @@ impl FsVerityHashValue for Sha512HashValue { const EMPTY: Self = Self([0; 64]); const ID: &str = "sha512"; } + +#[cfg(test)] +mod test { + use super::*; + + fn test_fsverity_hash() { + let len = size_of::(); + let hexlen = len * 2; + + let hex = H::EMPTY.to_hex(); + assert_eq!(hex.as_bytes(), [b'0'].repeat(hexlen)); + + assert_eq!(H::EMPTY.to_id(), format!("{}:{}", H::ID, hex)); + assert_eq!(format!("{:?}", H::EMPTY), format!("{}:{}", H::ID, hex)); + + assert_eq!(H::from_hex(&hex), Ok(H::EMPTY)); + + assert_eq!(H::from_hex("lol"), Err(FromHexError::OddLength)); + assert_eq!(H::from_hex("lolo"), Err(FromHexError::InvalidStringLength)); + assert_eq!( + H::from_hex([b'l'].repeat(hexlen)), + Err(FromHexError::InvalidHexCharacter { c: 'l', index: 0 }) + ); + + assert_eq!(H::from_object_dir_and_basename(0, &hex[2..]), Ok(H::EMPTY)); + + assert_eq!(H::from_object_dir_and_basename(0, &hex[2..]), Ok(H::EMPTY)); + + assert_eq!( + H::from_object_dir_and_basename(0, "lol"), + Err(FromHexError::InvalidStringLength) + ); + + assert_eq!( + H::from_object_dir_and_basename(0, [b'l'].repeat(hexlen - 2)), + Err(FromHexError::InvalidHexCharacter { c: 'l', index: 0 }) + ); + + assert_eq!( + H::from_object_pathname(format!("{}/{}", &hex[0..2], &hex[2..])), + Ok(H::EMPTY) + ); + + assert_eq!( + H::from_object_pathname(format!("../this/is/ignored/{}/{}", &hex[0..2], &hex[2..])), + Ok(H::EMPTY) + ); + + assert_eq!( + H::from_object_pathname(&hex), + Err(FromHexError::InvalidStringLength) + ); + + assert_eq!( + H::from_object_pathname("lol"), + Err(FromHexError::InvalidStringLength) + ); + + assert_eq!( + H::from_object_pathname([b'l'].repeat(hexlen + 1)), + Err(FromHexError::InvalidHexCharacter { c: 'l', index: 0 }) + ); + + assert_eq!( + H::from_object_pathname(format!("{}0{}", &hex[0..2], &hex[2..])), + Err(FromHexError::InvalidHexCharacter { c: '0', index: 2 }) + ); + } + + #[test] + fn test_sha256hashvalue() { + test_fsverity_hash::(); + } + + #[test] + fn test_sha512hashvalue() { + test_fsverity_hash::(); + } +} From 6c61761fbef17accd49a05c4637503163867714f Mon Sep 17 00:00:00 2001 From: Allison Karlitskaya Date: Wed, 16 Apr 2025 11:46:36 +0200 Subject: [PATCH 6/9] src: increase use of Self where possible Various bits of code here predate my having learned what Self. We're about to throw a bunch of generic type arguments absolutely everywhere, so increase our use of Self first to help make that less painful. Signed-off-by: Allison Karlitskaya --- src/fs.rs | 6 +++--- src/repository.rs | 22 +++++++++++----------- src/splitstream.rs | 16 ++++++++-------- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/fs.rs b/src/fs.rs index 968f6e17..f504e69e 100644 --- a/src/fs.rs +++ b/src/fs.rs @@ -176,7 +176,7 @@ impl FilesystemReader<'_> { st_uid: buf.st_uid, st_gid: buf.st_gid, st_mtim_sec: buf.st_mtime as i64, - xattrs: RefCell::new(FilesystemReader::read_xattrs(fd)?), + xattrs: RefCell::new(Self::read_xattrs(fd)?), }, )) } @@ -225,7 +225,7 @@ impl FilesystemReader<'_> { Mode::empty(), )?; - let (buf, stat) = FilesystemReader::stat(&fd, ifmt)?; + let (buf, stat) = Self::stat(&fd, ifmt)?; // NB: We could check `st_nlink > 1` to find out if we should track a file as a potential // hardlink or not, but some filesystems (like fuse-overlayfs) can report this incorrectly. @@ -249,7 +249,7 @@ impl FilesystemReader<'_> { Mode::empty(), )?; - let (_, stat) = FilesystemReader::stat(&fd, FileType::Directory)?; + let (_, stat) = Self::stat(&fd, FileType::Directory)?; let mut directory = Directory::new(stat); for item in Dir::read_from(&fd)? { diff --git a/src/repository.rs b/src/repository.rs index 79ee1f18..15a533f4 100644 --- a/src/repository.rs +++ b/src/repository.rs @@ -46,7 +46,7 @@ impl Repository { ) } - pub fn open_path(dirfd: impl AsFd, path: impl AsRef) -> Result { + pub fn open_path(dirfd: impl AsFd, path: impl AsRef) -> Result { let path = path.as_ref(); // O_PATH isn't enough because flock() @@ -56,17 +56,17 @@ impl Repository { flock(&repository, FlockOperation::LockShared) .context("Cannot lock composefs repository")?; - Ok(Repository { repository }) + Ok(Self { repository }) } - pub fn open_user() -> Result { + pub fn open_user() -> Result { let home = std::env::var("HOME").with_context(|| "$HOME must be set when in user mode")?; - Repository::open_path(CWD, PathBuf::from(home).join(".var/lib/composefs")) + Self::open_path(CWD, PathBuf::from(home).join(".var/lib/composefs")) } - pub fn open_system() -> Result { - Repository::open_path(CWD, PathBuf::from("/sysroot/composefs".to_string())) + pub fn open_system() -> Result { + Self::open_path(CWD, PathBuf::from("/sysroot/composefs".to_string())) } fn ensure_dir(&self, dir: impl AsRef) -> ErrnoResult<()> { @@ -212,7 +212,7 @@ impl Repository { }; let stream_path = format!("streams/{}", hex::encode(sha256)); let object_id = writer.done()?; - let object_path = Repository::format_object_path(&object_id); + let object_path = Self::format_object_path(&object_id); self.ensure_symlink(&stream_path, &object_path)?; if let Some(name) = reference { @@ -261,7 +261,7 @@ impl Repository { callback(&mut writer)?; let object_id = writer.done()?; - let object_path = Repository::format_object_path(&object_id); + let object_path = Self::format_object_path(&object_id); self.ensure_symlink(&stream_path, &object_path)?; object_id } @@ -411,11 +411,11 @@ impl Repository { let filename = entry.file_name(); if filename != c"." && filename != c".." { let dirfd = openat(&fd, filename, OFlags::RDONLY, Mode::empty())?; - Repository::walk_symlinkdir(dirfd, objects)?; + Self::walk_symlinkdir(dirfd, objects)?; } } FileType::Symlink => { - objects.insert(Repository::read_symlink_hashvalue(&fd, entry.file_name())?); + objects.insert(Self::read_symlink_hashvalue(&fd, entry.file_name())?); } _ => { bail!("Unexpected file type encountered"); @@ -447,7 +447,7 @@ impl Repository { OFlags::RDONLY | OFlags::DIRECTORY, Mode::empty(), )?; - Repository::walk_symlinkdir(refs, &mut objects)?; + Self::walk_symlinkdir(refs, &mut objects)?; for item in Dir::read_from(&category_fd)? { let entry = item?; diff --git a/src/splitstream.rs b/src/splitstream.rs index ed342c1f..a40c2769 100644 --- a/src/splitstream.rs +++ b/src/splitstream.rs @@ -78,12 +78,12 @@ impl std::fmt::Debug for SplitStreamWriter<'_> { } } -impl SplitStreamWriter<'_> { +impl<'a> SplitStreamWriter<'a> { pub fn new( - repo: &Repository, + repo: &'a Repository, refs: Option, sha256: Option, - ) -> SplitStreamWriter { + ) -> Self { // SAFETY: we surely can't get an error writing the header to a Vec let mut writer = Encoder::new(vec![], 0).unwrap(); @@ -97,7 +97,7 @@ impl SplitStreamWriter<'_> { } } - SplitStreamWriter { + Self { repo, inline_content: vec![], writer, @@ -113,7 +113,7 @@ impl SplitStreamWriter<'_> { /// flush any buffered inline data, taking new_value as the new value of the buffer fn flush_inline(&mut self, new_value: Vec) -> Result<()> { if !self.inline_content.is_empty() { - SplitStreamWriter::write_fragment( + Self::write_fragment( &mut self.writer, self.inline_content.len(), &self.inline_content, @@ -140,7 +140,7 @@ impl SplitStreamWriter<'_> { // external data becomes the start of a new inline block. self.flush_inline(padding)?; - SplitStreamWriter::write_fragment(&mut self.writer, 0, reference.as_bytes()) + Self::write_fragment(&mut self.writer, 0, reference.as_bytes()) } pub fn write_external(&mut self, data: &[u8], padding: Vec) -> Result<()> { @@ -214,7 +214,7 @@ enum ChunkType { } impl SplitStreamReader { - pub fn new(reader: R) -> Result> { + pub fn new(reader: R) -> Result { let mut decoder = Decoder::new(reader)?; let n_map_entries = { @@ -230,7 +230,7 @@ impl SplitStreamReader { refs.map.push(DigestMapEntry::read_from_io(&mut decoder)?); } - Ok(SplitStreamReader { + Ok(Self { decoder, refs, inline_bytes: 0, From 43584c17aa21818b77843932c8bc77fbcb52799e Mon Sep 17 00:00:00 2001 From: Allison Karlitskaya Date: Wed, 16 Apr 2025 16:30:26 +0200 Subject: [PATCH 7/9] src/dumpfile: minor reorg Let's pass a string instead of a Sha256HashValue to write_entry(). This will save us a substantial amount of grief with generics in the next commit. Signed-off-by: Allison Karlitskaya --- src/dumpfile.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/dumpfile.rs b/src/dumpfile.rs index 4bbc59f4..f60f459d 100644 --- a/src/dumpfile.rs +++ b/src/dumpfile.rs @@ -12,7 +12,7 @@ use anyhow::Result; use rustix::fs::FileType; use crate::{ - fsverity::{FsVerityHashValue, Sha256HashValue}, + fsverity::FsVerityHashValue, image::{Directory, FileSystem, Inode, Leaf, LeafContent, RegularFile, Stat}, }; @@ -49,7 +49,7 @@ fn write_entry( rdev: u64, payload: impl AsRef, content: &[u8], - digest: Option<&Sha256HashValue>, + digest: Option<&str>, ) -> fmt::Result { let mode = stat.st_mode | ifmt.as_raw_mode(); let uid = stat.st_uid; @@ -66,7 +66,7 @@ fn write_entry( write_escaped(writer, content)?; write!(writer, " ")?; if let Some(id) = digest { - write!(writer, "{}", id.to_hex())?; + write!(writer, "{}", id)?; } else { write_empty(writer)?; } @@ -131,7 +131,7 @@ pub fn write_leaf( 0, id.to_object_pathname(), &[], - Some(id), + Some(&id.to_hex()), ), LeafContent::BlockDevice(rdev) => write_entry( writer, From 500610071cd4092a856af96afb75684ff7af5fef Mon Sep 17 00:00:00 2001 From: Allison Karlitskaya Date: Wed, 16 Apr 2025 13:32:26 +0200 Subject: [PATCH 8/9] src: make everything generic over FsVerityHashValue Convert all of the many places that we've been using Sha256HashValue for fs-verity hashes to be generic over FsVerityHashValue instead. Work the changes out across the codebase. Due to some missing features in Rust, this change (while fairly mechnical) is also pretty large. See for example: - it's not possible to omit type parameters from the declaration of a function when they are irrelevant to the body of the function - https://github.com/rust-lang/rfcs/issues/424 Signed-off-by: Allison Karlitskaya --- src/bin/cfsctl.rs | 4 +- src/bin/composefs-setup-root.rs | 13 ++--- src/dumpfile.rs | 19 ++++--- src/erofs/reader.rs | 10 ++-- src/erofs/writer.rs | 42 +++++++++------- src/fs.rs | 75 +++++++++++++++++++++------- src/image.rs | 69 ++++++++++++++------------ src/oci/image.rs | 40 +++++++++------ src/oci/mod.rs | 87 +++++++++++++++++++-------------- src/oci/tar.rs | 24 +++++---- src/repository.rs | 85 +++++++++++++++----------------- src/selabel.rs | 17 ++++--- src/splitstream.rs | 68 +++++++++++++------------- tests/mkfs.rs | 18 ++++--- 14 files changed, 327 insertions(+), 244 deletions(-) diff --git a/src/bin/cfsctl.rs b/src/bin/cfsctl.rs index 0f5c5064..8cca5aaa 100644 --- a/src/bin/cfsctl.rs +++ b/src/bin/cfsctl.rs @@ -111,7 +111,7 @@ fn main() -> Result<()> { let args = App::parse(); - let repo = (if let Some(path) = args.repo { + let repo: Repository = (if let Some(path) = args.repo { Repository::open_path(CWD, path) } else if args.system { Repository::open_system() @@ -193,7 +193,7 @@ fn main() -> Result<()> { println!("{}", image_id.to_hex()); } Command::CreateDumpfile { ref path } => { - composefs::fs::create_dumpfile(path)?; + composefs::fs::create_dumpfile::(path)?; } Command::Mount { name, mountpoint } => { repo.mount(&name, &mountpoint)?; diff --git a/src/bin/composefs-setup-root.rs b/src/bin/composefs-setup-root.rs index ebd1c14e..479aa759 100644 --- a/src/bin/composefs-setup-root.rs +++ b/src/bin/composefs-setup-root.rs @@ -167,7 +167,7 @@ fn open_root_fs(path: &Path) -> Result { } fn mount_composefs_image(sysroot: &OwnedFd, name: &str) -> Result { - let repo = Repository::open_path(sysroot, "composefs")?; + let repo = Repository::::open_path(sysroot, "composefs")?; let image = repo.open_image(name)?; composefs_fsmount(image, name, repo.object_dir()?).context("Failed to mount composefs image") } @@ -200,11 +200,11 @@ fn mount_subdir( } // Implementation -fn parse_composefs_cmdline(cmdline: &[u8]) -> Result { +fn parse_composefs_cmdline(cmdline: &[u8]) -> Result { // TODO?: officially we need to understand quoting with double-quotes... for part in cmdline.split(|c| c.is_ascii_whitespace()) { if let Some(digest) = part.strip_prefix(b"composefs=") { - return Sha256HashValue::from_hex(digest).context("Parsing composefs="); + return H::from_hex(digest).context("Parsing composefs="); } } bail!("Unable to find composefs= cmdline parameter"); @@ -236,7 +236,7 @@ fn setup_root(args: Args) -> Result<()> { Some(cmdline) => cmdline.as_bytes(), None => &std::fs::read("/proc/cmdline")?, }; - let image = parse_composefs_cmdline(cmdline)?.to_hex(); + let image = parse_composefs_cmdline::(cmdline)?.to_hex(); let new_root = match args.root_fs { Some(path) => open_root_fs(&path).context("Failed to clone specified root fs")?, @@ -290,11 +290,12 @@ mod test { fn test_parse() { let failing = ["", "foo", "composefs", "composefs=foo"]; for case in failing { - assert!(parse_composefs_cmdline(case.as_bytes()).is_err()); + assert!(parse_composefs_cmdline::(case.as_bytes()).is_err()); } let digest = "8b7df143d91c716ecfa5fc1730022f6b421b05cedee8fd52b1fc65a96030ad52"; similar_asserts::assert_eq!( - parse_composefs_cmdline(format!("composefs={digest}").as_bytes()).unwrap(), + parse_composefs_cmdline::(format!("composefs={digest}").as_bytes()) + .unwrap(), Sha256HashValue::from_hex(digest).unwrap() ); } diff --git a/src/dumpfile.rs b/src/dumpfile.rs index f60f459d..f97d4364 100644 --- a/src/dumpfile.rs +++ b/src/dumpfile.rs @@ -101,11 +101,11 @@ pub fn write_directory( ) } -pub fn write_leaf( +pub fn write_leaf( writer: &mut impl fmt::Write, path: &Path, stat: &Stat, - content: &LeafContent, + content: &LeafContent, nlink: usize, ) -> fmt::Result { match content { @@ -204,8 +204,8 @@ pub fn write_hardlink(writer: &mut impl fmt::Write, path: &Path, target: &OsStr) Ok(()) } -struct DumpfileWriter<'a, W: Write> { - hardlinks: HashMap<*const Leaf, OsString>, +struct DumpfileWriter<'a, W: Write, ObjectID: FsVerityHashValue> { + hardlinks: HashMap<*const Leaf, OsString>, writer: &'a mut W, } @@ -215,7 +215,7 @@ fn writeln_fmt(writer: &mut impl Write, f: impl Fn(&mut String) -> fmt::Result) Ok(writeln!(writer, "{}", tmp)?) } -impl<'a, W: Write> DumpfileWriter<'a, W> { +impl<'a, W: Write, ObjectID: FsVerityHashValue> DumpfileWriter<'a, W, ObjectID> { fn new(writer: &'a mut W) -> Self { Self { hardlinks: HashMap::new(), @@ -223,7 +223,7 @@ impl<'a, W: Write> DumpfileWriter<'a, W> { } } - fn write_dir(&mut self, path: &mut PathBuf, dir: &Directory) -> Result<()> { + fn write_dir(&mut self, path: &mut PathBuf, dir: &Directory) -> Result<()> { // nlink is 2 + number of subdirectories // this is also true for the root dir since '..' is another self-ref let nlink = dir.inodes().fold(2, |count, inode| { @@ -256,7 +256,7 @@ impl<'a, W: Write> DumpfileWriter<'a, W> { Ok(()) } - fn write_leaf(&mut self, path: &Path, leaf: &Rc) -> Result<()> { + fn write_leaf(&mut self, path: &Path, leaf: &Rc>) -> Result<()> { let nlink = Rc::strong_count(leaf); if nlink > 1 { @@ -276,7 +276,10 @@ impl<'a, W: Write> DumpfileWriter<'a, W> { } } -pub fn write_dumpfile(writer: &mut W, fs: &FileSystem) -> Result<()> { +pub fn write_dumpfile( + writer: &mut impl Write, + fs: &FileSystem, +) -> Result<()> { // default pipe capacity on Linux is 16 pages (65536 bytes), but // sometimes the BufWriter will write more than its capacity... let mut buffer = BufWriter::with_capacity(32768, writer); diff --git a/src/erofs/reader.rs b/src/erofs/reader.rs index 6b668a76..d18422b1 100644 --- a/src/erofs/reader.rs +++ b/src/erofs/reader.rs @@ -12,7 +12,7 @@ use super::{ InodeXAttrHeader, ModeField, Superblock, XAttrHeader, }, }; -use crate::fsverity::Sha256HashValue; +use crate::fsverity::FsVerityHashValue; pub fn round_up(n: usize, to: usize) -> usize { (n + to - 1) & !(to - 1) @@ -486,13 +486,13 @@ pub enum ErofsReaderError { type ReadResult = Result; #[derive(Debug)] -pub struct ObjectCollector { +pub struct ObjectCollector { visited_nids: HashSet, nids_to_visit: BTreeSet, - objects: HashSet, + objects: HashSet, } -impl ObjectCollector { +impl ObjectCollector { fn visit_xattr(&mut self, attr: &XAttr) { // This is the index of "trusted". See XATTR_PREFIXES in format.rs. if attr.header.name_index != 4 { @@ -552,7 +552,7 @@ impl ObjectCollector { } } -pub fn collect_objects(image: &[u8]) -> ReadResult> { +pub fn collect_objects(image: &[u8]) -> ReadResult> { let img = Image::open(image); let mut this = ObjectCollector { visited_nids: HashSet::new(), diff --git a/src/erofs/writer.rs b/src/erofs/writer.rs index 89a19e78..53e13ad7 100644 --- a/src/erofs/writer.rs +++ b/src/erofs/writer.rs @@ -82,21 +82,21 @@ struct Directory<'a> { } #[derive(Debug)] -struct Leaf<'a> { - content: &'a image::LeafContent, +struct Leaf<'a, ObjectID: FsVerityHashValue> { + content: &'a image::LeafContent, nlink: usize, } #[derive(Debug)] -enum InodeContent<'a> { +enum InodeContent<'a, ObjectID: FsVerityHashValue> { Directory(Directory<'a>), - Leaf(Leaf<'a>), + Leaf(Leaf<'a, ObjectID>), } -struct Inode<'a> { +struct Inode<'a, ObjectID: FsVerityHashValue> { stat: &'a image::Stat, xattrs: InodeXAttrs, - content: InodeContent<'a>, + content: InodeContent<'a, ObjectID>, } impl XAttr { @@ -263,7 +263,7 @@ impl<'a> Directory<'a> { } } -impl Leaf<'_> { +impl Leaf<'_, ObjectID> { fn inode_meta(&self) -> (format::DataLayout, u32, u64, usize) { let (layout, u, size) = match &self.content { image::LeafContent::Regular(image::RegularFile::Inline(data)) => { @@ -299,7 +299,7 @@ impl Leaf<'_> { } } -impl Inode<'_> { +impl Inode<'_, ObjectID> { fn file_type(&self) -> format::FileType { match &self.content { InodeContent::Directory(..) => format::FileType::Directory, @@ -395,13 +395,13 @@ impl Inode<'_> { } } -struct InodeCollector<'a> { - inodes: Vec>, - hardlinks: HashMap<*const image::Leaf, usize>, +struct InodeCollector<'a, ObjectID: FsVerityHashValue> { + inodes: Vec>, + hardlinks: HashMap<*const image::Leaf, usize>, } -impl<'a> InodeCollector<'a> { - fn push_inode(&mut self, stat: &'a image::Stat, content: InodeContent<'a>) -> usize { +impl<'a, ObjectID: FsVerityHashValue> InodeCollector<'a, ObjectID> { + fn push_inode(&mut self, stat: &'a image::Stat, content: InodeContent<'a, ObjectID>) -> usize { let mut xattrs = InodeXAttrs::default(); // We need to record extra xattrs for some files. These come first. @@ -442,7 +442,7 @@ impl<'a> InodeCollector<'a> { inode } - fn collect_leaf(&mut self, leaf: &'a Rc) -> usize { + fn collect_leaf(&mut self, leaf: &'a Rc>) -> usize { let nlink = Rc::strong_count(leaf); if nlink > 1 { @@ -481,7 +481,7 @@ impl<'a> InodeCollector<'a> { entries.insert(point, entry); } - fn collect_dir(&mut self, dir: &'a image::Directory, parent: usize) -> usize { + fn collect_dir(&mut self, dir: &'a image::Directory, parent: usize) -> usize { // The root inode number needs to fit in a u16. That more or less compels us to write the // directory inode before the inode of the children of the directory. Reserve a slot. let me = self.push_inode(&dir.stat, InodeContent::Directory(Directory::default())); @@ -509,7 +509,7 @@ impl<'a> InodeCollector<'a> { me } - pub fn collect(fs: &'a image::FileSystem) -> Vec> { + pub fn collect(fs: &'a image::FileSystem) -> Vec> { let mut this = Self { inodes: vec![], hardlinks: HashMap::new(), @@ -525,7 +525,7 @@ impl<'a> InodeCollector<'a> { /// Takes a list of inodes where each inode contains only local xattr values, determines which /// xattrs (key, value) pairs appear more than once, and shares them. -fn share_xattrs(inodes: &mut [Inode]) -> Vec { +fn share_xattrs(inodes: &mut [Inode]) -> Vec { let mut xattrs: BTreeMap = BTreeMap::new(); // Collect all xattrs from the inodes @@ -563,7 +563,11 @@ fn share_xattrs(inodes: &mut [Inode]) -> Vec { xattrs.into_keys().collect() } -fn write_erofs(output: &mut impl Output, inodes: &[Inode], xattrs: &[XAttr]) { +fn write_erofs( + output: &mut impl Output, + inodes: &[Inode], + xattrs: &[XAttr], +) { // Write composefs header output.note_offset(Offset::Header); output.write_struct(format::ComposefsHeader { @@ -684,7 +688,7 @@ impl Output for FirstPass { } } -pub fn mkfs_erofs(fs: &image::FileSystem) -> Box<[u8]> { +pub fn mkfs_erofs(fs: &image::FileSystem) -> Box<[u8]> { // Create the intermediate representation: flattened inodes and shared xattrs let mut inodes = InodeCollector::collect(fs); let xattrs = share_xattrs(&mut inodes); diff --git a/src/fs.rs b/src/fs.rs index f504e69e..3cc0e28d 100644 --- a/src/fs.rs +++ b/src/fs.rs @@ -24,7 +24,7 @@ use rustix::{ use zerocopy::IntoBytes; use crate::{ - fsverity::{compute_verity, Sha256HashValue}, + fsverity::{compute_verity, FsVerityHashValue}, image::{Directory, FileSystem, Inode, Leaf, LeafContent, RegularFile, Stat}, repository::Repository, selabel::selabel, @@ -70,11 +70,11 @@ fn set_file_contents(dirfd: &OwnedFd, name: &OsStr, stat: &Stat, data: &[u8]) -> Ok(()) } -fn write_directory( - dir: &Directory, +fn write_directory( + dir: &Directory, dirfd: &OwnedFd, name: &OsStr, - repo: &Repository, + repo: &Repository, ) -> Result<()> { match mkdirat(dirfd, name, dir.stat.st_mode.into()) { Ok(()) | Err(Errno::EXIST) => {} @@ -85,7 +85,12 @@ fn write_directory( write_directory_contents(dir, &fd, repo) } -fn write_leaf(leaf: &Leaf, dirfd: &OwnedFd, name: &OsStr, repo: &Repository) -> Result<()> { +fn write_leaf( + leaf: &Leaf, + dirfd: &OwnedFd, + name: &OsStr, + repo: &Repository, +) -> Result<()> { let mode = leaf.stat.st_mode.into(); match &leaf.content { @@ -111,7 +116,11 @@ fn write_leaf(leaf: &Leaf, dirfd: &OwnedFd, name: &OsStr, repo: &Repository) -> Ok(()) } -fn write_directory_contents(dir: &Directory, fd: &OwnedFd, repo: &Repository) -> Result<()> { +fn write_directory_contents( + dir: &Directory, + fd: &OwnedFd, + repo: &Repository, +) -> Result<()> { for (name, inode) in dir.entries() { match inode { Inode::Directory(ref dir) => write_directory(dir, fd, name, repo), @@ -123,19 +132,23 @@ fn write_directory_contents(dir: &Directory, fd: &OwnedFd, repo: &Repository) -> } // NB: hardlinks not supported -pub fn write_to_path(repo: &Repository, dir: &Directory, output_dir: &Path) -> Result<()> { +pub fn write_to_path( + repo: &Repository, + dir: &Directory, + output_dir: &Path, +) -> Result<()> { let fd = openat(CWD, output_dir, OFlags::PATH | OFlags::DIRECTORY, 0.into())?; write_directory_contents(dir, &fd, repo) } #[derive(Debug)] -pub struct FilesystemReader<'repo> { - repo: Option<&'repo Repository>, - inodes: HashMap<(u64, u64), Rc>, +pub struct FilesystemReader<'repo, ObjectID: FsVerityHashValue> { + repo: Option<&'repo Repository>, + inodes: HashMap<(u64, u64), Rc>>, root_mtime: Option, } -impl FilesystemReader<'_> { +impl FilesystemReader<'_, ObjectID> { fn read_xattrs(fd: &OwnedFd) -> Result, Box<[u8]>>> { // flistxattr() and fgetxattr() don't work with with O_PATH fds, so go via /proc/self/fd. // Note: we want the symlink-following version of this call, which produces the correct @@ -181,7 +194,11 @@ impl FilesystemReader<'_> { )) } - fn read_leaf_content(&mut self, fd: OwnedFd, buf: rustix::fs::Stat) -> Result { + fn read_leaf_content( + &mut self, + fd: OwnedFd, + buf: rustix::fs::Stat, + ) -> Result> { let content = match FileType::from_raw_mode(buf.st_mode) { FileType::Directory | FileType::Unknown => unreachable!(), FileType::RegularFile => { @@ -212,7 +229,12 @@ impl FilesystemReader<'_> { Ok(content) } - fn read_leaf(&mut self, dirfd: &OwnedFd, name: &OsStr, ifmt: FileType) -> Result> { + fn read_leaf( + &mut self, + dirfd: &OwnedFd, + name: &OsStr, + ifmt: FileType, + ) -> Result>> { let oflags = match ifmt { FileType::RegularFile => OFlags::RDONLY, _ => OFlags::PATH, @@ -241,7 +263,11 @@ impl FilesystemReader<'_> { } } - pub fn read_directory(&mut self, dirfd: impl AsFd, name: &OsStr) -> Result { + pub fn read_directory( + &mut self, + dirfd: impl AsFd, + name: &OsStr, + ) -> Result> { let fd = openat( dirfd, name, @@ -268,7 +294,12 @@ impl FilesystemReader<'_> { Ok(directory) } - fn read_inode(&mut self, dirfd: &OwnedFd, name: &OsStr, ifmt: FileType) -> Result { + fn read_inode( + &mut self, + dirfd: &OwnedFd, + name: &OsStr, + ifmt: FileType, + ) -> Result> { if ifmt == FileType::Directory { let dir = self.read_directory(dirfd, name)?; Ok(Inode::Directory(Box::new(dir))) @@ -279,7 +310,10 @@ impl FilesystemReader<'_> { } } -pub fn read_from_path(path: &Path, repo: Option<&Repository>) -> Result { +pub fn read_from_path( + path: &Path, + repo: Option<&Repository>, +) -> Result> { let mut reader = FilesystemReader { repo, inodes: HashMap::new(), @@ -300,7 +334,10 @@ pub fn read_from_path(path: &Path, repo: Option<&Repository>) -> Result) -> Result { +pub fn create_image( + path: &Path, + repo: Option<&Repository>, +) -> Result { let fs = read_from_path(path, repo)?; let image = crate::erofs::writer::mkfs_erofs(&fs); if let Some(repo) = repo { @@ -310,8 +347,8 @@ pub fn create_image(path: &Path, repo: Option<&Repository>) -> Result Result<()> { - let fs = read_from_path(path, None)?; +pub fn create_dumpfile(path: &Path) -> Result<()> { + let fs = read_from_path::(path, None)?; super::dumpfile::write_dumpfile(&mut std::io::stdout(), &fs) } diff --git a/src/image.rs b/src/image.rs index 614916c1..15a8498f 100644 --- a/src/image.rs +++ b/src/image.rs @@ -8,7 +8,7 @@ use std::{ use thiserror::Error; -use crate::fsverity::Sha256HashValue; +use crate::fsverity::FsVerityHashValue; #[derive(Debug)] pub struct Stat { @@ -20,14 +20,14 @@ pub struct Stat { } #[derive(Debug)] -pub enum RegularFile { +pub enum RegularFile { Inline(Box<[u8]>), - External(Sha256HashValue, u64), + External(ObjectID, u64), } #[derive(Debug)] -pub enum LeafContent { - Regular(RegularFile), +pub enum LeafContent { + Regular(RegularFile), BlockDevice(u64), CharacterDevice(u64), Fifo, @@ -36,21 +36,21 @@ pub enum LeafContent { } #[derive(Debug)] -pub struct Leaf { +pub struct Leaf { pub stat: Stat, - pub content: LeafContent, + pub content: LeafContent, } #[derive(Debug)] -pub struct Directory { +pub struct Directory { pub stat: Stat, - entries: BTreeMap, Inode>, + entries: BTreeMap, Inode>, } #[derive(Debug)] -pub enum Inode { - Directory(Box), - Leaf(Rc), +pub enum Inode { + Directory(Box>), + Leaf(Rc>), } #[derive(Error, Debug)] @@ -67,7 +67,7 @@ pub enum ImageError { IsNotRegular(Box), } -impl Inode { +impl Inode { pub fn stat(&self) -> &Stat { match self { Inode::Directory(dir) => &dir.stat, @@ -76,7 +76,7 @@ impl Inode { } } -impl Directory { +impl Directory { pub fn new(stat: Stat) -> Self { Self { stat, @@ -85,7 +85,7 @@ impl Directory { } /// Iterates over all inodes in the current directory, in no particular order. - pub fn inodes(&self) -> impl Iterator + use<'_> { + pub fn inodes(&self) -> impl Iterator> + use<'_, ObjectID> { self.entries.values() } @@ -96,7 +96,8 @@ impl Directory { /// point. /// /// ``` - /// let fs = composefs::image::FileSystem::new(); + /// use composefs::{image::FileSystem, fsverity::Sha256HashValue}; + /// let fs = FileSystem::::new(); /// /// // populate the fs... /// @@ -104,13 +105,15 @@ impl Directory { /// // name: &OsStr, inode: &Inode /// } /// ``` - pub fn entries(&self) -> impl Iterator + use<'_> { + pub fn entries(&self) -> impl Iterator)> + use<'_, ObjectID> { self.entries.iter().map(|(k, v)| (k.as_ref(), v)) } /// Iterates over all entries in the current directory, in asciibetical order of name. The /// iterator returns pairs of `(&OsStr, &Inode)`. - pub fn sorted_entries(&self) -> impl Iterator + use<'_> { + pub fn sorted_entries( + &self, + ) -> impl Iterator)> + use<'_, ObjectID> { self.entries.iter().map(|(k, v)| (k.as_ref(), v)) } @@ -132,7 +135,7 @@ impl Directory { /// On success, this returns a reference to the named directory. /// /// On failure, can return any number of errors from ImageError. - pub fn get_directory(&self, pathname: &OsStr) -> Result<&Directory, ImageError> { + pub fn get_directory(&self, pathname: &OsStr) -> Result<&Directory, ImageError> { let path = Path::new(pathname); let mut dir = self; @@ -156,7 +159,10 @@ impl Directory { /// Gets a mutable reference to a subdirectory of this directory. /// /// This is the mutable version of `Directory::get_directory()`. - pub fn get_directory_mut(&mut self, pathname: &OsStr) -> Result<&mut Directory, ImageError> { + pub fn get_directory_mut( + &mut self, + pathname: &OsStr, + ) -> Result<&mut Directory, ImageError> { let path = Path::new(pathname); let mut dir = self; @@ -200,7 +206,7 @@ impl Directory { pub fn split<'d, 'n>( &'d self, pathname: &'n OsStr, - ) -> Result<(&'d Directory, &'n OsStr), ImageError> { + ) -> Result<(&'d Directory, &'n OsStr), ImageError> { let path = Path::new(pathname); let Some(filename) = path.file_name() else { @@ -222,7 +228,7 @@ impl Directory { pub fn split_mut<'d, 'n>( &'d mut self, pathname: &'n OsStr, - ) -> Result<(&'d mut Directory, &'n OsStr), ImageError> { + ) -> Result<(&'d mut Directory, &'n OsStr), ImageError> { let path = Path::new(pathname); let Some(filename) = path.file_name() else { @@ -252,7 +258,7 @@ impl Directory { /// is returned. /// /// On failure, can return any number of errors from ImageError. - pub fn ref_leaf(&self, filename: &OsStr) -> Result, ImageError> { + pub fn ref_leaf(&self, filename: &OsStr) -> Result>, ImageError> { match self.entries.get(filename) { Some(Inode::Leaf(leaf)) => Ok(Rc::clone(leaf)), Some(Inode::Directory(..)) => Err(ImageError::IsADirectory(Box::from(filename))), @@ -275,7 +281,10 @@ impl Directory { /// * an external reference, with size information /// /// On failure, can return any number of errors from ImageError. - pub fn get_file<'a>(&'a self, filename: &OsStr) -> Result<&'a RegularFile, ImageError> { + pub fn get_file<'a>( + &'a self, + filename: &OsStr, + ) -> Result<&'a RegularFile, ImageError> { match self.entries.get(filename) { Some(Inode::Leaf(leaf)) => match &leaf.content { LeafContent::Regular(file) => Ok(file), @@ -301,7 +310,7 @@ impl Directory { /// * `filename`: the filename in the current directory. If you need to support full /// pathnames then you should call `Directory::split()` first. /// * `inode`: the inode to store under the `filename` - pub fn merge(&mut self, filename: &OsStr, inode: Inode) { + pub fn merge(&mut self, filename: &OsStr, inode: Inode) { // If we're putting a directory on top of a directory, then update the stat information but // keep the old entries in place. if let Inode::Directory(new_dir) = inode { @@ -328,7 +337,7 @@ impl Directory { /// * `filename`: the filename in the current directory. If you need to support full /// pathnames then you should call `Directory::split()` first. /// * `inode`: the inode to store under the `filename` - pub fn insert(&mut self, filename: &OsStr, inode: Inode) { + pub fn insert(&mut self, filename: &OsStr, inode: Inode) { self.entries.insert(Box::from(filename), inode); } @@ -365,17 +374,17 @@ impl Directory { } #[derive(Debug)] -pub struct FileSystem { - pub root: Directory, +pub struct FileSystem { + pub root: Directory, } -impl Default for FileSystem { +impl Default for FileSystem { fn default() -> Self { Self::new() } } -impl FileSystem { +impl FileSystem { pub fn new() -> Self { Self { root: Directory::new(Stat { diff --git a/src/oci/image.rs b/src/oci/image.rs index 4401b681..b30fd210 100644 --- a/src/oci/image.rs +++ b/src/oci/image.rs @@ -6,7 +6,7 @@ use oci_spec::image::ImageConfiguration; use crate::{ dumpfile::write_dumpfile, erofs::writer::mkfs_erofs, - fsverity::Sha256HashValue, + fsverity::FsVerityHashValue, image::{Directory, FileSystem, Inode, Leaf}, oci::{ self, @@ -16,7 +16,10 @@ use crate::{ selabel::selabel, }; -pub fn process_entry(filesystem: &mut FileSystem, entry: TarEntry) -> Result<()> { +pub fn process_entry( + filesystem: &mut FileSystem, + entry: TarEntry, +) -> Result<()> { let inode = match entry.item { TarItem::Directory => Inode::Directory(Box::from(Directory::new(entry.stat))), TarItem::Leaf(content) => Inode::Leaf(Rc::new(Leaf { @@ -46,8 +49,11 @@ pub fn process_entry(filesystem: &mut FileSystem, entry: TarEntry) -> Result<()> Ok(()) } -pub fn compose_filesystem(repo: &Repository, layers: &[String]) -> Result { - let mut filesystem = FileSystem::new(); +pub fn compose_filesystem( + repo: &Repository, + layers: &[String], +) -> Result> { + let mut filesystem = FileSystem::::new(); for layer in layers { let mut split_stream = repo.open_stream(layer, None)?; @@ -62,19 +68,22 @@ pub fn compose_filesystem(repo: &Repository, layers: &[String]) -> Result Result<()> { +pub fn create_dumpfile( + repo: &Repository, + layers: &[String], +) -> Result<()> { let filesystem = compose_filesystem(repo, layers)?; let mut stdout = std::io::stdout(); write_dumpfile(&mut stdout, &filesystem)?; Ok(()) } -pub fn create_image( - repo: &Repository, +pub fn create_image( + repo: &Repository, config: &str, name: Option<&str>, - verity: Option<&Sha256HashValue>, -) -> Result { + verity: Option<&ObjectID>, +) -> Result { let mut filesystem = FileSystem::new(); let mut config_stream = repo.open_stream(config, verity)?; @@ -99,12 +108,15 @@ pub fn create_image( #[cfg(test)] mod test { - use crate::image::{LeafContent, RegularFile, Stat}; + use crate::{ + fsverity::Sha256HashValue, + image::{LeafContent, RegularFile, Stat}, + }; use std::{cell::RefCell, collections::BTreeMap, io::BufRead, path::PathBuf}; use super::*; - fn file_entry(path: &str) -> oci::tar::TarEntry { + fn file_entry(path: &str) -> oci::tar::TarEntry { oci::tar::TarEntry { path: PathBuf::from(path), stat: Stat { @@ -118,7 +130,7 @@ mod test { } } - fn dir_entry(path: &str) -> oci::tar::TarEntry { + fn dir_entry(path: &str) -> oci::tar::TarEntry { oci::tar::TarEntry { path: PathBuf::from(path), stat: Stat { @@ -132,7 +144,7 @@ mod test { } } - fn assert_files(fs: &FileSystem, expected: &[&str]) -> Result<()> { + fn assert_files(fs: &FileSystem, expected: &[&str]) -> Result<()> { let mut out = vec![]; write_dumpfile(&mut out, fs)?; let actual: Vec = out @@ -146,7 +158,7 @@ mod test { #[test] fn test_process_entry() -> Result<()> { - let mut fs = FileSystem::new(); + let mut fs = FileSystem::::new(); // both with and without leading slash should be supported process_entry(&mut fs, dir_entry("/a"))?; diff --git a/src/oci/mod.rs b/src/oci/mod.rs index 8093fc45..e740dbff 100644 --- a/src/oci/mod.rs +++ b/src/oci/mod.rs @@ -15,23 +15,26 @@ use tokio::io::AsyncReadExt; use crate::{ fs::write_to_path, - fsverity::{FsVerityHashValue, Sha256HashValue}, + fsverity::FsVerityHashValue, oci::tar::{get_entry, split_async}, repository::Repository, splitstream::DigestMap, util::{parse_sha256, Sha256Digest}, }; -pub fn import_layer( - repo: &Repository, +pub fn import_layer( + repo: &Repository, sha256: &Sha256Digest, name: Option<&str>, tar_stream: &mut impl Read, -) -> Result { +) -> Result { repo.ensure_stream(sha256, |writer| tar::split(tar_stream, writer), name) } -pub fn ls_layer(repo: &Repository, name: &str) -> Result<()> { +pub fn ls_layer( + repo: &Repository, + name: &str, +) -> Result<()> { let mut split_stream = repo.open_stream(name, None)?; while let Some(entry) = get_entry(&mut split_stream)? { @@ -41,8 +44,8 @@ pub fn ls_layer(repo: &Repository, name: &str) -> Result<()> { Ok(()) } -struct ImageOp<'repo> { - repo: &'repo Repository, +struct ImageOp<'repo, ObjectID: FsVerityHashValue> { + repo: &'repo Repository, proxy: ImageProxy, img: OpenedImage, progress: MultiProgress, @@ -62,10 +65,10 @@ fn sha256_from_digest(digest: &str) -> Result { } } -type ContentAndVerity = (Sha256Digest, Sha256HashValue); +type ContentAndVerity = (Sha256Digest, ObjectID); -impl<'repo> ImageOp<'repo> { - async fn new(repo: &'repo Repository, imgref: &str) -> Result { +impl<'repo, ObjectID: FsVerityHashValue> ImageOp<'repo, ObjectID> { + async fn new(repo: &'repo Repository, imgref: &str) -> Result { // See https://github.com/containers/skopeo/issues/2563 let skopeo_cmd = if imgref.starts_with("containers-storage:") { let mut cmd = Command::new("podman"); @@ -95,7 +98,7 @@ impl<'repo> ImageOp<'repo> { &self, layer_sha256: &Sha256Digest, descriptor: &Descriptor, - ) -> Result { + ) -> Result { // We need to use the per_manifest descriptor to download the compressed layer but it gets // stored in the repository via the per_config descriptor. Our return value is the // fsverity digest for the corresponding splitstream. @@ -142,7 +145,7 @@ impl<'repo> ImageOp<'repo> { &self, manifest_layers: &[Descriptor], descriptor: &Descriptor, - ) -> Result { + ) -> Result> { let config_sha256 = sha256_from_descriptor(descriptor)?; if let Some(config_id) = self.repo.check_stream(&config_sha256)? { // We already got this config? Nice. @@ -189,7 +192,7 @@ impl<'repo> ImageOp<'repo> { } } - pub async fn pull(&self) -> Result { + pub async fn pull(&self) -> Result> { let (_manifest_digest, raw_manifest) = self .proxy .fetch_manifest_raw_oci(&self.img) @@ -209,7 +212,11 @@ impl<'repo> ImageOp<'repo> { /// Pull the target image, and add the provided tag. If this is a mountable /// image (i.e. not an artifact), it is *not* unpacked by default. -pub async fn pull(repo: &Repository, imgref: &str, reference: Option<&str>) -> Result<()> { +pub async fn pull( + repo: &Repository, + imgref: &str, + reference: Option<&str>, +) -> Result<()> { let op = ImageOp::new(repo, imgref).await?; let (sha256, id) = op .pull() @@ -224,11 +231,11 @@ pub async fn pull(repo: &Repository, imgref: &str, reference: Option<&str>) -> R Ok(()) } -pub fn open_config( - repo: &Repository, +pub fn open_config( + repo: &Repository, name: &str, - verity: Option<&Sha256HashValue>, -) -> Result<(ImageConfiguration, DigestMap)> { + verity: Option<&ObjectID>, +) -> Result<(ImageConfiguration, DigestMap)> { let id = match verity { Some(id) => id, None => { @@ -251,10 +258,10 @@ fn hash(bytes: &[u8]) -> Sha256Digest { context.finalize().into() } -pub fn open_config_shallow( - repo: &Repository, +pub fn open_config_shallow( + repo: &Repository, name: &str, - verity: Option<&Sha256HashValue>, + verity: Option<&ObjectID>, ) -> Result { match verity { // with verity deep opens are just as fast as shallow ones @@ -272,11 +279,11 @@ pub fn open_config_shallow( } } -pub fn write_config( - repo: &Repository, +pub fn write_config( + repo: &Repository, config: &ImageConfiguration, - refs: DigestMap, -) -> Result { + refs: DigestMap, +) -> Result> { let json = config.to_string()?; let json_bytes = json.as_bytes(); let sha256 = hash(json_bytes); @@ -286,11 +293,11 @@ pub fn write_config( Ok((sha256, id)) } -pub fn seal( - repo: &Repository, +pub fn seal( + repo: &Repository, name: &str, - verity: Option<&Sha256HashValue>, -) -> Result { + verity: Option<&ObjectID>, +) -> Result> { let (mut config, refs) = open_config(repo, name, verity)?; let mut myconfig = config.config().clone().context("no config!")?; let labels = myconfig.labels_mut().get_or_insert_with(HashMap::new); @@ -300,11 +307,11 @@ pub fn seal( write_config(repo, &config, refs) } -pub fn mount( - repo: &Repository, +pub fn mount( + repo: &Repository, name: &str, mountpoint: &str, - verity: Option<&Sha256HashValue>, + verity: Option<&ObjectID>, ) -> Result<()> { let config = open_config_shallow(repo, name, verity)?; let Some(id) = config.get_config_annotation("containers.composefs.fsverity") else { @@ -313,7 +320,11 @@ pub fn mount( repo.mount(id, mountpoint) } -pub fn meta_layer(repo: &Repository, name: &str, verity: Option<&Sha256HashValue>) -> Result<()> { +pub fn meta_layer( + repo: &Repository, + name: &str, + verity: Option<&ObjectID>, +) -> Result<()> { let (config, refs) = open_config(repo, name, verity)?; let ids = config.rootfs().diff_ids(); @@ -330,10 +341,10 @@ pub fn meta_layer(repo: &Repository, name: &str, verity: Option<&Sha256HashValue } } -pub fn prepare_boot( - repo: &Repository, +pub fn prepare_boot( + repo: &Repository, name: &str, - verity: Option<&Sha256HashValue>, + verity: Option<&ObjectID>, output_dir: &Path, ) -> Result<()> { let (config, refs) = open_config(repo, name, verity)?; @@ -377,7 +388,7 @@ mod test { use rustix::fs::CWD; use sha2::{Digest, Sha256}; - use crate::{repository::Repository, test::tempdir}; + use crate::{fsverity::Sha256HashValue, repository::Repository, test::tempdir}; use super::*; @@ -410,7 +421,7 @@ mod test { let layer_id: [u8; 32] = context.finalize().into(); let repo_dir = tempdir(); - let repo = Repository::open_path(CWD, &repo_dir).unwrap(); + let repo = Repository::::open_path(CWD, &repo_dir).unwrap(); let id = import_layer(&repo, &layer_id, Some("name"), &mut layer.as_slice()).unwrap(); let mut dump = String::new(); diff --git a/src/oci/tar.rs b/src/oci/tar.rs index 2db4818f..08bbada5 100644 --- a/src/oci/tar.rs +++ b/src/oci/tar.rs @@ -15,6 +15,7 @@ use tokio::io::{AsyncRead, AsyncReadExt}; use crate::{ dumpfile, + fsverity::FsVerityHashValue, image::{LeafContent, RegularFile, Stat}, splitstream::{SplitStreamData, SplitStreamReader, SplitStreamWriter}, util::{read_exactish, read_exactish_async}, @@ -42,7 +43,10 @@ async fn read_header_async(reader: &mut (impl AsyncRead + Unpin)) -> Result(tar_stream: &mut R, writer: &mut SplitStreamWriter) -> Result<()> { +pub fn split( + tar_stream: &mut impl Read, + writer: &mut SplitStreamWriter, +) -> Result<()> { while let Some(header) = read_header(tar_stream)? { // the header always gets stored as inline data writer.write_inline(header.as_bytes()); @@ -69,9 +73,9 @@ pub fn split(tar_stream: &mut R, writer: &mut SplitStreamWriter) -> Res Ok(()) } -pub async fn split_async( +pub async fn split_async( mut tar_stream: impl AsyncRead + Unpin, - writer: &mut SplitStreamWriter<'_>, + writer: &mut SplitStreamWriter<'_, ObjectID>, ) -> Result<()> { while let Some(header) = read_header_async(&mut tar_stream).await? { // the header always gets stored as inline data @@ -100,20 +104,20 @@ pub async fn split_async( } #[derive(Debug)] -pub enum TarItem { +pub enum TarItem { Directory, - Leaf(LeafContent), + Leaf(LeafContent), Hardlink(OsString), } #[derive(Debug)] -pub struct TarEntry { +pub struct TarEntry { pub path: PathBuf, pub stat: Stat, - pub item: TarItem, + pub item: TarItem, } -impl fmt::Display for TarEntry { +impl fmt::Display for TarEntry { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { match self.item { TarItem::Hardlink(ref target) => dumpfile::write_hardlink(fmt, &self.path, target), @@ -156,7 +160,9 @@ fn symlink_target_from_tar(pax: Option>, gnu: Vec, short: &[u8]) - } } -pub fn get_entry(reader: &mut SplitStreamReader) -> Result> { +pub fn get_entry( + reader: &mut SplitStreamReader, +) -> Result>> { let mut gnu_longlink: Vec = vec![]; let mut gnu_longname: Vec = vec![]; let mut pax_longlink: Option> = None; diff --git a/src/repository.rs b/src/repository.rs index 15a533f4..66d343fa 100644 --- a/src/repository.rs +++ b/src/repository.rs @@ -20,7 +20,6 @@ use sha2::{Digest, Sha256}; use crate::{ fsverity::{ compute_verity, enable_verity, ensure_verity_equal, measure_verity, FsVerityHashValue, - Sha256HashValue, }, mount::mount_composefs_at, splitstream::{DigestMap, SplitStreamReader, SplitStreamWriter}, @@ -28,17 +27,18 @@ use crate::{ }; #[derive(Debug)] -pub struct Repository { +pub struct Repository { repository: OwnedFd, + _data: std::marker::PhantomData, } -impl Drop for Repository { +impl Drop for Repository { fn drop(&mut self) { flock(&self.repository, FlockOperation::Unlock).expect("repository unlock failed"); } } -impl Repository { +impl Repository { pub fn object_dir(&self) -> ErrnoResult { self.openat( "objects", @@ -56,7 +56,10 @@ impl Repository { flock(&repository, FlockOperation::LockShared) .context("Cannot lock composefs repository")?; - Ok(Self { repository }) + Ok(Self { + repository, + _data: std::marker::PhantomData, + }) } pub fn open_user() -> Result { @@ -76,8 +79,8 @@ impl Repository { }) } - pub fn ensure_object(&self, data: &[u8]) -> Result { - let digest: Sha256HashValue = compute_verity(data); + pub fn ensure_object(&self, data: &[u8]) -> Result { + let digest: ObjectID = compute_verity(data); let dir = PathBuf::from(format!("objects/{}", digest.to_object_dir())); let file = dir.join(digest.to_object_basename()); @@ -102,7 +105,7 @@ impl Repository { let ro_fd = open(proc_self_fd(&fd), OFlags::RDONLY, Mode::empty())?; drop(fd); - enable_verity::(&ro_fd).context("Enabling verity digest")?; + enable_verity::(&ro_fd).context("Enabling verity digest")?; ensure_verity_equal(&ro_fd, &digest).context("Double-checking verity digest")?; if let Err(err) = linkat( @@ -121,11 +124,7 @@ impl Repository { Ok(digest) } - fn open_with_verity( - &self, - filename: &str, - expected_verity: &Sha256HashValue, - ) -> Result { + fn open_with_verity(&self, filename: &str, expected_verity: &ObjectID) -> Result { let fd = self.openat(filename, OFlags::RDONLY)?; ensure_verity_equal(&fd, expected_verity)?; Ok(fd) @@ -137,16 +136,16 @@ impl Repository { pub fn create_stream( &self, sha256: Option, - maps: Option, - ) -> SplitStreamWriter { + maps: Option>, + ) -> SplitStreamWriter { SplitStreamWriter::new(self, maps, sha256) } - fn format_object_path(id: &Sha256HashValue) -> String { + fn format_object_path(id: &ObjectID) -> String { format!("objects/{}", id.to_object_pathname()) } - pub fn has_stream(&self, sha256: &Sha256Digest) -> Result> { + pub fn has_stream(&self, sha256: &Sha256Digest) -> Result> { let stream_path = format!("streams/{}", hex::encode(sha256)); match readlinkat(&self.repository, &stream_path, []) { @@ -163,7 +162,7 @@ impl Repository { bytes.starts_with(b"../"), "stream symlink has incorrect prefix" ); - Ok(Some(Sha256HashValue::from_object_pathname(bytes)?)) + Ok(Some(ObjectID::from_object_pathname(bytes)?)) } Err(Errno::NOENT) => Ok(None), Err(err) => Err(err)?, @@ -171,10 +170,10 @@ impl Repository { } /// Basically the same as has_stream() except that it performs expensive verification - pub fn check_stream(&self, sha256: &Sha256Digest) -> Result> { + pub fn check_stream(&self, sha256: &Sha256Digest) -> Result> { match self.openat(&format!("streams/{}", hex::encode(sha256)), OFlags::RDONLY) { Ok(stream) => { - let measured_verity: Sha256HashValue = measure_verity(&stream)?; + let measured_verity: ObjectID = measure_verity(&stream)?; let mut context = Sha256::new(); let mut split_stream = SplitStreamReader::new(File::from(stream))?; @@ -204,9 +203,9 @@ impl Repository { pub fn write_stream( &self, - writer: SplitStreamWriter, + writer: SplitStreamWriter, reference: Option<&str>, - ) -> Result { + ) -> Result { let Some((.., ref sha256)) = writer.sha256 else { bail!("Writer doesn't have sha256 enabled"); }; @@ -249,9 +248,9 @@ impl Repository { pub fn ensure_stream( &self, sha256: &Sha256Digest, - callback: impl FnOnce(&mut SplitStreamWriter) -> Result<()>, + callback: impl FnOnce(&mut SplitStreamWriter) -> Result<()>, reference: Option<&str>, - ) -> Result { + ) -> Result { let stream_path = format!("streams/{}", hex::encode(sha256)); let object_id = match self.has_stream(sha256)? { @@ -278,8 +277,8 @@ impl Repository { pub fn open_stream( &self, name: &str, - verity: Option<&Sha256HashValue>, - ) -> Result> { + verity: Option<&ObjectID>, + ) -> Result> { let filename = format!("streams/{}", name); let file = File::from(if let Some(verity_hash) = verity { @@ -291,14 +290,14 @@ impl Repository { SplitStreamReader::new(file) } - pub fn open_object(&self, id: &Sha256HashValue) -> Result { - self.open_with_verity(&format!("objects/{}", id.to_object_pathname()), id) + pub fn open_object(&self, id: &ObjectID) -> Result { + self.open_with_verity(&Self::format_object_path(id), id) } pub fn merge_splitstream( &self, name: &str, - verity: Option<&Sha256HashValue>, + verity: Option<&ObjectID>, stream: &mut impl Write, ) -> Result<()> { let mut split_stream = self.open_stream(name, verity)?; @@ -312,7 +311,7 @@ impl Repository { } /// this function is not safe for untrusted users - pub fn write_image(&self, name: Option<&str>, data: &[u8]) -> Result { + pub fn write_image(&self, name: Option<&str>, data: &[u8]) -> Result { let object_id = self.ensure_object(data)?; let object_path = Self::format_object_path(&object_id); @@ -329,7 +328,7 @@ impl Repository { } /// this function is not safe for untrusted users - pub fn import_image(&self, name: &str, image: &mut R) -> Result { + pub fn import_image(&self, name: &str, image: &mut R) -> Result { let mut data = vec![]; image.read_to_end(&mut data)?; self.write_image(Some(name), &data) @@ -340,7 +339,7 @@ impl Repository { if !name.contains("/") { // A name with no slashes in it is taken to be a sha256 fs-verity digest - ensure_verity_equal(&image, &Sha256HashValue::from_hex(name)?)?; + ensure_verity_equal(&image, &ObjectID::from_hex(name)?)?; } Ok(image) @@ -394,14 +393,12 @@ impl Repository { }) } - fn read_symlink_hashvalue(dirfd: &OwnedFd, name: &CStr) -> Result { + fn read_symlink_hashvalue(dirfd: &OwnedFd, name: &CStr) -> Result { let link_content = readlinkat(dirfd, name, [])?; - Ok(Sha256HashValue::from_object_pathname( - link_content.to_bytes(), - )?) + Ok(ObjectID::from_object_pathname(link_content.to_bytes())?) } - fn walk_symlinkdir(fd: OwnedFd, objects: &mut HashSet) -> Result<()> { + fn walk_symlinkdir(fd: OwnedFd, objects: &mut HashSet) -> Result<()> { for item in Dir::read_from(&fd)? { let entry = item?; // NB: the underlying filesystem must support returning filetype via direntry @@ -436,8 +433,8 @@ impl Repository { ) } - fn gc_category(&self, category: &str) -> Result> { - let mut objects = HashSet::::new(); + fn gc_category(&self, category: &str) -> Result> { + let mut objects = HashSet::new(); let category_fd = self.openat(category, OFlags::RDONLY | OFlags::DIRECTORY)?; @@ -475,7 +472,7 @@ impl Repository { Ok(objects) } - pub fn objects_for_image(&self, name: &str) -> Result> { + pub fn objects_for_image(&self, name: &str) -> Result> { let image = self.open_image(name)?; let mut data = vec![]; std::fs::File::from(image).read_to_end(&mut data)?; @@ -517,11 +514,9 @@ impl Repository { let entry = item?; let filename = entry.file_name(); if filename != c"." && filename != c".." { - let value = Sha256HashValue::from_object_dir_and_basename( - first_byte, - filename.to_bytes(), - )?; - if !objects.contains(&value) { + let id = + ObjectID::from_object_dir_and_basename(first_byte, filename.to_bytes())?; + if !objects.contains(&id) { println!("rm objects/{first_byte:02x}/{filename:?}"); } else { println!("# objects/{first_byte:02x}/{filename:?} lives"); diff --git a/src/selabel.rs b/src/selabel.rs index 71c742f1..a412a13e 100644 --- a/src/selabel.rs +++ b/src/selabel.rs @@ -11,6 +11,7 @@ use anyhow::{bail, ensure, Context, Result}; use regex_automata::{hybrid::dfa, util::syntax, Anchored, Input}; use crate::{ + fsverity::FsVerityHashValue, image::{Directory, FileSystem, ImageError, Inode, Leaf, LeafContent, RegularFile, Stat}, repository::Repository, }; @@ -105,10 +106,10 @@ struct Policy { contexts: Vec, } -pub fn openat<'a>( - dir: &'a Directory, +pub fn openat<'a, H: FsVerityHashValue>( + dir: &'a Directory, filename: impl AsRef, - repo: &Repository, + repo: &Repository, ) -> Result>> { match dir.get_file(filename.as_ref()) { Ok(RegularFile::Inline(data)) => Ok(Some(Box::new(&**data))), @@ -119,7 +120,7 @@ pub fn openat<'a>( } impl Policy { - pub fn build(dir: &Directory, repo: &Repository) -> Result { + pub fn build(dir: &Directory, repo: &Repository) -> Result { let mut aliases = HashMap::new(); let mut regexps = vec![]; let mut contexts = vec![]; @@ -201,7 +202,7 @@ fn relabel(stat: &Stat, path: &Path, ifmt: u8, policy: &mut Policy) { } } -fn relabel_leaf(leaf: &Leaf, path: &Path, policy: &mut Policy) { +fn relabel_leaf(leaf: &Leaf, path: &Path, policy: &mut Policy) { let ifmt = match leaf.content { LeafContent::Regular(..) => b'-', LeafContent::Fifo => b'p', // NB: 'pipe', not 'fifo' @@ -213,14 +214,14 @@ fn relabel_leaf(leaf: &Leaf, path: &Path, policy: &mut Policy) { relabel(&leaf.stat, path, ifmt, policy); } -fn relabel_inode(inode: &Inode, path: &mut PathBuf, policy: &mut Policy) { +fn relabel_inode(inode: &Inode, path: &mut PathBuf, policy: &mut Policy) { match inode { Inode::Directory(ref dir) => relabel_dir(dir, path, policy), Inode::Leaf(ref leaf) => relabel_leaf(leaf, path, policy), } } -fn relabel_dir(dir: &Directory, path: &mut PathBuf, policy: &mut Policy) { +fn relabel_dir(dir: &Directory, path: &mut PathBuf, policy: &mut Policy) { relabel(&dir.stat, path, b'd', policy); for (name, inode) in dir.sorted_entries() { @@ -245,7 +246,7 @@ fn parse_config(file: impl Read) -> Result> { Ok(None) } -pub fn selabel(fs: &mut FileSystem, repo: &Repository) -> Result<()> { +pub fn selabel(fs: &mut FileSystem, repo: &Repository) -> Result<()> { // if /etc/selinux/config doesn't exist then it's not an error let etc_selinux = match fs.root.get_directory("etc/selinux".as_ref()) { Err(ImageError::NotFound(..)) => return Ok(()), diff --git a/src/splitstream.rs b/src/splitstream.rs index a40c2769..0924ff06 100644 --- a/src/splitstream.rs +++ b/src/splitstream.rs @@ -11,42 +11,42 @@ use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; use zstd::stream::{read::Decoder, write::Encoder}; use crate::{ - fsverity::Sha256HashValue, + fsverity::FsVerityHashValue, repository::Repository, util::{read_exactish, Sha256Digest}, }; #[derive(Debug, FromBytes, Immutable, IntoBytes, KnownLayout)] #[repr(C)] -pub struct DigestMapEntry { +pub struct DigestMapEntry { pub body: Sha256Digest, - pub verity: Sha256HashValue, + pub verity: ObjectID, } #[derive(Debug)] -pub struct DigestMap { - pub map: Vec, +pub struct DigestMap { + pub map: Vec>, } -impl Default for DigestMap { +impl Default for DigestMap { fn default() -> Self { Self::new() } } -impl DigestMap { +impl DigestMap { pub fn new() -> Self { DigestMap { map: vec![] } } - pub fn lookup(&self, body: &Sha256Digest) -> Option<&Sha256HashValue> { + pub fn lookup(&self, body: &Sha256Digest) -> Option<&ObjectID> { match self.map.binary_search_by_key(body, |e| e.body) { Ok(idx) => Some(&self.map[idx].verity), Err(..) => None, } } - pub fn insert(&mut self, body: &Sha256Digest, verity: &Sha256HashValue) { + pub fn insert(&mut self, body: &Sha256Digest, verity: &ObjectID) { match self.map.binary_search_by_key(body, |e| e.body) { Ok(idx) => assert_eq!(self.map[idx].verity, *verity), // or else, bad things... Err(idx) => self.map.insert( @@ -60,14 +60,14 @@ impl DigestMap { } } -pub struct SplitStreamWriter<'a> { - repo: &'a Repository, +pub struct SplitStreamWriter<'a, ObjectID: FsVerityHashValue> { + repo: &'a Repository, inline_content: Vec, writer: Encoder<'a, Vec>, pub sha256: Option<(Sha256, Sha256Digest)>, } -impl std::fmt::Debug for SplitStreamWriter<'_> { +impl std::fmt::Debug for SplitStreamWriter<'_, ObjectID> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { // writer doesn't impl Debug f.debug_struct("SplitStreamWriter") @@ -78,10 +78,10 @@ impl std::fmt::Debug for SplitStreamWriter<'_> { } } -impl<'a> SplitStreamWriter<'a> { +impl<'a, ObjectID: FsVerityHashValue> SplitStreamWriter<'a, ObjectID> { pub fn new( - repo: &'a Repository, - refs: Option, + repo: &'a Repository, + refs: Option>, sha256: Option, ) -> Self { // SAFETY: we surely can't get an error writing the header to a Vec @@ -135,7 +135,7 @@ impl<'a> SplitStreamWriter<'a> { /// write a reference to external data to the stream. If the external data had padding in the /// stream which is not stored in the object then pass it here as well and it will be stored /// inline after the reference. - fn write_reference(&mut self, reference: Sha256HashValue, padding: Vec) -> Result<()> { + fn write_reference(&mut self, reference: &ObjectID, padding: Vec) -> Result<()> { // Flush the inline data before we store the external reference. Any padding from the // external data becomes the start of a new inline block. self.flush_inline(padding)?; @@ -149,10 +149,10 @@ impl<'a> SplitStreamWriter<'a> { sha256.update(&padding); } let id = self.repo.ensure_object(data)?; - self.write_reference(id, padding) + self.write_reference(&id, padding) } - pub fn done(mut self) -> Result { + pub fn done(mut self) -> Result { self.flush_inline(vec![])?; if let Some((context, expected)) = self.sha256 { @@ -166,19 +166,19 @@ impl<'a> SplitStreamWriter<'a> { } #[derive(Debug)] -pub enum SplitStreamData { +pub enum SplitStreamData { Inline(Box<[u8]>), - External(Sha256HashValue), + External(ObjectID), } // utility class to help read splitstreams -pub struct SplitStreamReader { +pub struct SplitStreamReader { decoder: Decoder<'static, BufReader>, - pub refs: DigestMap, + pub refs: DigestMap, inline_bytes: usize, } -impl std::fmt::Debug for SplitStreamReader { +impl std::fmt::Debug for SplitStreamReader { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { // decoder doesn't impl Debug f.debug_struct("SplitStreamReader") @@ -207,13 +207,13 @@ fn read_into_vec(reader: &mut impl Read, vec: &mut Vec, size: usize) -> Resu Ok(()) } -enum ChunkType { +enum ChunkType { Eof, Inline, - External(Sha256HashValue), + External(ObjectID), } -impl SplitStreamReader { +impl SplitStreamReader { pub fn new(reader: R) -> Result { let mut decoder = Decoder::new(reader)?; @@ -223,7 +223,7 @@ impl SplitStreamReader { u64::from_le_bytes(buf) } as usize; - let mut refs = DigestMap { + let mut refs = DigestMap:: { map: Vec::with_capacity(n_map_entries), }; for _ in 0..n_map_entries { @@ -242,7 +242,7 @@ impl SplitStreamReader { eof_ok: bool, ext_ok: bool, expected_bytes: usize, - ) -> Result { + ) -> Result> { if self.inline_bytes == 0 { match read_u64_le(&mut self.decoder)? { None => { @@ -255,7 +255,7 @@ impl SplitStreamReader { if !ext_ok { bail!("Unexpected external reference when parsing splitstream"); } - let id = Sha256HashValue::read_from_io(&mut self.decoder)?; + let id = ObjectID::read_from_io(&mut self.decoder)?; return Ok(ChunkType::External(id)); } Some(size) => { @@ -296,7 +296,7 @@ impl SplitStreamReader { &mut self, actual_size: usize, stored_size: usize, - ) -> Result { + ) -> Result> { if let ChunkType::External(id) = self.ensure_chunk(false, true, stored_size)? { // ...and the padding if actual_size < stored_size { @@ -315,7 +315,7 @@ impl SplitStreamReader { pub fn cat( &mut self, output: &mut impl Write, - mut load_data: impl FnMut(&Sha256HashValue) -> Result>, + mut load_data: impl FnMut(&ObjectID) -> Result>, ) -> Result<()> { let mut buffer = vec![]; @@ -334,7 +334,7 @@ impl SplitStreamReader { } } - pub fn get_object_refs(&mut self, mut callback: impl FnMut(&Sha256HashValue)) -> Result<()> { + pub fn get_object_refs(&mut self, mut callback: impl FnMut(&ObjectID)) -> Result<()> { let mut buffer = vec![]; for entry in &self.refs.map { @@ -361,7 +361,7 @@ impl SplitStreamReader { } } - pub fn lookup(&self, body: &Sha256Digest) -> Result<&Sha256HashValue> { + pub fn lookup(&self, body: &Sha256Digest) -> Result<&ObjectID> { match self.refs.lookup(body) { Some(id) => Ok(id), None => bail!("Reference is not found in splitstream"), @@ -369,7 +369,7 @@ impl SplitStreamReader { } } -impl Read for SplitStreamReader { +impl Read for SplitStreamReader { fn read(&mut self, data: &mut [u8]) -> std::io::Result { match self.ensure_chunk(true, false, 1) { Ok(ChunkType::Eof) => Ok(0), diff --git a/tests/mkfs.rs b/tests/mkfs.rs index c53ed85b..f37cef02 100644 --- a/tests/mkfs.rs +++ b/tests/mkfs.rs @@ -17,7 +17,7 @@ use composefs::{ image::{Directory, FileSystem, Inode, Leaf, LeafContent, RegularFile, Stat}, }; -fn debug_fs(mut fs: FileSystem) -> String { +fn debug_fs(mut fs: FileSystem) -> String { fs.done(); let image = mkfs_erofs(&fs); let mut output = vec![]; @@ -25,16 +25,20 @@ fn debug_fs(mut fs: FileSystem) -> String { String::from_utf8(output).unwrap() } -fn empty(_fs: &mut FileSystem) {} +fn empty(_fs: &mut FileSystem) {} #[test] fn test_empty() { - let mut fs = FileSystem::new(); + let mut fs = FileSystem::::new(); empty(&mut fs); insta::assert_snapshot!(debug_fs(fs)); } -fn add_leaf(dir: &mut Directory, name: impl AsRef, content: LeafContent) { +fn add_leaf( + dir: &mut Directory, + name: impl AsRef, + content: LeafContent, +) { dir.insert( name.as_ref(), Inode::Leaf(Rc::new(Leaf { @@ -50,7 +54,7 @@ fn add_leaf(dir: &mut Directory, name: impl AsRef, content: LeafContent) ); } -fn simple(fs: &mut FileSystem) { +fn simple(fs: &mut FileSystem) { let ext_id = Sha256HashValue::from_hex( "5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a", ) @@ -78,12 +82,12 @@ fn simple(fs: &mut FileSystem) { #[test] fn test_simple() { - let mut fs = FileSystem::new(); + let mut fs = FileSystem::::new(); simple(&mut fs); insta::assert_snapshot!(debug_fs(fs)); } -fn foreach_case(f: fn(&FileSystem)) { +fn foreach_case(f: fn(&FileSystem)) { for case in [empty, simple] { let mut fs = FileSystem::new(); case(&mut fs); From 80b7c66fed33dc54426ce7a958da46231f339b03 Mon Sep 17 00:00:00 2001 From: Allison Karlitskaya Date: Thu, 17 Apr 2025 19:39:28 +0200 Subject: [PATCH 9/9] src: use "impl trait" syntax for args where possible This syntax is a bit less verbose than the fully-spelled-out generic form. I'm surprised by how few sites this was possible for... Signed-off-by: Allison Karlitskaya --- src/dumpfile.rs | 4 ++-- src/erofs/writer.rs | 6 +++--- src/oci/image.rs | 5 +---- src/oci/mod.rs | 4 ++-- src/oci/tar.rs | 8 ++++---- tests/mkfs.rs | 4 ++-- 6 files changed, 14 insertions(+), 17 deletions(-) diff --git a/src/dumpfile.rs b/src/dumpfile.rs index f97d4364..475ceaf3 100644 --- a/src/dumpfile.rs +++ b/src/dumpfile.rs @@ -101,11 +101,11 @@ pub fn write_directory( ) } -pub fn write_leaf( +pub fn write_leaf( writer: &mut impl fmt::Write, path: &Path, stat: &Stat, - content: &LeafContent, + content: &LeafContent, nlink: usize, ) -> fmt::Result { match content { diff --git a/src/erofs/writer.rs b/src/erofs/writer.rs index 53e13ad7..e752eace 100644 --- a/src/erofs/writer.rs +++ b/src/erofs/writer.rs @@ -525,7 +525,7 @@ impl<'a, ObjectID: FsVerityHashValue> InodeCollector<'a, ObjectID> { /// Takes a list of inodes where each inode contains only local xattr values, determines which /// xattrs (key, value) pairs appear more than once, and shares them. -fn share_xattrs(inodes: &mut [Inode]) -> Vec { +fn share_xattrs(inodes: &mut [Inode]) -> Vec { let mut xattrs: BTreeMap = BTreeMap::new(); // Collect all xattrs from the inodes @@ -563,9 +563,9 @@ fn share_xattrs(inodes: &mut [Inode]) -> xattrs.into_keys().collect() } -fn write_erofs( +fn write_erofs( output: &mut impl Output, - inodes: &[Inode], + inodes: &[Inode], xattrs: &[XAttr], ) { // Write composefs header diff --git a/src/oci/image.rs b/src/oci/image.rs index b30fd210..de115814 100644 --- a/src/oci/image.rs +++ b/src/oci/image.rs @@ -68,10 +68,7 @@ pub fn compose_filesystem( Ok(filesystem) } -pub fn create_dumpfile( - repo: &Repository, - layers: &[String], -) -> Result<()> { +pub fn create_dumpfile(repo: &Repository, layers: &[String]) -> Result<()> { let filesystem = compose_filesystem(repo, layers)?; let mut stdout = std::io::stdout(); write_dumpfile(&mut stdout, &filesystem)?; diff --git a/src/oci/mod.rs b/src/oci/mod.rs index e740dbff..63497416 100644 --- a/src/oci/mod.rs +++ b/src/oci/mod.rs @@ -212,8 +212,8 @@ impl<'repo, ObjectID: FsVerityHashValue> ImageOp<'repo, ObjectID> { /// Pull the target image, and add the provided tag. If this is a mountable /// image (i.e. not an artifact), it is *not* unpacked by default. -pub async fn pull( - repo: &Repository, +pub async fn pull( + repo: &Repository, imgref: &str, reference: Option<&str>, ) -> Result<()> { diff --git a/src/oci/tar.rs b/src/oci/tar.rs index 08bbada5..4f3a0cb5 100644 --- a/src/oci/tar.rs +++ b/src/oci/tar.rs @@ -43,9 +43,9 @@ async fn read_header_async(reader: &mut (impl AsyncRead + Unpin)) -> Result( +pub fn split( tar_stream: &mut impl Read, - writer: &mut SplitStreamWriter, + writer: &mut SplitStreamWriter, ) -> Result<()> { while let Some(header) = read_header(tar_stream)? { // the header always gets stored as inline data @@ -73,9 +73,9 @@ pub fn split( Ok(()) } -pub async fn split_async( +pub async fn split_async( mut tar_stream: impl AsyncRead + Unpin, - writer: &mut SplitStreamWriter<'_, ObjectID>, + writer: &mut SplitStreamWriter<'_, impl FsVerityHashValue>, ) -> Result<()> { while let Some(header) = read_header_async(&mut tar_stream).await? { // the header always gets stored as inline data diff --git a/tests/mkfs.rs b/tests/mkfs.rs index f37cef02..69b8c541 100644 --- a/tests/mkfs.rs +++ b/tests/mkfs.rs @@ -17,7 +17,7 @@ use composefs::{ image::{Directory, FileSystem, Inode, Leaf, LeafContent, RegularFile, Stat}, }; -fn debug_fs(mut fs: FileSystem) -> String { +fn debug_fs(mut fs: FileSystem) -> String { fs.done(); let image = mkfs_erofs(&fs); let mut output = vec![]; @@ -25,7 +25,7 @@ fn debug_fs(mut fs: FileSystem) -> String String::from_utf8(output).unwrap() } -fn empty(_fs: &mut FileSystem) {} +fn empty(_fs: &mut FileSystem) {} #[test] fn test_empty() {