diff --git a/Cargo.toml b/Cargo.toml index 0461c9d9..8c45cfe2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,7 +36,7 @@ thiserror = "2.0.0" tokio = "1.24.2" toml = "0.8.0" xxhash-rust = { version = "0.8.2", features = ["xxh32"] } -zerocopy = { version = "0.8.0", features = ["derive"] } +zerocopy = { version = "0.8.0", features = ["derive", "std"] } zstd = "0.13.0" [dev-dependencies] diff --git a/src/bin/cfsctl.rs b/src/bin/cfsctl.rs index 09cd7372..8cca5aaa 100644 --- a/src/bin/cfsctl.rs +++ b/src/bin/cfsctl.rs @@ -5,7 +5,12 @@ use clap::{Parser, Subcommand}; use rustix::fs::CWD; -use composefs::{oci, repository::Repository, util::parse_sha256}; +use composefs::{ + fsverity::{FsVerityHashValue, Sha256HashValue}, + oci, + repository::Repository, + util::parse_sha256, +}; /// cfsctl #[derive(Debug, Parser)] @@ -106,7 +111,7 @@ fn main() -> Result<()> { let args = App::parse(); - let repo = (if let Some(path) = args.repo { + let repo: Repository = (if let Some(path) = args.repo { Repository::open_path(CWD, path) } else if args.system { Repository::open_system() @@ -130,7 +135,7 @@ fn main() -> Result<()> { } Command::ImportImage { reference } => { let image_id = repo.import_image(&reference, &mut std::io::stdin())?; - println!("{}", hex::encode(image_id)); + println!("{}", image_id.to_hex()); } Command::Oci { cmd: oci_cmd } => match oci_cmd { OciCommand::ImportLayer { name, sha256 } => { @@ -140,7 +145,7 @@ fn main() -> Result<()> { name.as_deref(), &mut std::io::stdin(), )?; - println!("{}", hex::encode(object_id)); + println!("{}", object_id.to_hex()); } OciCommand::LsLayer { name } => { oci::ls_layer(&repo, &name)?; @@ -150,7 +155,7 @@ fn main() -> Result<()> { } OciCommand::CreateImage { config, name } => { let image_id = oci::image::create_image(&repo, &config, name.as_deref(), None)?; - println!("{}", hex::encode(image_id)); + println!("{}", image_id.to_hex()); } OciCommand::Pull { ref image, name } => { let runtime = tokio::runtime::Builder::new_current_thread() @@ -161,10 +166,13 @@ fn main() -> Result<()> { runtime.block_on(async move { oci::pull(&repo, image, name.as_deref()).await })?; } OciCommand::Seal { verity, ref name } => { - let (sha256, verity) = - oci::seal(&repo, name, verity.map(parse_sha256).transpose()?.as_ref())?; + let (sha256, verity) = oci::seal( + &repo, + name, + verity.map(Sha256HashValue::from_hex).transpose()?.as_ref(), + )?; println!("sha256 {}", hex::encode(sha256)); - println!("verity {}", hex::encode(verity)); + println!("verity {}", verity.to_hex()); } OciCommand::Mount { ref name, @@ -182,10 +190,10 @@ fn main() -> Result<()> { }, Command::CreateImage { ref path } => { let image_id = composefs::fs::create_image(path, Some(&repo))?; - println!("{}", hex::encode(image_id)); + println!("{}", image_id.to_hex()); } Command::CreateDumpfile { ref path } => { - composefs::fs::create_dumpfile(path)?; + composefs::fs::create_dumpfile::(path)?; } Command::Mount { name, mountpoint } => { repo.mount(&name, &mountpoint)?; @@ -193,7 +201,7 @@ fn main() -> Result<()> { Command::ImageObjects { name } => { let objects = repo.objects_for_image(&name)?; for object in objects { - println!("{}", hex::encode(object)); + println!("{}", object.to_hex()); } } Command::GC => { diff --git a/src/bin/composefs-setup-root.rs b/src/bin/composefs-setup-root.rs index e356dabc..479aa759 100644 --- a/src/bin/composefs-setup-root.rs +++ b/src/bin/composefs-setup-root.rs @@ -21,7 +21,7 @@ use rustix::{ use serde::Deserialize; use composefs::{ - fsverity::Sha256HashValue, + fsverity::{FsVerityHashValue, Sha256HashValue}, mount::{composefs_fsmount, mount_at, FsHandle}, mountcompat::{overlayfs_set_fd, overlayfs_set_lower_and_data_fds, prepare_mount}, repository::Repository, @@ -167,7 +167,7 @@ fn open_root_fs(path: &Path) -> Result { } fn mount_composefs_image(sysroot: &OwnedFd, name: &str) -> Result { - let repo = Repository::open_path(sysroot, "composefs")?; + let repo = Repository::::open_path(sysroot, "composefs")?; let image = repo.open_image(name)?; composefs_fsmount(image, name, repo.object_dir()?).context("Failed to mount composefs image") } @@ -200,13 +200,11 @@ fn mount_subdir( } // Implementation -fn parse_composefs_cmdline(cmdline: &[u8]) -> Result { +fn parse_composefs_cmdline(cmdline: &[u8]) -> Result { // TODO?: officially we need to understand quoting with double-quotes... for part in cmdline.split(|c| c.is_ascii_whitespace()) { if let Some(digest) = part.strip_prefix(b"composefs=") { - let mut value = [0; 32]; - hex::decode_to_slice(digest, &mut value).context("Parsing composefs=")?; - return Ok(value); + return H::from_hex(digest).context("Parsing composefs="); } } bail!("Unable to find composefs= cmdline parameter"); @@ -238,7 +236,7 @@ fn setup_root(args: Args) -> Result<()> { Some(cmdline) => cmdline.as_bytes(), None => &std::fs::read("/proc/cmdline")?, }; - let image = hex::encode(parse_composefs_cmdline(cmdline)?); + let image = parse_composefs_cmdline::(cmdline)?.to_hex(); let new_root = match args.root_fs { Some(path) => open_root_fs(&path).context("Failed to clone specified root fs")?, @@ -284,18 +282,21 @@ fn main() -> Result<()> { setup_root(args) } -#[test] -fn test_parse() { - let failing = ["", "foo", "composefs", "composefs=foo"]; - for case in failing { - assert!(parse_composefs_cmdline(case.as_bytes()).is_err()); +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_parse() { + let failing = ["", "foo", "composefs", "composefs=foo"]; + for case in failing { + assert!(parse_composefs_cmdline::(case.as_bytes()).is_err()); + } + let digest = "8b7df143d91c716ecfa5fc1730022f6b421b05cedee8fd52b1fc65a96030ad52"; + similar_asserts::assert_eq!( + parse_composefs_cmdline::(format!("composefs={digest}").as_bytes()) + .unwrap(), + Sha256HashValue::from_hex(digest).unwrap() + ); } - let digest = "8b7df143d91c716ecfa5fc1730022f6b421b05cedee8fd52b1fc65a96030ad52"; - let digest_bytes = hex::decode(digest).unwrap(); - similar_asserts::assert_eq!( - parse_composefs_cmdline(format!("composefs={digest}").as_bytes()) - .unwrap() - .as_slice(), - &digest_bytes - ); } diff --git a/src/dumpfile.rs b/src/dumpfile.rs index d2baa331..475ceaf3 100644 --- a/src/dumpfile.rs +++ b/src/dumpfile.rs @@ -12,7 +12,7 @@ use anyhow::Result; use rustix::fs::FileType; use crate::{ - fsverity::Sha256HashValue, + fsverity::FsVerityHashValue, image::{Directory, FileSystem, Inode, Leaf, LeafContent, RegularFile, Stat}, }; @@ -49,7 +49,7 @@ fn write_entry( rdev: u64, payload: impl AsRef, content: &[u8], - digest: Option<&Sha256HashValue>, + digest: Option<&str>, ) -> fmt::Result { let mode = stat.st_mode | ifmt.as_raw_mode(); let uid = stat.st_uid; @@ -66,7 +66,7 @@ fn write_entry( write_escaped(writer, content)?; write!(writer, " ")?; if let Some(id) = digest { - write!(writer, "{}", hex::encode(id))?; + write!(writer, "{}", id)?; } else { write_empty(writer)?; } @@ -105,7 +105,7 @@ pub fn write_leaf( writer: &mut impl fmt::Write, path: &Path, stat: &Stat, - content: &LeafContent, + content: &LeafContent, nlink: usize, ) -> fmt::Result { match content { @@ -129,9 +129,9 @@ pub fn write_leaf( *size, nlink, 0, - format!("{:02x}/{}", id[0], hex::encode(&id[1..])), + id.to_object_pathname(), &[], - Some(id), + Some(&id.to_hex()), ), LeafContent::BlockDevice(rdev) => write_entry( writer, @@ -204,8 +204,8 @@ pub fn write_hardlink(writer: &mut impl fmt::Write, path: &Path, target: &OsStr) Ok(()) } -struct DumpfileWriter<'a, W: Write> { - hardlinks: HashMap<*const Leaf, OsString>, +struct DumpfileWriter<'a, W: Write, ObjectID: FsVerityHashValue> { + hardlinks: HashMap<*const Leaf, OsString>, writer: &'a mut W, } @@ -215,7 +215,7 @@ fn writeln_fmt(writer: &mut impl Write, f: impl Fn(&mut String) -> fmt::Result) Ok(writeln!(writer, "{}", tmp)?) } -impl<'a, W: Write> DumpfileWriter<'a, W> { +impl<'a, W: Write, ObjectID: FsVerityHashValue> DumpfileWriter<'a, W, ObjectID> { fn new(writer: &'a mut W) -> Self { Self { hardlinks: HashMap::new(), @@ -223,7 +223,7 @@ impl<'a, W: Write> DumpfileWriter<'a, W> { } } - fn write_dir(&mut self, path: &mut PathBuf, dir: &Directory) -> Result<()> { + fn write_dir(&mut self, path: &mut PathBuf, dir: &Directory) -> Result<()> { // nlink is 2 + number of subdirectories // this is also true for the root dir since '..' is another self-ref let nlink = dir.inodes().fold(2, |count, inode| { @@ -256,7 +256,7 @@ impl<'a, W: Write> DumpfileWriter<'a, W> { Ok(()) } - fn write_leaf(&mut self, path: &Path, leaf: &Rc) -> Result<()> { + fn write_leaf(&mut self, path: &Path, leaf: &Rc>) -> Result<()> { let nlink = Rc::strong_count(leaf); if nlink > 1 { @@ -276,7 +276,10 @@ impl<'a, W: Write> DumpfileWriter<'a, W> { } } -pub fn write_dumpfile(writer: &mut W, fs: &FileSystem) -> Result<()> { +pub fn write_dumpfile( + writer: &mut impl Write, + fs: &FileSystem, +) -> Result<()> { // default pipe capacity on Linux is 16 pages (65536 bytes), but // sometimes the BufWriter will write more than its capacity... let mut buffer = BufWriter::with_capacity(32768, writer); diff --git a/src/erofs/composefs.rs b/src/erofs/composefs.rs new file mode 100644 index 00000000..b293bf60 --- /dev/null +++ b/src/erofs/composefs.rs @@ -0,0 +1,33 @@ +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; + +use crate::fsverity::FsVerityHashValue; + +/* From linux/fs/overlayfs/overlayfs.h struct ovl_metacopy */ +#[derive(Debug, FromBytes, Immutable, KnownLayout, IntoBytes)] +#[repr(C)] +pub(super) struct OverlayMetacopy { + version: u8, + len: u8, + flags: u8, + digest_algo: u8, + pub(super) digest: H, +} + +impl OverlayMetacopy { + pub(super) fn new(digest: &H) -> Self { + Self { + version: 0, + len: size_of::() as u8, + flags: 0, + digest_algo: H::ALGORITHM, + digest: digest.clone(), + } + } + + pub(super) fn valid(&self) -> bool { + self.version == 0 + && self.len == size_of::() as u8 + && self.flags == 0 + && self.digest_algo == H::ALGORITHM + } +} diff --git a/src/erofs/mod.rs b/src/erofs/mod.rs index 8ef9c1b0..fac88cd5 100644 --- a/src/erofs/mod.rs +++ b/src/erofs/mod.rs @@ -1,3 +1,4 @@ +pub mod composefs; pub mod debug; pub mod format; pub mod reader; diff --git a/src/erofs/reader.rs b/src/erofs/reader.rs index 76dc77fb..d18422b1 100644 --- a/src/erofs/reader.rs +++ b/src/erofs/reader.rs @@ -5,11 +5,14 @@ use std::ops::Range; use thiserror::Error; use zerocopy::{little_endian::U32, FromBytes, Immutable, KnownLayout}; -use super::format::{ - CompactInodeHeader, ComposefsHeader, DataLayout, DirectoryEntryHeader, ExtendedInodeHeader, - InodeXAttrHeader, ModeField, Superblock, XAttrHeader, +use super::{ + composefs::OverlayMetacopy, + format::{ + CompactInodeHeader, ComposefsHeader, DataLayout, DirectoryEntryHeader, ExtendedInodeHeader, + InodeXAttrHeader, ModeField, Superblock, XAttrHeader, + }, }; -use crate::fsverity::Sha256HashValue; +use crate::fsverity::FsVerityHashValue; pub fn round_up(n: usize, to: usize) -> usize { (n + to - 1) & !(to - 1) @@ -483,26 +486,25 @@ pub enum ErofsReaderError { type ReadResult = Result; #[derive(Debug)] -pub struct ObjectCollector { +pub struct ObjectCollector { visited_nids: HashSet, nids_to_visit: BTreeSet, - objects: HashSet, + objects: HashSet, } -impl ObjectCollector { +impl ObjectCollector { fn visit_xattr(&mut self, attr: &XAttr) { - // TODO: "4" is a bit magic, isn't it? + // This is the index of "trusted". See XATTR_PREFIXES in format.rs. if attr.header.name_index != 4 { return; } if attr.suffix() != b"overlay.metacopy" { return; } - let value = attr.value(); - // TODO: oh look, more magic values... - if value.len() == 36 && value[..4] == [0, 36, 0, 1] { - // SAFETY: We already checked that the length is 4 + 32 - self.objects.insert(value[4..].try_into().unwrap()); + if let Ok(value) = OverlayMetacopy::read_from_bytes(attr.value()) { + if value.valid() { + self.objects.insert(value.digest); + } } } @@ -550,7 +552,7 @@ impl ObjectCollector { } } -pub fn collect_objects(image: &[u8]) -> ReadResult> { +pub fn collect_objects(image: &[u8]) -> ReadResult> { let img = Image::open(image); let mut this = ObjectCollector { visited_nids: HashSet::new(), diff --git a/src/erofs/writer.rs b/src/erofs/writer.rs index 19d06224..e752eace 100644 --- a/src/erofs/writer.rs +++ b/src/erofs/writer.rs @@ -11,7 +11,8 @@ use xxhash_rust::xxh32::xxh32; use zerocopy::{Immutable, IntoBytes}; use crate::{ - erofs::{format, reader::round_up}, + erofs::{composefs::OverlayMetacopy, format, reader::round_up}, + fsverity::FsVerityHashValue, image, }; @@ -81,21 +82,21 @@ struct Directory<'a> { } #[derive(Debug)] -struct Leaf<'a> { - content: &'a image::LeafContent, +struct Leaf<'a, ObjectID: FsVerityHashValue> { + content: &'a image::LeafContent, nlink: usize, } #[derive(Debug)] -enum InodeContent<'a> { +enum InodeContent<'a, ObjectID: FsVerityHashValue> { Directory(Directory<'a>), - Leaf(Leaf<'a>), + Leaf(Leaf<'a, ObjectID>), } -struct Inode<'a> { +struct Inode<'a, ObjectID: FsVerityHashValue> { stat: &'a image::Stat, xattrs: InodeXAttrs, - content: InodeContent<'a>, + content: InodeContent<'a, ObjectID>, } impl XAttr { @@ -262,7 +263,7 @@ impl<'a> Directory<'a> { } } -impl Leaf<'_> { +impl Leaf<'_, ObjectID> { fn inode_meta(&self) -> (format::DataLayout, u32, u64, usize) { let (layout, u, size) = match &self.content { image::LeafContent::Regular(image::RegularFile::Inline(data)) => { @@ -298,7 +299,7 @@ impl Leaf<'_> { } } -impl Inode<'_> { +impl Inode<'_, ObjectID> { fn file_type(&self) -> format::FileType { match &self.content { InodeContent::Directory(..) => format::FileType::Directory, @@ -394,13 +395,13 @@ impl Inode<'_> { } } -struct InodeCollector<'a> { - inodes: Vec>, - hardlinks: HashMap<*const image::Leaf, usize>, +struct InodeCollector<'a, ObjectID: FsVerityHashValue> { + inodes: Vec>, + hardlinks: HashMap<*const image::Leaf, usize>, } -impl<'a> InodeCollector<'a> { - fn push_inode(&mut self, stat: &'a image::Stat, content: InodeContent<'a>) -> usize { +impl<'a, ObjectID: FsVerityHashValue> InodeCollector<'a, ObjectID> { + fn push_inode(&mut self, stat: &'a image::Stat, content: InodeContent<'a, ObjectID>) -> usize { let mut xattrs = InodeXAttrs::default(); // We need to record extra xattrs for some files. These come first. @@ -409,10 +410,12 @@ impl<'a> InodeCollector<'a> { .. }) = content { - let metacopy = [&[0, 36, 0, 1], &id[..]].concat(); - xattrs.add(b"trusted.overlay.metacopy", &metacopy); + xattrs.add( + b"trusted.overlay.metacopy", + OverlayMetacopy::new(id).as_bytes(), + ); - let redirect = format!("/{:02x}/{}", id[0], hex::encode(&id[1..])); + let redirect = format!("/{}", id.to_object_pathname()); xattrs.add(b"trusted.overlay.redirect", redirect.as_bytes()); } @@ -439,7 +442,7 @@ impl<'a> InodeCollector<'a> { inode } - fn collect_leaf(&mut self, leaf: &'a Rc) -> usize { + fn collect_leaf(&mut self, leaf: &'a Rc>) -> usize { let nlink = Rc::strong_count(leaf); if nlink > 1 { @@ -478,7 +481,7 @@ impl<'a> InodeCollector<'a> { entries.insert(point, entry); } - fn collect_dir(&mut self, dir: &'a image::Directory, parent: usize) -> usize { + fn collect_dir(&mut self, dir: &'a image::Directory, parent: usize) -> usize { // The root inode number needs to fit in a u16. That more or less compels us to write the // directory inode before the inode of the children of the directory. Reserve a slot. let me = self.push_inode(&dir.stat, InodeContent::Directory(Directory::default())); @@ -506,7 +509,7 @@ impl<'a> InodeCollector<'a> { me } - pub fn collect(fs: &'a image::FileSystem) -> Vec> { + pub fn collect(fs: &'a image::FileSystem) -> Vec> { let mut this = Self { inodes: vec![], hardlinks: HashMap::new(), @@ -522,7 +525,7 @@ impl<'a> InodeCollector<'a> { /// Takes a list of inodes where each inode contains only local xattr values, determines which /// xattrs (key, value) pairs appear more than once, and shares them. -fn share_xattrs(inodes: &mut [Inode]) -> Vec { +fn share_xattrs(inodes: &mut [Inode]) -> Vec { let mut xattrs: BTreeMap = BTreeMap::new(); // Collect all xattrs from the inodes @@ -560,7 +563,11 @@ fn share_xattrs(inodes: &mut [Inode]) -> Vec { xattrs.into_keys().collect() } -fn write_erofs(output: &mut impl Output, inodes: &[Inode], xattrs: &[XAttr]) { +fn write_erofs( + output: &mut impl Output, + inodes: &[Inode], + xattrs: &[XAttr], +) { // Write composefs header output.note_offset(Offset::Header); output.write_struct(format::ComposefsHeader { @@ -681,7 +688,7 @@ impl Output for FirstPass { } } -pub fn mkfs_erofs(fs: &image::FileSystem) -> Box<[u8]> { +pub fn mkfs_erofs(fs: &image::FileSystem) -> Box<[u8]> { // Create the intermediate representation: flattened inodes and shared xattrs let mut inodes = InodeCollector::collect(fs); let xattrs = share_xattrs(&mut inodes); diff --git a/src/fs.rs b/src/fs.rs index 968f6e17..3cc0e28d 100644 --- a/src/fs.rs +++ b/src/fs.rs @@ -24,7 +24,7 @@ use rustix::{ use zerocopy::IntoBytes; use crate::{ - fsverity::{compute_verity, Sha256HashValue}, + fsverity::{compute_verity, FsVerityHashValue}, image::{Directory, FileSystem, Inode, Leaf, LeafContent, RegularFile, Stat}, repository::Repository, selabel::selabel, @@ -70,11 +70,11 @@ fn set_file_contents(dirfd: &OwnedFd, name: &OsStr, stat: &Stat, data: &[u8]) -> Ok(()) } -fn write_directory( - dir: &Directory, +fn write_directory( + dir: &Directory, dirfd: &OwnedFd, name: &OsStr, - repo: &Repository, + repo: &Repository, ) -> Result<()> { match mkdirat(dirfd, name, dir.stat.st_mode.into()) { Ok(()) | Err(Errno::EXIST) => {} @@ -85,7 +85,12 @@ fn write_directory( write_directory_contents(dir, &fd, repo) } -fn write_leaf(leaf: &Leaf, dirfd: &OwnedFd, name: &OsStr, repo: &Repository) -> Result<()> { +fn write_leaf( + leaf: &Leaf, + dirfd: &OwnedFd, + name: &OsStr, + repo: &Repository, +) -> Result<()> { let mode = leaf.stat.st_mode.into(); match &leaf.content { @@ -111,7 +116,11 @@ fn write_leaf(leaf: &Leaf, dirfd: &OwnedFd, name: &OsStr, repo: &Repository) -> Ok(()) } -fn write_directory_contents(dir: &Directory, fd: &OwnedFd, repo: &Repository) -> Result<()> { +fn write_directory_contents( + dir: &Directory, + fd: &OwnedFd, + repo: &Repository, +) -> Result<()> { for (name, inode) in dir.entries() { match inode { Inode::Directory(ref dir) => write_directory(dir, fd, name, repo), @@ -123,19 +132,23 @@ fn write_directory_contents(dir: &Directory, fd: &OwnedFd, repo: &Repository) -> } // NB: hardlinks not supported -pub fn write_to_path(repo: &Repository, dir: &Directory, output_dir: &Path) -> Result<()> { +pub fn write_to_path( + repo: &Repository, + dir: &Directory, + output_dir: &Path, +) -> Result<()> { let fd = openat(CWD, output_dir, OFlags::PATH | OFlags::DIRECTORY, 0.into())?; write_directory_contents(dir, &fd, repo) } #[derive(Debug)] -pub struct FilesystemReader<'repo> { - repo: Option<&'repo Repository>, - inodes: HashMap<(u64, u64), Rc>, +pub struct FilesystemReader<'repo, ObjectID: FsVerityHashValue> { + repo: Option<&'repo Repository>, + inodes: HashMap<(u64, u64), Rc>>, root_mtime: Option, } -impl FilesystemReader<'_> { +impl FilesystemReader<'_, ObjectID> { fn read_xattrs(fd: &OwnedFd) -> Result, Box<[u8]>>> { // flistxattr() and fgetxattr() don't work with with O_PATH fds, so go via /proc/self/fd. // Note: we want the symlink-following version of this call, which produces the correct @@ -176,12 +189,16 @@ impl FilesystemReader<'_> { st_uid: buf.st_uid, st_gid: buf.st_gid, st_mtim_sec: buf.st_mtime as i64, - xattrs: RefCell::new(FilesystemReader::read_xattrs(fd)?), + xattrs: RefCell::new(Self::read_xattrs(fd)?), }, )) } - fn read_leaf_content(&mut self, fd: OwnedFd, buf: rustix::fs::Stat) -> Result { + fn read_leaf_content( + &mut self, + fd: OwnedFd, + buf: rustix::fs::Stat, + ) -> Result> { let content = match FileType::from_raw_mode(buf.st_mode) { FileType::Directory | FileType::Unknown => unreachable!(), FileType::RegularFile => { @@ -212,7 +229,12 @@ impl FilesystemReader<'_> { Ok(content) } - fn read_leaf(&mut self, dirfd: &OwnedFd, name: &OsStr, ifmt: FileType) -> Result> { + fn read_leaf( + &mut self, + dirfd: &OwnedFd, + name: &OsStr, + ifmt: FileType, + ) -> Result>> { let oflags = match ifmt { FileType::RegularFile => OFlags::RDONLY, _ => OFlags::PATH, @@ -225,7 +247,7 @@ impl FilesystemReader<'_> { Mode::empty(), )?; - let (buf, stat) = FilesystemReader::stat(&fd, ifmt)?; + let (buf, stat) = Self::stat(&fd, ifmt)?; // NB: We could check `st_nlink > 1` to find out if we should track a file as a potential // hardlink or not, but some filesystems (like fuse-overlayfs) can report this incorrectly. @@ -241,7 +263,11 @@ impl FilesystemReader<'_> { } } - pub fn read_directory(&mut self, dirfd: impl AsFd, name: &OsStr) -> Result { + pub fn read_directory( + &mut self, + dirfd: impl AsFd, + name: &OsStr, + ) -> Result> { let fd = openat( dirfd, name, @@ -249,7 +275,7 @@ impl FilesystemReader<'_> { Mode::empty(), )?; - let (_, stat) = FilesystemReader::stat(&fd, FileType::Directory)?; + let (_, stat) = Self::stat(&fd, FileType::Directory)?; let mut directory = Directory::new(stat); for item in Dir::read_from(&fd)? { @@ -268,7 +294,12 @@ impl FilesystemReader<'_> { Ok(directory) } - fn read_inode(&mut self, dirfd: &OwnedFd, name: &OsStr, ifmt: FileType) -> Result { + fn read_inode( + &mut self, + dirfd: &OwnedFd, + name: &OsStr, + ifmt: FileType, + ) -> Result> { if ifmt == FileType::Directory { let dir = self.read_directory(dirfd, name)?; Ok(Inode::Directory(Box::new(dir))) @@ -279,7 +310,10 @@ impl FilesystemReader<'_> { } } -pub fn read_from_path(path: &Path, repo: Option<&Repository>) -> Result { +pub fn read_from_path( + path: &Path, + repo: Option<&Repository>, +) -> Result> { let mut reader = FilesystemReader { repo, inodes: HashMap::new(), @@ -300,7 +334,10 @@ pub fn read_from_path(path: &Path, repo: Option<&Repository>) -> Result) -> Result { +pub fn create_image( + path: &Path, + repo: Option<&Repository>, +) -> Result { let fs = read_from_path(path, repo)?; let image = crate::erofs::writer::mkfs_erofs(&fs); if let Some(repo) = repo { @@ -310,8 +347,8 @@ pub fn create_image(path: &Path, repo: Option<&Repository>) -> Result Result<()> { - let fs = read_from_path(path, None)?; +pub fn create_dumpfile(path: &Path) -> Result<()> { + let fs = read_from_path::(path, None)?; super::dumpfile::write_dumpfile(&mut std::io::stdout(), &fs) } diff --git a/src/fsverity/digest.rs b/src/fsverity/digest.rs index f085744b..861fc92c 100644 --- a/src/fsverity/digest.rs +++ b/src/fsverity/digest.rs @@ -64,7 +64,7 @@ impl FsVerityHasher { // We had a complete value, but now we're adding new data. // This means that we need to add a new hash layer... let mut new_layer = FsVerityLayer::new(); - new_layer.add_data(value.as_ref()); + new_layer.add_data(value.as_bytes()); self.layers.push(new_layer); } @@ -76,7 +76,7 @@ impl FsVerityHasher { for layer in self.layers.iter_mut() { // We have a layer we need to hash this value into - layer.add_data(value.as_ref()); + layer.add_data(value.as_bytes()); if layer.remaining != 0 { return; } @@ -97,7 +97,7 @@ impl FsVerityHasher { for layer in self.layers.iter_mut() { // We have a layer we need to hash this value into if value != H::EMPTY { - layer.add_data(value.as_ref()); + layer.add_data(value.as_bytes()); } if layer.remaining != (1 << LG_BLKSZ) { // ...but now this layer itself is complete, so get the value of *it*. @@ -143,7 +143,7 @@ impl FsVerityHasher { context.update(0u8.to_le_bytes()); /* salt_size */ context.update([0; 4]); /* reserved */ context.update(self.n_bytes.to_le_bytes()); - context.update(self.root_hash()); + context.update(self.root_hash().as_bytes()); context.update([0].repeat(64 - size_of::())); context.update([0; 32]); /* salt */ context.update([0; 144]); /* reserved */ @@ -162,12 +162,12 @@ mod tests { #[test] fn test_digest() { assert_eq!( - hex::encode(FsVerityHasher::::hash(b"hello world")), + FsVerityHasher::::hash(b"hello world").to_hex(), "1e2eaa4202d750a41174ee454970b92c1bc2f925b1e35076d8c7d5f56362ba64" ); assert_eq!( - hex::encode(FsVerityHasher::::hash(b"hello world")), + FsVerityHasher::::hash(b"hello world").to_hex(), "18430270729d162d4e469daca123ae61893db4b0583d8f7081e3bf4f92b88ba514e7982f10733fb6aa895195c5ae8fd2eb2c47a8be05513ce5a0c51a6f570409" ); } diff --git a/src/fsverity/hashvalue.rs b/src/fsverity/hashvalue.rs new file mode 100644 index 00000000..e75f1800 --- /dev/null +++ b/src/fsverity/hashvalue.rs @@ -0,0 +1,206 @@ +use core::{fmt, hash::Hash}; + +use hex::FromHexError; +use sha2::{digest::FixedOutputReset, digest::Output, Digest, Sha256, Sha512}; +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout, Unaligned}; + +pub trait FsVerityHashValue +where + Self: Clone, + Self: From>, + Self: FromBytes + Immutable + IntoBytes + KnownLayout + Unaligned, + Self: Hash + Eq, + Self: fmt::Debug, +{ + type Digest: Digest + FixedOutputReset + fmt::Debug; + const ALGORITHM: u8; + const EMPTY: Self; + const ID: &str; + + fn from_hex(hex: impl AsRef<[u8]>) -> Result { + let mut value = Self::EMPTY; + hex::decode_to_slice(hex.as_ref(), value.as_mut_bytes())?; + Ok(value) + } + + fn from_object_dir_and_basename( + dirnum: u8, + basename: impl AsRef<[u8]>, + ) -> Result { + let expected_size = 2 * (size_of::() - 1); + let bytes = basename.as_ref(); + if bytes.len() != expected_size { + return Err(FromHexError::InvalidStringLength); + } + let mut result = Self::EMPTY; + result.as_mut_bytes()[0] = dirnum; + hex::decode_to_slice(bytes, &mut result.as_mut_bytes()[1..])?; + Ok(result) + } + + fn from_object_pathname(pathname: impl AsRef<[u8]>) -> Result { + // We want to the trailing part of "....../xx/yyyyyy" where xxyyyyyy is our hex length + let min_size = 2 * size_of::() + 1; + let bytes = pathname.as_ref(); + if bytes.len() < min_size { + return Err(FromHexError::InvalidStringLength); + } + + let trailing = &bytes[bytes.len() - min_size..]; + let mut result = Self::EMPTY; + hex::decode_to_slice(&trailing[0..2], &mut result.as_mut_bytes()[0..1])?; + if trailing[2] != b'/' { + return Err(FromHexError::InvalidHexCharacter { + c: trailing[2] as char, + index: 2, + }); + } + hex::decode_to_slice(&trailing[3..], &mut result.as_mut_bytes()[1..])?; + Ok(result) + } + + fn to_object_pathname(&self) -> String { + format!("{:02x}/{}", self.as_bytes()[0], self.to_object_basename()) + } + + fn to_object_dir(&self) -> String { + format!("{:02x}", self.as_bytes()[0]) + } + + fn to_object_basename(&self) -> String { + hex::encode(&self.as_bytes()[1..]) + } + + fn to_hex(&self) -> String { + hex::encode(self.as_bytes()) + } + + fn to_id(&self) -> String { + format!("{}:{}", Self::ID, self.to_hex()) + } +} + +impl fmt::Debug for Sha256HashValue { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "sha256:{}", self.to_hex()) + } +} + +impl fmt::Debug for Sha512HashValue { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "sha512:{}", self.to_hex()) + } +} + +#[derive(Clone, Eq, FromBytes, Hash, Immutable, IntoBytes, KnownLayout, PartialEq, Unaligned)] +#[repr(C)] +pub struct Sha256HashValue([u8; 32]); + +impl From> for Sha256HashValue { + fn from(value: Output) -> Self { + Self(value.into()) + } +} + +impl FsVerityHashValue for Sha256HashValue { + type Digest = Sha256; + const ALGORITHM: u8 = 1; + const EMPTY: Self = Self([0; 32]); + const ID: &str = "sha256"; +} + +#[derive(Clone, Eq, FromBytes, Hash, Immutable, IntoBytes, KnownLayout, PartialEq, Unaligned)] +#[repr(C)] +pub struct Sha512HashValue([u8; 64]); + +impl From> for Sha512HashValue { + fn from(value: Output) -> Self { + Self(value.into()) + } +} + +impl FsVerityHashValue for Sha512HashValue { + type Digest = Sha512; + const ALGORITHM: u8 = 2; + const EMPTY: Self = Self([0; 64]); + const ID: &str = "sha512"; +} + +#[cfg(test)] +mod test { + use super::*; + + fn test_fsverity_hash() { + let len = size_of::(); + let hexlen = len * 2; + + let hex = H::EMPTY.to_hex(); + assert_eq!(hex.as_bytes(), [b'0'].repeat(hexlen)); + + assert_eq!(H::EMPTY.to_id(), format!("{}:{}", H::ID, hex)); + assert_eq!(format!("{:?}", H::EMPTY), format!("{}:{}", H::ID, hex)); + + assert_eq!(H::from_hex(&hex), Ok(H::EMPTY)); + + assert_eq!(H::from_hex("lol"), Err(FromHexError::OddLength)); + assert_eq!(H::from_hex("lolo"), Err(FromHexError::InvalidStringLength)); + assert_eq!( + H::from_hex([b'l'].repeat(hexlen)), + Err(FromHexError::InvalidHexCharacter { c: 'l', index: 0 }) + ); + + assert_eq!(H::from_object_dir_and_basename(0, &hex[2..]), Ok(H::EMPTY)); + + assert_eq!(H::from_object_dir_and_basename(0, &hex[2..]), Ok(H::EMPTY)); + + assert_eq!( + H::from_object_dir_and_basename(0, "lol"), + Err(FromHexError::InvalidStringLength) + ); + + assert_eq!( + H::from_object_dir_and_basename(0, [b'l'].repeat(hexlen - 2)), + Err(FromHexError::InvalidHexCharacter { c: 'l', index: 0 }) + ); + + assert_eq!( + H::from_object_pathname(format!("{}/{}", &hex[0..2], &hex[2..])), + Ok(H::EMPTY) + ); + + assert_eq!( + H::from_object_pathname(format!("../this/is/ignored/{}/{}", &hex[0..2], &hex[2..])), + Ok(H::EMPTY) + ); + + assert_eq!( + H::from_object_pathname(&hex), + Err(FromHexError::InvalidStringLength) + ); + + assert_eq!( + H::from_object_pathname("lol"), + Err(FromHexError::InvalidStringLength) + ); + + assert_eq!( + H::from_object_pathname([b'l'].repeat(hexlen + 1)), + Err(FromHexError::InvalidHexCharacter { c: 'l', index: 0 }) + ); + + assert_eq!( + H::from_object_pathname(format!("{}0{}", &hex[0..2], &hex[2..])), + Err(FromHexError::InvalidHexCharacter { c: '0', index: 2 }) + ); + } + + #[test] + fn test_sha256hashvalue() { + test_fsverity_hash::(); + } + + #[test] + fn test_sha512hashvalue() { + test_fsverity_hash::(); + } +} diff --git a/src/fsverity/mod.rs b/src/fsverity/mod.rs index f8eb4a64..880f29a3 100644 --- a/src/fsverity/mod.rs +++ b/src/fsverity/mod.rs @@ -1,36 +1,12 @@ mod digest; +mod hashvalue; mod ioctl; use std::{io::Error, os::fd::AsFd}; -use sha2::{digest::FixedOutputReset, digest::Output, Digest, Sha256, Sha512}; use thiserror::Error; -pub trait FsVerityHashValue -where - Self: Eq + AsRef<[u8]> + Clone, - Self: From>, -{ - type Digest: Digest + FixedOutputReset + std::fmt::Debug; - const ALGORITHM: u8; - const EMPTY: Self; -} - -pub type Sha256HashValue = [u8; 32]; - -impl FsVerityHashValue for Sha256HashValue { - type Digest = Sha256; - const ALGORITHM: u8 = 1; - const EMPTY: Self = [0; 32]; -} - -pub type Sha512HashValue = [u8; 64]; - -impl FsVerityHashValue for Sha512HashValue { - type Digest = Sha512; - const ALGORITHM: u8 = 2; - const EMPTY: Self = [0; 64]; -} +pub use hashvalue::{FsVerityHashValue, Sha256HashValue, Sha512HashValue}; /// Measuring fsverity failed. #[derive(Error, Debug)] // can't derive PartialEq because of std::io::Error @@ -156,8 +132,8 @@ pub fn ensure_verity_equal( Ok(()) } else { Err(CompareVerityError::DigestMismatch { - expected: hex::encode(expected), - found: hex::encode(found.as_ref()), + expected: expected.to_hex(), + found: found.to_hex(), }) } } @@ -166,10 +142,7 @@ pub fn ensure_verity_equal( mod tests { use std::{collections::BTreeSet, io::Write}; - use crate::{ - test::tempfile, - util::{parse_sha256, proc_self_fd}, - }; + use crate::{test::tempfile, util::proc_self_fd}; use rustix::{ fd::OwnedFd, fs::{open, Mode, OFlags}, @@ -224,26 +197,33 @@ mod tests { )); assert_eq!( - hex::encode(measure_verity::(&tf).unwrap()), + measure_verity::(&tf).unwrap().to_hex(), "1e2eaa4202d750a41174ee454970b92c1bc2f925b1e35076d8c7d5f56362ba64" ); assert_eq!( - hex::encode(measure_verity_opt::(&tf).unwrap().unwrap()), + measure_verity_opt::(&tf) + .unwrap() + .unwrap() + .to_hex(), "1e2eaa4202d750a41174ee454970b92c1bc2f925b1e35076d8c7d5f56362ba64" ); ensure_verity_equal( &tf, - &parse_sha256("1e2eaa4202d750a41174ee454970b92c1bc2f925b1e35076d8c7d5f56362ba64") - .unwrap(), + &Sha256HashValue::from_hex( + "1e2eaa4202d750a41174ee454970b92c1bc2f925b1e35076d8c7d5f56362ba64", + ) + .unwrap(), ) .unwrap(); let Err(CompareVerityError::DigestMismatch { expected, found }) = ensure_verity_equal( &tf, - &parse_sha256("1e2eaa4202d750a41174ee454970b92c1bc2f925b1e35076d8c7000000000000") - .unwrap(), + &Sha256HashValue::from_hex( + "1e2eaa4202d750a41174ee454970b92c1bc2f925b1e35076d8c7000000000000", + ) + .unwrap(), ) else { panic!("Didn't fail with expected error"); }; diff --git a/src/image.rs b/src/image.rs index 614916c1..15a8498f 100644 --- a/src/image.rs +++ b/src/image.rs @@ -8,7 +8,7 @@ use std::{ use thiserror::Error; -use crate::fsverity::Sha256HashValue; +use crate::fsverity::FsVerityHashValue; #[derive(Debug)] pub struct Stat { @@ -20,14 +20,14 @@ pub struct Stat { } #[derive(Debug)] -pub enum RegularFile { +pub enum RegularFile { Inline(Box<[u8]>), - External(Sha256HashValue, u64), + External(ObjectID, u64), } #[derive(Debug)] -pub enum LeafContent { - Regular(RegularFile), +pub enum LeafContent { + Regular(RegularFile), BlockDevice(u64), CharacterDevice(u64), Fifo, @@ -36,21 +36,21 @@ pub enum LeafContent { } #[derive(Debug)] -pub struct Leaf { +pub struct Leaf { pub stat: Stat, - pub content: LeafContent, + pub content: LeafContent, } #[derive(Debug)] -pub struct Directory { +pub struct Directory { pub stat: Stat, - entries: BTreeMap, Inode>, + entries: BTreeMap, Inode>, } #[derive(Debug)] -pub enum Inode { - Directory(Box), - Leaf(Rc), +pub enum Inode { + Directory(Box>), + Leaf(Rc>), } #[derive(Error, Debug)] @@ -67,7 +67,7 @@ pub enum ImageError { IsNotRegular(Box), } -impl Inode { +impl Inode { pub fn stat(&self) -> &Stat { match self { Inode::Directory(dir) => &dir.stat, @@ -76,7 +76,7 @@ impl Inode { } } -impl Directory { +impl Directory { pub fn new(stat: Stat) -> Self { Self { stat, @@ -85,7 +85,7 @@ impl Directory { } /// Iterates over all inodes in the current directory, in no particular order. - pub fn inodes(&self) -> impl Iterator + use<'_> { + pub fn inodes(&self) -> impl Iterator> + use<'_, ObjectID> { self.entries.values() } @@ -96,7 +96,8 @@ impl Directory { /// point. /// /// ``` - /// let fs = composefs::image::FileSystem::new(); + /// use composefs::{image::FileSystem, fsverity::Sha256HashValue}; + /// let fs = FileSystem::::new(); /// /// // populate the fs... /// @@ -104,13 +105,15 @@ impl Directory { /// // name: &OsStr, inode: &Inode /// } /// ``` - pub fn entries(&self) -> impl Iterator + use<'_> { + pub fn entries(&self) -> impl Iterator)> + use<'_, ObjectID> { self.entries.iter().map(|(k, v)| (k.as_ref(), v)) } /// Iterates over all entries in the current directory, in asciibetical order of name. The /// iterator returns pairs of `(&OsStr, &Inode)`. - pub fn sorted_entries(&self) -> impl Iterator + use<'_> { + pub fn sorted_entries( + &self, + ) -> impl Iterator)> + use<'_, ObjectID> { self.entries.iter().map(|(k, v)| (k.as_ref(), v)) } @@ -132,7 +135,7 @@ impl Directory { /// On success, this returns a reference to the named directory. /// /// On failure, can return any number of errors from ImageError. - pub fn get_directory(&self, pathname: &OsStr) -> Result<&Directory, ImageError> { + pub fn get_directory(&self, pathname: &OsStr) -> Result<&Directory, ImageError> { let path = Path::new(pathname); let mut dir = self; @@ -156,7 +159,10 @@ impl Directory { /// Gets a mutable reference to a subdirectory of this directory. /// /// This is the mutable version of `Directory::get_directory()`. - pub fn get_directory_mut(&mut self, pathname: &OsStr) -> Result<&mut Directory, ImageError> { + pub fn get_directory_mut( + &mut self, + pathname: &OsStr, + ) -> Result<&mut Directory, ImageError> { let path = Path::new(pathname); let mut dir = self; @@ -200,7 +206,7 @@ impl Directory { pub fn split<'d, 'n>( &'d self, pathname: &'n OsStr, - ) -> Result<(&'d Directory, &'n OsStr), ImageError> { + ) -> Result<(&'d Directory, &'n OsStr), ImageError> { let path = Path::new(pathname); let Some(filename) = path.file_name() else { @@ -222,7 +228,7 @@ impl Directory { pub fn split_mut<'d, 'n>( &'d mut self, pathname: &'n OsStr, - ) -> Result<(&'d mut Directory, &'n OsStr), ImageError> { + ) -> Result<(&'d mut Directory, &'n OsStr), ImageError> { let path = Path::new(pathname); let Some(filename) = path.file_name() else { @@ -252,7 +258,7 @@ impl Directory { /// is returned. /// /// On failure, can return any number of errors from ImageError. - pub fn ref_leaf(&self, filename: &OsStr) -> Result, ImageError> { + pub fn ref_leaf(&self, filename: &OsStr) -> Result>, ImageError> { match self.entries.get(filename) { Some(Inode::Leaf(leaf)) => Ok(Rc::clone(leaf)), Some(Inode::Directory(..)) => Err(ImageError::IsADirectory(Box::from(filename))), @@ -275,7 +281,10 @@ impl Directory { /// * an external reference, with size information /// /// On failure, can return any number of errors from ImageError. - pub fn get_file<'a>(&'a self, filename: &OsStr) -> Result<&'a RegularFile, ImageError> { + pub fn get_file<'a>( + &'a self, + filename: &OsStr, + ) -> Result<&'a RegularFile, ImageError> { match self.entries.get(filename) { Some(Inode::Leaf(leaf)) => match &leaf.content { LeafContent::Regular(file) => Ok(file), @@ -301,7 +310,7 @@ impl Directory { /// * `filename`: the filename in the current directory. If you need to support full /// pathnames then you should call `Directory::split()` first. /// * `inode`: the inode to store under the `filename` - pub fn merge(&mut self, filename: &OsStr, inode: Inode) { + pub fn merge(&mut self, filename: &OsStr, inode: Inode) { // If we're putting a directory on top of a directory, then update the stat information but // keep the old entries in place. if let Inode::Directory(new_dir) = inode { @@ -328,7 +337,7 @@ impl Directory { /// * `filename`: the filename in the current directory. If you need to support full /// pathnames then you should call `Directory::split()` first. /// * `inode`: the inode to store under the `filename` - pub fn insert(&mut self, filename: &OsStr, inode: Inode) { + pub fn insert(&mut self, filename: &OsStr, inode: Inode) { self.entries.insert(Box::from(filename), inode); } @@ -365,17 +374,17 @@ impl Directory { } #[derive(Debug)] -pub struct FileSystem { - pub root: Directory, +pub struct FileSystem { + pub root: Directory, } -impl Default for FileSystem { +impl Default for FileSystem { fn default() -> Self { Self::new() } } -impl FileSystem { +impl FileSystem { pub fn new() -> Self { Self { root: Directory::new(Stat { diff --git a/src/oci/image.rs b/src/oci/image.rs index bd5baecd..de115814 100644 --- a/src/oci/image.rs +++ b/src/oci/image.rs @@ -6,7 +6,7 @@ use oci_spec::image::ImageConfiguration; use crate::{ dumpfile::write_dumpfile, erofs::writer::mkfs_erofs, - fsverity::Sha256HashValue, + fsverity::FsVerityHashValue, image::{Directory, FileSystem, Inode, Leaf}, oci::{ self, @@ -16,7 +16,10 @@ use crate::{ selabel::selabel, }; -pub fn process_entry(filesystem: &mut FileSystem, entry: TarEntry) -> Result<()> { +pub fn process_entry( + filesystem: &mut FileSystem, + entry: TarEntry, +) -> Result<()> { let inode = match entry.item { TarItem::Directory => Inode::Directory(Box::from(Directory::new(entry.stat))), TarItem::Leaf(content) => Inode::Leaf(Rc::new(Leaf { @@ -46,8 +49,11 @@ pub fn process_entry(filesystem: &mut FileSystem, entry: TarEntry) -> Result<()> Ok(()) } -pub fn compose_filesystem(repo: &Repository, layers: &[String]) -> Result { - let mut filesystem = FileSystem::new(); +pub fn compose_filesystem( + repo: &Repository, + layers: &[String], +) -> Result> { + let mut filesystem = FileSystem::::new(); for layer in layers { let mut split_stream = repo.open_stream(layer, None)?; @@ -62,19 +68,19 @@ pub fn compose_filesystem(repo: &Repository, layers: &[String]) -> Result Result<()> { +pub fn create_dumpfile(repo: &Repository, layers: &[String]) -> Result<()> { let filesystem = compose_filesystem(repo, layers)?; let mut stdout = std::io::stdout(); write_dumpfile(&mut stdout, &filesystem)?; Ok(()) } -pub fn create_image( - repo: &Repository, +pub fn create_image( + repo: &Repository, config: &str, name: Option<&str>, - verity: Option<&Sha256HashValue>, -) -> Result { + verity: Option<&ObjectID>, +) -> Result { let mut filesystem = FileSystem::new(); let mut config_stream = repo.open_stream(config, verity)?; @@ -98,82 +104,85 @@ pub fn create_image( } #[cfg(test)] -use crate::image::{LeafContent, RegularFile, Stat}; -#[cfg(test)] -use std::{cell::RefCell, collections::BTreeMap, io::BufRead, path::PathBuf}; - -#[cfg(test)] -fn file_entry(path: &str) -> oci::tar::TarEntry { - oci::tar::TarEntry { - path: PathBuf::from(path), - stat: Stat { - st_mode: 0o644, - st_uid: 0, - st_gid: 0, - st_mtim_sec: 0, - xattrs: RefCell::new(BTreeMap::new()), - }, - item: oci::tar::TarItem::Leaf(LeafContent::Regular(RegularFile::Inline([].into()))), +mod test { + use crate::{ + fsverity::Sha256HashValue, + image::{LeafContent, RegularFile, Stat}, + }; + use std::{cell::RefCell, collections::BTreeMap, io::BufRead, path::PathBuf}; + + use super::*; + + fn file_entry(path: &str) -> oci::tar::TarEntry { + oci::tar::TarEntry { + path: PathBuf::from(path), + stat: Stat { + st_mode: 0o644, + st_uid: 0, + st_gid: 0, + st_mtim_sec: 0, + xattrs: RefCell::new(BTreeMap::new()), + }, + item: oci::tar::TarItem::Leaf(LeafContent::Regular(RegularFile::Inline([].into()))), + } } -} -#[cfg(test)] -fn dir_entry(path: &str) -> oci::tar::TarEntry { - oci::tar::TarEntry { - path: PathBuf::from(path), - stat: Stat { - st_mode: 0o755, - st_uid: 0, - st_gid: 0, - st_mtim_sec: 0, - xattrs: RefCell::new(BTreeMap::new()), - }, - item: oci::tar::TarItem::Directory, + fn dir_entry(path: &str) -> oci::tar::TarEntry { + oci::tar::TarEntry { + path: PathBuf::from(path), + stat: Stat { + st_mode: 0o755, + st_uid: 0, + st_gid: 0, + st_mtim_sec: 0, + xattrs: RefCell::new(BTreeMap::new()), + }, + item: oci::tar::TarItem::Directory, + } } -} -#[cfg(test)] -fn assert_files(fs: &FileSystem, expected: &[&str]) -> Result<()> { - let mut out = vec![]; - write_dumpfile(&mut out, fs)?; - let actual: Vec = out - .lines() - .map(|line| line.unwrap().split_once(' ').unwrap().0.into()) - .collect(); - - similar_asserts::assert_eq!(actual, expected); - Ok(()) -} + fn assert_files(fs: &FileSystem, expected: &[&str]) -> Result<()> { + let mut out = vec![]; + write_dumpfile(&mut out, fs)?; + let actual: Vec = out + .lines() + .map(|line| line.unwrap().split_once(' ').unwrap().0.into()) + .collect(); -#[test] -fn test_process_entry() -> Result<()> { - let mut fs = FileSystem::new(); - - // both with and without leading slash should be supported - process_entry(&mut fs, dir_entry("/a"))?; - process_entry(&mut fs, dir_entry("b"))?; - process_entry(&mut fs, dir_entry("c"))?; - assert_files(&fs, &["/", "/a", "/b", "/c"])?; - - // add some files - process_entry(&mut fs, file_entry("/a/b"))?; - process_entry(&mut fs, file_entry("/a/c"))?; - process_entry(&mut fs, file_entry("/b/a"))?; - process_entry(&mut fs, file_entry("/b/c"))?; - process_entry(&mut fs, file_entry("/c/a"))?; - process_entry(&mut fs, file_entry("/c/c"))?; - assert_files( - &fs, - &[ - "/", "/a", "/a/b", "/a/c", "/b", "/b/a", "/b/c", "/c", "/c/a", "/c/c", - ], - )?; - - // try some whiteouts - process_entry(&mut fs, file_entry(".wh.a"))?; // entire dir - process_entry(&mut fs, file_entry("/b/.wh..wh.opq"))?; // opaque dir - process_entry(&mut fs, file_entry("/c/.wh.c"))?; // single file - assert_files(&fs, &["/", "/b", "/c", "/c/a"])?; + similar_asserts::assert_eq!(actual, expected); + Ok(()) + } - Ok(()) + #[test] + fn test_process_entry() -> Result<()> { + let mut fs = FileSystem::::new(); + + // both with and without leading slash should be supported + process_entry(&mut fs, dir_entry("/a"))?; + process_entry(&mut fs, dir_entry("b"))?; + process_entry(&mut fs, dir_entry("c"))?; + assert_files(&fs, &["/", "/a", "/b", "/c"])?; + + // add some files + process_entry(&mut fs, file_entry("/a/b"))?; + process_entry(&mut fs, file_entry("/a/c"))?; + process_entry(&mut fs, file_entry("/b/a"))?; + process_entry(&mut fs, file_entry("/b/c"))?; + process_entry(&mut fs, file_entry("/c/a"))?; + process_entry(&mut fs, file_entry("/c/c"))?; + assert_files( + &fs, + &[ + "/", "/a", "/a/b", "/a/c", "/b", "/b/a", "/b/c", "/c", "/c/a", "/c/c", + ], + )?; + + // try some whiteouts + process_entry(&mut fs, file_entry(".wh.a"))?; // entire dir + process_entry(&mut fs, file_entry("/b/.wh..wh.opq"))?; // opaque dir + process_entry(&mut fs, file_entry("/c/.wh.c"))?; // single file + assert_files(&fs, &["/", "/b", "/c", "/c/a"])?; + + Ok(()) + } } diff --git a/src/oci/mod.rs b/src/oci/mod.rs index 087a067f..63497416 100644 --- a/src/oci/mod.rs +++ b/src/oci/mod.rs @@ -15,23 +15,26 @@ use tokio::io::AsyncReadExt; use crate::{ fs::write_to_path, - fsverity::Sha256HashValue, + fsverity::FsVerityHashValue, oci::tar::{get_entry, split_async}, repository::Repository, splitstream::DigestMap, - util::parse_sha256, + util::{parse_sha256, Sha256Digest}, }; -pub fn import_layer( - repo: &Repository, - sha256: &Sha256HashValue, +pub fn import_layer( + repo: &Repository, + sha256: &Sha256Digest, name: Option<&str>, tar_stream: &mut impl Read, -) -> Result { +) -> Result { repo.ensure_stream(sha256, |writer| tar::split(tar_stream, writer), name) } -pub fn ls_layer(repo: &Repository, name: &str) -> Result<()> { +pub fn ls_layer( + repo: &Repository, + name: &str, +) -> Result<()> { let mut split_stream = repo.open_stream(name, None)?; while let Some(entry) = get_entry(&mut split_stream)? { @@ -41,31 +44,31 @@ pub fn ls_layer(repo: &Repository, name: &str) -> Result<()> { Ok(()) } -struct ImageOp<'repo> { - repo: &'repo Repository, +struct ImageOp<'repo, ObjectID: FsVerityHashValue> { + repo: &'repo Repository, proxy: ImageProxy, img: OpenedImage, progress: MultiProgress, } -fn sha256_from_descriptor(descriptor: &Descriptor) -> Result { +fn sha256_from_descriptor(descriptor: &Descriptor) -> Result { let Some(digest) = descriptor.as_digest_sha256() else { bail!("Descriptor in oci config is not sha256"); }; Ok(parse_sha256(digest)?) } -fn sha256_from_digest(digest: &str) -> Result { +fn sha256_from_digest(digest: &str) -> Result { match digest.strip_prefix("sha256:") { Some(rest) => Ok(parse_sha256(rest)?), None => bail!("Manifest has non-sha256 digest"), } } -type ContentAndVerity = (Sha256HashValue, Sha256HashValue); +type ContentAndVerity = (Sha256Digest, ObjectID); -impl<'repo> ImageOp<'repo> { - async fn new(repo: &'repo Repository, imgref: &str) -> Result { +impl<'repo, ObjectID: FsVerityHashValue> ImageOp<'repo, ObjectID> { + async fn new(repo: &'repo Repository, imgref: &str) -> Result { // See https://github.com/containers/skopeo/issues/2563 let skopeo_cmd = if imgref.starts_with("containers-storage:") { let mut cmd = Command::new("podman"); @@ -93,16 +96,16 @@ impl<'repo> ImageOp<'repo> { pub async fn ensure_layer( &self, - layer_sha256: &Sha256HashValue, + layer_sha256: &Sha256Digest, descriptor: &Descriptor, - ) -> Result { + ) -> Result { // We need to use the per_manifest descriptor to download the compressed layer but it gets // stored in the repository via the per_config descriptor. Our return value is the // fsverity digest for the corresponding splitstream. if let Some(layer_id) = self.repo.check_stream(layer_sha256)? { self.progress - .println(format!("Already have layer {}", hex::encode(layer_sha256)))?; + .println(format!("Already have layer {layer_sha256:?}"))?; Ok(layer_id) } else { // Otherwise, we need to fetch it... @@ -142,7 +145,7 @@ impl<'repo> ImageOp<'repo> { &self, manifest_layers: &[Descriptor], descriptor: &Descriptor, - ) -> Result { + ) -> Result> { let config_sha256 = sha256_from_descriptor(descriptor)?; if let Some(config_id) = self.repo.check_stream(&config_sha256)? { // We already got this config? Nice. @@ -189,7 +192,7 @@ impl<'repo> ImageOp<'repo> { } } - pub async fn pull(&self) -> Result<(Sha256HashValue, Sha256HashValue)> { + pub async fn pull(&self) -> Result> { let (_manifest_digest, raw_manifest) = self .proxy .fetch_manifest_raw_oci(&self.img) @@ -209,7 +212,11 @@ impl<'repo> ImageOp<'repo> { /// Pull the target image, and add the provided tag. If this is a mountable /// image (i.e. not an artifact), it is *not* unpacked by default. -pub async fn pull(repo: &Repository, imgref: &str, reference: Option<&str>) -> Result<()> { +pub async fn pull( + repo: &Repository, + imgref: &str, + reference: Option<&str>, +) -> Result<()> { let op = ImageOp::new(repo, imgref).await?; let (sha256, id) = op .pull() @@ -220,15 +227,15 @@ pub async fn pull(repo: &Repository, imgref: &str, reference: Option<&str>) -> R repo.name_stream(sha256, name)?; } println!("sha256 {}", hex::encode(sha256)); - println!("verity {}", hex::encode(id)); + println!("verity {}", id.to_hex()); Ok(()) } -pub fn open_config( - repo: &Repository, +pub fn open_config( + repo: &Repository, name: &str, - verity: Option<&Sha256HashValue>, -) -> Result<(ImageConfiguration, DigestMap)> { + verity: Option<&ObjectID>, +) -> Result<(ImageConfiguration, DigestMap)> { let id = match verity { Some(id) => id, None => { @@ -245,16 +252,16 @@ pub fn open_config( Ok((config, stream.refs)) } -fn hash(bytes: &[u8]) -> Sha256HashValue { +fn hash(bytes: &[u8]) -> Sha256Digest { let mut context = Sha256::new(); context.update(bytes); context.finalize().into() } -pub fn open_config_shallow( - repo: &Repository, +pub fn open_config_shallow( + repo: &Repository, name: &str, - verity: Option<&Sha256HashValue>, + verity: Option<&ObjectID>, ) -> Result { match verity { // with verity deep opens are just as fast as shallow ones @@ -272,11 +279,11 @@ pub fn open_config_shallow( } } -pub fn write_config( - repo: &Repository, +pub fn write_config( + repo: &Repository, config: &ImageConfiguration, - refs: DigestMap, -) -> Result<(Sha256HashValue, Sha256HashValue)> { + refs: DigestMap, +) -> Result> { let json = config.to_string()?; let json_bytes = json.as_bytes(); let sha256 = hash(json_bytes); @@ -286,25 +293,25 @@ pub fn write_config( Ok((sha256, id)) } -pub fn seal( - repo: &Repository, +pub fn seal( + repo: &Repository, name: &str, - verity: Option<&Sha256HashValue>, -) -> Result<(Sha256HashValue, Sha256HashValue)> { + verity: Option<&ObjectID>, +) -> Result> { let (mut config, refs) = open_config(repo, name, verity)?; let mut myconfig = config.config().clone().context("no config!")?; let labels = myconfig.labels_mut().get_or_insert_with(HashMap::new); let id = crate::oci::image::create_image(repo, name, None, verity)?; - labels.insert("containers.composefs.fsverity".to_string(), hex::encode(id)); + labels.insert("containers.composefs.fsverity".to_string(), id.to_hex()); config.set_config(Some(myconfig)); write_config(repo, &config, refs) } -pub fn mount( - repo: &Repository, +pub fn mount( + repo: &Repository, name: &str, mountpoint: &str, - verity: Option<&Sha256HashValue>, + verity: Option<&ObjectID>, ) -> Result<()> { let config = open_config_shallow(repo, name, verity)?; let Some(id) = config.get_config_annotation("containers.composefs.fsverity") else { @@ -313,7 +320,11 @@ pub fn mount( repo.mount(id, mountpoint) } -pub fn meta_layer(repo: &Repository, name: &str, verity: Option<&Sha256HashValue>) -> Result<()> { +pub fn meta_layer( + repo: &Repository, + name: &str, + verity: Option<&ObjectID>, +) -> Result<()> { let (config, refs) = open_config(repo, name, verity)?; let ids = config.rootfs().diff_ids(); @@ -330,10 +341,10 @@ pub fn meta_layer(repo: &Repository, name: &str, verity: Option<&Sha256HashValue } } -pub fn prepare_boot( - repo: &Repository, +pub fn prepare_boot( + repo: &Repository, name: &str, - verity: Option<&Sha256HashValue>, + verity: Option<&ObjectID>, output_dir: &Path, ) -> Result<()> { let (config, refs) = open_config(repo, name, verity)?; @@ -377,7 +388,7 @@ mod test { use rustix::fs::CWD; use sha2::{Digest, Sha256}; - use crate::{repository::Repository, test::tempdir}; + use crate::{fsverity::Sha256HashValue, repository::Repository, test::tempdir}; use super::*; @@ -410,7 +421,7 @@ mod test { let layer_id: [u8; 32] = context.finalize().into(); let repo_dir = tempdir(); - let repo = Repository::open_path(CWD, &repo_dir).unwrap(); + let repo = Repository::::open_path(CWD, &repo_dir).unwrap(); let id = import_layer(&repo, &layer_id, Some("name"), &mut layer.as_slice()).unwrap(); let mut dump = String::new(); diff --git a/src/oci/tar.rs b/src/oci/tar.rs index 718a1670..4f3a0cb5 100644 --- a/src/oci/tar.rs +++ b/src/oci/tar.rs @@ -15,6 +15,7 @@ use tokio::io::{AsyncRead, AsyncReadExt}; use crate::{ dumpfile, + fsverity::FsVerityHashValue, image::{LeafContent, RegularFile, Stat}, splitstream::{SplitStreamData, SplitStreamReader, SplitStreamWriter}, util::{read_exactish, read_exactish_async}, @@ -42,7 +43,10 @@ async fn read_header_async(reader: &mut (impl AsyncRead + Unpin)) -> Result(tar_stream: &mut R, writer: &mut SplitStreamWriter) -> Result<()> { +pub fn split( + tar_stream: &mut impl Read, + writer: &mut SplitStreamWriter, +) -> Result<()> { while let Some(header) = read_header(tar_stream)? { // the header always gets stored as inline data writer.write_inline(header.as_bytes()); @@ -71,7 +75,7 @@ pub fn split(tar_stream: &mut R, writer: &mut SplitStreamWriter) -> Res pub async fn split_async( mut tar_stream: impl AsyncRead + Unpin, - writer: &mut SplitStreamWriter<'_>, + writer: &mut SplitStreamWriter<'_, impl FsVerityHashValue>, ) -> Result<()> { while let Some(header) = read_header_async(&mut tar_stream).await? { // the header always gets stored as inline data @@ -100,20 +104,20 @@ pub async fn split_async( } #[derive(Debug)] -pub enum TarItem { +pub enum TarItem { Directory, - Leaf(LeafContent), + Leaf(LeafContent), Hardlink(OsString), } #[derive(Debug)] -pub struct TarEntry { +pub struct TarEntry { pub path: PathBuf, pub stat: Stat, - pub item: TarItem, + pub item: TarItem, } -impl fmt::Display for TarEntry { +impl fmt::Display for TarEntry { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { match self.item { TarItem::Hardlink(ref target) => dumpfile::write_hardlink(fmt, &self.path, target), @@ -156,7 +160,9 @@ fn symlink_target_from_tar(pax: Option>, gnu: Vec, short: &[u8]) - } } -pub fn get_entry(reader: &mut SplitStreamReader) -> Result> { +pub fn get_entry( + reader: &mut SplitStreamReader, +) -> Result>> { let mut gnu_longlink: Vec = vec![]; let mut gnu_longname: Vec = vec![]; let mut pax_longlink: Option> = None; @@ -183,11 +189,7 @@ pub fn get_entry(reader: &mut SplitStreamReader) -> Result bail!( - "Unsupported external-chunked entry {:?} {}", - header, - hex::encode(id) - ), + _ => bail!("Unsupported external-chunked entry {header:?} {id:?}"), }, SplitStreamData::Inline(content) => match header.entry_type() { EntryType::GNULongLink => { diff --git a/src/repository.rs b/src/repository.rs index ac24f8fe..66d343fa 100644 --- a/src/repository.rs +++ b/src/repository.rs @@ -20,25 +20,25 @@ use sha2::{Digest, Sha256}; use crate::{ fsverity::{ compute_verity, enable_verity, ensure_verity_equal, measure_verity, FsVerityHashValue, - Sha256HashValue, }, mount::mount_composefs_at, splitstream::{DigestMap, SplitStreamReader, SplitStreamWriter}, - util::{parse_sha256, proc_self_fd}, + util::{proc_self_fd, Sha256Digest}, }; #[derive(Debug)] -pub struct Repository { +pub struct Repository { repository: OwnedFd, + _data: std::marker::PhantomData, } -impl Drop for Repository { +impl Drop for Repository { fn drop(&mut self) { flock(&self.repository, FlockOperation::Unlock).expect("repository unlock failed"); } } -impl Repository { +impl Repository { pub fn object_dir(&self) -> ErrnoResult { self.openat( "objects", @@ -46,7 +46,7 @@ impl Repository { ) } - pub fn open_path(dirfd: impl AsFd, path: impl AsRef) -> Result { + pub fn open_path(dirfd: impl AsFd, path: impl AsRef) -> Result { let path = path.as_ref(); // O_PATH isn't enough because flock() @@ -56,17 +56,20 @@ impl Repository { flock(&repository, FlockOperation::LockShared) .context("Cannot lock composefs repository")?; - Ok(Repository { repository }) + Ok(Self { + repository, + _data: std::marker::PhantomData, + }) } - pub fn open_user() -> Result { + pub fn open_user() -> Result { let home = std::env::var("HOME").with_context(|| "$HOME must be set when in user mode")?; - Repository::open_path(CWD, PathBuf::from(home).join(".var/lib/composefs")) + Self::open_path(CWD, PathBuf::from(home).join(".var/lib/composefs")) } - pub fn open_system() -> Result { - Repository::open_path(CWD, PathBuf::from("/sysroot/composefs".to_string())) + pub fn open_system() -> Result { + Self::open_path(CWD, PathBuf::from("/sysroot/composefs".to_string())) } fn ensure_dir(&self, dir: impl AsRef) -> ErrnoResult<()> { @@ -76,10 +79,10 @@ impl Repository { }) } - pub fn ensure_object(&self, data: &[u8]) -> Result { - let digest: Sha256HashValue = compute_verity(data); - let dir = PathBuf::from(format!("objects/{:02x}", digest[0])); - let file = dir.join(hex::encode(&digest[1..])); + pub fn ensure_object(&self, data: &[u8]) -> Result { + let digest: ObjectID = compute_verity(data); + let dir = PathBuf::from(format!("objects/{}", digest.to_object_dir())); + let file = dir.join(digest.to_object_basename()); // fairly common... if accessat(&self.repository, &file, Access::READ_OK, AtFlags::empty()) == Ok(()) { @@ -102,7 +105,7 @@ impl Repository { let ro_fd = open(proc_self_fd(&fd), OFlags::RDONLY, Mode::empty())?; drop(fd); - enable_verity::(&ro_fd).context("Enabling verity digest")?; + enable_verity::(&ro_fd).context("Enabling verity digest")?; ensure_verity_equal(&ro_fd, &digest).context("Double-checking verity digest")?; if let Err(err) = linkat( @@ -121,11 +124,7 @@ impl Repository { Ok(digest) } - fn open_with_verity( - &self, - filename: &str, - expected_verity: &Sha256HashValue, - ) -> Result { + fn open_with_verity(&self, filename: &str, expected_verity: &ObjectID) -> Result { let fd = self.openat(filename, OFlags::RDONLY)?; ensure_verity_equal(&fd, expected_verity)?; Ok(fd) @@ -136,37 +135,17 @@ impl Repository { /// store the result. pub fn create_stream( &self, - sha256: Option, - maps: Option, - ) -> SplitStreamWriter { + sha256: Option, + maps: Option>, + ) -> SplitStreamWriter { SplitStreamWriter::new(self, maps, sha256) } - fn parse_object_path(path: impl AsRef<[u8]>) -> Result { - // "objects/0c/9513d99b120ee9a709c4d6554d938f6b2b7e213cf5b26f2e255c0b77e40379" - let bytes = path.as_ref(); - ensure!(bytes.len() == 73, "stream symlink has incorrect length"); - ensure!( - bytes.starts_with(b"objects/"), - "stream symlink has incorrect prefix" - ); - ensure!( - bytes[10] == b'/', - "stream symlink has incorrect path separator" - ); - let mut result = Sha256HashValue::EMPTY; - hex::decode_to_slice(&bytes[8..10], &mut result[..1]) - .context("stream symlink has incorrect format")?; - hex::decode_to_slice(&bytes[11..], &mut result[1..]) - .context("stream symlink has incorrect format")?; - Ok(result) + fn format_object_path(id: &ObjectID) -> String { + format!("objects/{}", id.to_object_pathname()) } - fn format_object_path(id: &Sha256HashValue) -> String { - format!("objects/{:02x}/{}", id[0], hex::encode(&id[1..])) - } - - pub fn has_stream(&self, sha256: &Sha256HashValue) -> Result> { + pub fn has_stream(&self, sha256: &Sha256Digest) -> Result> { let stream_path = format!("streams/{}", hex::encode(sha256)); match readlinkat(&self.repository, &stream_path, []) { @@ -183,7 +162,7 @@ impl Repository { bytes.starts_with(b"../"), "stream symlink has incorrect prefix" ); - Ok(Some(Repository::parse_object_path(&bytes[3..])?)) + Ok(Some(ObjectID::from_object_pathname(bytes)?)) } Err(Errno::NOENT) => Ok(None), Err(err) => Err(err)?, @@ -191,16 +170,16 @@ impl Repository { } /// Basically the same as has_stream() except that it performs expensive verification - pub fn check_stream(&self, sha256: &Sha256HashValue) -> Result> { + pub fn check_stream(&self, sha256: &Sha256Digest) -> Result> { match self.openat(&format!("streams/{}", hex::encode(sha256)), OFlags::RDONLY) { Ok(stream) => { - let measured_verity: Sha256HashValue = measure_verity(&stream)?; + let measured_verity: ObjectID = measure_verity(&stream)?; let mut context = Sha256::new(); let mut split_stream = SplitStreamReader::new(File::from(stream))?; // check the verity of all linked streams for entry in &split_stream.refs.map { - if self.check_stream(&entry.body)? != Some(entry.verity) { + if self.check_stream(&entry.body)?.as_ref() != Some(&entry.verity) { bail!("reference mismatch"); } } @@ -224,15 +203,15 @@ impl Repository { pub fn write_stream( &self, - writer: SplitStreamWriter, + writer: SplitStreamWriter, reference: Option<&str>, - ) -> Result { + ) -> Result { let Some((.., ref sha256)) = writer.sha256 else { bail!("Writer doesn't have sha256 enabled"); }; let stream_path = format!("streams/{}", hex::encode(sha256)); let object_id = writer.done()?; - let object_path = Repository::format_object_path(&object_id); + let object_path = Self::format_object_path(&object_id); self.ensure_symlink(&stream_path, &object_path)?; if let Some(name) = reference { @@ -245,7 +224,7 @@ impl Repository { /// Assign the given name to a stream. The stream must already exist. After this operation it /// will be possible to refer to the stream by its new name 'refs/{name}'. - pub fn name_stream(&self, sha256: Sha256HashValue, name: &str) -> Result<()> { + pub fn name_stream(&self, sha256: Sha256Digest, name: &str) -> Result<()> { let stream_path = format!("streams/{}", hex::encode(sha256)); let reference_path = format!("streams/refs/{name}"); self.symlink(&reference_path, &stream_path)?; @@ -268,10 +247,10 @@ impl Repository { /// ID will be used when referring to the stream from other linked streams. pub fn ensure_stream( &self, - sha256: &Sha256HashValue, - callback: impl FnOnce(&mut SplitStreamWriter) -> Result<()>, + sha256: &Sha256Digest, + callback: impl FnOnce(&mut SplitStreamWriter) -> Result<()>, reference: Option<&str>, - ) -> Result { + ) -> Result { let stream_path = format!("streams/{}", hex::encode(sha256)); let object_id = match self.has_stream(sha256)? { @@ -281,7 +260,7 @@ impl Repository { callback(&mut writer)?; let object_id = writer.done()?; - let object_path = Repository::format_object_path(&object_id); + let object_path = Self::format_object_path(&object_id); self.ensure_symlink(&stream_path, &object_path)?; object_id } @@ -298,8 +277,8 @@ impl Repository { pub fn open_stream( &self, name: &str, - verity: Option<&Sha256HashValue>, - ) -> Result> { + verity: Option<&ObjectID>, + ) -> Result> { let filename = format!("streams/{}", name); let file = File::from(if let Some(verity_hash) = verity { @@ -311,17 +290,14 @@ impl Repository { SplitStreamReader::new(file) } - pub fn open_object(&self, id: &Sha256HashValue) -> Result { - self.open_with_verity( - &format!("objects/{:02x}/{}", id[0], hex::encode(&id[1..])), - id, - ) + pub fn open_object(&self, id: &ObjectID) -> Result { + self.open_with_verity(&Self::format_object_path(id), id) } pub fn merge_splitstream( &self, name: &str, - verity: Option<&Sha256HashValue>, + verity: Option<&ObjectID>, stream: &mut impl Write, ) -> Result<()> { let mut split_stream = self.open_stream(name, verity)?; @@ -335,15 +311,11 @@ impl Repository { } /// this function is not safe for untrusted users - pub fn write_image(&self, name: Option<&str>, data: &[u8]) -> Result { + pub fn write_image(&self, name: Option<&str>, data: &[u8]) -> Result { let object_id = self.ensure_object(data)?; - let object_path = format!( - "objects/{:02x}/{}", - object_id[0], - hex::encode(&object_id[1..]) - ); - let image_path = format!("images/{}", hex::encode(object_id)); + let object_path = Self::format_object_path(&object_id); + let image_path = format!("images/{}", object_id.to_hex()); self.ensure_symlink(&image_path, &object_path)?; @@ -356,7 +328,7 @@ impl Repository { } /// this function is not safe for untrusted users - pub fn import_image(&self, name: &str, image: &mut R) -> Result { + pub fn import_image(&self, name: &str, image: &mut R) -> Result { let mut data = vec![]; image.read_to_end(&mut data)?; self.write_image(Some(name), &data) @@ -367,7 +339,7 @@ impl Repository { if !name.contains("/") { // A name with no slashes in it is taken to be a sha256 fs-verity digest - ensure_verity_equal(&image, &parse_sha256(name)?)?; + ensure_verity_equal(&image, &ObjectID::from_hex(name)?)?; } Ok(image) @@ -421,22 +393,12 @@ impl Repository { }) } - fn read_symlink_hashvalue(dirfd: &OwnedFd, name: &CStr) -> Result { + fn read_symlink_hashvalue(dirfd: &OwnedFd, name: &CStr) -> Result { let link_content = readlinkat(dirfd, name, [])?; - let link_bytes = link_content.to_bytes(); - let link_size = link_bytes.len(); - // XXX: check correctness of leading ../? - // XXX: or is that something for fsck? - if link_size > 64 { - let mut value = Sha256HashValue::EMPTY; - hex::decode_to_slice(&link_bytes[link_size - 64..link_size], &mut value)?; - Ok(value) - } else { - bail!("symlink has wrong format") - } + Ok(ObjectID::from_object_pathname(link_content.to_bytes())?) } - fn walk_symlinkdir(fd: OwnedFd, objects: &mut HashSet) -> Result<()> { + fn walk_symlinkdir(fd: OwnedFd, objects: &mut HashSet) -> Result<()> { for item in Dir::read_from(&fd)? { let entry = item?; // NB: the underlying filesystem must support returning filetype via direntry @@ -446,11 +408,11 @@ impl Repository { let filename = entry.file_name(); if filename != c"." && filename != c".." { let dirfd = openat(&fd, filename, OFlags::RDONLY, Mode::empty())?; - Repository::walk_symlinkdir(dirfd, objects)?; + Self::walk_symlinkdir(dirfd, objects)?; } } FileType::Symlink => { - objects.insert(Repository::read_symlink_hashvalue(&fd, entry.file_name())?); + objects.insert(Self::read_symlink_hashvalue(&fd, entry.file_name())?); } _ => { bail!("Unexpected file type encountered"); @@ -471,8 +433,8 @@ impl Repository { ) } - fn gc_category(&self, category: &str) -> Result> { - let mut objects = HashSet::::new(); + fn gc_category(&self, category: &str) -> Result> { + let mut objects = HashSet::new(); let category_fd = self.openat(category, OFlags::RDONLY | OFlags::DIRECTORY)?; @@ -482,7 +444,7 @@ impl Repository { OFlags::RDONLY | OFlags::DIRECTORY, Mode::empty(), )?; - Repository::walk_symlinkdir(refs, &mut objects)?; + Self::walk_symlinkdir(refs, &mut objects)?; for item in Dir::read_from(&category_fd)? { let entry = item?; @@ -491,19 +453,26 @@ impl Repository { if entry.file_type() != FileType::Symlink { bail!("category directory contains non-symlink"); } + + // TODO: we need to sort this out. the symlink itself might be a sha256 content ID + // (as for splitstreams), not an object/ to be preserved. + continue; + + /* let mut value = Sha256HashValue::EMPTY; hex::decode_to_slice(filename.to_bytes(), &mut value)?; if !objects.contains(&value) { println!("rm {}/{:?}", category, filename); } + */ } } Ok(objects) } - pub fn objects_for_image(&self, name: &str) -> Result> { + pub fn objects_for_image(&self, name: &str) -> Result> { let image = self.open_image(name)?; let mut data = vec![]; std::fs::File::from(image).read_to_end(&mut data)?; @@ -516,19 +485,19 @@ impl Repository { let mut objects = HashSet::new(); for ref object in self.gc_category("images")? { - println!("{} lives as an image", hex::encode(object)); - objects.insert(*object); - objects.extend(self.objects_for_image(&hex::encode(object))?); + println!("{object:?} lives as an image"); + objects.insert(object.clone()); + objects.extend(self.objects_for_image(&object.to_hex())?); } for object in self.gc_category("streams")? { - println!("{} lives as a stream", hex::encode(object)); - objects.insert(object); + println!("{object:?} lives as a stream"); + objects.insert(object.clone()); - let mut split_stream = self.open_stream(&hex::encode(object), None)?; + let mut split_stream = self.open_stream(&object.to_hex(), None)?; split_stream.get_object_refs(|id| { - println!(" with {}", hex::encode(*id)); - objects.insert(*id); + println!(" with {id:?}"); + objects.insert(id.clone()); })?; } @@ -545,10 +514,9 @@ impl Repository { let entry = item?; let filename = entry.file_name(); if filename != c"." && filename != c".." { - let mut value = Sha256HashValue::EMPTY; - value[0] = first_byte; - hex::decode_to_slice(filename.to_bytes(), &mut value[1..])?; - if !objects.contains(&value) { + let id = + ObjectID::from_object_dir_and_basename(first_byte, filename.to_bytes())?; + if !objects.contains(&id) { println!("rm objects/{first_byte:02x}/{filename:?}"); } else { println!("# objects/{first_byte:02x}/{filename:?} lives"); diff --git a/src/selabel.rs b/src/selabel.rs index 71c742f1..a412a13e 100644 --- a/src/selabel.rs +++ b/src/selabel.rs @@ -11,6 +11,7 @@ use anyhow::{bail, ensure, Context, Result}; use regex_automata::{hybrid::dfa, util::syntax, Anchored, Input}; use crate::{ + fsverity::FsVerityHashValue, image::{Directory, FileSystem, ImageError, Inode, Leaf, LeafContent, RegularFile, Stat}, repository::Repository, }; @@ -105,10 +106,10 @@ struct Policy { contexts: Vec, } -pub fn openat<'a>( - dir: &'a Directory, +pub fn openat<'a, H: FsVerityHashValue>( + dir: &'a Directory, filename: impl AsRef, - repo: &Repository, + repo: &Repository, ) -> Result>> { match dir.get_file(filename.as_ref()) { Ok(RegularFile::Inline(data)) => Ok(Some(Box::new(&**data))), @@ -119,7 +120,7 @@ pub fn openat<'a>( } impl Policy { - pub fn build(dir: &Directory, repo: &Repository) -> Result { + pub fn build(dir: &Directory, repo: &Repository) -> Result { let mut aliases = HashMap::new(); let mut regexps = vec![]; let mut contexts = vec![]; @@ -201,7 +202,7 @@ fn relabel(stat: &Stat, path: &Path, ifmt: u8, policy: &mut Policy) { } } -fn relabel_leaf(leaf: &Leaf, path: &Path, policy: &mut Policy) { +fn relabel_leaf(leaf: &Leaf, path: &Path, policy: &mut Policy) { let ifmt = match leaf.content { LeafContent::Regular(..) => b'-', LeafContent::Fifo => b'p', // NB: 'pipe', not 'fifo' @@ -213,14 +214,14 @@ fn relabel_leaf(leaf: &Leaf, path: &Path, policy: &mut Policy) { relabel(&leaf.stat, path, ifmt, policy); } -fn relabel_inode(inode: &Inode, path: &mut PathBuf, policy: &mut Policy) { +fn relabel_inode(inode: &Inode, path: &mut PathBuf, policy: &mut Policy) { match inode { Inode::Directory(ref dir) => relabel_dir(dir, path, policy), Inode::Leaf(ref leaf) => relabel_leaf(leaf, path, policy), } } -fn relabel_dir(dir: &Directory, path: &mut PathBuf, policy: &mut Policy) { +fn relabel_dir(dir: &Directory, path: &mut PathBuf, policy: &mut Policy) { relabel(&dir.stat, path, b'd', policy); for (name, inode) in dir.sorted_entries() { @@ -245,7 +246,7 @@ fn parse_config(file: impl Read) -> Result> { Ok(None) } -pub fn selabel(fs: &mut FileSystem, repo: &Repository) -> Result<()> { +pub fn selabel(fs: &mut FileSystem, repo: &Repository) -> Result<()> { // if /etc/selinux/config doesn't exist then it's not an error let etc_selinux = match fs.root.get_directory("etc/selinux".as_ref()) { Err(ImageError::NotFound(..)) => return Ok(()), diff --git a/src/splitstream.rs b/src/splitstream.rs index 319ffe09..0924ff06 100644 --- a/src/splitstream.rs +++ b/src/splitstream.rs @@ -7,65 +7,67 @@ use std::io::{BufReader, Read, Write}; use anyhow::{bail, Result}; use sha2::{Digest, Sha256}; +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; use zstd::stream::{read::Decoder, write::Encoder}; use crate::{ - fsverity::{FsVerityHashValue, Sha256HashValue}, + fsverity::FsVerityHashValue, repository::Repository, - util::read_exactish, + util::{read_exactish, Sha256Digest}, }; -#[derive(Debug)] -pub struct DigestMapEntry { - pub body: Sha256HashValue, - pub verity: Sha256HashValue, +#[derive(Debug, FromBytes, Immutable, IntoBytes, KnownLayout)] +#[repr(C)] +pub struct DigestMapEntry { + pub body: Sha256Digest, + pub verity: ObjectID, } #[derive(Debug)] -pub struct DigestMap { - pub map: Vec, +pub struct DigestMap { + pub map: Vec>, } -impl Default for DigestMap { +impl Default for DigestMap { fn default() -> Self { Self::new() } } -impl DigestMap { +impl DigestMap { pub fn new() -> Self { DigestMap { map: vec![] } } - pub fn lookup(&self, body: &Sha256HashValue) -> Option<&Sha256HashValue> { + pub fn lookup(&self, body: &Sha256Digest) -> Option<&ObjectID> { match self.map.binary_search_by_key(body, |e| e.body) { Ok(idx) => Some(&self.map[idx].verity), Err(..) => None, } } - pub fn insert(&mut self, body: &Sha256HashValue, verity: &Sha256HashValue) { + pub fn insert(&mut self, body: &Sha256Digest, verity: &ObjectID) { match self.map.binary_search_by_key(body, |e| e.body) { Ok(idx) => assert_eq!(self.map[idx].verity, *verity), // or else, bad things... Err(idx) => self.map.insert( idx, DigestMapEntry { body: *body, - verity: *verity, + verity: verity.clone(), }, ), } } } -pub struct SplitStreamWriter<'a> { - repo: &'a Repository, +pub struct SplitStreamWriter<'a, ObjectID: FsVerityHashValue> { + repo: &'a Repository, inline_content: Vec, writer: Encoder<'a, Vec>, - pub sha256: Option<(Sha256, Sha256HashValue)>, + pub sha256: Option<(Sha256, Sha256Digest)>, } -impl std::fmt::Debug for SplitStreamWriter<'_> { +impl std::fmt::Debug for SplitStreamWriter<'_, ObjectID> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { // writer doesn't impl Debug f.debug_struct("SplitStreamWriter") @@ -76,29 +78,26 @@ impl std::fmt::Debug for SplitStreamWriter<'_> { } } -impl SplitStreamWriter<'_> { +impl<'a, ObjectID: FsVerityHashValue> SplitStreamWriter<'a, ObjectID> { pub fn new( - repo: &Repository, - refs: Option, - sha256: Option, - ) -> SplitStreamWriter { + repo: &'a Repository, + refs: Option>, + sha256: Option, + ) -> Self { // SAFETY: we surely can't get an error writing the header to a Vec let mut writer = Encoder::new(vec![], 0).unwrap(); match refs { Some(DigestMap { map }) => { writer.write_all(&(map.len() as u64).to_le_bytes()).unwrap(); - for ref entry in map { - writer.write_all(&entry.body).unwrap(); - writer.write_all(&entry.verity).unwrap(); - } + writer.write_all(map.as_bytes()).unwrap(); } None => { writer.write_all(&0u64.to_le_bytes()).unwrap(); } } - SplitStreamWriter { + Self { repo, inline_content: vec![], writer, @@ -114,7 +113,7 @@ impl SplitStreamWriter<'_> { /// flush any buffered inline data, taking new_value as the new value of the buffer fn flush_inline(&mut self, new_value: Vec) -> Result<()> { if !self.inline_content.is_empty() { - SplitStreamWriter::write_fragment( + Self::write_fragment( &mut self.writer, self.inline_content.len(), &self.inline_content, @@ -136,12 +135,12 @@ impl SplitStreamWriter<'_> { /// write a reference to external data to the stream. If the external data had padding in the /// stream which is not stored in the object then pass it here as well and it will be stored /// inline after the reference. - fn write_reference(&mut self, reference: Sha256HashValue, padding: Vec) -> Result<()> { + fn write_reference(&mut self, reference: &ObjectID, padding: Vec) -> Result<()> { // Flush the inline data before we store the external reference. Any padding from the // external data becomes the start of a new inline block. self.flush_inline(padding)?; - SplitStreamWriter::write_fragment(&mut self.writer, 0, &reference) + Self::write_fragment(&mut self.writer, 0, reference.as_bytes()) } pub fn write_external(&mut self, data: &[u8], padding: Vec) -> Result<()> { @@ -150,14 +149,14 @@ impl SplitStreamWriter<'_> { sha256.update(&padding); } let id = self.repo.ensure_object(data)?; - self.write_reference(id, padding) + self.write_reference(&id, padding) } - pub fn done(mut self) -> Result { + pub fn done(mut self) -> Result { self.flush_inline(vec![])?; if let Some((context, expected)) = self.sha256 { - if Into::::into(context.finalize()) != expected { + if Into::::into(context.finalize()) != expected { bail!("Content doesn't have expected SHA256 hash value!"); } } @@ -167,19 +166,19 @@ impl SplitStreamWriter<'_> { } #[derive(Debug)] -pub enum SplitStreamData { +pub enum SplitStreamData { Inline(Box<[u8]>), - External(Sha256HashValue), + External(ObjectID), } // utility class to help read splitstreams -pub struct SplitStreamReader { +pub struct SplitStreamReader { decoder: Decoder<'static, BufReader>, - pub refs: DigestMap, + pub refs: DigestMap, inline_bytes: usize, } -impl std::fmt::Debug for SplitStreamReader { +impl std::fmt::Debug for SplitStreamReader { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { // decoder doesn't impl Debug f.debug_struct("SplitStreamReader") @@ -208,14 +207,14 @@ fn read_into_vec(reader: &mut impl Read, vec: &mut Vec, size: usize) -> Resu Ok(()) } -enum ChunkType { +enum ChunkType { Eof, Inline, - External(Sha256HashValue), + External(ObjectID), } -impl SplitStreamReader { - pub fn new(reader: R) -> Result> { +impl SplitStreamReader { + pub fn new(reader: R) -> Result { let mut decoder = Decoder::new(reader)?; let n_map_entries = { @@ -224,19 +223,14 @@ impl SplitStreamReader { u64::from_le_bytes(buf) } as usize; - let mut refs = DigestMap { + let mut refs = DigestMap:: { map: Vec::with_capacity(n_map_entries), }; for _ in 0..n_map_entries { - let mut body = [0u8; 32]; - let mut verity = [0u8; 32]; - - decoder.read_exact(&mut body)?; - decoder.read_exact(&mut verity)?; - refs.map.push(DigestMapEntry { body, verity }); + refs.map.push(DigestMapEntry::read_from_io(&mut decoder)?); } - Ok(SplitStreamReader { + Ok(Self { decoder, refs, inline_bytes: 0, @@ -248,7 +242,7 @@ impl SplitStreamReader { eof_ok: bool, ext_ok: bool, expected_bytes: usize, - ) -> Result { + ) -> Result> { if self.inline_bytes == 0 { match read_u64_le(&mut self.decoder)? { None => { @@ -261,8 +255,7 @@ impl SplitStreamReader { if !ext_ok { bail!("Unexpected external reference when parsing splitstream"); } - let mut id = Sha256HashValue::EMPTY; - self.decoder.read_exact(&mut id)?; + let id = ObjectID::read_from_io(&mut self.decoder)?; return Ok(ChunkType::External(id)); } Some(size) => { @@ -303,7 +296,7 @@ impl SplitStreamReader { &mut self, actual_size: usize, stored_size: usize, - ) -> Result { + ) -> Result> { if let ChunkType::External(id) = self.ensure_chunk(false, true, stored_size)? { // ...and the padding if actual_size < stored_size { @@ -322,7 +315,7 @@ impl SplitStreamReader { pub fn cat( &mut self, output: &mut impl Write, - mut load_data: impl FnMut(&Sha256HashValue) -> Result>, + mut load_data: impl FnMut(&ObjectID) -> Result>, ) -> Result<()> { let mut buffer = vec![]; @@ -341,7 +334,7 @@ impl SplitStreamReader { } } - pub fn get_object_refs(&mut self, mut callback: impl FnMut(&Sha256HashValue)) -> Result<()> { + pub fn get_object_refs(&mut self, mut callback: impl FnMut(&ObjectID)) -> Result<()> { let mut buffer = vec![]; for entry in &self.refs.map { @@ -362,13 +355,13 @@ impl SplitStreamReader { } } - pub fn get_stream_refs(&mut self, mut callback: impl FnMut(&Sha256HashValue)) { + pub fn get_stream_refs(&mut self, mut callback: impl FnMut(&Sha256Digest)) { for entry in &self.refs.map { callback(&entry.body); } } - pub fn lookup(&self, body: &Sha256HashValue) -> Result<&Sha256HashValue> { + pub fn lookup(&self, body: &Sha256Digest) -> Result<&ObjectID> { match self.refs.lookup(body) { Some(id) => Ok(id), None => bail!("Reference is not found in splitstream"), @@ -376,7 +369,7 @@ impl SplitStreamReader { } } -impl Read for SplitStreamReader { +impl Read for SplitStreamReader { fn read(&mut self, data: &mut [u8]) -> std::io::Result { match self.ensure_chunk(true, false, 1) { Ok(ChunkType::Eof) => Ok(0), diff --git a/src/util.rs b/src/util.rs index bafa58e9..1e3006eb 100644 --- a/src/util.rs +++ b/src/util.rs @@ -5,8 +5,6 @@ use std::{ use tokio::io::{AsyncRead, AsyncReadExt}; -use crate::fsverity::{FsVerityHashValue, Sha256HashValue}; - /// Formats a string like "/proc/self/fd/3" for the given fd. This can be used to work with kernel /// APIs that don't directly accept file descriptors. /// @@ -83,14 +81,17 @@ pub(crate) async fn read_exactish_async( Ok(true) } -/// Parse a string containing a SHA256 digest in hexidecimal form into a Sha256HashValue. +/// A utility type representing a SHA-256 digest in binary. +pub type Sha256Digest = [u8; 32]; + +/// Parse a string containing a SHA256 digest in hexidecimal form into a Sha256Digest. /// /// The string must contain exactly 64 characters and consist entirely of [0-9a-f], case /// insensitive. /// /// In case of a failure to parse the string, this function returns ErrorKind::InvalidInput. -pub fn parse_sha256(string: impl AsRef) -> Result { - let mut value = Sha256HashValue::EMPTY; +pub fn parse_sha256(string: impl AsRef) -> Result { + let mut value = [0u8; 32]; hex::decode_to_slice(string.as_ref(), &mut value) .map_err(|source| Error::new(ErrorKind::InvalidInput, source))?; Ok(value) diff --git a/tests/mkfs.rs b/tests/mkfs.rs index 61992e3a..69b8c541 100644 --- a/tests/mkfs.rs +++ b/tests/mkfs.rs @@ -13,10 +13,11 @@ use tempfile::NamedTempFile; use composefs::{ dumpfile::write_dumpfile, erofs::{debug::debug_img, writer::mkfs_erofs}, + fsverity::{FsVerityHashValue, Sha256HashValue}, image::{Directory, FileSystem, Inode, Leaf, LeafContent, RegularFile, Stat}, }; -fn debug_fs(mut fs: FileSystem) -> String { +fn debug_fs(mut fs: FileSystem) -> String { fs.done(); let image = mkfs_erofs(&fs); let mut output = vec![]; @@ -24,16 +25,20 @@ fn debug_fs(mut fs: FileSystem) -> String { String::from_utf8(output).unwrap() } -fn empty(_fs: &mut FileSystem) {} +fn empty(_fs: &mut FileSystem) {} #[test] fn test_empty() { - let mut fs = FileSystem::new(); + let mut fs = FileSystem::::new(); empty(&mut fs); insta::assert_snapshot!(debug_fs(fs)); } -fn add_leaf(dir: &mut Directory, name: impl AsRef, content: LeafContent) { +fn add_leaf( + dir: &mut Directory, + name: impl AsRef, + content: LeafContent, +) { dir.insert( name.as_ref(), Inode::Leaf(Rc::new(Leaf { @@ -49,7 +54,11 @@ fn add_leaf(dir: &mut Directory, name: impl AsRef, content: LeafContent) ); } -fn simple(fs: &mut FileSystem) { +fn simple(fs: &mut FileSystem) { + let ext_id = Sha256HashValue::from_hex( + "5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a", + ) + .unwrap(); add_leaf(&mut fs.root, "fifo", LeafContent::Fifo); add_leaf( &mut fs.root, @@ -59,7 +68,7 @@ fn simple(fs: &mut FileSystem) { add_leaf( &mut fs.root, "regular-external", - LeafContent::Regular(RegularFile::External([0x5a; 32], 1234)), + LeafContent::Regular(RegularFile::External(ext_id, 1234)), ); add_leaf(&mut fs.root, "chrdev", LeafContent::CharacterDevice(123)); add_leaf(&mut fs.root, "blkdev", LeafContent::BlockDevice(123)); @@ -73,12 +82,12 @@ fn simple(fs: &mut FileSystem) { #[test] fn test_simple() { - let mut fs = FileSystem::new(); + let mut fs = FileSystem::::new(); simple(&mut fs); insta::assert_snapshot!(debug_fs(fs)); } -fn foreach_case(f: fn(&FileSystem)) { +fn foreach_case(f: fn(&FileSystem)) { for case in [empty, simple] { let mut fs = FileSystem::new(); case(&mut fs);