diff --git a/lib/vfs/src/filesystem_ext.rs b/lib/vfs/src/filesystem_ext.rs index 0aa28dd3093..e416630a1ba 100644 --- a/lib/vfs/src/filesystem_ext.rs +++ b/lib/vfs/src/filesystem_ext.rs @@ -49,7 +49,7 @@ pub trait FileSystemExt { } #[async_trait::async_trait] -impl FileSystemExt for F { +impl FileSystemExt for F { fn exists(&self, path: impl AsRef) -> bool { self.metadata(path.as_ref()).is_ok() } @@ -112,7 +112,7 @@ impl FileSystemExt for F { let _ = self .new_open_options() .create(true) - .append(true) + .write(true) .open(path)?; Ok(()) @@ -140,7 +140,10 @@ impl FileSystemExt for F { } } -fn create_dir_all(fs: &impl FileSystem, path: &Path) -> Result<(), FsError> { +fn create_dir_all(fs: &F, path: &Path) -> Result<(), FsError> +where + F: FileSystem + ?Sized, +{ if let Some(parent) = path.parent() { create_dir_all(fs, parent)?; } diff --git a/lib/vfs/src/filesystems.rs b/lib/vfs/src/filesystems.rs index ee60c82a0e0..d6e37ed2233 100644 --- a/lib/vfs/src/filesystems.rs +++ b/lib/vfs/src/filesystems.rs @@ -1,90 +1,104 @@ use crate::FileSystem; +use std::ops::ControlFlow; /// A chain of one or more [`FileSystem`]s. -// FIXME(Michael-F-Bryan): We could remove this trait's HRTBs and lifetimes if -// we had access to GATs, but our MSRV is currently 1.64 and GATs require 1.65. -pub trait FileSystems<'a>: 'a { - type Iter: IntoIterator + 'a; - - fn iter_filesystems(&'a self) -> Self::Iter; +pub trait FileSystems { + // FIXME(Michael-F-Bryan): Rewrite this to use GATs and an external iterator + // when we bump the MSRV to 1.65 or higher. + fn for_each_filesystems(&self, func: F) -> Option + where + F: FnMut(&dyn FileSystem) -> ControlFlow; } -impl<'a, S> FileSystems<'a> for &'a S +impl<'b, S> FileSystems for &'b S where - S: FileSystems<'a> + 'a, + S: FileSystems + 'b, { - type Iter = >::Iter; - - fn iter_filesystems(&'a self) -> Self::Iter { - (**self).iter_filesystems() + fn for_each_filesystems(&self, func: F) -> Option + where + F: FnMut(&dyn FileSystem) -> ControlFlow, + { + (**self).for_each_filesystems(func) } } -impl<'a, F> FileSystems<'a> for Vec +impl FileSystems for Vec where - F: FileSystem + 'a, + T: FileSystem, { - type Iter = std::iter::Map, fn(&F) -> &dyn FileSystem>; - - fn iter_filesystems(&'a self) -> Self::Iter { - fn downcast(value: &T) -> &dyn FileSystem { - value - } - self.iter().map(downcast) + fn for_each_filesystems(&self, func: F) -> Option + where + F: FnMut(&dyn FileSystem) -> ControlFlow, + { + self[..].for_each_filesystems(func) } } -impl<'a, F, const N: usize> FileSystems<'a> for [F; N] +impl FileSystems for [T; N] where - F: FileSystem + 'a, + T: FileSystem, { - type Iter = [&'a dyn FileSystem; N]; - - fn iter_filesystems(&'a self) -> Self::Iter { - // a poor man's version of the unstable array::each_ref() - let mut i = 0; - [(); N].map(|()| { - let f = &self[i] as &dyn FileSystem; - i += 1; - f - }) + fn for_each_filesystems(&self, func: F) -> Option + where + F: FnMut(&dyn FileSystem) -> ControlFlow, + { + self[..].for_each_filesystems(func) } } -impl<'a> FileSystems<'a> for () { - type Iter = std::iter::Empty<&'a dyn FileSystem>; +impl FileSystems for [T] +where + T: FileSystem, +{ + fn for_each_filesystems(&self, mut func: F) -> Option + where + F: FnMut(&dyn FileSystem) -> ControlFlow, + { + for fs in self.iter() { + match func(fs) { + ControlFlow::Continue(_) => continue, + ControlFlow::Break(result) => return Some(result), + } + } - fn iter_filesystems(&'a self) -> Self::Iter { - std::iter::empty() + None } } -macro_rules! count { - ($($t:ident),* $(,)?) => { - 0 $( + count!(@$t) )* - }; - (@$t:ident) => { 1 }; +impl FileSystems for () { + fn for_each_filesystems(&self, _func: F) -> Option + where + F: FnMut(&dyn FileSystem) -> ControlFlow, + { + None + } } macro_rules! tuple_filesystems { ($first:ident $(, $rest:ident)* $(,)?) => { - impl<'a, $first, $( $rest ),*> FileSystems<'a> for ($first, $($rest),*) + impl<$first, $( $rest ),*> FileSystems for ($first, $($rest),*) where - $first: FileSystem + 'a, - $($rest: FileSystem + 'a),* + $first: FileSystem, + $($rest: FileSystem),* { - type Iter = [ &'a dyn FileSystem; { count!($first, $($rest),*) }]; - - fn iter_filesystems(&'a self) -> Self::Iter { + fn for_each_filesystems(&self, mut func: F) -> Option + where + F: FnMut(&dyn FileSystem) -> ControlFlow, + { #[allow(non_snake_case)] let ($first, $($rest),*) = &self; - [ - $first as &dyn FileSystem, - $( - $rest as &dyn FileSystem, - )* - ] + if let ControlFlow::Break(result) = func($first) { + return Some(result); + } + + $( + if let ControlFlow::Break(result) = func($rest) { + return Some(result); + } + )* + + None } } @@ -93,4 +107,4 @@ macro_rules! tuple_filesystems { () => {}; } -tuple_filesystems!(A, B, C, D, E, F, G, H, I, J, K); +tuple_filesystems!(F1, F2, F3, F4, F5, F6, F7, F8, F9, F10, F11, F12, F13, F14, F15, F16,); diff --git a/lib/vfs/src/lib.rs b/lib/vfs/src/lib.rs index 7a72402fbb1..fa6cbfb6bf0 100644 --- a/lib/vfs/src/lib.rs +++ b/lib/vfs/src/lib.rs @@ -6,7 +6,6 @@ use std::any::Any; use std::ffi::OsString; use std::fmt; use std::io; -use std::ops::Deref; use std::path::{Path, PathBuf}; use std::pin::Pin; use std::task::Context; @@ -94,40 +93,6 @@ impl dyn FileSystem + 'static { } } -impl FileSystem for P -where - P: Deref + std::fmt::Debug + Send + Sync + 'static, - F: FileSystem + ?Sized, -{ - fn read_dir(&self, path: &Path) -> Result { - (**self).read_dir(path) - } - - fn create_dir(&self, path: &Path) -> Result<()> { - (**self).create_dir(path) - } - - fn remove_dir(&self, path: &Path) -> Result<()> { - (**self).remove_dir(path) - } - - fn rename(&self, from: &Path, to: &Path) -> Result<()> { - (**self).rename(from, to) - } - - fn metadata(&self, path: &Path) -> Result { - (**self).metadata(path) - } - - fn remove_file(&self, path: &Path) -> Result<()> { - (**self).remove_file(path) - } - - fn new_open_options(&self) -> OpenOptions { - (**self).new_open_options() - } -} - pub trait FileOpener { fn open( &self, diff --git a/lib/vfs/src/overlay_fs.rs b/lib/vfs/src/overlay_fs.rs index d27a99b4f95..c3d8a7c8203 100644 --- a/lib/vfs/src/overlay_fs.rs +++ b/lib/vfs/src/overlay_fs.rs @@ -1,4 +1,4 @@ -use std::{fmt::Debug, path::Path}; +use std::{fmt::Debug, ops::ControlFlow, path::Path}; use crate::{ FileOpener, FileSystem, FileSystemExt, FileSystems, FsError, Metadata, OpenOptions, @@ -51,7 +51,7 @@ pub struct OverlayFileSystem { impl OverlayFileSystem where P: FileSystem, - S: for<'a> FileSystems<'a>, + S: FileSystems, { /// Create a new [`FileSystem`] using a primary [`crate::FileSystem`] and a /// chain of secondary [`FileSystems`]. @@ -82,53 +82,58 @@ where (self.primary, self.secondaries) } - /// Iterate over all filesystems in order of precedence. - pub fn iter(&self) -> impl Iterator + '_ { - std::iter::once(self.primary() as &dyn FileSystem) - .chain(self.secondaries().iter_filesystems()) - } - - /// Try to apply an operation to each [`FileSystem`] in order of precedence. - /// - /// This uses [`should_continue()`] to determine whether an error is fatal - /// and needs to be returned to the caller, or whether we should try the - /// next [`FileSystem`] in the chain. - fn for_each(&self, mut func: F) -> Result - where - F: FnMut(&dyn FileSystem) -> Result, - { - for fs in self.iter() { - match func(fs) { - Ok(result) => return Ok(result), - Err(e) if should_continue(e) => continue, - Err(other) => return Err(other), + fn permission_error_or_not_found(&self, path: &Path) -> Result<(), FsError> { + let result = self.secondaries.for_each_filesystems(|fs| { + if fs.exists(path) { + ControlFlow::Break(FsError::PermissionDenied) + } else { + ControlFlow::Continue(()) } - } + }); - Err(FsError::EntryNotFound) + Err(result.unwrap_or(FsError::EntryNotFound)) } } impl FileSystem for OverlayFileSystem where - P: FileSystem, - S: for<'a> crate::FileSystems<'a> + Send + Sync, + P: FileSystem + 'static, + S: crate::FileSystems + Send + Sync + 'static, { fn read_dir(&self, path: &Path) -> Result { let mut entries = Vec::new(); let mut had_at_least_one_success = false; - for fs in self.iter() { - match fs.read_dir(path) { + match self.primary.read_dir(path) { + Ok(r) => { + for entry in r { + entries.push(entry?); + } + had_at_least_one_success = true; + } + Err(e) if should_continue(e) => {} + Err(e) => return Err(e), + } + + let result = self + .secondaries + .for_each_filesystems(|fs| match fs.read_dir(path) { Ok(r) => { for entry in r { - entries.push(entry?); + match entry { + Ok(entry) => entries.push(entry), + Err(e) => return ControlFlow::Break(e), + } } had_at_least_one_success = true; + ControlFlow::Continue(()) } - Err(e) if should_continue(e) => continue, - Err(e) => return Err(e), - } + Err(e) if should_continue(e) => ControlFlow::Continue(()), + Err(e) => ControlFlow::Break(e), + }); + + if let Some(error) = result { + return Err(error); } if had_at_least_one_success { @@ -146,70 +151,55 @@ where fn create_dir(&self, path: &Path) -> Result<(), FsError> { match self.primary.create_dir(path) { - Ok(()) => return Ok(()), Err(e) if should_continue(e) => {} - Err(e) => return Err(e), + other => return other, } - for fs in self.secondaries.iter_filesystems() { - if fs.is_dir(path) { - return Err(FsError::PermissionDenied); - } - } - - Err(FsError::EntryNotFound) + self.permission_error_or_not_found(path) } fn remove_dir(&self, path: &Path) -> Result<(), FsError> { match self.primary.remove_dir(path) { - Ok(()) => return Ok(()), Err(e) if should_continue(e) => {} - Err(e) => return Err(e), + other => return other, } - for fs in self.secondaries.iter_filesystems() { - if fs.is_dir(path) { - return Err(FsError::PermissionDenied); - } - } - - Err(FsError::EntryNotFound) + self.permission_error_or_not_found(path) } fn rename(&self, from: &Path, to: &Path) -> Result<(), FsError> { match self.primary.rename(from, to) { - Ok(()) => return Ok(()), Err(e) if should_continue(e) => {} - Err(e) => return Err(e), + other => return other, } - for fs in self.secondaries.iter_filesystems() { - if fs.exists(from) { - return Err(FsError::PermissionDenied); - } - } - - Err(FsError::EntryNotFound) + self.permission_error_or_not_found(from) } fn metadata(&self, path: &Path) -> Result { - self.for_each(|fs| fs.metadata(path)) + match self.primary.metadata(path) { + Ok(meta) => return Ok(meta), + Err(e) if should_continue(e) => {} + Err(e) => return Err(e), + } + + let result = self + .secondaries + .for_each_filesystems(|fs| match fs.metadata(path) { + Err(e) if should_continue(e) => ControlFlow::Continue(()), + other => ControlFlow::Break(other), + }); + + result.unwrap_or(Err(FsError::EntryNotFound)) } fn remove_file(&self, path: &Path) -> Result<(), FsError> { match self.primary.remove_file(path) { - Ok(()) => return Ok(()), Err(e) if should_continue(e) => {} - Err(e) => return Err(e), - } - - for fs in self.secondaries.iter_filesystems() { - if fs.exists(path) { - return Err(FsError::PermissionDenied); - } + other => return other, } - Err(FsError::EntryNotFound) + self.permission_error_or_not_found(path) } fn new_open_options(&self) -> OpenOptions<'_> { @@ -220,37 +210,108 @@ where impl FileOpener for OverlayFileSystem where P: FileSystem, - S: for<'a> FileSystems<'a> + Send + Sync, + S: FileSystems + Send + Sync + 'static, { fn open( &self, path: &Path, conf: &OpenOptionsConfig, ) -> Result, FsError> { - // TODO: Re-work this method so that trying to create a file inside a - // secondary filesystem will actually create the file on the primary - // filesystem, running create_dir_all() if necessary. - self.for_each(|fs| fs.new_open_options().options(conf.clone()).open(path)) + match self + .primary + .new_open_options() + .options(conf.clone()) + .open(path) + { + Err(e) if should_continue(e) => {} + other => return other, + } + + if conf.create || conf.create_new && !self.exists(path) { + if let Some(parent) = path.parent() { + let is_secondary_dir = self + .secondaries + .for_each_filesystems(|fs| match fs.is_dir(parent) { + true => ControlFlow::Break(true), + false => ControlFlow::Continue(()), + }) + .unwrap_or(false); + if is_secondary_dir { + // We fall into the special case where you can create a file + // that looks like it is inside a secondary filesystem folder, + // but actually it gets created on the host + self.primary.create_dir_all(parent)?; + return self + .primary + .new_open_options() + .options(conf.clone()) + .open(path); + } + } + } + + if opening_would_require_mutations(&self.secondaries, path, conf) { + return Err(FsError::PermissionDenied); + } + + self.secondaries + .for_each_filesystems(|fs| { + match fs.new_open_options().options(conf.clone()).open(path) { + Err(e) if should_continue(e) => ControlFlow::Continue(()), + other => ControlFlow::Break(other), + } + }) + .unwrap_or(Err(FsError::EntryNotFound)) } } +fn opening_would_require_mutations( + secondaries: &S, + path: &Path, + conf: &OpenOptionsConfig, +) -> bool +where + S: FileSystems + Send + Sync, +{ + if conf.append || conf.write || conf.create_new { + return true; + } + + if conf.create { + // Would we create the file if it doesn't exist yet. + let already_exists = secondaries + .for_each_filesystems(|fs| match fs.is_file(path) { + true => ControlFlow::Break(()), + false => ControlFlow::Continue(()), + }) + .is_some(); + + if !already_exists { + return true; + } + } + + false +} + impl Debug for OverlayFileSystem where P: FileSystem, - S: for<'a> FileSystems<'a>, + S: FileSystems, { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { struct IterFilesystems<'a, S>(&'a S); impl<'a, S> Debug for IterFilesystems<'a, S> where - S: for<'b> FileSystems<'b>, + S: FileSystems, { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let mut f = f.debug_list(); - for fs in self.0.iter_filesystems() { + let _: Option<()> = self.0.for_each_filesystems(|fs| { f.entry(&fs); - } + ControlFlow::Continue(()) + }); f.finish() } @@ -272,22 +333,28 @@ fn should_continue(e: FsError) -> bool { mod tests { use std::{path::PathBuf, sync::Arc}; + use tempfile::TempDir; use tokio::io::AsyncWriteExt; use super::*; - use crate::{mem_fs::FileSystem as MemFS, FileSystem as _, FileSystemExt}; + use crate::{ + mem_fs::FileSystem as MemFS, webc_fs::WebcFileSystem, FileSystemExt, RootFileSystemBuilder, + UnionFileSystem, + }; #[test] - fn can_be_used_as_an_object() { + fn object_safe() { fn _box_with_memfs( fs: OverlayFileSystem>, ) -> Box { Box::new(fs) } - fn _arc( - fs: OverlayFileSystem, Vec>>, - ) -> Arc { + fn _arc(fs: OverlayFileSystem) -> Arc + where + A: FileSystem + 'static, + S: FileSystems + Send + Sync + Debug + 'static, + { Arc::new(fs) } } @@ -429,4 +496,80 @@ mod tests { ] ); } + + #[tokio::test] + async fn wasi_runner_use_case() { + // Set up some dummy files on the host + let temp = TempDir::new().unwrap(); + let first = temp.path().join("first"); + let file_txt = first.join("file.txt"); + let second = temp.path().join("second"); + std::fs::create_dir_all(&first).unwrap(); + std::fs::write(&file_txt, b"First!").unwrap(); + std::fs::create_dir_all(&second).unwrap(); + // configure the union FS so things are saved in memory by default, but + // certain folders are mapped to the host. + let mut primary = UnionFileSystem::default(); + let root = RootFileSystemBuilder::new().build(); + primary.mount("in-memory", "/", false, Box::new(root), None); + let mapped_dirs = [&first, &second]; + for dir in mapped_dirs { + let dir = dir.display().to_string(); + let fs = Box::new(crate::host_fs::FileSystem); + primary.mount("in-memory", &dir, false, fs, Some(&dir)); + } + // Set up the secondary file systems + let webc = webc::v1::WebCOwned::parse( + include_bytes!("../../c-api/examples/assets/python-0.1.0.wasmer").to_vec(), + &webc::v1::ParseOptions::default(), + ) + .unwrap(); + let webc = WebcFileSystem::init_all(Arc::new(webc)); + + let fs = OverlayFileSystem::new(primary, [webc]); + + let paths: Vec<_> = fs.walk("/").map(|entry| entry.path()).collect(); + println!("{paths:#?}"); + + // We should get all the normal directories from rootfs (primary) + assert!(fs.is_dir("/lib")); + assert!(fs.is_dir("/bin")); + assert!(fs.is_file("/dev/stdin")); + assert!(fs.is_file("/dev/stdout")); + // We also want to see files from the WEBC volumes (secondary) + assert!(fs.is_dir("/lib/python3.6")); + assert!(fs.is_file("/lib/python3.6/collections/__init__.py")); + // files on a secondary fs aren't writable + assert_eq!( + fs.new_open_options() + .append(true) + .open("/lib/python3.6/collections/__init__.py") + .unwrap_err(), + FsError::PermissionDenied + ); + // you are allowed to create files that look like they are in a secondary + // folder, though + fs.touch("/lib/python3.6/collections/something-else.py") + .unwrap(); + // But it'll be on the primary filesystem, not the secondary one + assert!(fs + .primary + .is_file("/lib/python3.6/collections/something-else.py")); + assert!(!fs.secondaries[0].is_file("/lib/python3.6/collections/something-else.py")); + // You can do the same thing with folders + fs.create_dir("/lib/python3.6/something-else".as_ref()) + .unwrap(); + assert!(fs.primary.is_dir("/lib/python3.6/something-else")); + assert!(!fs.secondaries[0].is_dir("/lib/python3.6/something-else")); + // you should also be able to read files mounted from the host + assert_eq!(fs.read_to_string(&file_txt).await.unwrap(), "First!"); + // Overwriting them is fine and we'll see the changes on the host + fs.write(&file_txt, "Updated").await.unwrap(); + assert_eq!(std::fs::read_to_string(&file_txt).unwrap(), "Updated"); + // The filesystem will see changes on the host that happened after it was + // set up + let another = second.join("another.txt"); + std::fs::write(&another, "asdf").unwrap(); + assert_eq!(fs.read_to_string(&another).await.unwrap(), "asdf"); + } }