diff --git a/Cargo.lock b/Cargo.lock index 66d928d5164..4af84bf85fd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5216,6 +5216,7 @@ dependencies = [ "lazy_static", "libc", "pin-project-lite", + "pretty_assertions", "slab", "tempfile", "thiserror", diff --git a/lib/vfs/Cargo.toml b/lib/vfs/Cargo.toml index 03c70a78ead..5b11cabb8d6 100644 --- a/lib/vfs/Cargo.toml +++ b/lib/vfs/Cargo.toml @@ -25,6 +25,7 @@ pin-project-lite = "0.2.9" indexmap = "1.9.2" [dev-dependencies] +pretty_assertions = "1.3.0" tempfile = "3.4.0" tokio = { version = "1", features = [ "io-util", "rt" ], default_features = false } diff --git a/lib/vfs/src/builder.rs b/lib/vfs/src/builder.rs index a6aa9143d87..3c1351782f5 100644 --- a/lib/vfs/src/builder.rs +++ b/lib/vfs/src/builder.rs @@ -133,7 +133,7 @@ mod test_builder { .unwrap(); assert_eq!(dev_zero.write(b"hello").await.unwrap(), 5); let mut buf = vec![1; 10]; - dev_zero.read(&mut buf[..]).await.unwrap(); + dev_zero.read_exact(&mut buf[..]).await.unwrap(); assert_eq!(buf, vec![0; 10]); assert!(dev_zero.get_special_fd().is_none()); diff --git a/lib/vfs/src/filesystems.rs b/lib/vfs/src/filesystems.rs new file mode 100644 index 00000000000..bf47eb2df8e --- /dev/null +++ b/lib/vfs/src/filesystems.rs @@ -0,0 +1,106 @@ +use crate::FileSystem; + +/// A chain of one or more [`FileSystem`]s. +pub trait FileSystems<'a>: 'a { + // FIXME(Michael-F-Bryan): Rewrite this to use GATs when we bump the MSRV to + // 1.65 or higher. That'll get rid of all the lifetimes and HRTBs. + type Iter: IntoIterator + 'a; + + /// Get something that can be used to iterate over the underlying + /// filesystems. + fn filesystems(&'a self) -> Self::Iter; +} + +impl<'a, 'b, S> FileSystems<'a> for &'b S +where + S: FileSystems<'a> + 'b, + 'b: 'a, +{ + type Iter = S::Iter; + + fn filesystems(&'a self) -> Self::Iter { + (**self).filesystems() + } +} + +impl<'a, T> FileSystems<'a> for Vec +where + T: FileSystem, +{ + type Iter = <[T] as FileSystems<'a>>::Iter; + + fn filesystems(&'a self) -> Self::Iter { + self[..].filesystems() + } +} + +impl<'a, T, const N: usize> FileSystems<'a> for [T; N] +where + T: FileSystem, +{ + type Iter = [&'a dyn FileSystem; N]; + + fn filesystems(&'a self) -> Self::Iter { + // TODO: rewrite this when array::each_ref() is stable + let mut i = 0; + [(); N].map(|_| { + let f = &self[i] as &dyn FileSystem; + i += 1; + f + }) + } +} + +impl<'a, T> FileSystems<'a> for [T] +where + T: FileSystem, +{ + type Iter = std::iter::Map, fn(&T) -> &dyn FileSystem>; + + fn filesystems(&'a self) -> Self::Iter { + self.iter().map(|fs| fs as &dyn FileSystem) + } +} + +impl<'a> FileSystems<'a> for () { + type Iter = std::iter::Empty<&'a dyn FileSystem>; + + fn filesystems(&'a self) -> Self::Iter { + std::iter::empty() + } +} + +macro_rules! count { + ($first:tt $($rest:tt)*) => { + 1 + count!($($rest)*) + }; + () => { 0 }; +} + +macro_rules! tuple_filesystems { + ($first:ident $(, $rest:ident)* $(,)?) => { + impl<'a, $first, $( $rest ),*> FileSystems<'a> for ($first, $($rest),*) + where + $first: FileSystem, + $($rest: FileSystem),* + { + type Iter = [&'a dyn FileSystem; count!($first $($rest)*)]; + + fn filesystems(&'a self) -> Self::Iter { + #[allow(non_snake_case)] + let ($first, $($rest),*) = self; + + [ + $first as &dyn FileSystem, + $($rest),* + ] + } + + } + + tuple_filesystems!($($rest),*); + }; + () => {}; +} + +tuple_filesystems!(F1, F2, F3, F4, F5, F6, F7, F8, F9, F10, F11, F12, F13, F14, F15, F16,); diff --git a/lib/vfs/src/host_fs.rs b/lib/vfs/src/host_fs.rs index 253f191b9e3..ef4f3fc4633 100644 --- a/lib/vfs/src/host_fs.rs +++ b/lib/vfs/src/host_fs.rs @@ -1365,7 +1365,5 @@ mod tests { .join("hello.txt")), "canonicalizing a crazily stupid path name", ); - - let _ = fs_extra::remove_items(&["./test_canonicalize"]); } } diff --git a/lib/vfs/src/lib.rs b/lib/vfs/src/lib.rs index 580905e5983..1f0fd36f5d9 100644 --- a/lib/vfs/src/lib.rs +++ b/lib/vfs/src/lib.rs @@ -1,3 +1,7 @@ +#[cfg(test)] +#[macro_use] +extern crate pretty_assertions; + use std::any::Any; use std::ffi::OsString; use std::fmt; @@ -25,6 +29,9 @@ pub mod tmp_fs; pub mod union_fs; pub mod zero_file; // tty_file -> see wasmer_wasi::tty_file +mod filesystems; +pub(crate) mod ops; +mod overlay_fs; pub mod pipe; #[cfg(feature = "static-fs")] pub mod static_fs; @@ -38,7 +45,9 @@ pub use builder::*; pub use combine_file::*; pub use dual_write_file::*; pub use empty_fs::*; +pub use filesystems::FileSystems; pub use null_file::*; +pub use overlay_fs::OverlayFileSystem; pub use passthru_fs::*; pub use pipe::*; pub use special_file::*; @@ -138,6 +147,20 @@ impl OpenOptionsConfig { pub const fn truncate(&self) -> bool { self.truncate } + + /// Would a file opened with this [`OpenOptionsConfig`] change files on the + /// filesystem. + pub const fn would_mutate(&self) -> bool { + let OpenOptionsConfig { + read: _, + write, + create_new, + create, + append, + truncate, + } = *self; + append || write || create || create_new || truncate + } } impl<'a> fmt::Debug for OpenOptions<'a> { @@ -170,36 +193,62 @@ impl<'a> OpenOptions<'a> { self.conf.clone() } + /// Use an existing [`OpenOptionsConfig`] to configure this [`OpenOptions`]. pub fn options(&mut self, options: OpenOptionsConfig) -> &mut Self { self.conf = options; self } + /// Sets the option for read access. + /// + /// This option, when true, will indicate that the file should be + /// `read`-able if opened. pub fn read(&mut self, read: bool) -> &mut Self { self.conf.read = read; self } + /// Sets the option for write access. + /// + /// This option, when true, will indicate that the file should be + /// `write`-able if opened. + /// + /// If the file already exists, any write calls on it will overwrite its + /// contents, without truncating it. pub fn write(&mut self, write: bool) -> &mut Self { self.conf.write = write; self } + /// Sets the option for the append mode. + /// + /// This option, when true, means that writes will append to a file instead + /// of overwriting previous contents. + /// Note that setting `.write(true).append(true)` has the same effect as + /// setting only `.append(true)`. pub fn append(&mut self, append: bool) -> &mut Self { self.conf.append = append; self } + /// Sets the option for truncating a previous file. + /// + /// If a file is successfully opened with this option set it will truncate + /// the file to 0 length if it already exists. + /// + /// The file must be opened with write access for truncate to work. pub fn truncate(&mut self, truncate: bool) -> &mut Self { self.conf.truncate = truncate; self } + /// Sets the option to create a new file, or open it if it already exists. pub fn create(&mut self, create: bool) -> &mut Self { self.conf.create = create; self } + /// Sets the option to create a new file, failing if it already exists. pub fn create_new(&mut self, create_new: bool) -> &mut Self { self.conf.create_new = create_new; self diff --git a/lib/vfs/src/mem_fs/filesystem.rs b/lib/vfs/src/mem_fs/filesystem.rs index e8a9dc178d9..92a505ad447 100644 --- a/lib/vfs/src/mem_fs/filesystem.rs +++ b/lib/vfs/src/mem_fs/filesystem.rs @@ -12,9 +12,8 @@ use std::sync::{Arc, RwLock}; /// The in-memory file system! /// -/// It's a thin wrapper around [`FileSystemInner`]. This `FileSystem` -/// type can be cloned, it's a light copy of the `FileSystemInner` -/// (which is behind a `Arc` + `RwLock`. +/// This `FileSystem` type can be cloned, it's a light copy of the +/// `FileSystemInner` (which is behind a `Arc` + `RwLock`). #[derive(Clone, Default)] pub struct FileSystem { pub(super) inner: Arc>, @@ -937,9 +936,25 @@ impl Default for FileSystemInner { } } +#[allow(dead_code)] // The `No` variant. +pub(super) enum DirectoryMustBeEmpty { + Yes, + No, +} + +impl DirectoryMustBeEmpty { + pub(super) fn yes(&self) -> bool { + matches!(self, Self::Yes) + } + + pub(super) fn no(&self) -> bool { + !self.yes() + } +} + #[cfg(test)] mod test_filesystem { - use crate::{mem_fs::*, DirEntry, FileSystem as FS, FileType, FsError}; + use crate::{mem_fs::*, ops, DirEntry, FileSystem as FS, FileType, FsError}; macro_rules! path { ($path:expr) => { @@ -1686,20 +1701,26 @@ mod test_filesystem { "canonicalizing a crazily stupid path name", ); } -} - -#[allow(dead_code)] // The `No` variant. -pub(super) enum DirectoryMustBeEmpty { - Yes, - No, -} -impl DirectoryMustBeEmpty { - pub(super) fn yes(&self) -> bool { - matches!(self, Self::Yes) - } + #[test] + #[ignore = "Not yet supported. See https://github.com/wasmerio/wasmer/issues/3678"] + fn mount_to_overlapping_directories() { + let top_level = FileSystem::default(); + ops::touch(&top_level, "/file.txt").unwrap(); + let nested = FileSystem::default(); + ops::touch(&nested, "/another-file.txt").unwrap(); + let top_level: Arc = Arc::new(top_level); + let nested: Arc = Arc::new(nested); - pub(super) fn no(&self) -> bool { - !self.yes() + let fs = FileSystem::default(); + fs.mount("/top-level".into(), &top_level, "/".into()) + .unwrap(); + fs.mount("/top-level/nested".into(), &nested, "/".into()) + .unwrap(); + + assert!(ops::is_dir(&fs, "/top-level")); + assert!(ops::is_file(&fs, "/top-level/file.txt")); + assert!(ops::is_dir(&fs, "/top-level/nested")); + assert!(ops::is_file(&fs, "/top-level/nested/another-file.txt")); } } diff --git a/lib/vfs/src/ops.rs b/lib/vfs/src/ops.rs new file mode 100644 index 00000000000..2c124d9b8e7 --- /dev/null +++ b/lib/vfs/src/ops.rs @@ -0,0 +1,231 @@ +//! Common [`FileSystem`] operations. +#![allow(dead_code)] // Most of these helpers are used during testing + +use std::{collections::VecDeque, path::Path}; + +use tokio::io::{AsyncReadExt, AsyncWriteExt}; + +use crate::{DirEntry, FileSystem, FsError}; + +/// Does this item exists? +pub fn exists(fs: &F, path: impl AsRef) -> bool +where + F: FileSystem + ?Sized, +{ + fs.metadata(path.as_ref()).is_ok() +} + +/// Does this path refer to a directory? +pub fn is_dir(fs: &F, path: impl AsRef) -> bool +where + F: FileSystem + ?Sized, +{ + match fs.metadata(path.as_ref()) { + Ok(meta) => meta.is_dir(), + Err(_) => false, + } +} + +/// Does this path refer to a file? +pub fn is_file(fs: &F, path: impl AsRef) -> bool +where + F: FileSystem + ?Sized, +{ + match fs.metadata(path.as_ref()) { + Ok(meta) => meta.is_file(), + Err(_) => false, + } +} + +/// Make sure a directory (and all its parents) exist. +/// +/// This is analogous to [`std::fs::create_dir_all()`]. +pub fn create_dir_all(fs: &F, path: impl AsRef) -> Result<(), FsError> +where + F: FileSystem + ?Sized, +{ + let path = path.as_ref(); + if let Some(parent) = path.parent() { + create_dir_all(fs, parent)?; + } + + if let Ok(metadata) = fs.metadata(path) { + if metadata.is_dir() { + return Ok(()); + } + if metadata.is_file() { + return Err(FsError::BaseNotDirectory); + } + } + + fs.create_dir(path) +} + +/// Asynchronously write some bytes to a file. +/// +/// This is analogous to [`std::fs::write()`]. +pub async fn write( + fs: &F, + path: impl AsRef + Send, + data: impl AsRef<[u8]> + Send, +) -> Result<(), FsError> +where + F: FileSystem + ?Sized, +{ + let path = path.as_ref(); + let data = data.as_ref(); + + let mut f = fs + .new_open_options() + .create(true) + .truncate(true) + .write(true) + .open(path)?; + + f.write_all(data).await?; + f.flush().await?; + + Ok(()) +} + +/// Asynchronously read a file's contents into memory. +/// +/// This is analogous to [`std::fs::read()`]. +pub async fn read(fs: &F, path: impl AsRef + Send) -> Result, FsError> +where + F: FileSystem + ?Sized, +{ + let mut f = fs.new_open_options().read(true).open(path.as_ref())?; + let mut buffer = Vec::new(); + f.read_to_end(&mut buffer).await?; + + Ok(buffer) +} + +/// Asynchronously read a file's contents into memory as a string. +/// +/// This is analogous to [`std::fs::read_to_string()`]. +pub async fn read_to_string(fs: &F, path: impl AsRef + Send) -> Result +where + F: FileSystem + ?Sized, +{ + let mut f = fs.new_open_options().read(true).open(path.as_ref())?; + let mut buffer = String::new(); + f.read_to_string(&mut buffer).await?; + + Ok(buffer) +} + +/// Update a file's modification and access times, creating the file if it +/// doesn't already exist. +pub fn touch(fs: &F, path: impl AsRef + Send) -> Result<(), FsError> +where + F: FileSystem + ?Sized, +{ + let _ = fs.new_open_options().create(true).write(true).open(path)?; + + Ok(()) +} + +/// Recursively iterate over all paths inside a directory, ignoring any +/// errors that may occur along the way. +pub fn walk(fs: &F, path: impl AsRef) -> Box + '_> +where + F: FileSystem + ?Sized, +{ + let path = path.as_ref(); + let mut dirs_to_visit: VecDeque<_> = fs + .read_dir(path) + .ok() + .into_iter() + .flatten() + .filter_map(|result| result.ok()) + .collect(); + + Box::new(std::iter::from_fn(move || { + let next = dirs_to_visit.pop_back()?; + + if let Ok(children) = fs.read_dir(&next.path) { + dirs_to_visit.extend(children.flatten()); + } + + Some(next) + })) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::mem_fs::FileSystem as MemFS; + use tokio::io::AsyncReadExt; + + #[tokio::test] + async fn write() { + let fs = MemFS::default(); + + super::write(&fs, "/file.txt", b"Hello, World!") + .await + .unwrap(); + + let mut contents = String::new(); + fs.new_open_options() + .read(true) + .open("/file.txt") + .unwrap() + .read_to_string(&mut contents) + .await + .unwrap(); + assert_eq!(contents, "Hello, World!"); + } + + #[tokio::test] + async fn read() { + let fs = MemFS::default(); + fs.new_open_options() + .create(true) + .write(true) + .open("/file.txt") + .unwrap() + .write_all(b"Hello, World!") + .await + .unwrap(); + + let contents = super::read_to_string(&fs, "/file.txt").await.unwrap(); + assert_eq!(contents, "Hello, World!"); + + let contents = super::read(&fs, "/file.txt").await.unwrap(); + assert_eq!(contents, b"Hello, World!"); + } + + #[tokio::test] + async fn create_dir_all() { + let fs = MemFS::default(); + super::write(&fs, "/file.txt", b"").await.unwrap(); + + assert!(!super::exists(&fs, "/really/nested/directory")); + super::create_dir_all(&fs, "/really/nested/directory").unwrap(); + assert!(super::exists(&fs, "/really/nested/directory")); + + // It's okay to create the same directory multiple times + super::create_dir_all(&fs, "/really/nested/directory").unwrap(); + + // You can't create a directory on top of a file + assert_eq!( + super::create_dir_all(&fs, "/file.txt").unwrap_err(), + FsError::BaseNotDirectory + ); + assert_eq!( + super::create_dir_all(&fs, "/file.txt/invalid/path").unwrap_err(), + FsError::BaseNotDirectory + ); + } + + #[tokio::test] + async fn touch() { + let fs = MemFS::default(); + + super::touch(&fs, "/file.txt").unwrap(); + + assert_eq!(super::read(&fs, "/file.txt").await.unwrap(), b""); + } +} diff --git a/lib/vfs/src/overlay_fs.rs b/lib/vfs/src/overlay_fs.rs new file mode 100644 index 00000000000..b1e96a8bd18 --- /dev/null +++ b/lib/vfs/src/overlay_fs.rs @@ -0,0 +1,615 @@ +use std::{fmt::Debug, path::Path}; + +use crate::{ + ops, FileOpener, FileSystem, FileSystems, FsError, Metadata, OpenOptions, OpenOptionsConfig, + ReadDir, VirtualFile, +}; + +/// A primary filesystem and chain of secondary filesystems that are overlayed +/// on top of each other. +/// +/// # Precedence +/// +/// The [`OverlayFileSystem`] will execute operations based on precedence. +/// +/// +/// Most importantly, this means earlier filesystems can shadow files and +/// directories that have a lower precedence. +/// +///# Examples +/// +/// Something useful to know is that the [`FileSystems`] trait is implemented +/// for both arrays and tuples. +/// +/// For example, if you want to create a [`crate::FileSystem`] which will +/// create files in-memory while still being able to read from the host, you +/// might do something like this: +/// +/// ```rust +/// use wasmer_vfs::{ +/// mem_fs::FileSystem as MemFS, +/// host_fs::FileSystem as HostFS, +/// OverlayFileSystem, +/// }; +/// let fs = OverlayFileSystem::new(MemFS::default(), [HostFS]); +/// +/// // This also has the benefit of storing the two values in-line with no extra +/// // overhead or indirection. +/// assert_eq!( +/// std::mem::size_of_val(&fs), +/// std::mem::size_of::<(MemFS, HostFS)>(), +/// ); +/// ``` +/// +/// A more complex example is +#[derive(Clone, PartialEq, Eq)] +pub struct OverlayFileSystem { + primary: P, + secondaries: S, +} + +impl OverlayFileSystem +where + P: FileSystem + 'static, + S: for<'a> FileSystems<'a> + Send + Sync + 'static, +{ + /// Create a new [`FileSystem`] using a primary [`crate::FileSystem`] and a + /// chain of secondary [`FileSystems`]. + pub fn new(primary: P, secondaries: S) -> Self { + OverlayFileSystem { + primary, + secondaries, + } + } + + /// Get a reference to the primary filesystem. + pub fn primary(&self) -> &P { + &self.primary + } + + /// Get a mutable reference to the primary filesystem. + pub fn primary_mut(&mut self) -> &mut P { + &mut self.primary + } + + /// Get a reference to the secondary filesystems. + pub fn secondaries(&self) -> &S { + &self.secondaries + } + + /// Get a mutable reference to the secondary filesystems. + pub fn secondaries_mut(&mut self) -> &mut S { + &mut self.secondaries + } + + /// Consume the [`OverlayFileSystem`], returning the underlying primary and + /// secondary filesystems. + pub fn into_inner(self) -> (P, S) { + (self.primary, self.secondaries) + } + + fn permission_error_or_not_found(&self, path: &Path) -> Result<(), FsError> { + for fs in self.secondaries.filesystems() { + if ops::exists(fs, path) { + return Err(FsError::PermissionDenied); + } + } + + Err(FsError::EntryNotFound) + } +} + +impl FileSystem for OverlayFileSystem +where + P: FileSystem + 'static, + S: for<'a> FileSystems<'a> + Send + Sync + 'static, +{ + fn read_dir(&self, path: &Path) -> Result { + let mut entries = Vec::new(); + let mut had_at_least_one_success = false; + + let filesystems = std::iter::once(&self.primary as &dyn FileSystem) + .into_iter() + .chain(self.secondaries().filesystems()); + + for fs in filesystems { + match fs.read_dir(path) { + Ok(r) => { + for entry in r { + entries.push(entry?); + } + had_at_least_one_success = true; + } + Err(e) + if { + let e = e; + matches!(e, FsError::EntryNotFound) + } => + { + continue + } + Err(e) => return Err(e), + } + } + + if had_at_least_one_success { + // Make sure later entries are removed in favour of earlier ones. + // Note: this sort is guaranteed to be stable, meaning filesystems + // "higher up" the chain will be further towards the start and kept + // when deduplicating. + entries.sort_by(|a, b| a.path.cmp(&b.path)); + entries.dedup_by(|a, b| a.path == b.path); + + Ok(ReadDir::new(entries)) + } else { + Err(FsError::BaseNotDirectory) + } + } + + fn create_dir(&self, path: &Path) -> Result<(), FsError> { + match self.primary.create_dir(path) { + Err(e) + if { + let e = e; + matches!(e, FsError::EntryNotFound) + } => {} + other => return other, + } + + self.permission_error_or_not_found(path) + } + + fn remove_dir(&self, path: &Path) -> Result<(), FsError> { + match self.primary.remove_dir(path) { + Err(e) + if { + let e = e; + matches!(e, FsError::EntryNotFound) + } => {} + other => return other, + } + + self.permission_error_or_not_found(path) + } + + fn rename(&self, from: &Path, to: &Path) -> Result<(), FsError> { + match self.primary.rename(from, to) { + Err(e) + if { + let e = e; + matches!(e, FsError::EntryNotFound) + } => {} + other => return other, + } + + self.permission_error_or_not_found(from) + } + + fn metadata(&self, path: &Path) -> Result { + match self.primary.metadata(path) { + Ok(meta) => return Ok(meta), + Err(e) + if { + let e = e; + matches!(e, FsError::EntryNotFound) + } => {} + Err(e) => return Err(e), + } + + for fs in self.secondaries.filesystems() { + match fs.metadata(path) { + Err(e) + if { + let e = e; + matches!(e, FsError::EntryNotFound) + } => + { + continue + } + other => return other, + } + } + + Err(FsError::EntryNotFound) + } + + fn remove_file(&self, path: &Path) -> Result<(), FsError> { + match self.primary.remove_file(path) { + Err(e) + if { + let e = e; + matches!(e, FsError::EntryNotFound) + } => {} + other => return other, + } + + self.permission_error_or_not_found(path) + } + + fn new_open_options(&self) -> OpenOptions<'_> { + OpenOptions::new(self) + } +} + +impl FileOpener for OverlayFileSystem +where + P: FileSystem, + S: for<'a> FileSystems<'a> + Send + Sync + 'static, +{ + fn open( + &self, + path: &Path, + conf: &OpenOptionsConfig, + ) -> Result, FsError> { + match self + .primary + .new_open_options() + .options(conf.clone()) + .open(path) + { + Err(e) + if { + let e = e; + matches!(e, FsError::EntryNotFound) + } => {} + other => return other, + } + + if (conf.create || conf.create_new) && !ops::exists(self, path) { + if let Some(parent) = path.parent() { + let parent_exists_on_secondary_fs = self + .secondaries + .filesystems() + .into_iter() + .any(|fs| ops::is_dir(fs, parent)); + if parent_exists_on_secondary_fs { + // We fall into the special case where you can create a file + // that looks like it is inside a secondary filesystem folder, + // but actually it gets created on the host + ops::create_dir_all(&self.primary, parent)?; + return self + .primary + .new_open_options() + .options(conf.clone()) + .open(path); + } else { + return Err(FsError::EntryNotFound); + } + } + } + + if opening_would_require_mutations(&self.secondaries, path, conf) { + return Err(FsError::PermissionDenied); + } + + for fs in self.secondaries.filesystems() { + match fs.new_open_options().options(conf.clone()).open(path) { + Err(e) + if { + let e = e; + matches!(e, FsError::EntryNotFound) + } => + { + continue + } + other => return other, + } + } + + Err(FsError::EntryNotFound) + } +} + +fn opening_would_require_mutations( + secondaries: &S, + path: &Path, + conf: &OpenOptionsConfig, +) -> bool +where + S: for<'a> FileSystems<'a> + Send + Sync, +{ + if conf.append || conf.write || conf.create_new | conf.truncate { + return true; + } + + if conf.create { + // Would we create the file if it doesn't exist yet? + let already_exists = secondaries + .filesystems() + .into_iter() + .any(|fs| ops::is_file(fs, path)); + + if !already_exists { + return true; + } + } + + false +} + +impl Debug for OverlayFileSystem +where + P: FileSystem, + S: for<'a> FileSystems<'a>, +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + struct IterFilesystems<'a, S>(&'a S); + impl<'a, S> Debug for IterFilesystems<'a, S> + where + S: for<'b> FileSystems<'b>, + { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut f = f.debug_list(); + + for fs in self.0.filesystems() { + f.entry(&fs); + } + + f.finish() + } + } + + f.debug_struct("OverlayFileSystem") + .field("primary", &self.primary) + .field("secondaries", &IterFilesystems(&self.secondaries)) + .finish() + } +} + +#[cfg(test)] +mod tests { + use std::{path::PathBuf, sync::Arc}; + + use tempfile::TempDir; + use tokio::io::AsyncWriteExt; + + use super::*; + use crate::{mem_fs::FileSystem as MemFS, webc_fs::WebcFileSystem, RootFileSystemBuilder}; + + #[test] + fn object_safe() { + fn _box_with_memfs( + fs: OverlayFileSystem>, + ) -> Box { + Box::new(fs) + } + + fn _arc(fs: OverlayFileSystem) -> Arc + where + A: FileSystem + 'static, + S: for<'a> FileSystems<'a> + Send + Sync + Debug + 'static, + { + Arc::new(fs) + } + } + + #[tokio::test] + async fn remove_directory() { + let primary = MemFS::default(); + let secondary = MemFS::default(); + let first = Path::new("/first"); + let second = Path::new("/second"); + let file_txt = second.join("file.txt"); + let third = Path::new("/third"); + primary.create_dir(first).unwrap(); + primary.create_dir(second).unwrap(); + primary + .new_open_options() + .create(true) + .write(true) + .open(&file_txt) + .unwrap() + .write_all(b"Hello, World!") + .await + .unwrap(); + secondary.create_dir(third).unwrap(); + + let overlay = OverlayFileSystem::new(primary, [secondary]); + + // Delete a folder on the primary filesystem + overlay.remove_dir(first).unwrap(); + assert_eq!( + overlay.primary().metadata(first).unwrap_err(), + FsError::EntryNotFound, + "Deleted from primary" + ); + assert!(!ops::exists(&overlay.secondaries[0], &second)); + + // Directory on the primary fs isn't empty + assert_eq!( + overlay.remove_dir(second).unwrap_err(), + FsError::DirectoryNotEmpty, + ); + + // Try to remove something on one of the overlay filesystems + assert_eq!( + overlay.remove_dir(third).unwrap_err(), + FsError::PermissionDenied, + ); + assert!(ops::exists(&overlay.secondaries[0], third)); + } + + #[tokio::test] + async fn open_files() { + let primary = MemFS::default(); + let secondary = MemFS::default(); + ops::create_dir_all(&primary, "/primary").unwrap(); + ops::touch(&primary, "/primary/read.txt").unwrap(); + ops::touch(&primary, "/primary/write.txt").unwrap(); + ops::create_dir_all(&secondary, "/secondary").unwrap(); + ops::touch(&secondary, "/secondary/read.txt").unwrap(); + ops::touch(&secondary, "/secondary/write.txt").unwrap(); + ops::create_dir_all(&secondary, "/primary").unwrap(); + ops::write(&secondary, "/primary/read.txt", "This is shadowed") + .await + .unwrap(); + + let fs = OverlayFileSystem::new(primary, [secondary]); + + // Any new files will be created on the primary fs + let _ = fs + .new_open_options() + .create(true) + .write(true) + .open("/new.txt") + .unwrap(); + assert!(ops::exists(&fs.primary, "/new.txt")); + assert!(!ops::exists(&fs.secondaries[0], "/new.txt")); + + // You can open a file for reading and writing on the primary fs + let _ = fs + .new_open_options() + .create(false) + .write(true) + .read(true) + .open("/primary/write.txt") + .unwrap(); + + // Files on the primary should always shadow the secondary + let content = ops::read_to_string(&fs, "/primary/read.txt").await.unwrap(); + assert_ne!(content, "This is shadowed"); + } + + #[test] + fn create_file_that_looks_like_it_is_in_a_secondary_filesystem_folder() { + let primary = MemFS::default(); + let secondary = MemFS::default(); + ops::create_dir_all(&secondary, "/path/to/").unwrap(); + assert!(!ops::is_dir(&primary, "/path/to/")); + let fs = OverlayFileSystem::new(primary, [secondary]); + + ops::touch(&fs, "/path/to/file.txt").unwrap(); + + assert!(ops::is_dir(&fs.primary, "/path/to/")); + assert!(ops::is_file(&fs.primary, "/path/to/file.txt")); + assert!(!ops::is_file(&fs.secondaries[0], "/path/to/file.txt")); + } + + #[tokio::test] + async fn listed_files_appear_overlayed() { + let primary = MemFS::default(); + let secondary = MemFS::default(); + let secondary_overlayed = MemFS::default(); + ops::create_dir_all(&primary, "/primary").unwrap(); + ops::touch(&primary, "/primary/read.txt").unwrap(); + ops::touch(&primary, "/primary/write.txt").unwrap(); + ops::create_dir_all(&secondary, "/secondary").unwrap(); + ops::touch(&secondary, "/secondary/read.txt").unwrap(); + ops::touch(&secondary, "/secondary/write.txt").unwrap(); + // This second "secondary" filesystem should share the same folders as + // the first one. + ops::create_dir_all(&secondary_overlayed, "/secondary").unwrap(); + ops::touch(&secondary_overlayed, "/secondary/overlayed.txt").unwrap(); + + let fs = OverlayFileSystem::new(primary, [secondary, secondary_overlayed]); + + let paths: Vec<_> = ops::walk(&fs, "/").map(|entry| entry.path()).collect(); + assert_eq!( + paths, + vec![ + PathBuf::from("/secondary"), + PathBuf::from("/secondary/write.txt"), + PathBuf::from("/secondary/read.txt"), + PathBuf::from("/secondary/overlayed.txt"), + PathBuf::from("/primary"), + PathBuf::from("/primary/write.txt"), + PathBuf::from("/primary/read.txt"), + ] + ); + } + + #[tokio::test] + async fn wasi_runner_use_case() { + // Set up some dummy files on the host + let temp = TempDir::new().unwrap(); + let first = temp.path().join("first"); + let file_txt = first.join("file.txt"); + let second = temp.path().join("second"); + std::fs::create_dir_all(&first).unwrap(); + std::fs::write(&file_txt, b"First!").unwrap(); + std::fs::create_dir_all(&second).unwrap(); + // configure the union FS so things are saved in memory by default + // (initialized with a set of unix-like folders), but certain folders + // are first to the host. + let primary = RootFileSystemBuilder::new().build(); + let host_fs: Arc = Arc::new(crate::host_fs::FileSystem); + let first_dirs = [(&first, "/first"), (&second, "/second")]; + for (host, guest) in first_dirs { + primary + .mount(PathBuf::from(guest), &host_fs, host.clone()) + .unwrap(); + } + // Set up the secondary file systems + let webc = webc::v1::WebCOwned::parse( + include_bytes!("../../c-api/examples/assets/python-0.1.0.wasmer").to_vec(), + &webc::v1::ParseOptions::default(), + ) + .unwrap(); + let webc = WebcFileSystem::init_all(Arc::new(webc)); + + let fs = OverlayFileSystem::new(primary, [webc]); + + // We should get all the normal directories from rootfs (primary) + assert!(ops::is_dir(&fs, "/lib")); + assert!(ops::is_dir(&fs, "/bin")); + assert!(ops::is_file(&fs, "/dev/stdin")); + assert!(ops::is_file(&fs, "/dev/stdout")); + // We also want to see files from the WEBC volumes (secondary) + assert!(ops::is_dir(&fs, "/lib/python3.6")); + assert!(ops::is_file(&fs, "/lib/python3.6/collections/__init__.py")); + // files on a secondary fs aren't writable + assert_eq!( + fs.new_open_options() + .append(true) + .open("/lib/python3.6/collections/__init__.py") + .unwrap_err(), + FsError::PermissionDenied, + ); + // you are allowed to create files that look like they are in a secondary + // folder, though + ops::touch(&fs, "/lib/python3.6/collections/something-else.py").unwrap(); + // But it'll be on the primary filesystem, not the secondary one + assert!(ops::is_file( + &fs.primary, + "/lib/python3.6/collections/something-else.py" + )); + assert!(!ops::is_file( + &fs.secondaries[0], + "/lib/python3.6/collections/something-else.py" + )); + // You can do the same thing with folders + fs.create_dir("/lib/python3.6/something-else".as_ref()) + .unwrap(); + assert!(ops::is_dir(&fs.primary, "/lib/python3.6/something-else")); + assert!(!ops::is_dir( + &fs.secondaries[0], + "/lib/python3.6/something-else" + )); + // It only works when you are directly inside an existing directory + // on the secondary filesystem, though + assert_eq!( + ops::touch(&fs, "/lib/python3.6/collections/this/doesnt/exist.txt").unwrap_err(), + FsError::EntryNotFound + ); + // you should also be able to read files mounted from the host + assert!(ops::is_dir(&fs, "/first")); + assert!(ops::is_file(&fs, "/first/file.txt")); + assert_eq!( + ops::read_to_string(&fs, "/first/file.txt").await.unwrap(), + "First!" + ); + // Overwriting them is fine and we'll see the changes on the host + ops::write(&fs, "/first/file.txt", "Updated").await.unwrap(); + assert_eq!(std::fs::read_to_string(&file_txt).unwrap(), "Updated"); + // The filesystem will see changes on the host that happened after it was + // set up + let another = second.join("another.txt"); + std::fs::write(&another, "asdf").unwrap(); + assert_eq!( + ops::read_to_string(&fs, "/second/another.txt") + .await + .unwrap(), + "asdf" + ); + } +} diff --git a/lib/vfs/src/passthru_fs.rs b/lib/vfs/src/passthru_fs.rs index c75f3265b7c..7164bee7718 100644 --- a/lib/vfs/src/passthru_fs.rs +++ b/lib/vfs/src/passthru_fs.rs @@ -70,7 +70,7 @@ mod test_builder { .create(true) .open("/foo.txt") .unwrap() - .write(b"hello") + .write_all(b"hello") .await .unwrap(); diff --git a/lib/vfs/src/union_fs.rs b/lib/vfs/src/union_fs.rs index 61df13517fc..9685f588110 100644 --- a/lib/vfs/src/union_fs.rs +++ b/lib/vfs/src/union_fs.rs @@ -460,14 +460,13 @@ impl FileOpener for UnionFileSystem { #[cfg(test)] mod tests { + use std::{path::Path, sync::Arc}; + use tokio::io::AsyncWriteExt; - use crate::host_fs::FileSystem; - use crate::mem_fs; - use crate::FileSystem as FileSystemTrait; - use crate::FsError; - use crate::UnionFileSystem; - use std::path::Path; + use crate::{ + host_fs::FileSystem, mem_fs, ops, FileSystem as FileSystemTrait, FsError, UnionFileSystem, + }; fn gen_filesystem() -> UnionFileSystem { let mut union = UnionFileSystem::new(); @@ -1073,4 +1072,34 @@ mod tests { let _ = fs_extra::remove_items(&["./test_canonicalize"]); } */ + + #[test] + #[ignore = "Not yet supported. See https://github.com/wasmerio/wasmer/issues/3678"] + fn mount_to_overlapping_directories() { + let top_level = mem_fs::FileSystem::default(); + ops::touch(&top_level, "/file.txt").unwrap(); + let nested = mem_fs::FileSystem::default(); + ops::touch(&nested, "/another-file.txt").unwrap(); + + let mut fs = UnionFileSystem::default(); + fs.mount( + "top-level", + "/", + false, + Box::new(top_level), + Some("/top-level"), + ); + fs.mount( + "nested", + "/", + false, + Box::new(nested), + Some("/top-level/nested"), + ); + + assert!(ops::is_dir(&fs, "/top-level")); + assert!(ops::is_file(&fs, "/top-level/file.txt")); + assert!(ops::is_dir(&fs, "/top-level/nested")); + assert!(ops::is_file(&fs, "/top-level/nested/another-file.txt")); + } }