Merge pull request #3677 from wasmerio/overlay-fs

Overlay FileSystem

lib/vfs/Cargo.toml

@@ -25,6 +25,7 @@ pin-project-lite = "0.2.9"
 indexmap = "1.9.2"
 
 [dev-dependencies]
+pretty_assertions = "1.3.0"
 tempfile = "3.4.0"
 tokio = { version = "1", features = [ "io-util", "rt" ], default_features = false }
 

lib/vfs/src/builder.rs

@@ -133,7 +133,7 @@ mod test_builder {
             .unwrap();
         assert_eq!(dev_zero.write(b"hello").await.unwrap(), 5);
         let mut buf = vec![1; 10];
-        dev_zero.read(&mut buf[..]).await.unwrap();
+        dev_zero.read_exact(&mut buf[..]).await.unwrap();
         assert_eq!(buf, vec![0; 10]);
         assert!(dev_zero.get_special_fd().is_none());
 

lib/vfs/src/filesystems.rs

@@ -0,0 +1,106 @@
+use crate::FileSystem;
+
+/// A chain of one or more [`FileSystem`]s.
+pub trait FileSystems<'a>: 'a {
+    // FIXME(Michael-F-Bryan): Rewrite this to use GATs when we bump the MSRV to
+    // 1.65 or higher. That'll get rid of all the lifetimes and HRTBs.
+    type Iter: IntoIterator<Item = &'a dyn FileSystem> + 'a;
+
+    /// Get something that can be used to iterate over the underlying
+    /// filesystems.
+    fn filesystems(&'a self) -> Self::Iter;
+}
+
+impl<'a, 'b, S> FileSystems<'a> for &'b S
+where
+    S: FileSystems<'a> + 'b,
+    'b: 'a,
+{
+    type Iter = S::Iter;
+
+    fn filesystems(&'a self) -> Self::Iter {
+        (**self).filesystems()
+    }
+}
+
+impl<'a, T> FileSystems<'a> for Vec<T>
+where
+    T: FileSystem,
+{
+    type Iter = <[T] as FileSystems<'a>>::Iter;
+
+    fn filesystems(&'a self) -> Self::Iter {
+        self[..].filesystems()
+    }
+}
+
+impl<'a, T, const N: usize> FileSystems<'a> for [T; N]
+where
+    T: FileSystem,
+{
+    type Iter = [&'a dyn FileSystem; N];
+
+    fn filesystems(&'a self) -> Self::Iter {
+        // TODO: rewrite this when array::each_ref() is stable
+        let mut i = 0;
+        [(); N].map(|_| {
+            let f = &self[i] as &dyn FileSystem;
+            i += 1;
+            f
+        })
+    }
+}
+
+impl<'a, T> FileSystems<'a> for [T]
+where
+    T: FileSystem,
+{
+    type Iter = std::iter::Map<std::slice::Iter<'a, T>, fn(&T) -> &dyn FileSystem>;
+
+    fn filesystems(&'a self) -> Self::Iter {
+        self.iter().map(|fs| fs as &dyn FileSystem)
+    }
+}
+
+impl<'a> FileSystems<'a> for () {
+    type Iter = std::iter::Empty<&'a dyn FileSystem>;
+
+    fn filesystems(&'a self) -> Self::Iter {
+        std::iter::empty()
+    }
+}
+
+macro_rules! count {
+    ($first:tt $($rest:tt)*) => {
+        1 + count!($($rest)*)
+    };
+    () => { 0 };
+}
+
+macro_rules! tuple_filesystems {
+    ($first:ident $(, $rest:ident)* $(,)?) => {
+        impl<'a, $first, $( $rest ),*> FileSystems<'a> for ($first, $($rest),*)
+        where
+            $first: FileSystem,
+            $($rest: FileSystem),*
+        {
+            type Iter = [&'a dyn FileSystem; count!($first $($rest)*)];
+
+            fn filesystems(&'a self) -> Self::Iter {
+                #[allow(non_snake_case)]
+                let ($first, $($rest),*) = self;
+
+                [
+                    $first as &dyn FileSystem,
+                    $($rest),*
+                ]
+            }
+
+        }
+
+        tuple_filesystems!($($rest),*);
+    };
+    () => {};
+}
+
+tuple_filesystems!(F1, F2, F3, F4, F5, F6, F7, F8, F9, F10, F11, F12, F13, F14, F15, F16,);

lib/vfs/src/host_fs.rs

@@ -1365,7 +1365,5 @@ mod tests {
                 .join("hello.txt")),
             "canonicalizing a crazily stupid path name",
         );
-
-        let _ = fs_extra::remove_items(&["./test_canonicalize"]);
     }
 }

lib/vfs/src/lib.rs

@@ -1,3 +1,7 @@
+#[cfg(test)]
+#[macro_use]
+extern crate pretty_assertions;
+
 use std::any::Any;
 use std::ffi::OsString;
 use std::fmt;
@@ -25,6 +29,9 @@ pub mod tmp_fs;
 pub mod union_fs;
 pub mod zero_file;
 // tty_file -> see wasmer_wasi::tty_file
+mod filesystems;
+pub(crate) mod ops;
+mod overlay_fs;
 pub mod pipe;
 #[cfg(feature = "static-fs")]
 pub mod static_fs;
@@ -38,7 +45,9 @@ pub use builder::*;
 pub use combine_file::*;
 pub use dual_write_file::*;
 pub use empty_fs::*;
+pub use filesystems::FileSystems;
 pub use null_file::*;
+pub use overlay_fs::OverlayFileSystem;
 pub use passthru_fs::*;
 pub use pipe::*;
 pub use special_file::*;
@@ -138,6 +147,20 @@ impl OpenOptionsConfig {
     pub const fn truncate(&self) -> bool {
         self.truncate
     }
+
+    /// Would a file opened with this [`OpenOptionsConfig`] change files on the
+    /// filesystem.
+    pub const fn would_mutate(&self) -> bool {
+        let OpenOptionsConfig {
+            read: _,
+            write,
+            create_new,
+            create,
+            append,
+            truncate,
+        } = *self;
+        append || write || create || create_new || truncate
+    }
 }
 
 impl<'a> fmt::Debug for OpenOptions<'a> {
@@ -170,36 +193,62 @@ impl<'a> OpenOptions<'a> {
         self.conf.clone()
     }
 
+    /// Use an existing [`OpenOptionsConfig`] to configure this [`OpenOptions`].
     pub fn options(&mut self, options: OpenOptionsConfig) -> &mut Self {
         self.conf = options;
         self
     }
 
+    /// Sets the option for read access.
+    ///
+    /// This option, when true, will indicate that the file should be
+    /// `read`-able if opened.
     pub fn read(&mut self, read: bool) -> &mut Self {
         self.conf.read = read;
         self
     }
 
+    /// Sets the option for write access.
+    ///
+    /// This option, when true, will indicate that the file should be
+    /// `write`-able if opened.
+    ///
+    /// If the file already exists, any write calls on it will overwrite its
+    /// contents, without truncating it.
     pub fn write(&mut self, write: bool) -> &mut Self {
         self.conf.write = write;
         self
     }
 
+    /// Sets the option for the append mode.
+    ///
+    /// This option, when true, means that writes will append to a file instead
+    /// of overwriting previous contents.
+    /// Note that setting `.write(true).append(true)` has the same effect as
+    /// setting only `.append(true)`.
     pub fn append(&mut self, append: bool) -> &mut Self {
         self.conf.append = append;
         self
     }
 
+    /// Sets the option for truncating a previous file.
+    ///
+    /// If a file is successfully opened with this option set it will truncate
+    /// the file to 0 length if it already exists.
+    ///
+    /// The file must be opened with write access for truncate to work.
     pub fn truncate(&mut self, truncate: bool) -> &mut Self {
         self.conf.truncate = truncate;
         self
     }
 
+    /// Sets the option to create a new file, or open it if it already exists.
     pub fn create(&mut self, create: bool) -> &mut Self {
         self.conf.create = create;
         self
     }
 
+    /// Sets the option to create a new file, failing if it already exists.
     pub fn create_new(&mut self, create_new: bool) -> &mut Self {
         self.conf.create_new = create_new;
         self

lib/vfs/src/mem_fs/filesystem.rs

@@ -12,9 +12,8 @@ use std::sync::{Arc, RwLock};
 
 /// The in-memory file system!
 ///
-/// It's a thin wrapper around [`FileSystemInner`]. This `FileSystem`
-/// type can be cloned, it's a light copy of the `FileSystemInner`
-/// (which is behind a `Arc` + `RwLock`.
+/// This `FileSystem` type can be cloned, it's a light copy of the
+/// `FileSystemInner` (which is behind a `Arc` + `RwLock`).
 #[derive(Clone, Default)]
 pub struct FileSystem {
     pub(super) inner: Arc<RwLock<FileSystemInner>>,
@@ -937,9 +936,25 @@ impl Default for FileSystemInner {
     }
 }
 
+#[allow(dead_code)] // The `No` variant.
+pub(super) enum DirectoryMustBeEmpty {
+    Yes,
+    No,
+}
+
+impl DirectoryMustBeEmpty {
+    pub(super) fn yes(&self) -> bool {
+        matches!(self, Self::Yes)
+    }
+
+    pub(super) fn no(&self) -> bool {
+        !self.yes()
+    }
+}
+
 #[cfg(test)]
 mod test_filesystem {
-    use crate::{mem_fs::*, DirEntry, FileSystem as FS, FileType, FsError};
+    use crate::{mem_fs::*, ops, DirEntry, FileSystem as FS, FileType, FsError};
 
     macro_rules! path {
         ($path:expr) => {
@@ -1686,20 +1701,26 @@ mod test_filesystem {
             "canonicalizing a crazily stupid path name",
         );
     }
-}
-
-#[allow(dead_code)] // The `No` variant.
-pub(super) enum DirectoryMustBeEmpty {
-    Yes,
-    No,
-}
 
-impl DirectoryMustBeEmpty {
-    pub(super) fn yes(&self) -> bool {
-        matches!(self, Self::Yes)
-    }
+    #[test]
+    #[ignore = "Not yet supported. See https://github.com/wasmerio/wasmer/issues/3678"]
+    fn mount_to_overlapping_directories() {
+        let top_level = FileSystem::default();
+        ops::touch(&top_level, "/file.txt").unwrap();
+        let nested = FileSystem::default();
+        ops::touch(&nested, "/another-file.txt").unwrap();
+        let top_level: Arc<dyn crate::FileSystem + Send + Sync> = Arc::new(top_level);
+        let nested: Arc<dyn crate::FileSystem + Send + Sync> = Arc::new(nested);
 
-    pub(super) fn no(&self) -> bool {
-        !self.yes()
+        let fs = FileSystem::default();
+        fs.mount("/top-level".into(), &top_level, "/".into())
+            .unwrap();
+        fs.mount("/top-level/nested".into(), &nested, "/".into())
+            .unwrap();
+
+        assert!(ops::is_dir(&fs, "/top-level"));
+        assert!(ops::is_file(&fs, "/top-level/file.txt"));
+        assert!(ops::is_dir(&fs, "/top-level/nested"));
+        assert!(ops::is_file(&fs, "/top-level/nested/another-file.txt"));
     }
 }