Webc Inheritance Fixes

lib/vfs/src/webc_fs.rs

@@ -20,9 +20,10 @@ pub struct WebcFileSystem<T>
 where
     T: std::fmt::Debug + Send + Sync + 'static,
 {
-    pub package: String,
     pub webc: Arc<T>,
     pub memory: Arc<MemFileSystem>,
+    top_level_dirs: Vec<String>,
+    volumes: Vec<webc::Volume<'static>>,
 }
 
 impl<T> WebcFileSystem<T>
@@ -31,18 +32,44 @@ where
     T: Deref<Target = WebC<'static>>,
 {
     pub fn init(webc: Arc<T>, package: &str) -> Self {
-        let fs = Self {
-            package: package.to_string(),
+        let mut fs = Self {
             webc: webc.clone(),
             memory: Arc::new(MemFileSystem::default()),
+            top_level_dirs: Vec::new(),
+            volumes: Vec::new(),
         };
+
         for volume in webc.get_volumes_for_package(package) {
+            if let Some(vol_ref) = webc.volumes.get(&volume) {
+                fs.volumes.push(vol_ref.clone());
+            }
             for directory in webc.list_directories(&volume) {
+                fs.top_level_dirs.push(directory.clone());
+                let _ = fs.create_dir(Path::new(&directory));
+            }
+        }
+        fs
+    }
+
+    pub fn init_all(webc: Arc<T>) -> Self {
+        let mut fs = Self {
+            webc: webc.clone(),
+            memory: Arc::new(MemFileSystem::default()),
+            top_level_dirs: Vec::new(),
+            volumes: webc.volumes.clone().into_values().collect(),
+        };
+        for (header, _) in webc.volumes.iter() {
+            for directory in webc.list_directories(header) {
+                fs.top_level_dirs.push(directory.clone());
                 let _ = fs.create_dir(Path::new(&directory));
             }
         }
         fs
     }
+
+    pub fn top_level_dirs(&self) -> &Vec<String> {
+        &self.top_level_dirs
+    }
 }
 
 /// Custom file opener, returns a WebCFile
@@ -66,7 +93,6 @@ where
                     .map_err(|_e| FsError::EntryNotFound)?;
 
                 Ok(Box::new(WebCFile {
-                    package: self.package.clone(),
                     volume,
                     webc: self.webc.clone(),
                     path: path.to_path_buf(),
@@ -75,8 +101,8 @@ where
                 }))
             }
             None => {
-                for volume in self.webc.get_volumes_for_package(&self.package) {
-                    let v = match self.webc.volumes.get(&volume) {
+                for (volume, _) in self.webc.volumes.iter() {
+                    let v = match self.webc.volumes.get(volume) {
                         Some(s) => s,
                         None => continue, // error
                     };
@@ -87,7 +113,6 @@ where
                     };
 
                     return Ok(Box::new(WebCFile {
-                        package: self.package.clone(),
                         volume: volume.clone(),
                         webc: self.webc.clone(),
                         path: path.to_path_buf(),
@@ -107,8 +132,6 @@ where
     T: std::fmt::Debug + Send + Sync + 'static,
 {
     pub webc: Arc<T>,
-    #[allow(dead_code)]
-    pub package: String,
     pub volume: String,
     #[allow(dead_code)]
     pub path: PathBuf,
@@ -281,10 +304,12 @@ where
     fn read_dir(&self, path: &Path) -> Result<ReadDir, FsError> {
         let path = normalizes_path(path);
         let read_dir_result = self
-            .webc
-            .read_dir(&self.package, &path)
+            .volumes
+            .iter()
+            .filter_map(|v| v.read_dir(&path).ok())
+            .next()
             .map(|o| transform_into_read_dir(Path::new(&path), o.as_ref()))
-            .map_err(|_| FsError::EntryNotFound);
+            .ok_or(FsError::EntryNotFound);
 
         match read_dir_result {
             Ok(o) => Ok(o),
@@ -299,7 +324,7 @@ where
     fn remove_dir(&self, path: &Path) -> Result<(), FsError> {
         let path = normalizes_path(path);
         let result = self.memory.remove_dir(Path::new(&path));
-        if self.webc.get_file_entry(&self.package, &path).is_some() {
+        if self.volumes.iter().any(|v| v.get_file_entry(&path).is_ok()) {
             Ok(())
         } else {
             result
@@ -309,23 +334,34 @@ where
         let from = normalizes_path(from);
         let to = normalizes_path(to);
         let result = self.memory.rename(Path::new(&from), Path::new(&to));
-        if self.webc.get_file_entry(&self.package, &from).is_some() {
+        if self.volumes.iter().any(|v| v.get_file_entry(&from).is_ok()) {
             Ok(())
         } else {
             result
         }
     }
     fn metadata(&self, path: &Path) -> Result<Metadata, FsError> {
         let path = normalizes_path(path);
-        if let Some(fs_entry) = self.webc.get_file_entry(&self.package, &path) {
+        if let Some(fs_entry) = self
+            .volumes
+            .iter()
+            .filter_map(|v| v.get_file_entry(&path).ok())
+            .next()
+        {
             Ok(Metadata {
                 ft: translate_file_type(FsEntryType::File),
                 accessed: 0,
                 created: 0,
                 modified: 0,
-                len: fs_entry.1.get_len(),
+                len: fs_entry.get_len(),
             })
-        } else if self.webc.read_dir(&self.package, &path).is_ok() {
+        } else if self
+            .volumes
+            .iter()
+            .filter_map(|v| v.read_dir(&path).ok())
+            .next()
+            .is_some()
+        {
             Ok(Metadata {
                 ft: translate_file_type(FsEntryType::Dir),
                 accessed: 0,
@@ -340,7 +376,13 @@ where
     fn remove_file(&self, path: &Path) -> Result<(), FsError> {
         let path = normalizes_path(path);
         let result = self.memory.remove_file(Path::new(&path));
-        if self.webc.get_file_entry(&self.package, &path).is_some() {
+        if self
+            .volumes
+            .iter()
+            .filter_map(|v| v.get_file_entry(&path).ok())
+            .next()
+            .is_some()
+        {
             Ok(())
         } else {
             result
@@ -351,15 +393,26 @@ where
     }
     fn symlink_metadata(&self, path: &Path) -> Result<Metadata, FsError> {
         let path = normalizes_path(path);
-        if let Some(fs_entry) = self.webc.get_file_entry(&self.package, &path) {
+        if let Some(fs_entry) = self
+            .volumes
+            .iter()
+            .filter_map(|v| v.get_file_entry(&path).ok())
+            .next()
+        {
             Ok(Metadata {
                 ft: translate_file_type(FsEntryType::File),
                 accessed: 0,
                 created: 0,
                 modified: 0,
-                len: fs_entry.1.get_len(),
+                len: fs_entry.get_len(),
             })
-        } else if self.webc.read_dir(&self.package, &path).is_ok() {
+        } else if self
+            .volumes
+            .iter()
+            .filter_map(|v| v.read_dir(&path).ok())
+            .next()
+            .is_some()
+        {
             Ok(Metadata {
                 ft: translate_file_type(FsEntryType::Dir),
                 accessed: 0,

lib/wasi/src/bin_factory/exec.rs

@@ -62,6 +62,7 @@ pub fn spawn_exec(
 
     // If the file system has not already been union'ed then do so
     env.state.fs.conditional_union(&binary);
+    tracing::debug!("{:?}", env.state.fs);
 
     // Now run the module
     spawn_exec_module(module, store, env, runtime)

lib/wasi/src/bin_factory/module_cache.rs

@@ -1,10 +1,16 @@
-use std::{cell::RefCell, collections::HashMap, ops::DerefMut, path::PathBuf, sync::RwLock};
+use std::{
+    cell::RefCell,
+    collections::{HashMap, HashSet},
+    ops::DerefMut,
+    path::PathBuf,
+    sync::RwLock,
+};
 
 use wasmer::Module;
 use wasmer_wasi_types::wasi::Snapshot0Clockid;
 
 use super::BinaryPackage;
-use crate::{syscalls::platform_clock_time_get, VirtualTaskManager, WasiRuntime};
+use crate::{syscalls::platform_clock_time_get, WasiRuntime};
 
 pub const DEFAULT_COMPILED_PATH: &str = "~/.wasmer/compiled";
 pub const DEFAULT_WEBC_PATH: &str = "~/.wasmer/webc";
@@ -83,12 +89,7 @@ impl ModuleCache {
     }
 
     // TODO: should return Result<_, anyhow::Error>
-    pub fn get_webc(
-        &self,
-        webc: &str,
-        runtime: &dyn WasiRuntime,
-        tasks: &dyn VirtualTaskManager,
-    ) -> Option<BinaryPackage> {
+    pub fn get_webc(&self, webc: &str, runtime: &dyn WasiRuntime) -> Option<BinaryPackage> {
         let name = webc.to_string();
         let now = platform_clock_time_get(Snapshot0Clockid::Monotonic, 1_000_000).unwrap() as u128;
 
@@ -109,8 +110,19 @@ impl ModuleCache {
 
         // Slow path
         let mut cache = self.cache_webc.write().unwrap();
+        self.get_webc_slow(webc, runtime, cache.deref_mut())
+    }
+
+    fn get_webc_slow(
+        &self,
+        webc: &str,
+        runtime: &dyn WasiRuntime,
+        cache: &mut HashMap<String, BinaryPackage>,
+    ) -> Option<BinaryPackage> {
+        let name = webc.to_string();
+        let now = platform_clock_time_get(Snapshot0Clockid::Monotonic, 1_000_000).unwrap() as u128;
 
-        // Check the cache
+        // Check the cache (again)
         if let Some(data) = cache.get(&name) {
             if let Some(when_cached) = data.when_cached.as_ref() {
                 let delta = now - *when_cached;
@@ -129,9 +141,27 @@ impl ModuleCache {
                 .map(|a| a.0)
                 .unwrap_or_else(|| name.as_str());
             let cache_webc_dir = self.cache_webc_dir.as_str();
-            if let Ok(data) =
-                crate::wapm::fetch_webc_task(cache_webc_dir, wapm_name, runtime, tasks)
-            {
+            if let Ok(mut data) = crate::wapm::fetch_webc_task(cache_webc_dir, wapm_name, runtime) {
+                // If the binary has no entry but it inherits from another module
+                // that does have an entry then we fall back to that inherited entry point
+                // (this convention is recursive down the list of inheritance until it finds the first entry point)
+                let mut already: HashSet<String> = Default::default();
+                while data.entry.is_none() {
+                    let mut inherits = data.uses.iter().filter_map(|webc| {
+                        if !already.contains(webc) {
+                            already.insert(webc.clone());
+                            self.get_webc_slow(webc, runtime, cache)
+                        } else {
+                            None
+                        }
+                    });
+                    if let Some(inherits) = inherits.next() {
+                        data.entry = inherits.entry.clone();
+                    } else {
+                        break;
+                    }
+                }
+
                 // If the package is the same then don't replace it
                 // as we don't want to duplicate the memory usage
                 if let Some(existing) = cache.get_mut(&name) {
@@ -277,9 +307,7 @@ mod tests {
 
         let mut store = Vec::new();
         for _ in 0..2 {
-            let webc = cache
-                .get_webc("sharrattj/dash", &rt, std::ops::Deref::deref(tasks))
-                .unwrap();
+            let webc = cache.get_webc("sharrattj/dash", &rt).unwrap();
             store.push(webc);
             tasks
                 .runtime()

lib/wasi/src/fs/mod.rs

@@ -318,7 +318,6 @@ impl FileSystem for WasiFsRoot {
 
 /// Warning, modifying these fields directly may cause invariants to break and
 /// should be considered unsafe.  These fields may be made private in a future release
-#[derive(Debug)]
 #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
 pub struct WasiFs {
     //pub repo: Repo,
@@ -1390,7 +1389,6 @@ impl WasiFs {
             _ => (),
         }
         let fd = self.get_fd(fd)?;
-        debug!("fdstat: {:?}", fd);
 
         let guard = fd.inode.read();
         let deref = guard.deref();
@@ -1752,6 +1750,18 @@ impl WasiFs {
     }
 }
 
+impl std::fmt::Debug for WasiFs {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        if let Ok(guard) = self.current_dir.try_lock() {
+            write!(f, "current_dir={} ", guard.as_str())?;
+        } else {
+            write!(f, "current_dir=(locked) ")?;
+        }
+        write!(f, "next_fd={} ", self.next_fd.load(Ordering::Relaxed))?;
+        write!(f, "{:?}", self.root_fs)
+    }
+}
+
 /// Returns the default filesystem backing
 pub fn default_fs_backing() -> Box<dyn wasmer_vfs::FileSystem + Send + Sync> {
     cfg_if::cfg_if! {

lib/wasi/src/os/command/builtins/cmd_wasmer.rs

@@ -10,7 +10,7 @@ use wasmer_wasi_types::wasi::Errno;
 use crate::{
     bin_factory::{spawn_exec, BinaryPackage, ModuleCache},
     syscalls::stderr_write,
-    VirtualTaskManager, VirtualTaskManagerExt, WasiEnv, WasiRuntime,
+    VirtualTaskManagerExt, WasiEnv, WasiRuntime,
 };
 
 const HELP: &str = r#"USAGE:
@@ -74,8 +74,7 @@ impl CmdWasmer {
             env.state = Arc::new(state);
 
             // Get the binary
-            let tasks = parent_ctx.data().tasks();
-            if let Some(binary) = self.get_package(what.clone(), tasks.deref()) {
+            if let Some(binary) = self.get_package(what.clone()) {
                 // Now run the module
                 spawn_exec(binary, name, store, env, &self.runtime, &self.cache)
             } else {
@@ -98,13 +97,8 @@ impl CmdWasmer {
         }
     }
 
-    pub fn get_package(
-        &self,
-        name: String,
-        tasks: &dyn VirtualTaskManager,
-    ) -> Option<BinaryPackage> {
-        self.cache
-            .get_webc(name.as_str(), self.runtime.deref(), tasks)
+    pub fn get_package(&self, name: String) -> Option<BinaryPackage> {
+        self.cache.get_webc(name.as_str(), self.runtime.deref())
     }
 }