diff --git a/lib/vfs/src/webc_fs.rs b/lib/vfs/src/webc_fs.rs index 9acad28557e..441e382be71 100644 --- a/lib/vfs/src/webc_fs.rs +++ b/lib/vfs/src/webc_fs.rs @@ -20,9 +20,10 @@ pub struct WebcFileSystem where T: std::fmt::Debug + Send + Sync + 'static, { - pub package: String, pub webc: Arc, pub memory: Arc, + top_level_dirs: Vec, + volumes: Vec>, } impl WebcFileSystem @@ -31,18 +32,44 @@ where T: Deref>, { pub fn init(webc: Arc, package: &str) -> Self { - let fs = Self { - package: package.to_string(), + let mut fs = Self { webc: webc.clone(), memory: Arc::new(MemFileSystem::default()), + top_level_dirs: Vec::new(), + volumes: Vec::new(), }; + for volume in webc.get_volumes_for_package(package) { + if let Some(vol_ref) = webc.volumes.get(&volume) { + fs.volumes.push(vol_ref.clone()); + } for directory in webc.list_directories(&volume) { + fs.top_level_dirs.push(directory.clone()); + let _ = fs.create_dir(Path::new(&directory)); + } + } + fs + } + + pub fn init_all(webc: Arc) -> Self { + let mut fs = Self { + webc: webc.clone(), + memory: Arc::new(MemFileSystem::default()), + top_level_dirs: Vec::new(), + volumes: webc.volumes.clone().into_values().collect(), + }; + for (header, _) in webc.volumes.iter() { + for directory in webc.list_directories(header) { + fs.top_level_dirs.push(directory.clone()); let _ = fs.create_dir(Path::new(&directory)); } } fs } + + pub fn top_level_dirs(&self) -> &Vec { + &self.top_level_dirs + } } /// Custom file opener, returns a WebCFile @@ -66,7 +93,6 @@ where .map_err(|_e| FsError::EntryNotFound)?; Ok(Box::new(WebCFile { - package: self.package.clone(), volume, webc: self.webc.clone(), path: path.to_path_buf(), @@ -75,8 +101,8 @@ where })) } None => { - for volume in self.webc.get_volumes_for_package(&self.package) { - let v = match self.webc.volumes.get(&volume) { + for (volume, _) in self.webc.volumes.iter() { + let v = match self.webc.volumes.get(volume) { Some(s) => s, None => continue, // error }; @@ -87,7 +113,6 @@ where }; return Ok(Box::new(WebCFile { - package: self.package.clone(), volume: volume.clone(), webc: self.webc.clone(), path: path.to_path_buf(), @@ -107,8 +132,6 @@ where T: std::fmt::Debug + Send + Sync + 'static, { pub webc: Arc, - #[allow(dead_code)] - pub package: String, pub volume: String, #[allow(dead_code)] pub path: PathBuf, @@ -281,10 +304,12 @@ where fn read_dir(&self, path: &Path) -> Result { let path = normalizes_path(path); let read_dir_result = self - .webc - .read_dir(&self.package, &path) + .volumes + .iter() + .filter_map(|v| v.read_dir(&path).ok()) + .next() .map(|o| transform_into_read_dir(Path::new(&path), o.as_ref())) - .map_err(|_| FsError::EntryNotFound); + .ok_or(FsError::EntryNotFound); match read_dir_result { Ok(o) => Ok(o), @@ -299,7 +324,7 @@ where fn remove_dir(&self, path: &Path) -> Result<(), FsError> { let path = normalizes_path(path); let result = self.memory.remove_dir(Path::new(&path)); - if self.webc.get_file_entry(&self.package, &path).is_some() { + if self.volumes.iter().any(|v| v.get_file_entry(&path).is_ok()) { Ok(()) } else { result @@ -309,7 +334,7 @@ where let from = normalizes_path(from); let to = normalizes_path(to); let result = self.memory.rename(Path::new(&from), Path::new(&to)); - if self.webc.get_file_entry(&self.package, &from).is_some() { + if self.volumes.iter().any(|v| v.get_file_entry(&from).is_ok()) { Ok(()) } else { result @@ -317,15 +342,26 @@ where } fn metadata(&self, path: &Path) -> Result { let path = normalizes_path(path); - if let Some(fs_entry) = self.webc.get_file_entry(&self.package, &path) { + if let Some(fs_entry) = self + .volumes + .iter() + .filter_map(|v| v.get_file_entry(&path).ok()) + .next() + { Ok(Metadata { ft: translate_file_type(FsEntryType::File), accessed: 0, created: 0, modified: 0, - len: fs_entry.1.get_len(), + len: fs_entry.get_len(), }) - } else if self.webc.read_dir(&self.package, &path).is_ok() { + } else if self + .volumes + .iter() + .filter_map(|v| v.read_dir(&path).ok()) + .next() + .is_some() + { Ok(Metadata { ft: translate_file_type(FsEntryType::Dir), accessed: 0, @@ -340,7 +376,13 @@ where fn remove_file(&self, path: &Path) -> Result<(), FsError> { let path = normalizes_path(path); let result = self.memory.remove_file(Path::new(&path)); - if self.webc.get_file_entry(&self.package, &path).is_some() { + if self + .volumes + .iter() + .filter_map(|v| v.get_file_entry(&path).ok()) + .next() + .is_some() + { Ok(()) } else { result @@ -351,15 +393,26 @@ where } fn symlink_metadata(&self, path: &Path) -> Result { let path = normalizes_path(path); - if let Some(fs_entry) = self.webc.get_file_entry(&self.package, &path) { + if let Some(fs_entry) = self + .volumes + .iter() + .filter_map(|v| v.get_file_entry(&path).ok()) + .next() + { Ok(Metadata { ft: translate_file_type(FsEntryType::File), accessed: 0, created: 0, modified: 0, - len: fs_entry.1.get_len(), + len: fs_entry.get_len(), }) - } else if self.webc.read_dir(&self.package, &path).is_ok() { + } else if self + .volumes + .iter() + .filter_map(|v| v.read_dir(&path).ok()) + .next() + .is_some() + { Ok(Metadata { ft: translate_file_type(FsEntryType::Dir), accessed: 0, diff --git a/lib/wasi/src/bin_factory/exec.rs b/lib/wasi/src/bin_factory/exec.rs index 047d1a7a6cf..c72a76d6e83 100644 --- a/lib/wasi/src/bin_factory/exec.rs +++ b/lib/wasi/src/bin_factory/exec.rs @@ -62,6 +62,7 @@ pub fn spawn_exec( // If the file system has not already been union'ed then do so env.state.fs.conditional_union(&binary); + tracing::debug!("{:?}", env.state.fs); // Now run the module spawn_exec_module(module, store, env, runtime) diff --git a/lib/wasi/src/bin_factory/module_cache.rs b/lib/wasi/src/bin_factory/module_cache.rs index 5e8ad83e788..7eb43100f01 100644 --- a/lib/wasi/src/bin_factory/module_cache.rs +++ b/lib/wasi/src/bin_factory/module_cache.rs @@ -1,10 +1,16 @@ -use std::{cell::RefCell, collections::HashMap, ops::DerefMut, path::PathBuf, sync::RwLock}; +use std::{ + cell::RefCell, + collections::{HashMap, HashSet}, + ops::DerefMut, + path::PathBuf, + sync::RwLock, +}; use wasmer::Module; use wasmer_wasi_types::wasi::Snapshot0Clockid; use super::BinaryPackage; -use crate::{syscalls::platform_clock_time_get, VirtualTaskManager, WasiRuntime}; +use crate::{syscalls::platform_clock_time_get, WasiRuntime}; pub const DEFAULT_COMPILED_PATH: &str = "~/.wasmer/compiled"; pub const DEFAULT_WEBC_PATH: &str = "~/.wasmer/webc"; @@ -83,12 +89,7 @@ impl ModuleCache { } // TODO: should return Result<_, anyhow::Error> - pub fn get_webc( - &self, - webc: &str, - runtime: &dyn WasiRuntime, - tasks: &dyn VirtualTaskManager, - ) -> Option { + pub fn get_webc(&self, webc: &str, runtime: &dyn WasiRuntime) -> Option { let name = webc.to_string(); let now = platform_clock_time_get(Snapshot0Clockid::Monotonic, 1_000_000).unwrap() as u128; @@ -109,8 +110,19 @@ impl ModuleCache { // Slow path let mut cache = self.cache_webc.write().unwrap(); + self.get_webc_slow(webc, runtime, cache.deref_mut()) + } + + fn get_webc_slow( + &self, + webc: &str, + runtime: &dyn WasiRuntime, + cache: &mut HashMap, + ) -> Option { + let name = webc.to_string(); + let now = platform_clock_time_get(Snapshot0Clockid::Monotonic, 1_000_000).unwrap() as u128; - // Check the cache + // Check the cache (again) if let Some(data) = cache.get(&name) { if let Some(when_cached) = data.when_cached.as_ref() { let delta = now - *when_cached; @@ -129,9 +141,27 @@ impl ModuleCache { .map(|a| a.0) .unwrap_or_else(|| name.as_str()); let cache_webc_dir = self.cache_webc_dir.as_str(); - if let Ok(data) = - crate::wapm::fetch_webc_task(cache_webc_dir, wapm_name, runtime, tasks) - { + if let Ok(mut data) = crate::wapm::fetch_webc_task(cache_webc_dir, wapm_name, runtime) { + // If the binary has no entry but it inherits from another module + // that does have an entry then we fall back to that inherited entry point + // (this convention is recursive down the list of inheritance until it finds the first entry point) + let mut already: HashSet = Default::default(); + while data.entry.is_none() { + let mut inherits = data.uses.iter().filter_map(|webc| { + if !already.contains(webc) { + already.insert(webc.clone()); + self.get_webc_slow(webc, runtime, cache) + } else { + None + } + }); + if let Some(inherits) = inherits.next() { + data.entry = inherits.entry.clone(); + } else { + break; + } + } + // If the package is the same then don't replace it // as we don't want to duplicate the memory usage if let Some(existing) = cache.get_mut(&name) { @@ -277,9 +307,7 @@ mod tests { let mut store = Vec::new(); for _ in 0..2 { - let webc = cache - .get_webc("sharrattj/dash", &rt, std::ops::Deref::deref(tasks)) - .unwrap(); + let webc = cache.get_webc("sharrattj/dash", &rt).unwrap(); store.push(webc); tasks .runtime() diff --git a/lib/wasi/src/fs/mod.rs b/lib/wasi/src/fs/mod.rs index f0de31a83dc..f04fafac096 100644 --- a/lib/wasi/src/fs/mod.rs +++ b/lib/wasi/src/fs/mod.rs @@ -318,7 +318,6 @@ impl FileSystem for WasiFsRoot { /// Warning, modifying these fields directly may cause invariants to break and /// should be considered unsafe. These fields may be made private in a future release -#[derive(Debug)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub struct WasiFs { //pub repo: Repo, @@ -1390,7 +1389,6 @@ impl WasiFs { _ => (), } let fd = self.get_fd(fd)?; - debug!("fdstat: {:?}", fd); let guard = fd.inode.read(); let deref = guard.deref(); @@ -1752,6 +1750,18 @@ impl WasiFs { } } +impl std::fmt::Debug for WasiFs { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if let Ok(guard) = self.current_dir.try_lock() { + write!(f, "current_dir={} ", guard.as_str())?; + } else { + write!(f, "current_dir=(locked) ")?; + } + write!(f, "next_fd={} ", self.next_fd.load(Ordering::Relaxed))?; + write!(f, "{:?}", self.root_fs) + } +} + /// Returns the default filesystem backing pub fn default_fs_backing() -> Box { cfg_if::cfg_if! { diff --git a/lib/wasi/src/os/command/builtins/cmd_wasmer.rs b/lib/wasi/src/os/command/builtins/cmd_wasmer.rs index eb184410d13..1b7954f2d99 100644 --- a/lib/wasi/src/os/command/builtins/cmd_wasmer.rs +++ b/lib/wasi/src/os/command/builtins/cmd_wasmer.rs @@ -10,7 +10,7 @@ use wasmer_wasi_types::wasi::Errno; use crate::{ bin_factory::{spawn_exec, BinaryPackage, ModuleCache}, syscalls::stderr_write, - VirtualTaskManager, VirtualTaskManagerExt, WasiEnv, WasiRuntime, + VirtualTaskManagerExt, WasiEnv, WasiRuntime, }; const HELP: &str = r#"USAGE: @@ -74,8 +74,7 @@ impl CmdWasmer { env.state = Arc::new(state); // Get the binary - let tasks = parent_ctx.data().tasks(); - if let Some(binary) = self.get_package(what.clone(), tasks.deref()) { + if let Some(binary) = self.get_package(what.clone()) { // Now run the module spawn_exec(binary, name, store, env, &self.runtime, &self.cache) } else { @@ -98,13 +97,8 @@ impl CmdWasmer { } } - pub fn get_package( - &self, - name: String, - tasks: &dyn VirtualTaskManager, - ) -> Option { - self.cache - .get_webc(name.as_str(), self.runtime.deref(), tasks) + pub fn get_package(&self, name: String) -> Option { + self.cache.get_webc(name.as_str(), self.runtime.deref()) } } diff --git a/lib/wasi/src/os/console/mod.rs b/lib/wasi/src/os/console/mod.rs index 9fca12aaca2..08283f4eae1 100644 --- a/lib/wasi/src/os/console/mod.rs +++ b/lib/wasi/src/os/console/mod.rs @@ -203,24 +203,22 @@ impl Console { tasks.block_on(self.draw_welcome()); } - let binary = if let Some(binary) = - self.compiled_modules - .get_webc(webc, self.runtime.deref(), tasks.deref()) - { - binary - } else { - let mut stderr = self.stderr.clone(); - tasks.block_on(async { - wasmer_vfs::AsyncWriteExt::write_all( - &mut stderr, - format!("package not found [{}]\r\n", webc).as_bytes(), - ) - .await - .ok(); - }); - tracing::debug!("failed to get webc dependency - {}", webc); - return Err(VirtualBusError::NotFound); - }; + let binary = + if let Some(binary) = self.compiled_modules.get_webc(webc, self.runtime.deref()) { + binary + } else { + let mut stderr = self.stderr.clone(); + tasks.block_on(async { + wasmer_vfs::AsyncWriteExt::write_all( + &mut stderr, + format!("package not found [{}]\r\n", webc).as_bytes(), + ) + .await + .ok(); + }); + tracing::debug!("failed to get webc dependency - {}", webc); + return Err(VirtualBusError::NotFound); + }; let wasi_process = env.process.clone(); diff --git a/lib/wasi/src/runners/wasi.rs b/lib/wasi/src/runners/wasi.rs index 1f5b828c865..07cc0c911c6 100644 --- a/lib/wasi/src/runners/wasi.rs +++ b/lib/wasi/src/runners/wasi.rs @@ -88,29 +88,12 @@ fn prepare_webc_env( command: &str, args: &[String], ) -> Result { - use webc::FsEntryType; - - let package_name = webc.get_package_name(); - let top_level_dirs = webc - .get_volumes_for_package(&package_name) - .into_iter() - .flat_map(|volume| { - webc.volumes - .get(&volume) - .unwrap() - .header - .top_level - .iter() - .filter(|e| e.fs_type == FsEntryType::Dir) - .map(|e| e.text.to_string()) - }) - .collect::>(); - - let filesystem = Box::new(WebcFileSystem::init(webc, &package_name)); - let mut builder = WasiEnv::builder(command).fs(filesystem).args(args); - for f_name in top_level_dirs.iter() { + let filesystem = Box::new(WebcFileSystem::init_all(webc)); + let mut builder = WasiEnv::builder(command).args(args); + for f_name in filesystem.top_level_dirs() { builder.add_preopen_build(|p| p.directory(f_name).read(true).write(true).create(true))?; } + builder.set_fs(filesystem); Ok(builder) } diff --git a/lib/wasi/src/state/env.rs b/lib/wasi/src/state/env.rs index ffb989d0d77..a5e3c2d9c5e 100644 --- a/lib/wasi/src/state/env.rs +++ b/lib/wasi/src/state/env.rs @@ -771,7 +771,7 @@ impl WasiEnv { while let Some(use_package) = use_packages.pop_back() { if let Some(package) = cmd_wasmer .as_ref() - .and_then(|cmd| cmd.get_package(use_package.clone(), self.tasks().deref())) + .and_then(|cmd| cmd.get_package(use_package.clone())) { // If its already been added make sure the version is correct let package_name = package.package_name.to_string(); diff --git a/lib/wasi/src/syscalls/wasi/fd_event.rs b/lib/wasi/src/syscalls/wasi/fd_event.rs index abd9ff1ff1e..2017627b794 100644 --- a/lib/wasi/src/syscalls/wasi/fd_event.rs +++ b/lib/wasi/src/syscalls/wasi/fd_event.rs @@ -32,7 +32,7 @@ pub fn fd_event( .fs .create_fd(rights, rights, Fdflags::empty(), 0, inode)); - debug!( + trace!( "wasi[{}:{}]::fd_event - event notifications created (fd={})", ctx.data().pid(), ctx.data().tid(), diff --git a/lib/wasi/src/syscalls/wasi/fd_fdstat_get.rs b/lib/wasi/src/syscalls/wasi/fd_fdstat_get.rs index cb1b9070fa2..b337868bbf7 100644 --- a/lib/wasi/src/syscalls/wasi/fd_fdstat_get.rs +++ b/lib/wasi/src/syscalls/wasi/fd_fdstat_get.rs @@ -14,7 +14,7 @@ pub fn fd_fdstat_get( fd: WasiFd, buf_ptr: WasmPtr, ) -> Errno { - debug!( + trace!( "wasi[{}:{}]::fd_fdstat_get: fd={}, buf_ptr={}", ctx.data().pid(), ctx.data().tid(), diff --git a/lib/wasi/src/syscalls/wasi/fd_prestat_dir_name.rs b/lib/wasi/src/syscalls/wasi/fd_prestat_dir_name.rs index 0e0a2a84609..70b5b350643 100644 --- a/lib/wasi/src/syscalls/wasi/fd_prestat_dir_name.rs +++ b/lib/wasi/src/syscalls/wasi/fd_prestat_dir_name.rs @@ -22,7 +22,6 @@ pub fn fd_prestat_dir_name( // check inode-val.is_preopened? - trace!("=> inode: {:?}", inode); let guard = inode.read(); match guard.deref() { Kind::Dir { .. } | Kind::Root { .. } => { diff --git a/lib/wasi/src/syscalls/wasi/path_filestat_get.rs b/lib/wasi/src/syscalls/wasi/path_filestat_get.rs index 1430530337c..e3f25eb53ce 100644 --- a/lib/wasi/src/syscalls/wasi/path_filestat_get.rs +++ b/lib/wasi/src/syscalls/wasi/path_filestat_get.rs @@ -27,7 +27,7 @@ pub fn path_filestat_get( let (memory, mut state, inodes) = env.get_memory_and_wasi_state_and_inodes(&ctx, 0); let mut path_string = unsafe { get_input_str!(&memory, path, path_len) }; - debug!( + trace!( "wasi[{}:{}]::path_filestat_get (fd={}, path={})", ctx.data().pid(), ctx.data().tid(), diff --git a/lib/wasi/src/wapm/mod.rs b/lib/wasi/src/wapm/mod.rs index 10f2e131b5b..227a70bc413 100644 --- a/lib/wasi/src/wapm/mod.rs +++ b/lib/wasi/src/wapm/mod.rs @@ -9,11 +9,11 @@ use wasmer_vfs::FileSystem; use tracing::*; #[allow(unused_imports)] use tracing::{error, warn}; -use webc::{Annotation, FsEntryType, UrlOrManifest, WebC}; +use webc::{Annotation, UrlOrManifest, WebC}; use crate::{ bin_factory::{BinaryPackage, BinaryPackageCommand}, - VirtualTaskManager, WasiRuntime, + WasiRuntime, }; #[cfg(feature = "wapm-tar")] @@ -27,7 +27,6 @@ pub(crate) fn fetch_webc_task( cache_dir: &str, webc: &str, runtime: &dyn WasiRuntime, - tasks: &dyn VirtualTaskManager, ) -> Result { let client = runtime .http_client() @@ -40,7 +39,10 @@ pub(crate) fn fetch_webc_task( async move { fetch_webc(&cache_dir, &webc, client).await } }; - let result = tasks.block_on(f).context("webc fetch task has died"); + let result = runtime + .task_manager() + .block_on(f) + .context("webc fetch task has died"); result.with_context(|| format!("could not fetch webc '{webc}'")) } @@ -60,6 +62,8 @@ async fn fetch_webc( .replace(WAPM_WEBC_VERSION_TAG, version.replace('\"', "'").as_str()), None => WAPM_WEBC_QUERY_LAST.replace(WAPM_WEBC_QUERY_TAG, name.replace('\"', "'").as_str()), }; + debug!("request: {}", url_query); + let url = format!( "{}{}", WAPM_WEBC_URL, @@ -82,6 +86,8 @@ async fn fetch_webc( let body = response.body.context("HTTP response with empty body")?; let data: WapmWebQuery = serde_json::from_slice(&body).context("Could not parse webc registry JSON data")?; + debug!("response: {:?}", data); + let PiritaVersionedDownload { url: download_url, version, @@ -354,27 +360,10 @@ where pck.version = version.clone().into(); } - // Add all the file system files - let top_level_dirs = webc - .get_volumes_for_package(&package_name) - .into_iter() - .flat_map(|volume| { - webc.volumes - .get(&volume) - .unwrap() - .header - .top_level - .iter() - .filter(|e| e.fs_type == FsEntryType::Dir) - .map(|e| e.text.to_string()) - }) - .collect::>(); - // Add the file system from the webc - pck.webc_fs = Some(Arc::new(wasmer_vfs::webc_fs::WebcFileSystem::init( - ownership.clone(), - &package_name, - ))); + let webc_fs = wasmer_vfs::webc_fs::WebcFileSystem::init_all(ownership.clone()); + let top_level_dirs = webc_fs.top_level_dirs().clone(); + pck.webc_fs = Some(Arc::new(webc_fs)); pck.webc_top_level_dirs = top_level_dirs; // Add the memory footprint of the file system