diff --git a/Cargo.lock b/Cargo.lock index f0139620867..2e25d324452 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1252,6 +1252,20 @@ dependencies = [ "parking_lot_core 0.9.10", ] +[[package]] +name = "dashmap" +version = "6.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "804c8821570c3f8b70230c2ba75ffa5c0f9a4189b9a432b6656c536712acae28" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils 0.8.20", + "hashbrown 0.14.5", + "lock_api 0.4.12", + "once_cell", + "parking_lot_core 0.9.10", +] + [[package]] name = "data-encoding" version = "2.6.0" @@ -4706,7 +4720,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e56dd856803e253c8f298af3f4d7eb0ae5e23a737252cd90bb4f3b435033b2d" dependencies = [ - "dashmap", + "dashmap 5.5.3", "futures 0.3.30", "lazy_static", "log 0.4.22", @@ -6013,6 +6027,7 @@ dependencies = [ "anyhow", "async-trait", "bytes 1.6.1", + "dashmap 6.0.1", "derivative", "dunce", "filetime", @@ -7080,7 +7095,7 @@ dependencies = [ "cfg-if 1.0.0", "corosensei", "crossbeam-queue 0.3.11", - "dashmap", + "dashmap 6.0.1", "derivative", "enum-iterator", "fnv", @@ -7114,7 +7129,7 @@ dependencies = [ "cfg-if 1.0.0", "chrono", "cooked-waker", - "dashmap", + "dashmap 6.0.1", "derivative", "futures 0.3.30", "getrandom", diff --git a/Cargo.toml b/Cargo.toml index 324c5008164..023e1038115 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -97,6 +97,7 @@ edge-schema = { version = "=0.1.0" } shared-buffer = "0.1.4" # Third-party crates +dashmap = "6.0.1" http = "1.0.0" hyper = "1" reqwest = { version = "0.12.0", default-features = false } diff --git a/lib/backend-api/src/client.rs b/lib/backend-api/src/client.rs index 092a09fb4d6..4f87af5b02f 100644 --- a/lib/backend-api/src/client.rs +++ b/lib/backend-api/src/client.rs @@ -52,6 +52,28 @@ impl WasmerClient { }) } + pub fn new_with_proxy( + graphql_endpoint: Url, + user_agent: &str, + proxy: reqwest::Proxy, + ) -> Result { + let builder = { + #[cfg(all(target_arch = "wasm32", target_os = "unknown"))] + let mut builder = reqwest::ClientBuilder::new(); + + #[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] + let builder = reqwest::ClientBuilder::new() + .connect_timeout(Duration::from_secs(10)) + .timeout(Duration::from_secs(90)); + + builder.proxy(proxy) + }; + + let client = builder.build().context("failed to create reqwest client")?; + + Self::new_with_client(client, graphql_endpoint, user_agent) + } + pub fn new(graphql_endpoint: Url, user_agent: &str) -> Result { #[cfg(all(target_arch = "wasm32", target_os = "unknown"))] let client = reqwest::Client::builder() diff --git a/lib/cli/src/commands/package/download.rs b/lib/cli/src/commands/package/download.rs index c4a79e24611..bc168fae95e 100644 --- a/lib/cli/src/commands/package/download.rs +++ b/lib/cli/src/commands/package/download.rs @@ -28,6 +28,10 @@ pub struct PackageDownload { #[clap(long)] pub quiet: bool, + /// proxy to use for downloading + #[clap(long)] + pub proxy: Option, + /// The package to download. package: PackageSource, } @@ -97,7 +101,13 @@ impl PackageDownload { // caveat: client_unauthennticated will use a token if provided, it // just won't fail if none is present. So, _unauthenticated() can actually // produce an authenticated client. - let client = self.env.client_unauthennticated()?; + let client = if let Some(proxy) = &self.proxy { + let proxy = reqwest::Proxy::all(proxy)?; + + self.env.client_unauthennticated_with_proxy(proxy)? + } else { + self.env.client_unauthennticated()? + }; let version = id.version_or_default().to_string(); let version = if version == "*" { @@ -299,6 +309,7 @@ mod tests { out_path: Some(out_path.clone()), package: "wasmer/hello@0.1.0".parse().unwrap(), quiet: true, + proxy: None, }; cmd.execute().unwrap(); diff --git a/lib/cli/src/config/env.rs b/lib/cli/src/config/env.rs index 0a3fbf9d1fc..496962c14f7 100644 --- a/lib/cli/src/config/env.rs +++ b/lib/cli/src/config/env.rs @@ -138,6 +138,26 @@ impl WasmerEnv { Ok(client) } + pub fn client_unauthennticated_with_proxy( + &self, + proxy: reqwest::Proxy, + ) -> Result { + let registry_url = self.registry_endpoint()?; + let client = wasmer_api::WasmerClient::new_with_proxy( + registry_url, + &DEFAULT_WASMER_CLI_USER_AGENT, + proxy, + )?; + + let client = if let Some(token) = self.token() { + client.with_auth_token(token) + } else { + client + }; + + Ok(client) + } + pub fn client(&self) -> Result { let client = self.client_unauthennticated()?; if client.auth_token().is_none() { diff --git a/lib/virtual-fs/Cargo.toml b/lib/virtual-fs/Cargo.toml index f6db418f880..20923cf9515 100644 --- a/lib/virtual-fs/Cargo.toml +++ b/lib/virtual-fs/Cargo.toml @@ -10,6 +10,7 @@ repository.workspace = true rust-version.workspace = true [dependencies] +dashmap.workspace = true dunce = "1.0.4" anyhow = { version = "1.0.66", optional = true } async-trait = { version = "^0.1" } diff --git a/lib/virtual-fs/src/arc_fs.rs b/lib/virtual-fs/src/arc_fs.rs index 605f814ddc8..4646226cfe2 100644 --- a/lib/virtual-fs/src/arc_fs.rs +++ b/lib/virtual-fs/src/arc_fs.rs @@ -53,4 +53,13 @@ impl FileSystem for ArcFileSystem { fn new_open_options(&self) -> OpenOptions { self.fs.new_open_options() } + + fn mount( + &self, + name: String, + path: &Path, + fs: Box, + ) -> Result<()> { + self.fs.mount(name, path, fs) + } } diff --git a/lib/virtual-fs/src/empty_fs.rs b/lib/virtual-fs/src/empty_fs.rs index 6745c337453..5c6fd956803 100644 --- a/lib/virtual-fs/src/empty_fs.rs +++ b/lib/virtual-fs/src/empty_fs.rs @@ -67,6 +67,15 @@ impl FileSystem for EmptyFileSystem { fn new_open_options(&self) -> OpenOptions { OpenOptions::new(self) } + + fn mount( + &self, + name: String, + path: &Path, + fs: Box, + ) -> Result<()> { + Err(FsError::Unsupported) + } } impl FileOpener for EmptyFileSystem { diff --git a/lib/virtual-fs/src/host_fs.rs b/lib/virtual-fs/src/host_fs.rs index cde3eba76cd..199ee9fb502 100644 --- a/lib/virtual-fs/src/host_fs.rs +++ b/lib/virtual-fs/src/host_fs.rs @@ -235,6 +235,15 @@ impl crate::FileSystem for FileSystem { .and_then(TryInto::try_into) .map_err(Into::into) } + + fn mount( + &self, + _name: String, + _path: &Path, + _fs: Box, + ) -> Result<()> { + Err(FsError::Unsupported) + } } impl TryInto for std::fs::Metadata { diff --git a/lib/virtual-fs/src/lib.rs b/lib/virtual-fs/src/lib.rs index 084fcda5a71..6e9b4438530 100644 --- a/lib/virtual-fs/src/lib.rs +++ b/lib/virtual-fs/src/lib.rs @@ -98,6 +98,9 @@ pub trait FileSystem: fmt::Debug + Send + Sync + 'static + Upcastable { fn remove_file(&self, path: &Path) -> Result<()>; fn new_open_options(&self) -> OpenOptions; + + fn mount(&self, name: String, path: &Path, fs: Box) + -> Result<()>; } impl dyn FileSystem + 'static { @@ -152,6 +155,15 @@ where fn new_open_options(&self) -> OpenOptions { (**self).new_open_options() } + + fn mount( + &self, + name: String, + path: &Path, + fs: Box, + ) -> Result<()> { + (**self).mount(name, path, fs) + } } pub trait FileOpener { @@ -510,6 +522,9 @@ pub enum FsError { /// Some other unhandled error. If you see this, it's probably a bug. #[error("unknown error found")] UnknownError, + /// Operation is not supported on this filesystem + #[error("unsupported")] + Unsupported, } impl From for FsError { @@ -570,6 +585,7 @@ impl From for io::Error { FsError::DirectoryNotEmpty => io::ErrorKind::Other, FsError::UnknownError => io::ErrorKind::Other, FsError::StorageFull => io::ErrorKind::Other, + FsError::Unsupported => io::ErrorKind::Unsupported, // NOTE: Add this once the "io_error_more" Rust feature is stabilized // FsError::StorageFull => io::ErrorKind::StorageFull, }; @@ -580,7 +596,7 @@ impl From for io::Error { #[derive(Debug)] pub struct ReadDir { // TODO: to do this properly we need some kind of callback to the core FS abstraction - data: Vec, + pub(crate) data: Vec, index: usize, } diff --git a/lib/virtual-fs/src/mem_fs/filesystem.rs b/lib/virtual-fs/src/mem_fs/filesystem.rs index 0358125e8d5..65b34ce542f 100644 --- a/lib/virtual-fs/src/mem_fs/filesystem.rs +++ b/lib/virtual-fs/src/mem_fs/filesystem.rs @@ -3,7 +3,7 @@ use self::offloaded_file::OffloadBackingStore; use super::*; -use crate::{DirEntry, FileSystem as _, FileType, FsError, Metadata, OpenOptions, ReadDir, Result}; +use crate::{DirEntry, FileType, FsError, Metadata, OpenOptions, ReadDir, Result}; use futures::future::{BoxFuture, Either}; use slab::Slab; use std::collections::VecDeque; @@ -680,6 +680,16 @@ impl crate::FileSystem for FileSystem { fn new_open_options(&self) -> OpenOptions { OpenOptions::new(self) } + + fn mount( + &self, + _name: String, + path: &Path, + fs: Box, + ) -> Result<()> { + let fs: Arc = Arc::new(fs); + self.mount(path.to_owned(), &fs, PathBuf::from("/")) + } } impl fmt::Debug for FileSystem { diff --git a/lib/virtual-fs/src/overlay_fs.rs b/lib/virtual-fs/src/overlay_fs.rs index a29511b80c5..90c1803ece6 100644 --- a/lib/virtual-fs/src/overlay_fs.rs +++ b/lib/virtual-fs/src/overlay_fs.rs @@ -414,6 +414,15 @@ where fn new_open_options(&self) -> OpenOptions<'_> { OpenOptions::new(self) } + + fn mount( + &self, + _name: String, + _path: &Path, + _fs: Box, + ) -> Result<(), FsError> { + Err(FsError::Unsupported) + } } impl FileOpener for OverlayFileSystem diff --git a/lib/virtual-fs/src/passthru_fs.rs b/lib/virtual-fs/src/passthru_fs.rs index 0d138a90075..0164d6c9e86 100644 --- a/lib/virtual-fs/src/passthru_fs.rs +++ b/lib/virtual-fs/src/passthru_fs.rs @@ -53,6 +53,15 @@ impl FileSystem for PassthruFileSystem { fn new_open_options(&self) -> OpenOptions { self.fs.new_open_options() } + + fn mount( + &self, + _name: String, + _path: &Path, + _fs: Box, + ) -> Result<()> { + Err(FsError::Unsupported) + } } #[cfg(test)] diff --git a/lib/virtual-fs/src/static_fs.rs b/lib/virtual-fs/src/static_fs.rs index 9a45702a4a6..10e303edc0c 100644 --- a/lib/virtual-fs/src/static_fs.rs +++ b/lib/virtual-fs/src/static_fs.rs @@ -382,6 +382,15 @@ impl FileSystem for StaticFileSystem { self.memory.symlink_metadata(Path::new(&path)) } } + + fn mount( + &self, + _name: String, + _path: &Path, + _fs: Box, + ) -> Result<(), FsError> { + Err(FsError::Unsupported) + } } fn normalizes_path(path: &Path) -> String { diff --git a/lib/virtual-fs/src/tmp_fs.rs b/lib/virtual-fs/src/tmp_fs.rs index 0f0667f402c..f16036bf29a 100644 --- a/lib/virtual-fs/src/tmp_fs.rs +++ b/lib/virtual-fs/src/tmp_fs.rs @@ -96,4 +96,13 @@ impl FileSystem for TmpFileSystem { fn new_open_options(&self) -> OpenOptions { self.fs.new_open_options() } + + fn mount( + &self, + name: String, + path: &Path, + fs: Box, + ) -> Result<()> { + FileSystem::mount(&self.fs, name, path, fs) + } } diff --git a/lib/virtual-fs/src/trace_fs.rs b/lib/virtual-fs/src/trace_fs.rs index 44aa5bbbbc5..ca132bd233c 100644 --- a/lib/virtual-fs/src/trace_fs.rs +++ b/lib/virtual-fs/src/trace_fs.rs @@ -1,5 +1,5 @@ use std::{ - path::PathBuf, + path::{Path, PathBuf}, pin::Pin, task::{Context, Poll}, }; @@ -87,6 +87,16 @@ where fn new_open_options(&self) -> crate::OpenOptions { crate::OpenOptions::new(self) } + + #[tracing::instrument(level = "trace", skip(self))] + fn mount( + &self, + name: String, + path: &Path, + fs: Box, + ) -> crate::Result<()> { + self.0.mount(name, path, fs) + } } impl FileOpener for TraceFileSystem diff --git a/lib/virtual-fs/src/union_fs.rs b/lib/virtual-fs/src/union_fs.rs index 4e6eff9f044..88d56ad061c 100644 --- a/lib/virtual-fs/src/union_fs.rs +++ b/lib/virtual-fs/src/union_fs.rs @@ -2,92 +2,38 @@ //! its not as simple as TmpFs. not currently used but was used by //! the previoulsy implementation of Deploy - now using TmpFs -use crate::*; - -use std::{ - path::Path, - sync::{Arc, Mutex, Weak}, -}; +use dashmap::DashMap; -use tracing::{debug, trace}; +use crate::*; -pub type TempHolding = Arc>>>>; +use std::{path::Path, sync::Arc}; #[derive(Debug)] pub struct MountPoint { - pub path: String, + pub path: PathBuf, pub name: String, - pub fs: Option>>, - pub weak_fs: Weak>, - pub temp_holding: TempHolding, - pub should_sanitize: bool, - pub new_path: Option, -} - -impl Clone for MountPoint { - fn clone(&self) -> Self { - Self { - path: self.path.clone(), - name: self.name.clone(), - fs: None, - weak_fs: self.weak_fs.clone(), - temp_holding: self.temp_holding.clone(), - should_sanitize: self.should_sanitize, - new_path: self.new_path.clone(), - } - } + pub fs: Arc>, } impl MountPoint { - pub fn fs(&self) -> Option>> { - match &self.fs { - Some(a) => Some(a.clone()), - None => self.weak_fs.upgrade(), - } + pub fn fs(&self) -> &(dyn FileSystem + Send + Sync) { + self.fs.as_ref() } - /// Tries to recover the internal `Weak` to a `Arc` - fn solidify(&mut self) { - if self.fs.is_none() { - self.fs = self.weak_fs.upgrade(); - } - { - let mut guard = self.temp_holding.lock().unwrap(); - let fs = guard.take(); - if self.fs.is_none() { - self.fs = fs; - } - } - } - - /// Returns a strong-referenced copy of the internal `Arc` - fn strong(&self) -> Option { - self.fs().map(|fs| StrongMountPoint { + pub fn mount_point_ref(&self) -> MountPointRef<'_> { + MountPointRef { path: self.path.clone(), name: self.name.clone(), - fs, - should_sanitize: self.should_sanitize, - new_path: self.new_path.clone(), - }) + fs: &self.fs, + } } } -/// A `strong` mount point holds a strong `Arc` reference to the filesystem -/// mounted at path `path`. -#[derive(Debug)] -pub struct StrongMountPoint { - pub path: String, - pub name: String, - pub fs: Arc>, - pub should_sanitize: bool, - pub new_path: Option, -} - /// Allows different filesystems of different types /// to be mounted at various mount points -#[derive(Debug, Default, Clone)] +#[derive(Debug, Default)] pub struct UnionFileSystem { - pub mounts: Vec, + pub mounts: DashMap, } impl UnionFileSystem { @@ -98,350 +44,239 @@ impl UnionFileSystem { pub fn clear(&mut self) { self.mounts.clear(); } -} -impl UnionFileSystem { - pub fn mount( - &mut self, - name: &str, - path: &str, - should_sanitize: bool, - fs: Box, - new_path: Option<&str>, - ) { - self.unmount(path); - let mut path = path.to_string(); - if !path.starts_with('/') { - path.insert(0, '/'); - } - if !path.ends_with('/') { - path += "/"; - } - let new_path = new_path.map(|new_path| { - let mut new_path = new_path.to_string(); - if !new_path.ends_with('/') { - new_path += "/"; - } - new_path - }); - let fs = Arc::new(fs); - - let mount = MountPoint { - path, - name: name.to_string(), - fs: None, - weak_fs: Arc::downgrade(&fs), - temp_holding: Arc::new(Mutex::new(Some(fs.clone()))), - should_sanitize, - new_path, - }; - - self.mounts.push(mount); + fn is_root(&self) -> bool { + self.mounts.len() == 1 && self.mounts.contains_key(&PathBuf::from("/")) } - pub fn unmount(&mut self, path: &str) { - let path1 = path.to_string(); - let mut path2 = path1; - if !path2.starts_with('/') { - path2.insert(0, '/'); + fn prepare_path(&self, path: &Path) -> PathBuf { + if self.is_root() { + path.to_owned() + } else { + path.strip_prefix(PathBuf::from("/")) + .unwrap_or(path) + .to_owned() } - let mut path3 = path2.clone(); - if !path3.ends_with('/') { - path3.push('/') - } - if path2.ends_with('/') { - path2 = (path2[..(path2.len() - 1)]).to_string(); - } - - self.mounts - .retain(|mount| mount.path != path2 && mount.path != path3); } +} - fn read_dir_internal(&self, path: &Path) -> Result { - let path = path.to_string_lossy(); - - let mut ret = None; - for (path, mount) in filter_mounts(&self.mounts, path.as_ref()) { - match mount.fs.read_dir(Path::new(path.as_str())) { - Ok(dir) => { - if ret.is_none() { - ret = Some(Vec::new()); - } - let ret = ret.as_mut().unwrap(); - for sub in dir.flatten() { - ret.push(sub); - } - } - Err(err) => { - debug!("failed to read dir - {}", err); - } - } - } +impl UnionFileSystem { + #[allow(clippy::type_complexity)] + fn find_mount( + &self, + path: PathBuf, + ) -> Option<(PathBuf, PathBuf, Arc>)> { + let mut components = path.components().collect::>(); - match ret { - Some(mut ret) => { - ret.sort_by(|a, b| match (a.metadata.as_ref(), b.metadata.as_ref()) { - (Ok(a), Ok(b)) => a.modified.cmp(&b.modified), - _ => std::cmp::Ordering::Equal, - }); - Ok(ReadDir::new(ret)) - } - None => Err(FsError::EntryNotFound), - } - } + if let Some(c) = components.first().copied() { + components.remove(0); - /// Deletes all mount points that do not have `sanitize` set in the options - pub fn sanitize(mut self) -> Self { - self.solidify(); - self.mounts.retain(|mount| !mount.should_sanitize); - self - } + let sub_path = components.into_iter().collect::(); - /// Tries to recover the internal `Weak` to a `Arc` - pub fn solidify(&mut self) { - for mount in self.mounts.iter_mut() { - mount.solidify(); + if let Some(mount) = self.mounts.get(&PathBuf::from(c.as_os_str())) { + return Some(( + PathBuf::from(c.as_os_str()), + PathBuf::from("/").join(sub_path), + mount.fs.clone(), + )); + } } + + None } } impl FileSystem for UnionFileSystem { fn readlink(&self, path: &Path) -> Result { - debug!("readlink: path={}", path.display()); - let mut ret_error = FsError::EntryNotFound; - let path = path.to_string_lossy(); - for (path_inner, mount) in filter_mounts(&self.mounts, path.as_ref()) { - match mount.fs.readlink(Path::new(path_inner.as_str())) { - Ok(ret) => { - return Ok(ret); - } - Err(err) => { - // This fixes a bug when attempting to create the directory /usr when it does not exist - // on the x86 version of memfs - // TODO: patch virtual-fs and remove - if let FsError::NotAFile = &err { - ret_error = FsError::EntryNotFound; - } else { - debug!("readlink failed: (path={}) - {}", path, err); - ret_error = err; - } - } - } + let path = self.prepare_path(path); + + if path.as_os_str().is_empty() { + Err(FsError::NotAFile) + } else if let Some((_, path, fs)) = self.find_mount(path.to_owned()) { + fs.readlink(&path) + } else { + Err(FsError::EntryNotFound) } - debug!("readlink: failed={}", ret_error); - Err(ret_error) } fn read_dir(&self, path: &Path) -> Result { - debug!("read_dir: path={}", path.display()); - self.read_dir_internal(path) + let path = self.prepare_path(path); + + if path.as_os_str().is_empty() { + let entries = self + .mounts + .iter() + .map(|i| DirEntry { + path: PathBuf::from("/").join(i.key()), + metadata: Ok(Metadata { + ft: FileType::new_dir(), + accessed: 0, + created: 0, + modified: 0, + len: 0, + }), + }) + .collect::>(); + + Ok(ReadDir::new(entries)) + } else if let Some((prefix, path, fs)) = self.find_mount(path.to_owned()) { + let mut entries = fs.read_dir(&path)?; + + for entry in &mut entries.data { + let path: PathBuf = entry.path.components().skip(1).collect(); + entry.path = PathBuf::from("/").join(PathBuf::from(&prefix).join(path)); + } + + Ok(entries) + } else { + Err(FsError::EntryNotFound) + } } + fn create_dir(&self, path: &Path) -> Result<()> { - debug!("create_dir: path={}", path.display()); - if path.parent().is_none() { - return Err(FsError::BaseNotDirectory); - } - if self.read_dir_internal(path).is_ok() { - //return Err(FsError::AlreadyExists); - return Ok(()); - } + let path = self.prepare_path(path); - let path = path.to_string_lossy(); - let mut ret_error = FsError::EntryNotFound; - for (path, mount) in filter_mounts(&self.mounts, path.as_ref()) { - match mount.fs.create_dir(Path::new(path.as_str())) { - Ok(ret) => { - return Ok(ret); - } - Err(err) => { - ret_error = err; + if path.as_os_str().is_empty() { + Ok(()) + } else if let Some((_, path, fs)) = self.find_mount(path.to_owned()) { + let result = fs.create_dir(&path); + + if let Err(e) = result { + if e == FsError::AlreadyExists { + return Ok(()); } } + + result + } else { + Err(FsError::EntryNotFound) } - Err(ret_error) } fn remove_dir(&self, path: &Path) -> Result<()> { - debug!("remove_dir: path={}", path.display()); - if path.parent().is_none() { - return Err(FsError::BaseNotDirectory); + let path = self.prepare_path(path); + + if path.as_os_str().is_empty() { + Err(FsError::PermissionDenied) + } else if let Some((_, path, fs)) = self.find_mount(path.to_owned()) { + fs.remove_dir(&path) + } else { + Err(FsError::EntryNotFound) } - // https://github.com/rust-lang/rust/issues/86442 - // DirectoryNotEmpty is not implemented consistently - if path.is_dir() && self.read_dir(path).map(|s| !s.is_empty()).unwrap_or(false) { - return Err(FsError::DirectoryNotEmpty); - } - let mut ret_error = FsError::EntryNotFound; - let path = path.to_string_lossy(); - for (path, mount) in filter_mounts(&self.mounts, path.as_ref()) { - match mount.fs.remove_dir(Path::new(path.as_str())) { - Ok(ret) => { - return Ok(ret); - } - Err(err) => { - ret_error = err; - } - } - } - Err(ret_error) } fn rename<'a>(&'a self, from: &'a Path, to: &'a Path) -> BoxFuture<'a, Result<()>> { - Box::pin(async { - println!("rename: from={} to={}", from.display(), to.display()); - if from.parent().is_none() { - return Err(FsError::BaseNotDirectory); - } - if to.parent().is_none() { - return Err(FsError::BaseNotDirectory); - } - let mut ret_error = FsError::EntryNotFound; - let from = from.to_string_lossy(); - let to = to.to_string_lossy(); - #[cfg(target_os = "windows")] - let to = to.replace('\\', "/"); - for (path, mount) in filter_mounts(&self.mounts, from.as_ref()) { - let mut to = if to.starts_with(mount.path.as_str()) { - (to[mount.path.len()..]).to_string() - } else { - ret_error = FsError::UnknownError; - continue; - }; - if !to.starts_with('/') { - to = format!("/{}", to); - } - match mount - .fs - .rename(Path::new(&path), Path::new(to.as_str())) - .await - { - Ok(ret) => { - trace!("rename ok"); - return Ok(ret); - } - Err(err) => { - trace!("rename error (from={}, to={}) - {}", from, to, err); - ret_error = err; - } - } + Box::pin(async move { + let from = self.prepare_path(from); + let to = self.prepare_path(to); + + if from.as_os_str().is_empty() { + Err(FsError::PermissionDenied) + } else if let Some((prefix, path, fs)) = self.find_mount(from.to_owned()) { + let to = to.strip_prefix(prefix).map_err(|_| FsError::InvalidInput)?; + + let to = PathBuf::from("/").join(to); + + fs.rename(&path, &to).await + } else { + Err(FsError::EntryNotFound) } - trace!("rename failed - {}", ret_error); - Err(ret_error) }) } fn metadata(&self, path: &Path) -> Result { - debug!("metadata: path={}", path.display()); - let mut ret_error = FsError::EntryNotFound; - let path = path.to_string_lossy(); - for (path, mount) in filter_mounts(&self.mounts, path.as_ref()) { - match mount.fs.metadata(Path::new(path.as_str())) { - Ok(ret) => { - return Ok(ret); - } - Err(err) => { - // This fixes a bug when attempting to create the directory /usr when it does not exist - // on the x86 version of memfs - // TODO: patch virtual-fs and remove - if let FsError::NotAFile = &err { - ret_error = FsError::EntryNotFound; - } else { - debug!("metadata failed: (path={}) - {}", path, err); - ret_error = err; - } - } - } + let path = self.prepare_path(path); + + if path.as_os_str().is_empty() { + Ok(Metadata { + ft: FileType::new_dir(), + accessed: 0, + created: 0, + modified: 0, + len: 0, + }) + } else if let Some((_, path, fs)) = self.find_mount(path.to_owned()) { + fs.metadata(&path) + } else { + Err(FsError::EntryNotFound) } - Err(ret_error) } fn symlink_metadata(&self, path: &Path) -> Result { - debug!("symlink_metadata: path={}", path.display()); - let mut ret_error = FsError::EntryNotFound; - let path = path.to_string_lossy(); - for (path_inner, mount) in filter_mounts(&self.mounts, path.as_ref()) { - match mount.fs.symlink_metadata(Path::new(path_inner.as_str())) { - Ok(ret) => { - return Ok(ret); - } - Err(err) => { - // This fixes a bug when attempting to create the directory /usr when it does not exist - // on the x86 version of memfs - // TODO: patch virtual-fs and remove - if let FsError::NotAFile = &err { - ret_error = FsError::EntryNotFound; - } else { - debug!("metadata failed: (path={}) - {}", path, err); - ret_error = err; - } - } - } + let path = self.prepare_path(path); + + if path.as_os_str().is_empty() { + Ok(Metadata { + ft: FileType::new_dir(), + accessed: 0, + created: 0, + modified: 0, + len: 0, + }) + } else if let Some((_, path, fs)) = self.find_mount(path.to_owned()) { + fs.symlink_metadata(&path) + } else { + Err(FsError::EntryNotFound) } - debug!("symlink_metadata: failed={}", ret_error); - Err(ret_error) } fn remove_file(&self, path: &Path) -> Result<()> { - println!("remove_file: path={}", path.display()); - let mut ret_error = FsError::EntryNotFound; - let path = path.to_string_lossy(); - for (path, mount) in filter_mounts(&self.mounts, path.as_ref()) { - match mount.fs.remove_file(Path::new(path.as_str())) { - Ok(ret) => { - return Ok(ret); - } - Err(err) => { - println!("returning error {err:?}"); - ret_error = err; - } - } + let path = self.prepare_path(path); + + if path.as_os_str().is_empty() { + Err(FsError::NotAFile) + } else if let Some((_, path, fs)) = self.find_mount(path.to_owned()) { + fs.remove_file(&path) + } else { + Err(FsError::EntryNotFound) } - Err(ret_error) } fn new_open_options(&self) -> OpenOptions { OpenOptions::new(self) } -} -fn filter_mounts( - mounts: &[MountPoint], - target: &str, -) -> impl Iterator { - // On Windows, Path might use \ instead of /, wich mill messup the matching of mount points - #[cfg(target_os = "windows")] - let target = target.replace('\\', "/"); - - let mut biggest_path = 0usize; - let mut ret = Vec::new(); - for mount in mounts.iter().rev() { - let mut test_mount_path1 = mount.path.clone(); - if !test_mount_path1.ends_with('/') { - test_mount_path1.push('/'); - } + fn mount( + &self, + name: String, + path: &Path, + fs: Box, + ) -> Result<()> { + let mut components = path.components().collect::>(); + if let Some(c) = components.first().copied() { + components.remove(0); - let mut test_mount_path2 = mount.path.clone(); - if test_mount_path2.ends_with('/') { - test_mount_path2 = test_mount_path2[..(test_mount_path2.len() - 1)].to_string(); - } + let sub_path = components.into_iter().collect::(); - if target == test_mount_path1 || target == test_mount_path2 { - if let Some(mount) = mount.strong() { - biggest_path = biggest_path.max(mount.path.len()); - let mut path = "/".to_string(); - if let Some(ref np) = mount.new_path { - path = np.to_string(); - } - ret.push((path, mount)); - } - } else if target.starts_with(test_mount_path1.as_str()) { - if let Some(mount) = mount.strong() { - biggest_path = biggest_path.max(mount.path.len()); - let path = &target[test_mount_path2.len()..]; - let mut path = path.to_string(); - if let Some(ref np) = mount.new_path { - path = format!("{}{}", np, &path[1..]); - } - ret.push((path, mount)); + if let Some(mount) = self.mounts.get(&PathBuf::from(c.as_os_str())) { + return mount.fs.mount(name, sub_path.as_path(), fs); } + + let fs = if sub_path.components().next().is_none() { + fs + } else { + let union = UnionFileSystem::new(); + union.mount(name.clone(), sub_path.as_path(), fs)?; + + Box::new(union) + }; + + let fs = Arc::new(fs); + + let mount = MountPoint { + path: PathBuf::from(c.as_os_str()), + name, + fs, + }; + + self.mounts.insert(PathBuf::from(c.as_os_str()), mount); + } else { + return Err(FsError::EntryNotFound); } + + Ok(()) } - ret.retain(|(_, b)| b.path.len() >= biggest_path); - ret.into_iter() +} + +#[derive(Debug)] +pub struct MountPointRef<'a> { + pub path: PathBuf, + pub name: String, + pub fs: &'a dyn FileSystem, } impl FileOpener for UnionFileSystem { @@ -450,53 +285,39 @@ impl FileOpener for UnionFileSystem { path: &Path, conf: &OpenOptionsConfig, ) -> Result> { - debug!("open: path={}", path.display()); - let mut ret_err = FsError::EntryNotFound; - let path = path.to_string_lossy(); - - if conf.create() || conf.create_new() { - for (path, mount) in filter_mounts(&self.mounts, path.as_ref()) { - if let Ok(mut ret) = mount - .fs - .new_open_options() - .truncate(conf.truncate()) - .append(conf.append()) - .read(conf.read()) - .write(conf.write()) - .open(path) - { - if conf.create_new() { - ret.unlink().ok(); - continue; - } - return Ok(ret); - } - } - } - for (path, mount) in filter_mounts(&self.mounts, path.as_ref()) { - match mount.fs.new_open_options().options(conf.clone()).open(path) { - Ok(ret) => return Ok(ret), - Err(err) if ret_err == FsError::EntryNotFound => { - ret_err = err; - } - _ => {} + let path = self.prepare_path(path); + + if path.as_os_str().is_empty() { + Err(FsError::NotAFile) + } else { + let parent = path.parent().unwrap(); + let file_name = path.file_name().unwrap(); + if let Some((_, path, fs)) = self.find_mount(parent.to_owned()) { + fs.new_open_options() + .options(conf.clone()) + .open(path.join(file_name)) + } else { + Err(FsError::EntryNotFound) } } - Err(ret_err) } } #[cfg(test)] mod tests { - use std::path::Path; + use std::{ + collections::HashSet, + path::{Path, PathBuf}, + }; use tokio::io::AsyncWriteExt; - use crate::{mem_fs, ops, FileSystem as FileSystemTrait, FsError, UnionFileSystem}; + use crate::{mem_fs, FileSystem as FileSystemTrait, FsError, UnionFileSystem}; + + use super::{FileOpener, OpenOptionsConfig}; fn gen_filesystem() -> UnionFileSystem { - let mut union = UnionFileSystem::new(); - // fs.mount("/", Box::new(mem_fs::FileSystem::default())); + let union = UnionFileSystem::new(); let a = mem_fs::FileSystem::default(); let b = mem_fs::FileSystem::default(); let c = mem_fs::FileSystem::default(); @@ -506,18 +327,177 @@ mod tests { let g = mem_fs::FileSystem::default(); let h = mem_fs::FileSystem::default(); - union.mount("mem_fs_1", "/test_new_filesystem", false, Box::new(a), None); - union.mount("mem_fs_2", "/test_create_dir", false, Box::new(b), None); - union.mount("mem_fs_3", "/test_remove_dir", false, Box::new(c), None); - union.mount("mem_fs_4", "/test_rename", false, Box::new(d), None); - union.mount("mem_fs_5", "/test_metadata", false, Box::new(e), None); - union.mount("mem_fs_6", "/test_remove_file", false, Box::new(f), None); - union.mount("mem_fs_6", "/test_readdir", false, Box::new(g), None); - union.mount("mem_fs_6", "/test_canonicalize", false, Box::new(h), None); + union + .mount( + "mem_fs_1".to_string(), + PathBuf::from("/test_new_filesystem").as_path(), + Box::new(a), + ) + .unwrap(); + union + .mount( + "mem_fs_2".to_string(), + PathBuf::from("/test_create_dir").as_path(), + Box::new(b), + ) + .unwrap(); + union + .mount( + "mem_fs_3".to_string(), + PathBuf::from("/test_remove_dir").as_path(), + Box::new(c), + ) + .unwrap(); + union + .mount( + "mem_fs_4".to_string(), + PathBuf::from("/test_rename").as_path(), + Box::new(d), + ) + .unwrap(); + union + .mount( + "mem_fs_5".to_string(), + PathBuf::from("/test_metadata").as_path(), + Box::new(e), + ) + .unwrap(); + union + .mount( + "mem_fs_6".to_string(), + PathBuf::from("/test_remove_file").as_path(), + Box::new(f), + ) + .unwrap(); + union + .mount( + "mem_fs_6".to_string(), + PathBuf::from("/test_readdir").as_path(), + Box::new(g), + ) + .unwrap(); + union + .mount( + "mem_fs_6".to_string(), + PathBuf::from("/test_canonicalize").as_path(), + Box::new(h), + ) + .unwrap(); + + union + } + + fn gen_nested_filesystem() -> UnionFileSystem { + let union = UnionFileSystem::new(); + let a = mem_fs::FileSystem::default(); + a.open( + &PathBuf::from("/data-a.txt"), + &OpenOptionsConfig { + read: true, + write: true, + create_new: false, + create: true, + append: false, + truncate: false, + }, + ) + .unwrap(); + let b = mem_fs::FileSystem::default(); + b.open( + &PathBuf::from("/data-b.txt"), + &OpenOptionsConfig { + read: true, + write: true, + create_new: false, + create: true, + append: false, + truncate: false, + }, + ) + .unwrap(); + + union + .mount( + "mem_fs_1".to_string(), + PathBuf::from("/app/a").as_path(), + Box::new(a), + ) + .unwrap(); + union + .mount( + "mem_fs_2".to_string(), + PathBuf::from("/app/b").as_path(), + Box::new(b), + ) + .unwrap(); union } + #[tokio::test] + async fn test_nested_read_dir() { + let fs = gen_nested_filesystem(); + + let root_contents: Vec = fs + .read_dir(&PathBuf::from("/")) + .unwrap() + .map(|e| e.unwrap().path.clone()) + .collect(); + assert_eq!(root_contents, vec![PathBuf::from("/app")]); + + let app_contents: HashSet = fs + .read_dir(&PathBuf::from("/app")) + .unwrap() + .map(|e| e.unwrap().path) + .collect(); + assert_eq!( + app_contents, + HashSet::from_iter([PathBuf::from("/app/a"), PathBuf::from("/app/b")].into_iter()) + ); + + let a_contents: Vec = fs + .read_dir(&PathBuf::from("/app/a")) + .unwrap() + .map(|e| e.unwrap().path.clone()) + .collect(); + assert_eq!(a_contents, vec![PathBuf::from("/app/a/data-a.txt")]); + + let b_contents: Vec = fs + .read_dir(&PathBuf::from("/app/b")) + .unwrap() + .map(|e| e.unwrap().path) + .collect(); + assert_eq!(b_contents, vec![PathBuf::from("/app/b/data-b.txt")]); + } + + #[tokio::test] + async fn test_nested_metadata() { + let fs = gen_nested_filesystem(); + + assert!(fs.metadata(&PathBuf::from("/")).is_ok()); + assert!(fs.metadata(&PathBuf::from("/app")).is_ok()); + assert!(fs.metadata(&PathBuf::from("/app/a")).is_ok()); + assert!(fs.metadata(&PathBuf::from("/app/b")).is_ok()); + assert!(fs.metadata(&PathBuf::from("/app/a/data-a.txt")).is_ok()); + assert!(fs.metadata(&PathBuf::from("/app/b/data-b.txt")).is_ok()); + } + + #[tokio::test] + async fn test_nested_symlink_metadata() { + let fs = gen_nested_filesystem(); + + assert!(fs.symlink_metadata(&PathBuf::from("/")).is_ok()); + assert!(fs.symlink_metadata(&PathBuf::from("/app")).is_ok()); + assert!(fs.symlink_metadata(&PathBuf::from("/app/a")).is_ok()); + assert!(fs.symlink_metadata(&PathBuf::from("/app/b")).is_ok()); + assert!(fs + .symlink_metadata(&PathBuf::from("/app/a/data-a.txt")) + .is_ok()); + assert!(fs + .symlink_metadata(&PathBuf::from("/app/b/data-b.txt")) + .is_ok()); + } + #[tokio::test] async fn test_new_filesystem() { let fs = gen_filesystem(); @@ -540,19 +520,9 @@ mod tests { async fn test_create_dir() { let fs = gen_filesystem(); - assert_eq!( - fs.create_dir(Path::new("/")), - Err(FsError::BaseNotDirectory), - "creating a directory that has no parent", - ); + assert_eq!(fs.create_dir(Path::new("/")), Ok(()),); - let _ = fs_extra::remove_items(&["/test_create_dir"]); - - assert_eq!( - fs.create_dir(Path::new("/test_create_dir")), - Ok(()), - "creating a directory", - ); + assert_eq!(fs.create_dir(Path::new("/test_create_dir")), Ok(())); assert_eq!( fs.create_dir(Path::new("/test_create_dir/foo")), @@ -597,12 +567,10 @@ mod tests { async fn test_remove_dir() { let fs = gen_filesystem(); - let _ = fs_extra::remove_items(&["/test_remove_dir"]); - assert_eq!( fs.remove_dir(Path::new("/")), - Err(FsError::BaseNotDirectory), - "removing a directory that has no parent", + Err(FsError::PermissionDenied), + "cannot remove the root directory", ); assert_eq!( @@ -611,11 +579,7 @@ mod tests { "cannot remove a directory that doesn't exist", ); - assert_eq!( - fs.create_dir(Path::new("/test_remove_dir")), - Ok(()), - "creating a directory", - ); + assert_eq!(fs.create_dir(Path::new("/test_remove_dir")), Ok(())); assert_eq!( fs.create_dir(Path::new("/test_remove_dir/foo")), @@ -656,8 +620,6 @@ mod tests { !read_dir_names(&fs, "/test_remove_dir").contains(&"foo".to_string()), "the foo directory still exists" ); - - let _ = fs_extra::remove_items(&["/test_remove_dir"]); } fn read_dir_names(fs: &dyn crate::FileSystem, path: &str) -> Vec { @@ -671,20 +633,18 @@ mod tests { async fn test_rename() { let fs = gen_filesystem(); - let _ = fs_extra::remove_items(&["./test_rename"]); - assert_eq!( fs.rename(Path::new("/"), Path::new("/bar")).await, - Err(FsError::BaseNotDirectory), + Err(FsError::PermissionDenied), "renaming a directory that has no parent", ); assert_eq!( fs.rename(Path::new("/foo"), Path::new("/")).await, - Err(FsError::BaseNotDirectory), + Err(FsError::EntryNotFound), "renaming to a directory that has no parent", ); - assert_eq!(fs.create_dir(Path::new("/test_rename")), Ok(())); + assert_eq!(fs.create_dir(Path::new("/test_rename")), Ok(()),); assert_eq!(fs.create_dir(Path::new("/test_rename/foo")), Ok(())); assert_eq!(fs.create_dir(Path::new("/test_rename/foo/qux")), Ok(())); @@ -836,15 +796,11 @@ mod tests { let fs = gen_filesystem(); - let _ = fs_extra::remove_items(&["/test_metadata"]); - - assert_eq!(fs.create_dir(Path::new("/test_metadata")), Ok(())); - let root_metadata = fs.metadata(Path::new("/test_metadata")).unwrap(); assert!(root_metadata.ft.dir); - assert!(root_metadata.accessed == root_metadata.created); - assert!(root_metadata.modified == root_metadata.created); + assert_eq!(root_metadata.accessed, root_metadata.created); + assert_eq!(root_metadata.modified, root_metadata.created); assert!(root_metadata.modified > 0); assert_eq!(fs.create_dir(Path::new("/test_metadata/foo")), Ok(())); @@ -888,10 +844,6 @@ mod tests { async fn test_remove_file() { let fs = gen_filesystem(); - let _ = fs_extra::remove_items(&["./test_remove_file"]); - - assert!(fs.create_dir(Path::new("/test_remove_file")).is_ok()); - assert!( fs.new_open_options() .write(true) @@ -1094,34 +1046,4 @@ mod tests { let _ = fs_extra::remove_items(&["./test_canonicalize"]); } */ - - #[tokio::test] - #[ignore = "Not yet supported. See https://github.com/wasmerio/wasmer/issues/3678"] - async fn mount_to_overlapping_directories() { - let top_level = mem_fs::FileSystem::default(); - ops::touch(&top_level, "/file.txt").unwrap(); - let nested = mem_fs::FileSystem::default(); - ops::touch(&nested, "/another-file.txt").unwrap(); - - let mut fs = UnionFileSystem::default(); - fs.mount( - "top-level", - "/", - false, - Box::new(top_level), - Some("/top-level"), - ); - fs.mount( - "nested", - "/", - false, - Box::new(nested), - Some("/top-level/nested"), - ); - - assert!(ops::is_dir(&fs, "/top-level")); - assert!(ops::is_file(&fs, "/top-level/file.txt")); - assert!(ops::is_dir(&fs, "/top-level/nested")); - assert!(ops::is_file(&fs, "/top-level/nested/another-file.txt")); - } } diff --git a/lib/virtual-fs/src/webc_volume_fs.rs b/lib/virtual-fs/src/webc_volume_fs.rs index 7ce0638590e..745a0570e39 100644 --- a/lib/virtual-fs/src/webc_volume_fs.rs +++ b/lib/virtual-fs/src/webc_volume_fs.rs @@ -155,6 +155,15 @@ impl FileSystem for WebcVolumeFileSystem { fn new_open_options(&self) -> crate::OpenOptions { crate::OpenOptions::new(self) } + + fn mount( + &self, + _name: String, + _path: &Path, + _fs: Box, + ) -> Result<(), FsError> { + Err(FsError::Unsupported) + } } impl FileOpener for WebcVolumeFileSystem { diff --git a/lib/vm/Cargo.toml b/lib/vm/Cargo.toml index a89893b7e40..2239d5db2f7 100644 --- a/lib/vm/Cargo.toml +++ b/lib/vm/Cargo.toml @@ -14,6 +14,7 @@ version.workspace = true [dependencies] memoffset.workspace = true +dashmap.workspace = true wasmer-types = { path = "../types", version = "=4.3.5" } libc.workspace = true indexmap = { version = "1.6" } @@ -28,7 +29,6 @@ lazy_static = "1.4.0" region = { version = "3.0.2" } corosensei = { version = "0.1.2" } derivative = { version = "^2" } -dashmap = { version = "5.4" } fnv = "1.0.3" # - Optional shared dependencies. tracing = { version = "0.1", optional = true } diff --git a/lib/wasix/Cargo.toml b/lib/wasix/Cargo.toml index 3a5f3e0dbce..6fb82e1b962 100644 --- a/lib/wasix/Cargo.toml +++ b/lib/wasix/Cargo.toml @@ -24,6 +24,7 @@ wasmer-emscripten = { path = "../emscripten", version = "=4.3.5", optional = tru wasmer-config = { version = "0.5.0", path = "../config" } http.workspace = true +dashmap.workspace = true xxhash-rust = { version = "0.8.8", features = ["xxh64"] } rusty_pool = { version = "0.7.0", optional = true } cfg-if = "1.0" @@ -65,7 +66,6 @@ heapless = "0.7.16" once_cell = "1.17.0" pin-project = "1.0.12" semver = "1.0.17" -dashmap = "5.4.0" tempfile = "3.6.0" num_enum = "0.5.7" # Used by the WCGI runner diff --git a/lib/wasix/src/fs/mod.rs b/lib/wasix/src/fs/mod.rs index a1fe86e05d0..9ab56c8d7ff 100644 --- a/lib/wasix/src/fs/mod.rs +++ b/lib/wasix/src/fs/mod.rs @@ -376,6 +376,17 @@ impl FileSystem for WasiFsRoot { WasiFsRoot::Backing(fs) => fs.new_open_options(), } } + fn mount( + &self, + name: String, + path: &Path, + fs: Box, + ) -> virtual_fs::Result<()> { + match self { + WasiFsRoot::Sandbox(f) => f.mount(name, path, fs), + WasiFsRoot::Backing(f) => f.mount(name, path, fs), + } + } } /// Merge the contents of one filesystem into another. @@ -2154,6 +2165,14 @@ impl FileSystem for FallbackFileSystem { fn new_open_options(&self) -> virtual_fs::OpenOptions { Self::fail(); } + fn mount( + &self, + _name: String, + _path: &Path, + _fs: Box, + ) -> virtual_fs::Result<()> { + Self::fail() + } } pub fn virtual_file_type_to_wasi_file_type(file_type: virtual_fs::FileType) -> Filetype { @@ -2222,5 +2241,6 @@ pub fn fs_error_into_wasi_err(fs_error: FsError) -> Errno { FsError::DirectoryNotEmpty => Errno::Notempty, FsError::StorageFull => Errno::Overflow, FsError::Lock | FsError::UnknownError => Errno::Io, + FsError::Unsupported => Errno::Notsup, } } diff --git a/lib/wasix/src/runners/wasi_common.rs b/lib/wasix/src/runners/wasi_common.rs index d1302b11718..82b2135f98a 100644 --- a/lib/wasix/src/runners/wasi_common.rs +++ b/lib/wasix/src/runners/wasi_common.rs @@ -166,8 +166,7 @@ fn build_directory_mappings( })?; } - root_fs - .mount(guest_path.clone(), fs, "/".into()) + TmpFileSystem::mount(root_fs, guest_path.clone(), fs, "/".into()) .with_context(|| format!("Unable to mount \"{}\"", guest_path.display()))?; } } @@ -348,6 +347,19 @@ impl virtual_fs::FileSystem for RelativeOrAbsolutePathHack { fn new_open_options(&self) -> virtual_fs::OpenOptions { virtual_fs::OpenOptions::new(self) } + + fn mount( + &self, + name: String, + path: &Path, + fs: Box, + ) -> virtual_fs::Result<()> { + let name_ref = &name; + let f_ref = &Arc::new(fs); + self.execute(path, move |f, p| { + f.mount(name_ref.clone(), p, Box::new(f_ref.clone())) + }) + } } impl virtual_fs::FileOpener for RelativeOrAbsolutePathHack { diff --git a/lib/wasix/src/runtime/package_loader/load_package_tree.rs b/lib/wasix/src/runtime/package_loader/load_package_tree.rs index f635b2b35c6..4ccf27bae09 100644 --- a/lib/wasix/src/runtime/package_loader/load_package_tree.rs +++ b/lib/wasix/src/runtime/package_loader/load_package_tree.rs @@ -363,7 +363,7 @@ fn filesystem_v3( let mut mountings: Vec<_> = pkg.filesystem.iter().collect(); mountings.sort_by_key(|m| std::cmp::Reverse(m.mount_path.as_path())); - let mut union_fs = UnionFileSystem::new(); + let union_fs = UnionFileSystem::new(); for ResolvedFileSystemMapping { mount_path, @@ -396,13 +396,7 @@ fn filesystem_v3( })?; let webc_vol = WebcVolumeFileSystem::new(volume.clone()); - union_fs.mount( - volume_name, - mount_path.to_str().unwrap(), - false, - Box::new(webc_vol), - None, - ); + union_fs.mount(volume_name.clone(), mount_path, Box::new(webc_vol))?; } let fs = OverlayFileSystem::new(virtual_fs::EmptyFileSystem::default(), [union_fs]); @@ -590,6 +584,16 @@ where fn new_open_options(&self) -> virtual_fs::OpenOptions { virtual_fs::OpenOptions::new(self) } + + fn mount( + &self, + name: String, + path: &Path, + fs: Box, + ) -> virtual_fs::Result<()> { + let path = self.path(path)?; + self.inner.mount(name, path.as_path(), fs) + } } impl virtual_fs::FileOpener for MappedPathFileSystem diff --git a/lib/wasix/tests/runners.rs b/lib/wasix/tests/runners.rs index 5f1f8826114..b58c41e5998 100644 --- a/lib/wasix/tests/runners.rs +++ b/lib/wasix/tests/runners.rs @@ -12,7 +12,7 @@ use reqwest::Client; use tokio::runtime::Handle; use wasmer::Engine; use wasmer_wasix::{ - http::HttpClient, + http::{reqwest::get_proxy, HttpClient}, runners::Runner, runtime::{ module_cache::{FileSystemCache, ModuleCache, SharedCache}, @@ -229,10 +229,16 @@ async fn download_cached(url: &str) -> bytes::Bytes { } fn client() -> Client { - Client::builder() - .connect_timeout(Duration::from_secs(30)) - .build() - .unwrap() + let builder = { + let mut builder = reqwest::ClientBuilder::new().connect_timeout(Duration::from_secs(30)); + if let Some(proxy) = get_proxy().unwrap() { + builder = builder.proxy(proxy); + } + builder + }; + let client = builder.build().unwrap(); + + client } #[cfg(not(target_os = "windows"))] diff --git a/tests/integration/cli/src/fixtures.rs b/tests/integration/cli/src/fixtures.rs index f5a438bbb73..193e3c0d483 100644 --- a/tests/integration/cli/src/fixtures.rs +++ b/tests/integration/cli/src/fixtures.rs @@ -8,6 +8,12 @@ pub fn resources() -> PathBuf { Path::new(env!("CARGO_MANIFEST_DIR")).join("resources") } +pub fn packages() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests") + .join("packages") +} + pub fn php() -> (PathBuf, PathBuf, PathBuf) { let root = Path::new(env!("CARGO_MANIFEST_DIR")); let resources = resources().join("php"); diff --git a/tests/integration/cli/tests/packages/nested-mounted-paths/a/data-a.txt b/tests/integration/cli/tests/packages/nested-mounted-paths/a/data-a.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/cli/tests/packages/nested-mounted-paths/b/data-b.txt b/tests/integration/cli/tests/packages/nested-mounted-paths/b/data-b.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/cli/tests/packages/nested-mounted-paths/main.c b/tests/integration/cli/tests/packages/nested-mounted-paths/main.c new file mode 100644 index 00000000000..46854e25f8a --- /dev/null +++ b/tests/integration/cli/tests/packages/nested-mounted-paths/main.c @@ -0,0 +1,83 @@ +#include +#include +#include +#include +#include +#include +#include + +int main() +{ + DIR *dir; + struct dirent *entry; + + printf("/:\n"); + dir = opendir("/"); + + if (dir != NULL) + { + while ((entry = readdir(dir)) != NULL) + { + printf("%s\n", entry->d_name); + } + closedir(dir); + } + else + { + perror("opendir"); + return 1; + } + + printf("\n/app:\n"); + dir = opendir("/app"); + + if (dir != NULL) + { + while ((entry = readdir(dir)) != NULL) + { + printf("%s\n", entry->d_name); + } + closedir(dir); + } + else + { + perror("opendir"); + return 1; + } + + printf("\n/app/a:\n"); + dir = opendir("/app/a"); + + if (dir != NULL) + { + while ((entry = readdir(dir)) != NULL) + { + printf("%s\n", entry->d_name); + } + closedir(dir); + } + else + { + perror("opendir"); + return 1; + } + + printf("\n/app/b:\n"); + dir = opendir("/app/b"); + + if (dir != NULL) + { + while ((entry = readdir(dir)) != NULL) + { + printf("%s\n", entry->d_name); + } + closedir(dir); + } + else + { + perror("opendir"); + return 1; + } + + return 0; +} \ No newline at end of file diff --git a/tests/integration/cli/tests/packages/nested-mounted-paths/main.o b/tests/integration/cli/tests/packages/nested-mounted-paths/main.o new file mode 100644 index 00000000000..d60e1c81520 Binary files /dev/null and b/tests/integration/cli/tests/packages/nested-mounted-paths/main.o differ diff --git a/tests/integration/cli/tests/packages/nested-mounted-paths/out.webc b/tests/integration/cli/tests/packages/nested-mounted-paths/out.webc new file mode 100644 index 00000000000..5f036237b32 Binary files /dev/null and b/tests/integration/cli/tests/packages/nested-mounted-paths/out.webc differ diff --git a/tests/integration/cli/tests/packages/nested-mounted-paths/wasmer.toml b/tests/integration/cli/tests/packages/nested-mounted-paths/wasmer.toml new file mode 100644 index 00000000000..57a92009a67 --- /dev/null +++ b/tests/integration/cli/tests/packages/nested-mounted-paths/wasmer.toml @@ -0,0 +1,11 @@ +[[module]] +name = "main" +source = "main.o" + +[[command]] +name = "main" +module = "main" + +[fs] +"/app/a" = "a" +"/app/b" = "b" diff --git a/tests/integration/cli/tests/run.rs b/tests/integration/cli/tests/run.rs index 38000ac017b..2da4bb75f5f 100644 --- a/tests/integration/cli/tests/run.rs +++ b/tests/integration/cli/tests/run.rs @@ -15,7 +15,7 @@ use reqwest::{blocking::Client, IntoUrl}; use tempfile::TempDir; use wasmer_integration_tests_cli::{ asset_path, - fixtures::{self, php, resources}, + fixtures::{self, packages, php, resources}, get_wasmer_path, }; @@ -47,6 +47,61 @@ static CACHE_RUST_LOG: Lazy = Lazy::new(|| { .join(",") }); +#[test] +fn nested_mounted_paths() { + let package = packages().join("nested-mounted-paths"); + + let webc = package.join("out.webc"); + + let host_output = Command::new(get_wasmer_path()) + .arg("run") + .arg(package) + .output() + .unwrap(); + let host_stdout = host_output.stdout; + + let webc_output = Command::new(get_wasmer_path()) + .arg("run") + .arg(webc) + .arg(".") + .output() + .unwrap(); + let webc_stdout = webc_output.stdout; + + let expected = "/: +. +.. +.app +.private +app +bin +dev +etc +tmp + +/app: +. +.. +a +b + +/app/a: +. +.. +data-a.txt + +/app/b: +. +.. +data-b.txt +" + .as_bytes() + .to_vec(); + + assert_eq!(&host_stdout, &expected); + assert_eq!(&webc_stdout, &expected); +} + #[test] fn run_python_create_temp_dir_in_subprocess() { let resources = resources().join("python").join("temp-dir-in-child"); diff --git a/tests/lib/wast/src/wasi_wast.rs b/tests/lib/wast/src/wasi_wast.rs index 0eba68a2470..4491fbc82d4 100644 --- a/tests/lib/wast/src/wasi_wast.rs +++ b/tests/lib/wast/src/wasi_wast.rs @@ -246,14 +246,38 @@ impl<'a> WasiTest<'a> { let e = mem_fs::FileSystem::default(); let f = mem_fs::FileSystem::default(); - let mut union = union_fs::UnionFileSystem::new(); - - union.mount("mem_fs", "/test_fs", false, Box::new(a), None); - union.mount("mem_fs_2", "/snapshot1", false, Box::new(b), None); - union.mount("mem_fs_3", "/tests", false, Box::new(c), None); - union.mount("mem_fs_4", "/nightly_2022_10_18", false, Box::new(d), None); - union.mount("mem_fs_5", "/unstable", false, Box::new(e), None); - union.mount("mem_fs_6", "/.tmp_wasmer_wast_0", false, Box::new(f), None); + let union = union_fs::UnionFileSystem::new(); + + union.mount( + "mem_fs".to_string(), + PathBuf::from("/test_fs").as_ref(), + Box::new(a), + )?; + union.mount( + "mem_fs_2".to_string(), + PathBuf::from("/snapshot1").as_ref(), + Box::new(b), + )?; + union.mount( + "mem_fs_3".to_string(), + PathBuf::from("/tests").as_ref(), + Box::new(c), + )?; + union.mount( + "mem_fs_4".to_string(), + PathBuf::from("/nightly_2022_10_18").as_ref(), + Box::new(d), + )?; + union.mount( + "mem_fs_5".to_string(), + PathBuf::from("/unstable").as_ref(), + Box::new(e), + )?; + union.mount( + "mem_fs_6".to_string(), + PathBuf::from("/.tmp_wasmer_wast_0").as_ref(), + Box::new(f), + )?; Box::new(union) }