Merge pull request #5008 from wasmerio/allow-nested-mounted-paths

Allow nested mounted paths in `UnionFilesystem`

Cargo.toml

@@ -97,6 +97,7 @@ edge-schema = { version = "=0.1.0" }
 shared-buffer = "0.1.4"
 
 # Third-party crates
+dashmap = "6.0.1"
 http = "1.0.0"
 hyper = "1"
 reqwest = { version = "0.12.0", default-features = false }

lib/backend-api/src/client.rs

@@ -52,6 +52,28 @@ impl WasmerClient {
         })
     }
 
+    pub fn new_with_proxy(
+        graphql_endpoint: Url,
+        user_agent: &str,
+        proxy: reqwest::Proxy,
+    ) -> Result<Self, anyhow::Error> {
+        let builder = {
+            #[cfg(all(target_arch = "wasm32", target_os = "unknown"))]
+            let mut builder = reqwest::ClientBuilder::new();
+
+            #[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))]
+            let builder = reqwest::ClientBuilder::new()
+                .connect_timeout(Duration::from_secs(10))
+                .timeout(Duration::from_secs(90));
+
+            builder.proxy(proxy)
+        };
+
+        let client = builder.build().context("failed to create reqwest client")?;
+
+        Self::new_with_client(client, graphql_endpoint, user_agent)
+    }
+
     pub fn new(graphql_endpoint: Url, user_agent: &str) -> Result<Self, anyhow::Error> {
         #[cfg(all(target_arch = "wasm32", target_os = "unknown"))]
         let client = reqwest::Client::builder()

lib/cli/src/commands/package/download.rs

@@ -28,6 +28,10 @@ pub struct PackageDownload {
     #[clap(long)]
     pub quiet: bool,
 
+    /// proxy to use for downloading
+    #[clap(long)]
+    pub proxy: Option<String>,
+
     /// The package to download.
     package: PackageSource,
 }
@@ -97,7 +101,13 @@ impl PackageDownload {
                 // caveat: client_unauthennticated will use a token if provided, it
                 // just won't fail if none is present. So, _unauthenticated() can actually
                 // produce an authenticated client.
-                let client = self.env.client_unauthennticated()?;
+                let client = if let Some(proxy) = &self.proxy {
+                    let proxy = reqwest::Proxy::all(proxy)?;
+
+                    self.env.client_unauthennticated_with_proxy(proxy)?
+                } else {
+                    self.env.client_unauthennticated()?
+                };
 
                 let version = id.version_or_default().to_string();
                 let version = if version == "*" {
@@ -299,6 +309,7 @@ mod tests {
             out_path: Some(out_path.clone()),
             package: "wasmer/[email protected]".parse().unwrap(),
             quiet: true,
+            proxy: None,
         };
 
         cmd.execute().unwrap();

lib/cli/src/config/env.rs

@@ -138,6 +138,26 @@ impl WasmerEnv {
         Ok(client)
     }
 
+    pub fn client_unauthennticated_with_proxy(
+        &self,
+        proxy: reqwest::Proxy,
+    ) -> Result<WasmerClient, anyhow::Error> {
+        let registry_url = self.registry_endpoint()?;
+        let client = wasmer_api::WasmerClient::new_with_proxy(
+            registry_url,
+            &DEFAULT_WASMER_CLI_USER_AGENT,
+            proxy,
+        )?;
+
+        let client = if let Some(token) = self.token() {
+            client.with_auth_token(token)
+        } else {
+            client
+        };
+
+        Ok(client)
+    }
+
     pub fn client(&self) -> Result<WasmerClient, anyhow::Error> {
         let client = self.client_unauthennticated()?;
         if client.auth_token().is_none() {

lib/virtual-fs/Cargo.toml

@@ -10,6 +10,7 @@ repository.workspace = true
 rust-version.workspace = true
 
 [dependencies]
+dashmap.workspace = true
 dunce = "1.0.4"
 anyhow = { version = "1.0.66", optional = true }
 async-trait = { version = "^0.1" }

lib/virtual-fs/src/arc_fs.rs

@@ -53,4 +53,13 @@ impl FileSystem for ArcFileSystem {
     fn new_open_options(&self) -> OpenOptions {
         self.fs.new_open_options()
     }
+
+    fn mount(
+        &self,
+        name: String,
+        path: &Path,
+        fs: Box<dyn crate::FileSystem + Send + Sync>,
+    ) -> Result<()> {
+        self.fs.mount(name, path, fs)
+    }
 }

lib/virtual-fs/src/empty_fs.rs

@@ -67,6 +67,15 @@ impl FileSystem for EmptyFileSystem {
     fn new_open_options(&self) -> OpenOptions {
         OpenOptions::new(self)
     }
+
+    fn mount(
+        &self,
+        name: String,
+        path: &Path,
+        fs: Box<dyn crate::FileSystem + Send + Sync>,
+    ) -> Result<()> {
+        Err(FsError::Unsupported)
+    }
 }
 
 impl FileOpener for EmptyFileSystem {

lib/virtual-fs/src/host_fs.rs

@@ -235,6 +235,15 @@ impl crate::FileSystem for FileSystem {
             .and_then(TryInto::try_into)
             .map_err(Into::into)
     }
+
+    fn mount(
+        &self,
+        _name: String,
+        _path: &Path,
+        _fs: Box<dyn crate::FileSystem + Send + Sync>,
+    ) -> Result<()> {
+        Err(FsError::Unsupported)
+    }
 }
 
 impl TryInto<Metadata> for std::fs::Metadata {

lib/virtual-fs/src/lib.rs

@@ -98,6 +98,9 @@ pub trait FileSystem: fmt::Debug + Send + Sync + 'static + Upcastable {
     fn remove_file(&self, path: &Path) -> Result<()>;
 
     fn new_open_options(&self) -> OpenOptions;
+
+    fn mount(&self, name: String, path: &Path, fs: Box<dyn FileSystem + Send + Sync>)
+        -> Result<()>;
 }
 
 impl dyn FileSystem + 'static {
@@ -152,6 +155,15 @@ where
     fn new_open_options(&self) -> OpenOptions {
         (**self).new_open_options()
     }
+
+    fn mount(
+        &self,
+        name: String,
+        path: &Path,
+        fs: Box<dyn FileSystem + Send + Sync>,
+    ) -> Result<()> {
+        (**self).mount(name, path, fs)
+    }
 }
 
 pub trait FileOpener {
@@ -510,6 +522,9 @@ pub enum FsError {
     /// Some other unhandled error. If you see this, it's probably a bug.
     #[error("unknown error found")]
     UnknownError,
+    /// Operation is not supported on this filesystem
+    #[error("unsupported")]
+    Unsupported,
 }
 
 impl From<io::Error> for FsError {
@@ -570,6 +585,7 @@ impl From<FsError> for io::Error {
             FsError::DirectoryNotEmpty => io::ErrorKind::Other,
             FsError::UnknownError => io::ErrorKind::Other,
             FsError::StorageFull => io::ErrorKind::Other,
+            FsError::Unsupported => io::ErrorKind::Unsupported,
             // NOTE: Add this once the "io_error_more" Rust feature is stabilized
             // FsError::StorageFull => io::ErrorKind::StorageFull,
         };
@@ -580,7 +596,7 @@ impl From<FsError> for io::Error {
 #[derive(Debug)]
 pub struct ReadDir {
     // TODO: to do this properly we need some kind of callback to the core FS abstraction
-    data: Vec<DirEntry>,
+    pub(crate) data: Vec<DirEntry>,
     index: usize,
 }
 

lib/virtual-fs/src/mem_fs/filesystem.rs

@@ -3,7 +3,7 @@
 use self::offloaded_file::OffloadBackingStore;
 
 use super::*;
-use crate::{DirEntry, FileSystem as _, FileType, FsError, Metadata, OpenOptions, ReadDir, Result};
+use crate::{DirEntry, FileType, FsError, Metadata, OpenOptions, ReadDir, Result};
 use futures::future::{BoxFuture, Either};
 use slab::Slab;
 use std::collections::VecDeque;
@@ -680,6 +680,16 @@ impl crate::FileSystem for FileSystem {
     fn new_open_options(&self) -> OpenOptions {
         OpenOptions::new(self)
     }
+
+    fn mount(
+        &self,
+        _name: String,
+        path: &Path,
+        fs: Box<dyn crate::FileSystem + Send + Sync>,
+    ) -> Result<()> {
+        let fs: Arc<dyn crate::FileSystem + Send + Sync> = Arc::new(fs);
+        self.mount(path.to_owned(), &fs, PathBuf::from("/"))
+    }
 }
 
 impl fmt::Debug for FileSystem {

lib/virtual-fs/src/overlay_fs.rs

@@ -414,6 +414,15 @@ where
     fn new_open_options(&self) -> OpenOptions<'_> {
         OpenOptions::new(self)
     }
+
+    fn mount(
+        &self,
+        _name: String,
+        _path: &Path,
+        _fs: Box<dyn FileSystem + Send + Sync>,
+    ) -> Result<(), FsError> {
+        Err(FsError::Unsupported)
+    }
 }
 
 impl<P, S> FileOpener for OverlayFileSystem<P, S>

lib/virtual-fs/src/passthru_fs.rs

@@ -53,6 +53,15 @@ impl FileSystem for PassthruFileSystem {
     fn new_open_options(&self) -> OpenOptions {
         self.fs.new_open_options()
     }
+
+    fn mount(
+        &self,
+        _name: String,
+        _path: &Path,
+        _fs: Box<dyn FileSystem + Send + Sync>,
+    ) -> Result<()> {
+        Err(FsError::Unsupported)
+    }
 }
 
 #[cfg(test)]

lib/virtual-fs/src/static_fs.rs

@@ -382,6 +382,15 @@ impl FileSystem for StaticFileSystem {
             self.memory.symlink_metadata(Path::new(&path))
         }
     }
+
+    fn mount(
+        &self,
+        _name: String,
+        _path: &Path,
+        _fs: Box<dyn FileSystem + Send + Sync>,
+    ) -> Result<(), FsError> {
+        Err(FsError::Unsupported)
+    }
 }
 
 fn normalizes_path(path: &Path) -> String {

lib/virtual-fs/src/tmp_fs.rs

@@ -96,4 +96,13 @@ impl FileSystem for TmpFileSystem {
     fn new_open_options(&self) -> OpenOptions {
         self.fs.new_open_options()
     }
+
+    fn mount(
+        &self,
+        name: String,
+        path: &Path,
+        fs: Box<dyn FileSystem + Send + Sync>,
+    ) -> Result<()> {
+        FileSystem::mount(&self.fs, name, path, fs)
+    }
 }

lib/virtual-fs/src/trace_fs.rs

@@ -1,5 +1,5 @@
 use std::{
-    path::PathBuf,
+    path::{Path, PathBuf},
     pin::Pin,
     task::{Context, Poll},
 };
@@ -87,6 +87,16 @@ where
     fn new_open_options(&self) -> crate::OpenOptions {
         crate::OpenOptions::new(self)
     }
+
+    #[tracing::instrument(level = "trace", skip(self))]
+    fn mount(
+        &self,
+        name: String,
+        path: &Path,
+        fs: Box<dyn FileSystem + Send + Sync>,
+    ) -> crate::Result<()> {
+        self.0.mount(name, path, fs)
+    }
 }
 
 impl<F> FileOpener for TraceFileSystem<F>