Merge pull request #3913 from wasmerio/fs-table

Add filesystem mappings to the resolver

Cargo.toml

@@ -78,6 +78,11 @@ repository = "https://github.com/wasmerio/wasmer"
 rust-version = "1.67"
 version = "4.0.0-beta.1"
 
+[workspace.dependencies]
+webc = { version = "5.0.4", default-features = false }
+wapm-targz-to-pirita = "0.3.1"
+wasmer-toml = "0.6.0"
+
 [build-dependencies]
 test-generator = { path = "tests/lib/test-generator" }
 build-deps = "0.1.4"

deny.toml

@@ -111,7 +111,8 @@ confidence-threshold = 0.8
 exceptions = [
     # Each entry is the crate and version constraint, and its specific allow
     # list
-    { name = "webc", allow = ["BUSL-1.1"] }
+    { name = "webc", allow = ["BUSL-1.1"] },
+    { name = "wapm-targz-to-pirita", allow = ["BUSL-1.1"] },
 ]
 
 

lib/cli/Cargo.toml

@@ -45,8 +45,8 @@ virtual-fs  = { version = "0.4.0", path = "../virtual-fs", default-features = fa
 virtual-net  = { version = "0.3.0", path = "../virtual-net" }
 
 # Wasmer-owned dependencies.
-webc = { version = "5.0" }
-wapm-targz-to-pirita = "0.2.1"
+webc = { workspace = true }
+wapm-targz-to-pirita = { workspace = true }
 wasmer-deploy-cli = { version = "0.1.9", default-features = false }
 
 # Third-party dependencies.
@@ -67,7 +67,7 @@ dirs = { version = "4.0" }
 serde_json = { version = "1.0" }
 target-lexicon = { version = "0.12", features = ["std"] }
 prettytable-rs = "0.10.0"
-wasmer-toml = "0.6.0"
+wasmer-toml = { workspace = true }
 indexmap = "1.9.2"
 walkdir = "2.3.2"
 regex = "1.6.0"

lib/cli/src/commands/create_exe.rs

@@ -514,7 +514,7 @@ fn serialize_volume_to_webc_v1(volume: &WebcVolume) -> Vec<u8> {
                     if let Some(contents) = volume.read_file(&*path) {
                         files.insert(
                             webc::v1::DirOrFile::File(path.to_string().into()),
-                            contents.into(),
+                            contents.to_vec(),
                         );
                     }
                 }

lib/cli/src/commands/run.rs

@@ -22,7 +22,7 @@ use sha2::{Digest, Sha256};
 use tempfile::NamedTempFile;
 use tokio::runtime::Handle;
 use url::Url;
-use wapm_targz_to_pirita::{FileMap, TransformManifestFunctions};
+use wapm_targz_to_pirita::{webc::v1::DirOrFile, FileMap, TransformManifestFunctions};
 use wasmer::{
     DeserializeError, Engine, Function, Imports, Instance, Module, Store, Type, TypedFunction,
     Value,
@@ -44,7 +44,7 @@ use wasmer_wasix::{
     },
     Runtime,
 };
-use webc::{metadata::Manifest, v1::DirOrFile, Container};
+use webc::{metadata::Manifest, Container};
 
 use crate::{commands::run::wasi::Wasi, error::PrettyError, store::StoreOptions};
 
@@ -610,7 +610,7 @@ fn construct_webc_in_memory(dir: &Path) -> Result<Vec<u8>, Error> {
     }
 
     let functions = TransformManifestFunctions::default();
-    let webc = wapm_targz_to_pirita::generate_webc_file(files, dir, None, &functions)?;
+    let webc = wapm_targz_to_pirita::generate_webc_file(files, dir, &functions)?;
 
     Ok(webc)
 }

lib/wasix/Cargo.toml

@@ -29,8 +29,8 @@ bincode = { version = "1.3" }
 chrono = { version = "^0.4", default-features = false, features = [ "wasmbind", "std", "clock" ], optional = true }
 derivative = { version = "^2" }
 bytes = "1"
-webc = { version = "5.0.2", default-features = false }
-serde_cbor = { version = "0.11.2", optional = true }
+webc = { workspace = true }
+serde_cbor = { version = "0.11.2" }
 anyhow = { version = "1.0.66" }
 lazy_static = "1.4"
 sha2 = { version = "0.10" }
@@ -65,6 +65,7 @@ wcgi-host = { version = "0.1.2", optional = true }
 tower-http = { version = "0.4.0", features = ["trace", "util", "catch-panic", "cors"], optional = true }
 tower = { version = "0.4.13", features = ["make", "util"], optional = true }
 url = "2.3.1"
+petgraph = "0.6.3"
 
 [target.'cfg(not(target_arch = "riscv64"))'.dependencies.reqwest]
 version = "0.11"
@@ -96,6 +97,7 @@ wasm-bindgen = ">= 0.2.74, < 0.2.85"
 wasmer = { path = "../api", version = "=4.0.0-beta.1", default-features = false, features = ["wat", "js-serializable-module"] }
 tokio = { version = "1", features = [ "sync", "macros", "rt" ], default_features = false }
 pretty_assertions = "1.3.0"
+wapm-targz-to-pirita = { workspace = true }
 
 [target.'cfg(target_arch = "wasm32")'.dev-dependencies]
 wasm-bindgen-test = "0.3.0"
@@ -110,13 +112,13 @@ default = ["sys-default"]
 
 time = ["tokio/time"]
 
-webc_runner = ["serde_cbor"]
+webc_runner = []
 webc_runner_rt_wasi = []
 webc_runner_rt_wcgi = ["hyper", "wcgi", "wcgi-host", "tower", "tower-http"]
 webc_runner_rt_emscripten = ["wasmer-emscripten"]
 
 sys = ["webc/mmap", "time"]
-sys-default = ["sys", "logging", "host-fs", "sys-poll", "sys-thread", "host-vnet", "host-threads", "host-reqwest" ]
+sys-default = ["sys", "logging", "host-fs", "sys-poll", "sys-thread", "host-vnet", "host-threads", "host-reqwest"]
 sys-poll = []
 sys-thread = ["tokio/rt", "tokio/time", "tokio/rt-multi-thread"]
 
@@ -127,21 +129,14 @@ js = ["virtual-fs/no-time", "getrandom/js", "chrono"]
 js-default = ["js"]
 test-js = ["js", "wasmer/wat"]
 
-host-vnet = [ "virtual-net/host-net" ]
+host-vnet = ["virtual-net/host-net"]
 host-threads = []
 host-reqwest = ["reqwest"]
 host-fs = ["virtual-fs/host-fs"]
 
 logging = ["tracing/log"]
-disable-all-logging = [
-    "tracing/release_max_level_off",
-    "tracing/max_level_off"
-]
-enable-serde = [
-    "typetag",
-    "virtual-fs/enable-serde",
-    "wasmer-wasix-types/enable-serde",
-]
+disable-all-logging = ["tracing/release_max_level_off", "tracing/max_level_off"]
+enable-serde = ["typetag", "virtual-fs/enable-serde", "wasmer-wasix-types/enable-serde"]
 
 [package.metadata.docs.rs]
 features = ["wasmer/sys"]

lib/wasix/src/bin_factory/binary_package.rs

@@ -136,3 +136,103 @@ impl BinaryPackage {
         })
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use std::{collections::BTreeMap, path::Path};
+
+    use tempfile::TempDir;
+    use virtual_fs::AsyncReadExt;
+    use wapm_targz_to_pirita::{webc::v1::DirOrFile, FileMap, TransformManifestFunctions};
+
+    use crate::{runtime::task_manager::VirtualTaskManager, PluggableRuntime};
+
+    use super::*;
+
+    fn task_manager() -> Arc<dyn VirtualTaskManager + Send + Sync> {
+        cfg_if::cfg_if! {
+            if #[cfg(feature = "sys-threads")] {
+                Arc::new(crate::runtime::task_manager::tokio::TokioTaskManager::new(tokio::runtime::Handle::current()))
+            } else {
+                unimplemented!("Unable to get the task manager")
+            }
+        }
+    }
+
+    #[tokio::test]
+    #[cfg_attr(
+        not(feature = "sys-threads"),
+        ignore = "The tokio task manager isn't available on this platform"
+    )]
+    async fn fs_table_can_map_directories_to_different_names() {
+        let temp = TempDir::new().unwrap();
+        let wasmer_toml = r#"
+            [package]
+            name = "some/package"
+            version = "0.0.0"
+            description = "a dummy package"
+
+            [fs]
+            "/public" = "./out"
+        "#;
+        std::fs::write(temp.path().join("wasmer.toml"), wasmer_toml).unwrap();
+        let out = temp.path().join("out");
+        std::fs::create_dir_all(&out).unwrap();
+        let file_txt = "Hello, World!";
+        std::fs::write(out.join("file.txt"), file_txt).unwrap();
+        let webc = construct_webc_in_memory(temp.path());
+        let webc = Container::from_bytes(webc).unwrap();
+        let tasks = task_manager();
+        let runtime = PluggableRuntime::new(tasks);
+
+        let pkg = BinaryPackage::from_webc(&webc, &runtime).await.unwrap();
+
+        // We should have mapped "./out/file.txt" on the host to
+        // "/public/file.txt" on the guest.
+        let mut f = pkg
+            .webc_fs
+            .new_open_options()
+            .read(true)
+            .open("/public/file.txt")
+            .unwrap();
+        let mut buffer = String::new();
+        f.read_to_string(&mut buffer).await.unwrap();
+        assert_eq!(buffer, file_txt);
+    }
+
+    fn construct_webc_in_memory(dir: &Path) -> Vec<u8> {
+        let mut files = BTreeMap::new();
+        load_files_from_disk(&mut files, dir, dir);
+
+        let wasmer_toml = DirOrFile::File("wasmer.toml".into());
+        if let Some(toml_data) = files.remove(&wasmer_toml) {
+            // HACK(Michael-F-Bryan): The version of wapm-targz-to-pirita we are
+            // using doesn't know we renamed "wapm.toml" to "wasmer.toml", so we
+            // manually patch things up if people have already migrated their
+            // projects.
+            files
+                .entry(DirOrFile::File("wapm.toml".into()))
+                .or_insert(toml_data);
+        }
+
+        let functions = TransformManifestFunctions::default();
+        wapm_targz_to_pirita::generate_webc_file(files, dir, &functions).unwrap()
+    }
+
+    fn load_files_from_disk(files: &mut FileMap, dir: &Path, base: &Path) {
+        let entries = dir.read_dir().unwrap();
+
+        for entry in entries {
+            let path = entry.unwrap().path();
+            let relative_path = path.strip_prefix(base).unwrap().to_path_buf();
+
+            if path.is_dir() {
+                load_files_from_disk(files, &path, base);
+                files.insert(DirOrFile::Dir(relative_path), Vec::new());
+            } else if path.is_file() {
+                let data = std::fs::read(&path).unwrap();
+                files.insert(DirOrFile::File(relative_path), data);
+            }
+        }
+    }
+}

lib/wasix/src/runtime/package_loader/builtin_loader.rs

@@ -372,6 +372,7 @@ mod tests {
                 dependencies: Vec::new(),
                 commands: Vec::new(),
                 entrypoint: Some("asdf".to_string()),
+                filesystem: Vec::new(),
             },
             dist: DistributionInfo {
                 webc: "https://wapm.io/python/python".parse().unwrap(),