Merge pull request #4982 from wasmerio/feat/host-backed-webc-take-3

Mount [fs] entries as additional mapped directories when running a local package

lib/cli/src/commands/auth/login/mod.rs

@@ -181,7 +181,9 @@ impl Login {
             }
             #[cfg(test)]
             {
+                // prevent unused binding warning
                 _ = user;
+
                 false
             }
         } else {

lib/cli/src/commands/run/mod.rs

@@ -99,7 +99,7 @@ impl Run {
     }
 
     #[tracing::instrument(level = "debug", name = "wasmer_run", skip_all)]
-    fn execute_inner(self, output: Output) -> Result<(), Error> {
+    fn execute_inner(mut self, output: Output) -> Result<(), Error> {
         let pb = ProgressBar::new_spinner();
         pb.set_draw_target(output.draw_target());
         pb.enable_steady_tick(TICK);
@@ -157,6 +157,12 @@ impl Run {
 
         let target = self.input.resolve_target(&monitoring_runtime, &pb)?;
 
+        if let ExecutableTarget::Package(ref pkg) = target {
+            self.wasi
+                .mapped_dirs
+                .extend(pkg.additional_host_mapped_directories.clone());
+        }
+
         pb.finish_and_clear();
 
         // push the TTY state so we can restore it after the program finishes
@@ -1005,7 +1011,10 @@ impl wasmer_wasix::runtime::package_loader::PackageLoader for MonitoringPackageL
         &self,
         root: &Container,
         resolution: &wasmer_wasix::runtime::resolver::Resolution,
+        root_is_local_dir: bool,
     ) -> Result<BinaryPackage, Error> {
-        self.inner.load_package_tree(root, resolution).await
+        self.inner
+            .load_package_tree(root, resolution, root_is_local_dir)
+            .await
     }
 }

lib/wasix/Cargo.toml

@@ -103,6 +103,7 @@ web-sys = { version = "0.3.64", features = [
 ahash = "0.8.11"
 hyper-util = { version = "0.1.5", features = ["server", "server-graceful", "tokio", "service", "client"], optional = true }
 http-body-util = { version="0.1.1", optional = true }
+toml = "0.8"
 
 [target.'cfg(not(target_arch = "riscv64"))'.dependencies.reqwest]
 workspace = true

lib/wasix/src/bin_factory/binary_package.rs

@@ -9,6 +9,7 @@ use wasmer_config::package::{PackageHash, PackageId, PackageSource};
 use webc::{compat::SharedBytes, Container};
 
 use crate::{
+    runners::MappedDirectory,
     runtime::resolver::{PackageInfo, ResolveError},
     Runtime,
 };
@@ -75,6 +76,8 @@ pub struct BinaryPackage {
     pub commands: Vec<BinaryPackageCommand>,
     pub uses: Vec<String>,
     pub file_system_memory_footprint: u64,
+
+    pub additional_host_mapped_directories: Vec<MappedDirectory>,
 }
 
 impl BinaryPackage {
@@ -91,20 +94,38 @@ impl BinaryPackage {
         let id = PackageId::Hash(PackageHash::from_sha256_bytes(hash));
 
         let manifest_path = dir.join("wasmer.toml");
-        let webc = webc::wasmer_package::Package::from_manifest(manifest_path)?;
+        let webc = webc::wasmer_package::Package::from_manifest(&manifest_path)?;
         let container = Container::from(webc);
         let manifest = container.manifest();
 
         let root = PackageInfo::from_manifest(id, manifest, container.version())?;
         let root_id = root.id.clone();
 
         let resolution = crate::runtime::resolver::resolve(&root_id, &root, &*source).await?;
-        let pkg = rt
+        let mut pkg = rt
             .package_loader()
-            .load_package_tree(&container, &resolution)
+            .load_package_tree(&container, &resolution, true)
             .await
             .map_err(|e| anyhow::anyhow!(e))?;
 
+        // HACK: webc has no way to return its deserialized manifest to us, so we need to do it again here
+        // We already read and parsed the manifest once, so it'll succeed again. Unwrapping is safe at this point.
+        let wasmer_toml = std::fs::read_to_string(&manifest_path).unwrap();
+        let wasmer_toml: wasmer_config::package::Manifest = toml::from_str(&wasmer_toml).unwrap();
+        pkg.additional_host_mapped_directories.extend(
+            wasmer_toml
+                .fs
+                .into_iter()
+                .map(|(guest, host)| {
+                    anyhow::Ok(MappedDirectory {
+                        host: dir.join(host).canonicalize()?,
+                        guest,
+                    })
+                })
+                .collect::<Result<Vec<_>, _>>()?
+                .into_iter(),
+        );
+
         Ok(pkg)
     }
 
@@ -132,7 +153,7 @@ impl BinaryPackage {
         let resolution = crate::runtime::resolver::resolve(&root_id, &root, &*source).await?;
         let pkg = rt
             .package_loader()
-            .load_package_tree(container, &resolution)
+            .load_package_tree(container, &resolution, false)
             .await
             .map_err(|e| anyhow::anyhow!(e))?;
 
@@ -162,7 +183,7 @@ impl BinaryPackage {
             .context("Dependency resolution failed")?;
         let pkg = runtime
             .package_loader()
-            .load_package_tree(&root, &resolution)
+            .load_package_tree(&root, &resolution, false)
             .await
             .map_err(|e| anyhow::anyhow!(e))?;
 
@@ -255,7 +276,12 @@ mod tests {
                 .with_shared_http_client(runtime.http_client().unwrap().clone()),
         );
 
-        let pkg = BinaryPackage::from_dir(temp.path(), &runtime)
+        let pkg = webc::wasmer_package::Package::from_manifest(&manifest).unwrap();
+        let data = pkg.serialize().unwrap();
+        let webc_path = temp.path().join("package.webc");
+        std::fs::write(&webc_path, data).unwrap();
+
+        let pkg = BinaryPackage::from_webc(&Container::from_disk(&webc_path).unwrap(), &runtime)
             .await
             .unwrap();
 

lib/wasix/src/runtime/package_loader/builtin_loader.rs

@@ -362,8 +362,9 @@ impl PackageLoader for BuiltinPackageLoader {
         &self,
         root: &Container,
         resolution: &Resolution,
+        root_is_local_dir: bool,
     ) -> Result<BinaryPackage, Error> {
-        super::load_package_tree(root, self, resolution).await
+        super::load_package_tree(root, self, resolution, root_is_local_dir).await
     }
 }
 

lib/wasix/src/runtime/package_loader/load_package_tree.rs

@@ -36,10 +36,11 @@ pub async fn load_package_tree(
     root: &Container,
     loader: &dyn PackageLoader,
     resolution: &Resolution,
+    root_is_local_dir: bool,
 ) -> Result<BinaryPackage, Error> {
     let mut containers = fetch_dependencies(loader, &resolution.package, &resolution.graph).await?;
     containers.insert(resolution.package.root_package.clone(), root.clone());
-    let fs = filesystem(&containers, &resolution.package)?;
+    let fs = filesystem(&containers, &resolution.package, root_is_local_dir)?;
 
     let root = &resolution.package.root_package;
     let commands: Vec<BinaryPackageCommand> =
@@ -61,6 +62,8 @@ pub async fn load_package_tree(
         commands,
         uses: Vec::new(),
         file_system_memory_footprint,
+
+        additional_host_mapped_directories: vec![],
     };
 
     Ok(loaded)
@@ -316,6 +319,7 @@ fn count_file_system(fs: &dyn FileSystem, path: &Path) -> u64 {
 fn filesystem(
     packages: &HashMap<PackageId, Container>,
     pkg: &ResolvedPackage,
+    root_is_local_dir: bool,
 ) -> Result<Box<dyn FileSystem + Send + Sync>, Error> {
     if pkg.filesystem.is_empty() {
         return Ok(Box::new(OverlayFileSystem::<
@@ -342,16 +346,17 @@ fn filesystem(
     }
 
     if found_v3 && !found_v2 {
-        filesystem_v3(packages, pkg)
+        filesystem_v3(packages, pkg, root_is_local_dir)
     } else {
-        filesystem_v2(packages, pkg)
+        filesystem_v2(packages, pkg, root_is_local_dir)
     }
 }
 
 /// Build the filesystem for webc v3 packages.
 fn filesystem_v3(
     packages: &HashMap<PackageId, Container>,
     pkg: &ResolvedPackage,
+    root_is_local_dir: bool,
 ) -> Result<Box<dyn FileSystem + Send + Sync>, Error> {
     let mut volumes: HashMap<&PackageId, BTreeMap<String, Volume>> = HashMap::new();
 
@@ -367,6 +372,10 @@ fn filesystem_v3(
         ..
     } in &pkg.filesystem
     {
+        if *package == pkg.root_package && root_is_local_dir {
+            continue;
+        }
+
         // Note: We want to reuse existing Volume instances if we can. That way
         // we can keep the memory usage down. A webc::compat::Volume is
         // reference-counted, anyway.
@@ -427,6 +436,7 @@ fn filesystem_v3(
 fn filesystem_v2(
     packages: &HashMap<PackageId, Container>,
     pkg: &ResolvedPackage,
+    root_is_local_dir: bool,
 ) -> Result<Box<dyn FileSystem + Send + Sync>, Error> {
     let mut filesystems = Vec::new();
     let mut volumes: HashMap<&PackageId, BTreeMap<String, Volume>> = HashMap::new();
@@ -441,6 +451,10 @@ fn filesystem_v2(
         original_path,
     } in &pkg.filesystem
     {
+        if *package == pkg.root_package && root_is_local_dir {
+            continue;
+        }
+
         // Note: We want to reuse existing Volume instances if we can. That way
         // we can keep the memory usage down. A webc::compat::Volume is
         // reference-counted, anyway.

lib/wasix/src/runtime/package_loader/types.rs

@@ -20,6 +20,7 @@ pub trait PackageLoader: Send + Sync + Debug {
         &self,
         root: &Container,
         resolution: &Resolution,
+        root_is_local_dir: bool,
     ) -> Result<BinaryPackage, Error>;
 }
 
@@ -37,7 +38,10 @@ where
         &self,
         root: &Container,
         resolution: &Resolution,
+        root_is_local_dir: bool,
     ) -> Result<BinaryPackage, Error> {
-        (**self).load_package_tree(root, resolution).await
+        (**self)
+            .load_package_tree(root, resolution, root_is_local_dir)
+            .await
     }
 }

lib/wasix/src/runtime/package_loader/unsupported.rs

@@ -23,6 +23,7 @@ impl PackageLoader for UnsupportedPackageLoader {
         &self,
         _root: &Container,
         _resolution: &Resolution,
+        _root_is_local_dir: bool,
     ) -> Result<BinaryPackage, Error> {
         Err(Error::new(Unsupported))
     }

tests/integration/cli/tests/run.rs

@@ -188,7 +188,7 @@ fn test_wasmer_run_works_with_dir() {
     let temp_dir = tempfile::TempDir::new().unwrap();
     let qjs_path = temp_dir.path().join("qjs.wasm");
 
-    std::fs::copy(fixtures::qjs(), &qjs_path).unwrap();
+    std::fs::copy(fixtures::qjs(), qjs_path).unwrap();
     std::fs::copy(
         fixtures::qjs_wasmer_toml(),
         temp_dir.path().join("wasmer.toml"),
@@ -945,6 +945,53 @@ fn run_a_package_that_uses_an_atom_from_a_dependency() {
     assert.success().stdout(contains("Hello, World!"));
 }
 
+#[test]
+fn local_package_has_write_access_to_its_volumes() {
+    let temp = tempfile::tempdir().unwrap();
+
+    std::fs::write(
+        temp.path().join("wasmer.toml"),
+        r#"
+[dependencies]
+"python/python" = "*"
+
+[fs]
+"/mounted" = "."
+
+[[command]]
+name = "run"
+module = "python/python:python"
+runner = "wasi"
+
+[command.annotations.wasi]
+main-args = ["/mounted/script.py"]
+
+        "#,
+    )
+    .unwrap();
+
+    std::fs::write(
+        temp.path().join("script.py"),
+        r#"
+file = open("/mounted/hello.txt", "w")
+file.write("Hello, world!")
+        "#,
+    )
+    .unwrap();
+
+    Command::new(get_wasmer_path())
+        .arg("run")
+        .arg(temp.path())
+        .arg("--registry=wasmer.io")
+        .env("RUST_LOG", &*RUST_LOG)
+        .assert()
+        .success();
+
+    let file_contents =
+        String::from_utf8(std::fs::read(temp.path().join("hello.txt")).unwrap()).unwrap();
+    assert_eq!(file_contents, "Hello, world!");
+}
+
 fn project_root() -> &'static Path {
     Path::new(env!("CARGO_MANIFEST_DIR"))
         .ancestors()