From ef64bd386cef97e59da53897a6a9e8d8afe3e83a Mon Sep 17 00:00:00 2001 From: Arshia Ghafoori Date: Wed, 31 Jul 2024 20:34:02 +0400 Subject: [PATCH 1/4] Mount [fs] entries as additional mapped directories when running a local package --- Cargo.lock | 1 + lib/cli/src/commands/auth/login/mod.rs | 3 +++ lib/cli/src/commands/package/download.rs | 2 -- lib/cli/src/commands/run/mod.rs | 13 ++++++++-- lib/wasix/Cargo.toml | 1 + lib/wasix/src/bin_factory/binary_package.rs | 24 +++++++++++++++---- .../runtime/package_loader/builtin_loader.rs | 3 ++- .../package_loader/load_package_tree.rs | 20 +++++++++++++--- lib/wasix/src/runtime/package_loader/types.rs | 6 ++++- .../src/runtime/package_loader/unsupported.rs | 1 + 10 files changed, 60 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 64d702f0645..33610f2c0dd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7153,6 +7153,7 @@ dependencies = [ "thiserror", "tokio 1.38.1", "tokio-stream", + "toml 0.8.15", "tower", "tower-http", "tracing", diff --git a/lib/cli/src/commands/auth/login/mod.rs b/lib/cli/src/commands/auth/login/mod.rs index 0c3a2bf8dee..a8795c132d8 100644 --- a/lib/cli/src/commands/auth/login/mod.rs +++ b/lib/cli/src/commands/auth/login/mod.rs @@ -181,6 +181,9 @@ impl Login { } #[cfg(test)] { + // prevent unused binding warning + _ = user; + false } } else { diff --git a/lib/cli/src/commands/package/download.rs b/lib/cli/src/commands/package/download.rs index 8a9a7a48ea9..c4a79e24611 100644 --- a/lib/cli/src/commands/package/download.rs +++ b/lib/cli/src/commands/package/download.rs @@ -279,8 +279,6 @@ impl PackageDownload { #[cfg(test)] mod tests { - use crate::config::UserRegistry; - use super::*; /// Download a package from the dev registry. diff --git a/lib/cli/src/commands/run/mod.rs b/lib/cli/src/commands/run/mod.rs index b4193837a40..cd6bb370c3f 100644 --- a/lib/cli/src/commands/run/mod.rs +++ b/lib/cli/src/commands/run/mod.rs @@ -98,7 +98,7 @@ impl Run { } #[tracing::instrument(level = "debug", name = "wasmer_run", skip_all)] - fn execute_inner(self, output: Output) -> Result<(), Error> { + fn execute_inner(mut self, output: Output) -> Result<(), Error> { let pb = ProgressBar::new_spinner(); pb.set_draw_target(output.draw_target()); pb.enable_steady_tick(TICK); @@ -152,6 +152,12 @@ impl Run { let target = self.input.resolve_target(&monitoring_runtime, &pb)?; + if let ExecutableTarget::Package(ref pkg) = target { + self.wasi + .mapped_dirs + .extend(pkg.additional_host_mapped_directories.clone()); + } + pb.finish_and_clear(); // push the TTY state so we can restore it after the program finishes @@ -1000,7 +1006,10 @@ impl wasmer_wasix::runtime::package_loader::PackageLoader for MonitoringPackageL &self, root: &Container, resolution: &wasmer_wasix::runtime::resolver::Resolution, + root_is_local_dir: bool, ) -> Result { - self.inner.load_package_tree(root, resolution).await + self.inner + .load_package_tree(root, resolution, root_is_local_dir) + .await } } diff --git a/lib/wasix/Cargo.toml b/lib/wasix/Cargo.toml index ff1da6ed517..ccd0447a264 100644 --- a/lib/wasix/Cargo.toml +++ b/lib/wasix/Cargo.toml @@ -103,6 +103,7 @@ web-sys = { version = "0.3.64", features = [ ahash = "0.8.11" hyper-util = { version = "0.1.5", features = ["server", "server-graceful", "tokio", "service", "client"], optional = true } http-body-util = { version="0.1.1", optional = true } +toml = "0.8" [target.'cfg(not(target_arch = "riscv64"))'.dependencies.reqwest] workspace = true diff --git a/lib/wasix/src/bin_factory/binary_package.rs b/lib/wasix/src/bin_factory/binary_package.rs index 876e60b8687..a189bbd396d 100644 --- a/lib/wasix/src/bin_factory/binary_package.rs +++ b/lib/wasix/src/bin_factory/binary_package.rs @@ -9,6 +9,7 @@ use wasmer_config::package::{PackageHash, PackageId, PackageSource}; use webc::{compat::SharedBytes, Container}; use crate::{ + runners::MappedDirectory, runtime::resolver::{PackageInfo, ResolveError}, Runtime, }; @@ -75,6 +76,8 @@ pub struct BinaryPackage { pub commands: Vec, pub uses: Vec, pub file_system_memory_footprint: u64, + + pub additional_host_mapped_directories: Vec, } impl BinaryPackage { @@ -91,7 +94,7 @@ impl BinaryPackage { let id = PackageId::Hash(PackageHash::from_sha256_bytes(hash)); let manifest_path = dir.join("wasmer.toml"); - let webc = webc::wasmer_package::Package::from_manifest(manifest_path)?; + let webc = webc::wasmer_package::Package::from_manifest(&manifest_path)?; let container = Container::from(webc); let manifest = container.manifest(); @@ -99,12 +102,23 @@ impl BinaryPackage { let root_id = root.id.clone(); let resolution = crate::runtime::resolver::resolve(&root_id, &root, &*source).await?; - let pkg = rt + let mut pkg = rt .package_loader() - .load_package_tree(&container, &resolution) + .load_package_tree(&container, &resolution, true) .await .map_err(|e| anyhow::anyhow!(e))?; + // HACK: webc has no way to return its deserialized manifest to us, so we need to do it again here + // We already read and parsed the manifest once, so it'll succeed again. Unwrapping is safe at this point. + let wasmer_toml = std::fs::read_to_string(&manifest_path).unwrap(); + let wasmer_toml: wasmer_config::package::Manifest = toml::from_str(&wasmer_toml).unwrap(); + pkg.additional_host_mapped_directories.extend( + wasmer_toml + .fs + .into_iter() + .map(|(guest, host)| MappedDirectory { host, guest }), + ); + Ok(pkg) } @@ -132,7 +146,7 @@ impl BinaryPackage { let resolution = crate::runtime::resolver::resolve(&root_id, &root, &*source).await?; let pkg = rt .package_loader() - .load_package_tree(container, &resolution) + .load_package_tree(container, &resolution, false) .await .map_err(|e| anyhow::anyhow!(e))?; @@ -162,7 +176,7 @@ impl BinaryPackage { .context("Dependency resolution failed")?; let pkg = runtime .package_loader() - .load_package_tree(&root, &resolution) + .load_package_tree(&root, &resolution, false) .await .map_err(|e| anyhow::anyhow!(e))?; diff --git a/lib/wasix/src/runtime/package_loader/builtin_loader.rs b/lib/wasix/src/runtime/package_loader/builtin_loader.rs index 521b10a89a3..cf38f7c0f9d 100644 --- a/lib/wasix/src/runtime/package_loader/builtin_loader.rs +++ b/lib/wasix/src/runtime/package_loader/builtin_loader.rs @@ -262,8 +262,9 @@ impl PackageLoader for BuiltinPackageLoader { &self, root: &Container, resolution: &Resolution, + root_is_local_dir: bool, ) -> Result { - super::load_package_tree(root, self, resolution).await + super::load_package_tree(root, self, resolution, root_is_local_dir).await } } diff --git a/lib/wasix/src/runtime/package_loader/load_package_tree.rs b/lib/wasix/src/runtime/package_loader/load_package_tree.rs index 0b1f3a82e4e..f635b2b35c6 100644 --- a/lib/wasix/src/runtime/package_loader/load_package_tree.rs +++ b/lib/wasix/src/runtime/package_loader/load_package_tree.rs @@ -36,10 +36,11 @@ pub async fn load_package_tree( root: &Container, loader: &dyn PackageLoader, resolution: &Resolution, + root_is_local_dir: bool, ) -> Result { let mut containers = fetch_dependencies(loader, &resolution.package, &resolution.graph).await?; containers.insert(resolution.package.root_package.clone(), root.clone()); - let fs = filesystem(&containers, &resolution.package)?; + let fs = filesystem(&containers, &resolution.package, root_is_local_dir)?; let root = &resolution.package.root_package; let commands: Vec = @@ -61,6 +62,8 @@ pub async fn load_package_tree( commands, uses: Vec::new(), file_system_memory_footprint, + + additional_host_mapped_directories: vec![], }; Ok(loaded) @@ -316,6 +319,7 @@ fn count_file_system(fs: &dyn FileSystem, path: &Path) -> u64 { fn filesystem( packages: &HashMap, pkg: &ResolvedPackage, + root_is_local_dir: bool, ) -> Result, Error> { if pkg.filesystem.is_empty() { return Ok(Box::new(OverlayFileSystem::< @@ -342,9 +346,9 @@ fn filesystem( } if found_v3 && !found_v2 { - filesystem_v3(packages, pkg) + filesystem_v3(packages, pkg, root_is_local_dir) } else { - filesystem_v2(packages, pkg) + filesystem_v2(packages, pkg, root_is_local_dir) } } @@ -352,6 +356,7 @@ fn filesystem( fn filesystem_v3( packages: &HashMap, pkg: &ResolvedPackage, + root_is_local_dir: bool, ) -> Result, Error> { let mut volumes: HashMap<&PackageId, BTreeMap> = HashMap::new(); @@ -367,6 +372,10 @@ fn filesystem_v3( .. } in &pkg.filesystem { + if *package == pkg.root_package && root_is_local_dir { + continue; + } + // Note: We want to reuse existing Volume instances if we can. That way // we can keep the memory usage down. A webc::compat::Volume is // reference-counted, anyway. @@ -427,6 +436,7 @@ fn filesystem_v3( fn filesystem_v2( packages: &HashMap, pkg: &ResolvedPackage, + root_is_local_dir: bool, ) -> Result, Error> { let mut filesystems = Vec::new(); let mut volumes: HashMap<&PackageId, BTreeMap> = HashMap::new(); @@ -441,6 +451,10 @@ fn filesystem_v2( original_path, } in &pkg.filesystem { + if *package == pkg.root_package && root_is_local_dir { + continue; + } + // Note: We want to reuse existing Volume instances if we can. That way // we can keep the memory usage down. A webc::compat::Volume is // reference-counted, anyway. diff --git a/lib/wasix/src/runtime/package_loader/types.rs b/lib/wasix/src/runtime/package_loader/types.rs index afb3e924828..2147f8402c6 100644 --- a/lib/wasix/src/runtime/package_loader/types.rs +++ b/lib/wasix/src/runtime/package_loader/types.rs @@ -20,6 +20,7 @@ pub trait PackageLoader: Send + Sync + Debug { &self, root: &Container, resolution: &Resolution, + root_is_local_dir: bool, ) -> Result; } @@ -37,7 +38,10 @@ where &self, root: &Container, resolution: &Resolution, + root_is_local_dir: bool, ) -> Result { - (**self).load_package_tree(root, resolution).await + (**self) + .load_package_tree(root, resolution, root_is_local_dir) + .await } } diff --git a/lib/wasix/src/runtime/package_loader/unsupported.rs b/lib/wasix/src/runtime/package_loader/unsupported.rs index be5488b1edb..c0c245ff216 100644 --- a/lib/wasix/src/runtime/package_loader/unsupported.rs +++ b/lib/wasix/src/runtime/package_loader/unsupported.rs @@ -23,6 +23,7 @@ impl PackageLoader for UnsupportedPackageLoader { &self, _root: &Container, _resolution: &Resolution, + _root_is_local_dir: bool, ) -> Result { Err(Error::new(Unsupported)) } From 55529107d7a90a23319cc0596bd4fc6059de1b8f Mon Sep 17 00:00:00 2001 From: Arshia Ghafoori Date: Thu, 1 Aug 2024 12:20:13 +0400 Subject: [PATCH 2/4] Fix "fs_table_can_map_directories_to_different_names" test --- lib/wasix/src/bin_factory/binary_package.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/wasix/src/bin_factory/binary_package.rs b/lib/wasix/src/bin_factory/binary_package.rs index a189bbd396d..9a91e52f476 100644 --- a/lib/wasix/src/bin_factory/binary_package.rs +++ b/lib/wasix/src/bin_factory/binary_package.rs @@ -269,7 +269,12 @@ mod tests { .with_shared_http_client(runtime.http_client().unwrap().clone()), ); - let pkg = BinaryPackage::from_dir(temp.path(), &runtime) + let pkg = webc::wasmer_package::Package::from_manifest(&manifest).unwrap(); + let data = pkg.serialize().unwrap(); + let webc_path = temp.path().join("package.webc"); + std::fs::write(&webc_path, data).unwrap(); + + let pkg = BinaryPackage::from_webc(&Container::from_disk(&webc_path).unwrap(), &runtime) .await .unwrap(); From c89ce17de13edcb80f6edb418ffbce1f7c7784d5 Mon Sep 17 00:00:00 2001 From: Arshia Ghafoori Date: Fri, 2 Aug 2024 11:50:20 +0330 Subject: [PATCH 3/4] Fix host volumes using the wrong host path --- lib/wasix/src/bin_factory/binary_package.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/wasix/src/bin_factory/binary_package.rs b/lib/wasix/src/bin_factory/binary_package.rs index 9a91e52f476..cf3548f6e7a 100644 --- a/lib/wasix/src/bin_factory/binary_package.rs +++ b/lib/wasix/src/bin_factory/binary_package.rs @@ -116,7 +116,14 @@ impl BinaryPackage { wasmer_toml .fs .into_iter() - .map(|(guest, host)| MappedDirectory { host, guest }), + .map(|(guest, host)| { + anyhow::Ok(MappedDirectory { + host: dir.join(host).canonicalize()?, + guest, + }) + }) + .collect::, _>>()? + .into_iter(), ); Ok(pkg) From 7c475b2a14c0b73cb0deb4b1d9a0211621ffc174 Mon Sep 17 00:00:00 2001 From: Arshia Ghafoori Date: Fri, 2 Aug 2024 11:50:34 +0330 Subject: [PATCH 4/4] Add integration test for local packages with host volumes --- tests/integration/cli/tests/run.rs | 49 +++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/tests/integration/cli/tests/run.rs b/tests/integration/cli/tests/run.rs index 117f193f17d..38000ac017b 100644 --- a/tests/integration/cli/tests/run.rs +++ b/tests/integration/cli/tests/run.rs @@ -188,7 +188,7 @@ fn test_wasmer_run_works_with_dir() { let temp_dir = tempfile::TempDir::new().unwrap(); let qjs_path = temp_dir.path().join("qjs.wasm"); - std::fs::copy(fixtures::qjs(), &qjs_path).unwrap(); + std::fs::copy(fixtures::qjs(), qjs_path).unwrap(); std::fs::copy( fixtures::qjs_wasmer_toml(), temp_dir.path().join("wasmer.toml"), @@ -945,6 +945,53 @@ fn run_a_package_that_uses_an_atom_from_a_dependency() { assert.success().stdout(contains("Hello, World!")); } +#[test] +fn local_package_has_write_access_to_its_volumes() { + let temp = tempfile::tempdir().unwrap(); + + std::fs::write( + temp.path().join("wasmer.toml"), + r#" +[dependencies] +"python/python" = "*" + +[fs] +"/mounted" = "." + +[[command]] +name = "run" +module = "python/python:python" +runner = "wasi" + +[command.annotations.wasi] +main-args = ["/mounted/script.py"] + + "#, + ) + .unwrap(); + + std::fs::write( + temp.path().join("script.py"), + r#" +file = open("/mounted/hello.txt", "w") +file.write("Hello, world!") + "#, + ) + .unwrap(); + + Command::new(get_wasmer_path()) + .arg("run") + .arg(temp.path()) + .arg("--registry=wasmer.io") + .env("RUST_LOG", &*RUST_LOG) + .assert() + .success(); + + let file_contents = + String::from_utf8(std::fs::read(temp.path().join("hello.txt")).unwrap()).unwrap(); + assert_eq!(file_contents, "Hello, world!"); +} + fn project_root() -> &'static Path { Path::new(env!("CARGO_MANIFEST_DIR")) .ancestors()