From 3b86439476d66c3047f1891d52ddbe6be90443db Mon Sep 17 00:00:00 2001 From: "M.Amin Rayej" Date: Fri, 7 Jun 2024 13:37:50 +0330 Subject: [PATCH 1/3] add support for spawning local wasm files --- lib/wasix/src/bin_factory/exec.rs | 16 ++- lib/wasix/src/bin_factory/mod.rs | 119 +++++++++++++++++++++- lib/wasix/src/syscalls/wasix/proc_exec.rs | 6 +- tests/wasix/cwd-to-home/expected | 1 - tests/wasix/cwd-to-home/run.sh | 2 +- tests/wasix/execv/main.c | 34 +++++++ tests/wasix/execv/run.sh | 5 + tests/wasix/test.sh | 3 +- 8 files changed, 176 insertions(+), 10 deletions(-) delete mode 100644 tests/wasix/cwd-to-home/expected create mode 100644 tests/wasix/execv/main.c create mode 100755 tests/wasix/execv/run.sh diff --git a/lib/wasix/src/bin_factory/exec.rs b/lib/wasix/src/bin_factory/exec.rs index fcdc7217259..39656bb1383 100644 --- a/lib/wasix/src/bin_factory/exec.rs +++ b/lib/wasix/src/bin_factory/exec.rs @@ -29,15 +29,25 @@ pub async fn spawn_exec( env: WasiEnv, runtime: &Arc, ) -> Result { + // Spawn union the file system + spawn_union_fs(&env, &binary).await?; + // Load the WASM let wasm = spawn_load_wasm(&env, &binary, name).await?; + spawn_exec_wasm(wasm, name, env, runtime).await +} + +#[tracing::instrument(level = "trace", skip_all, fields(%name))] +pub async fn spawn_exec_wasm( + wasm: &[u8], + name: &str, + env: WasiEnv, + runtime: &Arc, +) -> Result { // Load the module let module = spawn_load_module(&env, name, wasm, runtime).await?; - // Spawn union the file system - spawn_union_fs(&env, &binary).await?; - // Now run the module spawn_exec_module(module, env, runtime) } diff --git a/lib/wasix/src/bin_factory/mod.rs b/lib/wasix/src/bin_factory/mod.rs index 16311cd4fad..5b97a55c810 100644 --- a/lib/wasix/src/bin_factory/mod.rs +++ b/lib/wasix/src/bin_factory/mod.rs @@ -8,6 +8,7 @@ use std::{ }; use anyhow::Context; +use exec::spawn_exec_wasm; use virtual_fs::{AsyncReadExt, FileSystem}; use wasmer::FunctionEnvMut; use wasmer_wasix_types::wasi::Errno; @@ -52,8 +53,6 @@ impl BinFactory { cache.insert(name.to_string(), Some(binary)); } - // TODO: remove allow once BinFactory is refactored - // currently fine because a BinFactory is only used by a single process tree #[allow(clippy::await_holding_lock)] pub async fn get_binary( &self, @@ -145,6 +144,90 @@ impl BinFactory { } Err(SpawnError::BinaryNotFound { binary: name }) } + + pub fn spawn_executable<'a>( + &'a self, + name: String, + store: wasmer::Store, + env: WasiEnv, + ) -> Pin> + 'a>> { + Box::pin(async move { + // Find the binary (or die trying) and make the spawn type + let res = self + .get_executable(name.as_str(), Some(env.fs_root())) + .await + .ok_or_else(|| SpawnError::BinaryNotFound { + binary: name.clone(), + }); + if res.is_err() { + env.on_exit(Some(Errno::Noent.into())).await; + } + let executable = res?; + + // Execute + match executable { + Executable::Wasm(bytes) => { + spawn_exec_wasm(&bytes, name.as_str(), env, &self.runtime).await + } + Executable::BinaryPackage(pkg) => { + spawn_exec(pkg, name.as_str(), store, env, &self.runtime).await + } + } + }) + } + + // TODO: remove allow once BinFactory is refactored + // currently fine because a BinFactory is only used by a single process tree + #[allow(clippy::await_holding_lock)] + pub async fn get_executable( + &self, + name: &str, + fs: Option<&dyn FileSystem>, + ) -> Option { + let name = name.to_string(); + + // Fast path + { + let cache = self.local.read().unwrap(); + if let Some(data) = cache.get(&name) { + data.clone().map(Executable::BinaryPackage); + } + } + + // Slow path + let mut cache = self.local.write().unwrap(); + + // Check the cache + if let Some(data) = cache.get(&name) { + return data.clone().map(Executable::BinaryPackage); + } + + // Check the filesystem for the file + if name.starts_with('/') { + if let Some(fs) = fs { + match load_executable_from_filesystem(fs, name.as_ref(), self.runtime()).await { + Ok(executable) => { + if let Executable::BinaryPackage(pkg) = &executable { + cache.insert(name, Some(pkg.clone())); + } + + return Some(executable); + } + Err(e) => { + tracing::warn!( + path = name, + error = &*e, + "Unable to load the package from disk" + ); + } + } + } + } + + // NAK + cache.insert(name, None); + None + } } async fn load_package_from_filesystem( @@ -168,3 +251,35 @@ async fn load_package_from_filesystem( Ok(pkg) } + +pub enum Executable { + Wasm(bytes::Bytes), + BinaryPackage(BinaryPackage), +} + +async fn load_executable_from_filesystem( + fs: &dyn FileSystem, + path: &Path, + rt: &(dyn Runtime + Send + Sync), +) -> Result { + let mut f = fs + .new_open_options() + .read(true) + .open(path) + .context("Unable to open the file")?; + + let mut data = Vec::with_capacity(f.size() as usize); + f.read_to_end(&mut data).await.context("Read failed")?; + + let bytes: bytes::Bytes = data.into(); + + if let Ok(container) = Container::from_bytes(bytes.clone()) { + let pkg = BinaryPackage::from_webc(&container, rt) + .await + .context("Unable to load the package")?; + + Ok(Executable::BinaryPackage(pkg)) + } else { + Ok(Executable::Wasm(bytes)) + } +} diff --git a/lib/wasix/src/syscalls/wasix/proc_exec.rs b/lib/wasix/src/syscalls/wasix/proc_exec.rs index 8e2e71f0c51..6797d3338c3 100644 --- a/lib/wasix/src/syscalls/wasix/proc_exec.rs +++ b/lib/wasix/src/syscalls/wasix/proc_exec.rs @@ -117,7 +117,9 @@ pub fn proc_exec( let name_inner = name.clone(); __asyncify_light(ctx.data(), None, async { - let ret = bin_factory.spawn(name_inner, new_store, env).await; + let ret = bin_factory + .spawn_executable(name_inner, new_store, env) + .await; match ret { Ok(ret) => { trace!(%child_pid, "spawned sub-process"); @@ -213,7 +215,7 @@ pub fn proc_exec( let env = builder.take().unwrap(); // Spawn a new process with this current execution environment - InlineWaker::block_on(bin_factory.spawn(name, new_store, env)) + InlineWaker::block_on(bin_factory.spawn_executable(name, new_store, env)) } }; diff --git a/tests/wasix/cwd-to-home/expected b/tests/wasix/cwd-to-home/expected deleted file mode 100644 index c227083464f..00000000000 --- a/tests/wasix/cwd-to-home/expected +++ /dev/null @@ -1 +0,0 @@ -0 \ No newline at end of file diff --git a/tests/wasix/cwd-to-home/run.sh b/tests/wasix/cwd-to-home/run.sh index 474bd170e0d..e3f5a60846a 100755 --- a/tests/wasix/cwd-to-home/run.sh +++ b/tests/wasix/cwd-to-home/run.sh @@ -2,4 +2,4 @@ $WASMER -q run main.wasm --dir=. > output -diff -u output expected 1>/dev/null \ No newline at end of file +printf "0" | diff -u output - 1>/dev/null \ No newline at end of file diff --git a/tests/wasix/execv/main.c b/tests/wasix/execv/main.c new file mode 100644 index 00000000000..e430c0ba418 --- /dev/null +++ b/tests/wasix/execv/main.c @@ -0,0 +1,34 @@ +#include +#include +#include +#include + +int main(int argc, char *argv[]) +{ + if (argc > 1 && argv[1] != NULL) + { + return 0; + } + + pid_t pid = fork(); + if (pid == -1) + { + exit(EXIT_FAILURE); + } + else if (pid == 0) + { + char *newargv[] = {argv[0], "child", NULL}; + + execv("/code/main.wasm", newargv); + + exit(EXIT_FAILURE); + } + else + { + int status; + waitpid(pid, &status, 0); + printf("%d", status); + } + + return 0; +} \ No newline at end of file diff --git a/tests/wasix/execv/run.sh b/tests/wasix/execv/run.sh new file mode 100755 index 00000000000..2f9d5723c8d --- /dev/null +++ b/tests/wasix/execv/run.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +$WASMER -q run main.wasm --mapdir=/code:. > output + +printf "0" | diff -u output - 1>/dev/null \ No newline at end of file diff --git a/tests/wasix/test.sh b/tests/wasix/test.sh index f3349e46c4a..1d85eba2b1f 100755 --- a/tests/wasix/test.sh +++ b/tests/wasix/test.sh @@ -77,6 +77,7 @@ while read dir; do cmd="cd $dir; \ $CC $CFLAGS $LDFLAGS -o main.wasm main.c; \ + wasm-opt -O4 --asyncify -g main.wasm -o main.wasm; \ ./run.sh" if bash -c "$cmd"; then @@ -85,6 +86,6 @@ while read dir; do printf "\rTesting $dir ❌\n" status=1 fi -done < <(find . -mindepth 1 -maxdepth 1 -type d) +done < <(find . -mindepth 1 -maxdepth 1 -type d | sort) exit $status \ No newline at end of file From 812d962484c1cfac935a2a0f906d8522a87d683f Mon Sep 17 00:00:00 2001 From: "M.Amin Rayej" Date: Fri, 7 Jun 2024 14:57:00 +0330 Subject: [PATCH 2/3] refactor and deduplicate code --- lib/wasix/src/bin_factory/exec.rs | 4 -- lib/wasix/src/bin_factory/mod.rs | 68 +++---------------------------- 2 files changed, 6 insertions(+), 66 deletions(-) diff --git a/lib/wasix/src/bin_factory/exec.rs b/lib/wasix/src/bin_factory/exec.rs index 39656bb1383..2dd675796f3 100644 --- a/lib/wasix/src/bin_factory/exec.rs +++ b/lib/wasix/src/bin_factory/exec.rs @@ -29,10 +29,8 @@ pub async fn spawn_exec( env: WasiEnv, runtime: &Arc, ) -> Result { - // Spawn union the file system spawn_union_fs(&env, &binary).await?; - // Load the WASM let wasm = spawn_load_wasm(&env, &binary, name).await?; spawn_exec_wasm(wasm, name, env, runtime).await @@ -45,10 +43,8 @@ pub async fn spawn_exec_wasm( env: WasiEnv, runtime: &Arc, ) -> Result { - // Load the module let module = spawn_load_module(&env, name, wasm, runtime).await?; - // Now run the module spawn_exec_module(module, env, runtime) } diff --git a/lib/wasix/src/bin_factory/mod.rs b/lib/wasix/src/bin_factory/mod.rs index 5b97a55c810..e99bb1a7be7 100644 --- a/lib/wasix/src/bin_factory/mod.rs +++ b/lib/wasix/src/bin_factory/mod.rs @@ -59,46 +59,12 @@ impl BinFactory { name: &str, fs: Option<&dyn FileSystem>, ) -> Option { - let name = name.to_string(); - - // Fast path - { - let cache = self.local.read().unwrap(); - if let Some(data) = cache.get(&name) { - return data.clone(); - } - } - - // Slow path - let mut cache = self.local.write().unwrap(); - - // Check the cache - if let Some(data) = cache.get(&name) { - return data.clone(); - } - - // Check the filesystem for the file - if name.starts_with('/') { - if let Some(fs) = fs { - match load_package_from_filesystem(fs, name.as_ref(), self.runtime()).await { - Ok(pkg) => { - cache.insert(name, Some(pkg.clone())); - return Some(pkg); - } - Err(e) => { - tracing::warn!( - path = name, - error = &*e, - "Unable to load the package from disk" - ); - } - } - } - } - - // NAK - cache.insert(name, None); - None + self.get_executable(name, fs) + .await + .and_then(|executable| match executable { + Executable::Wasm(_) => None, + Executable::BinaryPackage(pkg) => Some(pkg), + }) } pub fn spawn<'a>( @@ -230,28 +196,6 @@ impl BinFactory { } } -async fn load_package_from_filesystem( - fs: &dyn FileSystem, - path: &Path, - rt: &(dyn Runtime + Send + Sync), -) -> Result { - let mut f = fs - .new_open_options() - .read(true) - .open(path) - .context("Unable to open the file")?; - - let mut data = Vec::with_capacity(f.size() as usize); - f.read_to_end(&mut data).await.context("Read failed")?; - - let container = Container::from_bytes(data).context("Unable to parse the WEBC file")?; - let pkg = BinaryPackage::from_webc(&container, rt) - .await - .context("Unable to load the package")?; - - Ok(pkg) -} - pub enum Executable { Wasm(bytes::Bytes), BinaryPackage(BinaryPackage), From 5e4df24c125bde7eed6e9716bef29a569c00b7d0 Mon Sep 17 00:00:00 2001 From: "M.Amin Rayej" Date: Fri, 7 Jun 2024 15:00:46 +0330 Subject: [PATCH 3/3] more refactor --- lib/wasix/src/bin_factory/mod.rs | 44 ++++++----------------- lib/wasix/src/syscalls/wasix/proc_exec.rs | 6 ++-- 2 files changed, 12 insertions(+), 38 deletions(-) diff --git a/lib/wasix/src/bin_factory/mod.rs b/lib/wasix/src/bin_factory/mod.rs index e99bb1a7be7..0cb8cee8e0d 100644 --- a/lib/wasix/src/bin_factory/mod.rs +++ b/lib/wasix/src/bin_factory/mod.rs @@ -76,7 +76,7 @@ impl BinFactory { Box::pin(async move { // Find the binary (or die trying) and make the spawn type let res = self - .get_binary(name.as_str(), Some(env.fs_root())) + .get_executable(name.as_str(), Some(env.fs_root())) .await .ok_or_else(|| SpawnError::BinaryNotFound { binary: name.clone(), @@ -84,10 +84,17 @@ impl BinFactory { if res.is_err() { env.on_exit(Some(Errno::Noent.into())).await; } - let binary = res?; + let executable = res?; // Execute - spawn_exec(binary, name.as_str(), store, env, &self.runtime).await + match executable { + Executable::Wasm(bytes) => { + spawn_exec_wasm(&bytes, name.as_str(), env, &self.runtime).await + } + Executable::BinaryPackage(pkg) => { + spawn_exec(pkg, name.as_str(), store, env, &self.runtime).await + } + } }) } @@ -111,37 +118,6 @@ impl BinFactory { Err(SpawnError::BinaryNotFound { binary: name }) } - pub fn spawn_executable<'a>( - &'a self, - name: String, - store: wasmer::Store, - env: WasiEnv, - ) -> Pin> + 'a>> { - Box::pin(async move { - // Find the binary (or die trying) and make the spawn type - let res = self - .get_executable(name.as_str(), Some(env.fs_root())) - .await - .ok_or_else(|| SpawnError::BinaryNotFound { - binary: name.clone(), - }); - if res.is_err() { - env.on_exit(Some(Errno::Noent.into())).await; - } - let executable = res?; - - // Execute - match executable { - Executable::Wasm(bytes) => { - spawn_exec_wasm(&bytes, name.as_str(), env, &self.runtime).await - } - Executable::BinaryPackage(pkg) => { - spawn_exec(pkg, name.as_str(), store, env, &self.runtime).await - } - } - }) - } - // TODO: remove allow once BinFactory is refactored // currently fine because a BinFactory is only used by a single process tree #[allow(clippy::await_holding_lock)] diff --git a/lib/wasix/src/syscalls/wasix/proc_exec.rs b/lib/wasix/src/syscalls/wasix/proc_exec.rs index 6797d3338c3..8e2e71f0c51 100644 --- a/lib/wasix/src/syscalls/wasix/proc_exec.rs +++ b/lib/wasix/src/syscalls/wasix/proc_exec.rs @@ -117,9 +117,7 @@ pub fn proc_exec( let name_inner = name.clone(); __asyncify_light(ctx.data(), None, async { - let ret = bin_factory - .spawn_executable(name_inner, new_store, env) - .await; + let ret = bin_factory.spawn(name_inner, new_store, env).await; match ret { Ok(ret) => { trace!(%child_pid, "spawned sub-process"); @@ -215,7 +213,7 @@ pub fn proc_exec( let env = builder.take().unwrap(); // Spawn a new process with this current execution environment - InlineWaker::block_on(bin_factory.spawn_executable(name, new_store, env)) + InlineWaker::block_on(bin_factory.spawn(name, new_store, env)) } };