Merge #2941

2941:  Implementation of WASIX and a fully networking for Web Assembly r=epilys a=epilys

- Introduced the virtual BUS interface used for RPC between web assembly apps
- Introduced the virtual networking interface used to implement networking
  for web assembly apps
- Implemented a local implementation of the virtual networking
  (available behind the feature toggle 'host-net' on the 'wasi' package)
- Fixed up some of the examples from the wasmer3 branch
- Refactored the WASI implementations so they support wasm64-wasi
- WASIX is behind its own namespaces for both 32bit and 64bit implementations

Original PR by `@john-sharratt` #2930

Co-authored-by: Johnathan Sharratt <[email protected]>
Co-authored-by: ptitSeb <[email protected]>
Co-authored-by: John Sharratt's Shared Account <[email protected]>

Cargo.toml

@@ -47,10 +47,13 @@ members = [
     "lib/engine-staticlib",
     "lib/object",
     "lib/vfs",
+    "lib/vnet",
+    "lib/vbus",
     "lib/vm",
     "lib/wasi",
     "lib/wasi-types",
     "lib/wasi-experimental-io-devices",
+    "lib/wasi-local-networking",
     "lib/types",
     "tests/wasi-wast",
     "tests/lib/wast",

benches/static_and_dynamic_functions.rs

@@ -32,10 +32,10 @@ static BASIC_WAT: &str = r#"(module
 )"#;
 
 pub fn run_basic_static_function(store: &Store, compiler_name: &str, c: &mut Criterion) {
-    let module = Module::new(&store, BASIC_WAT).unwrap();
+    let module = Module::new(store, BASIC_WAT).unwrap();
     let import_object = imports! {
         "env" => {
-            "multiply" => Function::new_native(&store, |a: i32, b: i32| a * b),
+            "multiply" => Function::new_native(store, |a: i32, b: i32| a * b),
         },
     };
     let instance = Instance::new(&module, &import_object).unwrap();
@@ -93,10 +93,10 @@ pub fn run_basic_static_function(store: &Store, compiler_name: &str, c: &mut Cri
 }
 
 pub fn run_basic_dynamic_function(store: &Store, compiler_name: &str, c: &mut Criterion) {
-    let module = Module::new(&store, BASIC_WAT).unwrap();
+    let module = Module::new(store, BASIC_WAT).unwrap();
     let import_object = imports! {
         "env" => {
-            "multiply" => Function::new_native(&store, |a: i32, b: i32| a * b),
+            "multiply" => Function::new_native(store, |a: i32, b: i32| a * b),
         },
     };
     let instance = Instance::new(&module, &import_object).unwrap();

examples/wasi_pipes.rs

@@ -11,6 +11,7 @@
 //!
 //! Ready?
 
+use std::io::{Read, Write};
 use wasmer::{Instance, Module, Store};
 use wasmer_compiler_cranelift::Cranelift;
 use wasmer_engine_universal::Universal;
@@ -36,11 +37,11 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
 
     println!("Creating `WasiEnv`...");
     // First, we create the `WasiEnv` with the stdio pipes
-    let input = Pipe::new();
-    let output = Pipe::new();
+    let mut input = Pipe::new();
+    let mut output = Pipe::new();
     let mut wasi_env = WasiState::new("hello")
-        .stdin(Box::new(input))
-        .stdout(Box::new(output))
+        .stdin(Box::new(input.clone()))
+        .stdout(Box::new(output.clone()))
         .finalize()?;
 
     println!("Instantiating module with WASI imports...");
@@ -51,29 +52,19 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
 
     let msg = "racecar go zoom";
     println!("Writing \"{}\" to the WASI stdin...", msg);
-    // To write to the stdin, we need a mutable reference to the pipe
-    //
-    // We access WasiState in a nested scope to ensure we're not holding
-    // the mutex after we need it.
-    {
-        let mut state = wasi_env.state();
-        let wasi_stdin = state.fs.stdin_mut()?.as_mut().unwrap();
-        // Then we can write to it!
-        writeln!(wasi_stdin, "{}", msg)?;
-    }
+    // To write to the stdin
+    writeln!(input, "{}", msg)?;
 
     println!("Call WASI `_start` function...");
     // And we just call the `_start` function!
     let start = instance.exports.get_function("_start")?;
     start.call(&[])?;
 
     println!("Reading from the WASI stdout...");
-    // To read from the stdout, we again need a mutable reference to the pipe
-    let mut state = wasi_env.state();
-    let wasi_stdout = state.fs.stdout_mut()?.as_mut().unwrap();
-    // Then we can read from it!
+
+    // To read from the stdout
     let mut buf = String::new();
-    wasi_stdout.read_to_string(&mut buf)?;
+    output.read_to_string(&mut buf)?;
     println!("Read \"{}\" from the WASI stdout!", buf.trim());
 
     Ok(())

lib/api/src/js/externals/memory.rs

@@ -240,7 +240,7 @@ impl Memory {
     /// Used by tests
     #[doc(hidden)]
     pub fn uint8view(&self) -> js_sys::Uint8Array {
-        self.view.clone()
+        js_sys::Uint8Array::new(&self.vm_memory.memory.buffer())
     }
 
     pub(crate) fn from_vm_export(store: &Store, vm_memory: VMMemory) -> Self {
@@ -276,16 +276,17 @@ impl Memory {
     /// This method is guaranteed to be safe (from the host side) in the face of
     /// concurrent writes.
     pub fn read(&self, offset: u64, buf: &mut [u8]) -> Result<(), MemoryAccessError> {
+        let view = self.uint8view();
         let offset: u32 = offset.try_into().map_err(|_| MemoryAccessError::Overflow)?;
         let len: u32 = buf
             .len()
             .try_into()
             .map_err(|_| MemoryAccessError::Overflow)?;
         let end = offset.checked_add(len).ok_or(MemoryAccessError::Overflow)?;
-        if end > self.view.length() {
+        if end > view.length() {
             Err(MemoryAccessError::HeapOutOfBounds)?;
         }
-        self.view.subarray(offset, end).copy_to(buf);
+        view.subarray(offset, end).copy_to(buf);
         Ok(())
     }
 
@@ -304,13 +305,14 @@ impl Memory {
         offset: u64,
         buf: &'a mut [MaybeUninit<u8>],
     ) -> Result<&'a mut [u8], MemoryAccessError> {
+        let view = self.uint8view();
         let offset: u32 = offset.try_into().map_err(|_| MemoryAccessError::Overflow)?;
         let len: u32 = buf
             .len()
             .try_into()
             .map_err(|_| MemoryAccessError::Overflow)?;
         let end = offset.checked_add(len).ok_or(MemoryAccessError::Overflow)?;
-        if end > self.view.length() {
+        if end > view.length() {
             Err(MemoryAccessError::HeapOutOfBounds)?;
         }
 
@@ -321,7 +323,7 @@ impl Memory {
         }
         let buf = unsafe { slice::from_raw_parts_mut(buf.as_mut_ptr() as *mut u8, buf.len()) };
 
-        self.view.subarray(offset, end).copy_to(buf);
+        view.subarray(offset, end).copy_to(buf);
         Ok(buf)
     }
 
@@ -333,16 +335,18 @@ impl Memory {
     /// This method is guaranteed to be safe (from the host side) in the face of
     /// concurrent reads/writes.
     pub fn write(&self, offset: u64, data: &[u8]) -> Result<(), MemoryAccessError> {
+        let view = self.uint8view();
         let offset: u32 = offset.try_into().map_err(|_| MemoryAccessError::Overflow)?;
         let len: u32 = data
             .len()
             .try_into()
             .map_err(|_| MemoryAccessError::Overflow)?;
+        let view = self.uint8view();
         let end = offset.checked_add(len).ok_or(MemoryAccessError::Overflow)?;
-        if end > self.view.length() {
+        if end > view.length() {
             Err(MemoryAccessError::HeapOutOfBounds)?;
         }
-        self.view.subarray(offset, end).copy_from(data);
+        view.subarray(offset, end).copy_from(data);
         Ok(())
     }
 }

lib/api/src/js/mem_access.rs

@@ -10,7 +10,7 @@ use std::{
     string::FromUtf8Error,
 };
 use thiserror::Error;
-use wasmer_types::ValueType;
+use wasmer_types::{MemorySize, ValueType};
 
 /// Error for invalid [`Memory`] access.
 #[derive(Clone, Copy, Debug, Error)]
@@ -85,6 +85,17 @@ impl<'a, T: ValueType> WasmRef<'a, T> {
         WasmPtr::new(self.offset)
     }
 
+    /// Get a `WasmPtr` fror this `WasmRef`.
+    #[inline]
+    pub fn as_ptr<M: MemorySize>(self) -> WasmPtr<T, M> {
+        let offset: M::Offset = self
+            .offset
+            .try_into()
+            .map_err(|_| "invalid offset into memory")
+            .unwrap();
+        WasmPtr::<T, M>::new(offset)
+    }
+
     /// Get a reference to the Wasm memory backing this reference.
     #[inline]
     pub fn memory(self) -> &'a Memory {

lib/api/src/js/native.rs

@@ -28,6 +28,7 @@ pub struct TypedFunction<Args = (), Rets = ()> {
 }
 
 unsafe impl<Args, Rets> Send for TypedFunction<Args, Rets> {}
+unsafe impl<Args, Rets> Sync for TypedFunction<Args, Rets> {}
 
 impl<Args, Rets> TypedFunction<Args, Rets>
 where