From 2fe6e6f03971e2ade7df97d01306173205863cdd Mon Sep 17 00:00:00 2001 From: losfair Date: Sat, 15 Feb 2020 01:31:33 +0800 Subject: [PATCH 1/6] Global trampoline buffer. --- lib/runtime-core/src/trampoline_x64.rs | 51 ++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/lib/runtime-core/src/trampoline_x64.rs b/lib/runtime-core/src/trampoline_x64.rs index 3d07484c715..c549498f3a3 100644 --- a/lib/runtime-core/src/trampoline_x64.rs +++ b/lib/runtime-core/src/trampoline_x64.rs @@ -9,6 +9,8 @@ use crate::loader::CodeMemory; use crate::vm::Ctx; use std::fmt; +use std::ptr::NonNull; +use std::sync::atomic::{AtomicUsize, Ordering}; use std::{mem, slice}; lazy_static! { @@ -29,6 +31,50 @@ lazy_static! { mem::transmute(ptr) } }; + + static ref TRAMPOLINES: TrampBuffer = TrampBuffer::new(); +} + +struct TrampBuffer { + buffer: CodeMemory, + len: AtomicUsize, +} + +impl TrampBuffer { + /// Creates a trampoline buffer. + fn new() -> TrampBuffer { + // Pre-allocate 64 MiB of virtual memory for code. + let mem = CodeMemory::new(64 * 1048576); + mem.make_writable_executable(); + TrampBuffer { + buffer: mem, + len: AtomicUsize::new(0), + } + } + + /// Bump allocation. Copies `buf` to the end of this code memory. + /// + /// FIXME: Proper storage recycling. + fn append(&self, buf: &[u8]) -> Option> { + let begin = self.len.fetch_add(buf.len(), Ordering::SeqCst); + let end = begin + buf.len(); + + // Length overflowed. Revert and return None. + if end > self.buffer.len() { + self.len.fetch_sub(buf.len(), Ordering::SeqCst); + return None; + } + + // Now we have unique ownership to `self.buffer[begin..end]`. + let slice = unsafe { + std::slice::from_raw_parts_mut( + self.buffer.get_backing_ptr().offset(begin as _), + buf.len(), + ) + }; + slice.copy_from_slice(buf); + Some(NonNull::new(slice.as_mut_ptr()).unwrap()) + } } /// An opaque type for pointers to a callable memory location. @@ -219,6 +265,11 @@ impl TrampolineBufferBuilder { idx } + /// Appends to the global trampoline buffer. + pub fn append_global(self) -> Option> { + TRAMPOLINES.append(&self.code) + } + /// Consumes the builder and builds the trampoline buffer. pub fn build(self) -> TrampolineBuffer { get_context(); // ensure lazy initialization is completed From 12373bb87289f8e68b14bd2c2bbc8887bea922e9 Mon Sep 17 00:00:00 2001 From: losfair Date: Sat, 15 Feb 2020 01:31:49 +0800 Subject: [PATCH 2/6] Func::new_polymorphic --- lib/runtime-core-tests/tests/imports.rs | 31 ++++++++++- lib/runtime-core/src/loader.rs | 20 ++++++- lib/runtime-core/src/typed_func.rs | 74 ++++++++++++++++++++++++- lib/runtime-core/src/vm.rs | 4 +- 4 files changed, 122 insertions(+), 7 deletions(-) diff --git a/lib/runtime-core-tests/tests/imports.rs b/lib/runtime-core-tests/tests/imports.rs index b461ad2b758..42a09d84460 100644 --- a/lib/runtime-core-tests/tests/imports.rs +++ b/lib/runtime-core-tests/tests/imports.rs @@ -1,6 +1,13 @@ +use std::sync::Arc; use wasmer_runtime_core::{ - compile_with, error::RuntimeError, imports, memory::Memory, typed_func::Func, - types::MemoryDescriptor, units::Pages, vm, Instance, + compile_with, + error::RuntimeError, + imports, + memory::Memory, + typed_func::Func, + types::{FuncSig, MemoryDescriptor, Type, Value}, + units::Pages, + vm, Instance, }; use wasmer_runtime_core_tests::{get_compiler, wat2wasm}; @@ -68,6 +75,7 @@ fn imported_functions_forms(test: &dyn Fn(&Instance)) { (import "env" "memory" (memory 1 1)) (import "env" "callback_fn" (func $callback_fn (type $type))) (import "env" "callback_closure" (func $callback_closure (type $type))) + (import "env" "callback_closure_polymorphic" (func $callback_closure_polymorphic (type $type))) (import "env" "callback_closure_with_env" (func $callback_closure_with_env (type $type))) (import "env" "callback_fn_with_vmctx" (func $callback_fn_with_vmctx (type $type))) (import "env" "callback_closure_with_vmctx" (func $callback_closure_with_vmctx (type $type))) @@ -86,6 +94,10 @@ fn imported_functions_forms(test: &dyn Fn(&Instance)) { get_local 0 call $callback_closure) + (func (export "function_closure_polymorphic") (type $type) + get_local 0 + call $callback_closure_polymorphic) + (func (export "function_closure_with_env") (type $type) get_local 0 call $callback_closure_with_env) @@ -142,6 +154,16 @@ fn imported_functions_forms(test: &dyn Fn(&Instance)) { Ok(n + 1) }), + "callback_closure_polymorphic" => Func::::new_polymorphic( + Arc::new(FuncSig::new(vec![Type::I32], vec![Type::I32])), + |_, params| -> Vec { + match params[0] { + Value::I32(x) => vec![Value::I32(x + 1)], + _ => unreachable!() + } + } + ), + // Closure with a captured environment (a single variable + an instance of `Memory`). "callback_closure_with_env" => Func::new(move |n: i32| -> Result { let shift_ = shift + memory.view::()[0].get(); @@ -236,6 +258,11 @@ macro_rules! test { test!(test_fn, function_fn, Ok(2)); test!(test_closure, function_closure, Ok(2)); +test!( + test_closure_polymorphic, + function_closure_polymorphic, + Ok(2) +); test!( test_closure_with_env, function_closure_with_env, diff --git a/lib/runtime-core/src/loader.rs b/lib/runtime-core/src/loader.rs index f516643d063..ea1ca0130ac 100644 --- a/lib/runtime-core/src/loader.rs +++ b/lib/runtime-core/src/loader.rs @@ -1,7 +1,9 @@ //! The loader module functions are used to load an instance. use crate::{backend::RunnableModule, module::ModuleInfo, types::Type, types::Value, vm::Ctx}; #[cfg(unix)] -use libc::{mmap, mprotect, munmap, MAP_ANON, MAP_PRIVATE, PROT_EXEC, PROT_READ, PROT_WRITE}; +use libc::{ + mmap, mprotect, munmap, MAP_ANON, MAP_NORESERVE, MAP_PRIVATE, PROT_EXEC, PROT_READ, PROT_WRITE, +}; use std::{ fmt::Debug, ops::{Deref, DerefMut}, @@ -169,7 +171,7 @@ impl CodeMemory { std::ptr::null_mut(), size, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANON, + MAP_PRIVATE | MAP_ANON | MAP_NORESERVE, -1, 0, ) @@ -196,6 +198,20 @@ impl CodeMemory { panic!("cannot set code memory to writable"); } } + + /// Makes this code memory both writable and executable. + /// + /// Avoid using this if a combination `make_executable` and `make_writable` can be used. + pub fn make_writable_executable(&self) { + if unsafe { mprotect(self.ptr as _, self.size, PROT_READ | PROT_WRITE | PROT_EXEC) } != 0 { + panic!("cannot set code memory to writable and executable"); + } + } + + /// Returns the backing pointer of this code memory. + pub fn get_backing_ptr(&self) -> *mut u8 { + self.ptr + } } #[cfg(unix)] diff --git a/lib/runtime-core/src/typed_func.rs b/lib/runtime-core/src/typed_func.rs index 513ede7fbc0..2edcf6644b3 100644 --- a/lib/runtime-core/src/typed_func.rs +++ b/lib/runtime-core/src/typed_func.rs @@ -4,7 +4,7 @@ use crate::{ error::RuntimeError, export::{Context, Export, FuncPointer}, import::IsExport, - types::{FuncSig, NativeWasmType, Type, WasmExternType}, + types::{FuncSig, NativeWasmType, Type, Value, WasmExternType}, vm, }; use std::{ @@ -240,6 +240,78 @@ where _phantom: PhantomData, } } + + /// Creates a polymorphic function. + #[allow(unused_variables)] + #[cfg(all(unix, target_arch = "x86_64"))] + pub fn new_polymorphic(signature: Arc, func: F) -> Func<'a, Args, Rets, Host> + where + F: Fn(&mut vm::Ctx, &[Value]) -> Vec + 'static, + { + use crate::trampoline_x64::*; + use std::convert::TryFrom; + + struct PolymorphicContext { + arg_types: Vec, + func: Box Vec>, + } + unsafe extern "C" fn enter_host_polymorphic( + ctx: *const CallContext, + args: *const u64, + ) -> u64 { + let ctx = &*(ctx as *const PolymorphicContext); + let vmctx = &mut *(*args.offset(0) as *mut vm::Ctx); + let args: Vec = ctx + .arg_types + .iter() + .enumerate() + .map(|(i, t)| { + let i = i + 1; // skip vmctx + match *t { + Type::I32 => Value::I32(*args.offset(i as _) as i32), + Type::I64 => Value::I64(*args.offset(i as _) as i64), + Type::F32 => Value::F32(f32::from_bits(*args.offset(i as _) as u32)), + Type::F64 => Value::F64(f64::from_bits(*args.offset(i as _) as u64)), + Type::V128 => { + panic!("enter_host_polymorphic: 128-bit types are not supported") + } + } + }) + .collect(); + let rets = (ctx.func)(vmctx, &args); + if rets.len() == 0 { + 0 + } else if rets.len() == 1 { + u64::try_from(rets[0].to_u128()).expect( + "128-bit return value from polymorphic host functions is not yet supported", + ) + } else { + panic!( + "multiple return values from polymorphic host functions is not yet supported" + ); + } + } + let mut builder = TrampolineBufferBuilder::new(); + let ctx = Box::new(PolymorphicContext { + arg_types: signature.params().to_vec(), + func: Box::new(func), + }); + builder.add_callinfo_trampoline( + enter_host_polymorphic, + Box::into_raw(ctx) as *const _, + (signature.params().len() + 1) as u32, // +vmctx + ); + let ptr = builder + .append_global() + .expect("cannot bump-allocate global trampoline memory"); + Func { + inner: Host(()), + func: ptr.cast::(), + func_env: None, + vmctx: ptr::null_mut(), + _phantom: PhantomData, + } + } } impl<'a, Args, Rets, Inner> Func<'a, Args, Rets, Inner> diff --git a/lib/runtime-core/src/vm.rs b/lib/runtime-core/src/vm.rs index 2a39bdec10f..a25ee24ff6b 100644 --- a/lib/runtime-core/src/vm.rs +++ b/lib/runtime-core/src/vm.rs @@ -545,13 +545,13 @@ impl Ctx { /// `typed_func` module within the `wrap` functions, to wrap imported /// functions. #[repr(transparent)] -pub struct Func(pub(self) *mut c_void); +pub struct Func(*mut c_void); /// Represents a function environment pointer, like a captured /// environment of a closure. It is mostly used in the `typed_func` /// module within the `wrap` functions, to wrap imported functions. #[repr(transparent)] -pub struct FuncEnv(pub(self) *mut c_void); +pub struct FuncEnv(*mut c_void); /// Represents a function context. It is used by imported functions /// only. From 5f4561e5efdee87a3a26e520fcde77c8231cbe04 Mon Sep 17 00:00:00 2001 From: losfair Date: Sun, 16 Feb 2020 00:28:43 +0800 Subject: [PATCH 3/6] Fix compilation error on Aarch64. --- lib/runtime-core/src/typed_func.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/runtime-core/src/typed_func.rs b/lib/runtime-core/src/typed_func.rs index 2edcf6644b3..84fbe8addd7 100644 --- a/lib/runtime-core/src/typed_func.rs +++ b/lib/runtime-core/src/typed_func.rs @@ -4,7 +4,7 @@ use crate::{ error::RuntimeError, export::{Context, Export, FuncPointer}, import::IsExport, - types::{FuncSig, NativeWasmType, Type, Value, WasmExternType}, + types::{FuncSig, NativeWasmType, Type, WasmExternType}, vm, }; use std::{ @@ -246,9 +246,10 @@ where #[cfg(all(unix, target_arch = "x86_64"))] pub fn new_polymorphic(signature: Arc, func: F) -> Func<'a, Args, Rets, Host> where - F: Fn(&mut vm::Ctx, &[Value]) -> Vec + 'static, + F: Fn(&mut vm::Ctx, &[crate::types::Value]) -> Vec + 'static, { use crate::trampoline_x64::*; + use crate::types::Value; use std::convert::TryFrom; struct PolymorphicContext { From ad20a008e095a471d85a3dccebfa5b7d9ecae4af Mon Sep 17 00:00:00 2001 From: Ivan Enderlin Date: Mon, 17 Feb 2020 15:30:25 +0100 Subject: [PATCH 4/6] fix(runtime-core) Use explicit `dyn` for trait objects. --- lib/runtime-core/src/typed_func.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/runtime-core/src/typed_func.rs b/lib/runtime-core/src/typed_func.rs index 84fbe8addd7..eca73f61b86 100644 --- a/lib/runtime-core/src/typed_func.rs +++ b/lib/runtime-core/src/typed_func.rs @@ -254,7 +254,7 @@ where struct PolymorphicContext { arg_types: Vec, - func: Box Vec>, + func: Box Vec>, } unsafe extern "C" fn enter_host_polymorphic( ctx: *const CallContext, From 2ee1e80f3b15f1c266bdfeeb828e5c1aed2919a3 Mon Sep 17 00:00:00 2001 From: Ivan Enderlin Date: Mon, 17 Feb 2020 15:43:14 +0100 Subject: [PATCH 5/6] feat(runtime-core) Allow dynamic signature for polymorphic host functions. This patch adds a new field in `Func`: `signature`. It contains the signature of the host function. For non-polymorphic host functions, the signature is computed from the `Args` and `Rets` implementation parameters at compile-time. For polymorphic host functions though, to be fully dynamic, the signature given to `new_polymorphic` is used in `Func` as the correct signature. --- lib/runtime-core-tests/tests/imports.rs | 2 +- lib/runtime-core/src/typed_func.rs | 25 +++++++++++++++++++++---- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/lib/runtime-core-tests/tests/imports.rs b/lib/runtime-core-tests/tests/imports.rs index 42a09d84460..ce3cea2c4a9 100644 --- a/lib/runtime-core-tests/tests/imports.rs +++ b/lib/runtime-core-tests/tests/imports.rs @@ -154,7 +154,7 @@ fn imported_functions_forms(test: &dyn Fn(&Instance)) { Ok(n + 1) }), - "callback_closure_polymorphic" => Func::::new_polymorphic( + "callback_closure_polymorphic" => Func::new_polymorphic( Arc::new(FuncSig::new(vec![Type::I32], vec![Type::I32])), |_, params| -> Vec { match params[0] { diff --git a/lib/runtime-core/src/typed_func.rs b/lib/runtime-core/src/typed_func.rs index eca73f61b86..4abad5a1741 100644 --- a/lib/runtime-core/src/typed_func.rs +++ b/lib/runtime-core/src/typed_func.rs @@ -189,9 +189,21 @@ where /// Represents a function that can be used by WebAssembly. pub struct Func<'a, Args = (), Rets = (), Inner: Kind = Wasm> { inner: Inner, + + /// The function pointer. func: NonNull, + + /// The function environment. func_env: Option>, + + /// The famous `vm::Ctx`. vmctx: *mut vm::Ctx, + + /// The signature is usually infered from `Args` and `Rets`. In + /// case of polymorphic function, the signature is only known at + /// runtime. + signature: Arc, + _phantom: PhantomData<(&'a (), Args, Rets)>, } @@ -214,6 +226,7 @@ where func, func_env, vmctx, + signature: Arc::new(FuncSig::new(Args::types(), Rets::types())), _phantom: PhantomData, } } @@ -225,7 +238,7 @@ where Rets: WasmTypeList, { /// Creates a new `Func`. - pub fn new(func: F) -> Func<'a, Args, Rets, Host> + pub fn new(func: F) -> Self where Kind: ExternalFunctionKind, F: ExternalFunction, @@ -237,14 +250,17 @@ where func, func_env, vmctx: ptr::null_mut(), + signature: Arc::new(FuncSig::new(Args::types(), Rets::types())), _phantom: PhantomData, } } +} +impl<'a> Func<'a, (), (), Host> { /// Creates a polymorphic function. #[allow(unused_variables)] #[cfg(all(unix, target_arch = "x86_64"))] - pub fn new_polymorphic(signature: Arc, func: F) -> Func<'a, Args, Rets, Host> + pub fn new_polymorphic(signature: Arc, func: F) -> Self where F: Fn(&mut vm::Ctx, &[crate::types::Value]) -> Vec + 'static, { @@ -305,11 +321,13 @@ where let ptr = builder .append_global() .expect("cannot bump-allocate global trampoline memory"); + Func { inner: Host(()), func: ptr.cast::(), func_env: None, vmctx: ptr::null_mut(), + signature, _phantom: PhantomData, } } @@ -751,12 +769,11 @@ where func_env @ Some(_) => Context::ExternalWithEnv(self.vmctx, func_env), None => Context::Internal, }; - let signature = Arc::new(FuncSig::new(Args::types(), Rets::types())); Export::Function { func, ctx, - signature, + signature: self.signature.clone(), } } } From b67acbc0e3828a8b99ebf956839f1731dd7b8af0 Mon Sep 17 00:00:00 2001 From: losfair Date: Tue, 25 Feb 2020 01:19:19 +0800 Subject: [PATCH 6/6] Add `ErasedFunc` for type-erased functions. --- lib/runtime-core-tests/tests/imports.rs | 4 +- lib/runtime-core/src/typed_func.rs | 75 +++++++++++++++++++++---- 2 files changed, 66 insertions(+), 13 deletions(-) diff --git a/lib/runtime-core-tests/tests/imports.rs b/lib/runtime-core-tests/tests/imports.rs index ce3cea2c4a9..17c82097ad9 100644 --- a/lib/runtime-core-tests/tests/imports.rs +++ b/lib/runtime-core-tests/tests/imports.rs @@ -4,7 +4,7 @@ use wasmer_runtime_core::{ error::RuntimeError, imports, memory::Memory, - typed_func::Func, + typed_func::{ErasedFunc, Func}, types::{FuncSig, MemoryDescriptor, Type, Value}, units::Pages, vm, Instance, @@ -154,7 +154,7 @@ fn imported_functions_forms(test: &dyn Fn(&Instance)) { Ok(n + 1) }), - "callback_closure_polymorphic" => Func::new_polymorphic( + "callback_closure_polymorphic" => ErasedFunc::new_polymorphic( Arc::new(FuncSig::new(vec![Type::I32], vec![Type::I32])), |_, params| -> Vec { match params[0] { diff --git a/lib/runtime-core/src/typed_func.rs b/lib/runtime-core/src/typed_func.rs index 39c8b45e93b..6a3558f6d5d 100644 --- a/lib/runtime-core/src/typed_func.rs +++ b/lib/runtime-core/src/typed_func.rs @@ -190,6 +190,32 @@ where } } +/// Represents a type-erased function provided by either the host or the WebAssembly program. +#[allow(dead_code)] +pub struct ErasedFunc<'a> { + inner: Box, + + /// The function pointer. + func: NonNull, + + /// The function environment. + func_env: Option>, + + /// The famous `vm::Ctx`. + vmctx: *mut vm::Ctx, + + /// The runtime signature of this function. + /// + /// When converted from a `Func`, this is determined by the static `Args` and `Rets` type parameters. + /// otherwise the signature is dynamically assigned during `ErasedFunc` creation, usually when creating + /// a polymorphic host function. + signature: Arc, + + _phantom: PhantomData<&'a ()>, +} + +unsafe impl<'a> Send for ErasedFunc<'a> {} + /// Represents a function that can be used by WebAssembly. pub struct Func<'a, Args = (), Rets = (), Inner: Kind = Wasm> { inner: Inner, @@ -203,17 +229,30 @@ pub struct Func<'a, Args = (), Rets = (), Inner: Kind = Wasm> { /// The famous `vm::Ctx`. vmctx: *mut vm::Ctx, - /// The signature is usually infered from `Args` and `Rets`. In - /// case of polymorphic function, the signature is only known at - /// runtime. - signature: Arc, - _phantom: PhantomData<(&'a (), Args, Rets)>, } unsafe impl<'a, Args, Rets> Send for Func<'a, Args, Rets, Wasm> {} unsafe impl<'a, Args, Rets> Send for Func<'a, Args, Rets, Host> {} +impl<'a, Args, Rets, Inner> From> for ErasedFunc<'a> +where + Args: WasmTypeList, + Rets: WasmTypeList, + Inner: Kind + 'static, +{ + fn from(that: Func<'a, Args, Rets, Inner>) -> ErasedFunc<'a> { + ErasedFunc { + inner: Box::new(that.inner), + func: that.func, + func_env: that.func_env, + vmctx: that.vmctx, + signature: Arc::new(FuncSig::new(Args::types(), Rets::types())), + _phantom: PhantomData, + } + } +} + impl<'a, Args, Rets> Func<'a, Args, Rets, Wasm> where Args: WasmTypeList, @@ -230,7 +269,6 @@ where func, func_env, vmctx, - signature: Arc::new(FuncSig::new(Args::types(), Rets::types())), _phantom: PhantomData, } } @@ -254,13 +292,12 @@ where func, func_env, vmctx: ptr::null_mut(), - signature: Arc::new(FuncSig::new(Args::types(), Rets::types())), _phantom: PhantomData, } } } -impl<'a> Func<'a, (), (), Host> { +impl<'a> ErasedFunc<'a> { /// Creates a polymorphic function. #[allow(unused_variables)] #[cfg(all(unix, target_arch = "x86_64"))] @@ -326,8 +363,8 @@ impl<'a> Func<'a, (), (), Host> { .append_global() .expect("cannot bump-allocate global trampoline memory"); - Func { - inner: Host(()), + ErasedFunc { + inner: Box::new(Host(())), func: ptr.cast::(), func_env: None, vmctx: ptr::null_mut(), @@ -765,6 +802,22 @@ impl_traits!([C] S24, A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T impl_traits!([C] S25, A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V, W, X, Y); impl_traits!([C] S26, A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V, W, X, Y, Z); +impl<'a> IsExport for ErasedFunc<'a> { + fn to_export(&self) -> Export { + let func = unsafe { FuncPointer::new(self.func.as_ptr()) }; + let ctx = match self.func_env { + func_env @ Some(_) => Context::ExternalWithEnv(self.vmctx, func_env), + None => Context::Internal, + }; + + Export::Function { + func, + ctx, + signature: self.signature.clone(), + } + } +} + impl<'a, Args, Rets, Inner> IsExport for Func<'a, Args, Rets, Inner> where Args: WasmTypeList, @@ -781,7 +834,7 @@ where Export::Function { func, ctx, - signature: self.signature.clone(), + signature: Arc::new(FuncSig::new(Args::types(), Rets::types())), } } }