diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f4f1d6f6d8..f0b13ac7a7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## **[Unreleased]** +- [#1217](https://github.com/wasmerio/wasmer/pull/1217) Polymorphic host functions based on dynamic trampoline generation. - [#1252](https://github.com/wasmerio/wasmer/pull/1252) Allow `/` in wasi `--mapdir` wasm path. - [#1212](https://github.com/wasmerio/wasmer/pull/1212) Add support for GDB JIT debugging: - Add `--generate-debug-info` and `-g` flags to `wasmer run` to generate debug information during compilation. The debug info is passed via the GDB JIT interface to a debugger to allow source-level debugging of Wasm files. Currently only available on clif-backend. diff --git a/lib/runtime-core-tests/tests/imports.rs b/lib/runtime-core-tests/tests/imports.rs index b461ad2b758..1af995368eb 100644 --- a/lib/runtime-core-tests/tests/imports.rs +++ b/lib/runtime-core-tests/tests/imports.rs @@ -1,6 +1,13 @@ +use std::sync::Arc; use wasmer_runtime_core::{ - compile_with, error::RuntimeError, imports, memory::Memory, typed_func::Func, - types::MemoryDescriptor, units::Pages, vm, Instance, + compile_with, + error::RuntimeError, + imports, + memory::Memory, + typed_func::{DynamicFunc, Func}, + types::{FuncSig, MemoryDescriptor, Type, Value}, + units::Pages, + vm, Instance, }; use wasmer_runtime_core_tests::{get_compiler, wat2wasm}; @@ -68,6 +75,7 @@ fn imported_functions_forms(test: &dyn Fn(&Instance)) { (import "env" "memory" (memory 1 1)) (import "env" "callback_fn" (func $callback_fn (type $type))) (import "env" "callback_closure" (func $callback_closure (type $type))) + (import "env" "callback_closure_dynamic" (func $callback_closure_dynamic (type $type))) (import "env" "callback_closure_with_env" (func $callback_closure_with_env (type $type))) (import "env" "callback_fn_with_vmctx" (func $callback_fn_with_vmctx (type $type))) (import "env" "callback_closure_with_vmctx" (func $callback_closure_with_vmctx (type $type))) @@ -86,6 +94,10 @@ fn imported_functions_forms(test: &dyn Fn(&Instance)) { get_local 0 call $callback_closure) + (func (export "function_closure_dynamic") (type $type) + get_local 0 + call $callback_closure_dynamic) + (func (export "function_closure_with_env") (type $type) get_local 0 call $callback_closure_with_env) @@ -142,6 +154,16 @@ fn imported_functions_forms(test: &dyn Fn(&Instance)) { Ok(n + 1) }), + "callback_closure_dynamic" => DynamicFunc::new( + Arc::new(FuncSig::new(vec![Type::I32], vec![Type::I32])), + |_, params| -> Vec { + match params[0] { + Value::I32(x) => vec![Value::I32(x + 1)], + _ => unreachable!() + } + } + ), + // Closure with a captured environment (a single variable + an instance of `Memory`). "callback_closure_with_env" => Func::new(move |n: i32| -> Result { let shift_ = shift + memory.view::()[0].get(); @@ -236,6 +258,7 @@ macro_rules! test { test!(test_fn, function_fn, Ok(2)); test!(test_closure, function_closure, Ok(2)); +test!(test_closure_dynamic, function_closure_dynamic, Ok(2)); test!( test_closure_with_env, function_closure_with_env, diff --git a/lib/runtime-core/src/loader.rs b/lib/runtime-core/src/loader.rs index f516643d063..738c8dfef83 100644 --- a/lib/runtime-core/src/loader.rs +++ b/lib/runtime-core/src/loader.rs @@ -1,7 +1,9 @@ //! The loader module functions are used to load an instance. use crate::{backend::RunnableModule, module::ModuleInfo, types::Type, types::Value, vm::Ctx}; #[cfg(unix)] -use libc::{mmap, mprotect, munmap, MAP_ANON, MAP_PRIVATE, PROT_EXEC, PROT_READ, PROT_WRITE}; +use libc::{ + mmap, mprotect, munmap, MAP_ANON, MAP_NORESERVE, MAP_PRIVATE, PROT_EXEC, PROT_READ, PROT_WRITE, +}; use std::{ fmt::Debug, ops::{Deref, DerefMut}, @@ -138,12 +140,12 @@ impl CodeMemory { unimplemented!("CodeMemory::new"); } - /// Makes this code memory executable. + /// Makes this code memory executable and not writable. pub fn make_executable(&self) { unimplemented!("CodeMemory::make_executable"); } - /// Makes this code memory writable. + /// Makes this code memory writable and not executable. pub fn make_writable(&self) { unimplemented!("CodeMemory::make_writable"); } @@ -169,7 +171,7 @@ impl CodeMemory { std::ptr::null_mut(), size, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANON, + MAP_PRIVATE | MAP_ANON | MAP_NORESERVE, -1, 0, ) @@ -183,19 +185,33 @@ impl CodeMemory { } } - /// Makes this code memory executable. + /// Makes this code memory executable and not writable. pub fn make_executable(&self) { if unsafe { mprotect(self.ptr as _, self.size, PROT_READ | PROT_EXEC) } != 0 { panic!("cannot set code memory to executable"); } } - /// Makes this code memory writable. + /// Makes this code memory writable and not executable. pub fn make_writable(&self) { if unsafe { mprotect(self.ptr as _, self.size, PROT_READ | PROT_WRITE) } != 0 { panic!("cannot set code memory to writable"); } } + + /// Makes this code memory both writable and executable. + /// + /// Avoid using this if a combination `make_executable` and `make_writable` can be used. + pub fn make_writable_executable(&self) { + if unsafe { mprotect(self.ptr as _, self.size, PROT_READ | PROT_WRITE | PROT_EXEC) } != 0 { + panic!("cannot set code memory to writable and executable"); + } + } + + /// Returns the backing pointer of this code memory. + pub fn get_backing_ptr(&self) -> *mut u8 { + self.ptr + } } #[cfg(unix)] diff --git a/lib/runtime-core/src/trampoline_x64.rs b/lib/runtime-core/src/trampoline_x64.rs index 3d07484c715..04809832151 100644 --- a/lib/runtime-core/src/trampoline_x64.rs +++ b/lib/runtime-core/src/trampoline_x64.rs @@ -8,7 +8,10 @@ use crate::loader::CodeMemory; use crate::vm::Ctx; +use std::collections::BTreeMap; use std::fmt; +use std::ptr::NonNull; +use std::sync::Mutex; use std::{mem, slice}; lazy_static! { @@ -29,6 +32,96 @@ lazy_static! { mem::transmute(ptr) } }; + + static ref TRAMPOLINES: TrampBuffer = TrampBuffer::new(64 * 1048576); +} + +/// The global trampoline buffer. +struct TrampBuffer { + /// A fixed-(virtual)-size executable+writable buffer for storing trampolines. + buffer: CodeMemory, + + /// Allocation state. + alloc: Mutex, +} + +/// The allocation state of a `TrampBuffer`. +struct AllocState { + /// Records all allocated blocks in `buffer`. + /// + /// Maps the start address of each block to its end address. + blocks: BTreeMap, +} + +impl TrampBuffer { + /// Creates a trampoline buffer with a given (virtual) size. + fn new(size: usize) -> TrampBuffer { + let mem = CodeMemory::new(size); + mem.make_writable_executable(); + TrampBuffer { + buffer: mem, + alloc: Mutex::new(AllocState { + blocks: BTreeMap::new(), + }), + } + } + + /// Removes a previously-`insert`ed trampoline. + /// + /// For safety, refer to the public interface `TrampolineBufferBuilder::remove_global`. + unsafe fn remove(&self, start: NonNull) { + let start = start.as_ptr() as usize - self.buffer.get_backing_ptr() as usize; + let mut alloc = self.alloc.lock().unwrap(); + alloc + .blocks + .remove(&start) + .expect("TrampBuffer::remove(): Attempting to remove a non-existent allocation."); + } + + /// Allocates a region of executable memory and copies `buf` to the end of this region. + /// + /// Returns `None` if no memory is available. + fn insert(&self, buf: &[u8]) -> Option> { + // First, assume an available start position... + let mut assumed_start: usize = 0; + + let mut alloc = self.alloc.lock().unwrap(); + let mut found = false; + + // Then, try invalidating that assumption... + for (&start, &end) in &alloc.blocks { + if start - assumed_start < buf.len() { + // Unavailable. Move to next free block. + assumed_start = end; + } else { + // This free block can be used. + found = true; + break; + } + } + + if !found { + // No previous free blocks were found. Try allocating at the end. + if self.buffer.len() - assumed_start < buf.len() { + // No more free space. Cannot allocate. + return None; + } + } + + // Now we know `assumed_start` is valid. + let start = assumed_start; + alloc.blocks.insert(start, start + buf.len()); + + // We have unique ownership to `self.buffer[start..start + buf.len()]`. + let slice = unsafe { + std::slice::from_raw_parts_mut( + self.buffer.get_backing_ptr().offset(start as _), + buf.len(), + ) + }; + slice.copy_from_slice(buf); + Some(NonNull::new(slice.as_mut_ptr()).unwrap()) + } } /// An opaque type for pointers to a callable memory location. @@ -219,6 +312,27 @@ impl TrampolineBufferBuilder { idx } + /// Inserts this trampoline to the global trampoline buffer. + pub fn insert_global(self) -> Option> { + TRAMPOLINES.insert(&self.code) + } + + /// Removes the trampoline pointed to by `ptr` from the global trampoline buffer. Panics if `ptr` + /// does not point to any trampoline. + /// + /// # Safety + /// + /// Calling this function invalidates the trampoline `ptr` points to and recycles its memory. You + /// should ensure that `ptr` isn't used after calling `remove_global`. + pub unsafe fn remove_global(ptr: NonNull) { + TRAMPOLINES.remove(ptr); + } + + /// Gets the current (non-executable) code in this builder. + pub fn code(&self) -> &[u8] { + &self.code + } + /// Consumes the builder and builds the trampoline buffer. pub fn build(self) -> TrampolineBuffer { get_context(); // ensure lazy initialization is completed @@ -292,4 +406,85 @@ mod tests { }; assert_eq!(ret, 136); } + + #[test] + fn test_many_global_trampolines() { + unsafe extern "C" fn inner(n: *const CallContext, args: *const u64) -> u64 { + let n = n as usize; + let mut result: u64 = 0; + for i in 0..n { + result += *args.offset(i as _); + } + result + } + + // Use the smallest possible buffer size (page size) to check memory releasing logic. + let buffer = TrampBuffer::new(4096); + + // Validate the previous trampoline instead of the current one to ensure that no overwrite happened. + let mut prev: Option<(NonNull, u64)> = None; + + for i in 0..5000usize { + let mut builder = TrampolineBufferBuilder::new(); + let n = i % 8; + builder.add_callinfo_trampoline(inner, n as _, n as _); + let ptr = buffer + .insert(builder.code()) + .expect("cannot insert new code into global buffer"); + + if let Some((ptr, expected)) = prev.take() { + use std::mem::transmute; + + // Test different argument counts. + unsafe { + match expected { + 0 => { + let f = transmute::<_, extern "C" fn() -> u64>(ptr); + assert_eq!(f(), 0); + } + 1 => { + let f = transmute::<_, extern "C" fn(u64) -> u64>(ptr); + assert_eq!(f(1), 1); + } + 3 => { + let f = transmute::<_, extern "C" fn(u64, u64) -> u64>(ptr); + assert_eq!(f(1, 2), 3); + } + 6 => { + let f = transmute::<_, extern "C" fn(u64, u64, u64) -> u64>(ptr); + assert_eq!(f(1, 2, 3), 6); + } + 10 => { + let f = transmute::<_, extern "C" fn(u64, u64, u64, u64) -> u64>(ptr); + assert_eq!(f(1, 2, 3, 4), 10); + } + 15 => { + let f = + transmute::<_, extern "C" fn(u64, u64, u64, u64, u64) -> u64>(ptr); + assert_eq!(f(1, 2, 3, 4, 5), 15); + } + 21 => { + let f = transmute::< + _, + extern "C" fn(u64, u64, u64, u64, u64, u64) -> u64, + >(ptr); + assert_eq!(f(1, 2, 3, 4, 5, 6), 21); + } + 28 => { + let f = transmute::< + _, + extern "C" fn(u64, u64, u64, u64, u64, u64, u64) -> u64, + >(ptr); + assert_eq!(f(1, 2, 3, 4, 5, 6, 7), 28); + } + _ => unreachable!(), + } + buffer.remove(ptr); + } + } + + let expected = (0..=n as u64).sum(); + prev = Some((ptr, expected)) + } + } } diff --git a/lib/runtime-core/src/typed_func.rs b/lib/runtime-core/src/typed_func.rs index ad4b5078edb..0b29d54eadb 100644 --- a/lib/runtime-core/src/typed_func.rs +++ b/lib/runtime-core/src/typed_func.rs @@ -190,18 +190,68 @@ where } } +/// Represents a type-erased function provided by either the host or the WebAssembly program. +pub struct DynamicFunc<'a> { + _inner: Box, + + /// The function pointer. + func: NonNull, + + /// The function environment. + func_env: Option>, + + /// The famous `vm::Ctx`. + vmctx: *mut vm::Ctx, + + /// The runtime signature of this function. + /// + /// When converted from a `Func`, this is determined by the static `Args` and `Rets` type parameters. + /// otherwise the signature is dynamically assigned during `DynamicFunc` creation, usually when creating + /// a polymorphic host function. + signature: Arc, + + _phantom: PhantomData<&'a ()>, +} + +unsafe impl<'a> Send for DynamicFunc<'a> {} + /// Represents a function that can be used by WebAssembly. pub struct Func<'a, Args = (), Rets = (), Inner: Kind = Wasm> { inner: Inner, + + /// The function pointer. func: NonNull, + + /// The function environment. func_env: Option>, + + /// The famous `vm::Ctx`. vmctx: *mut vm::Ctx, + _phantom: PhantomData<(&'a (), Args, Rets)>, } unsafe impl<'a, Args, Rets> Send for Func<'a, Args, Rets, Wasm> {} unsafe impl<'a, Args, Rets> Send for Func<'a, Args, Rets, Host> {} +impl<'a, Args, Rets, Inner> From> for DynamicFunc<'a> +where + Args: WasmTypeList, + Rets: WasmTypeList, + Inner: Kind + 'static, +{ + fn from(that: Func<'a, Args, Rets, Inner>) -> DynamicFunc<'a> { + DynamicFunc { + _inner: Box::new(that.inner), + func: that.func, + func_env: that.func_env, + vmctx: that.vmctx, + signature: Arc::new(FuncSig::new(Args::types(), Rets::types())), + _phantom: PhantomData, + } + } +} + impl<'a, Args, Rets> Func<'a, Args, Rets, Wasm> where Args: WasmTypeList, @@ -229,7 +279,7 @@ where Rets: WasmTypeList, { /// Creates a new `Func`. - pub fn new(func: F) -> Func<'a, Args, Rets, Host> + pub fn new(func: F) -> Self where Kind: HostFunctionKind, F: HostFunction, @@ -246,6 +296,106 @@ where } } +impl<'a> DynamicFunc<'a> { + /// Creates a dynamic function that is polymorphic over its argument and return types. + #[allow(unused_variables)] + #[cfg(all(unix, target_arch = "x86_64"))] + pub fn new(signature: Arc, func: F) -> Self + where + F: Fn(&mut vm::Ctx, &[crate::types::Value]) -> Vec + 'static, + { + use crate::trampoline_x64::{CallContext, TrampolineBufferBuilder}; + use crate::types::Value; + use std::convert::TryFrom; + + struct PolymorphicContext { + arg_types: Vec, + func: Box Vec>, + } + unsafe extern "C" fn enter_host_polymorphic( + ctx: *const CallContext, + args: *const u64, + ) -> u64 { + let ctx = &*(ctx as *const PolymorphicContext); + let vmctx = &mut *(*args.offset(0) as *mut vm::Ctx); + let args: Vec = ctx + .arg_types + .iter() + .enumerate() + .map(|(i, t)| { + let i = i + 1; // skip vmctx + match *t { + Type::I32 => Value::I32(*args.offset(i as _) as i32), + Type::I64 => Value::I64(*args.offset(i as _) as i64), + Type::F32 => Value::F32(f32::from_bits(*args.offset(i as _) as u32)), + Type::F64 => Value::F64(f64::from_bits(*args.offset(i as _) as u64)), + Type::V128 => { + todo!("enter_host_polymorphic: 128-bit types are not supported") + } + } + }) + .collect(); + let rets = (ctx.func)(vmctx, &args); + if rets.len() == 0 { + 0 + } else if rets.len() == 1 { + u64::try_from(rets[0].to_u128()).expect( + "128-bit return value from polymorphic host functions is not yet supported", + ) + } else { + panic!( + "multiple return values from polymorphic host functions is not yet supported" + ); + } + } + + // Disable "fat" closures for possible future changes. + if mem::size_of::() != 0 { + unimplemented!("DynamicFunc with captured environment is not yet supported"); + } + + let mut builder = TrampolineBufferBuilder::new(); + let ctx: Box = Box::new(PolymorphicContext { + arg_types: signature.params().to_vec(), + func: Box::new(func), + }); + let ctx = Box::into_raw(ctx); + builder.add_callinfo_trampoline( + enter_host_polymorphic, + ctx as *const _, + (signature.params().len() + 1) as u32, // +vmctx + ); + let ptr = builder + .insert_global() + .expect("cannot bump-allocate global trampoline memory"); + + struct AutoRelease { + ptr: NonNull, + ctx: *mut PolymorphicContext, + } + + impl Drop for AutoRelease { + fn drop(&mut self) { + unsafe { + TrampolineBufferBuilder::remove_global(self.ptr); + Box::from_raw(self.ctx); + } + } + } + + impl Kind for AutoRelease {} + + DynamicFunc { + _inner: Box::new(AutoRelease { ptr, ctx }), + func: ptr.cast::(), + func_env: None, + vmctx: ptr::null_mut(), + signature, + _phantom: PhantomData, + } + } +} + impl<'a, Args, Rets, Inner> Func<'a, Args, Rets, Inner> where Args: WasmTypeList, @@ -674,6 +824,22 @@ impl_traits!([C] S24, A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T impl_traits!([C] S25, A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V, W, X, Y); impl_traits!([C] S26, A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V, W, X, Y, Z); +impl<'a> IsExport for DynamicFunc<'a> { + fn to_export(&self) -> Export { + let func = unsafe { FuncPointer::new(self.func.as_ptr()) }; + let ctx = match self.func_env { + func_env @ Some(_) => Context::ExternalWithEnv(self.vmctx, func_env), + None => Context::Internal, + }; + + Export::Function { + func, + ctx, + signature: self.signature.clone(), + } + } +} + impl<'a, Args, Rets, Inner> IsExport for Func<'a, Args, Rets, Inner> where Args: WasmTypeList, @@ -686,12 +852,11 @@ where func_env @ Some(_) => Context::ExternalWithEnv(self.vmctx, func_env), None => Context::Internal, }; - let signature = Arc::new(FuncSig::new(Args::types(), Rets::types())); Export::Function { func, ctx, - signature, + signature: Arc::new(FuncSig::new(Args::types(), Rets::types())), } } } @@ -798,4 +963,18 @@ mod tests { }, }; } + + #[test] + fn test_many_new_dynamics() { + use crate::types::{FuncSig, Type}; + + // Check that generating a lot (1M) of polymorphic functions doesn't use up the executable buffer. + for _ in 0..1000000 { + let arglist = vec![Type::I32; 100]; + DynamicFunc::new( + Arc::new(FuncSig::new(arglist, vec![Type::I32])), + |_, _| unreachable!(), + ); + } + } } diff --git a/lib/runtime-core/src/vm.rs b/lib/runtime-core/src/vm.rs index 12c5ec41f22..73b9b2e6113 100644 --- a/lib/runtime-core/src/vm.rs +++ b/lib/runtime-core/src/vm.rs @@ -545,13 +545,13 @@ impl Ctx { /// `typed_func` module within the `wrap` functions, to wrap imported /// functions. #[repr(transparent)] -pub struct Func(pub(self) *mut c_void); +pub struct Func(*mut c_void); /// Represents a function environment pointer, like a captured /// environment of a closure. It is mostly used in the `typed_func` /// module within the `wrap` functions, to wrap imported functions. #[repr(transparent)] -pub struct FuncEnv(pub(self) *mut c_void); +pub struct FuncEnv(*mut c_void); /// Represents a function context. It is used by imported functions /// only.