From cd6594df69646aa7e3c7ebfd9da826738ba383f8 Mon Sep 17 00:00:00 2001 From: Edoardo Marangoni Date: Mon, 7 Apr 2025 12:59:30 +0200 Subject: [PATCH 01/19] feat(llvm): Use debug symbols when generating function names --- lib/compiler-llvm/src/compiler.rs | 54 +++++++++++++++++++++++++++++-- 1 file changed, 51 insertions(+), 3 deletions(-) diff --git a/lib/compiler-llvm/src/compiler.rs b/lib/compiler-llvm/src/compiler.rs index 4232054e872..24873f76762 100644 --- a/lib/compiler-llvm/src/compiler.rs +++ b/lib/compiler-llvm/src/compiler.rs @@ -9,7 +9,7 @@ use inkwell::targets::FileType; use inkwell::DLLStorageClass; use rayon::iter::ParallelBridge; use rayon::prelude::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator}; -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; use wasmer_compiler::types::function::{Compilation, UnwindInfo}; use wasmer_compiler::types::module::CompileModuleInfo; @@ -24,7 +24,7 @@ use wasmer_compiler::{ }; use wasmer_types::entity::{EntityRef, PrimaryMap}; use wasmer_types::target::Target; -use wasmer_types::{CompileError, FunctionIndex, LocalFunctionIndex, SignatureIndex}; +use wasmer_types::{CompileError, FunctionIndex, LocalFunctionIndex, ModuleInfo, SignatureIndex}; use wasmer_vm::LibCall; /// A compiler that compiles a WebAssembly module with LLVM, translating the Wasm to LLVM IR, @@ -82,6 +82,52 @@ impl SymbolRegistry for ShortNames { } } +struct ModuleBasedSymbolRegistry { + wasm_module: Arc, + local_func_names: HashMap, + short_names: ShortNames, +} + +impl ModuleBasedSymbolRegistry { + fn new(wasm_module: Arc) -> Self { + let local_func_names = HashMap::from_iter( + wasm_module + .function_names + .iter() + .map(|(f, v)| (wasm_module.local_func_index(f.clone()), v)) + .filter(|(f, _)| f.is_some()) + .map(|(f, v)| (v.clone(), f.unwrap())), + ); + Self { + wasm_module, + local_func_names, + short_names: ShortNames {}, + } + } +} + +impl SymbolRegistry for ModuleBasedSymbolRegistry { + fn symbol_to_name(&self, symbol: Symbol) -> String { + match symbol { + Symbol::LocalFunction(index) => self + .wasm_module + .function_names + .get(&self.wasm_module.func_index(index)) + .map(|v| v.clone()) + .unwrap_or(self.short_names.symbol_to_name(symbol)), + _ => self.short_names.symbol_to_name(symbol), + } + } + + fn name_to_symbol(&self, name: &str) -> Option { + if let Some(idx) = self.local_func_names.get(name) { + Some(Symbol::LocalFunction(*idx)) + } else { + self.short_names.name_to_symbol(name) + } + } +} + impl LLVMCompiler { #[allow(clippy::too_many_arguments)] fn compile_native_object( @@ -273,6 +319,8 @@ impl Compiler for LLVMCompiler { HashSet::default() }; + let symbol_registry = ModuleBasedSymbolRegistry::new(module.clone()); + let functions = function_body_inputs .iter() .collect::)>>() @@ -294,7 +342,7 @@ impl Compiler for LLVMCompiler { self.config(), memory_styles, table_styles, - &ShortNames {}, + &symbol_registry, ) }, ) From de69b95a59be711036c5374d5ceb84ac3ea1c648 Mon Sep 17 00:00:00 2001 From: Edoardo Marangoni Date: Wed, 9 Apr 2025 12:47:58 +0200 Subject: [PATCH 02/19] feat(llvm): Add new `pass_params` (g0m0) optimization --- lib/cli/src/backend.rs | 128 +++- lib/cli/src/commands/run/mod.rs | 8 +- lib/compiler-llvm/src/abi/aarch64_systemv.rs | 74 ++- lib/compiler-llvm/src/abi/mod.rs | 38 +- lib/compiler-llvm/src/abi/x86_64_systemv.rs | 76 ++- lib/compiler-llvm/src/compiler.rs | 11 +- lib/compiler-llvm/src/config.rs | 10 + lib/compiler-llvm/src/trampoline/wasm.rs | 143 ++++- lib/compiler-llvm/src/translator/code.rs | 590 +++++++++++++----- .../src/translator/intrinsics.rs | 20 +- lib/vm/src/instance/mod.rs | 25 +- 11 files changed, 911 insertions(+), 212 deletions(-) diff --git a/lib/cli/src/backend.rs b/lib/cli/src/backend.rs index 806db18bb61..43dd2a45b70 100644 --- a/lib/cli/src/backend.rs +++ b/lib/cli/src/backend.rs @@ -193,6 +193,11 @@ pub struct RuntimeOptions { #[clap(long)] llvm_debug_dir: Option, + /// Only available for the LLVM compiler. Enable the "pass-params" optimization, where the first (#0) + /// global and the first (#0) memory passed between guest functions as explicit parameters. + #[clap(long)] + enable_pass_params_opt: bool, + #[clap(flatten)] features: WasmFeatures, } @@ -267,7 +272,7 @@ impl RuntimeOptions { let backend = backends.first().unwrap(); let backend_kind = wasmer::BackendKind::from(backend); let required_features = wasmer::Engine::default_features_for_backend(&backend_kind, target); - backend.get_engine(target, &required_features) + backend.get_engine(target, &required_features, self) } pub fn get_engine_for_module(&self, module_contents: &[u8], target: &Target) -> Result { @@ -310,7 +315,7 @@ impl RuntimeOptions { filtered_backends .first() .unwrap() - .get_engine(target, required_features) + .get_engine(target, required_features, self) } #[cfg(feature = "compiler")] @@ -436,6 +441,11 @@ impl RuntimeOptions { }; use wasmer_types::entity::EntityRef; let mut config = LLVM::new(); + + if self.enable_pass_params_opt { + config.enable_pass_params_opt(); + } + struct Callbacks { debug_dir: PathBuf, } @@ -592,7 +602,12 @@ impl BackendType { } /// Get an engine for this backend type - pub fn get_engine(&self, target: &Target, features: &Features) -> Result { + pub fn get_engine( + &self, + target: &Target, + features: &Features, + runtime_opts: &RuntimeOptions, + ) -> Result { match self { #[cfg(feature = "singlepass")] Self::Singlepass => { @@ -616,7 +631,112 @@ impl BackendType { } #[cfg(feature = "llvm")] Self::LLVM => { - let config = wasmer_compiler_llvm::LLVM::new(); + use std::{fmt, fs::File, io::Write}; + + use wasmer_compiler_llvm::{ + CompiledKind, InkwellMemoryBuffer, InkwellModule, LLVMCallbacks, LLVM, + }; + use wasmer_types::entity::EntityRef; + + let mut config = wasmer_compiler_llvm::LLVM::new(); + + struct Callbacks { + debug_dir: PathBuf, + } + impl Callbacks { + fn new(debug_dir: PathBuf) -> Result { + // Create the debug dir in case it doesn't exist + std::fs::create_dir_all(&debug_dir)?; + Ok(Self { debug_dir }) + } + } + // Converts a kind into a filename, that we will use to dump + // the contents of the IR object file to. + fn types_to_signature(types: &[Type]) -> String { + types + .iter() + .map(|ty| match ty { + Type::I32 => "i".to_string(), + Type::I64 => "I".to_string(), + Type::F32 => "f".to_string(), + Type::F64 => "F".to_string(), + Type::V128 => "v".to_string(), + Type::ExternRef => "e".to_string(), + Type::FuncRef => "r".to_string(), + Type::ExceptionRef => "x".to_string(), + }) + .collect::>() + .join("") + } + // Converts a kind into a filename, that we will use to dump + // the contents of the IR object file to. + fn function_kind_to_filename(kind: &CompiledKind) -> String { + match kind { + CompiledKind::Local(local_index) => { + format!("function_{}", local_index.index()) + } + CompiledKind::FunctionCallTrampoline(func_type) => format!( + "trampoline_call_{}_{}", + types_to_signature(func_type.params()), + types_to_signature(func_type.results()) + ), + CompiledKind::DynamicFunctionTrampoline(func_type) => format!( + "trampoline_dynamic_{}_{}", + types_to_signature(func_type.params()), + types_to_signature(func_type.results()) + ), + CompiledKind::Module => "module".into(), + } + } + impl LLVMCallbacks for Callbacks { + fn preopt_ir(&self, kind: &CompiledKind, module: &InkwellModule) { + let mut path = self.debug_dir.clone(); + path.push(format!("{}.preopt.ll", function_kind_to_filename(kind))); + module + .print_to_file(&path) + .expect("Error while dumping pre optimized LLVM IR"); + } + fn postopt_ir(&self, kind: &CompiledKind, module: &InkwellModule) { + let mut path = self.debug_dir.clone(); + path.push(format!("{}.postopt.ll", function_kind_to_filename(kind))); + module + .print_to_file(&path) + .expect("Error while dumping post optimized LLVM IR"); + } + fn obj_memory_buffer( + &self, + kind: &CompiledKind, + memory_buffer: &InkwellMemoryBuffer, + ) { + let mut path = self.debug_dir.clone(); + path.push(format!("{}.o", function_kind_to_filename(kind))); + let mem_buf_slice = memory_buffer.as_slice(); + let mut file = File::create(path) + .expect("Error while creating debug object file from LLVM IR"); + let mut pos = 0; + while pos < mem_buf_slice.len() { + pos += file.write(&mem_buf_slice[pos..]).unwrap(); + } + } + } + + impl fmt::Debug for Callbacks { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "LLVMCallbacks") + } + } + + if let Some(ref llvm_debug_dir) = runtime_opts.llvm_debug_dir { + config.callbacks(Some(Arc::new(Callbacks::new(llvm_debug_dir.clone())?))); + } + if runtime_opts.enable_verifier { + config.enable_verifier(); + } + + if runtime_opts.enable_pass_params_opt { + config.enable_pass_params_opt(); + } + let engine = wasmer_compiler::EngineBuilder::new(config) .set_features(Some(features.clone())) .set_target(Some(target.clone())) diff --git a/lib/cli/src/commands/run/mod.rs b/lib/cli/src/commands/run/mod.rs index 09144a88f3c..6053ec946c6 100644 --- a/lib/cli/src/commands/run/mod.rs +++ b/lib/cli/src/commands/run/mod.rs @@ -260,9 +260,11 @@ impl Run { let engine_id = filtered_backends[0].to_string(); // Get a new engine that's compatible with the required features - if let Ok(new_engine) = - filtered_backends[0].get_engine(&Target::default(), &features) - { + if let Ok(new_engine) = filtered_backends[0].get_engine( + &Target::default(), + &features, + &self.rt, + ) { tracing::info!( "The command '{}' requires to run the Wasm module with the features {:?}. The backends available are {}. Choosing {}.", cmd.name(), diff --git a/lib/compiler-llvm/src/abi/aarch64_systemv.rs b/lib/compiler-llvm/src/abi/aarch64_systemv.rs index dce0fc86d20..d5d14fc7efd 100644 --- a/lib/compiler-llvm/src/abi/aarch64_systemv.rs +++ b/lib/compiler-llvm/src/abi/aarch64_systemv.rs @@ -15,13 +15,15 @@ use wasmer_vm::VMOffsets; use std::convert::TryInto; +use super::{FunctionKind, LocalFunctionG0M0params}; + /// Implementation of the [`Abi`] trait for the Aarch64 ABI on Linux. pub struct Aarch64SystemV {} impl Abi for Aarch64SystemV { // Given a function definition, retrieve the parameter that is the vmctx pointer. fn get_vmctx_ptr_param<'ctx>(&self, func_value: &FunctionValue<'ctx>) -> PointerValue<'ctx> { - func_value + let param = func_value .get_nth_param(u32::from( func_value .get_enum_attribute( @@ -30,8 +32,48 @@ impl Abi for Aarch64SystemV { ) .is_some(), )) - .unwrap() - .into_pointer_value() + .unwrap(); + //param.set_name("vmctx"); + + param.into_pointer_value() + } + + /// Given a function definition, retrieve the parameter that is the pointer to the first -- + /// number 0 -- local global. + fn get_g0_ptr_param<'ctx>(&self, func_value: &FunctionValue<'ctx>) -> IntValue<'ctx> { + // g0 is always after the vmctx. + let vmctx_idx = u32::from( + func_value + .get_enum_attribute( + AttributeLoc::Param(0), + Attribute::get_named_enum_kind_id("sret"), + ) + .is_some(), + ); + + let param = func_value.get_nth_param(vmctx_idx + 1).unwrap(); + param.set_name("g0"); + + param.into_int_value() + } + + /// Given a function definition, retrieve the parameter that is the pointer to the first -- + /// number 0 -- local memory. + fn get_m0_ptr_param<'ctx>(&self, func_value: &FunctionValue<'ctx>) -> PointerValue<'ctx> { + // m0 is always after g0. + let vmctx_idx = u32::from( + func_value + .get_enum_attribute( + AttributeLoc::Param(0), + Attribute::get_named_enum_kind_id("sret"), + ) + .is_some(), + ); + + let param = func_value.get_nth_param(vmctx_idx + 2).unwrap(); + param.set_name("m0_base_ptr"); + + param.into_pointer_value() } // Given a wasm function type, produce an llvm function declaration. @@ -41,11 +83,19 @@ impl Abi for Aarch64SystemV { intrinsics: &Intrinsics<'ctx>, offsets: Option<&VMOffsets>, sig: &FuncSig, + function_kind: Option, ) -> Result<(FunctionType<'ctx>, Vec<(Attribute, AttributeLoc)>), CompileError> { let user_param_types = sig.params().iter().map(|&ty| type_to_llvm(intrinsics, ty)); - let param_types = - std::iter::once(Ok(intrinsics.ptr_ty.as_basic_type_enum())).chain(user_param_types); + let mut param_types = vec![Ok(intrinsics.ptr_ty.as_basic_type_enum())]; + if function_kind.is_some_and(|v| v.is_local()) { + // The value of g0 + param_types.push(Ok(intrinsics.i32_ty.as_basic_type_enum())); + // The base pointer to m0 + param_types.push(Ok(intrinsics.ptr_ty.as_basic_type_enum())); + } + + let param_types = param_types.into_iter().chain(user_param_types); let vmctx_attributes = |i: u32| { vec![ @@ -261,6 +311,7 @@ impl Abi for Aarch64SystemV { ctx_ptr: PointerValue<'ctx>, values: &[BasicValueEnum<'ctx>], intrinsics: &Intrinsics<'ctx>, + g0m0: LocalFunctionG0M0params<'ctx>, ) -> Result>, CompileError> { // If it's an sret, allocate the return space. let sret = if llvm_fn_ty.get_return_type().is_none() && func_sig.results().len() > 1 { @@ -277,14 +328,21 @@ impl Abi for Aarch64SystemV { None }; - let values = std::iter::once(ctx_ptr.as_basic_value_enum()).chain(values.iter().copied()); + let mut args = vec![ctx_ptr.as_basic_value_enum()]; + + if let Some((g0, m0)) = g0m0 { + args.push(g0.into()); + args.push(m0.into()); + } + + let args = args.into_iter().chain(values.iter().copied()); let ret = if let Some(sret) = sret { std::iter::once(sret.as_basic_value_enum()) - .chain(values) + .chain(args) .collect() } else { - values.collect() + args.collect() }; Ok(ret) diff --git a/lib/compiler-llvm/src/abi/mod.rs b/lib/compiler-llvm/src/abi/mod.rs index 6e3a46b98dd..f4863d1d4d1 100644 --- a/lib/compiler-llvm/src/abi/mod.rs +++ b/lib/compiler-llvm/src/abi/mod.rs @@ -12,7 +12,7 @@ use inkwell::{ context::Context, targets::TargetMachine, types::FunctionType, - values::{BasicValueEnum, CallSiteValue, FunctionValue, PointerValue}, + values::{BasicValueEnum, CallSiteValue, FunctionValue, IntValue, PointerValue}, }; use wasmer_types::CompileError; use wasmer_types::FunctionType as FuncSig; @@ -37,6 +37,25 @@ pub fn get_abi(target_machine: &TargetMachine) -> Box { } } +#[derive(Debug)] +pub(crate) enum FunctionKind { + Local, + Imported, +} + +impl FunctionKind { + /// Returns `true` if the function kind is [`Local`]. + /// + /// [`Local`]: FunctionKind::Local + #[must_use] + pub(crate) fn is_local(&self) -> bool { + matches!(self, Self::Local) + } +} + +/// The two additional parameters needed for g0m0 optimization. +pub(crate) type LocalFunctionG0M0params<'ctx> = Option<(IntValue<'ctx>, PointerValue<'ctx>)>; + /// We need to produce different LLVM IR for different platforms. (Contrary to /// popular knowledge LLVM IR is not intended to be portable in that way.) This /// trait deals with differences between function signatures on different @@ -45,13 +64,29 @@ pub trait Abi { /// Given a function definition, retrieve the parameter that is the vmctx pointer. fn get_vmctx_ptr_param<'ctx>(&self, func_value: &FunctionValue<'ctx>) -> PointerValue<'ctx>; + /// Given a function definition, retrieve the parameter that is the pointer to the first -- + /// number 0 -- local global. + #[allow(unused)] + fn get_g0_ptr_param<'ctx>(&self, func_value: &FunctionValue<'ctx>) -> IntValue<'ctx>; + + /// Given a function definition, retrieve the parameter that is the pointer to the first -- + /// number 0 -- local memory. + /// + /// # Notes + /// This function assumes that g0m0 is enabled. + fn get_m0_ptr_param<'ctx>(&self, func_value: &FunctionValue<'ctx>) -> PointerValue<'ctx>; + /// Given a wasm function type, produce an llvm function declaration. + /// + /// # Notes + /// This function assumes that g0m0 is enabled. fn func_type_to_llvm<'ctx>( &self, context: &'ctx Context, intrinsics: &Intrinsics<'ctx>, offsets: Option<&VMOffsets>, sig: &FuncSig, + function_kind: Option, ) -> Result<(FunctionType<'ctx>, Vec<(Attribute, AttributeLoc)>), CompileError>; /// Marshall wasm stack values into function parameters. @@ -63,6 +98,7 @@ pub trait Abi { ctx_ptr: PointerValue<'ctx>, values: &[BasicValueEnum<'ctx>], intrinsics: &Intrinsics<'ctx>, + g0m0: LocalFunctionG0M0params<'ctx>, ) -> Result>, CompileError>; /// Given a CallSite, extract the returned values and return them in a Vec. diff --git a/lib/compiler-llvm/src/abi/x86_64_systemv.rs b/lib/compiler-llvm/src/abi/x86_64_systemv.rs index 33502dc337e..128b6c82da1 100644 --- a/lib/compiler-llvm/src/abi/x86_64_systemv.rs +++ b/lib/compiler-llvm/src/abi/x86_64_systemv.rs @@ -17,13 +17,15 @@ use wasmer_vm::VMOffsets; use std::convert::TryInto; +use super::{FunctionKind, LocalFunctionG0M0params}; + /// Implementation of the [`Abi`] trait for the AMD64 SystemV ABI. pub struct X86_64SystemV {} impl Abi for X86_64SystemV { - // Given a function definition, retrieve the parameter that is the vmctx pointer. + /// Given a function definition, retrieve the parameter that is the vmctx pointer. fn get_vmctx_ptr_param<'ctx>(&self, func_value: &FunctionValue<'ctx>) -> PointerValue<'ctx> { - func_value + let param = func_value .get_nth_param(u32::from( func_value .get_enum_attribute( @@ -32,8 +34,48 @@ impl Abi for X86_64SystemV { ) .is_some(), )) - .unwrap() - .into_pointer_value() + .unwrap(); + //param.set_name("vmctx"); + + param.into_pointer_value() + } + + /// Given a function definition, retrieve the parameter that is the pointer to the first -- + /// number 0 -- local global. + fn get_g0_ptr_param<'ctx>(&self, func_value: &FunctionValue<'ctx>) -> IntValue<'ctx> { + // g0 is always after the vmctx. + let vmctx_idx = u32::from( + func_value + .get_enum_attribute( + AttributeLoc::Param(0), + Attribute::get_named_enum_kind_id("sret"), + ) + .is_some(), + ); + + let param = func_value.get_nth_param(vmctx_idx + 1).unwrap(); + param.set_name("g0"); + + param.into_int_value() + } + + /// Given a function definition, retrieve the parameter that is the pointer to the first -- + /// number 0 -- local memory. + fn get_m0_ptr_param<'ctx>(&self, func_value: &FunctionValue<'ctx>) -> PointerValue<'ctx> { + // m0 is always after g0. + let vmctx_idx = u32::from( + func_value + .get_enum_attribute( + AttributeLoc::Param(0), + Attribute::get_named_enum_kind_id("sret"), + ) + .is_some(), + ); + + let param = func_value.get_nth_param(vmctx_idx + 2).unwrap(); + param.set_name("m0_base_ptr"); + + param.into_pointer_value() } // Given a wasm function type, produce an llvm function declaration. @@ -43,11 +85,19 @@ impl Abi for X86_64SystemV { intrinsics: &Intrinsics<'ctx>, offsets: Option<&VMOffsets>, sig: &FuncSig, + function_kind: Option, ) -> Result<(FunctionType<'ctx>, Vec<(Attribute, AttributeLoc)>), CompileError> { let user_param_types = sig.params().iter().map(|&ty| type_to_llvm(intrinsics, ty)); - let param_types = - std::iter::once(Ok(intrinsics.ptr_ty.as_basic_type_enum())).chain(user_param_types); + let mut param_types = vec![Ok(intrinsics.ptr_ty.as_basic_type_enum())]; + if function_kind.is_some_and(|v| v.is_local()) { + // The value of g0 + param_types.push(Ok(intrinsics.i32_ty.as_basic_type_enum())); + // The base pointer to m0 + param_types.push(Ok(intrinsics.ptr_ty.as_basic_type_enum())); + } + + let param_types = param_types.into_iter().chain(user_param_types); // TODO: figure out how many bytes long vmctx is, and mark it dereferenceable. (no need to mark it nonnull once we do this.) let vmctx_attributes = |i: u32| { @@ -296,6 +346,7 @@ impl Abi for X86_64SystemV { ctx_ptr: PointerValue<'ctx>, values: &[BasicValueEnum<'ctx>], intrinsics: &Intrinsics<'ctx>, + g0m0: LocalFunctionG0M0params<'ctx>, ) -> Result>, CompileError> { // If it's an sret, allocate the return space. let sret = if llvm_fn_ty.get_return_type().is_none() && func_sig.results().len() > 1 { @@ -312,14 +363,21 @@ impl Abi for X86_64SystemV { None }; - let values = std::iter::once(ctx_ptr.as_basic_value_enum()).chain(values.iter().copied()); + let mut args = vec![ctx_ptr.as_basic_value_enum()]; + + if let Some((g0, m0)) = g0m0 { + args.push(g0.into()); + args.push(m0.into()); + } + + let args = args.into_iter().chain(values.iter().copied()); let ret = if let Some(sret) = sret { std::iter::once(sret.as_basic_value_enum()) - .chain(values) + .chain(args) .collect() } else { - values.collect() + args.collect() }; Ok(ret) diff --git a/lib/compiler-llvm/src/compiler.rs b/lib/compiler-llvm/src/compiler.rs index 24873f76762..9ef3efc0ff8 100644 --- a/lib/compiler-llvm/src/compiler.rs +++ b/lib/compiler-llvm/src/compiler.rs @@ -173,7 +173,12 @@ impl LLVMCompiler { }, |func_trampoline, (i, sig)| { let name = symbol_registry.symbol_to_name(Symbol::FunctionCallTrampoline(i)); - let module = func_trampoline.trampoline_to_module(sig, self.config(), &name)?; + let module = func_trampoline.trampoline_to_module( + sig, + self.config(), + &name, + compile_info, + )?; Ok(module.write_bitcode_to_memory().as_slice().to_vec()) }, ); @@ -456,7 +461,9 @@ impl Compiler for LLVMCompiler { let target_machine = self.config().target_machine(target); FuncTrampoline::new(target_machine, binary_format).unwrap() }, - |func_trampoline, sig| func_trampoline.trampoline(sig, self.config(), ""), + |func_trampoline, sig| { + func_trampoline.trampoline(sig, self.config(), "", compile_info) + }, ) .collect::>() .into_iter() diff --git a/lib/compiler-llvm/src/config.rs b/lib/compiler-llvm/src/config.rs index f9223c32344..167d968ad62 100644 --- a/lib/compiler-llvm/src/config.rs +++ b/lib/compiler-llvm/src/config.rs @@ -43,6 +43,7 @@ pub trait LLVMCallbacks: Debug + Send + Sync { #[derive(Debug, Clone)] pub struct LLVM { pub(crate) enable_nan_canonicalization: bool, + pub(crate) enable_g0m0_opt: bool, pub(crate) enable_verifier: bool, pub(crate) opt_level: LLVMOptLevel, is_pic: bool, @@ -62,6 +63,7 @@ impl LLVM { is_pic: false, callbacks: None, middlewares: vec![], + enable_g0m0_opt: false, } } @@ -71,6 +73,14 @@ impl LLVM { self } + /// (warning: experimental) Pass the value of the first (#0) global and the base pointer of the + /// first (#0) memory as parameter between guest functions. + pub fn enable_pass_params_opt(&mut self) -> &mut Self { + // internally, the "pass_params" opt is known as g0m0 opt. + self.enable_g0m0_opt = true; + self + } + /// Callbacks that will triggered in the different compilation /// phases in LLVM. pub fn callbacks(&mut self, callbacks: Option>) -> &mut Self { diff --git a/lib/compiler-llvm/src/trampoline/wasm.rs b/lib/compiler-llvm/src/trampoline/wasm.rs index 010820beb1a..11d8ac4ac6a 100644 --- a/lib/compiler-llvm/src/trampoline/wasm.rs +++ b/lib/compiler-llvm/src/trampoline/wasm.rs @@ -1,5 +1,5 @@ use crate::{ - abi::{get_abi, Abi}, + abi::{get_abi, Abi, FunctionKind}, config::{CompiledKind, LLVM}, error::{err, err_nt}, object_file::{load_object_file, CompiledFunction}, @@ -17,8 +17,11 @@ use inkwell::{ }; use std::{cmp, convert::TryInto}; use target_lexicon::BinaryFormat; -use wasmer_compiler::types::{function::FunctionBody, relocation::RelocationTarget}; +use wasmer_compiler::types::{ + function::FunctionBody, module::CompileModuleInfo, relocation::RelocationTarget, +}; use wasmer_types::{CompileError, FunctionType as FuncType, LocalFunctionIndex}; +use wasmer_vm::MemoryStyle; pub struct FuncTrampoline { ctx: Context, @@ -59,6 +62,7 @@ impl FuncTrampoline { ty: &FuncType, config: &LLVM, name: &str, + compile_info: &CompileModuleInfo, ) -> Result { // The function type, used for the callbacks. let function = CompiledKind::FunctionCallTrampoline(ty.clone()); @@ -70,9 +74,15 @@ impl FuncTrampoline { module.set_data_layout(&target_data.get_data_layout()); let intrinsics = Intrinsics::declare(&module, &self.ctx, &target_data, &self.binary_fmt); + let func_kind = if config.enable_g0m0_opt { + Some(FunctionKind::Local) + } else { + None + }; + let (callee_ty, callee_attrs) = self.abi - .func_type_to_llvm(&self.ctx, &intrinsics, None, ty)?; + .func_type_to_llvm(&self.ctx, &intrinsics, None, ty, func_kind)?; let trampoline_ty = intrinsics.void_ty.fn_type( &[ intrinsics.ptr_ty.into(), // vmctx ptr @@ -95,6 +105,8 @@ impl FuncTrampoline { trampoline_func.add_attribute(AttributeLoc::Function, intrinsics.uwtable); trampoline_func.add_attribute(AttributeLoc::Function, intrinsics.frame_pointer); self.generate_trampoline( + config, + compile_info, trampoline_func, ty, callee_ty, @@ -141,8 +153,9 @@ impl FuncTrampoline { ty: &FuncType, config: &LLVM, name: &str, + compile_info: &CompileModuleInfo, ) -> Result { - let module = self.trampoline_to_module(ty, config, name)?; + let module = self.trampoline_to_module(ty, config, name, compile_info)?; let function = CompiledKind::FunctionCallTrampoline(ty.clone()); let target_machine = &self.target_machine; @@ -222,7 +235,7 @@ impl FuncTrampoline { let (trampoline_ty, trampoline_attrs) = self.abi - .func_type_to_llvm(&self.ctx, &intrinsics, None, ty)?; + .func_type_to_llvm(&self.ctx, &intrinsics, None, ty, None)?; let trampoline_func = module.add_function(name, trampoline_ty, Some(Linkage::External)); for (attr, attr_loc) in trampoline_attrs { trampoline_func.add_attribute(attr_loc, attr); @@ -335,6 +348,8 @@ impl FuncTrampoline { fn generate_trampoline<'ctx>( &self, + config: &LLVM, + compile_info: &CompileModuleInfo, trampoline_func: FunctionValue, func_sig: &FuncType, llvm_func_type: FunctionType, @@ -360,8 +375,11 @@ impl FuncTrampoline { } }; - let mut args_vec: Vec = - Vec::with_capacity(func_sig.params().len() + 1); + let mut args_vec: Vec = Vec::with_capacity(if config.enable_g0m0_opt { + func_sig.params().len() + 3 + } else { + func_sig.params().len() + 1 + }); if self.abi.is_sret(func_sig)? { let basic_types: Vec<_> = func_sig @@ -376,6 +394,117 @@ impl FuncTrampoline { args_vec.push(callee_vmctx_ptr.into()); + if config.enable_g0m0_opt { + let wasm_module = &compile_info.module; + let memory_styles = &compile_info.memory_styles; + let callee_vmctx_ptr_value = callee_vmctx_ptr.into_pointer_value(); + // get value of G0, get a pointer to M0's base + + let offsets = wasmer_vm::VMOffsets::new(8, wasm_module); + + let global_index = wasmer_types::GlobalIndex::from_u32(0); + let global_type = wasm_module.globals[global_index]; + let global_value_type = global_type.ty; + let global_mutability = global_type.mutability; + + let offset = + if let Some(local_global_index) = wasm_module.local_global_index(global_index) { + offsets.vmctx_vmglobal_definition(local_global_index) + } else { + offsets.vmctx_vmglobal_import(global_index) + }; + let offset = intrinsics.i32_ty.const_int(offset.into(), false); + let global_ptr = { + let global_ptr_ptr = unsafe { + err!(builder.build_gep(intrinsics.i8_ty, callee_vmctx_ptr_value, &[offset], "")) + }; + let global_ptr_ptr = + err!(builder.build_bit_cast(global_ptr_ptr, intrinsics.ptr_ty, "")) + .into_pointer_value(); + let global_ptr = err!(builder.build_load(intrinsics.ptr_ty, global_ptr_ptr, "")) + .into_pointer_value(); + + global_ptr + }; + + let global_ptr = err!(builder.build_bit_cast( + global_ptr, + type_to_llvm_ptr(intrinsics, global_value_type)?, + "", + )) + .into_pointer_value(); + + let global_value = match global_mutability { + wasmer_types::Mutability::Const => { + let value = err!(builder.build_load( + type_to_llvm(intrinsics, global_value_type)?, + global_ptr, + "g0", + )); + value + } + wasmer_types::Mutability::Var => { + err!(builder.build_load( + type_to_llvm(intrinsics, global_value_type)?, + global_ptr, + "" + )) + } + }; + + global_value.set_name("trmpl_g0"); + args_vec.push(global_value.into()); + + // load mem + let memory_index = wasmer_types::MemoryIndex::from_u32(0); + let memory_definition_ptr = if let Some(local_memory_index) = + wasm_module.local_memory_index(memory_index) + { + let offset = offsets.vmctx_vmmemory_definition(local_memory_index); + let offset = intrinsics.i32_ty.const_int(offset.into(), false); + unsafe { + err!(builder.build_gep(intrinsics.i8_ty, callee_vmctx_ptr_value, &[offset], "")) + } + } else { + let offset = offsets.vmctx_vmmemory_import(memory_index); + let offset = intrinsics.i32_ty.const_int(offset.into(), false); + let memory_definition_ptr_ptr = unsafe { + err!(builder.build_gep(intrinsics.i8_ty, callee_vmctx_ptr_value, &[offset], "")) + }; + let memory_definition_ptr_ptr = + err!(builder.build_bit_cast(memory_definition_ptr_ptr, intrinsics.ptr_ty, "",)) + .into_pointer_value(); + let memory_definition_ptr = + err!(builder.build_load(intrinsics.ptr_ty, memory_definition_ptr_ptr, "")) + .into_pointer_value(); + + memory_definition_ptr + }; + let memory_definition_ptr = + err!(builder.build_bit_cast(memory_definition_ptr, intrinsics.ptr_ty, "",)) + .into_pointer_value(); + let base_ptr = err!(builder.build_struct_gep( + intrinsics.vmmemory_definition_ty, + memory_definition_ptr, + intrinsics.vmmemory_definition_base_element, + "", + )); + + let memory_style = &memory_styles[memory_index]; + let base_ptr = if let MemoryStyle::Dynamic { .. } = memory_style { + base_ptr + } else { + let base_ptr = + err!(builder.build_load(intrinsics.ptr_ty, base_ptr, "")).into_pointer_value(); + + base_ptr + }; + + base_ptr.set_name("trmpl_m0_base_ptr"); + + args_vec.push(base_ptr.into()); + } + for (i, param_ty) in func_sig.params().iter().enumerate() { let index = intrinsics.i32_ty.const_int(i as _, false); let item_pointer = unsafe { diff --git a/lib/compiler-llvm/src/translator/code.rs b/lib/compiler-llvm/src/translator/code.rs index 1f1b7e61a67..db1a08f5527 100644 --- a/lib/compiler-llvm/src/translator/code.rs +++ b/lib/compiler-llvm/src/translator/code.rs @@ -16,8 +16,9 @@ use inkwell::{ targets::{FileType, TargetMachine}, types::{BasicType, BasicTypeEnum, FloatMathType, IntType, PointerType, VectorType}, values::{ - AnyValue, BasicMetadataValueEnum, BasicValue, BasicValueEnum, FloatValue, FunctionValue, - InstructionOpcode, InstructionValue, IntValue, PhiValue, PointerValue, VectorValue, + AnyValue, BasicMetadataValueEnum, BasicValue, BasicValueEnum, CallSiteValue, FloatValue, + FunctionValue, InstructionOpcode, InstructionValue, IntValue, PhiValue, PointerValue, + VectorValue, }, AddressSpace, AtomicOrdering, AtomicRMWBinOp, DLLStorageClass, FloatPredicate, IntPredicate, }; @@ -26,7 +27,7 @@ use smallvec::SmallVec; use target_lexicon::BinaryFormat; use crate::{ - abi::{get_abi, Abi}, + abi::{get_abi, Abi, FunctionKind, LocalFunctionG0M0params}, config::{CompiledKind, LLVM}, error::{err, err_nt}, object_file::{load_object_file, CompiledFunction}, @@ -100,6 +101,14 @@ impl FuncTranslator { let func_index = wasm_module.func_index(*local_func_index); let function_name = symbol_registry.symbol_to_name(Symbol::LocalFunction(*local_func_index)); + + let g0m0_is_enabled = config.enable_g0m0_opt; + let func_kind = if g0m0_is_enabled { + Some(FunctionKind::Local) + } else { + None + }; + let module_name = match wasm_module.name.as_ref() { None => format!(" function {function_name}"), Some(module_name) => format!("module {module_name} function {function_name}"), @@ -119,9 +128,13 @@ impl FuncTranslator { // TODO: pointer width let offsets = VMOffsets::new(8, wasm_module); let intrinsics = Intrinsics::declare(&module, &self.ctx, &target_data, &self.binary_fmt); - let (func_type, func_attrs) = - self.abi - .func_type_to_llvm(&self.ctx, &intrinsics, Some(&offsets), wasm_fn_type)?; + let (func_type, func_attrs) = self.abi.func_type_to_llvm( + &self.ctx, + &intrinsics, + Some(&offsets), + wasm_fn_type, + func_kind, + )?; let func = module.add_function(&function_name, func_type, Some(Linkage::External)); for (attr, attr_loc) in &func_attrs { @@ -189,9 +202,17 @@ impl FuncTranslator { let mut params = vec![]; let first_param = if func_type.get_return_type().is_none() && wasm_fn_type.results().len() > 1 { - 2 + if g0m0_is_enabled { + 4 + } else { + 2 + } } else { - 1 + if g0m0_is_enabled { + 3 + } else { + 1 + } }; let mut is_first_alloca = true; let mut insert_alloca = |ty, name| -> Result { @@ -230,7 +251,21 @@ impl FuncTranslator { let mut params_locals = params.clone(); params_locals.extend(locals.iter().cloned()); + let mut g0m0_params = None; + + if g0m0_is_enabled { + let value = self.abi.get_g0_ptr_param(&func); + let g0 = insert_alloca(intrinsics.i32_ty.as_basic_type_enum(), "g0")?; + err!(cache_builder.build_store(g0, value)); + g0.set_name("g0"); + let m0 = self.abi.get_m0_ptr_param(&func); + m0.set_name("m0_base_ptr"); + + g0m0_params = Some((g0, m0)); + } + let mut fcg = LLVMFunctionCodeGenerator { + g0m0: g0m0_params, context: &self.ctx, builder, alloca_builder, @@ -310,6 +345,16 @@ impl FuncTranslator { passes.push("simplifycfg"); passes.push("mem2reg"); + let llvm_dump_path = std::env::var("WASMER_LLVM_DUMP_DIR"); + if let Ok(ref llvm_dump_path) = llvm_dump_path { + let path = std::path::Path::new(llvm_dump_path); + if !path.exists() { + std::fs::create_dir_all(path).unwrap() + } + let path = path.join(format!("{function_name}.ll")); + _ = module.print_to_file(path).unwrap(); + } + module .run_passes( passes.join(",").as_str(), @@ -318,6 +363,14 @@ impl FuncTranslator { ) .unwrap(); + if let Ok(ref llvm_dump_path) = llvm_dump_path { + if !passes.is_empty() { + let path = + std::path::Path::new(llvm_dump_path).join(format!("{function_name}_opt.ll")); + _ = module.print_to_file(path).unwrap(); + } + } + if let Some(ref callbacks) = config.callbacks { callbacks.postopt_ir(&function, &module); } @@ -358,6 +411,7 @@ impl FuncTranslator { } let mem_buf_slice = memory_buffer.as_slice(); + load_object_file( mem_buf_slice, &self.func_section, @@ -1171,7 +1225,9 @@ impl<'ctx, 'a> LLVMFunctionCodeGenerator<'ctx, 'a> { let offset = err!(builder.build_int_add(var_offset, imm_offset, "")); // Look up the memory base (as pointer) and bounds (as unsigned integer). - let base_ptr = + let base_ptr = if let Some((_, ref m0)) = self.g0m0 { + m0.clone() + } else { match self .ctx .memory(memory_index, intrinsics, self.module, self.memory_styles)? @@ -1284,7 +1340,8 @@ impl<'ctx, 'a> LLVMFunctionCodeGenerator<'ctx, 'a> { ptr_to_base } MemoryCache::Static { base_ptr } => base_ptr, - }; + } + }; let value_ptr = unsafe { err!(builder.build_gep(self.intrinsics.i8_ty, base_ptr, &[offset], "")) }; err_nt!(builder @@ -1485,6 +1542,262 @@ impl<'ctx, 'a> LLVMFunctionCodeGenerator<'ctx, 'a> { panic!() } } + + fn build_g0m0_indirect_call( + &mut self, + table_index: u32, + ctx_ptr: PointerValue<'ctx>, + func_type: &FunctionType, + func_ptr: PointerValue<'ctx>, + func_index: IntValue<'ctx>, + ) -> Result<(), CompileError> { + let Some((g0, m0)) = self.g0m0 else { + return Err(CompileError::Codegen(format!( + "Call to build_g0m0_indirect_call without g0m0 parameters!" + ))); + }; + + let mut local_func_indices = vec![]; + let mut foreign_func_indices = vec![]; + + for t in &self.wasm_module.table_initializers { + if t.table_index.as_u32() == table_index { + for (func_in_table_idx, func_idx) in t.elements.iter().enumerate() { + if self.wasm_module.local_func_index(*func_idx).is_some() { + local_func_indices.push(func_in_table_idx) + } else { + foreign_func_indices.push(func_in_table_idx) + } + } + break; + } + } + + // removed with mem2reg. + // + let g0_value = err!(self.builder.build_load(self.intrinsics.i32_ty, g0, "g0")); + + let needs_switch = self.g0m0.is_some() + && !local_func_indices.is_empty() + && !foreign_func_indices.is_empty(); + + if needs_switch { + let foreign_idx_block = self + .context + .append_basic_block(self.function, "foreign_call_block"); + let local_idx_block = self + .context + .append_basic_block(self.function, "local_call_block"); + let unreachable_indirect_call_branch_block = self + .context + .append_basic_block(self.function, "unreachable_indirect_call_branch"); + + let cont = self.context.append_basic_block(self.function, "cont"); + + err!(self.builder.build_switch( + func_index, + unreachable_indirect_call_branch_block, + &local_func_indices + .into_iter() + .map(|v| ( + self.intrinsics.i32_ty.const_int(v as _, false), + local_idx_block.clone() + )) + .chain(foreign_func_indices.into_iter().map(|v| ( + self.intrinsics.i32_ty.const_int(v as _, false), + foreign_idx_block.clone() + ))) + .collect::>() + )); + + self.builder + .position_at_end(unreachable_indirect_call_branch_block); + err!(self.builder.build_unreachable()); + + //let current_block = self.builder.get_insert_block().unwrap(); + self.builder.position_at_end(local_idx_block); + let local_call_site = self.build_indirect_call( + ctx_ptr, + func_type, + func_ptr, + Some(FunctionKind::Local), + Some((g0_value.into_int_value(), m0)), + )?; + + let local_rets = self.abi.rets_from_call( + &self.builder, + self.intrinsics, + local_call_site, + func_type, + )?; + + err!(self.builder.build_unconditional_branch(cont)); + + self.builder.position_at_end(foreign_idx_block); + let foreign_call_site = self.build_indirect_call( + ctx_ptr, + func_type, + func_ptr, + Some(FunctionKind::Imported), + None, + )?; + + let foreign_rets = self.abi.rets_from_call( + &self.builder, + self.intrinsics, + foreign_call_site, + func_type, + )?; + + err!(self.builder.build_unconditional_branch(cont)); + + self.builder.position_at_end(cont); + + for i in 0..foreign_rets.len() { + let f_i = foreign_rets[i]; + let l_i = local_rets[i]; + let ty = f_i.get_type(); + let v = err!(self.builder.build_phi(ty, "")); + v.add_incoming(&[(&f_i, foreign_idx_block), (&l_i, local_idx_block)]); + self.state.push1(v.as_basic_value()); + } + } else if foreign_func_indices.is_empty() { + let call_site = self.build_indirect_call( + ctx_ptr, + func_type, + func_ptr, + Some(FunctionKind::Local), + Some((g0_value.into_int_value(), m0)), + )?; + + self.abi + .rets_from_call(&self.builder, self.intrinsics, call_site, func_type)? + .iter() + .for_each(|ret| self.state.push1(*ret)); + } else { + let call_site = self.build_indirect_call( + ctx_ptr, + func_type, + func_ptr, + Some(FunctionKind::Imported), + Some((g0_value.into_int_value(), m0)), + )?; + self.abi + .rets_from_call(&self.builder, self.intrinsics, call_site, func_type)? + .iter() + .for_each(|ret| self.state.push1(*ret)); + } + + Ok(()) + } + + fn build_indirect_call( + &mut self, + ctx_ptr: PointerValue<'ctx>, + func_type: &FunctionType, + func_ptr: PointerValue<'ctx>, + func_kind: Option, + g0m0_params: LocalFunctionG0M0params<'ctx>, + ) -> Result, CompileError> { + let (llvm_func_type, llvm_func_attrs) = self.abi.func_type_to_llvm( + self.context, + self.intrinsics, + Some(self.ctx.get_offsets()), + func_type, + func_kind, + )?; + + let params = self.state.popn_save_extra(func_type.params().len())?; + + // Apply pending canonicalizations. + let params = params + .iter() + .zip(func_type.params().iter()) + .map(|((v, info), wasm_ty)| match wasm_ty { + Type::F32 => err_nt!(self.builder.build_bit_cast( + self.apply_pending_canonicalization(*v, *info)?, + self.intrinsics.f32_ty, + "", + )), + Type::F64 => err_nt!(self.builder.build_bit_cast( + self.apply_pending_canonicalization(*v, *info)?, + self.intrinsics.f64_ty, + "", + )), + Type::V128 => self.apply_pending_canonicalization(*v, *info), + _ => Ok(*v), + }) + .collect::, _>>()?; + + let params = self.abi.args_to_call( + &self.alloca_builder, + func_type, + &llvm_func_type, + ctx_ptr, + params.as_slice(), + self.intrinsics, + g0m0_params, + )?; + + let typed_func_ptr = err!(self.builder.build_pointer_cast( + func_ptr, + self.context.ptr_type(AddressSpace::default()), + "typed_func_ptr", + )); + + /* + if self.track_state { + if let Some(offset) = opcode_offset { + let mut stackmaps = self.stackmaps.borrow_mut(); + emit_stack_map( + &info, + self.intrinsics, + self.builder, + self.index, + &mut *stackmaps, + StackmapEntryKind::Call, + &self.locals, + state, + ctx, + offset, + ) + } + } + */ + + let call_site_local = if let Some(lpad) = self.state.get_landingpad() { + let then_block = self.context.append_basic_block(self.function, "then_block"); + + let ret = err!(self.builder.build_indirect_invoke( + llvm_func_type, + typed_func_ptr, + params.as_slice(), + then_block, + lpad, + "", + )); + + self.builder.position_at_end(then_block); + ret + } else { + err!(self.builder.build_indirect_call( + llvm_func_type, + typed_func_ptr, + params + .iter() + .copied() + .map(Into::into) + .collect::>() + .as_slice(), + "indirect_call", + )) + }; + for (attr, attr_loc) in llvm_func_attrs { + call_site_local.add_attribute(attr_loc, attr); + } + + Ok(call_site_local) + } } /* @@ -1573,6 +1886,7 @@ fn finalize_opcode_stack_map<'ctx>( */ pub struct LLVMFunctionCodeGenerator<'ctx, 'a> { + g0m0: Option<(PointerValue<'ctx>, PointerValue<'ctx>)>, context: &'ctx Context, builder: Builder<'ctx>, alloca_builder: Builder<'ctx>, @@ -2306,52 +2620,78 @@ impl<'ctx, 'a> LLVMFunctionCodeGenerator<'ctx, 'a> { } Operator::GlobalGet { global_index } => { - let global_index = GlobalIndex::from_u32(global_index); - match self - .ctx - .global(global_index, self.intrinsics, self.module)? - { - GlobalCache::Const { value } => { - self.state.push1(*value); - } - GlobalCache::Mut { - ptr_to_value, - value_type, - } => { - let value = err!(self.builder.build_load(*value_type, *ptr_to_value, "")); - tbaa_label( - self.module, - self.intrinsics, - format!("global {}", global_index.as_u32()), - value.as_instruction_value().unwrap(), - ); - self.state.push1(value); + if self.g0m0.is_some() && global_index == 0 { + let Some((g0, _)) = self.g0m0 else { + unreachable!() + }; + + // Removed with mem2reg. + let value = + err!(self + .builder + .build_load(self.intrinsics.i32_ty, g0.clone(), "")); + + self.state.push1(value); + } else { + let global_index = GlobalIndex::from_u32(global_index); + match self + .ctx + .global(global_index, self.intrinsics, self.module)? + { + GlobalCache::Const { value } => { + self.state.push1(*value); + } + GlobalCache::Mut { + ptr_to_value, + value_type, + } => { + let value = + err!(self.builder.build_load(*value_type, *ptr_to_value, "")); + tbaa_label( + self.module, + self.intrinsics, + format!("global {}", global_index.as_u32()), + value.as_instruction_value().unwrap(), + ); + self.state.push1(value); + } } } } Operator::GlobalSet { global_index } => { - let global_index = GlobalIndex::from_u32(global_index); - match self - .ctx - .global(global_index, self.intrinsics, self.module)? - { - GlobalCache::Const { value: _ } => { - return Err(CompileError::Codegen(format!( - "global.set on immutable global index {}", - global_index.as_u32() - ))) - } - GlobalCache::Mut { ptr_to_value, .. } => { - let ptr_to_value = *ptr_to_value; - let (value, info) = self.state.pop1_extra()?; - let value = self.apply_pending_canonicalization(value, info)?; - let store = err!(self.builder.build_store(ptr_to_value, value)); - tbaa_label( - self.module, - self.intrinsics, - format!("global {}", global_index.as_u32()), - store, - ); + if self.g0m0.is_some() && global_index == 0 { + let Some((g0, _)) = self.g0m0 else { + unreachable!() + }; + let ptr_to_value = g0.clone(); + let (value, info) = self.state.pop1_extra()?; + let value = self.apply_pending_canonicalization(value, info)?; + let store = err!(self.builder.build_store(ptr_to_value, value)); + tbaa_label(self.module, self.intrinsics, format!("global 0",), store); + } else { + let global_index = GlobalIndex::from_u32(global_index); + match self + .ctx + .global(global_index, self.intrinsics, self.module)? + { + GlobalCache::Const { value: _ } => { + return Err(CompileError::Codegen(format!( + "global.set on immutable global index {}", + global_index.as_u32() + ))) + } + GlobalCache::Mut { ptr_to_value, .. } => { + let ptr_to_value = *ptr_to_value; + let (value, info) = self.state.pop1_extra()?; + let value = self.apply_pending_canonicalization(value, info)?; + let store = err!(self.builder.build_store(ptr_to_value, value)); + tbaa_label( + self.module, + self.intrinsics, + format!("global {}", global_index.as_u32()), + store, + ); + } } } } @@ -2405,15 +2745,28 @@ impl<'ctx, 'a> LLVMFunctionCodeGenerator<'ctx, 'a> { let sigindex = &self.wasm_module.functions[func_index]; let func_type = &self.wasm_module.signatures[*sigindex]; + let mut g0m0_params = None; + let FunctionCache { func, llvm_func_type, vmctx: callee_vmctx, attrs, } = if let Some(local_func_index) = self.wasm_module.local_func_index(func_index) { + if let Some((g0, m0)) = &self.g0m0 { + // removed with mem2reg. + let value = + err!(self + .builder + .build_load(self.intrinsics.i32_ty, g0.clone(), "")); + + g0m0_params = Some((value.into_int_value(), m0.clone())); + } + let function_name = self .symbol_registry .symbol_to_name(Symbol::LocalFunction(local_func_index)); + self.ctx.local_func( local_func_index, func_index, @@ -2466,6 +2819,7 @@ impl<'ctx, 'a> LLVMFunctionCodeGenerator<'ctx, 'a> { callee_vmctx.into_pointer_value(), params.as_slice(), self.intrinsics, + g0m0_params, )?; /* @@ -2756,120 +3110,28 @@ impl<'ctx, 'a> LLVMFunctionCodeGenerator<'ctx, 'a> { err!(self.builder.build_unreachable()); self.builder.position_at_end(continue_block); - let (llvm_func_type, llvm_func_attrs) = self.abi.func_type_to_llvm( - self.context, - self.intrinsics, - Some(self.ctx.get_offsets()), - func_type, - )?; - - let params = self.state.popn_save_extra(func_type.params().len())?; - - // Apply pending canonicalizations. - let params = params - .iter() - .zip(func_type.params().iter()) - .map(|((v, info), wasm_ty)| match wasm_ty { - Type::F32 => err_nt!(self.builder.build_bit_cast( - self.apply_pending_canonicalization(*v, *info)?, - self.intrinsics.f32_ty, - "", - )), - Type::F64 => err_nt!(self.builder.build_bit_cast( - self.apply_pending_canonicalization(*v, *info)?, - self.intrinsics.f64_ty, - "", - )), - Type::V128 => self.apply_pending_canonicalization(*v, *info), - _ => Ok(*v), - }) - .collect::, _>>()?; - - let params = self.abi.args_to_call( - &self.alloca_builder, - func_type, - &llvm_func_type, - ctx_ptr.into_pointer_value(), - params.as_slice(), - self.intrinsics, - )?; - - let typed_func_ptr = err!(self.builder.build_pointer_cast( - func_ptr, - self.context.ptr_type(AddressSpace::default()), - "typed_func_ptr", - )); - - /* - if self.track_state { - if let Some(offset) = opcode_offset { - let mut stackmaps = self.stackmaps.borrow_mut(); - emit_stack_map( - &info, - self.intrinsics, - self.builder, - self.index, - &mut *stackmaps, - StackmapEntryKind::Call, - &self.locals, - state, - ctx, - offset, - ) - } - } - */ - - let call_site = if let Some(lpad) = self.state.get_landingpad() { - let then_block = self.context.append_basic_block(self.function, "then_block"); - - let ret = err!(self.builder.build_indirect_invoke( - llvm_func_type, - typed_func_ptr, - params.as_slice(), - then_block, - lpad, - "", - )); - - self.builder.position_at_end(then_block); - ret + if self.g0m0.is_some() { + self.build_g0m0_indirect_call( + table_index, + ctx_ptr.into_pointer_value(), + func_type, + func_ptr, + func_index, + )?; } else { - err!(self.builder.build_indirect_call( - llvm_func_type, - typed_func_ptr, - params - .iter() - .copied() - .map(Into::into) - .collect::>() - .as_slice(), - "indirect_call", - )) - }; - for (attr, attr_loc) in llvm_func_attrs { - call_site.add_attribute(attr_loc, attr); - } - /* - if self.track_state { - if let Some(offset) = opcode_offset { - let mut stackmaps = self.stackmaps.borrow_mut(); - finalize_opcode_stack_map( - self.intrinsics, - self.builder, - self.index, - &mut *stackmaps, - StackmapEntryKind::Call, - offset, - ) - } - } - */ + let call_site = self.build_indirect_call( + ctx_ptr.into_pointer_value(), + func_type, + func_ptr, + None, + None, + )?; - self.abi - .rets_from_call(&self.builder, self.intrinsics, call_site, func_type)? - .iter() - .for_each(|ret| self.state.push1(*ret)); + self.abi + .rets_from_call(&self.builder, self.intrinsics, call_site, func_type)? + .iter() + .for_each(|ret| self.state.push1(*ret)); + } } /*************************** diff --git a/lib/compiler-llvm/src/translator/intrinsics.rs b/lib/compiler-llvm/src/translator/intrinsics.rs index 5ec8494e4bc..6b5a4de4617 100644 --- a/lib/compiler-llvm/src/translator/intrinsics.rs +++ b/lib/compiler-llvm/src/translator/intrinsics.rs @@ -1695,9 +1695,13 @@ impl<'ctx, 'a> CtxType<'ctx, 'a> { Entry::Occupied(entry) => entry.into_mut(), Entry::Vacant(entry) => { debug_assert!(module.get_function(function_name).is_none()); - let (llvm_func_type, llvm_func_attrs) = - self.abi - .func_type_to_llvm(context, intrinsics, Some(offsets), func_type)?; + let (llvm_func_type, llvm_func_attrs) = self.abi.func_type_to_llvm( + context, + intrinsics, + Some(offsets), + func_type, + Some(crate::abi::FunctionKind::Local), + )?; let func = module.add_function(function_name, llvm_func_type, Some(Linkage::External)); for (attr, attr_loc) in &llvm_func_attrs { @@ -1730,9 +1734,13 @@ impl<'ctx, 'a> CtxType<'ctx, 'a> { match cached_functions.entry(function_index) { Entry::Occupied(entry) => Ok(entry.into_mut()), Entry::Vacant(entry) => { - let (llvm_func_type, llvm_func_attrs) = - self.abi - .func_type_to_llvm(context, intrinsics, Some(offsets), func_type)?; + let (llvm_func_type, llvm_func_attrs) = self.abi.func_type_to_llvm( + context, + intrinsics, + Some(offsets), + func_type, + None, + )?; debug_assert!(wasm_module.local_func_index(function_index).is_none()); let offset = offsets.vmctx_vmfunction_import(function_index); let offset = intrinsics.i32_ty.const_int(offset.into(), false); diff --git a/lib/vm/src/instance/mod.rs b/lib/vm/src/instance/mod.rs index 40f7a5d97c7..3b9d9f3a5d1 100644 --- a/lib/vm/src/instance/mod.rs +++ b/lib/vm/src/instance/mod.rs @@ -12,7 +12,7 @@ use crate::export::VMExtern; use crate::imports::Imports; use crate::store::{InternalStoreHandle, StoreObjects}; use crate::table::TableElement; -use crate::trap::{catch_traps, Trap, TrapCode}; +use crate::trap::{Trap, TrapCode}; use crate::vmcontext::{ memory32_atomic_check32, memory32_atomic_check64, memory_copy, memory_fill, VMBuiltinFunctionsArray, VMCallerCheckedAnyfunc, VMContext, VMFunctionContext, @@ -20,7 +20,7 @@ use crate::vmcontext::{ VMMemoryImport, VMSharedSignatureIndex, VMTableDefinition, VMTableImport, VMTagImport, VMTrampoline, }; -use crate::{FunctionBodyPtr, MaybeInstanceOwned, TrapHandlerFn, VMFunctionBody, VMTag}; +use crate::{wasmer_call_trampoline, FunctionBodyPtr, MaybeInstanceOwned, TrapHandlerFn, VMTag}; use crate::{LinearMemory, NotifyLocation}; use crate::{VMConfig, VMFuncRef, VMFunction, VMGlobal, VMMemory, VMTable}; pub use allocator::InstanceAllocator; @@ -371,13 +371,22 @@ impl Instance { } }; - // Make the call. + let sig = self.module.functions[start_index]; + let trampoline = self.function_call_trampolines[sig]; + let values_vec = vec![].as_mut_ptr() as *mut u8; + unsafe { - catch_traps(trap_handler, config, move || { - mem::transmute::<*const VMFunctionBody, unsafe extern "C" fn(VMFunctionContext)>( - callee_address, - )(callee_vmctx) - }) + // Even though we already know the type of the function we need to call, in certain + // specific cases trampoline prepare callee arguments for specific optimizations, such + // as passing g0 and m0_base_ptr as paramters. + wasmer_call_trampoline( + trap_handler, + config, + callee_vmctx, + trampoline, + callee_address, + values_vec, + ) } } From 197c240a64d9bb3f642747b32d6334808ccbbd08 Mon Sep 17 00:00:00 2001 From: Edoardo Marangoni Date: Wed, 9 Apr 2025 13:18:40 +0200 Subject: [PATCH 03/19] chore: Make linter happy --- lib/compiler-llvm/src/abi/mod.rs | 1 + lib/compiler-llvm/src/compiler.rs | 4 +- lib/compiler-llvm/src/trampoline/wasm.rs | 17 +++--- lib/compiler-llvm/src/translator/code.rs | 68 +++++++++++------------- lib/vm/src/instance/mod.rs | 2 +- 5 files changed, 43 insertions(+), 49 deletions(-) diff --git a/lib/compiler-llvm/src/abi/mod.rs b/lib/compiler-llvm/src/abi/mod.rs index f4863d1d4d1..c4fd354369d 100644 --- a/lib/compiler-llvm/src/abi/mod.rs +++ b/lib/compiler-llvm/src/abi/mod.rs @@ -90,6 +90,7 @@ pub trait Abi { ) -> Result<(FunctionType<'ctx>, Vec<(Attribute, AttributeLoc)>), CompileError>; /// Marshall wasm stack values into function parameters. + #[allow(clippy::too_many_arguments)] fn args_to_call<'ctx>( &self, alloca_builder: &Builder<'ctx>, diff --git a/lib/compiler-llvm/src/compiler.rs b/lib/compiler-llvm/src/compiler.rs index 9ef3efc0ff8..c41e80e7091 100644 --- a/lib/compiler-llvm/src/compiler.rs +++ b/lib/compiler-llvm/src/compiler.rs @@ -94,7 +94,7 @@ impl ModuleBasedSymbolRegistry { wasm_module .function_names .iter() - .map(|(f, v)| (wasm_module.local_func_index(f.clone()), v)) + .map(|(f, v)| (wasm_module.local_func_index(*f), v)) .filter(|(f, _)| f.is_some()) .map(|(f, v)| (v.clone(), f.unwrap())), ); @@ -113,7 +113,7 @@ impl SymbolRegistry for ModuleBasedSymbolRegistry { .wasm_module .function_names .get(&self.wasm_module.func_index(index)) - .map(|v| v.clone()) + .cloned() .unwrap_or(self.short_names.symbol_to_name(symbol)), _ => self.short_names.symbol_to_name(symbol), } diff --git a/lib/compiler-llvm/src/trampoline/wasm.rs b/lib/compiler-llvm/src/trampoline/wasm.rs index 11d8ac4ac6a..97d85894996 100644 --- a/lib/compiler-llvm/src/trampoline/wasm.rs +++ b/lib/compiler-llvm/src/trampoline/wasm.rs @@ -346,6 +346,7 @@ impl FuncTrampoline { }) } + #[allow(clippy::too_many_arguments)] fn generate_trampoline<'ctx>( &self, config: &LLVM, @@ -375,11 +376,12 @@ impl FuncTrampoline { } }; - let mut args_vec: Vec = Vec::with_capacity(if config.enable_g0m0_opt { - func_sig.params().len() + 3 - } else { - func_sig.params().len() + 1 - }); + let mut args_vec: Vec = + Vec::with_capacity(if config.enable_g0m0_opt { + func_sig.params().len() + 3 + } else { + func_sig.params().len() + 1 + }); if self.abi.is_sret(func_sig)? { let basic_types: Vec<_> = func_sig @@ -436,12 +438,11 @@ impl FuncTrampoline { let global_value = match global_mutability { wasmer_types::Mutability::Const => { - let value = err!(builder.build_load( + err!(builder.build_load( type_to_llvm(intrinsics, global_value_type)?, global_ptr, "g0", - )); - value + )) } wasmer_types::Mutability::Var => { err!(builder.build_load( diff --git a/lib/compiler-llvm/src/translator/code.rs b/lib/compiler-llvm/src/translator/code.rs index db1a08f5527..fbd9f813068 100644 --- a/lib/compiler-llvm/src/translator/code.rs +++ b/lib/compiler-llvm/src/translator/code.rs @@ -207,12 +207,10 @@ impl FuncTranslator { } else { 2 } + } else if g0m0_is_enabled { + 3 } else { - if g0m0_is_enabled { - 3 - } else { - 1 - } + 1 }; let mut is_first_alloca = true; let mut insert_alloca = |ty, name| -> Result { @@ -345,15 +343,15 @@ impl FuncTranslator { passes.push("simplifycfg"); passes.push("mem2reg"); - let llvm_dump_path = std::env::var("WASMER_LLVM_DUMP_DIR"); - if let Ok(ref llvm_dump_path) = llvm_dump_path { - let path = std::path::Path::new(llvm_dump_path); - if !path.exists() { - std::fs::create_dir_all(path).unwrap() - } - let path = path.join(format!("{function_name}.ll")); - _ = module.print_to_file(path).unwrap(); - } + //let llvm_dump_path = std::env::var("WASMER_LLVM_DUMP_DIR"); + //if let Ok(ref llvm_dump_path) = llvm_dump_path { + // let path = std::path::Path::new(llvm_dump_path); + // if !path.exists() { + // std::fs::create_dir_all(path).unwrap() + // } + // let path = path.join(format!("{function_name}.ll")); + // _ = module.print_to_file(path).unwrap(); + //} module .run_passes( @@ -363,13 +361,13 @@ impl FuncTranslator { ) .unwrap(); - if let Ok(ref llvm_dump_path) = llvm_dump_path { - if !passes.is_empty() { - let path = - std::path::Path::new(llvm_dump_path).join(format!("{function_name}_opt.ll")); - _ = module.print_to_file(path).unwrap(); - } - } + //if let Ok(ref llvm_dump_path) = llvm_dump_path { + // if !passes.is_empty() { + // let path = + // std::path::Path::new(llvm_dump_path).join(format!("{function_name}_opt.ll")); + // _ = module.print_to_file(path).unwrap(); + // } + //} if let Some(ref callbacks) = config.callbacks { callbacks.postopt_ir(&function, &module); @@ -1226,7 +1224,7 @@ impl<'ctx, 'a> LLVMFunctionCodeGenerator<'ctx, 'a> { // Look up the memory base (as pointer) and bounds (as unsigned integer). let base_ptr = if let Some((_, ref m0)) = self.g0m0 { - m0.clone() + *m0 } else { match self .ctx @@ -1552,9 +1550,9 @@ impl<'ctx, 'a> LLVMFunctionCodeGenerator<'ctx, 'a> { func_index: IntValue<'ctx>, ) -> Result<(), CompileError> { let Some((g0, m0)) = self.g0m0 else { - return Err(CompileError::Codegen(format!( - "Call to build_g0m0_indirect_call without g0m0 parameters!" - ))); + return Err(CompileError::Codegen( + "Call to build_g0m0_indirect_call without g0m0 parameters!".to_string(), + )); }; let mut local_func_indices = vec![]; @@ -1601,11 +1599,11 @@ impl<'ctx, 'a> LLVMFunctionCodeGenerator<'ctx, 'a> { .into_iter() .map(|v| ( self.intrinsics.i32_ty.const_int(v as _, false), - local_idx_block.clone() + local_idx_block )) .chain(foreign_func_indices.into_iter().map(|v| ( self.intrinsics.i32_ty.const_int(v as _, false), - foreign_idx_block.clone() + foreign_idx_block ))) .collect::>() )); @@ -2626,10 +2624,7 @@ impl<'ctx, 'a> LLVMFunctionCodeGenerator<'ctx, 'a> { }; // Removed with mem2reg. - let value = - err!(self - .builder - .build_load(self.intrinsics.i32_ty, g0.clone(), "")); + let value = err!(self.builder.build_load(self.intrinsics.i32_ty, g0, "")); self.state.push1(value); } else { @@ -2663,11 +2658,11 @@ impl<'ctx, 'a> LLVMFunctionCodeGenerator<'ctx, 'a> { let Some((g0, _)) = self.g0m0 else { unreachable!() }; - let ptr_to_value = g0.clone(); + let ptr_to_value = g0; let (value, info) = self.state.pop1_extra()?; let value = self.apply_pending_canonicalization(value, info)?; let store = err!(self.builder.build_store(ptr_to_value, value)); - tbaa_label(self.module, self.intrinsics, format!("global 0",), store); + tbaa_label(self.module, self.intrinsics, "global 0".to_string(), store); } else { let global_index = GlobalIndex::from_u32(global_index); match self @@ -2755,12 +2750,9 @@ impl<'ctx, 'a> LLVMFunctionCodeGenerator<'ctx, 'a> { } = if let Some(local_func_index) = self.wasm_module.local_func_index(func_index) { if let Some((g0, m0)) = &self.g0m0 { // removed with mem2reg. - let value = - err!(self - .builder - .build_load(self.intrinsics.i32_ty, g0.clone(), "")); + let value = err!(self.builder.build_load(self.intrinsics.i32_ty, *g0, "")); - g0m0_params = Some((value.into_int_value(), m0.clone())); + g0m0_params = Some((value.into_int_value(), *m0)); } let function_name = self diff --git a/lib/vm/src/instance/mod.rs b/lib/vm/src/instance/mod.rs index 3b9d9f3a5d1..7fcab05a697 100644 --- a/lib/vm/src/instance/mod.rs +++ b/lib/vm/src/instance/mod.rs @@ -373,7 +373,7 @@ impl Instance { let sig = self.module.functions[start_index]; let trampoline = self.function_call_trampolines[sig]; - let values_vec = vec![].as_mut_ptr() as *mut u8; + let values_vec = vec![].as_mut_ptr(); unsafe { // Even though we already know the type of the function we need to call, in certain From 77bdd28671a26add87664035d58f5f2b7620da79 Mon Sep 17 00:00:00 2001 From: Edoardo Marangoni Date: Tue, 22 Apr 2025 15:22:05 +0200 Subject: [PATCH 04/19] fix(llvm): Rename `FunctionKind` to `G0M0FunctionKind`, fix generation of LLVM types with g0m0 enabled --- lib/compiler-llvm/src/abi/aarch64_systemv.rs | 4 ++-- lib/compiler-llvm/src/abi/mod.rs | 6 +++--- lib/compiler-llvm/src/abi/x86_64_systemv.rs | 4 ++-- lib/compiler-llvm/src/trampoline/wasm.rs | 4 ++-- lib/compiler-llvm/src/translator/code.rs | 16 ++++++++-------- lib/compiler-llvm/src/translator/intrinsics.rs | 10 +++++++++- 6 files changed, 26 insertions(+), 18 deletions(-) diff --git a/lib/compiler-llvm/src/abi/aarch64_systemv.rs b/lib/compiler-llvm/src/abi/aarch64_systemv.rs index d5d14fc7efd..2076f629622 100644 --- a/lib/compiler-llvm/src/abi/aarch64_systemv.rs +++ b/lib/compiler-llvm/src/abi/aarch64_systemv.rs @@ -15,7 +15,7 @@ use wasmer_vm::VMOffsets; use std::convert::TryInto; -use super::{FunctionKind, LocalFunctionG0M0params}; +use super::{G0M0FunctionKind, LocalFunctionG0M0params}; /// Implementation of the [`Abi`] trait for the Aarch64 ABI on Linux. pub struct Aarch64SystemV {} @@ -83,7 +83,7 @@ impl Abi for Aarch64SystemV { intrinsics: &Intrinsics<'ctx>, offsets: Option<&VMOffsets>, sig: &FuncSig, - function_kind: Option, + function_kind: Option, ) -> Result<(FunctionType<'ctx>, Vec<(Attribute, AttributeLoc)>), CompileError> { let user_param_types = sig.params().iter().map(|&ty| type_to_llvm(intrinsics, ty)); diff --git a/lib/compiler-llvm/src/abi/mod.rs b/lib/compiler-llvm/src/abi/mod.rs index c4fd354369d..1bc6446e4ac 100644 --- a/lib/compiler-llvm/src/abi/mod.rs +++ b/lib/compiler-llvm/src/abi/mod.rs @@ -38,12 +38,12 @@ pub fn get_abi(target_machine: &TargetMachine) -> Box { } #[derive(Debug)] -pub(crate) enum FunctionKind { +pub(crate) enum G0M0FunctionKind { Local, Imported, } -impl FunctionKind { +impl G0M0FunctionKind { /// Returns `true` if the function kind is [`Local`]. /// /// [`Local`]: FunctionKind::Local @@ -86,7 +86,7 @@ pub trait Abi { intrinsics: &Intrinsics<'ctx>, offsets: Option<&VMOffsets>, sig: &FuncSig, - function_kind: Option, + function_kind: Option, ) -> Result<(FunctionType<'ctx>, Vec<(Attribute, AttributeLoc)>), CompileError>; /// Marshall wasm stack values into function parameters. diff --git a/lib/compiler-llvm/src/abi/x86_64_systemv.rs b/lib/compiler-llvm/src/abi/x86_64_systemv.rs index 128b6c82da1..f0bad7fb01f 100644 --- a/lib/compiler-llvm/src/abi/x86_64_systemv.rs +++ b/lib/compiler-llvm/src/abi/x86_64_systemv.rs @@ -17,7 +17,7 @@ use wasmer_vm::VMOffsets; use std::convert::TryInto; -use super::{FunctionKind, LocalFunctionG0M0params}; +use super::{G0M0FunctionKind, LocalFunctionG0M0params}; /// Implementation of the [`Abi`] trait for the AMD64 SystemV ABI. pub struct X86_64SystemV {} @@ -85,7 +85,7 @@ impl Abi for X86_64SystemV { intrinsics: &Intrinsics<'ctx>, offsets: Option<&VMOffsets>, sig: &FuncSig, - function_kind: Option, + function_kind: Option, ) -> Result<(FunctionType<'ctx>, Vec<(Attribute, AttributeLoc)>), CompileError> { let user_param_types = sig.params().iter().map(|&ty| type_to_llvm(intrinsics, ty)); diff --git a/lib/compiler-llvm/src/trampoline/wasm.rs b/lib/compiler-llvm/src/trampoline/wasm.rs index 97d85894996..3332c5d5661 100644 --- a/lib/compiler-llvm/src/trampoline/wasm.rs +++ b/lib/compiler-llvm/src/trampoline/wasm.rs @@ -1,5 +1,5 @@ use crate::{ - abi::{get_abi, Abi, FunctionKind}, + abi::{get_abi, Abi, G0M0FunctionKind}, config::{CompiledKind, LLVM}, error::{err, err_nt}, object_file::{load_object_file, CompiledFunction}, @@ -75,7 +75,7 @@ impl FuncTrampoline { let intrinsics = Intrinsics::declare(&module, &self.ctx, &target_data, &self.binary_fmt); let func_kind = if config.enable_g0m0_opt { - Some(FunctionKind::Local) + Some(G0M0FunctionKind::Local) } else { None }; diff --git a/lib/compiler-llvm/src/translator/code.rs b/lib/compiler-llvm/src/translator/code.rs index fbd9f813068..a9c6c76d1f6 100644 --- a/lib/compiler-llvm/src/translator/code.rs +++ b/lib/compiler-llvm/src/translator/code.rs @@ -27,7 +27,7 @@ use smallvec::SmallVec; use target_lexicon::BinaryFormat; use crate::{ - abi::{get_abi, Abi, FunctionKind, LocalFunctionG0M0params}, + abi::{get_abi, Abi, G0M0FunctionKind, LocalFunctionG0M0params}, config::{CompiledKind, LLVM}, error::{err, err_nt}, object_file::{load_object_file, CompiledFunction}, @@ -104,7 +104,7 @@ impl FuncTranslator { let g0m0_is_enabled = config.enable_g0m0_opt; let func_kind = if g0m0_is_enabled { - Some(FunctionKind::Local) + Some(G0M0FunctionKind::Local) } else { None }; @@ -271,7 +271,7 @@ impl FuncTranslator { state, function: func, locals: params_locals, - ctx: CtxType::new(wasm_module, &func, &cache_builder, &*self.abi), + ctx: CtxType::new(wasm_module, &func, &cache_builder, &*self.abi, config), unreachable_depth: 0, memory_styles, _table_styles, @@ -1618,7 +1618,7 @@ impl<'ctx, 'a> LLVMFunctionCodeGenerator<'ctx, 'a> { ctx_ptr, func_type, func_ptr, - Some(FunctionKind::Local), + Some(G0M0FunctionKind::Local), Some((g0_value.into_int_value(), m0)), )?; @@ -1636,7 +1636,7 @@ impl<'ctx, 'a> LLVMFunctionCodeGenerator<'ctx, 'a> { ctx_ptr, func_type, func_ptr, - Some(FunctionKind::Imported), + Some(G0M0FunctionKind::Imported), None, )?; @@ -1664,7 +1664,7 @@ impl<'ctx, 'a> LLVMFunctionCodeGenerator<'ctx, 'a> { ctx_ptr, func_type, func_ptr, - Some(FunctionKind::Local), + Some(G0M0FunctionKind::Local), Some((g0_value.into_int_value(), m0)), )?; @@ -1677,7 +1677,7 @@ impl<'ctx, 'a> LLVMFunctionCodeGenerator<'ctx, 'a> { ctx_ptr, func_type, func_ptr, - Some(FunctionKind::Imported), + Some(G0M0FunctionKind::Imported), Some((g0_value.into_int_value(), m0)), )?; self.abi @@ -1694,7 +1694,7 @@ impl<'ctx, 'a> LLVMFunctionCodeGenerator<'ctx, 'a> { ctx_ptr: PointerValue<'ctx>, func_type: &FunctionType, func_ptr: PointerValue<'ctx>, - func_kind: Option, + func_kind: Option, g0m0_params: LocalFunctionG0M0params<'ctx>, ) -> Result, CompileError> { let (llvm_func_type, llvm_func_attrs) = self.abi.func_type_to_llvm( diff --git a/lib/compiler-llvm/src/translator/intrinsics.rs b/lib/compiler-llvm/src/translator/intrinsics.rs index 6b5a4de4617..ac53bd1428d 100644 --- a/lib/compiler-llvm/src/translator/intrinsics.rs +++ b/lib/compiler-llvm/src/translator/intrinsics.rs @@ -6,6 +6,7 @@ use crate::abi::Abi; use crate::error::err; +use crate::LLVM; use inkwell::values::BasicMetadataValueEnum; use inkwell::{ attributes::{Attribute, AttributeLoc}, @@ -1211,6 +1212,7 @@ pub struct FunctionCache<'ctx> { pub struct CtxType<'ctx, 'a> { ctx_ptr_value: PointerValue<'ctx>, + config: &'a LLVM, wasm_module: &'a WasmerCompilerModule, cache_builder: &'a Builder<'ctx>, abi: &'a dyn Abi, @@ -1232,8 +1234,10 @@ impl<'ctx, 'a> CtxType<'ctx, 'a> { func_value: &FunctionValue<'ctx>, cache_builder: &'a Builder<'ctx>, abi: &'a dyn Abi, + config: &'a LLVM, ) -> CtxType<'ctx, 'a> { CtxType { + config, ctx_ptr_value: abi.get_vmctx_ptr_param(func_value), wasm_module, @@ -1700,7 +1704,11 @@ impl<'ctx, 'a> CtxType<'ctx, 'a> { intrinsics, Some(offsets), func_type, - Some(crate::abi::FunctionKind::Local), + if self.config.enable_g0m0_opt { + Some(crate::abi::G0M0FunctionKind::Local) + } else { + None + }, )?; let func = module.add_function(function_name, llvm_func_type, Some(Linkage::External)); From b9a71c239aa58fd3bd5c3a8c70028354c4d5dded Mon Sep 17 00:00:00 2001 From: Edoardo Marangoni Date: Thu, 10 Apr 2025 23:05:37 +0200 Subject: [PATCH 05/19] feat: Use `String` as return for `deterministic_id` and, in compilers, use current opts as part of the ID --- lib/api/src/backend/js/entities/engine.rs | 4 ++-- lib/api/src/backend/jsc/entities/engine.rs | 4 ++-- lib/api/src/backend/v8/entities/engine.rs | 4 ++-- lib/api/src/backend/wamr/entities/engine.rs | 4 ++-- lib/api/src/backend/wasmi/entities/engine.rs | 4 ++-- lib/api/src/entities/engine/inner.rs | 2 +- lib/api/src/entities/engine/mod.rs | 2 +- lib/compiler-cranelift/src/compiler.rs | 5 +++++ lib/compiler-llvm/src/compiler.rs | 19 +++++++++++++++++++ lib/compiler-singlepass/src/compiler.rs | 5 +++++ lib/compiler/src/compiler.rs | 5 ++++- lib/compiler/src/engine/inner.rs | 13 +++++++------ .../src/runtime/module_cache/filesystem.rs | 10 +++++----- lib/wasix/src/runtime/module_cache/shared.rs | 2 +- .../src/runtime/module_cache/thread_local.rs | 4 ++-- 15 files changed, 60 insertions(+), 27 deletions(-) diff --git a/lib/api/src/backend/js/entities/engine.rs b/lib/api/src/backend/js/entities/engine.rs index 96df07412ea..cdb2316c307 100644 --- a/lib/api/src/backend/js/entities/engine.rs +++ b/lib/api/src/backend/js/entities/engine.rs @@ -5,9 +5,9 @@ use wasmer_types::{target::Target, Features}; pub struct Engine; impl Engine { - pub(crate) fn deterministic_id(&self) -> &str { + pub(crate) fn deterministic_id(&self) -> String { // All js engines have the same id - "js-generic" + String::from("js-generic") } /// Returns the WebAssembly features supported by the JS engine. diff --git a/lib/api/src/backend/jsc/entities/engine.rs b/lib/api/src/backend/jsc/entities/engine.rs index 05b4fc924dd..c026f4b18fc 100644 --- a/lib/api/src/backend/jsc/entities/engine.rs +++ b/lib/api/src/backend/jsc/entities/engine.rs @@ -166,9 +166,9 @@ impl JSCEngine { } impl Engine { - pub(crate) fn deterministic_id(&self) -> &str { + pub(crate) fn deterministic_id(&self) -> String { // All js engines have the same id - "javascriptcore" + String::from("javascriptcore") } /// Returns the WebAssembly features supported by the JSC engine for the given target. diff --git a/lib/api/src/backend/v8/entities/engine.rs b/lib/api/src/backend/v8/entities/engine.rs index 75cc5f682f2..5e5e44de7c0 100644 --- a/lib/api/src/backend/v8/entities/engine.rs +++ b/lib/api/src/backend/v8/entities/engine.rs @@ -50,8 +50,8 @@ impl Engine { Self::default() } - pub(crate) fn deterministic_id(&self) -> &str { - "v8" + pub(crate) fn deterministic_id(&self) -> String { + String::from("v8") } /// Returns the WebAssembly features supported by the V8 engine. diff --git a/lib/api/src/backend/wamr/entities/engine.rs b/lib/api/src/backend/wamr/entities/engine.rs index 6f8e5eee56d..6e30b96cf35 100644 --- a/lib/api/src/backend/wamr/entities/engine.rs +++ b/lib/api/src/backend/wamr/entities/engine.rs @@ -36,8 +36,8 @@ impl Engine { Self::default() } - pub(crate) fn deterministic_id(&self) -> &str { - "wamr" + pub(crate) fn deterministic_id(&self) -> String { + String::from("wamr") } /// Returns the WebAssembly features supported by the WAMR engine. diff --git a/lib/api/src/backend/wasmi/entities/engine.rs b/lib/api/src/backend/wasmi/entities/engine.rs index 89d8244efd3..896d40fce37 100644 --- a/lib/api/src/backend/wasmi/entities/engine.rs +++ b/lib/api/src/backend/wasmi/entities/engine.rs @@ -36,8 +36,8 @@ impl Engine { Self::default() } - pub(crate) fn deterministic_id(&self) -> &str { - "wasmi" + pub(crate) fn deterministic_id(&self) -> String { + String::from("wasmi") } /// Returns the WebAssembly features supported by the WASMI engine. diff --git a/lib/api/src/entities/engine/inner.rs b/lib/api/src/entities/engine/inner.rs index 78c6cacaa99..8c0326e1d6c 100644 --- a/lib/api/src/entities/engine/inner.rs +++ b/lib/api/src/entities/engine/inner.rs @@ -17,7 +17,7 @@ gen_rt_ty!(Engine @derives Debug, Clone); impl BackendEngine { /// Returns the deterministic id of this engine. #[inline] - pub fn deterministic_id(&self) -> &str { + pub fn deterministic_id(&self) -> String { match_rt!(on self => s { s.deterministic_id() }) diff --git a/lib/api/src/entities/engine/mod.rs b/lib/api/src/entities/engine/mod.rs index 881813a872a..1979b7628c0 100644 --- a/lib/api/src/entities/engine/mod.rs +++ b/lib/api/src/entities/engine/mod.rs @@ -51,7 +51,7 @@ impl Engine { } /// Returns the deterministic id of this engine. - pub fn deterministic_id(&self) -> &str { + pub fn deterministic_id(&self) -> String { self.be.deterministic_id() } diff --git a/lib/compiler-cranelift/src/compiler.rs b/lib/compiler-cranelift/src/compiler.rs index ae682f67a42..78156ef832e 100644 --- a/lib/compiler-cranelift/src/compiler.rs +++ b/lib/compiler-cranelift/src/compiler.rs @@ -49,6 +49,7 @@ use wasmer_types::{ /// A compiler that compiles a WebAssembly module with Cranelift, translating the Wasm to Cranelift IR, /// optimizing it and then translating to assembly. +#[derive(Debug)] pub struct CraneliftCompiler { config: Cranelift, } @@ -70,6 +71,10 @@ impl Compiler for CraneliftCompiler { "cranelift" } + fn deterministic_id(&self) -> String { + String::from("cranelift") + } + /// Get the middlewares for this compiler fn get_middlewares(&self) -> &[Arc] { &self.config.middlewares diff --git a/lib/compiler-llvm/src/compiler.rs b/lib/compiler-llvm/src/compiler.rs index c41e80e7091..b9d5b385998 100644 --- a/lib/compiler-llvm/src/compiler.rs +++ b/lib/compiler-llvm/src/compiler.rs @@ -29,6 +29,7 @@ use wasmer_vm::LibCall; /// A compiler that compiles a WebAssembly module with LLVM, translating the Wasm to LLVM IR, /// optimizing it and then translating to assembly. +#[derive(Debug)] pub struct LLVMCompiler { config: LLVM, } @@ -261,6 +262,24 @@ impl Compiler for LLVMCompiler { "llvm" } + fn deterministic_id(&self) -> String { + let mut ret = format!( + "llvm-{}", + match self.config.opt_level { + inkwell::OptimizationLevel::None => "opt0", + inkwell::OptimizationLevel::Less => "optl", + inkwell::OptimizationLevel::Default => "optd", + inkwell::OptimizationLevel::Aggressive => "opta", + } + ); + + if self.config.enable_g0m0_opt { + ret.push_str("-g0m0"); + } + + ret + } + /// Get the middlewares for this compiler fn get_middlewares(&self) -> &[Arc] { &self.config.middlewares diff --git a/lib/compiler-singlepass/src/compiler.rs b/lib/compiler-singlepass/src/compiler.rs index e5733774ec5..a48140a466b 100644 --- a/lib/compiler-singlepass/src/compiler.rs +++ b/lib/compiler-singlepass/src/compiler.rs @@ -38,6 +38,7 @@ use wasmer_types::{ /// A compiler that compiles a WebAssembly module with Singlepass. /// It does the compilation in one pass +#[derive(Debug)] pub struct SinglepassCompiler { config: Singlepass, } @@ -59,6 +60,10 @@ impl Compiler for SinglepassCompiler { "singlepass" } + fn deterministic_id(&self) -> String { + String::from("singlepass") + } + /// Get the middlewares for this compiler fn get_middlewares(&self) -> &[Arc] { &self.config.middlewares diff --git a/lib/compiler/src/compiler.rs b/lib/compiler/src/compiler.rs index 8167c78a020..ed944a08c50 100644 --- a/lib/compiler/src/compiler.rs +++ b/lib/compiler/src/compiler.rs @@ -75,12 +75,15 @@ where } /// An implementation of a Compiler from parsed WebAssembly module to Compiled native code. -pub trait Compiler: Send { +pub trait Compiler: Send + std::fmt::Debug { /// Returns a descriptive name for this compiler. /// /// Note that this is an API breaking change since 3.0 fn name(&self) -> &str; + /// Returns the deterministic id of this compiler. Same compilers with different + /// optimizations map to different deterministic IDs. + fn deterministic_id(&self) -> String; /// Validates a module. /// /// It returns the a succesful Result in case is valid, `CompileError` in case is not. diff --git a/lib/compiler/src/engine/inner.rs b/lib/compiler/src/engine/inner.rs index 01f555ce1c0..41040f24e7c 100644 --- a/lib/compiler/src/engine/inner.rs +++ b/lib/compiler/src/engine/inner.rs @@ -94,12 +94,13 @@ impl Engine { } /// Returns the deterministic id of this engine - pub fn deterministic_id(&self) -> &str { - // TODO: add a `deterministic_id` to the Compiler, so two - // compilers can actually serialize into a different deterministic_id - // if their configuration is different (eg. LLVM with optimizations vs LLVM - // without optimizations) - self.name.as_str() + pub fn deterministic_id(&self) -> String { + let i = self.inner(); + if let Some(ref c) = i.compiler { + c.deterministic_id() + } else { + self.name.clone() + } } /// Create a headless `Engine` diff --git a/lib/wasix/src/runtime/module_cache/filesystem.rs b/lib/wasix/src/runtime/module_cache/filesystem.rs index 5528ff2261f..3571a2e759d 100644 --- a/lib/wasix/src/runtime/module_cache/filesystem.rs +++ b/lib/wasix/src/runtime/module_cache/filesystem.rs @@ -42,7 +42,7 @@ impl FileSystemCache { impl ModuleCache for FileSystemCache { #[tracing::instrument(level = "debug", skip_all, fields(% key))] async fn load(&self, key: ModuleHash, engine: &Engine) -> Result { - let path = self.path(key, engine.deterministic_id()); + let path = self.path(key, &engine.deterministic_id()); self.task_manager .runtime_handle() @@ -97,7 +97,7 @@ impl ModuleCache for FileSystemCache { engine: &Engine, module: &Module, ) -> Result<(), CacheError> { - let path = self.path(key, engine.deterministic_id()); + let path = self.path(key, &engine.deterministic_id()); self.task_manager .runtime_handle() @@ -221,7 +221,7 @@ mod tests { let module = Module::new(&engine, ADD_WAT).unwrap(); let cache = FileSystemCache::new(temp.path(), create_tokio_task_manager()); let key = ModuleHash::xxhash_from_bytes([0; 8]); - let expected_path = cache.path(key, engine.deterministic_id()); + let expected_path = cache.path(key, &engine.deterministic_id()); cache.save(key, &engine, &module).await.unwrap(); @@ -262,7 +262,7 @@ mod tests { let module = Module::new(&engine, ADD_WAT).unwrap(); let key = ModuleHash::xxhash_from_bytes([0; 8]); let cache = FileSystemCache::new(temp.path(), create_tokio_task_manager()); - let expected_path = cache.path(key, engine.deterministic_id()); + let expected_path = cache.path(key, &engine.deterministic_id()); std::fs::create_dir_all(expected_path.parent().unwrap()).unwrap(); let serialized = module.serialize().unwrap(); std::fs::write(&expected_path, &serialized).unwrap(); @@ -285,7 +285,7 @@ mod tests { let module = Module::new(&engine, ADD_WAT).unwrap(); let key = ModuleHash::xxhash_from_bytes([0; 8]); let cache = FileSystemCache::new(temp.path(), create_tokio_task_manager()); - let expected_path = cache.path(key, engine.deterministic_id()); + let expected_path = cache.path(key, &engine.deterministic_id()); std::fs::create_dir_all(expected_path.parent().unwrap()).unwrap(); let serialized = module.serialize().unwrap(); let mut encoder = weezl::encode::Encoder::new(weezl::BitOrder::Msb, 8); diff --git a/lib/wasix/src/runtime/module_cache/shared.rs b/lib/wasix/src/runtime/module_cache/shared.rs index 305e80ff654..101c401d778 100644 --- a/lib/wasix/src/runtime/module_cache/shared.rs +++ b/lib/wasix/src/runtime/module_cache/shared.rs @@ -20,7 +20,7 @@ impl SharedCache { impl ModuleCache for SharedCache { #[tracing::instrument(level = "debug", skip_all, fields(%key))] async fn load(&self, key: ModuleHash, engine: &Engine) -> Result { - let key = (key, engine.deterministic_id().to_string()); + let key = (key, engine.deterministic_id()); match self.modules.get(&key) { Some(m) => { diff --git a/lib/wasix/src/runtime/module_cache/thread_local.rs b/lib/wasix/src/runtime/module_cache/thread_local.rs index c23744ae19c..c873967af24 100644 --- a/lib/wasix/src/runtime/module_cache/thread_local.rs +++ b/lib/wasix/src/runtime/module_cache/thread_local.rs @@ -31,7 +31,7 @@ impl ThreadLocalCache { impl ModuleCache for ThreadLocalCache { #[tracing::instrument(level = "debug", skip_all, fields(%key))] async fn load(&self, key: ModuleHash, engine: &Engine) -> Result { - match self.lookup(key, engine.deterministic_id()) { + match self.lookup(key, &engine.deterministic_id()) { Some(m) => { tracing::debug!("Cache hit!"); Ok(m) @@ -47,7 +47,7 @@ impl ModuleCache for ThreadLocalCache { engine: &Engine, module: &Module, ) -> Result<(), CacheError> { - self.insert(key, module, engine.deterministic_id()); + self.insert(key, module, &engine.deterministic_id()); Ok(()) } } From 184b8d4f9dbc75e5deecd291113d0b22caf5e306 Mon Sep 17 00:00:00 2001 From: Edoardo Marangoni Date: Thu, 10 Apr 2025 23:06:32 +0200 Subject: [PATCH 06/19] feat(api): Add `with_opts` function to create new engines --- lib/api/src/entities/engine/mod.rs | 23 ++++++++++++++++++++++- lib/compiler-llvm/src/compiler.rs | 10 ++++++++++ lib/compiler/src/compiler.rs | 17 +++++++++++++++++ lib/compiler/src/engine/inner.rs | 30 ++++++++++++++++++++++++++++++ lib/types/src/target.rs | 8 ++++++++ 5 files changed, 87 insertions(+), 1 deletion(-) diff --git a/lib/api/src/entities/engine/mod.rs b/lib/api/src/entities/engine/mod.rs index 1979b7628c0..f709cb1a0ab 100644 --- a/lib/api/src/entities/engine/mod.rs +++ b/lib/api/src/entities/engine/mod.rs @@ -2,7 +2,10 @@ use bytes::Bytes; use std::{path::Path, sync::Arc}; -use wasmer_types::{target::Target, DeserializeError, Features}; +use wasmer_types::{ + target::{SuggestedCompilerOptimizations, Target}, + CompileError, DeserializeError, Features, +}; #[cfg(feature = "sys")] use wasmer_compiler::Artifact; @@ -223,4 +226,22 @@ impl Engine { ) -> Result, DeserializeError> { self.be.deserialize_from_file_unchecked(file_ref) } + + /// Add suggested optimizations to this engine. + /// + /// # Note + /// + /// Not every backend supports every optimization. This function may fail (i.e. not set the + /// suggested optimizations) silently if the underlying engine backend does not support one or + /// more optimizations. + pub fn with_opts( + &mut self, + suggested_opts: &SuggestedCompilerOptimizations, + ) -> Result<(), CompileError> { + match self.be { + #[cfg(feature = "sys")] + BackendEngine::Sys(ref mut e) => e.with_opts(suggested_opts), + _ => Ok(()), + } + } } diff --git a/lib/compiler-llvm/src/compiler.rs b/lib/compiler-llvm/src/compiler.rs index b9d5b385998..204ddb44e75 100644 --- a/lib/compiler-llvm/src/compiler.rs +++ b/lib/compiler-llvm/src/compiler.rs @@ -557,4 +557,14 @@ impl Compiler for LLVMCompiler { got, }) } + + fn with_opts( + &mut self, + suggested_compiler_opts: &wasmer_types::target::SuggestedCompilerOptimizations, + ) -> Result<(), CompileError> { + if suggested_compiler_opts.pass_params.is_some_and(|v| v) { + self.config.enable_g0m0_opt = true; + } + Ok(()) + } } diff --git a/lib/compiler/src/compiler.rs b/lib/compiler/src/compiler.rs index ed944a08c50..67658a6ac46 100644 --- a/lib/compiler/src/compiler.rs +++ b/lib/compiler/src/compiler.rs @@ -9,6 +9,7 @@ use crate::{ FunctionBodyData, ModuleTranslationState, }; use enumset::EnumSet; +use wasmer_types::target::SuggestedCompilerOptimizations; use wasmer_types::{ entity::PrimaryMap, error::CompileError, @@ -84,6 +85,22 @@ pub trait Compiler: Send + std::fmt::Debug { /// Returns the deterministic id of this compiler. Same compilers with different /// optimizations map to different deterministic IDs. fn deterministic_id(&self) -> String; + + /// Add suggested optimizations to this compiler. + /// + /// # Note + /// + /// Not every compiler supports every optimization. This function may fail (i.e. not set the + /// suggested optimizations) silently if the underlying compiler does not support one or + /// more optimizations. + fn with_opts( + &mut self, + suggested_compiler_opts: &SuggestedCompilerOptimizations, + ) -> Result<(), CompileError> { + _ = suggested_compiler_opts; + Ok(()) + } + /// Validates a module. /// /// It returns the a succesful Result in case is valid, `CompileError` in case is not. diff --git a/lib/compiler/src/engine/inner.rs b/lib/compiler/src/engine/inner.rs index 41040f24e7c..140561b8668 100644 --- a/lib/compiler/src/engine/inner.rs +++ b/lib/compiler/src/engine/inner.rs @@ -284,11 +284,31 @@ impl Engine { pub fn tunables(&self) -> &dyn Tunables { self.tunables.as_ref() } + + /// Add suggested optimizations to this engine. + /// + /// # Note + /// + /// Not every backend supports every optimization. This function may fail (i.e. not set the + /// suggested optimizations) silently if the underlying engine backend does not support one or + /// more optimizations. + pub fn with_opts( + &mut self, + suggested_opts: &wasmer_types::target::SuggestedCompilerOptimizations, + ) -> Result<(), CompileError> { + let mut i = self.inner_mut(); + if let Some(ref mut c) = i.compiler { + c.with_opts(suggested_opts)?; + } + + Ok(()) + } } impl std::fmt::Debug for Engine { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.debug_struct("Engine") + .field("inner", &self.inner) .field("target", &self.target) .field("engine_id", &self.engine_id) .field("name", &self.name) @@ -314,6 +334,16 @@ pub struct EngineInner { signatures: SignatureRegistry, } +impl std::fmt::Debug for EngineInner { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("EngineInner") + .field("compiler", &self.compiler) + .field("features", &self.features) + .field("signatures", &self.signatures) + .finish() + } +} + impl EngineInner { /// Gets the compiler associated to this engine. #[cfg(feature = "compiler")] diff --git a/lib/types/src/target.rs b/lib/types/src/target.rs index 5aec0d0e63b..a231733c407 100644 --- a/lib/types/src/target.rs +++ b/lib/types/src/target.rs @@ -229,3 +229,11 @@ impl Default for Target { } } } + +/// A set of suggested optimizations, which will be passed on to backends in the chain of +/// execution. +#[derive(Debug, Clone, Default)] +pub struct SuggestedCompilerOptimizations { + /// Suggest the "pass_params" (also known as `g0m0`) optimiation to be enabled. + pub pass_params: Option, +} From 2304461570389122274bbf94f91523d8ac270426 Mon Sep 17 00:00:00 2001 From: Edoardo Marangoni Date: Wed, 9 Apr 2025 15:22:25 +0200 Subject: [PATCH 07/19] feat(cli): Add `--profiler=perfmap` support --- lib/cli/src/backend.rs | 80 +++++++++++++++++++++++++- lib/compiler-cranelift/src/compiler.rs | 4 ++ lib/compiler-cranelift/src/config.rs | 6 ++ lib/compiler-llvm/src/compiler.rs | 4 ++ lib/compiler-llvm/src/config.rs | 6 ++ lib/compiler/src/compiler.rs | 11 ++++ lib/compiler/src/engine/artifact.rs | 2 + lib/compiler/src/engine/inner.rs | 42 +++++++++++++- 8 files changed, 150 insertions(+), 5 deletions(-) diff --git a/lib/cli/src/backend.rs b/lib/cli/src/backend.rs index 43dd2a45b70..e830b798629 100644 --- a/lib/cli/src/backend.rs +++ b/lib/cli/src/backend.rs @@ -6,9 +6,9 @@ // module. #![allow(dead_code, unused_imports, unused_variables)] -use std::path::PathBuf; use std::string::ToString; use std::sync::Arc; +use std::{path::PathBuf, str::FromStr}; use anyhow::{bail, Result}; #[cfg(feature = "sys")] @@ -187,6 +187,12 @@ pub struct RuntimeOptions { #[clap(long)] enable_verifier: bool, + /// Enable a profiler. + /// + /// Available for cranelift, LLVM and singlepass. + #[clap(long, value_enum)] + profiler: Option, + /// LLVM debug directory, where IR and object files will be written to. /// /// Only available for the LLVM compiler. @@ -202,6 +208,23 @@ pub struct RuntimeOptions { features: WasmFeatures, } +#[derive(Clone, Debug)] +pub enum Profiler { + /// Perfmap-based profilers. + Perfmap, +} + +impl FromStr for Profiler { + type Err = anyhow::Error; + + fn from_str(s: &str) -> std::result::Result { + match s.to_lowercase().as_str() { + "perfmap" => Ok(Self::Perfmap), + _ => Err(anyhow::anyhow!("Unrecognized profiler: {s}")), + } + } +} + impl RuntimeOptions { pub fn get_available_backends(&self) -> Result> { // If a specific backend is explicitly requested, use it @@ -422,6 +445,14 @@ impl RuntimeOptions { if self.enable_verifier { config.enable_verifier(); } + if let Some(p) = &self.profiler { + match p { + Profiler::Perfmap => { + config.enable_perfmap() + } + } + } + Box::new(config) } #[cfg(feature = "cranelift")] @@ -430,6 +461,13 @@ impl RuntimeOptions { if self.enable_verifier { config.enable_verifier(); } + if let Some(p) = &self.profiler { + match p { + Profiler::Perfmap => { + config.enable_perfmap() + } + } + } Box::new(config) } #[cfg(feature = "llvm")] @@ -538,6 +576,14 @@ impl RuntimeOptions { if self.enable_verifier { config.enable_verifier(); } + if let Some(p) = &self.profiler { + match p { + Profiler::Perfmap => { + config.enable_perfmap() + } + } + } + Box::new(config) } BackendType::V8 | BackendType::Wamr | BackendType::Wasmi => unreachable!(), @@ -611,7 +657,17 @@ impl BackendType { match self { #[cfg(feature = "singlepass")] Self::Singlepass => { - let config = wasmer_compiler_singlepass::Singlepass::new(); + let mut config = wasmer_compiler_singlepass::Singlepass::new(); + if runtime_opts.enable_verifier { + config.enable_verifier(); + } + if let Some(p) = &runtime_opts.profiler { + match p { + Profiler::Perfmap => { + config.enable_perfmap() + } + } + } let engine = wasmer_compiler::EngineBuilder::new(config) .set_features(Some(features.clone())) .set_target(Some(target.clone())) @@ -621,7 +677,17 @@ impl BackendType { } #[cfg(feature = "cranelift")] Self::Cranelift => { - let config = wasmer_compiler_cranelift::Cranelift::new(); + let mut config = wasmer_compiler_cranelift::Cranelift::new(); + if runtime_opts.enable_verifier { + config.enable_verifier(); + } + if let Some(p) = &runtime_opts.profiler { + match p { + Profiler::Perfmap => { + config.enable_perfmap() + } + } + } let engine = wasmer_compiler::EngineBuilder::new(config) .set_features(Some(features.clone())) .set_target(Some(target.clone())) @@ -737,6 +803,14 @@ impl BackendType { config.enable_pass_params_opt(); } + if let Some(p) = &runtime_opts.profiler { + match p { + Profiler::Perfmap => { + config.enable_perfmap() + } + } + } + let engine = wasmer_compiler::EngineBuilder::new(config) .set_features(Some(features.clone())) .set_target(Some(target.clone())) diff --git a/lib/compiler-cranelift/src/compiler.rs b/lib/compiler-cranelift/src/compiler.rs index ae682f67a42..99fbe72b4a7 100644 --- a/lib/compiler-cranelift/src/compiler.rs +++ b/lib/compiler-cranelift/src/compiler.rs @@ -70,6 +70,10 @@ impl Compiler for CraneliftCompiler { "cranelift" } + fn get_perfmap_enabled(&self) -> bool { + self.config.enable_perfmap + } + /// Get the middlewares for this compiler fn get_middlewares(&self) -> &[Arc] { &self.config.middlewares diff --git a/lib/compiler-cranelift/src/config.rs b/lib/compiler-cranelift/src/config.rs index b854efbb28c..a4baf3c22d2 100644 --- a/lib/compiler-cranelift/src/config.rs +++ b/lib/compiler-cranelift/src/config.rs @@ -33,6 +33,7 @@ pub enum CraneliftOptLevel { pub struct Cranelift { enable_nan_canonicalization: bool, enable_verifier: bool, + pub(crate) enable_perfmap: bool, enable_pic: bool, opt_level: CraneliftOptLevel, /// The middleware chain. @@ -49,6 +50,7 @@ impl Cranelift { opt_level: CraneliftOptLevel::Speed, enable_pic: false, middlewares: vec![], + enable_perfmap: false, } } @@ -194,6 +196,10 @@ impl CompilerConfig for Cranelift { self.enable_verifier = true; } + fn enable_perfmap(&mut self) { + self.enable_perfmap = true; + } + fn canonicalize_nans(&mut self, enable: bool) { self.enable_nan_canonicalization = enable; } diff --git a/lib/compiler-llvm/src/compiler.rs b/lib/compiler-llvm/src/compiler.rs index c41e80e7091..af385161695 100644 --- a/lib/compiler-llvm/src/compiler.rs +++ b/lib/compiler-llvm/src/compiler.rs @@ -261,6 +261,10 @@ impl Compiler for LLVMCompiler { "llvm" } + fn get_perfmap_enabled(&self) -> bool { + self.config.enable_perfmap + } + /// Get the middlewares for this compiler fn get_middlewares(&self) -> &[Arc] { &self.config.middlewares diff --git a/lib/compiler-llvm/src/config.rs b/lib/compiler-llvm/src/config.rs index 167d968ad62..19d8ffc3683 100644 --- a/lib/compiler-llvm/src/config.rs +++ b/lib/compiler-llvm/src/config.rs @@ -45,6 +45,7 @@ pub struct LLVM { pub(crate) enable_nan_canonicalization: bool, pub(crate) enable_g0m0_opt: bool, pub(crate) enable_verifier: bool, + pub(crate) enable_perfmap: bool, pub(crate) opt_level: LLVMOptLevel, is_pic: bool, pub(crate) callbacks: Option>, @@ -59,6 +60,7 @@ impl LLVM { Self { enable_nan_canonicalization: false, enable_verifier: false, + enable_perfmap: false, opt_level: LLVMOptLevel::Aggressive, is_pic: false, callbacks: None, @@ -300,6 +302,10 @@ impl CompilerConfig for LLVM { self.is_pic = true; } + fn enable_perfmap(&mut self) { + self.enable_perfmap = true + } + /// Whether to verify compiler IR. fn enable_verifier(&mut self) { self.enable_verifier = true; diff --git a/lib/compiler/src/compiler.rs b/lib/compiler/src/compiler.rs index 8167c78a020..2435c2a4b9d 100644 --- a/lib/compiler/src/compiler.rs +++ b/lib/compiler/src/compiler.rs @@ -39,6 +39,12 @@ pub trait CompilerConfig { // in case they create an IR that they can verify. } + /// Enable generation of perfmaps to sample the JIT compiled frames. + fn enable_perfmap(&mut self) { + // By default we do nothing, each backend will need to customize this + // in case they create an IR that they can verify. + } + /// Enable NaN canonicalization. /// /// NaN canonicalization is useful when trying to run WebAssembly @@ -155,4 +161,9 @@ pub trait Compiler: Send { fn get_cpu_features_used(&self, cpu_features: &EnumSet) -> EnumSet { *cpu_features } + + /// Get whether `perfmap` is enabled or not. + fn get_perfmap_enabled(&self) -> bool { + false + } } diff --git a/lib/compiler/src/engine/artifact.rs b/lib/compiler/src/engine/artifact.rs index ba62b7c8181..a24bf0841e5 100644 --- a/lib/compiler/src/engine/artifact.rs +++ b/lib/compiler/src/engine/artifact.rs @@ -446,6 +446,8 @@ impl Artifact { get_got_address(RelocationTarget::LibCall(wasmer_vm::LibCall::EHPersonality)), )?; + engine_inner.register_perfmap(&finished_functions, module_info)?; + // Make all code compiled thus far executable. engine_inner.publish_compiled_code(); diff --git a/lib/compiler/src/engine/inner.rs b/lib/compiler/src/engine/inner.rs index 01f555ce1c0..c31a1f1ffd2 100644 --- a/lib/compiler/src/engine/inner.rs +++ b/lib/compiler/src/engine/inner.rs @@ -17,8 +17,11 @@ use shared_buffer::OwnedBuffer; #[cfg(not(target_arch = "wasm32"))] use std::path::Path; -use std::sync::atomic::{AtomicUsize, Ordering::SeqCst}; use std::sync::{Arc, Mutex}; +use std::{ + io::Write, + sync::atomic::{AtomicUsize, Ordering::SeqCst}, +}; #[cfg(feature = "compiler")] use wasmer_types::Features; @@ -27,7 +30,7 @@ use wasmer_types::{ entity::PrimaryMap, DeserializeError, FunctionIndex, FunctionType, LocalFunctionIndex, SignatureIndex, }; -use wasmer_types::{target::Target, CompileError, HashAlgorithm}; +use wasmer_types::{target::Target, CompileError, HashAlgorithm, ModuleInfo}; #[cfg(not(target_arch = "wasm32"))] use wasmer_vm::{ @@ -487,6 +490,41 @@ impl EngineInner { .unwrap() .register_frame_info(frame_info); } + + pub(crate) fn register_perfmap( + &self, + finished_functions: &PrimaryMap, + module_info: &ModuleInfo, + ) -> Result<(), CompileError> { + if self + .compiler + .as_ref() + .is_some_and(|v| v.get_perfmap_enabled()) + { + let filename = format!("/tmp/perf-{}.map", std::process::id()); + let mut file = std::io::BufWriter::new(std::fs::File::create(filename).unwrap()); + + for (func_index, code) in finished_functions.iter() { + let func_index = module_info.func_index(func_index); + let name = if let Some(func_name) = module_info.function_names.get(&func_index) { + func_name.clone() + } else { + format!("{:p}", code.ptr.0) + }; + + let sanitized_name = name.replace(['\n', '\r'], "_"); + let line = format!( + "{:p} {:x} {}\n", + code.ptr.0 as *const _, code.length, sanitized_name + ); + write!(file, "{line}").map_err(|e| CompileError::Codegen(e.to_string()))?; + file.flush() + .map_err(|e| CompileError::Codegen(e.to_string()))?; + } + } + + Ok(()) + } } #[cfg(feature = "compiler")] From 6ecdefb729d362e9dd8cbacdaea97362fec0d555 Mon Sep 17 00:00:00 2001 From: Edoardo Marangoni Date: Tue, 22 Apr 2025 16:01:09 +0200 Subject: [PATCH 08/19] chore: Make linter happy --- lib/cli/src/backend.rs | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/lib/cli/src/backend.rs b/lib/cli/src/backend.rs index e830b798629..00025dd542b 100644 --- a/lib/cli/src/backend.rs +++ b/lib/cli/src/backend.rs @@ -447,9 +447,7 @@ impl RuntimeOptions { } if let Some(p) = &self.profiler { match p { - Profiler::Perfmap => { - config.enable_perfmap() - } + Profiler::Perfmap => config.enable_perfmap(), } } @@ -463,9 +461,7 @@ impl RuntimeOptions { } if let Some(p) = &self.profiler { match p { - Profiler::Perfmap => { - config.enable_perfmap() - } + Profiler::Perfmap => config.enable_perfmap(), } } Box::new(config) @@ -578,9 +574,7 @@ impl RuntimeOptions { } if let Some(p) = &self.profiler { match p { - Profiler::Perfmap => { - config.enable_perfmap() - } + Profiler::Perfmap => config.enable_perfmap(), } } @@ -663,9 +657,7 @@ impl BackendType { } if let Some(p) = &runtime_opts.profiler { match p { - Profiler::Perfmap => { - config.enable_perfmap() - } + Profiler::Perfmap => config.enable_perfmap(), } } let engine = wasmer_compiler::EngineBuilder::new(config) @@ -683,9 +675,7 @@ impl BackendType { } if let Some(p) = &runtime_opts.profiler { match p { - Profiler::Perfmap => { - config.enable_perfmap() - } + Profiler::Perfmap => config.enable_perfmap(), } } let engine = wasmer_compiler::EngineBuilder::new(config) @@ -805,9 +795,7 @@ impl BackendType { if let Some(p) = &runtime_opts.profiler { match p { - Profiler::Perfmap => { - config.enable_perfmap() - } + Profiler::Perfmap => config.enable_perfmap(), } } From f25342be28aef91516c3eefd02f4c71ee06c0cf3 Mon Sep 17 00:00:00 2001 From: Edoardo Marangoni Date: Tue, 22 Apr 2025 16:14:38 +0200 Subject: [PATCH 09/19] feat(wasmer_config): Add `SuggestedCompilerOptimizations` field to Manifest --- lib/cli/src/commands/init.rs | 1 + lib/config/src/package/mod.rs | 22 ++++++++++ lib/package/src/convert/webc_to_package.rs | 25 ++++++++++- lib/package/src/package/manifest.rs | 48 +++++++++++++++++++--- lib/package/src/package/package.rs | 15 ++++++- 5 files changed, 103 insertions(+), 8 deletions(-) diff --git a/lib/cli/src/commands/init.rs b/lib/cli/src/commands/init.rs index d32866638b0..26b416bb3ac 100644 --- a/lib/cli/src/commands/init.rs +++ b/lib/cli/src/commands/init.rs @@ -491,6 +491,7 @@ async fn construct_manifest( map.insert("wasi".to_string(), "0.1.0-unstable".to_string()); map }), + annotations: None, }]; let mut pkg = wasmer_config::package::Package::builder( diff --git a/lib/config/src/package/mod.rs b/lib/config/src/package/mod.rs index e21e7e1d2ca..2fb81ed7d05 100644 --- a/lib/config/src/package/mod.rs +++ b/lib/config/src/package/mod.rs @@ -503,6 +503,26 @@ pub struct Module { /// Interface definitions that can be used to generate bindings to this /// module. pub bindings: Option, + /// Miscellaneous annotations from the user. + #[serde(skip_serializing_if = "Option::is_none")] + pub annotations: Option, +} + +/// Miscellaneous annotations specified by the user. +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)] +pub struct UserAnnotations { + pub suggested_compiler_optimizations: SuggestedCompilerOptimizations, +} + +/// Suggested optimization that might be operated on the module when (and if) compiled. +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)] +pub struct SuggestedCompilerOptimizations { + pub pass_params: Option, +} + +impl SuggestedCompilerOptimizations { + pub const KEY: &'static str = "suggested_compiler_optimizations"; + pub const PASS_PARAMS_KEY: &'static str = "pass_params"; } /// The interface exposed by a [`Module`]. @@ -1007,6 +1027,7 @@ mod tests { interfaces: None, kind: Some("https://webc.org/kind/wasi".to_string()), source: Path::new("test.wasm").to_path_buf(), + annotations: None, }], commands: Vec::new(), fs: vec![ @@ -1062,6 +1083,7 @@ module = "mod" wit_exports: PathBuf::from("exports.wit"), wit_bindgen: "0.0.0".parse().unwrap() })), + annotations: None }, ); } diff --git a/lib/package/src/convert/webc_to_package.rs b/lib/package/src/convert/webc_to_package.rs index c15cbc17f23..6fac5c63080 100644 --- a/lib/package/src/convert/webc_to_package.rs +++ b/lib/package/src/convert/webc_to_package.rs @@ -1,6 +1,8 @@ use std::path::Path; -use wasmer_config::package::ModuleReference; +use wasmer_config::package::{ + UserAnnotations, ModuleReference, SuggestedCompilerOptimizations, +}; use webc::Container; @@ -150,6 +152,26 @@ pub fn webc_to_package_dir(webc: &Container, target_dir: &Path) -> Result<(), Co let relative_path = format!("./{module_dir_name}/{atom_name}"); + let mut annotations = None; + + if let Some(manifest_atom) = manifest.atoms.get(&atom_name) { + if let Some(sco) = manifest_atom + .annotations + .get(SuggestedCompilerOptimizations::KEY) + { + if let Some((_, v)) = sco.as_map().and_then(|v| { + v.iter() + .find(|(k, _)| k.as_text().is_some_and(|v| v == SuggestedCompilerOptimizations::PASS_PARAMS_KEY)) + }) { + annotations = Some(UserAnnotations { + suggested_compiler_optimizations: SuggestedCompilerOptimizations { + pass_params: Some(v.as_bool().unwrap_or_default()), + }, + }); + } + } + } + pkg_manifest.modules.push(wasmer_config::package::Module { name: atom_name, source: relative_path.into(), @@ -157,6 +179,7 @@ pub fn webc_to_package_dir(webc: &Container, target_dir: &Path) -> Result<(), Co kind: None, interfaces: None, bindings: None, + annotations, }); } } diff --git a/lib/package/src/package/manifest.rs b/lib/package/src/package/manifest.rs index c9d9519ea54..17e8ee3fc5a 100644 --- a/lib/package/src/package/manifest.rs +++ b/lib/package/src/package/manifest.rs @@ -3,13 +3,14 @@ use std::{ path::{Path, PathBuf}, }; -use ciborium::Value; +use ciborium::{cbor, Value}; use semver::VersionReq; use sha2::Digest; use shared_buffer::{MmapError, OwnedBuffer}; use url::Url; #[allow(deprecated)] use wasmer_config::package::{CommandV1, CommandV2, Manifest as WasmerManifest, Package}; +use wasmer_config::package::{UserAnnotations, SuggestedCompilerOptimizations}; use webc::{ indexmap::{self, IndexMap}, metadata::AtomSignature, @@ -160,7 +161,14 @@ pub(crate) fn wasmer_manifest_to_webc( /// take a `wasmer.toml` manifest and convert it to the `*.webc` equivalent. pub(crate) fn in_memory_wasmer_manifest_to_webc( manifest: &WasmerManifest, - atoms: &BTreeMap, OwnedBuffer)>, + atoms: &BTreeMap< + String, + ( + Option, + OwnedBuffer, + Option<&UserAnnotations>, + ), + >, ) -> Result<(WebcManifest, BTreeMap), ManifestError> { let use_map = transform_dependencies(&manifest.dependencies)?; @@ -279,27 +287,55 @@ fn transform_atoms( error, })?; - atom_entries.insert(name.clone(), (module.kind.clone(), file)); + atom_entries.insert( + name.clone(), + (module.kind.clone(), file, module.annotations.as_ref()), + ); } transform_atoms_shared(&atom_entries) } fn transform_in_memory_atoms( - atoms: &BTreeMap, OwnedBuffer)>, + atoms: &BTreeMap< + String, + ( + Option, + OwnedBuffer, + Option<&UserAnnotations>, + ), + >, ) -> Result<(IndexMap, Atoms), ManifestError> { transform_atoms_shared(atoms) } fn transform_atoms_shared( - atoms: &BTreeMap, OwnedBuffer)>, + atoms: &BTreeMap< + String, + ( + Option, + OwnedBuffer, + Option<&UserAnnotations>, + ), + >, ) -> Result<(IndexMap, Atoms), ManifestError> { let mut atom_files = BTreeMap::new(); let mut metadata = IndexMap::new(); - for (name, (kind, content)) in atoms.iter() { + for (name, (kind, content, misc_annotations)) in atoms.iter() { // Create atom with annotations including Wasm features if available let mut annotations = IndexMap::new(); + if let Some(misc_annotations) = misc_annotations { + if let Some(pass_params) = misc_annotations + .suggested_compiler_optimizations + .pass_params + { + annotations.insert( + SuggestedCompilerOptimizations::KEY.to_string(), + cbor!({"pass_params" => pass_params}).unwrap(), + ); + } + } // Detect required WebAssembly features by analyzing the module binary let features_result = wasmer_types::Features::detect_from_wasm(content); diff --git a/lib/package/src/package/package.rs b/lib/package/src/package/package.rs index 4ca0f6955b1..74f8dc664dc 100644 --- a/lib/package/src/package/package.rs +++ b/lib/package/src/package/package.rs @@ -355,8 +355,21 @@ impl Package { let volumes = new_volumes; + let mut annotated_atoms = BTreeMap::new(); + + for (atom_name, (a, b)) in atoms { + let annotations = + if let Some(module) = manifest.modules.iter().find(|v| v.name == atom_name) { + module.annotations.as_ref() + } else { + None + }; + + annotated_atoms.insert(atom_name, (a, b, annotations)); + } + let (mut manifest, atoms) = - super::manifest::in_memory_wasmer_manifest_to_webc(&manifest, &atoms)?; + super::manifest::in_memory_wasmer_manifest_to_webc(&manifest, &annotated_atoms)?; if let Some(entry) = manifest.package.get_mut(Wapm::KEY) { let mut wapm: Wapm = entry.deserialized()?; From 4c5e5ecccdd4c95832153c324f9b905dff50d13b Mon Sep 17 00:00:00 2001 From: Edoardo Marangoni Date: Tue, 22 Apr 2025 16:15:01 +0200 Subject: [PATCH 10/19] feat(wasmer-types): Minor changes to comments for `SuggestedCompilerOptimizations` --- lib/types/src/target.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/types/src/target.rs b/lib/types/src/target.rs index a231733c407..7a53a4f7af1 100644 --- a/lib/types/src/target.rs +++ b/lib/types/src/target.rs @@ -230,10 +230,12 @@ impl Default for Target { } } -/// A set of suggested optimizations, which will be passed on to backends in the chain of -/// execution. -#[derive(Debug, Clone, Default)] +/// Suggested optimization that might be operated on the module when (and if) compiled. +/// +// Note: This type is a copy of `wasmer_config::package::SuggestedCompilerOptimizations`, so to +// avoid dependencies on `wasmer_config` for crates that already depend on `wasmer_types`. +#[derive(Clone, Debug, PartialEq, Eq, Default)] pub struct SuggestedCompilerOptimizations { - /// Suggest the "pass_params" (also known as `g0m0`) optimiation to be enabled. + /// Suggest the `pass_params` (also known as g0m0) optimization pass. pub pass_params: Option, } From 0a04f3990b205968213c01830d662d02459aa77a Mon Sep 17 00:00:00 2001 From: Edoardo Marangoni Date: Tue, 22 Apr 2025 16:15:42 +0200 Subject: [PATCH 11/19] feat(wasix): Extract compiler optimizations suggestions from commands --- lib/wasix/src/bin_factory/binary_package.rs | 7 ++- lib/wasix/src/runtime/mod.rs | 31 ++++++++++-- .../package_loader/load_package_tree.rs | 50 +++++++++++++++++-- 3 files changed, 81 insertions(+), 7 deletions(-) diff --git a/lib/wasix/src/bin_factory/binary_package.rs b/lib/wasix/src/bin_factory/binary_package.rs index 78649b14829..6688c712b9c 100644 --- a/lib/wasix/src/bin_factory/binary_package.rs +++ b/lib/wasix/src/bin_factory/binary_package.rs @@ -4,7 +4,9 @@ use anyhow::Context; use once_cell::sync::OnceCell; use sha2::Digest; use virtual_fs::FileSystem; -use wasmer_config::package::{PackageHash, PackageId, PackageSource}; +use wasmer_config::package::{ + PackageHash, PackageId, PackageSource, SuggestedCompilerOptimizations, +}; use wasmer_package::package::Package; use webc::compat::SharedBytes; use webc::Container; @@ -24,6 +26,7 @@ pub struct BinaryPackageCommand { pub(crate) atom: SharedBytes, hash: ModuleHash, features: Option, + pub suggested_compiler_optimizations: SuggestedCompilerOptimizations, } impl BinaryPackageCommand { @@ -33,6 +36,7 @@ impl BinaryPackageCommand { atom: SharedBytes, hash: ModuleHash, features: Option, + suggested_compiler_optimizations: SuggestedCompilerOptimizations, ) -> Self { Self { name, @@ -40,6 +44,7 @@ impl BinaryPackageCommand { atom, hash, features, + suggested_compiler_optimizations, } } diff --git a/lib/wasix/src/runtime/mod.rs b/lib/wasix/src/runtime/mod.rs index cfe2b974ce6..cc90a94e556 100644 --- a/lib/wasix/src/runtime/mod.rs +++ b/lib/wasix/src/runtime/mod.rs @@ -5,7 +5,10 @@ pub mod task_manager; pub use self::task_manager::{SpawnMemoryType, VirtualTaskManager}; use self::{module_cache::CacheError, task_manager::InlineWaker}; -use wasmer_types::ModuleHash; +use wasmer_config::package::SuggestedCompilerOptimizations; +use wasmer_types::{ + target::SuggestedCompilerOptimizations as WasmerSuggestedCompilerOptimizations, ModuleHash, +}; use std::{ fmt, @@ -15,7 +18,7 @@ use std::{ use futures::future::BoxFuture; use virtual_net::{DynVirtualNetworking, VirtualNetworking}; -use wasmer::{Module, RuntimeError}; +use wasmer::{CompileError, Module, RuntimeError}; use wasmer_wasix_types::wasi::ExitCode; #[cfg(feature = "journal")] @@ -77,6 +80,17 @@ where wasmer::Engine::default() } + fn engine_with_suggested_opts( + &self, + suggested_opts: &SuggestedCompilerOptimizations, + ) -> Result { + let mut engine = self.engine(); + engine.with_opts(&WasmerSuggestedCompilerOptimizations { + pass_params: suggested_opts.pass_params, + })?; + Ok(engine) + } + /// Create a new [`wasmer::Store`]. fn new_store(&self) -> wasmer::Store { cfg_if::cfg_if! { @@ -106,11 +120,22 @@ where &self, cmd: &BinaryPackageCommand, ) -> BoxFuture<'_, Result> { - let engine = self.engine(); let hash = *cmd.hash(); let wasm = cmd.atom(); let module_cache = self.module_cache(); + let engine = match self.engine_with_suggested_opts(&cmd.suggested_compiler_optimizations) { + Ok(engine) => engine, + Err(error) => { + return Box::pin(async move { + Err(SpawnError::CompileError { + module_hash: hash, + error, + }) + }) + } + }; + let task = async move { load_module(&engine, &module_cache, &wasm, hash).await }; Box::pin(task) diff --git a/lib/wasix/src/runtime/package_loader/load_package_tree.rs b/lib/wasix/src/runtime/package_loader/load_package_tree.rs index e81c0197be8..743fb8ac392 100644 --- a/lib/wasix/src/runtime/package_loader/load_package_tree.rs +++ b/lib/wasix/src/runtime/package_loader/load_package_tree.rs @@ -10,7 +10,7 @@ use futures::{future::BoxFuture, StreamExt, TryStreamExt}; use once_cell::sync::OnceCell; use petgraph::visit::EdgeRef; use virtual_fs::{FileSystem, OverlayFileSystem, UnionFileSystem, WebcVolumeFileSystem}; -use wasmer_config::package::PackageId; +use wasmer_config::package::{PackageId, SuggestedCompilerOptimizations}; use wasmer_package::utils::wasm_annotations_to_features; use webc::metadata::annotations::Atom as AtomAnnotation; use webc::{Container, Volume}; @@ -208,12 +208,47 @@ fn load_binary_command( None }; - let cmd = - BinaryPackageCommand::new(command_name.to_string(), cmd.clone(), atom, hash, features); + let suggested_compiler_optimizations = + if let Some(atom_metadata) = webc.manifest().atoms.get(&atom_name) { + extract_suggested_compiler_opts_from_atom_metadata(atom_metadata) + } else { + wasmer_config::package::SuggestedCompilerOptimizations::default() + }; + + let cmd = BinaryPackageCommand::new( + command_name.to_string(), + cmd.clone(), + atom, + hash, + features, + suggested_compiler_optimizations, + ); Ok(Some(cmd)) } +fn extract_suggested_compiler_opts_from_atom_metadata( + atom_metadata: &webc::metadata::Atom, +) -> wasmer_config::package::SuggestedCompilerOptimizations { + let mut ret = SuggestedCompilerOptimizations::default(); + + if let Some(sco) = atom_metadata + .annotations + .get(SuggestedCompilerOptimizations::KEY) + { + if let Some((_, v)) = sco.as_map().and_then(|v| { + v.iter().find(|(k, _)| { + k.as_text() + .is_some_and(|v| v == SuggestedCompilerOptimizations::PASS_PARAMS_KEY) + }) + }) { + ret.pass_params = v.as_bool() + } + } + + ret +} + fn atom_name_for_command( command_name: &str, cmd: &webc::metadata::Command, @@ -281,12 +316,21 @@ fn legacy_atom_hack( None }; + // Get WebAssembly features from manifest atom annotations + let suggested_opts_from_manifest = if let Some(atom_metadata) = webc.manifest().atoms.get(&name) + { + extract_suggested_compiler_opts_from_atom_metadata(atom_metadata) + } else { + SuggestedCompilerOptimizations::default() + }; + Ok(Some(BinaryPackageCommand::new( command_name.to_string(), metadata.clone(), atom, hash, features, + suggested_opts_from_manifest, ))) } From 88633cb6206e8e609fa7bcd7afd4fcba9a206b2d Mon Sep 17 00:00:00 2001 From: Edoardo Marangoni Date: Tue, 22 Apr 2025 16:15:49 +0200 Subject: [PATCH 12/19] chore: Minor changes --- lib/compiler/src/compiler.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/compiler/src/compiler.rs b/lib/compiler/src/compiler.rs index 67658a6ac46..13f6486517e 100644 --- a/lib/compiler/src/compiler.rs +++ b/lib/compiler/src/compiler.rs @@ -9,11 +9,10 @@ use crate::{ FunctionBodyData, ModuleTranslationState, }; use enumset::EnumSet; -use wasmer_types::target::SuggestedCompilerOptimizations; use wasmer_types::{ entity::PrimaryMap, error::CompileError, - target::{CpuFeature, Target}, + target::{CpuFeature, SuggestedCompilerOptimizations, Target}, Features, LocalFunctionIndex, }; #[cfg(feature = "translator")] From 79f2135de868b5742f81038fd4e9e89b56649443 Mon Sep 17 00:00:00 2001 From: Edoardo Marangoni Date: Tue, 22 Apr 2025 16:29:41 +0200 Subject: [PATCH 13/19] fix(compiler): Feature-gate `register_perfmap` for non-wasm32 targets --- lib/compiler/src/engine/artifact.rs | 5 ++++- lib/compiler/src/engine/inner.rs | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/compiler/src/engine/artifact.rs b/lib/compiler/src/engine/artifact.rs index a24bf0841e5..a00cca0dad1 100644 --- a/lib/compiler/src/engine/artifact.rs +++ b/lib/compiler/src/engine/artifact.rs @@ -446,7 +446,10 @@ impl Artifact { get_got_address(RelocationTarget::LibCall(wasmer_vm::LibCall::EHPersonality)), )?; - engine_inner.register_perfmap(&finished_functions, module_info)?; + #[cfg(not(target_arch = "wasm32"))] + { + engine_inner.register_perfmap(&finished_functions, module_info)?; + } // Make all code compiled thus far executable. engine_inner.publish_compiled_code(); diff --git a/lib/compiler/src/engine/inner.rs b/lib/compiler/src/engine/inner.rs index c31a1f1ffd2..36da69b6135 100644 --- a/lib/compiler/src/engine/inner.rs +++ b/lib/compiler/src/engine/inner.rs @@ -491,6 +491,7 @@ impl EngineInner { .register_frame_info(frame_info); } + #[cfg(not(target_arch = "wasm32"))] pub(crate) fn register_perfmap( &self, finished_functions: &PrimaryMap, From f3cf1aaada2bb290a3b7ef9bbcf75f9f64b55851 Mon Sep 17 00:00:00 2001 From: Edoardo Marangoni Date: Tue, 22 Apr 2025 16:34:32 +0200 Subject: [PATCH 14/19] chore: Make linter happy --- lib/package/src/convert/webc_to_package.rs | 11 ++++---- lib/package/src/package/manifest.rs | 29 +++------------------- 2 files changed, 10 insertions(+), 30 deletions(-) diff --git a/lib/package/src/convert/webc_to_package.rs b/lib/package/src/convert/webc_to_package.rs index 6fac5c63080..81dea85b909 100644 --- a/lib/package/src/convert/webc_to_package.rs +++ b/lib/package/src/convert/webc_to_package.rs @@ -1,8 +1,6 @@ use std::path::Path; -use wasmer_config::package::{ - UserAnnotations, ModuleReference, SuggestedCompilerOptimizations, -}; +use wasmer_config::package::{ModuleReference, SuggestedCompilerOptimizations, UserAnnotations}; use webc::Container; @@ -160,8 +158,11 @@ pub fn webc_to_package_dir(webc: &Container, target_dir: &Path) -> Result<(), Co .get(SuggestedCompilerOptimizations::KEY) { if let Some((_, v)) = sco.as_map().and_then(|v| { - v.iter() - .find(|(k, _)| k.as_text().is_some_and(|v| v == SuggestedCompilerOptimizations::PASS_PARAMS_KEY)) + v.iter().find(|(k, _)| { + k.as_text().is_some_and(|v| { + v == SuggestedCompilerOptimizations::PASS_PARAMS_KEY + }) + }) }) { annotations = Some(UserAnnotations { suggested_compiler_optimizations: SuggestedCompilerOptimizations { diff --git a/lib/package/src/package/manifest.rs b/lib/package/src/package/manifest.rs index 17e8ee3fc5a..e716e46332d 100644 --- a/lib/package/src/package/manifest.rs +++ b/lib/package/src/package/manifest.rs @@ -10,7 +10,7 @@ use shared_buffer::{MmapError, OwnedBuffer}; use url::Url; #[allow(deprecated)] use wasmer_config::package::{CommandV1, CommandV2, Manifest as WasmerManifest, Package}; -use wasmer_config::package::{UserAnnotations, SuggestedCompilerOptimizations}; +use wasmer_config::package::{SuggestedCompilerOptimizations, UserAnnotations}; use webc::{ indexmap::{self, IndexMap}, metadata::AtomSignature, @@ -161,14 +161,7 @@ pub(crate) fn wasmer_manifest_to_webc( /// take a `wasmer.toml` manifest and convert it to the `*.webc` equivalent. pub(crate) fn in_memory_wasmer_manifest_to_webc( manifest: &WasmerManifest, - atoms: &BTreeMap< - String, - ( - Option, - OwnedBuffer, - Option<&UserAnnotations>, - ), - >, + atoms: &BTreeMap, OwnedBuffer, Option<&UserAnnotations>)>, ) -> Result<(WebcManifest, BTreeMap), ManifestError> { let use_map = transform_dependencies(&manifest.dependencies)?; @@ -297,27 +290,13 @@ fn transform_atoms( } fn transform_in_memory_atoms( - atoms: &BTreeMap< - String, - ( - Option, - OwnedBuffer, - Option<&UserAnnotations>, - ), - >, + atoms: &BTreeMap, OwnedBuffer, Option<&UserAnnotations>)>, ) -> Result<(IndexMap, Atoms), ManifestError> { transform_atoms_shared(atoms) } fn transform_atoms_shared( - atoms: &BTreeMap< - String, - ( - Option, - OwnedBuffer, - Option<&UserAnnotations>, - ), - >, + atoms: &BTreeMap, OwnedBuffer, Option<&UserAnnotations>)>, ) -> Result<(IndexMap, Atoms), ManifestError> { let mut atom_files = BTreeMap::new(); let mut metadata = IndexMap::new(); From 7e881aa49d52c9bbdc8c2390be52afa887e6dfc2 Mon Sep 17 00:00:00 2001 From: Edoardo Marangoni Date: Tue, 22 Apr 2025 17:07:36 +0200 Subject: [PATCH 15/19] fix: Feature-gate fields and calls related to compilers --- lib/compiler/src/engine/inner.rs | 43 +++++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/lib/compiler/src/engine/inner.rs b/lib/compiler/src/engine/inner.rs index 140561b8668..a0459497013 100644 --- a/lib/compiler/src/engine/inner.rs +++ b/lib/compiler/src/engine/inner.rs @@ -96,10 +96,18 @@ impl Engine { /// Returns the deterministic id of this engine pub fn deterministic_id(&self) -> String { let i = self.inner(); - if let Some(ref c) = i.compiler { - c.deterministic_id() - } else { - self.name.clone() + #[cfg(feature = "compiler")] + { + if let Some(ref c) = i.compiler { + return c.deterministic_id(); + } else { + return self.name.clone(); + } + } + + #[allow(unreachable_code)] + { + return self.name.to_string(); } } @@ -296,9 +304,12 @@ impl Engine { &mut self, suggested_opts: &wasmer_types::target::SuggestedCompilerOptimizations, ) -> Result<(), CompileError> { - let mut i = self.inner_mut(); - if let Some(ref mut c) = i.compiler { - c.with_opts(suggested_opts)?; + #[cfg(feature = "compiler")] + { + let mut i = self.inner_mut(); + if let Some(ref mut c) = i.compiler { + c.with_opts(suggested_opts)?; + } } Ok(()) @@ -336,11 +347,19 @@ pub struct EngineInner { impl std::fmt::Debug for EngineInner { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - f.debug_struct("EngineInner") - .field("compiler", &self.compiler) - .field("features", &self.features) - .field("signatures", &self.signatures) - .finish() + let mut formatter = f.debug_struct("EngineInner"); + #[cfg(feature = "compiler")] + { + formatter.field("compiler", &self.compiler); + formatter.field("features", &self.features); + } + + #[cfg(not(target_arch = "wasm32"))] + { + formatter.field("signatures", &self.signatures); + } + + formatter.finish() } } From 8e7c3165cc25b65af8eb3070086e38c7ab12f1b2 Mon Sep 17 00:00:00 2001 From: Edoardo Marangoni Date: Tue, 22 Apr 2025 17:47:31 +0200 Subject: [PATCH 16/19] chore: Make linter happy --- lib/compiler/src/engine/inner.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compiler/src/engine/inner.rs b/lib/compiler/src/engine/inner.rs index a0459497013..3239c21c59f 100644 --- a/lib/compiler/src/engine/inner.rs +++ b/lib/compiler/src/engine/inner.rs @@ -107,7 +107,7 @@ impl Engine { #[allow(unreachable_code)] { - return self.name.to_string(); + self.name.to_string() } } From 09b1f1527d815fbafea02c218ea0e867af515760 Mon Sep 17 00:00:00 2001 From: Edoardo Marangoni Date: Tue, 22 Apr 2025 18:32:32 +0200 Subject: [PATCH 17/19] fix: Rename `wasmer_types::SuggestedCompilerOptimizations` to `UserCompilerOptimizations` --- lib/api/src/entities/engine/mod.rs | 4 ++-- lib/compiler-llvm/src/compiler.rs | 2 +- lib/compiler/src/compiler.rs | 4 ++-- lib/compiler/src/engine/inner.rs | 2 +- lib/types/src/target.rs | 4 ++-- lib/wasix/src/runtime/mod.rs | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/api/src/entities/engine/mod.rs b/lib/api/src/entities/engine/mod.rs index f709cb1a0ab..13c443b16cf 100644 --- a/lib/api/src/entities/engine/mod.rs +++ b/lib/api/src/entities/engine/mod.rs @@ -3,7 +3,7 @@ use bytes::Bytes; use std::{path::Path, sync::Arc}; use wasmer_types::{ - target::{SuggestedCompilerOptimizations, Target}, + target::{UserCompilerOptimizations, Target}, CompileError, DeserializeError, Features, }; @@ -236,7 +236,7 @@ impl Engine { /// more optimizations. pub fn with_opts( &mut self, - suggested_opts: &SuggestedCompilerOptimizations, + suggested_opts: &UserCompilerOptimizations, ) -> Result<(), CompileError> { match self.be { #[cfg(feature = "sys")] diff --git a/lib/compiler-llvm/src/compiler.rs b/lib/compiler-llvm/src/compiler.rs index 204ddb44e75..1d8a9a0dac7 100644 --- a/lib/compiler-llvm/src/compiler.rs +++ b/lib/compiler-llvm/src/compiler.rs @@ -560,7 +560,7 @@ impl Compiler for LLVMCompiler { fn with_opts( &mut self, - suggested_compiler_opts: &wasmer_types::target::SuggestedCompilerOptimizations, + suggested_compiler_opts: &wasmer_types::target::UserCompilerOptimizations, ) -> Result<(), CompileError> { if suggested_compiler_opts.pass_params.is_some_and(|v| v) { self.config.enable_g0m0_opt = true; diff --git a/lib/compiler/src/compiler.rs b/lib/compiler/src/compiler.rs index 13f6486517e..7cd04c8119b 100644 --- a/lib/compiler/src/compiler.rs +++ b/lib/compiler/src/compiler.rs @@ -12,7 +12,7 @@ use enumset::EnumSet; use wasmer_types::{ entity::PrimaryMap, error::CompileError, - target::{CpuFeature, SuggestedCompilerOptimizations, Target}, + target::{CpuFeature, UserCompilerOptimizations, Target}, Features, LocalFunctionIndex, }; #[cfg(feature = "translator")] @@ -94,7 +94,7 @@ pub trait Compiler: Send + std::fmt::Debug { /// more optimizations. fn with_opts( &mut self, - suggested_compiler_opts: &SuggestedCompilerOptimizations, + suggested_compiler_opts: &UserCompilerOptimizations, ) -> Result<(), CompileError> { _ = suggested_compiler_opts; Ok(()) diff --git a/lib/compiler/src/engine/inner.rs b/lib/compiler/src/engine/inner.rs index 3239c21c59f..7785aa7fb7f 100644 --- a/lib/compiler/src/engine/inner.rs +++ b/lib/compiler/src/engine/inner.rs @@ -302,7 +302,7 @@ impl Engine { /// more optimizations. pub fn with_opts( &mut self, - suggested_opts: &wasmer_types::target::SuggestedCompilerOptimizations, + suggested_opts: &wasmer_types::target::UserCompilerOptimizations, ) -> Result<(), CompileError> { #[cfg(feature = "compiler")] { diff --git a/lib/types/src/target.rs b/lib/types/src/target.rs index 7a53a4f7af1..739d16a3be1 100644 --- a/lib/types/src/target.rs +++ b/lib/types/src/target.rs @@ -230,12 +230,12 @@ impl Default for Target { } } -/// Suggested optimization that might be operated on the module when (and if) compiled. +/// User-suggested optimization that might be operated on the module when (and if) compiled. /// // Note: This type is a copy of `wasmer_config::package::SuggestedCompilerOptimizations`, so to // avoid dependencies on `wasmer_config` for crates that already depend on `wasmer_types`. #[derive(Clone, Debug, PartialEq, Eq, Default)] -pub struct SuggestedCompilerOptimizations { +pub struct UserCompilerOptimizations { /// Suggest the `pass_params` (also known as g0m0) optimization pass. pub pass_params: Option, } diff --git a/lib/wasix/src/runtime/mod.rs b/lib/wasix/src/runtime/mod.rs index cc90a94e556..917f97bde43 100644 --- a/lib/wasix/src/runtime/mod.rs +++ b/lib/wasix/src/runtime/mod.rs @@ -7,7 +7,7 @@ pub use self::task_manager::{SpawnMemoryType, VirtualTaskManager}; use self::{module_cache::CacheError, task_manager::InlineWaker}; use wasmer_config::package::SuggestedCompilerOptimizations; use wasmer_types::{ - target::SuggestedCompilerOptimizations as WasmerSuggestedCompilerOptimizations, ModuleHash, + target::UserCompilerOptimizations as WasmerSuggestedCompilerOptimizations, ModuleHash, }; use std::{ From 0ba63b9400b5c1cc6cb5cfe3042d32515ea54de9 Mon Sep 17 00:00:00 2001 From: Edoardo Marangoni Date: Tue, 22 Apr 2025 18:37:57 +0200 Subject: [PATCH 18/19] chore: Make linter happy --- lib/api/src/entities/engine/mod.rs | 2 +- lib/compiler/src/compiler.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/api/src/entities/engine/mod.rs b/lib/api/src/entities/engine/mod.rs index 13c443b16cf..96cec93b4df 100644 --- a/lib/api/src/entities/engine/mod.rs +++ b/lib/api/src/entities/engine/mod.rs @@ -3,7 +3,7 @@ use bytes::Bytes; use std::{path::Path, sync::Arc}; use wasmer_types::{ - target::{UserCompilerOptimizations, Target}, + target::{Target, UserCompilerOptimizations}, CompileError, DeserializeError, Features, }; diff --git a/lib/compiler/src/compiler.rs b/lib/compiler/src/compiler.rs index 7cd04c8119b..c1113408034 100644 --- a/lib/compiler/src/compiler.rs +++ b/lib/compiler/src/compiler.rs @@ -12,7 +12,7 @@ use enumset::EnumSet; use wasmer_types::{ entity::PrimaryMap, error::CompileError, - target::{CpuFeature, UserCompilerOptimizations, Target}, + target::{CpuFeature, Target, UserCompilerOptimizations}, Features, LocalFunctionIndex, }; #[cfg(feature = "translator")] From f1b3b824008cbb6785b29a84a58669a3f947c602 Mon Sep 17 00:00:00 2001 From: Edoardo Marangoni Date: Wed, 23 Apr 2025 10:18:18 +0200 Subject: [PATCH 19/19] chore: Fix missing closing delimiter, make linter happy --- lib/compiler-cranelift/src/compiler.rs | 1 + lib/compiler-llvm/src/compiler.rs | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compiler-cranelift/src/compiler.rs b/lib/compiler-cranelift/src/compiler.rs index 30b6dc0c06f..42d6bfa82b6 100644 --- a/lib/compiler-cranelift/src/compiler.rs +++ b/lib/compiler-cranelift/src/compiler.rs @@ -73,6 +73,7 @@ impl Compiler for CraneliftCompiler { fn get_perfmap_enabled(&self) -> bool { self.config.enable_perfmap + } fn deterministic_id(&self) -> String { String::from("cranelift") diff --git a/lib/compiler-llvm/src/compiler.rs b/lib/compiler-llvm/src/compiler.rs index 2902f2cd654..fc828826460 100644 --- a/lib/compiler-llvm/src/compiler.rs +++ b/lib/compiler-llvm/src/compiler.rs @@ -262,7 +262,6 @@ impl Compiler for LLVMCompiler { "llvm" } - fn get_perfmap_enabled(&self) -> bool { self.config.enable_perfmap }