diff --git a/lib/api/src/backend/js/entities/engine.rs b/lib/api/src/backend/js/entities/engine.rs index 96df07412ea..cdb2316c307 100644 --- a/lib/api/src/backend/js/entities/engine.rs +++ b/lib/api/src/backend/js/entities/engine.rs @@ -5,9 +5,9 @@ use wasmer_types::{target::Target, Features}; pub struct Engine; impl Engine { - pub(crate) fn deterministic_id(&self) -> &str { + pub(crate) fn deterministic_id(&self) -> String { // All js engines have the same id - "js-generic" + String::from("js-generic") } /// Returns the WebAssembly features supported by the JS engine. diff --git a/lib/api/src/backend/jsc/entities/engine.rs b/lib/api/src/backend/jsc/entities/engine.rs index 05b4fc924dd..c026f4b18fc 100644 --- a/lib/api/src/backend/jsc/entities/engine.rs +++ b/lib/api/src/backend/jsc/entities/engine.rs @@ -166,9 +166,9 @@ impl JSCEngine { } impl Engine { - pub(crate) fn deterministic_id(&self) -> &str { + pub(crate) fn deterministic_id(&self) -> String { // All js engines have the same id - "javascriptcore" + String::from("javascriptcore") } /// Returns the WebAssembly features supported by the JSC engine for the given target. diff --git a/lib/api/src/backend/v8/entities/engine.rs b/lib/api/src/backend/v8/entities/engine.rs index 75cc5f682f2..5e5e44de7c0 100644 --- a/lib/api/src/backend/v8/entities/engine.rs +++ b/lib/api/src/backend/v8/entities/engine.rs @@ -50,8 +50,8 @@ impl Engine { Self::default() } - pub(crate) fn deterministic_id(&self) -> &str { - "v8" + pub(crate) fn deterministic_id(&self) -> String { + String::from("v8") } /// Returns the WebAssembly features supported by the V8 engine. diff --git a/lib/api/src/backend/wamr/entities/engine.rs b/lib/api/src/backend/wamr/entities/engine.rs index 6f8e5eee56d..6e30b96cf35 100644 --- a/lib/api/src/backend/wamr/entities/engine.rs +++ b/lib/api/src/backend/wamr/entities/engine.rs @@ -36,8 +36,8 @@ impl Engine { Self::default() } - pub(crate) fn deterministic_id(&self) -> &str { - "wamr" + pub(crate) fn deterministic_id(&self) -> String { + String::from("wamr") } /// Returns the WebAssembly features supported by the WAMR engine. diff --git a/lib/api/src/backend/wasmi/entities/engine.rs b/lib/api/src/backend/wasmi/entities/engine.rs index 89d8244efd3..896d40fce37 100644 --- a/lib/api/src/backend/wasmi/entities/engine.rs +++ b/lib/api/src/backend/wasmi/entities/engine.rs @@ -36,8 +36,8 @@ impl Engine { Self::default() } - pub(crate) fn deterministic_id(&self) -> &str { - "wasmi" + pub(crate) fn deterministic_id(&self) -> String { + String::from("wasmi") } /// Returns the WebAssembly features supported by the WASMI engine. diff --git a/lib/api/src/entities/engine/inner.rs b/lib/api/src/entities/engine/inner.rs index 78c6cacaa99..8c0326e1d6c 100644 --- a/lib/api/src/entities/engine/inner.rs +++ b/lib/api/src/entities/engine/inner.rs @@ -17,7 +17,7 @@ gen_rt_ty!(Engine @derives Debug, Clone); impl BackendEngine { /// Returns the deterministic id of this engine. #[inline] - pub fn deterministic_id(&self) -> &str { + pub fn deterministic_id(&self) -> String { match_rt!(on self => s { s.deterministic_id() }) diff --git a/lib/api/src/entities/engine/mod.rs b/lib/api/src/entities/engine/mod.rs index 881813a872a..96cec93b4df 100644 --- a/lib/api/src/entities/engine/mod.rs +++ b/lib/api/src/entities/engine/mod.rs @@ -2,7 +2,10 @@ use bytes::Bytes; use std::{path::Path, sync::Arc}; -use wasmer_types::{target::Target, DeserializeError, Features}; +use wasmer_types::{ + target::{Target, UserCompilerOptimizations}, + CompileError, DeserializeError, Features, +}; #[cfg(feature = "sys")] use wasmer_compiler::Artifact; @@ -51,7 +54,7 @@ impl Engine { } /// Returns the deterministic id of this engine. - pub fn deterministic_id(&self) -> &str { + pub fn deterministic_id(&self) -> String { self.be.deterministic_id() } @@ -223,4 +226,22 @@ impl Engine { ) -> Result, DeserializeError> { self.be.deserialize_from_file_unchecked(file_ref) } + + /// Add suggested optimizations to this engine. + /// + /// # Note + /// + /// Not every backend supports every optimization. This function may fail (i.e. not set the + /// suggested optimizations) silently if the underlying engine backend does not support one or + /// more optimizations. + pub fn with_opts( + &mut self, + suggested_opts: &UserCompilerOptimizations, + ) -> Result<(), CompileError> { + match self.be { + #[cfg(feature = "sys")] + BackendEngine::Sys(ref mut e) => e.with_opts(suggested_opts), + _ => Ok(()), + } + } } diff --git a/lib/cli/src/backend.rs b/lib/cli/src/backend.rs index 806db18bb61..00025dd542b 100644 --- a/lib/cli/src/backend.rs +++ b/lib/cli/src/backend.rs @@ -6,9 +6,9 @@ // module. #![allow(dead_code, unused_imports, unused_variables)] -use std::path::PathBuf; use std::string::ToString; use std::sync::Arc; +use std::{path::PathBuf, str::FromStr}; use anyhow::{bail, Result}; #[cfg(feature = "sys")] @@ -187,16 +187,44 @@ pub struct RuntimeOptions { #[clap(long)] enable_verifier: bool, + /// Enable a profiler. + /// + /// Available for cranelift, LLVM and singlepass. + #[clap(long, value_enum)] + profiler: Option, + /// LLVM debug directory, where IR and object files will be written to. /// /// Only available for the LLVM compiler. #[clap(long)] llvm_debug_dir: Option, + /// Only available for the LLVM compiler. Enable the "pass-params" optimization, where the first (#0) + /// global and the first (#0) memory passed between guest functions as explicit parameters. + #[clap(long)] + enable_pass_params_opt: bool, + #[clap(flatten)] features: WasmFeatures, } +#[derive(Clone, Debug)] +pub enum Profiler { + /// Perfmap-based profilers. + Perfmap, +} + +impl FromStr for Profiler { + type Err = anyhow::Error; + + fn from_str(s: &str) -> std::result::Result { + match s.to_lowercase().as_str() { + "perfmap" => Ok(Self::Perfmap), + _ => Err(anyhow::anyhow!("Unrecognized profiler: {s}")), + } + } +} + impl RuntimeOptions { pub fn get_available_backends(&self) -> Result> { // If a specific backend is explicitly requested, use it @@ -267,7 +295,7 @@ impl RuntimeOptions { let backend = backends.first().unwrap(); let backend_kind = wasmer::BackendKind::from(backend); let required_features = wasmer::Engine::default_features_for_backend(&backend_kind, target); - backend.get_engine(target, &required_features) + backend.get_engine(target, &required_features, self) } pub fn get_engine_for_module(&self, module_contents: &[u8], target: &Target) -> Result { @@ -310,7 +338,7 @@ impl RuntimeOptions { filtered_backends .first() .unwrap() - .get_engine(target, required_features) + .get_engine(target, required_features, self) } #[cfg(feature = "compiler")] @@ -417,6 +445,12 @@ impl RuntimeOptions { if self.enable_verifier { config.enable_verifier(); } + if let Some(p) = &self.profiler { + match p { + Profiler::Perfmap => config.enable_perfmap(), + } + } + Box::new(config) } #[cfg(feature = "cranelift")] @@ -425,6 +459,11 @@ impl RuntimeOptions { if self.enable_verifier { config.enable_verifier(); } + if let Some(p) = &self.profiler { + match p { + Profiler::Perfmap => config.enable_perfmap(), + } + } Box::new(config) } #[cfg(feature = "llvm")] @@ -436,6 +475,11 @@ impl RuntimeOptions { }; use wasmer_types::entity::EntityRef; let mut config = LLVM::new(); + + if self.enable_pass_params_opt { + config.enable_pass_params_opt(); + } + struct Callbacks { debug_dir: PathBuf, } @@ -528,6 +572,12 @@ impl RuntimeOptions { if self.enable_verifier { config.enable_verifier(); } + if let Some(p) = &self.profiler { + match p { + Profiler::Perfmap => config.enable_perfmap(), + } + } + Box::new(config) } BackendType::V8 | BackendType::Wamr | BackendType::Wasmi => unreachable!(), @@ -592,11 +642,24 @@ impl BackendType { } /// Get an engine for this backend type - pub fn get_engine(&self, target: &Target, features: &Features) -> Result { + pub fn get_engine( + &self, + target: &Target, + features: &Features, + runtime_opts: &RuntimeOptions, + ) -> Result { match self { #[cfg(feature = "singlepass")] Self::Singlepass => { - let config = wasmer_compiler_singlepass::Singlepass::new(); + let mut config = wasmer_compiler_singlepass::Singlepass::new(); + if runtime_opts.enable_verifier { + config.enable_verifier(); + } + if let Some(p) = &runtime_opts.profiler { + match p { + Profiler::Perfmap => config.enable_perfmap(), + } + } let engine = wasmer_compiler::EngineBuilder::new(config) .set_features(Some(features.clone())) .set_target(Some(target.clone())) @@ -606,7 +669,15 @@ impl BackendType { } #[cfg(feature = "cranelift")] Self::Cranelift => { - let config = wasmer_compiler_cranelift::Cranelift::new(); + let mut config = wasmer_compiler_cranelift::Cranelift::new(); + if runtime_opts.enable_verifier { + config.enable_verifier(); + } + if let Some(p) = &runtime_opts.profiler { + match p { + Profiler::Perfmap => config.enable_perfmap(), + } + } let engine = wasmer_compiler::EngineBuilder::new(config) .set_features(Some(features.clone())) .set_target(Some(target.clone())) @@ -616,7 +687,118 @@ impl BackendType { } #[cfg(feature = "llvm")] Self::LLVM => { - let config = wasmer_compiler_llvm::LLVM::new(); + use std::{fmt, fs::File, io::Write}; + + use wasmer_compiler_llvm::{ + CompiledKind, InkwellMemoryBuffer, InkwellModule, LLVMCallbacks, LLVM, + }; + use wasmer_types::entity::EntityRef; + + let mut config = wasmer_compiler_llvm::LLVM::new(); + + struct Callbacks { + debug_dir: PathBuf, + } + impl Callbacks { + fn new(debug_dir: PathBuf) -> Result { + // Create the debug dir in case it doesn't exist + std::fs::create_dir_all(&debug_dir)?; + Ok(Self { debug_dir }) + } + } + // Converts a kind into a filename, that we will use to dump + // the contents of the IR object file to. + fn types_to_signature(types: &[Type]) -> String { + types + .iter() + .map(|ty| match ty { + Type::I32 => "i".to_string(), + Type::I64 => "I".to_string(), + Type::F32 => "f".to_string(), + Type::F64 => "F".to_string(), + Type::V128 => "v".to_string(), + Type::ExternRef => "e".to_string(), + Type::FuncRef => "r".to_string(), + Type::ExceptionRef => "x".to_string(), + }) + .collect::>() + .join("") + } + // Converts a kind into a filename, that we will use to dump + // the contents of the IR object file to. + fn function_kind_to_filename(kind: &CompiledKind) -> String { + match kind { + CompiledKind::Local(local_index) => { + format!("function_{}", local_index.index()) + } + CompiledKind::FunctionCallTrampoline(func_type) => format!( + "trampoline_call_{}_{}", + types_to_signature(func_type.params()), + types_to_signature(func_type.results()) + ), + CompiledKind::DynamicFunctionTrampoline(func_type) => format!( + "trampoline_dynamic_{}_{}", + types_to_signature(func_type.params()), + types_to_signature(func_type.results()) + ), + CompiledKind::Module => "module".into(), + } + } + impl LLVMCallbacks for Callbacks { + fn preopt_ir(&self, kind: &CompiledKind, module: &InkwellModule) { + let mut path = self.debug_dir.clone(); + path.push(format!("{}.preopt.ll", function_kind_to_filename(kind))); + module + .print_to_file(&path) + .expect("Error while dumping pre optimized LLVM IR"); + } + fn postopt_ir(&self, kind: &CompiledKind, module: &InkwellModule) { + let mut path = self.debug_dir.clone(); + path.push(format!("{}.postopt.ll", function_kind_to_filename(kind))); + module + .print_to_file(&path) + .expect("Error while dumping post optimized LLVM IR"); + } + fn obj_memory_buffer( + &self, + kind: &CompiledKind, + memory_buffer: &InkwellMemoryBuffer, + ) { + let mut path = self.debug_dir.clone(); + path.push(format!("{}.o", function_kind_to_filename(kind))); + let mem_buf_slice = memory_buffer.as_slice(); + let mut file = File::create(path) + .expect("Error while creating debug object file from LLVM IR"); + let mut pos = 0; + while pos < mem_buf_slice.len() { + pos += file.write(&mem_buf_slice[pos..]).unwrap(); + } + } + } + + impl fmt::Debug for Callbacks { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "LLVMCallbacks") + } + } + + if let Some(ref llvm_debug_dir) = runtime_opts.llvm_debug_dir { + config.callbacks(Some(Arc::new(Callbacks::new(llvm_debug_dir.clone())?))); + } + if runtime_opts.enable_verifier { + config.enable_verifier(); + } + + if runtime_opts.enable_pass_params_opt { + config.enable_pass_params_opt(); + } + + if let Some(p) = &runtime_opts.profiler { + match p { + Profiler::Perfmap => config.enable_perfmap(), + } + } + let engine = wasmer_compiler::EngineBuilder::new(config) .set_features(Some(features.clone())) .set_target(Some(target.clone())) diff --git a/lib/cli/src/commands/init.rs b/lib/cli/src/commands/init.rs index d32866638b0..26b416bb3ac 100644 --- a/lib/cli/src/commands/init.rs +++ b/lib/cli/src/commands/init.rs @@ -491,6 +491,7 @@ async fn construct_manifest( map.insert("wasi".to_string(), "0.1.0-unstable".to_string()); map }), + annotations: None, }]; let mut pkg = wasmer_config::package::Package::builder( diff --git a/lib/cli/src/commands/run/mod.rs b/lib/cli/src/commands/run/mod.rs index 09144a88f3c..6053ec946c6 100644 --- a/lib/cli/src/commands/run/mod.rs +++ b/lib/cli/src/commands/run/mod.rs @@ -260,9 +260,11 @@ impl Run { let engine_id = filtered_backends[0].to_string(); // Get a new engine that's compatible with the required features - if let Ok(new_engine) = - filtered_backends[0].get_engine(&Target::default(), &features) - { + if let Ok(new_engine) = filtered_backends[0].get_engine( + &Target::default(), + &features, + &self.rt, + ) { tracing::info!( "The command '{}' requires to run the Wasm module with the features {:?}. The backends available are {}. Choosing {}.", cmd.name(), diff --git a/lib/compiler-cranelift/src/compiler.rs b/lib/compiler-cranelift/src/compiler.rs index ae682f67a42..42d6bfa82b6 100644 --- a/lib/compiler-cranelift/src/compiler.rs +++ b/lib/compiler-cranelift/src/compiler.rs @@ -49,6 +49,7 @@ use wasmer_types::{ /// A compiler that compiles a WebAssembly module with Cranelift, translating the Wasm to Cranelift IR, /// optimizing it and then translating to assembly. +#[derive(Debug)] pub struct CraneliftCompiler { config: Cranelift, } @@ -70,6 +71,14 @@ impl Compiler for CraneliftCompiler { "cranelift" } + fn get_perfmap_enabled(&self) -> bool { + self.config.enable_perfmap + } + + fn deterministic_id(&self) -> String { + String::from("cranelift") + } + /// Get the middlewares for this compiler fn get_middlewares(&self) -> &[Arc] { &self.config.middlewares diff --git a/lib/compiler-cranelift/src/config.rs b/lib/compiler-cranelift/src/config.rs index b854efbb28c..a4baf3c22d2 100644 --- a/lib/compiler-cranelift/src/config.rs +++ b/lib/compiler-cranelift/src/config.rs @@ -33,6 +33,7 @@ pub enum CraneliftOptLevel { pub struct Cranelift { enable_nan_canonicalization: bool, enable_verifier: bool, + pub(crate) enable_perfmap: bool, enable_pic: bool, opt_level: CraneliftOptLevel, /// The middleware chain. @@ -49,6 +50,7 @@ impl Cranelift { opt_level: CraneliftOptLevel::Speed, enable_pic: false, middlewares: vec![], + enable_perfmap: false, } } @@ -194,6 +196,10 @@ impl CompilerConfig for Cranelift { self.enable_verifier = true; } + fn enable_perfmap(&mut self) { + self.enable_perfmap = true; + } + fn canonicalize_nans(&mut self, enable: bool) { self.enable_nan_canonicalization = enable; } diff --git a/lib/compiler-llvm/src/abi/aarch64_systemv.rs b/lib/compiler-llvm/src/abi/aarch64_systemv.rs index dce0fc86d20..2076f629622 100644 --- a/lib/compiler-llvm/src/abi/aarch64_systemv.rs +++ b/lib/compiler-llvm/src/abi/aarch64_systemv.rs @@ -15,13 +15,15 @@ use wasmer_vm::VMOffsets; use std::convert::TryInto; +use super::{G0M0FunctionKind, LocalFunctionG0M0params}; + /// Implementation of the [`Abi`] trait for the Aarch64 ABI on Linux. pub struct Aarch64SystemV {} impl Abi for Aarch64SystemV { // Given a function definition, retrieve the parameter that is the vmctx pointer. fn get_vmctx_ptr_param<'ctx>(&self, func_value: &FunctionValue<'ctx>) -> PointerValue<'ctx> { - func_value + let param = func_value .get_nth_param(u32::from( func_value .get_enum_attribute( @@ -30,8 +32,48 @@ impl Abi for Aarch64SystemV { ) .is_some(), )) - .unwrap() - .into_pointer_value() + .unwrap(); + //param.set_name("vmctx"); + + param.into_pointer_value() + } + + /// Given a function definition, retrieve the parameter that is the pointer to the first -- + /// number 0 -- local global. + fn get_g0_ptr_param<'ctx>(&self, func_value: &FunctionValue<'ctx>) -> IntValue<'ctx> { + // g0 is always after the vmctx. + let vmctx_idx = u32::from( + func_value + .get_enum_attribute( + AttributeLoc::Param(0), + Attribute::get_named_enum_kind_id("sret"), + ) + .is_some(), + ); + + let param = func_value.get_nth_param(vmctx_idx + 1).unwrap(); + param.set_name("g0"); + + param.into_int_value() + } + + /// Given a function definition, retrieve the parameter that is the pointer to the first -- + /// number 0 -- local memory. + fn get_m0_ptr_param<'ctx>(&self, func_value: &FunctionValue<'ctx>) -> PointerValue<'ctx> { + // m0 is always after g0. + let vmctx_idx = u32::from( + func_value + .get_enum_attribute( + AttributeLoc::Param(0), + Attribute::get_named_enum_kind_id("sret"), + ) + .is_some(), + ); + + let param = func_value.get_nth_param(vmctx_idx + 2).unwrap(); + param.set_name("m0_base_ptr"); + + param.into_pointer_value() } // Given a wasm function type, produce an llvm function declaration. @@ -41,11 +83,19 @@ impl Abi for Aarch64SystemV { intrinsics: &Intrinsics<'ctx>, offsets: Option<&VMOffsets>, sig: &FuncSig, + function_kind: Option, ) -> Result<(FunctionType<'ctx>, Vec<(Attribute, AttributeLoc)>), CompileError> { let user_param_types = sig.params().iter().map(|&ty| type_to_llvm(intrinsics, ty)); - let param_types = - std::iter::once(Ok(intrinsics.ptr_ty.as_basic_type_enum())).chain(user_param_types); + let mut param_types = vec![Ok(intrinsics.ptr_ty.as_basic_type_enum())]; + if function_kind.is_some_and(|v| v.is_local()) { + // The value of g0 + param_types.push(Ok(intrinsics.i32_ty.as_basic_type_enum())); + // The base pointer to m0 + param_types.push(Ok(intrinsics.ptr_ty.as_basic_type_enum())); + } + + let param_types = param_types.into_iter().chain(user_param_types); let vmctx_attributes = |i: u32| { vec![ @@ -261,6 +311,7 @@ impl Abi for Aarch64SystemV { ctx_ptr: PointerValue<'ctx>, values: &[BasicValueEnum<'ctx>], intrinsics: &Intrinsics<'ctx>, + g0m0: LocalFunctionG0M0params<'ctx>, ) -> Result>, CompileError> { // If it's an sret, allocate the return space. let sret = if llvm_fn_ty.get_return_type().is_none() && func_sig.results().len() > 1 { @@ -277,14 +328,21 @@ impl Abi for Aarch64SystemV { None }; - let values = std::iter::once(ctx_ptr.as_basic_value_enum()).chain(values.iter().copied()); + let mut args = vec![ctx_ptr.as_basic_value_enum()]; + + if let Some((g0, m0)) = g0m0 { + args.push(g0.into()); + args.push(m0.into()); + } + + let args = args.into_iter().chain(values.iter().copied()); let ret = if let Some(sret) = sret { std::iter::once(sret.as_basic_value_enum()) - .chain(values) + .chain(args) .collect() } else { - values.collect() + args.collect() }; Ok(ret) diff --git a/lib/compiler-llvm/src/abi/mod.rs b/lib/compiler-llvm/src/abi/mod.rs index 6e3a46b98dd..1bc6446e4ac 100644 --- a/lib/compiler-llvm/src/abi/mod.rs +++ b/lib/compiler-llvm/src/abi/mod.rs @@ -12,7 +12,7 @@ use inkwell::{ context::Context, targets::TargetMachine, types::FunctionType, - values::{BasicValueEnum, CallSiteValue, FunctionValue, PointerValue}, + values::{BasicValueEnum, CallSiteValue, FunctionValue, IntValue, PointerValue}, }; use wasmer_types::CompileError; use wasmer_types::FunctionType as FuncSig; @@ -37,6 +37,25 @@ pub fn get_abi(target_machine: &TargetMachine) -> Box { } } +#[derive(Debug)] +pub(crate) enum G0M0FunctionKind { + Local, + Imported, +} + +impl G0M0FunctionKind { + /// Returns `true` if the function kind is [`Local`]. + /// + /// [`Local`]: FunctionKind::Local + #[must_use] + pub(crate) fn is_local(&self) -> bool { + matches!(self, Self::Local) + } +} + +/// The two additional parameters needed for g0m0 optimization. +pub(crate) type LocalFunctionG0M0params<'ctx> = Option<(IntValue<'ctx>, PointerValue<'ctx>)>; + /// We need to produce different LLVM IR for different platforms. (Contrary to /// popular knowledge LLVM IR is not intended to be portable in that way.) This /// trait deals with differences between function signatures on different @@ -45,16 +64,33 @@ pub trait Abi { /// Given a function definition, retrieve the parameter that is the vmctx pointer. fn get_vmctx_ptr_param<'ctx>(&self, func_value: &FunctionValue<'ctx>) -> PointerValue<'ctx>; + /// Given a function definition, retrieve the parameter that is the pointer to the first -- + /// number 0 -- local global. + #[allow(unused)] + fn get_g0_ptr_param<'ctx>(&self, func_value: &FunctionValue<'ctx>) -> IntValue<'ctx>; + + /// Given a function definition, retrieve the parameter that is the pointer to the first -- + /// number 0 -- local memory. + /// + /// # Notes + /// This function assumes that g0m0 is enabled. + fn get_m0_ptr_param<'ctx>(&self, func_value: &FunctionValue<'ctx>) -> PointerValue<'ctx>; + /// Given a wasm function type, produce an llvm function declaration. + /// + /// # Notes + /// This function assumes that g0m0 is enabled. fn func_type_to_llvm<'ctx>( &self, context: &'ctx Context, intrinsics: &Intrinsics<'ctx>, offsets: Option<&VMOffsets>, sig: &FuncSig, + function_kind: Option, ) -> Result<(FunctionType<'ctx>, Vec<(Attribute, AttributeLoc)>), CompileError>; /// Marshall wasm stack values into function parameters. + #[allow(clippy::too_many_arguments)] fn args_to_call<'ctx>( &self, alloca_builder: &Builder<'ctx>, @@ -63,6 +99,7 @@ pub trait Abi { ctx_ptr: PointerValue<'ctx>, values: &[BasicValueEnum<'ctx>], intrinsics: &Intrinsics<'ctx>, + g0m0: LocalFunctionG0M0params<'ctx>, ) -> Result>, CompileError>; /// Given a CallSite, extract the returned values and return them in a Vec. diff --git a/lib/compiler-llvm/src/abi/x86_64_systemv.rs b/lib/compiler-llvm/src/abi/x86_64_systemv.rs index 33502dc337e..f0bad7fb01f 100644 --- a/lib/compiler-llvm/src/abi/x86_64_systemv.rs +++ b/lib/compiler-llvm/src/abi/x86_64_systemv.rs @@ -17,13 +17,15 @@ use wasmer_vm::VMOffsets; use std::convert::TryInto; +use super::{G0M0FunctionKind, LocalFunctionG0M0params}; + /// Implementation of the [`Abi`] trait for the AMD64 SystemV ABI. pub struct X86_64SystemV {} impl Abi for X86_64SystemV { - // Given a function definition, retrieve the parameter that is the vmctx pointer. + /// Given a function definition, retrieve the parameter that is the vmctx pointer. fn get_vmctx_ptr_param<'ctx>(&self, func_value: &FunctionValue<'ctx>) -> PointerValue<'ctx> { - func_value + let param = func_value .get_nth_param(u32::from( func_value .get_enum_attribute( @@ -32,8 +34,48 @@ impl Abi for X86_64SystemV { ) .is_some(), )) - .unwrap() - .into_pointer_value() + .unwrap(); + //param.set_name("vmctx"); + + param.into_pointer_value() + } + + /// Given a function definition, retrieve the parameter that is the pointer to the first -- + /// number 0 -- local global. + fn get_g0_ptr_param<'ctx>(&self, func_value: &FunctionValue<'ctx>) -> IntValue<'ctx> { + // g0 is always after the vmctx. + let vmctx_idx = u32::from( + func_value + .get_enum_attribute( + AttributeLoc::Param(0), + Attribute::get_named_enum_kind_id("sret"), + ) + .is_some(), + ); + + let param = func_value.get_nth_param(vmctx_idx + 1).unwrap(); + param.set_name("g0"); + + param.into_int_value() + } + + /// Given a function definition, retrieve the parameter that is the pointer to the first -- + /// number 0 -- local memory. + fn get_m0_ptr_param<'ctx>(&self, func_value: &FunctionValue<'ctx>) -> PointerValue<'ctx> { + // m0 is always after g0. + let vmctx_idx = u32::from( + func_value + .get_enum_attribute( + AttributeLoc::Param(0), + Attribute::get_named_enum_kind_id("sret"), + ) + .is_some(), + ); + + let param = func_value.get_nth_param(vmctx_idx + 2).unwrap(); + param.set_name("m0_base_ptr"); + + param.into_pointer_value() } // Given a wasm function type, produce an llvm function declaration. @@ -43,11 +85,19 @@ impl Abi for X86_64SystemV { intrinsics: &Intrinsics<'ctx>, offsets: Option<&VMOffsets>, sig: &FuncSig, + function_kind: Option, ) -> Result<(FunctionType<'ctx>, Vec<(Attribute, AttributeLoc)>), CompileError> { let user_param_types = sig.params().iter().map(|&ty| type_to_llvm(intrinsics, ty)); - let param_types = - std::iter::once(Ok(intrinsics.ptr_ty.as_basic_type_enum())).chain(user_param_types); + let mut param_types = vec![Ok(intrinsics.ptr_ty.as_basic_type_enum())]; + if function_kind.is_some_and(|v| v.is_local()) { + // The value of g0 + param_types.push(Ok(intrinsics.i32_ty.as_basic_type_enum())); + // The base pointer to m0 + param_types.push(Ok(intrinsics.ptr_ty.as_basic_type_enum())); + } + + let param_types = param_types.into_iter().chain(user_param_types); // TODO: figure out how many bytes long vmctx is, and mark it dereferenceable. (no need to mark it nonnull once we do this.) let vmctx_attributes = |i: u32| { @@ -296,6 +346,7 @@ impl Abi for X86_64SystemV { ctx_ptr: PointerValue<'ctx>, values: &[BasicValueEnum<'ctx>], intrinsics: &Intrinsics<'ctx>, + g0m0: LocalFunctionG0M0params<'ctx>, ) -> Result>, CompileError> { // If it's an sret, allocate the return space. let sret = if llvm_fn_ty.get_return_type().is_none() && func_sig.results().len() > 1 { @@ -312,14 +363,21 @@ impl Abi for X86_64SystemV { None }; - let values = std::iter::once(ctx_ptr.as_basic_value_enum()).chain(values.iter().copied()); + let mut args = vec![ctx_ptr.as_basic_value_enum()]; + + if let Some((g0, m0)) = g0m0 { + args.push(g0.into()); + args.push(m0.into()); + } + + let args = args.into_iter().chain(values.iter().copied()); let ret = if let Some(sret) = sret { std::iter::once(sret.as_basic_value_enum()) - .chain(values) + .chain(args) .collect() } else { - values.collect() + args.collect() }; Ok(ret) diff --git a/lib/compiler-llvm/src/compiler.rs b/lib/compiler-llvm/src/compiler.rs index 4232054e872..fc828826460 100644 --- a/lib/compiler-llvm/src/compiler.rs +++ b/lib/compiler-llvm/src/compiler.rs @@ -9,7 +9,7 @@ use inkwell::targets::FileType; use inkwell::DLLStorageClass; use rayon::iter::ParallelBridge; use rayon::prelude::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator}; -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; use wasmer_compiler::types::function::{Compilation, UnwindInfo}; use wasmer_compiler::types::module::CompileModuleInfo; @@ -24,11 +24,12 @@ use wasmer_compiler::{ }; use wasmer_types::entity::{EntityRef, PrimaryMap}; use wasmer_types::target::Target; -use wasmer_types::{CompileError, FunctionIndex, LocalFunctionIndex, SignatureIndex}; +use wasmer_types::{CompileError, FunctionIndex, LocalFunctionIndex, ModuleInfo, SignatureIndex}; use wasmer_vm::LibCall; /// A compiler that compiles a WebAssembly module with LLVM, translating the Wasm to LLVM IR, /// optimizing it and then translating to assembly. +#[derive(Debug)] pub struct LLVMCompiler { config: LLVM, } @@ -82,6 +83,52 @@ impl SymbolRegistry for ShortNames { } } +struct ModuleBasedSymbolRegistry { + wasm_module: Arc, + local_func_names: HashMap, + short_names: ShortNames, +} + +impl ModuleBasedSymbolRegistry { + fn new(wasm_module: Arc) -> Self { + let local_func_names = HashMap::from_iter( + wasm_module + .function_names + .iter() + .map(|(f, v)| (wasm_module.local_func_index(*f), v)) + .filter(|(f, _)| f.is_some()) + .map(|(f, v)| (v.clone(), f.unwrap())), + ); + Self { + wasm_module, + local_func_names, + short_names: ShortNames {}, + } + } +} + +impl SymbolRegistry for ModuleBasedSymbolRegistry { + fn symbol_to_name(&self, symbol: Symbol) -> String { + match symbol { + Symbol::LocalFunction(index) => self + .wasm_module + .function_names + .get(&self.wasm_module.func_index(index)) + .cloned() + .unwrap_or(self.short_names.symbol_to_name(symbol)), + _ => self.short_names.symbol_to_name(symbol), + } + } + + fn name_to_symbol(&self, name: &str) -> Option { + if let Some(idx) = self.local_func_names.get(name) { + Some(Symbol::LocalFunction(*idx)) + } else { + self.short_names.name_to_symbol(name) + } + } +} + impl LLVMCompiler { #[allow(clippy::too_many_arguments)] fn compile_native_object( @@ -127,7 +174,12 @@ impl LLVMCompiler { }, |func_trampoline, (i, sig)| { let name = symbol_registry.symbol_to_name(Symbol::FunctionCallTrampoline(i)); - let module = func_trampoline.trampoline_to_module(sig, self.config(), &name)?; + let module = func_trampoline.trampoline_to_module( + sig, + self.config(), + &name, + compile_info, + )?; Ok(module.write_bitcode_to_memory().as_slice().to_vec()) }, ); @@ -210,6 +262,28 @@ impl Compiler for LLVMCompiler { "llvm" } + fn get_perfmap_enabled(&self) -> bool { + self.config.enable_perfmap + } + + fn deterministic_id(&self) -> String { + let mut ret = format!( + "llvm-{}", + match self.config.opt_level { + inkwell::OptimizationLevel::None => "opt0", + inkwell::OptimizationLevel::Less => "optl", + inkwell::OptimizationLevel::Default => "optd", + inkwell::OptimizationLevel::Aggressive => "opta", + } + ); + + if self.config.enable_g0m0_opt { + ret.push_str("-g0m0"); + } + + ret + } + /// Get the middlewares for this compiler fn get_middlewares(&self) -> &[Arc] { &self.config.middlewares @@ -273,6 +347,8 @@ impl Compiler for LLVMCompiler { HashSet::default() }; + let symbol_registry = ModuleBasedSymbolRegistry::new(module.clone()); + let functions = function_body_inputs .iter() .collect::)>>() @@ -294,7 +370,7 @@ impl Compiler for LLVMCompiler { self.config(), memory_styles, table_styles, - &ShortNames {}, + &symbol_registry, ) }, ) @@ -408,7 +484,9 @@ impl Compiler for LLVMCompiler { let target_machine = self.config().target_machine(target); FuncTrampoline::new(target_machine, binary_format).unwrap() }, - |func_trampoline, sig| func_trampoline.trampoline(sig, self.config(), ""), + |func_trampoline, sig| { + func_trampoline.trampoline(sig, self.config(), "", compile_info) + }, ) .collect::>() .into_iter() @@ -483,4 +561,14 @@ impl Compiler for LLVMCompiler { got, }) } + + fn with_opts( + &mut self, + suggested_compiler_opts: &wasmer_types::target::UserCompilerOptimizations, + ) -> Result<(), CompileError> { + if suggested_compiler_opts.pass_params.is_some_and(|v| v) { + self.config.enable_g0m0_opt = true; + } + Ok(()) + } } diff --git a/lib/compiler-llvm/src/config.rs b/lib/compiler-llvm/src/config.rs index f9223c32344..19d8ffc3683 100644 --- a/lib/compiler-llvm/src/config.rs +++ b/lib/compiler-llvm/src/config.rs @@ -43,7 +43,9 @@ pub trait LLVMCallbacks: Debug + Send + Sync { #[derive(Debug, Clone)] pub struct LLVM { pub(crate) enable_nan_canonicalization: bool, + pub(crate) enable_g0m0_opt: bool, pub(crate) enable_verifier: bool, + pub(crate) enable_perfmap: bool, pub(crate) opt_level: LLVMOptLevel, is_pic: bool, pub(crate) callbacks: Option>, @@ -58,10 +60,12 @@ impl LLVM { Self { enable_nan_canonicalization: false, enable_verifier: false, + enable_perfmap: false, opt_level: LLVMOptLevel::Aggressive, is_pic: false, callbacks: None, middlewares: vec![], + enable_g0m0_opt: false, } } @@ -71,6 +75,14 @@ impl LLVM { self } + /// (warning: experimental) Pass the value of the first (#0) global and the base pointer of the + /// first (#0) memory as parameter between guest functions. + pub fn enable_pass_params_opt(&mut self) -> &mut Self { + // internally, the "pass_params" opt is known as g0m0 opt. + self.enable_g0m0_opt = true; + self + } + /// Callbacks that will triggered in the different compilation /// phases in LLVM. pub fn callbacks(&mut self, callbacks: Option>) -> &mut Self { @@ -290,6 +302,10 @@ impl CompilerConfig for LLVM { self.is_pic = true; } + fn enable_perfmap(&mut self) { + self.enable_perfmap = true + } + /// Whether to verify compiler IR. fn enable_verifier(&mut self) { self.enable_verifier = true; diff --git a/lib/compiler-llvm/src/trampoline/wasm.rs b/lib/compiler-llvm/src/trampoline/wasm.rs index 010820beb1a..3332c5d5661 100644 --- a/lib/compiler-llvm/src/trampoline/wasm.rs +++ b/lib/compiler-llvm/src/trampoline/wasm.rs @@ -1,5 +1,5 @@ use crate::{ - abi::{get_abi, Abi}, + abi::{get_abi, Abi, G0M0FunctionKind}, config::{CompiledKind, LLVM}, error::{err, err_nt}, object_file::{load_object_file, CompiledFunction}, @@ -17,8 +17,11 @@ use inkwell::{ }; use std::{cmp, convert::TryInto}; use target_lexicon::BinaryFormat; -use wasmer_compiler::types::{function::FunctionBody, relocation::RelocationTarget}; +use wasmer_compiler::types::{ + function::FunctionBody, module::CompileModuleInfo, relocation::RelocationTarget, +}; use wasmer_types::{CompileError, FunctionType as FuncType, LocalFunctionIndex}; +use wasmer_vm::MemoryStyle; pub struct FuncTrampoline { ctx: Context, @@ -59,6 +62,7 @@ impl FuncTrampoline { ty: &FuncType, config: &LLVM, name: &str, + compile_info: &CompileModuleInfo, ) -> Result { // The function type, used for the callbacks. let function = CompiledKind::FunctionCallTrampoline(ty.clone()); @@ -70,9 +74,15 @@ impl FuncTrampoline { module.set_data_layout(&target_data.get_data_layout()); let intrinsics = Intrinsics::declare(&module, &self.ctx, &target_data, &self.binary_fmt); + let func_kind = if config.enable_g0m0_opt { + Some(G0M0FunctionKind::Local) + } else { + None + }; + let (callee_ty, callee_attrs) = self.abi - .func_type_to_llvm(&self.ctx, &intrinsics, None, ty)?; + .func_type_to_llvm(&self.ctx, &intrinsics, None, ty, func_kind)?; let trampoline_ty = intrinsics.void_ty.fn_type( &[ intrinsics.ptr_ty.into(), // vmctx ptr @@ -95,6 +105,8 @@ impl FuncTrampoline { trampoline_func.add_attribute(AttributeLoc::Function, intrinsics.uwtable); trampoline_func.add_attribute(AttributeLoc::Function, intrinsics.frame_pointer); self.generate_trampoline( + config, + compile_info, trampoline_func, ty, callee_ty, @@ -141,8 +153,9 @@ impl FuncTrampoline { ty: &FuncType, config: &LLVM, name: &str, + compile_info: &CompileModuleInfo, ) -> Result { - let module = self.trampoline_to_module(ty, config, name)?; + let module = self.trampoline_to_module(ty, config, name, compile_info)?; let function = CompiledKind::FunctionCallTrampoline(ty.clone()); let target_machine = &self.target_machine; @@ -222,7 +235,7 @@ impl FuncTrampoline { let (trampoline_ty, trampoline_attrs) = self.abi - .func_type_to_llvm(&self.ctx, &intrinsics, None, ty)?; + .func_type_to_llvm(&self.ctx, &intrinsics, None, ty, None)?; let trampoline_func = module.add_function(name, trampoline_ty, Some(Linkage::External)); for (attr, attr_loc) in trampoline_attrs { trampoline_func.add_attribute(attr_loc, attr); @@ -333,8 +346,11 @@ impl FuncTrampoline { }) } + #[allow(clippy::too_many_arguments)] fn generate_trampoline<'ctx>( &self, + config: &LLVM, + compile_info: &CompileModuleInfo, trampoline_func: FunctionValue, func_sig: &FuncType, llvm_func_type: FunctionType, @@ -361,7 +377,11 @@ impl FuncTrampoline { }; let mut args_vec: Vec = - Vec::with_capacity(func_sig.params().len() + 1); + Vec::with_capacity(if config.enable_g0m0_opt { + func_sig.params().len() + 3 + } else { + func_sig.params().len() + 1 + }); if self.abi.is_sret(func_sig)? { let basic_types: Vec<_> = func_sig @@ -376,6 +396,116 @@ impl FuncTrampoline { args_vec.push(callee_vmctx_ptr.into()); + if config.enable_g0m0_opt { + let wasm_module = &compile_info.module; + let memory_styles = &compile_info.memory_styles; + let callee_vmctx_ptr_value = callee_vmctx_ptr.into_pointer_value(); + // get value of G0, get a pointer to M0's base + + let offsets = wasmer_vm::VMOffsets::new(8, wasm_module); + + let global_index = wasmer_types::GlobalIndex::from_u32(0); + let global_type = wasm_module.globals[global_index]; + let global_value_type = global_type.ty; + let global_mutability = global_type.mutability; + + let offset = + if let Some(local_global_index) = wasm_module.local_global_index(global_index) { + offsets.vmctx_vmglobal_definition(local_global_index) + } else { + offsets.vmctx_vmglobal_import(global_index) + }; + let offset = intrinsics.i32_ty.const_int(offset.into(), false); + let global_ptr = { + let global_ptr_ptr = unsafe { + err!(builder.build_gep(intrinsics.i8_ty, callee_vmctx_ptr_value, &[offset], "")) + }; + let global_ptr_ptr = + err!(builder.build_bit_cast(global_ptr_ptr, intrinsics.ptr_ty, "")) + .into_pointer_value(); + let global_ptr = err!(builder.build_load(intrinsics.ptr_ty, global_ptr_ptr, "")) + .into_pointer_value(); + + global_ptr + }; + + let global_ptr = err!(builder.build_bit_cast( + global_ptr, + type_to_llvm_ptr(intrinsics, global_value_type)?, + "", + )) + .into_pointer_value(); + + let global_value = match global_mutability { + wasmer_types::Mutability::Const => { + err!(builder.build_load( + type_to_llvm(intrinsics, global_value_type)?, + global_ptr, + "g0", + )) + } + wasmer_types::Mutability::Var => { + err!(builder.build_load( + type_to_llvm(intrinsics, global_value_type)?, + global_ptr, + "" + )) + } + }; + + global_value.set_name("trmpl_g0"); + args_vec.push(global_value.into()); + + // load mem + let memory_index = wasmer_types::MemoryIndex::from_u32(0); + let memory_definition_ptr = if let Some(local_memory_index) = + wasm_module.local_memory_index(memory_index) + { + let offset = offsets.vmctx_vmmemory_definition(local_memory_index); + let offset = intrinsics.i32_ty.const_int(offset.into(), false); + unsafe { + err!(builder.build_gep(intrinsics.i8_ty, callee_vmctx_ptr_value, &[offset], "")) + } + } else { + let offset = offsets.vmctx_vmmemory_import(memory_index); + let offset = intrinsics.i32_ty.const_int(offset.into(), false); + let memory_definition_ptr_ptr = unsafe { + err!(builder.build_gep(intrinsics.i8_ty, callee_vmctx_ptr_value, &[offset], "")) + }; + let memory_definition_ptr_ptr = + err!(builder.build_bit_cast(memory_definition_ptr_ptr, intrinsics.ptr_ty, "",)) + .into_pointer_value(); + let memory_definition_ptr = + err!(builder.build_load(intrinsics.ptr_ty, memory_definition_ptr_ptr, "")) + .into_pointer_value(); + + memory_definition_ptr + }; + let memory_definition_ptr = + err!(builder.build_bit_cast(memory_definition_ptr, intrinsics.ptr_ty, "",)) + .into_pointer_value(); + let base_ptr = err!(builder.build_struct_gep( + intrinsics.vmmemory_definition_ty, + memory_definition_ptr, + intrinsics.vmmemory_definition_base_element, + "", + )); + + let memory_style = &memory_styles[memory_index]; + let base_ptr = if let MemoryStyle::Dynamic { .. } = memory_style { + base_ptr + } else { + let base_ptr = + err!(builder.build_load(intrinsics.ptr_ty, base_ptr, "")).into_pointer_value(); + + base_ptr + }; + + base_ptr.set_name("trmpl_m0_base_ptr"); + + args_vec.push(base_ptr.into()); + } + for (i, param_ty) in func_sig.params().iter().enumerate() { let index = intrinsics.i32_ty.const_int(i as _, false); let item_pointer = unsafe { diff --git a/lib/compiler-llvm/src/translator/code.rs b/lib/compiler-llvm/src/translator/code.rs index 1f1b7e61a67..a9c6c76d1f6 100644 --- a/lib/compiler-llvm/src/translator/code.rs +++ b/lib/compiler-llvm/src/translator/code.rs @@ -16,8 +16,9 @@ use inkwell::{ targets::{FileType, TargetMachine}, types::{BasicType, BasicTypeEnum, FloatMathType, IntType, PointerType, VectorType}, values::{ - AnyValue, BasicMetadataValueEnum, BasicValue, BasicValueEnum, FloatValue, FunctionValue, - InstructionOpcode, InstructionValue, IntValue, PhiValue, PointerValue, VectorValue, + AnyValue, BasicMetadataValueEnum, BasicValue, BasicValueEnum, CallSiteValue, FloatValue, + FunctionValue, InstructionOpcode, InstructionValue, IntValue, PhiValue, PointerValue, + VectorValue, }, AddressSpace, AtomicOrdering, AtomicRMWBinOp, DLLStorageClass, FloatPredicate, IntPredicate, }; @@ -26,7 +27,7 @@ use smallvec::SmallVec; use target_lexicon::BinaryFormat; use crate::{ - abi::{get_abi, Abi}, + abi::{get_abi, Abi, G0M0FunctionKind, LocalFunctionG0M0params}, config::{CompiledKind, LLVM}, error::{err, err_nt}, object_file::{load_object_file, CompiledFunction}, @@ -100,6 +101,14 @@ impl FuncTranslator { let func_index = wasm_module.func_index(*local_func_index); let function_name = symbol_registry.symbol_to_name(Symbol::LocalFunction(*local_func_index)); + + let g0m0_is_enabled = config.enable_g0m0_opt; + let func_kind = if g0m0_is_enabled { + Some(G0M0FunctionKind::Local) + } else { + None + }; + let module_name = match wasm_module.name.as_ref() { None => format!(" function {function_name}"), Some(module_name) => format!("module {module_name} function {function_name}"), @@ -119,9 +128,13 @@ impl FuncTranslator { // TODO: pointer width let offsets = VMOffsets::new(8, wasm_module); let intrinsics = Intrinsics::declare(&module, &self.ctx, &target_data, &self.binary_fmt); - let (func_type, func_attrs) = - self.abi - .func_type_to_llvm(&self.ctx, &intrinsics, Some(&offsets), wasm_fn_type)?; + let (func_type, func_attrs) = self.abi.func_type_to_llvm( + &self.ctx, + &intrinsics, + Some(&offsets), + wasm_fn_type, + func_kind, + )?; let func = module.add_function(&function_name, func_type, Some(Linkage::External)); for (attr, attr_loc) in &func_attrs { @@ -189,7 +202,13 @@ impl FuncTranslator { let mut params = vec![]; let first_param = if func_type.get_return_type().is_none() && wasm_fn_type.results().len() > 1 { - 2 + if g0m0_is_enabled { + 4 + } else { + 2 + } + } else if g0m0_is_enabled { + 3 } else { 1 }; @@ -230,7 +249,21 @@ impl FuncTranslator { let mut params_locals = params.clone(); params_locals.extend(locals.iter().cloned()); + let mut g0m0_params = None; + + if g0m0_is_enabled { + let value = self.abi.get_g0_ptr_param(&func); + let g0 = insert_alloca(intrinsics.i32_ty.as_basic_type_enum(), "g0")?; + err!(cache_builder.build_store(g0, value)); + g0.set_name("g0"); + let m0 = self.abi.get_m0_ptr_param(&func); + m0.set_name("m0_base_ptr"); + + g0m0_params = Some((g0, m0)); + } + let mut fcg = LLVMFunctionCodeGenerator { + g0m0: g0m0_params, context: &self.ctx, builder, alloca_builder, @@ -238,7 +271,7 @@ impl FuncTranslator { state, function: func, locals: params_locals, - ctx: CtxType::new(wasm_module, &func, &cache_builder, &*self.abi), + ctx: CtxType::new(wasm_module, &func, &cache_builder, &*self.abi, config), unreachable_depth: 0, memory_styles, _table_styles, @@ -310,6 +343,16 @@ impl FuncTranslator { passes.push("simplifycfg"); passes.push("mem2reg"); + //let llvm_dump_path = std::env::var("WASMER_LLVM_DUMP_DIR"); + //if let Ok(ref llvm_dump_path) = llvm_dump_path { + // let path = std::path::Path::new(llvm_dump_path); + // if !path.exists() { + // std::fs::create_dir_all(path).unwrap() + // } + // let path = path.join(format!("{function_name}.ll")); + // _ = module.print_to_file(path).unwrap(); + //} + module .run_passes( passes.join(",").as_str(), @@ -318,6 +361,14 @@ impl FuncTranslator { ) .unwrap(); + //if let Ok(ref llvm_dump_path) = llvm_dump_path { + // if !passes.is_empty() { + // let path = + // std::path::Path::new(llvm_dump_path).join(format!("{function_name}_opt.ll")); + // _ = module.print_to_file(path).unwrap(); + // } + //} + if let Some(ref callbacks) = config.callbacks { callbacks.postopt_ir(&function, &module); } @@ -358,6 +409,7 @@ impl FuncTranslator { } let mem_buf_slice = memory_buffer.as_slice(); + load_object_file( mem_buf_slice, &self.func_section, @@ -1171,7 +1223,9 @@ impl<'ctx, 'a> LLVMFunctionCodeGenerator<'ctx, 'a> { let offset = err!(builder.build_int_add(var_offset, imm_offset, "")); // Look up the memory base (as pointer) and bounds (as unsigned integer). - let base_ptr = + let base_ptr = if let Some((_, ref m0)) = self.g0m0 { + *m0 + } else { match self .ctx .memory(memory_index, intrinsics, self.module, self.memory_styles)? @@ -1284,7 +1338,8 @@ impl<'ctx, 'a> LLVMFunctionCodeGenerator<'ctx, 'a> { ptr_to_base } MemoryCache::Static { base_ptr } => base_ptr, - }; + } + }; let value_ptr = unsafe { err!(builder.build_gep(self.intrinsics.i8_ty, base_ptr, &[offset], "")) }; err_nt!(builder @@ -1485,6 +1540,262 @@ impl<'ctx, 'a> LLVMFunctionCodeGenerator<'ctx, 'a> { panic!() } } + + fn build_g0m0_indirect_call( + &mut self, + table_index: u32, + ctx_ptr: PointerValue<'ctx>, + func_type: &FunctionType, + func_ptr: PointerValue<'ctx>, + func_index: IntValue<'ctx>, + ) -> Result<(), CompileError> { + let Some((g0, m0)) = self.g0m0 else { + return Err(CompileError::Codegen( + "Call to build_g0m0_indirect_call without g0m0 parameters!".to_string(), + )); + }; + + let mut local_func_indices = vec![]; + let mut foreign_func_indices = vec![]; + + for t in &self.wasm_module.table_initializers { + if t.table_index.as_u32() == table_index { + for (func_in_table_idx, func_idx) in t.elements.iter().enumerate() { + if self.wasm_module.local_func_index(*func_idx).is_some() { + local_func_indices.push(func_in_table_idx) + } else { + foreign_func_indices.push(func_in_table_idx) + } + } + break; + } + } + + // removed with mem2reg. + // + let g0_value = err!(self.builder.build_load(self.intrinsics.i32_ty, g0, "g0")); + + let needs_switch = self.g0m0.is_some() + && !local_func_indices.is_empty() + && !foreign_func_indices.is_empty(); + + if needs_switch { + let foreign_idx_block = self + .context + .append_basic_block(self.function, "foreign_call_block"); + let local_idx_block = self + .context + .append_basic_block(self.function, "local_call_block"); + let unreachable_indirect_call_branch_block = self + .context + .append_basic_block(self.function, "unreachable_indirect_call_branch"); + + let cont = self.context.append_basic_block(self.function, "cont"); + + err!(self.builder.build_switch( + func_index, + unreachable_indirect_call_branch_block, + &local_func_indices + .into_iter() + .map(|v| ( + self.intrinsics.i32_ty.const_int(v as _, false), + local_idx_block + )) + .chain(foreign_func_indices.into_iter().map(|v| ( + self.intrinsics.i32_ty.const_int(v as _, false), + foreign_idx_block + ))) + .collect::>() + )); + + self.builder + .position_at_end(unreachable_indirect_call_branch_block); + err!(self.builder.build_unreachable()); + + //let current_block = self.builder.get_insert_block().unwrap(); + self.builder.position_at_end(local_idx_block); + let local_call_site = self.build_indirect_call( + ctx_ptr, + func_type, + func_ptr, + Some(G0M0FunctionKind::Local), + Some((g0_value.into_int_value(), m0)), + )?; + + let local_rets = self.abi.rets_from_call( + &self.builder, + self.intrinsics, + local_call_site, + func_type, + )?; + + err!(self.builder.build_unconditional_branch(cont)); + + self.builder.position_at_end(foreign_idx_block); + let foreign_call_site = self.build_indirect_call( + ctx_ptr, + func_type, + func_ptr, + Some(G0M0FunctionKind::Imported), + None, + )?; + + let foreign_rets = self.abi.rets_from_call( + &self.builder, + self.intrinsics, + foreign_call_site, + func_type, + )?; + + err!(self.builder.build_unconditional_branch(cont)); + + self.builder.position_at_end(cont); + + for i in 0..foreign_rets.len() { + let f_i = foreign_rets[i]; + let l_i = local_rets[i]; + let ty = f_i.get_type(); + let v = err!(self.builder.build_phi(ty, "")); + v.add_incoming(&[(&f_i, foreign_idx_block), (&l_i, local_idx_block)]); + self.state.push1(v.as_basic_value()); + } + } else if foreign_func_indices.is_empty() { + let call_site = self.build_indirect_call( + ctx_ptr, + func_type, + func_ptr, + Some(G0M0FunctionKind::Local), + Some((g0_value.into_int_value(), m0)), + )?; + + self.abi + .rets_from_call(&self.builder, self.intrinsics, call_site, func_type)? + .iter() + .for_each(|ret| self.state.push1(*ret)); + } else { + let call_site = self.build_indirect_call( + ctx_ptr, + func_type, + func_ptr, + Some(G0M0FunctionKind::Imported), + Some((g0_value.into_int_value(), m0)), + )?; + self.abi + .rets_from_call(&self.builder, self.intrinsics, call_site, func_type)? + .iter() + .for_each(|ret| self.state.push1(*ret)); + } + + Ok(()) + } + + fn build_indirect_call( + &mut self, + ctx_ptr: PointerValue<'ctx>, + func_type: &FunctionType, + func_ptr: PointerValue<'ctx>, + func_kind: Option, + g0m0_params: LocalFunctionG0M0params<'ctx>, + ) -> Result, CompileError> { + let (llvm_func_type, llvm_func_attrs) = self.abi.func_type_to_llvm( + self.context, + self.intrinsics, + Some(self.ctx.get_offsets()), + func_type, + func_kind, + )?; + + let params = self.state.popn_save_extra(func_type.params().len())?; + + // Apply pending canonicalizations. + let params = params + .iter() + .zip(func_type.params().iter()) + .map(|((v, info), wasm_ty)| match wasm_ty { + Type::F32 => err_nt!(self.builder.build_bit_cast( + self.apply_pending_canonicalization(*v, *info)?, + self.intrinsics.f32_ty, + "", + )), + Type::F64 => err_nt!(self.builder.build_bit_cast( + self.apply_pending_canonicalization(*v, *info)?, + self.intrinsics.f64_ty, + "", + )), + Type::V128 => self.apply_pending_canonicalization(*v, *info), + _ => Ok(*v), + }) + .collect::, _>>()?; + + let params = self.abi.args_to_call( + &self.alloca_builder, + func_type, + &llvm_func_type, + ctx_ptr, + params.as_slice(), + self.intrinsics, + g0m0_params, + )?; + + let typed_func_ptr = err!(self.builder.build_pointer_cast( + func_ptr, + self.context.ptr_type(AddressSpace::default()), + "typed_func_ptr", + )); + + /* + if self.track_state { + if let Some(offset) = opcode_offset { + let mut stackmaps = self.stackmaps.borrow_mut(); + emit_stack_map( + &info, + self.intrinsics, + self.builder, + self.index, + &mut *stackmaps, + StackmapEntryKind::Call, + &self.locals, + state, + ctx, + offset, + ) + } + } + */ + + let call_site_local = if let Some(lpad) = self.state.get_landingpad() { + let then_block = self.context.append_basic_block(self.function, "then_block"); + + let ret = err!(self.builder.build_indirect_invoke( + llvm_func_type, + typed_func_ptr, + params.as_slice(), + then_block, + lpad, + "", + )); + + self.builder.position_at_end(then_block); + ret + } else { + err!(self.builder.build_indirect_call( + llvm_func_type, + typed_func_ptr, + params + .iter() + .copied() + .map(Into::into) + .collect::>() + .as_slice(), + "indirect_call", + )) + }; + for (attr, attr_loc) in llvm_func_attrs { + call_site_local.add_attribute(attr_loc, attr); + } + + Ok(call_site_local) + } } /* @@ -1573,6 +1884,7 @@ fn finalize_opcode_stack_map<'ctx>( */ pub struct LLVMFunctionCodeGenerator<'ctx, 'a> { + g0m0: Option<(PointerValue<'ctx>, PointerValue<'ctx>)>, context: &'ctx Context, builder: Builder<'ctx>, alloca_builder: Builder<'ctx>, @@ -2306,52 +2618,75 @@ impl<'ctx, 'a> LLVMFunctionCodeGenerator<'ctx, 'a> { } Operator::GlobalGet { global_index } => { - let global_index = GlobalIndex::from_u32(global_index); - match self - .ctx - .global(global_index, self.intrinsics, self.module)? - { - GlobalCache::Const { value } => { - self.state.push1(*value); - } - GlobalCache::Mut { - ptr_to_value, - value_type, - } => { - let value = err!(self.builder.build_load(*value_type, *ptr_to_value, "")); - tbaa_label( - self.module, - self.intrinsics, - format!("global {}", global_index.as_u32()), - value.as_instruction_value().unwrap(), - ); - self.state.push1(value); + if self.g0m0.is_some() && global_index == 0 { + let Some((g0, _)) = self.g0m0 else { + unreachable!() + }; + + // Removed with mem2reg. + let value = err!(self.builder.build_load(self.intrinsics.i32_ty, g0, "")); + + self.state.push1(value); + } else { + let global_index = GlobalIndex::from_u32(global_index); + match self + .ctx + .global(global_index, self.intrinsics, self.module)? + { + GlobalCache::Const { value } => { + self.state.push1(*value); + } + GlobalCache::Mut { + ptr_to_value, + value_type, + } => { + let value = + err!(self.builder.build_load(*value_type, *ptr_to_value, "")); + tbaa_label( + self.module, + self.intrinsics, + format!("global {}", global_index.as_u32()), + value.as_instruction_value().unwrap(), + ); + self.state.push1(value); + } } } } Operator::GlobalSet { global_index } => { - let global_index = GlobalIndex::from_u32(global_index); - match self - .ctx - .global(global_index, self.intrinsics, self.module)? - { - GlobalCache::Const { value: _ } => { - return Err(CompileError::Codegen(format!( - "global.set on immutable global index {}", - global_index.as_u32() - ))) - } - GlobalCache::Mut { ptr_to_value, .. } => { - let ptr_to_value = *ptr_to_value; - let (value, info) = self.state.pop1_extra()?; - let value = self.apply_pending_canonicalization(value, info)?; - let store = err!(self.builder.build_store(ptr_to_value, value)); - tbaa_label( - self.module, - self.intrinsics, - format!("global {}", global_index.as_u32()), - store, - ); + if self.g0m0.is_some() && global_index == 0 { + let Some((g0, _)) = self.g0m0 else { + unreachable!() + }; + let ptr_to_value = g0; + let (value, info) = self.state.pop1_extra()?; + let value = self.apply_pending_canonicalization(value, info)?; + let store = err!(self.builder.build_store(ptr_to_value, value)); + tbaa_label(self.module, self.intrinsics, "global 0".to_string(), store); + } else { + let global_index = GlobalIndex::from_u32(global_index); + match self + .ctx + .global(global_index, self.intrinsics, self.module)? + { + GlobalCache::Const { value: _ } => { + return Err(CompileError::Codegen(format!( + "global.set on immutable global index {}", + global_index.as_u32() + ))) + } + GlobalCache::Mut { ptr_to_value, .. } => { + let ptr_to_value = *ptr_to_value; + let (value, info) = self.state.pop1_extra()?; + let value = self.apply_pending_canonicalization(value, info)?; + let store = err!(self.builder.build_store(ptr_to_value, value)); + tbaa_label( + self.module, + self.intrinsics, + format!("global {}", global_index.as_u32()), + store, + ); + } } } } @@ -2405,15 +2740,25 @@ impl<'ctx, 'a> LLVMFunctionCodeGenerator<'ctx, 'a> { let sigindex = &self.wasm_module.functions[func_index]; let func_type = &self.wasm_module.signatures[*sigindex]; + let mut g0m0_params = None; + let FunctionCache { func, llvm_func_type, vmctx: callee_vmctx, attrs, } = if let Some(local_func_index) = self.wasm_module.local_func_index(func_index) { + if let Some((g0, m0)) = &self.g0m0 { + // removed with mem2reg. + let value = err!(self.builder.build_load(self.intrinsics.i32_ty, *g0, "")); + + g0m0_params = Some((value.into_int_value(), *m0)); + } + let function_name = self .symbol_registry .symbol_to_name(Symbol::LocalFunction(local_func_index)); + self.ctx.local_func( local_func_index, func_index, @@ -2466,6 +2811,7 @@ impl<'ctx, 'a> LLVMFunctionCodeGenerator<'ctx, 'a> { callee_vmctx.into_pointer_value(), params.as_slice(), self.intrinsics, + g0m0_params, )?; /* @@ -2756,120 +3102,28 @@ impl<'ctx, 'a> LLVMFunctionCodeGenerator<'ctx, 'a> { err!(self.builder.build_unreachable()); self.builder.position_at_end(continue_block); - let (llvm_func_type, llvm_func_attrs) = self.abi.func_type_to_llvm( - self.context, - self.intrinsics, - Some(self.ctx.get_offsets()), - func_type, - )?; - - let params = self.state.popn_save_extra(func_type.params().len())?; - - // Apply pending canonicalizations. - let params = params - .iter() - .zip(func_type.params().iter()) - .map(|((v, info), wasm_ty)| match wasm_ty { - Type::F32 => err_nt!(self.builder.build_bit_cast( - self.apply_pending_canonicalization(*v, *info)?, - self.intrinsics.f32_ty, - "", - )), - Type::F64 => err_nt!(self.builder.build_bit_cast( - self.apply_pending_canonicalization(*v, *info)?, - self.intrinsics.f64_ty, - "", - )), - Type::V128 => self.apply_pending_canonicalization(*v, *info), - _ => Ok(*v), - }) - .collect::, _>>()?; - - let params = self.abi.args_to_call( - &self.alloca_builder, - func_type, - &llvm_func_type, - ctx_ptr.into_pointer_value(), - params.as_slice(), - self.intrinsics, - )?; - - let typed_func_ptr = err!(self.builder.build_pointer_cast( - func_ptr, - self.context.ptr_type(AddressSpace::default()), - "typed_func_ptr", - )); - - /* - if self.track_state { - if let Some(offset) = opcode_offset { - let mut stackmaps = self.stackmaps.borrow_mut(); - emit_stack_map( - &info, - self.intrinsics, - self.builder, - self.index, - &mut *stackmaps, - StackmapEntryKind::Call, - &self.locals, - state, - ctx, - offset, - ) - } - } - */ - - let call_site = if let Some(lpad) = self.state.get_landingpad() { - let then_block = self.context.append_basic_block(self.function, "then_block"); - - let ret = err!(self.builder.build_indirect_invoke( - llvm_func_type, - typed_func_ptr, - params.as_slice(), - then_block, - lpad, - "", - )); - - self.builder.position_at_end(then_block); - ret + if self.g0m0.is_some() { + self.build_g0m0_indirect_call( + table_index, + ctx_ptr.into_pointer_value(), + func_type, + func_ptr, + func_index, + )?; } else { - err!(self.builder.build_indirect_call( - llvm_func_type, - typed_func_ptr, - params - .iter() - .copied() - .map(Into::into) - .collect::>() - .as_slice(), - "indirect_call", - )) - }; - for (attr, attr_loc) in llvm_func_attrs { - call_site.add_attribute(attr_loc, attr); - } - /* - if self.track_state { - if let Some(offset) = opcode_offset { - let mut stackmaps = self.stackmaps.borrow_mut(); - finalize_opcode_stack_map( - self.intrinsics, - self.builder, - self.index, - &mut *stackmaps, - StackmapEntryKind::Call, - offset, - ) - } - } - */ + let call_site = self.build_indirect_call( + ctx_ptr.into_pointer_value(), + func_type, + func_ptr, + None, + None, + )?; - self.abi - .rets_from_call(&self.builder, self.intrinsics, call_site, func_type)? - .iter() - .for_each(|ret| self.state.push1(*ret)); + self.abi + .rets_from_call(&self.builder, self.intrinsics, call_site, func_type)? + .iter() + .for_each(|ret| self.state.push1(*ret)); + } } /*************************** diff --git a/lib/compiler-llvm/src/translator/intrinsics.rs b/lib/compiler-llvm/src/translator/intrinsics.rs index 5ec8494e4bc..ac53bd1428d 100644 --- a/lib/compiler-llvm/src/translator/intrinsics.rs +++ b/lib/compiler-llvm/src/translator/intrinsics.rs @@ -6,6 +6,7 @@ use crate::abi::Abi; use crate::error::err; +use crate::LLVM; use inkwell::values::BasicMetadataValueEnum; use inkwell::{ attributes::{Attribute, AttributeLoc}, @@ -1211,6 +1212,7 @@ pub struct FunctionCache<'ctx> { pub struct CtxType<'ctx, 'a> { ctx_ptr_value: PointerValue<'ctx>, + config: &'a LLVM, wasm_module: &'a WasmerCompilerModule, cache_builder: &'a Builder<'ctx>, abi: &'a dyn Abi, @@ -1232,8 +1234,10 @@ impl<'ctx, 'a> CtxType<'ctx, 'a> { func_value: &FunctionValue<'ctx>, cache_builder: &'a Builder<'ctx>, abi: &'a dyn Abi, + config: &'a LLVM, ) -> CtxType<'ctx, 'a> { CtxType { + config, ctx_ptr_value: abi.get_vmctx_ptr_param(func_value), wasm_module, @@ -1695,9 +1699,17 @@ impl<'ctx, 'a> CtxType<'ctx, 'a> { Entry::Occupied(entry) => entry.into_mut(), Entry::Vacant(entry) => { debug_assert!(module.get_function(function_name).is_none()); - let (llvm_func_type, llvm_func_attrs) = - self.abi - .func_type_to_llvm(context, intrinsics, Some(offsets), func_type)?; + let (llvm_func_type, llvm_func_attrs) = self.abi.func_type_to_llvm( + context, + intrinsics, + Some(offsets), + func_type, + if self.config.enable_g0m0_opt { + Some(crate::abi::G0M0FunctionKind::Local) + } else { + None + }, + )?; let func = module.add_function(function_name, llvm_func_type, Some(Linkage::External)); for (attr, attr_loc) in &llvm_func_attrs { @@ -1730,9 +1742,13 @@ impl<'ctx, 'a> CtxType<'ctx, 'a> { match cached_functions.entry(function_index) { Entry::Occupied(entry) => Ok(entry.into_mut()), Entry::Vacant(entry) => { - let (llvm_func_type, llvm_func_attrs) = - self.abi - .func_type_to_llvm(context, intrinsics, Some(offsets), func_type)?; + let (llvm_func_type, llvm_func_attrs) = self.abi.func_type_to_llvm( + context, + intrinsics, + Some(offsets), + func_type, + None, + )?; debug_assert!(wasm_module.local_func_index(function_index).is_none()); let offset = offsets.vmctx_vmfunction_import(function_index); let offset = intrinsics.i32_ty.const_int(offset.into(), false); diff --git a/lib/compiler-singlepass/src/compiler.rs b/lib/compiler-singlepass/src/compiler.rs index e5733774ec5..a48140a466b 100644 --- a/lib/compiler-singlepass/src/compiler.rs +++ b/lib/compiler-singlepass/src/compiler.rs @@ -38,6 +38,7 @@ use wasmer_types::{ /// A compiler that compiles a WebAssembly module with Singlepass. /// It does the compilation in one pass +#[derive(Debug)] pub struct SinglepassCompiler { config: Singlepass, } @@ -59,6 +60,10 @@ impl Compiler for SinglepassCompiler { "singlepass" } + fn deterministic_id(&self) -> String { + String::from("singlepass") + } + /// Get the middlewares for this compiler fn get_middlewares(&self) -> &[Arc] { &self.config.middlewares diff --git a/lib/compiler/src/compiler.rs b/lib/compiler/src/compiler.rs index 8167c78a020..82d3976c7db 100644 --- a/lib/compiler/src/compiler.rs +++ b/lib/compiler/src/compiler.rs @@ -12,7 +12,7 @@ use enumset::EnumSet; use wasmer_types::{ entity::PrimaryMap, error::CompileError, - target::{CpuFeature, Target}, + target::{CpuFeature, Target, UserCompilerOptimizations}, Features, LocalFunctionIndex, }; #[cfg(feature = "translator")] @@ -39,6 +39,12 @@ pub trait CompilerConfig { // in case they create an IR that they can verify. } + /// Enable generation of perfmaps to sample the JIT compiled frames. + fn enable_perfmap(&mut self) { + // By default we do nothing, each backend will need to customize this + // in case they create an IR that they can verify. + } + /// Enable NaN canonicalization. /// /// NaN canonicalization is useful when trying to run WebAssembly @@ -75,12 +81,31 @@ where } /// An implementation of a Compiler from parsed WebAssembly module to Compiled native code. -pub trait Compiler: Send { +pub trait Compiler: Send + std::fmt::Debug { /// Returns a descriptive name for this compiler. /// /// Note that this is an API breaking change since 3.0 fn name(&self) -> &str; + /// Returns the deterministic id of this compiler. Same compilers with different + /// optimizations map to different deterministic IDs. + fn deterministic_id(&self) -> String; + + /// Add suggested optimizations to this compiler. + /// + /// # Note + /// + /// Not every compiler supports every optimization. This function may fail (i.e. not set the + /// suggested optimizations) silently if the underlying compiler does not support one or + /// more optimizations. + fn with_opts( + &mut self, + suggested_compiler_opts: &UserCompilerOptimizations, + ) -> Result<(), CompileError> { + _ = suggested_compiler_opts; + Ok(()) + } + /// Validates a module. /// /// It returns the a succesful Result in case is valid, `CompileError` in case is not. @@ -155,4 +180,9 @@ pub trait Compiler: Send { fn get_cpu_features_used(&self, cpu_features: &EnumSet) -> EnumSet { *cpu_features } + + /// Get whether `perfmap` is enabled or not. + fn get_perfmap_enabled(&self) -> bool { + false + } } diff --git a/lib/compiler/src/engine/artifact.rs b/lib/compiler/src/engine/artifact.rs index ba62b7c8181..a00cca0dad1 100644 --- a/lib/compiler/src/engine/artifact.rs +++ b/lib/compiler/src/engine/artifact.rs @@ -446,6 +446,11 @@ impl Artifact { get_got_address(RelocationTarget::LibCall(wasmer_vm::LibCall::EHPersonality)), )?; + #[cfg(not(target_arch = "wasm32"))] + { + engine_inner.register_perfmap(&finished_functions, module_info)?; + } + // Make all code compiled thus far executable. engine_inner.publish_compiled_code(); diff --git a/lib/compiler/src/engine/inner.rs b/lib/compiler/src/engine/inner.rs index 01f555ce1c0..e1747510fc8 100644 --- a/lib/compiler/src/engine/inner.rs +++ b/lib/compiler/src/engine/inner.rs @@ -17,8 +17,11 @@ use shared_buffer::OwnedBuffer; #[cfg(not(target_arch = "wasm32"))] use std::path::Path; -use std::sync::atomic::{AtomicUsize, Ordering::SeqCst}; use std::sync::{Arc, Mutex}; +use std::{ + io::Write, + sync::atomic::{AtomicUsize, Ordering::SeqCst}, +}; #[cfg(feature = "compiler")] use wasmer_types::Features; @@ -27,7 +30,7 @@ use wasmer_types::{ entity::PrimaryMap, DeserializeError, FunctionIndex, FunctionType, LocalFunctionIndex, SignatureIndex, }; -use wasmer_types::{target::Target, CompileError, HashAlgorithm}; +use wasmer_types::{target::Target, CompileError, HashAlgorithm, ModuleInfo}; #[cfg(not(target_arch = "wasm32"))] use wasmer_vm::{ @@ -94,12 +97,21 @@ impl Engine { } /// Returns the deterministic id of this engine - pub fn deterministic_id(&self) -> &str { - // TODO: add a `deterministic_id` to the Compiler, so two - // compilers can actually serialize into a different deterministic_id - // if their configuration is different (eg. LLVM with optimizations vs LLVM - // without optimizations) - self.name.as_str() + pub fn deterministic_id(&self) -> String { + let i = self.inner(); + #[cfg(feature = "compiler")] + { + if let Some(ref c) = i.compiler { + return c.deterministic_id(); + } else { + return self.name.clone(); + } + } + + #[allow(unreachable_code)] + { + self.name.to_string() + } } /// Create a headless `Engine` @@ -283,11 +295,34 @@ impl Engine { pub fn tunables(&self) -> &dyn Tunables { self.tunables.as_ref() } + + /// Add suggested optimizations to this engine. + /// + /// # Note + /// + /// Not every backend supports every optimization. This function may fail (i.e. not set the + /// suggested optimizations) silently if the underlying engine backend does not support one or + /// more optimizations. + pub fn with_opts( + &mut self, + suggested_opts: &wasmer_types::target::UserCompilerOptimizations, + ) -> Result<(), CompileError> { + #[cfg(feature = "compiler")] + { + let mut i = self.inner_mut(); + if let Some(ref mut c) = i.compiler { + c.with_opts(suggested_opts)?; + } + } + + Ok(()) + } } impl std::fmt::Debug for Engine { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.debug_struct("Engine") + .field("inner", &self.inner) .field("target", &self.target) .field("engine_id", &self.engine_id) .field("name", &self.name) @@ -313,6 +348,24 @@ pub struct EngineInner { signatures: SignatureRegistry, } +impl std::fmt::Debug for EngineInner { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let mut formatter = f.debug_struct("EngineInner"); + #[cfg(feature = "compiler")] + { + formatter.field("compiler", &self.compiler); + formatter.field("features", &self.features); + } + + #[cfg(not(target_arch = "wasm32"))] + { + formatter.field("signatures", &self.signatures); + } + + formatter.finish() + } +} + impl EngineInner { /// Gets the compiler associated to this engine. #[cfg(feature = "compiler")] @@ -487,6 +540,42 @@ impl EngineInner { .unwrap() .register_frame_info(frame_info); } + + #[cfg(not(target_arch = "wasm32"))] + pub(crate) fn register_perfmap( + &self, + finished_functions: &PrimaryMap, + module_info: &ModuleInfo, + ) -> Result<(), CompileError> { + if self + .compiler + .as_ref() + .is_some_and(|v| v.get_perfmap_enabled()) + { + let filename = format!("/tmp/perf-{}.map", std::process::id()); + let mut file = std::io::BufWriter::new(std::fs::File::create(filename).unwrap()); + + for (func_index, code) in finished_functions.iter() { + let func_index = module_info.func_index(func_index); + let name = if let Some(func_name) = module_info.function_names.get(&func_index) { + func_name.clone() + } else { + format!("{:p}", code.ptr.0) + }; + + let sanitized_name = name.replace(['\n', '\r'], "_"); + let line = format!( + "{:p} {:x} {}\n", + code.ptr.0 as *const _, code.length, sanitized_name + ); + write!(file, "{line}").map_err(|e| CompileError::Codegen(e.to_string()))?; + file.flush() + .map_err(|e| CompileError::Codegen(e.to_string()))?; + } + } + + Ok(()) + } } #[cfg(feature = "compiler")] diff --git a/lib/config/src/package/mod.rs b/lib/config/src/package/mod.rs index e21e7e1d2ca..2fb81ed7d05 100644 --- a/lib/config/src/package/mod.rs +++ b/lib/config/src/package/mod.rs @@ -503,6 +503,26 @@ pub struct Module { /// Interface definitions that can be used to generate bindings to this /// module. pub bindings: Option, + /// Miscellaneous annotations from the user. + #[serde(skip_serializing_if = "Option::is_none")] + pub annotations: Option, +} + +/// Miscellaneous annotations specified by the user. +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)] +pub struct UserAnnotations { + pub suggested_compiler_optimizations: SuggestedCompilerOptimizations, +} + +/// Suggested optimization that might be operated on the module when (and if) compiled. +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, Default)] +pub struct SuggestedCompilerOptimizations { + pub pass_params: Option, +} + +impl SuggestedCompilerOptimizations { + pub const KEY: &'static str = "suggested_compiler_optimizations"; + pub const PASS_PARAMS_KEY: &'static str = "pass_params"; } /// The interface exposed by a [`Module`]. @@ -1007,6 +1027,7 @@ mod tests { interfaces: None, kind: Some("https://webc.org/kind/wasi".to_string()), source: Path::new("test.wasm").to_path_buf(), + annotations: None, }], commands: Vec::new(), fs: vec![ @@ -1062,6 +1083,7 @@ module = "mod" wit_exports: PathBuf::from("exports.wit"), wit_bindgen: "0.0.0".parse().unwrap() })), + annotations: None }, ); } diff --git a/lib/package/src/convert/webc_to_package.rs b/lib/package/src/convert/webc_to_package.rs index c15cbc17f23..81dea85b909 100644 --- a/lib/package/src/convert/webc_to_package.rs +++ b/lib/package/src/convert/webc_to_package.rs @@ -1,6 +1,6 @@ use std::path::Path; -use wasmer_config::package::ModuleReference; +use wasmer_config::package::{ModuleReference, SuggestedCompilerOptimizations, UserAnnotations}; use webc::Container; @@ -150,6 +150,29 @@ pub fn webc_to_package_dir(webc: &Container, target_dir: &Path) -> Result<(), Co let relative_path = format!("./{module_dir_name}/{atom_name}"); + let mut annotations = None; + + if let Some(manifest_atom) = manifest.atoms.get(&atom_name) { + if let Some(sco) = manifest_atom + .annotations + .get(SuggestedCompilerOptimizations::KEY) + { + if let Some((_, v)) = sco.as_map().and_then(|v| { + v.iter().find(|(k, _)| { + k.as_text().is_some_and(|v| { + v == SuggestedCompilerOptimizations::PASS_PARAMS_KEY + }) + }) + }) { + annotations = Some(UserAnnotations { + suggested_compiler_optimizations: SuggestedCompilerOptimizations { + pass_params: Some(v.as_bool().unwrap_or_default()), + }, + }); + } + } + } + pkg_manifest.modules.push(wasmer_config::package::Module { name: atom_name, source: relative_path.into(), @@ -157,6 +180,7 @@ pub fn webc_to_package_dir(webc: &Container, target_dir: &Path) -> Result<(), Co kind: None, interfaces: None, bindings: None, + annotations, }); } } diff --git a/lib/package/src/package/manifest.rs b/lib/package/src/package/manifest.rs index c9d9519ea54..e716e46332d 100644 --- a/lib/package/src/package/manifest.rs +++ b/lib/package/src/package/manifest.rs @@ -3,13 +3,14 @@ use std::{ path::{Path, PathBuf}, }; -use ciborium::Value; +use ciborium::{cbor, Value}; use semver::VersionReq; use sha2::Digest; use shared_buffer::{MmapError, OwnedBuffer}; use url::Url; #[allow(deprecated)] use wasmer_config::package::{CommandV1, CommandV2, Manifest as WasmerManifest, Package}; +use wasmer_config::package::{SuggestedCompilerOptimizations, UserAnnotations}; use webc::{ indexmap::{self, IndexMap}, metadata::AtomSignature, @@ -160,7 +161,7 @@ pub(crate) fn wasmer_manifest_to_webc( /// take a `wasmer.toml` manifest and convert it to the `*.webc` equivalent. pub(crate) fn in_memory_wasmer_manifest_to_webc( manifest: &WasmerManifest, - atoms: &BTreeMap, OwnedBuffer)>, + atoms: &BTreeMap, OwnedBuffer, Option<&UserAnnotations>)>, ) -> Result<(WebcManifest, BTreeMap), ManifestError> { let use_map = transform_dependencies(&manifest.dependencies)?; @@ -279,27 +280,41 @@ fn transform_atoms( error, })?; - atom_entries.insert(name.clone(), (module.kind.clone(), file)); + atom_entries.insert( + name.clone(), + (module.kind.clone(), file, module.annotations.as_ref()), + ); } transform_atoms_shared(&atom_entries) } fn transform_in_memory_atoms( - atoms: &BTreeMap, OwnedBuffer)>, + atoms: &BTreeMap, OwnedBuffer, Option<&UserAnnotations>)>, ) -> Result<(IndexMap, Atoms), ManifestError> { transform_atoms_shared(atoms) } fn transform_atoms_shared( - atoms: &BTreeMap, OwnedBuffer)>, + atoms: &BTreeMap, OwnedBuffer, Option<&UserAnnotations>)>, ) -> Result<(IndexMap, Atoms), ManifestError> { let mut atom_files = BTreeMap::new(); let mut metadata = IndexMap::new(); - for (name, (kind, content)) in atoms.iter() { + for (name, (kind, content, misc_annotations)) in atoms.iter() { // Create atom with annotations including Wasm features if available let mut annotations = IndexMap::new(); + if let Some(misc_annotations) = misc_annotations { + if let Some(pass_params) = misc_annotations + .suggested_compiler_optimizations + .pass_params + { + annotations.insert( + SuggestedCompilerOptimizations::KEY.to_string(), + cbor!({"pass_params" => pass_params}).unwrap(), + ); + } + } // Detect required WebAssembly features by analyzing the module binary let features_result = wasmer_types::Features::detect_from_wasm(content); diff --git a/lib/package/src/package/package.rs b/lib/package/src/package/package.rs index 4ca0f6955b1..74f8dc664dc 100644 --- a/lib/package/src/package/package.rs +++ b/lib/package/src/package/package.rs @@ -355,8 +355,21 @@ impl Package { let volumes = new_volumes; + let mut annotated_atoms = BTreeMap::new(); + + for (atom_name, (a, b)) in atoms { + let annotations = + if let Some(module) = manifest.modules.iter().find(|v| v.name == atom_name) { + module.annotations.as_ref() + } else { + None + }; + + annotated_atoms.insert(atom_name, (a, b, annotations)); + } + let (mut manifest, atoms) = - super::manifest::in_memory_wasmer_manifest_to_webc(&manifest, &atoms)?; + super::manifest::in_memory_wasmer_manifest_to_webc(&manifest, &annotated_atoms)?; if let Some(entry) = manifest.package.get_mut(Wapm::KEY) { let mut wapm: Wapm = entry.deserialized()?; diff --git a/lib/types/src/target.rs b/lib/types/src/target.rs index 5aec0d0e63b..739d16a3be1 100644 --- a/lib/types/src/target.rs +++ b/lib/types/src/target.rs @@ -229,3 +229,13 @@ impl Default for Target { } } } + +/// User-suggested optimization that might be operated on the module when (and if) compiled. +/// +// Note: This type is a copy of `wasmer_config::package::SuggestedCompilerOptimizations`, so to +// avoid dependencies on `wasmer_config` for crates that already depend on `wasmer_types`. +#[derive(Clone, Debug, PartialEq, Eq, Default)] +pub struct UserCompilerOptimizations { + /// Suggest the `pass_params` (also known as g0m0) optimization pass. + pub pass_params: Option, +} diff --git a/lib/vm/src/instance/mod.rs b/lib/vm/src/instance/mod.rs index 40f7a5d97c7..7fcab05a697 100644 --- a/lib/vm/src/instance/mod.rs +++ b/lib/vm/src/instance/mod.rs @@ -12,7 +12,7 @@ use crate::export::VMExtern; use crate::imports::Imports; use crate::store::{InternalStoreHandle, StoreObjects}; use crate::table::TableElement; -use crate::trap::{catch_traps, Trap, TrapCode}; +use crate::trap::{Trap, TrapCode}; use crate::vmcontext::{ memory32_atomic_check32, memory32_atomic_check64, memory_copy, memory_fill, VMBuiltinFunctionsArray, VMCallerCheckedAnyfunc, VMContext, VMFunctionContext, @@ -20,7 +20,7 @@ use crate::vmcontext::{ VMMemoryImport, VMSharedSignatureIndex, VMTableDefinition, VMTableImport, VMTagImport, VMTrampoline, }; -use crate::{FunctionBodyPtr, MaybeInstanceOwned, TrapHandlerFn, VMFunctionBody, VMTag}; +use crate::{wasmer_call_trampoline, FunctionBodyPtr, MaybeInstanceOwned, TrapHandlerFn, VMTag}; use crate::{LinearMemory, NotifyLocation}; use crate::{VMConfig, VMFuncRef, VMFunction, VMGlobal, VMMemory, VMTable}; pub use allocator::InstanceAllocator; @@ -371,13 +371,22 @@ impl Instance { } }; - // Make the call. + let sig = self.module.functions[start_index]; + let trampoline = self.function_call_trampolines[sig]; + let values_vec = vec![].as_mut_ptr(); + unsafe { - catch_traps(trap_handler, config, move || { - mem::transmute::<*const VMFunctionBody, unsafe extern "C" fn(VMFunctionContext)>( - callee_address, - )(callee_vmctx) - }) + // Even though we already know the type of the function we need to call, in certain + // specific cases trampoline prepare callee arguments for specific optimizations, such + // as passing g0 and m0_base_ptr as paramters. + wasmer_call_trampoline( + trap_handler, + config, + callee_vmctx, + trampoline, + callee_address, + values_vec, + ) } } diff --git a/lib/wasix/src/bin_factory/binary_package.rs b/lib/wasix/src/bin_factory/binary_package.rs index 78649b14829..6688c712b9c 100644 --- a/lib/wasix/src/bin_factory/binary_package.rs +++ b/lib/wasix/src/bin_factory/binary_package.rs @@ -4,7 +4,9 @@ use anyhow::Context; use once_cell::sync::OnceCell; use sha2::Digest; use virtual_fs::FileSystem; -use wasmer_config::package::{PackageHash, PackageId, PackageSource}; +use wasmer_config::package::{ + PackageHash, PackageId, PackageSource, SuggestedCompilerOptimizations, +}; use wasmer_package::package::Package; use webc::compat::SharedBytes; use webc::Container; @@ -24,6 +26,7 @@ pub struct BinaryPackageCommand { pub(crate) atom: SharedBytes, hash: ModuleHash, features: Option, + pub suggested_compiler_optimizations: SuggestedCompilerOptimizations, } impl BinaryPackageCommand { @@ -33,6 +36,7 @@ impl BinaryPackageCommand { atom: SharedBytes, hash: ModuleHash, features: Option, + suggested_compiler_optimizations: SuggestedCompilerOptimizations, ) -> Self { Self { name, @@ -40,6 +44,7 @@ impl BinaryPackageCommand { atom, hash, features, + suggested_compiler_optimizations, } } diff --git a/lib/wasix/src/runtime/mod.rs b/lib/wasix/src/runtime/mod.rs index cfe2b974ce6..917f97bde43 100644 --- a/lib/wasix/src/runtime/mod.rs +++ b/lib/wasix/src/runtime/mod.rs @@ -5,7 +5,10 @@ pub mod task_manager; pub use self::task_manager::{SpawnMemoryType, VirtualTaskManager}; use self::{module_cache::CacheError, task_manager::InlineWaker}; -use wasmer_types::ModuleHash; +use wasmer_config::package::SuggestedCompilerOptimizations; +use wasmer_types::{ + target::UserCompilerOptimizations as WasmerSuggestedCompilerOptimizations, ModuleHash, +}; use std::{ fmt, @@ -15,7 +18,7 @@ use std::{ use futures::future::BoxFuture; use virtual_net::{DynVirtualNetworking, VirtualNetworking}; -use wasmer::{Module, RuntimeError}; +use wasmer::{CompileError, Module, RuntimeError}; use wasmer_wasix_types::wasi::ExitCode; #[cfg(feature = "journal")] @@ -77,6 +80,17 @@ where wasmer::Engine::default() } + fn engine_with_suggested_opts( + &self, + suggested_opts: &SuggestedCompilerOptimizations, + ) -> Result { + let mut engine = self.engine(); + engine.with_opts(&WasmerSuggestedCompilerOptimizations { + pass_params: suggested_opts.pass_params, + })?; + Ok(engine) + } + /// Create a new [`wasmer::Store`]. fn new_store(&self) -> wasmer::Store { cfg_if::cfg_if! { @@ -106,11 +120,22 @@ where &self, cmd: &BinaryPackageCommand, ) -> BoxFuture<'_, Result> { - let engine = self.engine(); let hash = *cmd.hash(); let wasm = cmd.atom(); let module_cache = self.module_cache(); + let engine = match self.engine_with_suggested_opts(&cmd.suggested_compiler_optimizations) { + Ok(engine) => engine, + Err(error) => { + return Box::pin(async move { + Err(SpawnError::CompileError { + module_hash: hash, + error, + }) + }) + } + }; + let task = async move { load_module(&engine, &module_cache, &wasm, hash).await }; Box::pin(task) diff --git a/lib/wasix/src/runtime/module_cache/filesystem.rs b/lib/wasix/src/runtime/module_cache/filesystem.rs index 5528ff2261f..3571a2e759d 100644 --- a/lib/wasix/src/runtime/module_cache/filesystem.rs +++ b/lib/wasix/src/runtime/module_cache/filesystem.rs @@ -42,7 +42,7 @@ impl FileSystemCache { impl ModuleCache for FileSystemCache { #[tracing::instrument(level = "debug", skip_all, fields(% key))] async fn load(&self, key: ModuleHash, engine: &Engine) -> Result { - let path = self.path(key, engine.deterministic_id()); + let path = self.path(key, &engine.deterministic_id()); self.task_manager .runtime_handle() @@ -97,7 +97,7 @@ impl ModuleCache for FileSystemCache { engine: &Engine, module: &Module, ) -> Result<(), CacheError> { - let path = self.path(key, engine.deterministic_id()); + let path = self.path(key, &engine.deterministic_id()); self.task_manager .runtime_handle() @@ -221,7 +221,7 @@ mod tests { let module = Module::new(&engine, ADD_WAT).unwrap(); let cache = FileSystemCache::new(temp.path(), create_tokio_task_manager()); let key = ModuleHash::xxhash_from_bytes([0; 8]); - let expected_path = cache.path(key, engine.deterministic_id()); + let expected_path = cache.path(key, &engine.deterministic_id()); cache.save(key, &engine, &module).await.unwrap(); @@ -262,7 +262,7 @@ mod tests { let module = Module::new(&engine, ADD_WAT).unwrap(); let key = ModuleHash::xxhash_from_bytes([0; 8]); let cache = FileSystemCache::new(temp.path(), create_tokio_task_manager()); - let expected_path = cache.path(key, engine.deterministic_id()); + let expected_path = cache.path(key, &engine.deterministic_id()); std::fs::create_dir_all(expected_path.parent().unwrap()).unwrap(); let serialized = module.serialize().unwrap(); std::fs::write(&expected_path, &serialized).unwrap(); @@ -285,7 +285,7 @@ mod tests { let module = Module::new(&engine, ADD_WAT).unwrap(); let key = ModuleHash::xxhash_from_bytes([0; 8]); let cache = FileSystemCache::new(temp.path(), create_tokio_task_manager()); - let expected_path = cache.path(key, engine.deterministic_id()); + let expected_path = cache.path(key, &engine.deterministic_id()); std::fs::create_dir_all(expected_path.parent().unwrap()).unwrap(); let serialized = module.serialize().unwrap(); let mut encoder = weezl::encode::Encoder::new(weezl::BitOrder::Msb, 8); diff --git a/lib/wasix/src/runtime/module_cache/shared.rs b/lib/wasix/src/runtime/module_cache/shared.rs index 305e80ff654..101c401d778 100644 --- a/lib/wasix/src/runtime/module_cache/shared.rs +++ b/lib/wasix/src/runtime/module_cache/shared.rs @@ -20,7 +20,7 @@ impl SharedCache { impl ModuleCache for SharedCache { #[tracing::instrument(level = "debug", skip_all, fields(%key))] async fn load(&self, key: ModuleHash, engine: &Engine) -> Result { - let key = (key, engine.deterministic_id().to_string()); + let key = (key, engine.deterministic_id()); match self.modules.get(&key) { Some(m) => { diff --git a/lib/wasix/src/runtime/module_cache/thread_local.rs b/lib/wasix/src/runtime/module_cache/thread_local.rs index c23744ae19c..c873967af24 100644 --- a/lib/wasix/src/runtime/module_cache/thread_local.rs +++ b/lib/wasix/src/runtime/module_cache/thread_local.rs @@ -31,7 +31,7 @@ impl ThreadLocalCache { impl ModuleCache for ThreadLocalCache { #[tracing::instrument(level = "debug", skip_all, fields(%key))] async fn load(&self, key: ModuleHash, engine: &Engine) -> Result { - match self.lookup(key, engine.deterministic_id()) { + match self.lookup(key, &engine.deterministic_id()) { Some(m) => { tracing::debug!("Cache hit!"); Ok(m) @@ -47,7 +47,7 @@ impl ModuleCache for ThreadLocalCache { engine: &Engine, module: &Module, ) -> Result<(), CacheError> { - self.insert(key, module, engine.deterministic_id()); + self.insert(key, module, &engine.deterministic_id()); Ok(()) } } diff --git a/lib/wasix/src/runtime/package_loader/load_package_tree.rs b/lib/wasix/src/runtime/package_loader/load_package_tree.rs index e81c0197be8..743fb8ac392 100644 --- a/lib/wasix/src/runtime/package_loader/load_package_tree.rs +++ b/lib/wasix/src/runtime/package_loader/load_package_tree.rs @@ -10,7 +10,7 @@ use futures::{future::BoxFuture, StreamExt, TryStreamExt}; use once_cell::sync::OnceCell; use petgraph::visit::EdgeRef; use virtual_fs::{FileSystem, OverlayFileSystem, UnionFileSystem, WebcVolumeFileSystem}; -use wasmer_config::package::PackageId; +use wasmer_config::package::{PackageId, SuggestedCompilerOptimizations}; use wasmer_package::utils::wasm_annotations_to_features; use webc::metadata::annotations::Atom as AtomAnnotation; use webc::{Container, Volume}; @@ -208,12 +208,47 @@ fn load_binary_command( None }; - let cmd = - BinaryPackageCommand::new(command_name.to_string(), cmd.clone(), atom, hash, features); + let suggested_compiler_optimizations = + if let Some(atom_metadata) = webc.manifest().atoms.get(&atom_name) { + extract_suggested_compiler_opts_from_atom_metadata(atom_metadata) + } else { + wasmer_config::package::SuggestedCompilerOptimizations::default() + }; + + let cmd = BinaryPackageCommand::new( + command_name.to_string(), + cmd.clone(), + atom, + hash, + features, + suggested_compiler_optimizations, + ); Ok(Some(cmd)) } +fn extract_suggested_compiler_opts_from_atom_metadata( + atom_metadata: &webc::metadata::Atom, +) -> wasmer_config::package::SuggestedCompilerOptimizations { + let mut ret = SuggestedCompilerOptimizations::default(); + + if let Some(sco) = atom_metadata + .annotations + .get(SuggestedCompilerOptimizations::KEY) + { + if let Some((_, v)) = sco.as_map().and_then(|v| { + v.iter().find(|(k, _)| { + k.as_text() + .is_some_and(|v| v == SuggestedCompilerOptimizations::PASS_PARAMS_KEY) + }) + }) { + ret.pass_params = v.as_bool() + } + } + + ret +} + fn atom_name_for_command( command_name: &str, cmd: &webc::metadata::Command, @@ -281,12 +316,21 @@ fn legacy_atom_hack( None }; + // Get WebAssembly features from manifest atom annotations + let suggested_opts_from_manifest = if let Some(atom_metadata) = webc.manifest().atoms.get(&name) + { + extract_suggested_compiler_opts_from_atom_metadata(atom_metadata) + } else { + SuggestedCompilerOptimizations::default() + }; + Ok(Some(BinaryPackageCommand::new( command_name.to_string(), metadata.clone(), atom, hash, features, + suggested_opts_from_manifest, ))) }