From d25f47d54d867fcdfefe4a8d70ca21d09cf6c61e Mon Sep 17 00:00:00 2001 From: Syrus Date: Sat, 2 May 2020 18:41:05 -0700 Subject: [PATCH 01/14] Got half-baked engine --- Cargo.lock | 30 ++++++ Cargo.toml | 1 + lib/api/Cargo.toml | 1 + lib/api/src/import_object.rs | 2 +- lib/api/src/instance.rs | 2 +- lib/api/src/lib.rs | 2 +- lib/api/src/module.rs | 9 +- lib/api/src/tunables.rs | 3 +- lib/engine/Cargo.toml | 34 +++++++ lib/engine/README.md | 12 +++ lib/engine/src/engine.rs | 49 +++++++++ lib/{jit => engine}/src/error.rs | 1 - lib/engine/src/lib.rs | 45 +++++++++ lib/engine/src/module.rs | 19 ++++ lib/{jit => engine}/src/resolver.rs | 0 lib/engine/src/serialize.rs | 107 ++++++++++++++++++++ lib/{jit => engine}/src/trap/error.rs | 0 lib/{jit => engine}/src/trap/frame_info.rs | 0 lib/engine/src/trap/mod.rs | 4 + lib/{jit => engine}/src/tunables.rs | 0 lib/jit/Cargo.toml | 1 + lib/jit/src/engine.rs | 38 +++---- lib/jit/src/lib.rs | 12 --- lib/jit/src/link.rs | 11 +-- lib/jit/src/module.rs | 53 +++++----- lib/jit/src/serialize.rs | 109 +-------------------- lib/jit/src/trap/mod.rs | 4 - 27 files changed, 361 insertions(+), 188 deletions(-) create mode 100644 lib/engine/Cargo.toml create mode 100644 lib/engine/README.md create mode 100644 lib/engine/src/engine.rs rename lib/{jit => engine}/src/error.rs (98%) create mode 100644 lib/engine/src/lib.rs create mode 100644 lib/engine/src/module.rs rename lib/{jit => engine}/src/resolver.rs (100%) create mode 100644 lib/engine/src/serialize.rs rename lib/{jit => engine}/src/trap/error.rs (100%) rename lib/{jit => engine}/src/trap/frame_info.rs (100%) create mode 100644 lib/engine/src/trap/mod.rs rename lib/{jit => engine}/src/tunables.rs (100%) delete mode 100644 lib/jit/src/trap/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 6c96d75a4ee..38d7a7fea12 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -353,6 +353,12 @@ dependencies = [ "generic-array", ] +[[package]] +name = "downcast-rs" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ba6eb47c2131e784a38b726eb54c1e1484904f013e576a25354d0124161af6" + [[package]] name = "either" version = "1.5.3" @@ -1465,6 +1471,7 @@ dependencies = [ "wasmer-compiler-cranelift", "wasmer-compiler-llvm", "wasmer-compiler-singlepass", + "wasmer-engine", "wasmer-jit", "wasmer-runtime", "wat", @@ -1488,6 +1495,7 @@ dependencies = [ "wasmer-compiler-cranelift", "wasmer-compiler-llvm", "wasmer-compiler-singlepass", + "wasmer-engine", "wasmer-jit", "wasmer-wasi", "wasmer-wasi-experimental-io-devices", @@ -1574,6 +1582,27 @@ dependencies = [ "wasmer-runtime", ] +[[package]] +name = "wasmer-engine" +version = "0.16.2" +dependencies = [ + "backtrace", + "bincode", + "downcast-rs", + "lazy_static", + "more-asserts", + "region", + "rustc-demangle", + "serde", + "serde_bytes", + "target-lexicon", + "thiserror", + "wasm-common", + "wasmer-compiler", + "wasmer-runtime", + "winapi", +] + [[package]] name = "wasmer-jit" version = "0.16.2" @@ -1590,6 +1619,7 @@ dependencies = [ "thiserror", "wasm-common", "wasmer-compiler", + "wasmer-engine", "wasmer-runtime", "winapi", ] diff --git a/Cargo.toml b/Cargo.toml index 3646e52215f..34d475d0be1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ wasmer-compiler = { path = "lib/compiler" } wasmer-compiler-cranelift = { path = "lib/compiler-cranelift", optional = true } wasmer-compiler-singlepass = { path = "lib/compiler-singlepass", optional = true } wasmer-compiler-llvm = { path = "lib/compiler-llvm", optional = true } +wasmer-engine = { path = "lib/engine" } wasmer-jit = { path = "lib/jit" } wasmer-wasi = { path = "lib/wasi", optional = true } wasmer-wasi-experimental-io-devices = { path = "lib/wasi-experimental-io-devices", optional = true } diff --git a/lib/api/Cargo.toml b/lib/api/Cargo.toml index cb543508103..74abb180c9d 100644 --- a/lib/api/Cargo.toml +++ b/lib/api/Cargo.toml @@ -14,6 +14,7 @@ wasmer-compiler-singlepass = { path = "../compiler-singlepass", version = "0.16. wasmer-compiler-cranelift = { path = "../compiler-cranelift", version = "0.16.2", optional = true } wasmer-compiler-llvm = { path = "../compiler-llvm", version = "0.16.2", optional = true } wasmer-compiler = { path = "../compiler", version = "0.16.2" } +wasmer-engine = { path = "../engine", version = "0.16.2" } wasmer-jit = { path = "../jit", version = "0.16.2" } wasm-common = { path = "../wasm-common", version = "0.16.2" } indexmap = { version = "1.3.2", features = ["serde-1"] } diff --git a/lib/api/src/import_object.rs b/lib/api/src/import_object.rs index 1c86af7915b..b5000c513b0 100644 --- a/lib/api/src/import_object.rs +++ b/lib/api/src/import_object.rs @@ -8,7 +8,7 @@ use std::{ ffi::c_void, sync::{Arc, Mutex}, }; -use wasmer_jit::Resolver; +use wasmer_engine::Resolver; use wasmer_runtime::Export; /// The `LikeNamespace` trait represents objects that act as a namespace for imports. diff --git a/lib/api/src/instance.rs b/lib/api/src/instance.rs index 64d382229a6..71f14dbef33 100644 --- a/lib/api/src/instance.rs +++ b/lib/api/src/instance.rs @@ -3,7 +3,7 @@ use crate::externals::Extern; use crate::module::Module; use crate::store::Store; use crate::InstantiationError; -use wasmer_jit::Resolver; +use wasmer_engine::Resolver; use wasmer_runtime::InstanceHandle; /// A WebAssembly Instance is a stateful, executable diff --git a/lib/api/src/lib.rs b/lib/api/src/lib.rs index 048e8913c2a..42625101901 100644 --- a/lib/api/src/lib.rs +++ b/lib/api/src/lib.rs @@ -28,7 +28,7 @@ pub use crate::types::{ pub use wasm_common::{ValueType, WasmExternType, WasmTypeList}; pub use wasmer_compiler::{CompilerConfig, Features, Target}; -pub use wasmer_jit::{ +pub use wasmer_engine::{ DeserializeError, InstantiationError, LinkError, RuntimeError, SerializeError, }; diff --git a/lib/api/src/module.rs b/lib/api/src/module.rs index 402e0647221..5bb23d6f6c8 100644 --- a/lib/api/src/module.rs +++ b/lib/api/src/module.rs @@ -6,7 +6,7 @@ use std::path::Path; use std::sync::Arc; use thiserror::Error; use wasmer_compiler::{CompileError, WasmError}; -use wasmer_jit::{CompiledModule, DeserializeError, Resolver, SerializeError}; +use wasmer_engine::{CompiledModule, DeserializeError, Resolver, SerializeError}; use wasmer_runtime::InstanceHandle; #[derive(Error, Debug)] @@ -30,7 +30,7 @@ pub enum IoCompileError { #[derive(Clone)] pub struct Module { store: Store, - compiled: Arc, + compiled: Arc, } impl Module { @@ -173,10 +173,11 @@ impl Module { Ok(Self::from_compiled_module(store, compiled)) } - fn from_compiled_module(store: &Store, compiled: CompiledModule) -> Self { + fn from_compiled_module(store: &Store, compiled: Arc) -> Self + { Module { store: store.clone(), - compiled: Arc::new(compiled), + compiled, } } diff --git a/lib/api/src/tunables.rs b/lib/api/src/tunables.rs index 1e73824ec43..3db5513ddd3 100644 --- a/lib/api/src/tunables.rs +++ b/lib/api/src/tunables.rs @@ -4,6 +4,7 @@ use target_lexicon::{OperatingSystem, PointerWidth, Triple, HOST}; use wasm_common::{MemoryType, Pages, TableType}; use wasmer_runtime::{LinearMemory, Table}; use wasmer_runtime::{MemoryPlan, MemoryStyle, TablePlan, TableStyle}; +use wasmer_engine::Tunables as BaseTunables; /// Tunable parameters for WebAssembly compilation. #[derive(Clone)] @@ -57,7 +58,7 @@ impl Tunables { } } -impl wasmer_jit::Tunables for Tunables { +impl BaseTunables for Tunables { /// Get a `MemoryPlan` for the provided `MemoryType` fn memory_plan(&self, memory: MemoryType) -> MemoryPlan { // A heap with a maximum that doesn't exceed the static memory bound specified by the diff --git a/lib/engine/Cargo.toml b/lib/engine/Cargo.toml new file mode 100644 index 00000000000..43d9bc59240 --- /dev/null +++ b/lib/engine/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "wasmer-engine" +version = "0.16.2" +authors = ["Wasmer Engineering Team "] +description = "Wasmer Engine abstraction" +license = "(Apache-2.0 WITH LLVM-exception) or MIT" +categories = ["wasm"] +keywords = ["webassembly", "wasm"] +repository = "https://github.com/wasmerio/wasmer" +readme = "README.md" +edition = "2018" + +[dependencies] +wasm-common = { path = "../wasm-common", version = "0.16.2" } +wasmer-compiler = { path = "../compiler", version = "0.16.2", default-features = false } +wasmer-runtime = { path = "../runtime", version = "0.16.2" } +target-lexicon = { version = "0.10.0", default-features = false } +# flexbuffers = { path = "../../../flatbuffers/rust/flexbuffers", version = "0.1.0" } +backtrace = "0.3.46" +rustc-demangle = "0.1.16" +more-asserts = "0.2.1" +thiserror = "1.0.16" +region = "2.1.2" +serde = { version = "1.0.106", sfeatures = ["derive", "rc"] } +serde_bytes = { version = "0.11.3" } +bincode = "1.2.1" +lazy_static = "1.4" +downcast-rs = "1.1.1" + +[target.'cfg(target_os = "windows")'.dependencies] +winapi = { version = "0.3.8", features = ["winnt", "impl-default"] } + +[badges] +maintenance = { status = "actively-developed" } diff --git a/lib/engine/README.md b/lib/engine/README.md new file mode 100644 index 00000000000..f45e09291e0 --- /dev/null +++ b/lib/engine/README.md @@ -0,0 +1,12 @@ +# Wasmer Engine + +The Wasmer Engine is the general abstraction for Engines in Wasmer. +It currently has two implementations: +* Wasmer JIT +* Wasmer Native + +### Acknowledgments + +This project borrowed some of the code of the trap implementation from the [wasmtime-api](https://crates.io/crates/wasmtime), the code since then has evolved significantly. + +Please check [Wasmer ATTRIBUTIONS](https://github.com/wasmerio/wasmer/blob/master/ATTRIBUTIONS.md) to further see licenses and other attributions of the project. diff --git a/lib/engine/src/engine.rs b/lib/engine/src/engine.rs new file mode 100644 index 00000000000..299fc687a30 --- /dev/null +++ b/lib/engine/src/engine.rs @@ -0,0 +1,49 @@ +//! JIT compilation. + +use crate::error::InstantiationError; +use crate::resolver::Resolver; +use crate::tunables::Tunables; +use crate::{CompiledModule, DeserializeError, SerializeError}; +use std::sync::Arc; +use wasm_common::FuncType; +use wasmer_compiler::CompileError; +use wasmer_runtime::{InstanceHandle, VMSharedSignatureIndex, VMTrampoline}; + +/// A unimplemented Wasmer `Engine`. +/// This trait is used by implementors to implement custom engines, +/// such as: JIT or Native. +pub trait Engine { + /// The `CompiledModule` type + type Product: CompiledModule; + + /// Get the tunables + fn tunables(&self) -> &Tunables; + + /// Register a signature + fn register_signature(&self, func_type: &FuncType) -> VMSharedSignatureIndex; + + /// Lookup a signature + fn lookup_signature(&self, sig: VMSharedSignatureIndex) -> Option; + + /// Retrieves a trampoline given a signature + fn trampoline(&self, sig: VMSharedSignatureIndex) -> Option; + + /// Validates a WebAssembly module + fn validate(&self, binary: &[u8]) -> Result<(), CompileError>; + + /// Compile a WebAssembly binary + fn compile(&self, binary: &[u8]) -> Result; + + /// Instantiates a WebAssembly module + fn instantiate( + &self, + compiled_module: &Arc, + resolver: &dyn Resolver, + ) -> Result; + + /// Serializes a WebAssembly module + fn serialize(&self, compiled_module: &Arc) -> Result, SerializeError>; + + /// Deserializes a WebAssembly module + fn deserialize(&self, bytes: &[u8]) -> Result; +} diff --git a/lib/jit/src/error.rs b/lib/engine/src/error.rs similarity index 98% rename from lib/jit/src/error.rs rename to lib/engine/src/error.rs index cb6e140137d..79ff95cf691 100644 --- a/lib/jit/src/error.rs +++ b/lib/engine/src/error.rs @@ -1,4 +1,3 @@ -// TODO: Move this errors into a common engine crate. //! The WebAssembly possible errors use crate::trap::RuntimeError; use thiserror::Error; diff --git a/lib/engine/src/lib.rs b/lib/engine/src/lib.rs new file mode 100644 index 00000000000..0f02366f35b --- /dev/null +++ b/lib/engine/src/lib.rs @@ -0,0 +1,45 @@ +//! Generic Engine abstraction for Wasmer Engines. + +#![deny(missing_docs, trivial_numeric_casts, unused_extern_crates)] +#![warn(unused_import_braces)] +#![cfg_attr(feature = "clippy", plugin(clippy(conf_file = "../../clippy.toml")))] +#![cfg_attr( + feature = "cargo-clippy", + allow(clippy::new_without_default, clippy::new_without_default) +)] +#![cfg_attr( + feature = "cargo-clippy", + warn( + clippy::float_arithmetic, + clippy::mut_mut, + clippy::nonminimal_bool, + clippy::option_map_unwrap_or, + clippy::option_map_unwrap_or_else, + clippy::print_stdout, + clippy::unicode_not_nfc, + clippy::use_self + ) +)] + +mod engine; +mod error; +mod module; +mod resolver; +mod serialize; +mod trap; +mod tunables; + +pub use crate::engine::Engine; +pub use crate::error::{ + DeserializeError, ImportError, InstantiationError, LinkError, SerializeError, +}; +pub use crate::module::CompiledModule; +pub use crate::resolver::{resolve_imports, NullResolver, Resolver}; +pub use crate::trap::*; +pub use crate::tunables::Tunables; +pub use crate::serialize::SerializableFunctionFrameInfo; + +pub use wasmer_compiler::CompilerConfig; + +/// Version number of this crate. +pub const VERSION: &str = env!("CARGO_PKG_VERSION"); diff --git a/lib/engine/src/module.rs b/lib/engine/src/module.rs new file mode 100644 index 00000000000..42c0bfd290a --- /dev/null +++ b/lib/engine/src/module.rs @@ -0,0 +1,19 @@ +use std::sync::Arc; +use wasmer_runtime::Module; + +use downcast_rs::{DowncastSync, impl_downcast}; + +/// The `CompiledModule` trait is used by engine implementors, such +/// as a JIT or Native execution. +pub trait CompiledModule: DowncastSync { + /// Return a reference-counting pointer to a module. + fn module(&self) -> &Arc; + + /// Return a reference-counting pointer to a module. + fn module_mut(&mut self) -> &mut Arc; + + /// Return a reference to a module. + fn module_ref(&self) -> &Module; +} + +impl_downcast!(sync CompiledModule); // `sync` => also produce `Arc` downcasts. diff --git a/lib/jit/src/resolver.rs b/lib/engine/src/resolver.rs similarity index 100% rename from lib/jit/src/resolver.rs rename to lib/engine/src/resolver.rs diff --git a/lib/engine/src/serialize.rs b/lib/engine/src/serialize.rs new file mode 100644 index 00000000000..42964a24ee2 --- /dev/null +++ b/lib/engine/src/serialize.rs @@ -0,0 +1,107 @@ +use serde::de::{Deserializer, Visitor}; +use serde::ser::Serializer; +use serde::{Deserialize, Serialize}; +use std::fmt; +use wasmer_compiler::CompiledFunctionFrameInfo; + +/// This is the unserialized verison of `CompiledFunctionFrameInfo`. +#[derive(Clone, Serialize, Deserialize)] +#[serde(transparent)] +#[repr(transparent)] +pub struct UnprocessedFunctionFrameInfo { + #[serde(with = "serde_bytes")] + bytes: Vec, +} + +impl UnprocessedFunctionFrameInfo { + /// Converts the `UnprocessedFunctionFrameInfo` to a `CompiledFunctionFrameInfo` + pub fn deserialize(&self) -> CompiledFunctionFrameInfo { + // let r = flexbuffers::Reader::get_root(&self.bytes).expect("Can't deserialize the info"); + // CompiledFunctionFrameInfo::deserialize(r).expect("Can't deserialize the info") + bincode::deserialize(&self.bytes).expect("Can't deserialize the info") + } + + /// Converts the `CompiledFunctionFrameInfo` to a `UnprocessedFunctionFrameInfo` + pub fn serialize(processed: &CompiledFunctionFrameInfo) -> Self { + // let mut s = flexbuffers::FlexbufferSerializer::new(); + // processed + // .serialize(&mut s) + // .expect("Can't serialize the info"); + // let bytes = s.take_buffer(); + let bytes = bincode::serialize(&processed).expect("Can't serialize the info"); + Self { bytes } + } +} + +/// We hold the frame info in two states, mainly because we want to +/// process it lazily to speed up execution. +/// +/// When a Trap occurs, we process the frame info lazily for each +/// function in the frame. That way we minimize as much as we can +/// the upfront effort. +/// +/// The data can also be processed upfront. This will happen in the case +/// of compiling at the same time that emiting the JIT. +/// In that case, we don't need to deserialize/process anything +/// as the data is already in memory. +#[derive(Clone)] +pub enum SerializableFunctionFrameInfo { + /// The unprocessed frame info (binary) + Unprocessed(UnprocessedFunctionFrameInfo), + /// The processed frame info (memory struct) + Processed(CompiledFunctionFrameInfo), +} + +impl SerializableFunctionFrameInfo { + /// Returns true if the extra function info is not yet + /// processed + pub fn is_unprocessed(&self) -> bool { + match self { + Self::Unprocessed(_) => true, + _ => false, + } + } +} + +// Below: +// The custom ser/de for `SerializableFunctionFrameInfo`. + +impl Serialize for SerializableFunctionFrameInfo { + fn serialize(&self, s: S) -> Result + where + S: Serializer, + { + let unprocessed = match self { + Self::Processed(processed) => UnprocessedFunctionFrameInfo::serialize(processed), + Self::Unprocessed(unprocessed) => unprocessed.clone(), + }; + s.serialize_bytes(&unprocessed.bytes) + } +} + +struct FunctionFrameInfoVisitor; + +impl<'de> Visitor<'de> for FunctionFrameInfoVisitor { + type Value = UnprocessedFunctionFrameInfo; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("bytes") + } + fn visit_byte_buf(self, v: Vec) -> Result { + Ok(UnprocessedFunctionFrameInfo { bytes: v }) + } + fn visit_bytes(self, v: &[u8]) -> Result { + Ok(UnprocessedFunctionFrameInfo { bytes: v.to_vec() }) + } +} + +impl<'de> Deserialize<'de> for SerializableFunctionFrameInfo { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + Ok(SerializableFunctionFrameInfo::Unprocessed( + deserializer.deserialize_byte_buf(FunctionFrameInfoVisitor)?, + )) + } +} diff --git a/lib/jit/src/trap/error.rs b/lib/engine/src/trap/error.rs similarity index 100% rename from lib/jit/src/trap/error.rs rename to lib/engine/src/trap/error.rs diff --git a/lib/jit/src/trap/frame_info.rs b/lib/engine/src/trap/frame_info.rs similarity index 100% rename from lib/jit/src/trap/frame_info.rs rename to lib/engine/src/trap/frame_info.rs diff --git a/lib/engine/src/trap/mod.rs b/lib/engine/src/trap/mod.rs new file mode 100644 index 00000000000..0aa0e015338 --- /dev/null +++ b/lib/engine/src/trap/mod.rs @@ -0,0 +1,4 @@ +mod error; +mod frame_info; +pub use error::RuntimeError; +pub use frame_info::{register as register_frame_info, FrameInfo, GlobalFrameInfoRegistration, FRAME_INFO}; diff --git a/lib/jit/src/tunables.rs b/lib/engine/src/tunables.rs similarity index 100% rename from lib/jit/src/tunables.rs rename to lib/engine/src/tunables.rs diff --git a/lib/jit/Cargo.toml b/lib/jit/Cargo.toml index 660bf6c6346..7123401d124 100644 --- a/lib/jit/Cargo.toml +++ b/lib/jit/Cargo.toml @@ -14,6 +14,7 @@ edition = "2018" wasm-common = { path = "../wasm-common", version = "0.16.2" } wasmer-compiler = { path = "../compiler", version = "0.16.2", default-features = false } wasmer-runtime = { path = "../runtime", version = "0.16.2" } +wasmer-engine = { path = "../engine", version = "0.16.2" } target-lexicon = { version = "0.10.0", default-features = false } # flexbuffers = { path = "../../../flatbuffers/rust/flexbuffers", version = "0.1.0" } backtrace = "0.3.46" diff --git a/lib/jit/src/engine.rs b/lib/jit/src/engine.rs index 9d5513b6093..e0901a62916 100644 --- a/lib/jit/src/engine.rs +++ b/lib/jit/src/engine.rs @@ -1,10 +1,7 @@ //! JIT compilation. -use crate::error::InstantiationError; -use crate::resolver::Resolver; -use crate::tunables::Tunables; -use crate::CodeMemory; -use crate::{CompiledModule, DeserializeError, SerializeError}; +use wasmer_engine::{Engine, InstantiationError, Resolver, Tunables, DeserializeError, SerializeError, CompiledModule as BaseCompiledModule}; +use crate::{CodeMemory, CompiledModule}; use std::cell::RefCell; use std::collections::HashMap; use std::sync::Arc; @@ -12,7 +9,7 @@ use wasm_common::entity::PrimaryMap; use wasm_common::{FuncType, LocalFuncIndex, MemoryIndex, TableIndex}; use wasmer_compiler::{ Compilation, CompileError, Compiler as BaseCompiler, CompilerConfig, FunctionBody, - FunctionBodyData, ModuleTranslationState, Target, + FunctionBodyData, ModuleTranslationState, }; use wasmer_runtime::{ InstanceHandle, MemoryPlan, Module, SignatureRegistry, TablePlan, VMFunctionBody, @@ -44,11 +41,6 @@ impl JITEngine { } } - /// Get the tunables - pub fn tunables(&self) -> &dyn Tunables { - &**self.tunables - } - pub(crate) fn compiler(&self) -> std::cell::Ref<'_, JITEngineInner> { self.inner.borrow() } @@ -57,6 +49,13 @@ impl JITEngine { self.inner.borrow_mut() } + + /// Get the tunables + pub fn tunables(&self) -> &dyn Tunables { + &**self.tunables + } +// } +// impl Engine for JITEngine { /// Register a signature pub fn register_signature(&self, func_type: &FuncType) -> VMSharedSignatureIndex { let compiler = self.compiler(); @@ -80,27 +79,30 @@ impl JITEngine { } /// Compile a WebAssembly binary - pub fn compile(&self, binary: &[u8]) -> Result { - CompiledModule::new(&self, binary) + pub fn compile(&self, binary: &[u8]) -> Result, CompileError> { + Ok(Arc::new(CompiledModule::new(&self, binary)?)) } /// Instantiates a WebAssembly module pub fn instantiate( &self, - compiled_module: &CompiledModule, + compiled_module: &Arc, resolver: &dyn Resolver, - ) -> Result { + ) -> Result + { + let compiled_module = compiled_module.downcast_ref::().unwrap(); unsafe { compiled_module.instantiate(&self, resolver, Box::new(())) } } /// Serializes a WebAssembly module - pub fn serialize(&self, compiled_module: &CompiledModule) -> Result, SerializeError> { + pub fn serialize(&self, compiled_module: &Arc) -> Result, SerializeError> { + let compiled_module = compiled_module.downcast_ref::().unwrap(); compiled_module.serialize() } /// Deserializes a WebAssembly module - pub fn deserialize(&self, bytes: &[u8]) -> Result { - CompiledModule::deserialize(&self, bytes) + pub fn deserialize(&self, bytes: &[u8]) -> Result, DeserializeError> { + Ok(Arc::new(CompiledModule::deserialize(&self, bytes)?)) } } diff --git a/lib/jit/src/lib.rs b/lib/jit/src/lib.rs index c1bcc81143f..0d0a026ec07 100644 --- a/lib/jit/src/lib.rs +++ b/lib/jit/src/lib.rs @@ -27,28 +27,16 @@ mod code_memory; mod engine; -mod error; mod function_table; mod link; mod module; -mod resolver; mod serialize; -mod trap; -mod tunables; pub use crate::code_memory::CodeMemory; pub use crate::engine::JITEngine; -pub use crate::error::{ - DeserializeError, ImportError, InstantiationError, LinkError, SerializeError, -}; pub use crate::function_table::FunctionTable; pub use crate::link::link_module; pub use crate::module::CompiledModule; -pub use crate::resolver::{resolve_imports, NullResolver, Resolver}; -pub use crate::trap::*; -pub use crate::tunables::Tunables; - -pub use wasmer_compiler::CompilerConfig; /// Version number of this crate. pub const VERSION: &str = env!("CARGO_PKG_VERSION"); diff --git a/lib/jit/src/link.rs b/lib/jit/src/link.rs index 12430189eae..78f75e08d0c 100644 --- a/lib/jit/src/link.rs +++ b/lib/jit/src/link.rs @@ -7,9 +7,8 @@ use wasmer_compiler::{JumpTable, JumpTableOffsets, RelocationKind, RelocationTar use wasmer_runtime::Module; use wasmer_runtime::VMFunctionBody; -/// Links a module that has been compiled with `compiled_module` in `wasmer-compiler::Compiler`. -/// -/// Performs all required relocations inside the function code, provided the necessary metadata. +/// Links a module, patching the allocated functions with the +/// requiredd relocations and jump tables. pub fn link_module( module: &Module, allocated_functions: &PrimaryMap, @@ -68,10 +67,8 @@ pub fn link_module( .wrapping_add(reloc_addend as u32); write_unaligned(reloc_address as *mut u32, reloc_delta_u32); } - RelocationKind::X86PCRelRodata4 => { - // ignore - } - _ => panic!("unsupported reloc kind"), + RelocationKind::X86PCRelRodata4 => {} + _ => panic!("Relocation kind unsupported"), } } } diff --git a/lib/jit/src/module.rs b/lib/jit/src/module.rs index 655980303cc..2a7b766814d 100644 --- a/lib/jit/src/module.rs +++ b/lib/jit/src/module.rs @@ -4,16 +4,14 @@ //! steps. use crate::engine::{JITEngine, JITEngineInner}; -use crate::error::{DeserializeError, SerializeError}; -use crate::error::{InstantiationError, LinkError}; +use wasmer_engine::{DeserializeError, SerializeError, InstantiationError, LinkError, SerializableFunctionFrameInfo, + resolve_imports, Resolver, register_frame_info, GlobalFrameInfoRegistration, RuntimeError, + CompiledModule as BaseCompiledModule +}; use crate::link::link_module; -use crate::resolver::{resolve_imports, Resolver}; use crate::serialize::{ - SerializableCompilation, SerializableFunctionFrameInfo, SerializableModule, + SerializableCompilation, SerializableModule, }; -use crate::trap::register as register_frame_info; -use crate::trap::GlobalFrameInfoRegistration; -use crate::trap::RuntimeError; use serde::{Deserialize, Serialize}; use std::any::Any; use std::sync::{Arc, Mutex}; @@ -168,15 +166,7 @@ impl CompiledModule { &self.serializable.table_plans } - /// Crate an `Instance` from this `CompiledModule`. - /// - /// Note that if only one instance of this module is needed, it may be more - /// efficient to call the top-level `instantiate`, since that avoids copying - /// the data initializers. - /// - /// # Unsafety - /// - /// See `InstanceHandle::new` + /// Instantiate the module pub unsafe fn instantiate( &self, jit: &JITEngine, @@ -227,21 +217,6 @@ impl CompiledModule { .map_err(|trap| InstantiationError::Start(RuntimeError::from_trap(trap))) } - /// Return a reference-counting pointer to a module. - pub fn module(&self) -> &Arc { - &self.serializable.module - } - - /// Return a reference-counting pointer to a module. - pub fn module_mut(&mut self) -> &mut Arc { - &mut self.serializable.module - } - - /// Return a reference to a module. - pub fn module_ref(&self) -> &Module { - &self.serializable.module - } - /// Register this module's stack frame information into the global scope. /// /// This is required to ensure that any traps can be properly symbolicated. @@ -259,6 +234,22 @@ impl CompiledModule { )); } } +unsafe impl Sync for CompiledModule {} +unsafe impl Send for CompiledModule {} + +impl BaseCompiledModule for CompiledModule { + fn module(&self) -> &Arc { + &self.serializable.module + } + + fn module_mut(&mut self) -> &mut Arc { + &mut self.serializable.module + } + + fn module_ref(&self) -> &Module { + &self.serializable.module + } +} /// Allocate memory for just the memories of the current module. fn create_memories( diff --git a/lib/jit/src/serialize.rs b/lib/jit/src/serialize.rs index 835775c47f9..7f5eb3db3a2 100644 --- a/lib/jit/src/serialize.rs +++ b/lib/jit/src/serialize.rs @@ -1,11 +1,8 @@ -use serde::de::{Deserializer, Visitor}; -use serde::ser::Serializer; use serde::{Deserialize, Serialize}; -use std::fmt; use std::sync::Arc; -use wasmer_compiler::{CompiledFunctionFrameInfo, FunctionBody, JumpTableOffsets, Relocation}; +use wasmer_compiler::{FunctionBody, JumpTableOffsets, Relocation}; use wasmer_runtime::Module; - +use wasmer_engine::SerializableFunctionFrameInfo; use wasm_common::entity::PrimaryMap; use wasm_common::{LocalFuncIndex, MemoryIndex, OwnedDataInitializer, TableIndex}; use wasmer_runtime::{MemoryPlan, TablePlan}; @@ -33,105 +30,3 @@ pub struct SerializableModule { pub memory_plans: PrimaryMap, pub table_plans: PrimaryMap, } - -/// This is the unserialized verison of `CompiledFunctionFrameInfo`. -#[derive(Clone, Serialize, Deserialize)] -#[serde(transparent)] -#[repr(transparent)] -pub struct UnprocessedFunctionFrameInfo { - #[serde(with = "serde_bytes")] - bytes: Vec, -} - -impl UnprocessedFunctionFrameInfo { - /// Converts the `UnprocessedFunctionFrameInfo` to a `CompiledFunctionFrameInfo` - pub fn deserialize(&self) -> CompiledFunctionFrameInfo { - // let r = flexbuffers::Reader::get_root(&self.bytes).expect("Can't deserialize the info"); - // CompiledFunctionFrameInfo::deserialize(r).expect("Can't deserialize the info") - bincode::deserialize(&self.bytes).expect("Can't deserialize the info") - } - - /// Converts the `CompiledFunctionFrameInfo` to a `UnprocessedFunctionFrameInfo` - pub fn serialize(processed: &CompiledFunctionFrameInfo) -> Self { - // let mut s = flexbuffers::FlexbufferSerializer::new(); - // processed - // .serialize(&mut s) - // .expect("Can't serialize the info"); - // let bytes = s.take_buffer(); - let bytes = bincode::serialize(&processed).expect("Can't serialize the info"); - Self { bytes } - } -} - -/// We hold the frame info in two states, mainly because we want to -/// process it lazily to speed up execution. -/// -/// When a Trap occurs, we process the frame info lazily for each -/// function in the frame. That way we minimize as much as we can -/// the upfront effort. -/// -/// The data can also be processed upfront. This will happen in the case -/// of compiling at the same time that emiting the JIT. -/// In that case, we don't need to deserialize/process anything -/// as the data is already in memory. -#[derive(Clone)] -pub enum SerializableFunctionFrameInfo { - /// The unprocessed frame info (binary) - Unprocessed(UnprocessedFunctionFrameInfo), - /// The processed frame info (memory struct) - Processed(CompiledFunctionFrameInfo), -} - -impl SerializableFunctionFrameInfo { - /// Returns true if the extra function info is not yet - /// processed - pub fn is_unprocessed(&self) -> bool { - match self { - Self::Unprocessed(_) => true, - _ => false, - } - } -} - -// Below: -// The custom ser/de for `SerializableFunctionFrameInfo`. - -impl Serialize for SerializableFunctionFrameInfo { - fn serialize(&self, s: S) -> Result - where - S: Serializer, - { - let unprocessed = match self { - Self::Processed(processed) => UnprocessedFunctionFrameInfo::serialize(processed), - Self::Unprocessed(unprocessed) => unprocessed.clone(), - }; - s.serialize_bytes(&unprocessed.bytes) - } -} - -struct FunctionFrameInfoVisitor; - -impl<'de> Visitor<'de> for FunctionFrameInfoVisitor { - type Value = UnprocessedFunctionFrameInfo; - - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str("bytes") - } - fn visit_byte_buf(self, v: Vec) -> Result { - Ok(UnprocessedFunctionFrameInfo { bytes: v }) - } - fn visit_bytes(self, v: &[u8]) -> Result { - Ok(UnprocessedFunctionFrameInfo { bytes: v.to_vec() }) - } -} - -impl<'de> Deserialize<'de> for SerializableFunctionFrameInfo { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - Ok(SerializableFunctionFrameInfo::Unprocessed( - deserializer.deserialize_byte_buf(FunctionFrameInfoVisitor)?, - )) - } -} diff --git a/lib/jit/src/trap/mod.rs b/lib/jit/src/trap/mod.rs deleted file mode 100644 index 1a37b64afe2..00000000000 --- a/lib/jit/src/trap/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -mod error; -mod frame_info; -pub use error::RuntimeError; -pub use frame_info::{register, FrameInfo, GlobalFrameInfoRegistration, FRAME_INFO}; From 4d9a5291aa6071b251e2e7b938b3f1301846c787 Mon Sep 17 00:00:00 2001 From: Syrus Date: Sat, 2 May 2020 19:05:48 -0700 Subject: [PATCH 02/14] =?UTF-8?q?Engine=20is=20working=20all=20the=20way?= =?UTF-8?q?=20=F0=9F=8E=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/api/src/externals.rs | 1 + lib/api/src/lib.rs | 5 +++-- lib/api/src/module.rs | 5 ++--- lib/api/src/resolver.rs | 4 ++-- lib/api/src/store.rs | 15 ++++++--------- lib/api/src/tunables.rs | 2 +- lib/engine/src/engine.rs | 11 ++++------- lib/engine/src/lib.rs | 2 +- lib/engine/src/module.rs | 4 ++-- lib/engine/src/trap/error.rs | 2 +- lib/engine/src/trap/mod.rs | 4 +++- lib/jit/src/engine.rs | 35 ++++++++++++++++++++--------------- lib/jit/src/module.rs | 13 ++++++------- lib/jit/src/serialize.rs | 6 +++--- src/compiler.rs | 5 +++-- tests/wast.rs | 5 +++-- 16 files changed, 61 insertions(+), 58 deletions(-) diff --git a/lib/api/src/externals.rs b/lib/api/src/externals.rs index 18629f329f5..26a4c8654b6 100644 --- a/lib/api/src/externals.rs +++ b/lib/api/src/externals.rs @@ -8,6 +8,7 @@ use crate::{ExternType, FuncType, GlobalType, MemoryType, TableType, ValType}; use std::cmp::max; use std::slice; use wasm_common::{Bytes, HostFunction, Pages, ValueType, WasmTypeList, WithEnv, WithoutEnv}; +use wasmer_engine::Engine; use wasmer_runtime::{ wasmer_call_trampoline, Export, ExportFunction, ExportGlobal, ExportMemory, ExportTable, LinearMemory, Table as RuntimeTable, VMCallerCheckedAnyfunc, VMContext, VMFunctionBody, diff --git a/lib/api/src/lib.rs b/lib/api/src/lib.rs index 42625101901..b87d7c02a21 100644 --- a/lib/api/src/lib.rs +++ b/lib/api/src/lib.rs @@ -19,7 +19,7 @@ pub use crate::instance::Instance; pub use crate::memory_view::MemoryView; pub use crate::module::Module; pub use crate::ptr::{Array, Item, WasmPtr}; -pub use crate::store::{Engine, Store, StoreObject}; +pub use crate::store::{Store, StoreObject}; pub use crate::tunables::Tunables; pub use crate::types::{ AnyRef, ExportType, ExternType, FuncType, GlobalType, HostInfo, HostRef, ImportType, @@ -29,8 +29,9 @@ pub use crate::types::{ pub use wasm_common::{ValueType, WasmExternType, WasmTypeList}; pub use wasmer_compiler::{CompilerConfig, Features, Target}; pub use wasmer_engine::{ - DeserializeError, InstantiationError, LinkError, RuntimeError, SerializeError, + DeserializeError, Engine, InstantiationError, LinkError, RuntimeError, SerializeError, }; +pub use wasmer_jit::JITEngine; #[cfg(feature = "compiler-singlepass")] pub use wasmer_compiler_singlepass::SinglepassConfig; diff --git a/lib/api/src/module.rs b/lib/api/src/module.rs index 5bb23d6f6c8..97a17524060 100644 --- a/lib/api/src/module.rs +++ b/lib/api/src/module.rs @@ -6,7 +6,7 @@ use std::path::Path; use std::sync::Arc; use thiserror::Error; use wasmer_compiler::{CompileError, WasmError}; -use wasmer_engine::{CompiledModule, DeserializeError, Resolver, SerializeError}; +use wasmer_engine::{CompiledModule, DeserializeError, Engine, Resolver, SerializeError}; use wasmer_runtime::InstanceHandle; #[derive(Error, Debug)] @@ -173,8 +173,7 @@ impl Module { Ok(Self::from_compiled_module(store, compiled)) } - fn from_compiled_module(store: &Store, compiled: Arc) -> Self - { + fn from_compiled_module(store: &Store, compiled: Arc) -> Self { Module { store: store.clone(), compiled, diff --git a/lib/api/src/resolver.rs b/lib/api/src/resolver.rs index 1d3d30baf2b..182bdcd8afc 100644 --- a/lib/api/src/resolver.rs +++ b/lib/api/src/resolver.rs @@ -1,7 +1,7 @@ -use wasmer_jit::Resolver; +use wasmer_engine::Resolver; use std::iter::FromIterator; -pub struct IndexResolver { +pub struct sIndexResolver { externs: Vec, } impl Resolver for IndexResolver { diff --git a/lib/api/src/store.rs b/lib/api/src/store.rs index 949dadbd51b..ae752be7d8a 100644 --- a/lib/api/src/store.rs +++ b/lib/api/src/store.rs @@ -1,23 +1,20 @@ use crate::tunables::Tunables; use std::sync::Arc; use wasmer_compiler::CompilerConfig; +use wasmer_engine::Engine; use wasmer_jit::JITEngine; -pub type Engine = JITEngine; - #[derive(Clone)] pub struct Store { - engine: Arc, + engine: Arc, } impl Store { - pub fn new(engine: &Engine) -> Store { - Store { - engine: Arc::new(engine.clone()), - } + pub fn new(engine: Arc) -> Store { + Store { engine } } - pub fn engine(&self) -> &Engine { + pub fn engine(&self) -> &Arc { &self.engine } @@ -75,7 +72,7 @@ impl Default for Store { fn default() -> Store { let config = Self::default_compiler_config(); let tunables = Tunables::for_target(config.target().triple()); - Store::new(&Engine::new(&config, tunables)) + Store::new(Arc::new(JITEngine::new(&config, tunables))) } } diff --git a/lib/api/src/tunables.rs b/lib/api/src/tunables.rs index 3db5513ddd3..1824d3d914e 100644 --- a/lib/api/src/tunables.rs +++ b/lib/api/src/tunables.rs @@ -2,9 +2,9 @@ use more_asserts::assert_ge; use std::cmp::min; use target_lexicon::{OperatingSystem, PointerWidth, Triple, HOST}; use wasm_common::{MemoryType, Pages, TableType}; +use wasmer_engine::Tunables as BaseTunables; use wasmer_runtime::{LinearMemory, Table}; use wasmer_runtime::{MemoryPlan, MemoryStyle, TablePlan, TableStyle}; -use wasmer_engine::Tunables as BaseTunables; /// Tunable parameters for WebAssembly compilation. #[derive(Clone)] diff --git a/lib/engine/src/engine.rs b/lib/engine/src/engine.rs index 299fc687a30..8bbccef4c26 100644 --- a/lib/engine/src/engine.rs +++ b/lib/engine/src/engine.rs @@ -13,9 +13,6 @@ use wasmer_runtime::{InstanceHandle, VMSharedSignatureIndex, VMTrampoline}; /// This trait is used by implementors to implement custom engines, /// such as: JIT or Native. pub trait Engine { - /// The `CompiledModule` type - type Product: CompiledModule; - /// Get the tunables fn tunables(&self) -> &Tunables; @@ -32,18 +29,18 @@ pub trait Engine { fn validate(&self, binary: &[u8]) -> Result<(), CompileError>; /// Compile a WebAssembly binary - fn compile(&self, binary: &[u8]) -> Result; + fn compile(&self, binary: &[u8]) -> Result, CompileError>; /// Instantiates a WebAssembly module fn instantiate( &self, - compiled_module: &Arc, + compiled_module: &Arc, resolver: &dyn Resolver, ) -> Result; /// Serializes a WebAssembly module - fn serialize(&self, compiled_module: &Arc) -> Result, SerializeError>; + fn serialize(&self, compiled_module: &Arc) -> Result, SerializeError>; /// Deserializes a WebAssembly module - fn deserialize(&self, bytes: &[u8]) -> Result; + fn deserialize(&self, bytes: &[u8]) -> Result, DeserializeError>; } diff --git a/lib/engine/src/lib.rs b/lib/engine/src/lib.rs index 0f02366f35b..436d59afffc 100644 --- a/lib/engine/src/lib.rs +++ b/lib/engine/src/lib.rs @@ -35,9 +35,9 @@ pub use crate::error::{ }; pub use crate::module::CompiledModule; pub use crate::resolver::{resolve_imports, NullResolver, Resolver}; +pub use crate::serialize::SerializableFunctionFrameInfo; pub use crate::trap::*; pub use crate::tunables::Tunables; -pub use crate::serialize::SerializableFunctionFrameInfo; pub use wasmer_compiler::CompilerConfig; diff --git a/lib/engine/src/module.rs b/lib/engine/src/module.rs index 42c0bfd290a..82b742c74f3 100644 --- a/lib/engine/src/module.rs +++ b/lib/engine/src/module.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use wasmer_runtime::Module; -use downcast_rs::{DowncastSync, impl_downcast}; +use downcast_rs::{impl_downcast, DowncastSync}; /// The `CompiledModule` trait is used by engine implementors, such /// as a JIT or Native execution. @@ -16,4 +16,4 @@ pub trait CompiledModule: DowncastSync { fn module_ref(&self) -> &Module; } -impl_downcast!(sync CompiledModule); // `sync` => also produce `Arc` downcasts. +impl_downcast!(sync CompiledModule); // `sync` => also produce `Arc` downcasts. diff --git a/lib/engine/src/trap/error.rs b/lib/engine/src/trap/error.rs index 0f9637a6d3e..7f066b6339e 100644 --- a/lib/engine/src/trap/error.rs +++ b/lib/engine/src/trap/error.rs @@ -27,7 +27,7 @@ impl RuntimeError { /// Creates a new `Trap` with `message`. /// # Example /// ``` - /// let trap = wasmer_jit::RuntimeError::new("unexpected error"); + /// let trap = wasmer_engine::RuntimeError::new("unexpected error"); /// assert_eq!("unexpected error", trap.message()); /// ``` pub fn new>(message: I) -> Self { diff --git a/lib/engine/src/trap/mod.rs b/lib/engine/src/trap/mod.rs index 0aa0e015338..d32440a8866 100644 --- a/lib/engine/src/trap/mod.rs +++ b/lib/engine/src/trap/mod.rs @@ -1,4 +1,6 @@ mod error; mod frame_info; pub use error::RuntimeError; -pub use frame_info::{register as register_frame_info, FrameInfo, GlobalFrameInfoRegistration, FRAME_INFO}; +pub use frame_info::{ + register as register_frame_info, FrameInfo, GlobalFrameInfoRegistration, FRAME_INFO, +}; diff --git a/lib/jit/src/engine.rs b/lib/jit/src/engine.rs index e0901a62916..3107acbd9d0 100644 --- a/lib/jit/src/engine.rs +++ b/lib/jit/src/engine.rs @@ -1,6 +1,5 @@ //! JIT compilation. -use wasmer_engine::{Engine, InstantiationError, Resolver, Tunables, DeserializeError, SerializeError, CompiledModule as BaseCompiledModule}; use crate::{CodeMemory, CompiledModule}; use std::cell::RefCell; use std::collections::HashMap; @@ -11,6 +10,10 @@ use wasmer_compiler::{ Compilation, CompileError, Compiler as BaseCompiler, CompilerConfig, FunctionBody, FunctionBodyData, ModuleTranslationState, }; +use wasmer_engine::{ + CompiledModule as BaseCompiledModule, DeserializeError, Engine, InstantiationError, Resolver, + SerializeError, Tunables, +}; use wasmer_runtime::{ InstanceHandle, MemoryPlan, Module, SignatureRegistry, TablePlan, VMFunctionBody, VMSharedSignatureIndex, VMTrampoline, @@ -48,60 +51,62 @@ impl JITEngine { pub(crate) fn compiler_mut(&self) -> std::cell::RefMut<'_, JITEngineInner> { self.inner.borrow_mut() } +} - +impl Engine for JITEngine { /// Get the tunables - pub fn tunables(&self) -> &dyn Tunables { + fn tunables(&self) -> &dyn Tunables { &**self.tunables } -// } -// impl Engine for JITEngine { + /// Register a signature - pub fn register_signature(&self, func_type: &FuncType) -> VMSharedSignatureIndex { + fn register_signature(&self, func_type: &FuncType) -> VMSharedSignatureIndex { let compiler = self.compiler(); compiler.signatures().register(func_type) } /// Lookup a signature - pub fn lookup_signature(&self, sig: VMSharedSignatureIndex) -> Option { + fn lookup_signature(&self, sig: VMSharedSignatureIndex) -> Option { let compiler = self.compiler(); compiler.signatures().lookup(sig) } /// Retrieves a trampoline given a signature - pub fn trampoline(&self, sig: VMSharedSignatureIndex) -> Option { + fn trampoline(&self, sig: VMSharedSignatureIndex) -> Option { self.compiler().trampoline(sig) } /// Validates a WebAssembly module - pub fn validate(&self, binary: &[u8]) -> Result<(), CompileError> { + fn validate(&self, binary: &[u8]) -> Result<(), CompileError> { self.compiler().validate(binary) } /// Compile a WebAssembly binary - pub fn compile(&self, binary: &[u8]) -> Result, CompileError> { + fn compile(&self, binary: &[u8]) -> Result, CompileError> { Ok(Arc::new(CompiledModule::new(&self, binary)?)) } /// Instantiates a WebAssembly module - pub fn instantiate( + fn instantiate( &self, compiled_module: &Arc, resolver: &dyn Resolver, - ) -> Result - { + ) -> Result { let compiled_module = compiled_module.downcast_ref::().unwrap(); unsafe { compiled_module.instantiate(&self, resolver, Box::new(())) } } /// Serializes a WebAssembly module - pub fn serialize(&self, compiled_module: &Arc) -> Result, SerializeError> { + fn serialize( + &self, + compiled_module: &Arc, + ) -> Result, SerializeError> { let compiled_module = compiled_module.downcast_ref::().unwrap(); compiled_module.serialize() } /// Deserializes a WebAssembly module - pub fn deserialize(&self, bytes: &[u8]) -> Result, DeserializeError> { + fn deserialize(&self, bytes: &[u8]) -> Result, DeserializeError> { Ok(Arc::new(CompiledModule::deserialize(&self, bytes)?)) } } diff --git a/lib/jit/src/module.rs b/lib/jit/src/module.rs index 2a7b766814d..67576d01f83 100644 --- a/lib/jit/src/module.rs +++ b/lib/jit/src/module.rs @@ -4,14 +4,8 @@ //! steps. use crate::engine::{JITEngine, JITEngineInner}; -use wasmer_engine::{DeserializeError, SerializeError, InstantiationError, LinkError, SerializableFunctionFrameInfo, - resolve_imports, Resolver, register_frame_info, GlobalFrameInfoRegistration, RuntimeError, - CompiledModule as BaseCompiledModule -}; use crate::link::link_module; -use crate::serialize::{ - SerializableCompilation, SerializableModule, -}; +use crate::serialize::{SerializableCompilation, SerializableModule}; use serde::{Deserialize, Serialize}; use std::any::Any; use std::sync::{Arc, Mutex}; @@ -22,6 +16,11 @@ use wasm_common::{ }; use wasmer_compiler::CompileError; use wasmer_compiler::ModuleEnvironment; +use wasmer_engine::{ + register_frame_info, resolve_imports, CompiledModule as BaseCompiledModule, DeserializeError, + Engine, GlobalFrameInfoRegistration, InstantiationError, LinkError, Resolver, RuntimeError, + SerializableFunctionFrameInfo, SerializeError, +}; use wasmer_runtime::{ InstanceHandle, LinearMemory, Module, SignatureRegistry, Table, VMFunctionBody, VMGlobalDefinition, VMSharedSignatureIndex, diff --git a/lib/jit/src/serialize.rs b/lib/jit/src/serialize.rs index 7f5eb3db3a2..a4a78af4ff3 100644 --- a/lib/jit/src/serialize.rs +++ b/lib/jit/src/serialize.rs @@ -1,10 +1,10 @@ use serde::{Deserialize, Serialize}; use std::sync::Arc; -use wasmer_compiler::{FunctionBody, JumpTableOffsets, Relocation}; -use wasmer_runtime::Module; -use wasmer_engine::SerializableFunctionFrameInfo; use wasm_common::entity::PrimaryMap; use wasm_common::{LocalFuncIndex, MemoryIndex, OwnedDataInitializer, TableIndex}; +use wasmer_compiler::{FunctionBody, JumpTableOffsets, Relocation}; +use wasmer_engine::SerializableFunctionFrameInfo; +use wasmer_runtime::Module; use wasmer_runtime::{MemoryPlan, TablePlan}; /// The compilation related data for a serialized modules diff --git a/src/compiler.rs b/src/compiler.rs index 016211e2aac..a2d1aee5a94 100644 --- a/src/compiler.rs +++ b/src/compiler.rs @@ -5,6 +5,7 @@ use crate::common::WasmFeatures; use anyhow::{bail, Error, Result}; use std::str::FromStr; use std::string::ToString; +use std::sync::Arc; use structopt::StructOpt; use wasmer::*; @@ -127,8 +128,8 @@ impl CompilerOptions { let compiler_name = compiler.to_string(); let compiler_config = self.get_config(compiler)?; let tunables = Tunables::for_target(compiler_config.target().triple()); - let engine = Engine::new(&*compiler_config, tunables); - let store = Store::new(&engine); + let engine = JITEngine::new(&*compiler_config, tunables); + let store = Store::new(Arc::new(engine)); Ok((store, compiler_name)) } } diff --git a/tests/wast.rs b/tests/wast.rs index bed0d1b6d3f..6f630c2f64b 100644 --- a/tests/wast.rs +++ b/tests/wast.rs @@ -1,6 +1,7 @@ use std::path::Path; +use std::sync::Arc; use test_utils::get_compiler_config_from_str; -use wasmer::{Engine, Features, Store, Tunables}; +use wasmer::{Features, JITEngine, Store, Tunables}; use wasmer_wast::Wast; // The generated tests (from build.rs) look like: @@ -23,7 +24,7 @@ fn run_wast(wast_path: &str, compiler: &str) -> anyhow::Result<()> { let compiler_config = get_compiler_config_from_str(compiler, try_nan_canonicalization, features); let tunables = Tunables::for_target(compiler_config.target().triple()); - let store = Store::new(&Engine::new(&*compiler_config, tunables)); + let store = Store::new(Arc::new(JITEngine::new(&*compiler_config, tunables))); let mut wast = Wast::new_with_spectest(store); let path = Path::new(wast_path); wast.run_file(path) From 143d965ac6a6a9e3f74b85751b85ab95a4767e72 Mon Sep 17 00:00:00 2001 From: Syrus Date: Sat, 2 May 2020 19:24:06 -0700 Subject: [PATCH 03/14] Renamed jit to Engine-jit --- Cargo.lock | 6 +++--- Cargo.toml | 4 ++-- lib/api/Cargo.toml | 2 +- lib/api/src/externals.rs | 2 +- lib/api/src/lib.rs | 2 +- lib/api/src/store.rs | 2 +- lib/{jit => engine-jit}/Cargo.toml | 4 ++-- lib/{jit => engine-jit}/README.md | 0 lib/{jit => engine-jit}/src/code_memory.rs | 0 lib/{jit => engine-jit}/src/engine.rs | 0 lib/{jit => engine-jit}/src/function_table.rs | 0 lib/{jit => engine-jit}/src/lib.rs | 0 lib/{jit => engine-jit}/src/link.rs | 0 lib/{jit => engine-jit}/src/module.rs | 0 lib/{jit => engine-jit}/src/serialize.rs | 0 15 files changed, 11 insertions(+), 11 deletions(-) rename lib/{jit => engine-jit}/Cargo.toml (94%) rename lib/{jit => engine-jit}/README.md (100%) rename lib/{jit => engine-jit}/src/code_memory.rs (100%) rename lib/{jit => engine-jit}/src/engine.rs (100%) rename lib/{jit => engine-jit}/src/function_table.rs (100%) rename lib/{jit => engine-jit}/src/lib.rs (100%) rename lib/{jit => engine-jit}/src/link.rs (100%) rename lib/{jit => engine-jit}/src/module.rs (100%) rename lib/{jit => engine-jit}/src/serialize.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index 38d7a7fea12..b4c1b785b29 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1472,7 +1472,7 @@ dependencies = [ "wasmer-compiler-llvm", "wasmer-compiler-singlepass", "wasmer-engine", - "wasmer-jit", + "wasmer-engine-jit", "wasmer-runtime", "wat", "winapi", @@ -1496,7 +1496,7 @@ dependencies = [ "wasmer-compiler-llvm", "wasmer-compiler-singlepass", "wasmer-engine", - "wasmer-jit", + "wasmer-engine-jit", "wasmer-wasi", "wasmer-wasi-experimental-io-devices", "wasmer-wast", @@ -1604,7 +1604,7 @@ dependencies = [ ] [[package]] -name = "wasmer-jit" +name = "wasmer-engine-jit" version = "0.16.2" dependencies = [ "backtrace", diff --git a/Cargo.toml b/Cargo.toml index 34d475d0be1..685ba323fe8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,7 +26,7 @@ wasmer-compiler-cranelift = { path = "lib/compiler-cranelift", optional = true } wasmer-compiler-singlepass = { path = "lib/compiler-singlepass", optional = true } wasmer-compiler-llvm = { path = "lib/compiler-llvm", optional = true } wasmer-engine = { path = "lib/engine" } -wasmer-jit = { path = "lib/jit" } +wasmer-engine-jit = { path = "lib/engine-jit" } wasmer-wasi = { path = "lib/wasi", optional = true } wasmer-wasi-experimental-io-devices = { path = "lib/wasi-experimental-io-devices", optional = true } wasmer-wast = { path = "tests/lib/wast", optional = true } @@ -50,7 +50,7 @@ anyhow = "1.0.28" test-utils = { path = "tests/lib/test-utils" } wasmer = { path = "lib/api" } wasmer-compiler = { path = "lib/compiler" } -wasmer-jit = { path = "lib/jit" } +wasmer-engine-jit = { path = "lib/engine-jit" } wasmer-wast = { path = "tests/lib/wast" } [features] diff --git a/lib/api/Cargo.toml b/lib/api/Cargo.toml index 74abb180c9d..9b3bcfa4ce3 100644 --- a/lib/api/Cargo.toml +++ b/lib/api/Cargo.toml @@ -15,7 +15,7 @@ wasmer-compiler-cranelift = { path = "../compiler-cranelift", version = "0.16.2" wasmer-compiler-llvm = { path = "../compiler-llvm", version = "0.16.2", optional = true } wasmer-compiler = { path = "../compiler", version = "0.16.2" } wasmer-engine = { path = "../engine", version = "0.16.2" } -wasmer-jit = { path = "../jit", version = "0.16.2" } +wasmer-engine-jit = { path = "../engine-jit", version = "0.16.2" } wasm-common = { path = "../wasm-common", version = "0.16.2" } indexmap = { version = "1.3.2", features = ["serde-1"] } cfg-if = "0.1.10" diff --git a/lib/api/src/externals.rs b/lib/api/src/externals.rs index 26a4c8654b6..0ab53fc0561 100644 --- a/lib/api/src/externals.rs +++ b/lib/api/src/externals.rs @@ -8,7 +8,7 @@ use crate::{ExternType, FuncType, GlobalType, MemoryType, TableType, ValType}; use std::cmp::max; use std::slice; use wasm_common::{Bytes, HostFunction, Pages, ValueType, WasmTypeList, WithEnv, WithoutEnv}; -use wasmer_engine::Engine; +use wasmer_engine::Engine as _; use wasmer_runtime::{ wasmer_call_trampoline, Export, ExportFunction, ExportGlobal, ExportMemory, ExportTable, LinearMemory, Table as RuntimeTable, VMCallerCheckedAnyfunc, VMContext, VMFunctionBody, diff --git a/lib/api/src/lib.rs b/lib/api/src/lib.rs index b87d7c02a21..6ca1ee0a053 100644 --- a/lib/api/src/lib.rs +++ b/lib/api/src/lib.rs @@ -31,7 +31,7 @@ pub use wasmer_compiler::{CompilerConfig, Features, Target}; pub use wasmer_engine::{ DeserializeError, Engine, InstantiationError, LinkError, RuntimeError, SerializeError, }; -pub use wasmer_jit::JITEngine; +pub use wasmer_engine_jit::JITEngine; #[cfg(feature = "compiler-singlepass")] pub use wasmer_compiler_singlepass::SinglepassConfig; diff --git a/lib/api/src/store.rs b/lib/api/src/store.rs index ae752be7d8a..73da1cad63f 100644 --- a/lib/api/src/store.rs +++ b/lib/api/src/store.rs @@ -2,7 +2,7 @@ use crate::tunables::Tunables; use std::sync::Arc; use wasmer_compiler::CompilerConfig; use wasmer_engine::Engine; -use wasmer_jit::JITEngine; +use wasmer_engine_jit::JITEngine; #[derive(Clone)] pub struct Store { diff --git a/lib/jit/Cargo.toml b/lib/engine-jit/Cargo.toml similarity index 94% rename from lib/jit/Cargo.toml rename to lib/engine-jit/Cargo.toml index 7123401d124..8b2f56fc39b 100644 --- a/lib/jit/Cargo.toml +++ b/lib/engine-jit/Cargo.toml @@ -1,8 +1,8 @@ [package] -name = "wasmer-jit" +name = "wasmer-engine-jit" version = "0.16.2" authors = ["Wasmer Engineering Team "] -description = "Wasmer JIT frontend" +description = "Wasmer JIT Engine" license = "(Apache-2.0 WITH LLVM-exception) or MIT" categories = ["wasm"] keywords = ["webassembly", "wasm"] diff --git a/lib/jit/README.md b/lib/engine-jit/README.md similarity index 100% rename from lib/jit/README.md rename to lib/engine-jit/README.md diff --git a/lib/jit/src/code_memory.rs b/lib/engine-jit/src/code_memory.rs similarity index 100% rename from lib/jit/src/code_memory.rs rename to lib/engine-jit/src/code_memory.rs diff --git a/lib/jit/src/engine.rs b/lib/engine-jit/src/engine.rs similarity index 100% rename from lib/jit/src/engine.rs rename to lib/engine-jit/src/engine.rs diff --git a/lib/jit/src/function_table.rs b/lib/engine-jit/src/function_table.rs similarity index 100% rename from lib/jit/src/function_table.rs rename to lib/engine-jit/src/function_table.rs diff --git a/lib/jit/src/lib.rs b/lib/engine-jit/src/lib.rs similarity index 100% rename from lib/jit/src/lib.rs rename to lib/engine-jit/src/lib.rs diff --git a/lib/jit/src/link.rs b/lib/engine-jit/src/link.rs similarity index 100% rename from lib/jit/src/link.rs rename to lib/engine-jit/src/link.rs diff --git a/lib/jit/src/module.rs b/lib/engine-jit/src/module.rs similarity index 100% rename from lib/jit/src/module.rs rename to lib/engine-jit/src/module.rs diff --git a/lib/jit/src/serialize.rs b/lib/engine-jit/src/serialize.rs similarity index 100% rename from lib/jit/src/serialize.rs rename to lib/engine-jit/src/serialize.rs From a0e3fdb1afbdb73237317ec28b68fdfa8eef8b1d Mon Sep 17 00:00:00 2001 From: Syrus Date: Sat, 2 May 2020 23:06:14 -0700 Subject: [PATCH 04/14] Make JIT engine optional in API --- build.rs | 6 +++++- lib/api/Cargo.toml | 14 ++++++++++++-- lib/api/src/lib.rs | 1 + lib/api/src/store.rs | 38 ++++++++++++++++++++++++++++++-------- 4 files changed, 48 insertions(+), 11 deletions(-) diff --git a/build.rs b/build.rs index bdcbb7c4068..aa85f428bf7 100644 --- a/build.rs +++ b/build.rs @@ -37,7 +37,11 @@ fn main() -> anyhow::Result<()> { with_backends(&mut spectests, &backends, |mut spectests| { with_test_module(&mut spectests, "spec", |spectests| { let _spec_tests = test_directory(spectests, "tests/wast/spec", wast_processor)?; - test_directory_module(spectests, "tests/wast/spec/proposals/multi-value", wast_processor)?; + test_directory_module( + spectests, + "tests/wast/spec/proposals/multi-value", + wast_processor, + )?; // test_directory_module(spectests, "tests/wast/spec/proposals/bulk-memory-operations", wast_processor)?; Ok(()) })?; diff --git a/lib/api/Cargo.toml b/lib/api/Cargo.toml index 9b3bcfa4ce3..7d60df9f378 100644 --- a/lib/api/Cargo.toml +++ b/lib/api/Cargo.toml @@ -15,7 +15,7 @@ wasmer-compiler-cranelift = { path = "../compiler-cranelift", version = "0.16.2" wasmer-compiler-llvm = { path = "../compiler-llvm", version = "0.16.2", optional = true } wasmer-compiler = { path = "../compiler", version = "0.16.2" } wasmer-engine = { path = "../engine", version = "0.16.2" } -wasmer-engine-jit = { path = "../engine-jit", version = "0.16.2" } +wasmer-engine-jit = { path = "../engine-jit", version = "0.16.2", optional = true } wasm-common = { path = "../wasm-common", version = "0.16.2" } indexmap = { version = "1.3.2", features = ["serde-1"] } cfg-if = "0.1.10" @@ -38,7 +38,17 @@ anyhow = "1.0.28" maintenance = { status = "actively-developed" } [features] -default = ["wat", "default-compiler-cranelift"] +default = [ + "wat", + "default-compiler-cranelift", + "default-engine-jit" +] +engine-jit = [ + "wasmer-engine-jit" +] +default-engine-jit = [ + "engine-jit" +] compiler-singlepass = [ "wasmer-compiler-singlepass" ] diff --git a/lib/api/src/lib.rs b/lib/api/src/lib.rs index 6ca1ee0a053..dfc645ba53c 100644 --- a/lib/api/src/lib.rs +++ b/lib/api/src/lib.rs @@ -31,6 +31,7 @@ pub use wasmer_compiler::{CompilerConfig, Features, Target}; pub use wasmer_engine::{ DeserializeError, Engine, InstantiationError, LinkError, RuntimeError, SerializeError, }; +#[cfg(feature = "engine-jit")] pub use wasmer_engine_jit::JITEngine; #[cfg(feature = "compiler-singlepass")] diff --git a/lib/api/src/store.rs b/lib/api/src/store.rs index 73da1cad63f..ebe9baf2b6c 100644 --- a/lib/api/src/store.rs +++ b/lib/api/src/store.rs @@ -2,7 +2,6 @@ use crate::tunables::Tunables; use std::sync::Arc; use wasmer_compiler::CompilerConfig; use wasmer_engine::Engine; -use wasmer_engine_jit::JITEngine; #[derive(Clone)] pub struct Store { @@ -54,6 +53,27 @@ impl Store { #[cfg(feature = "default-compiler-singlepass")] return wasmer_compiler_singlepass::SinglepassConfig::default(); } + + #[cfg(all( + any( + feature = "default-compiler-singlepass", + feature = "default-compiler-cranelift", + feature = "default-compiler-llvm", + ), + any(feature = "default-engine-jit", feature = "default-engine-native",) + ))] + pub fn default_engine() -> impl Engine { + #[cfg(all(feature = "default-engine-jit", feature = "default-engine-native",))] + compile_error!( + "The `default-engine-X` features are mutually exclusive. Please choose just one" + ); + + let config = Self::default_compiler_config(); + let tunables = Tunables::for_target(config.target().triple()); + + #[cfg(feature = "engine-jit")] + return wasmer_engine_jit::JITEngine::new(&config, tunables); + } } impl PartialEq for Store { @@ -63,16 +83,18 @@ impl PartialEq for Store { } // We only implement default if we have assigned a default compiler -#[cfg(any( - feature = "default-compiler-singlepass", - feature = "default-compiler-cranelift", - feature = "default-compiler-llvm", +#[cfg(all( + any( + feature = "default-compiler-singlepass", + feature = "default-compiler-cranelift", + feature = "default-compiler-llvm", + ), + any(feature = "default-engine-jit", feature = "default-engine-native",) ))] impl Default for Store { fn default() -> Store { - let config = Self::default_compiler_config(); - let tunables = Tunables::for_target(config.target().triple()); - Store::new(Arc::new(JITEngine::new(&config, tunables))) + let engine = Self::default_engine(); + Store::new(Arc::new(engine)) } } From c40ec78191cfd369665dea0c544f43cb66a0c9b1 Mon Sep 17 00:00:00 2001 From: Syrus Date: Sun, 3 May 2020 00:54:38 -0700 Subject: [PATCH 05/14] Trying to clean-up dependencies --- Cargo.toml | 19 ++++++++++++------- lib/cache/Cargo.toml | 2 +- lib/wasi/Cargo.toml | 2 +- tests/lib/wast/Cargo.toml | 6 +++++- 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 685ba323fe8..f49c5db0817 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,13 +20,13 @@ publish = false autoexamples = false [dependencies] -wasmer = { path = "lib/api" } +wasmer = { path = "lib/api", default-features = false } wasmer-compiler = { path = "lib/compiler" } wasmer-compiler-cranelift = { path = "lib/compiler-cranelift", optional = true } wasmer-compiler-singlepass = { path = "lib/compiler-singlepass", optional = true } wasmer-compiler-llvm = { path = "lib/compiler-llvm", optional = true } wasmer-engine = { path = "lib/engine" } -wasmer-engine-jit = { path = "lib/engine-jit" } +wasmer-engine-jit = { path = "lib/engine-jit", optional = true } wasmer-wasi = { path = "lib/wasi", optional = true } wasmer-wasi-experimental-io-devices = { path = "lib/wasi-experimental-io-devices", optional = true } wasmer-wast = { path = "tests/lib/wast", optional = true } @@ -48,16 +48,21 @@ rustc_version = "0.2" [dev-dependencies] anyhow = "1.0.28" test-utils = { path = "tests/lib/test-utils" } -wasmer = { path = "lib/api" } -wasmer-compiler = { path = "lib/compiler" } -wasmer-engine-jit = { path = "lib/engine-jit" } -wasmer-wast = { path = "tests/lib/wast" } +# wasmer = { path = "lib/api" } +# wasmer-compiler = { path = "lib/compiler" } +# wasmer-engine-jit = { path = "lib/engine-jit" } +# wasmer-wast = { path = "tests/lib/wast" } [features] # Don't add the backend features in default, please add them on the Makefile # since we might want to autoconfigure them depending on the availability on the host. # default = ["wasi"] -default = ["wast", "wasi", "experimental-io-devices", "compiler-cranelift", "cache"] +default = ["wast", "wasi", "wat", "experimental-io-devices", "compiler-cranelift", "cache", "engine-jit"] +wat = ["wasmer/wat"] +engine-jit = [ + "wasmer-engine-jit", + "wasmer/engine-jit", +] cache = ["wasmer-cache"] wast = ["wasmer-wast"] wasi = ["wasmer-wasi"] diff --git a/lib/cache/Cargo.toml b/lib/cache/Cargo.toml index 00b11670de4..c771b26ff34 100644 --- a/lib/cache/Cargo.toml +++ b/lib/cache/Cargo.toml @@ -11,7 +11,7 @@ keywords = ["webassembly", "wasm", "cache"] edition = "2018" [dependencies] -wasmer = { path = "../api", version = "0.16.2" } +wasmer = { path = "../api", version = "0.16.2", default-features = false } memmap = "0.7" hex = "0.4" thiserror = "1" diff --git a/lib/wasi/Cargo.toml b/lib/wasi/Cargo.toml index b83415e7039..82b64f592ca 100644 --- a/lib/wasi/Cargo.toml +++ b/lib/wasi/Cargo.toml @@ -20,7 +20,7 @@ getrandom = "0.1" time = "0.1" typetag = "0.1" serde = { version = "1", features = ["derive"] } -wasmer = { path = "../api", version = "0.16.2" } +wasmer = { path = "../api", version = "0.16.2", default-features = false } [target.'cfg(windows)'.dependencies] winapi = "0.3" diff --git a/tests/lib/wast/Cargo.toml b/tests/lib/wast/Cargo.toml index b3bfeb375a7..0d85802a987 100644 --- a/tests/lib/wast/Cargo.toml +++ b/tests/lib/wast/Cargo.toml @@ -12,9 +12,13 @@ edition = "2018" [dependencies] anyhow = "1.0.28" -wasmer = { path = "../../../lib/api", version = "0.16.2" } +wasmer = { path = "../../../lib/api", version = "0.16.2", default-features = false } wast = "14.0.0" thiserror = "1.0.15" +[features] +default = ["wat"] +wat = ["wasmer/wat"] + [badges] maintenance = { status = "actively-developed" } From 1d47cf9987a4cdbc7afa261d1632d681b046cb7a Mon Sep 17 00:00:00 2001 From: Syrus Date: Sun, 3 May 2020 01:58:54 -0700 Subject: [PATCH 06/14] Added support for the engine --- lib/engine-jit/src/engine.rs | 23 ++++++++- src/commands/run.rs | 96 +++++++++++++++++++++--------------- src/store.rs | 17 +++++-- 3 files changed, 91 insertions(+), 45 deletions(-) diff --git a/lib/engine-jit/src/engine.rs b/lib/engine-jit/src/engine.rs index 3107acbd9d0..cba0390835e 100644 --- a/lib/engine-jit/src/engine.rs +++ b/lib/engine-jit/src/engine.rs @@ -27,6 +27,8 @@ pub struct JITEngine { } impl JITEngine { + const MAGIC_HEADER: &'static [u8] = b"\0wasmer-jit"; + /// Create a new JIT Engine given config pub fn new(config: &C, tunables: impl Tunables + 'static) -> Self where @@ -51,6 +53,12 @@ impl JITEngine { pub(crate) fn compiler_mut(&self) -> std::cell::RefMut<'_, JITEngineInner> { self.inner.borrow_mut() } + + /// Check if the provided bytes look like a serialized + /// module by the `JITEngine` implementation. + pub fn is_deserializable(bytes: &[u8]) -> bool { + bytes.starts_with(Self::MAGIC_HEADER) + } } impl Engine for JITEngine { @@ -102,12 +110,23 @@ impl Engine for JITEngine { compiled_module: &Arc, ) -> Result, SerializeError> { let compiled_module = compiled_module.downcast_ref::().unwrap(); - compiled_module.serialize() + // We append the header + let mut serialized = Self::MAGIC_HEADER.to_vec(); + serialized.extend(compiled_module.serialize()?); + Ok(serialized) } /// Deserializes a WebAssembly module fn deserialize(&self, bytes: &[u8]) -> Result, DeserializeError> { - Ok(Arc::new(CompiledModule::deserialize(&self, bytes)?)) + if !Self::is_deserializable(bytes) { + return Err(DeserializeError::Incompatible( + "The provided bytes are not wasmer-jit".to_string(), + )); + } + Ok(Arc::new(CompiledModule::deserialize( + &self, + &bytes[Self::MAGIC_HEADER.len()..], + )?)) } } diff --git a/src/commands/run.rs b/src/commands/run.rs index fb59fdc7d7c..97323d3ca14 100644 --- a/src/commands/run.rs +++ b/src/commands/run.rs @@ -3,9 +3,12 @@ use crate::store::StoreOptions; use anyhow::{anyhow, bail, Context, Result}; use std::path::PathBuf; use std::str::FromStr; +use std::sync::Arc; use wasmer::*; #[cfg(feature = "cache")] use wasmer_cache::{Cache, FileSystemCache, IoDeserializeError, WasmHash}; +#[cfg(feature = "engine-jit")] +use wasmer_engine_jit::JITEngine; use structopt::StructOpt; @@ -78,46 +81,7 @@ impl Run { .context(format!("failed to run `{}`", self.path.display())) } fn inner_execute(&self) -> Result<()> { - let (store, compiler_name) = self.compiler.get_store()?; - let contents = std::fs::read(self.path.clone())?; - // We try to get it from cache, in case caching is enabled - // and the file length is greater than 4KB. - // For files smaller than 4KB caching is not worth, - // as it takes space and the speedup is minimal. - let mut module = - if cfg!(feature = "cache") && !self.disable_cache && contents.len() > 0x1000 { - let mut cache = self.get_cache(compiler_name)?; - // Try to get the hash from the provided `--cache-key`, otherwise - // generate one from the provided file `.wasm` contents. - let hash = self - .cache_key - .as_ref() - .and_then(|key| WasmHash::from_str(&key).ok()) - .unwrap_or(WasmHash::generate(&contents)); - let module = match unsafe { cache.load(&store, hash) } { - Ok(module) => module, - Err(e) => { - match e { - IoDeserializeError::Deserialize(e) => { - eprintln!("Warning: error while getting module from cache: {}", e); - } - IoDeserializeError::Io(_) => { - // Do not notify on IO errors - } - } - let module = Module::new(&store, &contents)?; - // Store the compiled Module in cache - cache.store(hash, module.clone())?; - module - } - }; - module - } else { - Module::new(&store, &contents)? - }; - // We set the name outside the cache, to make sure we dont cache the name - module.set_name(self.path.canonicalize()?.as_path().to_str().unwrap()); - + let module = self.get_module()?; // Do we want to invoke a function? if let Some(ref invoke) = self.invoke { let imports = imports! {}; @@ -164,6 +128,58 @@ impl Run { Ok(()) } + fn get_module(&self) -> Result { + let contents = std::fs::read(self.path.clone())?; + if JITEngine::is_deserializable(&contents) { + let (compiler_config, _compiler_name) = self.compiler.get_compiler_config()?; + let tunables = self.compiler.get_tunables(&*compiler_config); + let engine = JITEngine::new(&*compiler_config, tunables); + let store = Store::new(Arc::new(engine)); + let module = unsafe { Module::deserialize(&store, &contents)? }; + return Ok(module); + } + let (store, compiler_name) = self.compiler.get_store()?; + // We try to get it from cache, in case caching is enabled + // and the file length is greater than 4KB. + // For files smaller than 4KB caching is not worth, + // as it takes space and the speedup is minimal. + let mut module = + if cfg!(feature = "cache") && !self.disable_cache && contents.len() > 0x1000 { + let mut cache = self.get_cache(compiler_name)?; + // Try to get the hash from the provided `--cache-key`, otherwise + // generate one from the provided file `.wasm` contents. + let hash = self + .cache_key + .as_ref() + .and_then(|key| WasmHash::from_str(&key).ok()) + .unwrap_or(WasmHash::generate(&contents)); + let module = match unsafe { cache.load(&store, hash) } { + Ok(module) => module, + Err(e) => { + match e { + IoDeserializeError::Deserialize(e) => { + eprintln!("Warning: error while getting module from cache: {}", e); + } + IoDeserializeError::Io(_) => { + // Do not notify on IO errors + } + } + let module = Module::new(&store, &contents)?; + // Store the compiled Module in cache + cache.store(hash, module.clone())?; + module + } + }; + module + } else { + Module::new(&store, &contents)? + }; + // We set the name outside the cache, to make sure we dont cache the name + module.set_name(self.path.canonicalize()?.as_path().to_str().unwrap()); + + Ok(module) + } + fn invoke_function( &self, instance: &Instance, diff --git a/src/store.rs b/src/store.rs index aac46aa09c6..a40f157b06d 100644 --- a/src/store.rs +++ b/src/store.rs @@ -122,12 +122,23 @@ impl StoreOptions { return Ok(config); } - /// Get's the store - pub fn get_store(&self) -> Result<(Store, String)> { + /// Get's the compiler config + pub fn get_compiler_config(&self) -> Result<(Box, String)> { let compiler = self.get_compiler()?; let compiler_name = compiler.to_string(); let compiler_config = self.get_config(compiler)?; - let tunables = Tunables::for_target(compiler_config.target().triple()); + Ok((compiler_config, compiler_name)) + } + + /// Get's the tunables for the compiler target + pub fn get_tunables(&self, compiler_config: &dyn CompilerConfig) -> Tunables { + Tunables::for_target(compiler_config.target().triple()) + } + + /// Get's the store + pub fn get_store(&self) -> Result<(Store, String)> { + let (compiler_config, compiler_name) = self.get_compiler_config()?; + let tunables = self.get_tunables(&*compiler_config); let engine = JITEngine::new(&*compiler_config, tunables); let store = Store::new(Arc::new(engine)); Ok((store, compiler_name)) From 5e181b0da2035f6ff11e0aa4cf80a5ba4863a651 Mon Sep 17 00:00:00 2001 From: Syrus Date: Sun, 3 May 2020 20:55:07 -0700 Subject: [PATCH 07/14] Removed unused file --- lib/api/src/resolver.rs | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 lib/api/src/resolver.rs diff --git a/lib/api/src/resolver.rs b/lib/api/src/resolver.rs deleted file mode 100644 index 182bdcd8afc..00000000000 --- a/lib/api/src/resolver.rs +++ /dev/null @@ -1,21 +0,0 @@ -use wasmer_engine::Resolver; -use std::iter::FromIterator; - -pub struct sIndexResolver { - externs: Vec, -} -impl Resolver for IndexResolver { - fn resolve(&self, index: u32, _module: &str, _name: &str) -> Option { - self.externs.get(index as usize).map(|extern_| extern_.to_export()) - } -} - -impl FromIterator for IndexResolver { - fn from_iter>(iter: I) -> Self { - let mut externs = Vec::new(); - for extern_ in iter { - externs.push(extern_); - } - IndexResolver { externs } - } -} From c99c971e7cdb36e38d6ec8eb8aa7f4d01b2bbdc0 Mon Sep 17 00:00:00 2001 From: Syrus Date: Sun, 3 May 2020 23:23:40 -0700 Subject: [PATCH 08/14] Removed dangling CompilerConfig --- lib/engine/src/lib.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/engine/src/lib.rs b/lib/engine/src/lib.rs index 436d59afffc..97d9c322d60 100644 --- a/lib/engine/src/lib.rs +++ b/lib/engine/src/lib.rs @@ -39,7 +39,5 @@ pub use crate::serialize::SerializableFunctionFrameInfo; pub use crate::trap::*; pub use crate::tunables::Tunables; -pub use wasmer_compiler::CompilerConfig; - /// Version number of this crate. pub const VERSION: &str = env!("CARGO_PKG_VERSION"); From 08985ae75b698b177b9616573331764e425f674b Mon Sep 17 00:00:00 2001 From: Syrus Akbary Date: Mon, 4 May 2020 14:29:42 -0700 Subject: [PATCH 09/14] Update lib/engine-jit/src/link.rs Co-authored-by: nlewycky --- lib/engine-jit/src/link.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/engine-jit/src/link.rs b/lib/engine-jit/src/link.rs index 7df28a699c9..d011886e474 100644 --- a/lib/engine-jit/src/link.rs +++ b/lib/engine-jit/src/link.rs @@ -11,7 +11,7 @@ use wasmer_runtime::Module; use wasmer_runtime::VMFunctionBody; /// Links a module, patching the allocated functions with the -/// requiredd relocations and jump tables. +/// required relocations and jump tables. pub fn link_module( module: &Module, allocated_functions: &PrimaryMap, From 9e5103f3af1b2939e237f2e18516deb070a25dee Mon Sep 17 00:00:00 2001 From: Syrus Date: Tue, 5 May 2020 15:30:52 -0700 Subject: [PATCH 10/14] Simplified engine traits --- lib/api/src/module.rs | 8 ++++++-- lib/engine-jit/src/engine.rs | 4 ++-- lib/engine/src/engine.rs | 4 ++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/api/src/module.rs b/lib/api/src/module.rs index 599e743ea76..50a08bfcc53 100644 --- a/lib/api/src/module.rs +++ b/lib/api/src/module.rs @@ -1,6 +1,7 @@ use crate::store::Store; use crate::types::{ExportType, ImportType}; use crate::InstantiationError; +use std::borrow::Borrow; use std::io; use std::path::Path; use std::sync::Arc; @@ -155,7 +156,7 @@ impl Module { /// let serialized = module.serialize()?; /// ``` pub fn serialize(&self) -> Result, SerializeError> { - self.store.engine().serialize(&self.compiled) + self.store.engine().serialize(self.compiled.borrow()) } /// Deserializes a a serialized Module binary into a `Module`. @@ -197,7 +198,10 @@ impl Module { resolver: &dyn Resolver, ) -> Result { unsafe { - let instance_handle = self.store.engine().instantiate(&self.compiled, resolver)?; + let instance_handle = self + .store + .engine() + .instantiate(self.compiled.borrow(), resolver)?; // After the instance handle is created, we need to initialize // the data, call the start function and so. However, if any diff --git a/lib/engine-jit/src/engine.rs b/lib/engine-jit/src/engine.rs index aa4e48ecdcc..aa5c08303ac 100644 --- a/lib/engine-jit/src/engine.rs +++ b/lib/engine-jit/src/engine.rs @@ -123,7 +123,7 @@ impl Engine for JITEngine { /// Instantiates a WebAssembly module unsafe fn instantiate( &self, - compiled_module: &Arc, + compiled_module: &dyn BaseCompiledModule, resolver: &dyn Resolver, ) -> Result { let compiled_module = compiled_module.downcast_ref::().unwrap(); @@ -133,7 +133,7 @@ impl Engine for JITEngine { /// Serializes a WebAssembly module fn serialize( &self, - compiled_module: &Arc, + compiled_module: &dyn BaseCompiledModule, ) -> Result, SerializeError> { let compiled_module = compiled_module.downcast_ref::().unwrap(); // We append the header diff --git a/lib/engine/src/engine.rs b/lib/engine/src/engine.rs index 8aae502b40b..709d5fae4aa 100644 --- a/lib/engine/src/engine.rs +++ b/lib/engine/src/engine.rs @@ -34,12 +34,12 @@ pub trait Engine { /// Instantiates a WebAssembly module unsafe fn instantiate( &self, - compiled_module: &Arc, + compiled_module: &dyn CompiledModule, resolver: &dyn Resolver, ) -> Result; /// Serializes a WebAssembly module - fn serialize(&self, compiled_module: &Arc) -> Result, SerializeError>; + fn serialize(&self, compiled_module: &dyn CompiledModule) -> Result, SerializeError>; /// Deserializes a WebAssembly module fn deserialize(&self, bytes: &[u8]) -> Result, DeserializeError>; From 456d7e31d28e9c34eae1fa5c433289c447e6fffa Mon Sep 17 00:00:00 2001 From: Syrus Date: Tue, 5 May 2020 15:46:07 -0700 Subject: [PATCH 11/14] Make types more consistent --- lib/api/src/module.rs | 2 +- lib/engine-jit/src/module.rs | 20 ++++++++------------ lib/engine/src/module.rs | 7 ++----- lib/engine/src/trap/frame_info.rs | 2 +- 4 files changed, 12 insertions(+), 19 deletions(-) diff --git a/lib/api/src/module.rs b/lib/api/src/module.rs index 50a08bfcc53..d7f53afc9b0 100644 --- a/lib/api/src/module.rs +++ b/lib/api/src/module.rs @@ -244,7 +244,7 @@ impl Module { /// ``` pub fn set_name(&mut self, name: &str) { let compiled = Arc::get_mut(&mut self.compiled).unwrap(); - Arc::get_mut(compiled.module_mut()).unwrap().name = Some(name.to_string()); + compiled.module_mut().name = Some(name.to_string()); } /// Returns an iterator over the imported types in the Module. diff --git a/lib/engine-jit/src/module.rs b/lib/engine-jit/src/module.rs index 37e86384f66..d6b99164960 100644 --- a/lib/engine-jit/src/module.rs +++ b/lib/engine-jit/src/module.rs @@ -209,7 +209,7 @@ impl CompiledModule { let tunables = jit.tunables(); let sig_registry: &SignatureRegistry = jit_compiler.signatures(); let imports = resolve_imports( - &self.serializable.module, + &self.module(), &sig_registry, resolver, self.memory_plans(), @@ -218,15 +218,15 @@ impl CompiledModule { .map_err(InstantiationError::Link)?; let finished_memories = tunables - .create_memories(&self.serializable.module, self.memory_plans()) + .create_memories(&self.module(), self.memory_plans()) .map_err(InstantiationError::Link)? .into_boxed_slice(); let finished_tables = tunables - .create_tables(&self.serializable.module, self.table_plans()) + .create_tables(&self.module(), self.table_plans()) .map_err(InstantiationError::Link)? .into_boxed_slice(); let finished_globals = tunables - .create_globals(&self.serializable.module) + .create_globals(&self.module()) .map_err(InstantiationError::Link)? .into_boxed_slice(); @@ -268,7 +268,7 @@ impl CompiledModule { let frame_infos = &self.serializable.compilation.function_frame_info; let finished_functions = &self.finished_functions; *info = Some(register_frame_info( - &self.module(), + self.serializable.module.clone(), finished_functions, frame_infos.clone(), )); @@ -289,15 +289,11 @@ impl BaseCompiledModule for CompiledModule { .map_err(|trap| InstantiationError::Start(RuntimeError::from_trap(trap))) } - fn module(&self) -> &Arc { + fn module(&self) -> &Module { &self.serializable.module } - fn module_mut(&mut self) -> &mut Arc { - &mut self.serializable.module - } - - fn module_ref(&self) -> &Module { - &self.serializable.module + fn module_mut(&mut self) -> &mut Module { + Arc::get_mut(&mut self.serializable.module).unwrap() } } diff --git a/lib/engine/src/module.rs b/lib/engine/src/module.rs index 95720d11a2a..2ff523eb635 100644 --- a/lib/engine/src/module.rs +++ b/lib/engine/src/module.rs @@ -19,13 +19,10 @@ pub trait CompiledModule: DowncastSync { ) -> Result<(), InstantiationError>; /// Return a reference-counting pointer to a module. - fn module(&self) -> &Arc; + fn module(&self) -> &Module; /// Return a reference-counting pointer to a module. - fn module_mut(&mut self) -> &mut Arc; - - /// Return a reference to a module. - fn module_ref(&self) -> &Module; + fn module_mut(&mut self) -> &mut Module; } impl_downcast!(sync CompiledModule); // `sync` => also produce `Arc` downcasts. diff --git a/lib/engine/src/trap/frame_info.rs b/lib/engine/src/trap/frame_info.rs index 1d0d34300ca..5678b9c327e 100644 --- a/lib/engine/src/trap/frame_info.rs +++ b/lib/engine/src/trap/frame_info.rs @@ -225,7 +225,7 @@ impl Drop for GlobalFrameInfoRegistration { /// then `None` will be returned. Otherwise the returned object, when /// dropped, will be used to unregister all name information from this map. pub fn register( - module: &Arc, + module: Arc, finished_functions: &BoxedSlice, frame_infos: PrimaryMap, ) -> Option { From b422c2327573fea932c8da83a812b0fd18edc923 Mon Sep 17 00:00:00 2001 From: Syrus Date: Tue, 5 May 2020 15:51:47 -0700 Subject: [PATCH 12/14] Covered use case of no engines available --- src/store.rs | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/store.rs b/src/store.rs index 08f37a1ebf9..9d40aeece81 100644 --- a/src/store.rs +++ b/src/store.rs @@ -9,8 +9,6 @@ use std::sync::Arc; use structopt::StructOpt; use wasmer::*; use wasmer_compiler::CompilerConfig; -#[cfg(feature = "jit")] -use wasmer_engine_jit::JITEngine; #[derive(Debug, Clone, StructOpt)] /// The compiler options @@ -66,7 +64,7 @@ impl FromStr for Compiler { } } -#[cfg(feature = "compiler")] +#[cfg(all(feature = "compiler", feature = "engine"))] impl StoreOptions { fn get_compiler(&self) -> Result { if self.cranelift { @@ -141,13 +139,14 @@ impl StoreOptions { let (compiler_config, compiler_name) = self.get_compiler_config()?; let tunables = self.get_tunables(&*compiler_config); #[cfg(feature = "jit")] - let engine = JITEngine::new(&*compiler_config, tunables); + let engine = wasmer_engine_jit::JITEngine::new(&*compiler_config, tunables); let store = Store::new(Arc::new(engine)); Ok((store, compiler_name)) } } -#[cfg(not(feature = "compiler"))] +// If we don't have a compiler, but we have an engine +#[cfg(all(not(feature = "compiler"), feature = "engine"))] impl StoreOptions { /// Get the store (headless engine) pub fn get_store(&self) -> Result<(Store, String)> { @@ -158,3 +157,12 @@ impl StoreOptions { Ok((store, "headless".to_string())) } } + +// If we don't have neither a compiler nor an engine +#[cfg(not(all(feature = "compiler", feature = "engine")))] +impl StoreOptions { + /// Get the store (headless engine) + pub fn get_store(&self) -> Result<(Store, String)> { + bail!("No engines are enabled"); + } +} From 0de4a22fced7d302c772ed4497f21d8813b295ec Mon Sep 17 00:00:00 2001 From: Syrus Date: Tue, 5 May 2020 15:53:45 -0700 Subject: [PATCH 13/14] Fixed StoreOptions with no engines --- src/store.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/store.rs b/src/store.rs index 9d40aeece81..a814c3540ae 100644 --- a/src/store.rs +++ b/src/store.rs @@ -158,8 +158,8 @@ impl StoreOptions { } } -// If we don't have neither a compiler nor an engine -#[cfg(not(all(feature = "compiler", feature = "engine")))] +// If we don't have any engine enabled +#[cfg(not(feature = "engine"))] impl StoreOptions { /// Get the store (headless engine) pub fn get_store(&self) -> Result<(Store, String)> { From 60e8350214d9056b77125e44cfa015cbeb0921c9 Mon Sep 17 00:00:00 2001 From: Syrus Date: Tue, 5 May 2020 16:12:53 -0700 Subject: [PATCH 14/14] Fixed Send/Sync in CompiledModule --- lib/engine-jit/src/module.rs | 3 --- lib/engine/src/module.rs | 6 +++--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/lib/engine-jit/src/module.rs b/lib/engine-jit/src/module.rs index d6b99164960..9673bf589d5 100644 --- a/lib/engine-jit/src/module.rs +++ b/lib/engine-jit/src/module.rs @@ -275,9 +275,6 @@ impl CompiledModule { } } -unsafe impl Sync for CompiledModule {} -unsafe impl Send for CompiledModule {} - impl BaseCompiledModule for CompiledModule { unsafe fn finish_instantiation( &self, diff --git a/lib/engine/src/module.rs b/lib/engine/src/module.rs index 2ff523eb635..a1e7510eb52 100644 --- a/lib/engine/src/module.rs +++ b/lib/engine/src/module.rs @@ -3,11 +3,11 @@ use std::sync::Arc; use wasmer_runtime::InstanceHandle; use wasmer_runtime::Module; -use downcast_rs::{impl_downcast, DowncastSync}; +use downcast_rs::{impl_downcast, Downcast}; /// The `CompiledModule` trait is used by engine implementors, such /// as a JIT or Native execution. -pub trait CompiledModule: DowncastSync { +pub trait CompiledModule: Downcast { /// Finish instantiation of a `InstanceHandle` /// /// # Unsafety @@ -25,4 +25,4 @@ pub trait CompiledModule: DowncastSync { fn module_mut(&mut self) -> &mut Module; } -impl_downcast!(sync CompiledModule); // `sync` => also produce `Arc` downcasts. +impl_downcast!(CompiledModule); // `sync` => also produce `Arc` downcasts.