Merge #2250

2250: Use rkyv for JIT Engine r=syrusakbary a=syrusakbary Since we are going to switch serialization methods, the new one doesn’t have a slowdown if we read the data directly. Thus, we can always operate with processed frame infos rather than creating a lazy structure that will be deserialized when needed. Try it: ```shell git clone https://github.com/wasmerio/wasmer.git -b universal-compiled-info cd wasmer make build-wasmer ./target/release/wasmer run lib/c-api/tests/assets/qjs.wasm --disable-cache -- -h ``` Co-authored-by: Syrus Akbary <[email protected]>
wasmerio · Apr 30, 2021 · 8012f3f · 8012f3f
2 parents 70c8dd6 + ab55989
commit 8012f3f
Show file tree

Hide file tree

Showing 30 changed files with 320 additions and 349 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock
diff --git a/lib/compiler/Cargo.toml b/lib/compiler/Cargo.toml
@@ -21,7 +21,7 @@ serde = { version = "1.0", features = ["derive"], optional = true }
 thiserror = "1.0"
 serde_bytes = { version = "0.11", optional = true }
 smallvec = "1.6" 
-rkyv = { version = "0.6", optional = true }
+rkyv = { version = "0.6.1", optional = true }
 loupe = "0.1"
 
 [features]
@@ -33,7 +33,7 @@ translator = ["wasmparser"]
 std = ["wasmer-types/std"]
 core = ["hashbrown", "wasmer-types/core"]
 enable-serde = ["serde", "serde_bytes", "wasmer-types/enable-serde"]
-enable-rkyv = ["rkyv", "wasmer-vm/enable-rkyv"]
+enable-rkyv = ["rkyv", "wasmer-vm/enable-rkyv", "wasmer-types/enable-rkyv"]
 
 [badges]
 maintenance = { status = "experimental" }
diff --git a/lib/compiler/src/address_map.rs b/lib/compiler/src/address_map.rs
@@ -4,11 +4,17 @@
 use crate::lib::std::vec::Vec;
 use crate::sourceloc::SourceLoc;
 use loupe::MemoryUsage;
+#[cfg(feature = "enable-rkyv")]
+use rkyv::{Archive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize};
 #[cfg(feature = "enable-serde")]
 use serde::{Deserialize, Serialize};
 
 /// Single source location to generated address mapping.
 #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
+#[cfg_attr(
+    feature = "enable-rkyv",
+    derive(RkyvSerialize, RkyvDeserialize, Archive)
+)]
 #[derive(Debug, Clone, PartialEq, Eq, MemoryUsage)]
 pub struct InstructionAddressMap {
     /// Original source location.
@@ -23,6 +29,10 @@ pub struct InstructionAddressMap {
 
 /// Function and its instructions addresses mappings.
 #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
+#[cfg_attr(
+    feature = "enable-rkyv",
+    derive(RkyvSerialize, RkyvDeserialize, Archive)
+)]
 #[derive(Debug, Clone, PartialEq, Eq, Default, MemoryUsage)]
 pub struct FunctionAddressMap {
     /// Instructions maps.

diff --git a/lib/compiler/src/function.rs b/lib/compiler/src/function.rs
@@ -13,6 +13,8 @@ use crate::section::{CustomSection, SectionIndex};
 use crate::trap::TrapInformation;
 use crate::{CompiledFunctionUnwindInfo, FunctionAddressMap, JumpTableOffsets, Relocation};
 use loupe::MemoryUsage;
+#[cfg(feature = "enable-rkyv")]
+use rkyv::{Archive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize};
 #[cfg(feature = "enable-serde")]
 use serde::{Deserialize, Serialize};
 use wasmer_types::entity::PrimaryMap;
@@ -23,6 +25,10 @@ use wasmer_types::{FunctionIndex, LocalFunctionIndex, SignatureIndex};
 /// This structure is only used for reconstructing
 /// the frame information after a `Trap`.
 #[cfg_attr(feature = "enable-serde", derive(Deserialize, Serialize))]
+#[cfg_attr(
+    feature = "enable-rkyv",
+    derive(RkyvSerialize, RkyvDeserialize, Archive)
+)]
 #[derive(Debug, Clone, PartialEq, Eq, Default, MemoryUsage)]
 pub struct CompiledFunctionFrameInfo {
     /// The traps (in the function body).
@@ -36,6 +42,10 @@ pub struct CompiledFunctionFrameInfo {
 
 /// The function body.
 #[cfg_attr(feature = "enable-serde", derive(Deserialize, Serialize))]
+#[cfg_attr(
+    feature = "enable-rkyv",
+    derive(RkyvSerialize, RkyvDeserialize, Archive)
+)]
 #[derive(Debug, Clone, PartialEq, Eq, MemoryUsage)]
 pub struct FunctionBody {
     /// The function body bytes.
@@ -52,6 +62,10 @@ pub struct FunctionBody {
 /// (function bytecode body, relocations, traps, jump tables
 /// and unwind information).
 #[cfg_attr(feature = "enable-serde", derive(Deserialize, Serialize))]
+#[cfg_attr(
+    feature = "enable-rkyv",
+    derive(RkyvSerialize, RkyvDeserialize, Archive)
+)]
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub struct CompiledFunction {
     /// The function body.
@@ -80,6 +94,10 @@ pub type CustomSections = PrimaryMap<SectionIndex, CustomSection>;
 /// In the future this structure may also hold other information useful
 /// for debugging.
 #[cfg_attr(feature = "enable-serde", derive(Deserialize, Serialize))]
+#[cfg_attr(
+    feature = "enable-rkyv",
+    derive(RkyvSerialize, RkyvDeserialize, Archive)
+)]
 #[derive(Debug, PartialEq, Eq, Clone, MemoryUsage)]
 pub struct Dwarf {
     /// The section index in the [`Compilation`] that corresponds to the exception frames.

diff --git a/lib/compiler/src/jump_table.rs b/lib/compiler/src/jump_table.rs
@@ -6,6 +6,8 @@
 
 use super::CodeOffset;
 use loupe::MemoryUsage;
+#[cfg(feature = "enable-rkyv")]
+use rkyv::{Archive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize};
 #[cfg(feature = "enable-serde")]
 use serde::{Deserialize, Serialize};
 use wasmer_types::entity::{entity_impl, SecondaryMap};
@@ -15,6 +17,10 @@ use wasmer_types::entity::{entity_impl, SecondaryMap};
 /// `JumpTable`s are used for indirect branching and are specialized for dense,
 /// 0-based jump offsets.
 #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
+#[cfg_attr(
+    feature = "enable-rkyv",
+    derive(RkyvSerialize, RkyvDeserialize, Archive)
+)]
 #[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, MemoryUsage)]
 pub struct JumpTable(u32);
 

diff --git a/lib/compiler/src/relocation.rs b/lib/compiler/src/relocation.rs
@@ -14,6 +14,8 @@ use crate::lib::std::vec::Vec;
 use crate::section::SectionIndex;
 use crate::{Addend, CodeOffset, JumpTable};
 use loupe::MemoryUsage;
+#[cfg(feature = "enable-rkyv")]
+use rkyv::{Archive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize};
 #[cfg(feature = "enable-serde")]
 use serde::{Deserialize, Serialize};
 use wasmer_types::entity::PrimaryMap;
@@ -22,6 +24,10 @@ use wasmer_vm::libcalls::LibCall;
 
 /// Relocation kinds for every ISA.
 #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
+#[cfg_attr(
+    feature = "enable-rkyv",
+    derive(RkyvSerialize, RkyvDeserialize, Archive)
+)]
 #[derive(Copy, Clone, Debug, PartialEq, Eq, MemoryUsage)]
 pub enum RelocationKind {
     /// absolute 4-byte
@@ -80,6 +86,10 @@ impl fmt::Display for RelocationKind {
 
 /// A record of a relocation to perform.
 #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
+#[cfg_attr(
+    feature = "enable-rkyv",
+    derive(RkyvSerialize, RkyvDeserialize, Archive)
+)]
 #[derive(Debug, Clone, PartialEq, Eq, MemoryUsage)]
 pub struct Relocation {
     /// The relocation kind.
@@ -94,6 +104,10 @@ pub struct Relocation {
 
 /// Destination function. Can be either user function or some special one, like `memory.grow`.
 #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
+#[cfg_attr(
+    feature = "enable-rkyv",
+    derive(RkyvSerialize, RkyvDeserialize, Archive)
+)]
 #[derive(Debug, Copy, Clone, PartialEq, Eq, MemoryUsage)]
 pub enum RelocationTarget {
     /// A relocation to a function defined locally in the wasm (not an imported one).

diff --git a/lib/compiler/src/section.rs b/lib/compiler/src/section.rs
@@ -8,21 +8,38 @@
 use crate::lib::std::vec::Vec;
 use crate::Relocation;
 use loupe::MemoryUsage;
+#[cfg(feature = "enable-rkyv")]
+use rkyv::{Archive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize};
 #[cfg(feature = "enable-serde")]
 use serde::{Deserialize, Serialize};
 use wasmer_types::entity::entity_impl;
 
 /// Index type of a Section defined inside a WebAssembly `Compilation`.
 #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
+#[cfg_attr(
+    feature = "enable-rkyv",
+    derive(RkyvSerialize, RkyvDeserialize, Archive)
+)]
+#[cfg_attr(
+    feature = "enable-rkyv",
+    archive(derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug))
+)]
 #[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug, MemoryUsage)]
 pub struct SectionIndex(u32);
 
 entity_impl!(SectionIndex);
 
+#[cfg(feature = "enable-rkyv")]
+entity_impl!(ArchivedSectionIndex);
+
 /// Custom section Protection.
 ///
 /// Determines how a custom section may be used.
 #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
+#[cfg_attr(
+    feature = "enable-rkyv",
+    derive(RkyvSerialize, RkyvDeserialize, Archive)
+)]
 #[derive(Debug, Clone, PartialEq, Eq, MemoryUsage)]
 pub enum CustomSectionProtection {
     /// A custom section with read permission.
@@ -37,6 +54,10 @@ pub enum CustomSectionProtection {
 /// This is used so compilers can store arbitrary information
 /// in the emitted module.
 #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
+#[cfg_attr(
+    feature = "enable-rkyv",
+    derive(RkyvSerialize, RkyvDeserialize, Archive)
+)]
 #[derive(Debug, Clone, PartialEq, Eq, MemoryUsage)]
 pub struct CustomSection {
     /// Memory protection that applies to this section.
@@ -56,6 +77,10 @@ pub struct CustomSection {
 
 /// The bytes in the section.
 #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
+#[cfg_attr(
+    feature = "enable-rkyv",
+    derive(RkyvSerialize, RkyvDeserialize, Archive)
+)]
 #[derive(Debug, Clone, PartialEq, Eq, Default, MemoryUsage)]
 pub struct SectionBody(#[cfg_attr(feature = "enable-serde", serde(with = "serde_bytes"))] Vec<u8>);
 

diff --git a/lib/compiler/src/sourceloc.rs b/lib/compiler/src/sourceloc.rs
@@ -9,6 +9,8 @@
 
 use crate::lib::std::fmt;
 use loupe::MemoryUsage;
+#[cfg(feature = "enable-rkyv")]
+use rkyv::{Archive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize};
 #[cfg(feature = "enable-serde")]
 use serde::{Deserialize, Serialize};
 
@@ -21,6 +23,10 @@ use serde::{Deserialize, Serialize};
     derive(Serialize, Deserialize),
     serde(transparent)
 )]
+#[cfg_attr(
+    feature = "enable-rkyv",
+    derive(RkyvSerialize, RkyvDeserialize, Archive)
+)]
 #[repr(transparent)]
 #[derive(Debug, Clone, Copy, PartialEq, Eq, MemoryUsage)]
 pub struct SourceLoc(u32);

diff --git a/lib/compiler/src/trap.rs b/lib/compiler/src/trap.rs
@@ -1,11 +1,17 @@
 use crate::CodeOffset;
 use loupe::MemoryUsage;
+#[cfg(feature = "enable-rkyv")]
+use rkyv::{Archive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize};
 #[cfg(feature = "enable-serde")]
 use serde::{Deserialize, Serialize};
 use wasmer_vm::TrapCode;
 
 /// Information about trap.
 #[cfg_attr(feature = "enable-serde", derive(Deserialize, Serialize))]
+#[cfg_attr(
+    feature = "enable-rkyv",
+    derive(RkyvSerialize, RkyvDeserialize, Archive)
+)]
 #[derive(Clone, Debug, PartialEq, Eq, MemoryUsage)]
 pub struct TrapInformation {
     /// The offset of the trapping instruction in native code. It is relative to the beginning of the function.

diff --git a/lib/compiler/src/unwind.rs b/lib/compiler/src/unwind.rs
@@ -7,6 +7,8 @@
 //! [Learn more](https://en.wikipedia.org/wiki/Call_stack).
 use crate::lib::std::vec::Vec;
 use loupe::MemoryUsage;
+#[cfg(feature = "enable-rkyv")]
+use rkyv::{Archive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize};
 #[cfg(feature = "enable-serde")]
 use serde::{Deserialize, Serialize};
 
@@ -18,6 +20,10 @@ use serde::{Deserialize, Serialize};
 ///
 /// [unwind info]: https://docs.microsoft.com/en-us/cpp/build/exception-handling-x64?view=vs-2019
 #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
+#[cfg_attr(
+    feature = "enable-rkyv",
+    derive(RkyvSerialize, RkyvDeserialize, Archive)
+)]
 #[derive(Debug, Clone, PartialEq, Eq, MemoryUsage)]
 pub enum CompiledFunctionUnwindInfo {
     /// Windows UNWIND_INFO.