From 14ee38a24e46ac794990c12b7b1a951d6384fa50 Mon Sep 17 00:00:00 2001 From: Manos Pitsidianakis Date: Wed, 13 Jul 2022 22:29:45 +0300 Subject: [PATCH 1/4] compiler: move Symbol, SymbolRegistry to wasmer-types --- lib/compiler-llvm/src/compiler.rs | 6 +- lib/compiler-llvm/src/translator/code.rs | 4 +- lib/compiler/src/compiler.rs | 31 +--- lib/compiler/src/lib.rs | 2 +- lib/object/src/module.rs | 6 +- lib/types/src/compilation/mod.rs | 1 + lib/types/src/compilation/symbols.rs | 208 +++++++++++++++++++++++ lib/types/src/lib.rs | 1 + 8 files changed, 220 insertions(+), 39 deletions(-) create mode 100644 lib/types/src/compilation/symbols.rs diff --git a/lib/compiler-llvm/src/compiler.rs b/lib/compiler-llvm/src/compiler.rs index e63ea126c0d..7ff8ba78b5e 100644 --- a/lib/compiler-llvm/src/compiler.rs +++ b/lib/compiler-llvm/src/compiler.rs @@ -10,14 +10,12 @@ use inkwell::DLLStorageClass; use rayon::iter::ParallelBridge; use rayon::prelude::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator}; use std::sync::Arc; -use wasmer_compiler::{ - Compiler, FunctionBodyData, ModuleMiddleware, ModuleTranslationState, Symbol, SymbolRegistry, -}; +use wasmer_compiler::{Compiler, FunctionBodyData, ModuleMiddleware, ModuleTranslationState}; use wasmer_types::entity::{EntityRef, PrimaryMap}; use wasmer_types::{ Compilation, CompileError, CompileModuleInfo, CustomSection, CustomSectionProtection, Dwarf, FunctionIndex, LocalFunctionIndex, RelocationTarget, SectionBody, SectionIndex, SignatureIndex, - Target, + Symbol, SymbolRegistry, Target, }; //use std::sync::Mutex; diff --git a/lib/compiler-llvm/src/translator/code.rs b/lib/compiler-llvm/src/translator/code.rs index 85b6edc56a3..0d372e95e04 100644 --- a/lib/compiler-llvm/src/translator/code.rs +++ b/lib/compiler-llvm/src/translator/code.rs @@ -28,12 +28,12 @@ use std::convert::TryFrom; use wasmer_compiler::wasmparser::{MemoryImmediate, Operator}; use wasmer_compiler::{ from_binaryreadererror_wasmerror, wptype_to_type, FunctionBinaryReader, FunctionBodyData, - MiddlewareBinaryReader, ModuleMiddlewareChain, ModuleTranslationState, Symbol, SymbolRegistry, + MiddlewareBinaryReader, ModuleMiddlewareChain, ModuleTranslationState, }; use wasmer_types::entity::PrimaryMap; use wasmer_types::{ CompileError, FunctionIndex, FunctionType, GlobalIndex, LocalFunctionIndex, MemoryIndex, - ModuleInfo, RelocationTarget, SignatureIndex, TableIndex, Type, + ModuleInfo, RelocationTarget, SignatureIndex, Symbol, SymbolRegistry, TableIndex, Type, }; use wasmer_vm::{MemoryStyle, TableStyle, VMOffsets}; diff --git a/lib/compiler/src/compiler.rs b/lib/compiler/src/compiler.rs index 72fad88306a..3c15bd7cad3 100644 --- a/lib/compiler/src/compiler.rs +++ b/lib/compiler/src/compiler.rs @@ -8,11 +8,11 @@ use crate::FunctionBodyData; use crate::ModuleTranslationState; use wasmer_types::compilation::function::Compilation; use wasmer_types::compilation::module::CompileModuleInfo; +use wasmer_types::compilation::symbols::SymbolRegistry; use wasmer_types::compilation::target::Target; use wasmer_types::entity::PrimaryMap; use wasmer_types::error::CompileError; -use wasmer_types::SectionIndex; -use wasmer_types::{Features, FunctionIndex, LocalFunctionIndex, SignatureIndex}; +use wasmer_types::{Features, LocalFunctionIndex}; use wasmparser::{Validator, WasmFeatures}; /// The compiler configuration options. @@ -144,30 +144,3 @@ pub trait Compiler: Send { /// Get the middlewares for this compiler fn get_middlewares(&self) -> &[Arc]; } - -/// The kinds of wasmer_types objects that might be found in a native object file. -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum Symbol { - /// A function defined in the wasm. - LocalFunction(LocalFunctionIndex), - - /// A wasm section. - Section(SectionIndex), - - /// The function call trampoline for a given signature. - FunctionCallTrampoline(SignatureIndex), - - /// The dynamic function trampoline for a given function. - DynamicFunctionTrampoline(FunctionIndex), -} - -/// This trait facilitates symbol name lookups in a native object file. -pub trait SymbolRegistry: Send + Sync { - /// Given a `Symbol` it returns the name for that symbol in the object file - fn symbol_to_name(&self, symbol: Symbol) -> String; - - /// Given a name it returns the `Symbol` for that name in the object file - /// - /// This function is the inverse of [`SymbolRegistry::symbol_to_name`] - fn name_to_symbol(&self, name: &str) -> Option; -} diff --git a/lib/compiler/src/lib.rs b/lib/compiler/src/lib.rs index 09a32d32e30..fbf9d369a65 100644 --- a/lib/compiler/src/lib.rs +++ b/lib/compiler/src/lib.rs @@ -70,7 +70,7 @@ mod compiler; #[macro_use] mod translator; #[cfg(feature = "translator")] -pub use crate::compiler::{Compiler, CompilerConfig, Symbol, SymbolRegistry}; +pub use crate::compiler::{Compiler, CompilerConfig}; #[cfg(feature = "translator")] pub use crate::translator::{ from_binaryreadererror_wasmerror, translate_module, wptype_to_type, FunctionBinaryReader, diff --git a/lib/object/src/module.rs b/lib/object/src/module.rs index 20e65b168bd..40cb005af2c 100644 --- a/lib/object/src/module.rs +++ b/lib/object/src/module.rs @@ -6,13 +6,13 @@ use object::{ elf, macho, RelocationEncoding, RelocationKind, SectionKind, SymbolFlags, SymbolKind, SymbolScope, }; -use wasmer_compiler::{Symbol, SymbolRegistry}; use wasmer_types::entity::PrimaryMap; use wasmer_types::LocalFunctionIndex; use wasmer_types::{ Architecture, BinaryFormat, Compilation, CustomSectionProtection, Endianness, RelocationKind as Reloc, RelocationTarget, SectionIndex, Triple, }; +use wasmer_types::{Symbol, SymbolRegistry}; const DWARF_SECTION_NAME: &[u8] = b".eh_frame"; @@ -111,7 +111,7 @@ pub fn emit_data( /// # Usage /// /// ```rust -/// # use wasmer_compiler::SymbolRegistry; +/// # use wasmer_types::SymbolRegistry; /// # use wasmer_types::{Compilation, Triple}; /// # use wasmer_object::ObjectError; /// use wasmer_object::{get_object_for_target, emit_compilation}; @@ -390,7 +390,7 @@ pub fn emit_compilation( /// # Usage /// /// ```rust -/// # use wasmer_compiler::SymbolRegistry; +/// # use wasmer_types::SymbolRegistry; /// # use wasmer_types::{Compilation, Triple}; /// # use wasmer_object::ObjectError; /// use wasmer_object::{get_object_for_target, emit_compilation}; diff --git a/lib/types/src/compilation/mod.rs b/lib/types/src/compilation/mod.rs index 49712fcf570..87ad57b4dc8 100644 --- a/lib/types/src/compilation/mod.rs +++ b/lib/types/src/compilation/mod.rs @@ -6,6 +6,7 @@ pub mod module; pub mod relocation; pub mod section; pub mod sourceloc; +pub mod symbols; pub mod target; pub mod trap; pub mod unwind; diff --git a/lib/types/src/compilation/symbols.rs b/lib/types/src/compilation/symbols.rs new file mode 100644 index 00000000000..7201b60f5c6 --- /dev/null +++ b/lib/types/src/compilation/symbols.rs @@ -0,0 +1,208 @@ +//! This module define the required structures for compilation symbols. +use crate::{ + entity::{EntityRef, PrimaryMap}, + CompileModuleInfo, DeserializeError, FunctionIndex, LocalFunctionIndex, OwnedDataInitializer, + SectionIndex, SerializeError, SignatureIndex, +}; +use rkyv::{ + archived_value, de::deserializers::SharedDeserializeMap, ser::serializers::AllocSerializer, + ser::Serializer as RkyvSerializer, Archive, Deserialize as RkyvDeserialize, + Serialize as RkyvSerialize, +}; +#[cfg(feature = "enable-serde")] +use serde::{Deserialize, Serialize}; + +/// The kinds of wasmer_types objects that might be found in a native object file. +#[derive( + RkyvSerialize, + RkyvDeserialize, + Archive, + Copy, + Clone, + PartialEq, + Eq, + Hash, + PartialOrd, + Ord, + Debug, +)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] +#[archive(as = "Self")] +pub enum Symbol { + /// A function defined in the wasm. + LocalFunction(LocalFunctionIndex), + + /// A wasm section. + Section(SectionIndex), + + /// The function call trampoline for a given signature. + FunctionCallTrampoline(SignatureIndex), + + /// The dynamic function trampoline for a given function. + DynamicFunctionTrampoline(FunctionIndex), +} + +/// This trait facilitates symbol name lookups in a native object file. +pub trait SymbolRegistry: Send + Sync { + /// Given a `Symbol` it returns the name for that symbol in the object file + fn symbol_to_name(&self, symbol: Symbol) -> String; + + /// Given a name it returns the `Symbol` for that name in the object file + /// + /// This function is the inverse of [`SymbolRegistry::symbol_to_name`] + fn name_to_symbol(&self, name: &str) -> Option; +} + +/// Serializable struct that represents the compiled metadata. +#[derive(Debug, RkyvSerialize, RkyvDeserialize, Archive)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] +pub struct ModuleMetadata { + /// FIXME + pub compile_info: CompileModuleInfo, + /// FIXME + pub prefix: String, + /// FIXME + pub data_initializers: Box<[OwnedDataInitializer]>, + /// The function body lengths (used to find function by address) + pub function_body_lengths: PrimaryMap, + /// FIXME + pub cpu_features: u64, +} + +/// FIXME +pub struct ModuleMetadataSymbolRegistry { + /// FIXME + pub prefix: String, +} + +impl ModuleMetadata { + /// FIXME + pub fn split(&mut self) -> (&mut CompileModuleInfo, ModuleMetadataSymbolRegistry) { + let compile_info = &mut self.compile_info; + let symbol_registry = ModuleMetadataSymbolRegistry { + prefix: self.prefix.clone(), + }; + (compile_info, symbol_registry) + } + + /// FIXME + pub fn get_symbol_registry(&self) -> ModuleMetadataSymbolRegistry { + ModuleMetadataSymbolRegistry { + prefix: self.prefix.clone(), + } + } + /// Serialize a Module into bytes + /// The bytes will have the following format: + /// RKYV serialization (any length) + POS (8 bytes) + pub fn serialize(&self) -> Result, SerializeError> { + let mut serializer = AllocSerializer::<4096>::default(); + let pos = serializer + .serialize_value(self) + .map_err(|err| SerializeError::Generic(format!("{}", err)))? as u64; + let mut serialized_data = serializer.into_serializer().into_inner(); + serialized_data.extend_from_slice(&pos.to_le_bytes()); + Ok(serialized_data.to_vec()) + } + + /// Deserialize a Module from a slice. + /// The slice must have the following format: + /// RKYV serialization (any length) + POS (8 bytes) + /// + /// # Safety + /// + /// This method is unsafe since it deserializes data directly + /// from memory. + /// Right now we are not doing any extra work for validation, but + /// `rkyv` has an option to do bytecheck on the serialized data before + /// serializing (via `rkyv::check_archived_value`). + pub unsafe fn deserialize(metadata_slice: &[u8]) -> Result { + let archived = Self::archive_from_slice(metadata_slice)?; + Self::deserialize_from_archive(archived) + } + + /// # Safety + /// + /// This method is unsafe. + /// Please check `ModuleMetadata::deserialize` for more details. + unsafe fn archive_from_slice( + metadata_slice: &[u8], + ) -> Result<&ArchivedModuleMetadata, DeserializeError> { + if metadata_slice.len() < 8 { + return Err(DeserializeError::Incompatible( + "invalid serialized ModuleMetadata".into(), + )); + } + let mut pos: [u8; 8] = Default::default(); + pos.copy_from_slice(&metadata_slice[metadata_slice.len() - 8..metadata_slice.len()]); + let pos: u64 = u64::from_le_bytes(pos); + Ok(archived_value::( + &metadata_slice[..metadata_slice.len() - 8], + pos as usize, + )) + } + + /// Deserialize a compilation module from an archive + pub fn deserialize_from_archive( + archived: &ArchivedModuleMetadata, + ) -> Result { + let mut deserializer = SharedDeserializeMap::new(); + RkyvDeserialize::deserialize(archived, &mut deserializer) + .map_err(|e| DeserializeError::CorruptedBinary(format!("{:?}", e))) + } +} + +impl SymbolRegistry for ModuleMetadataSymbolRegistry { + fn symbol_to_name(&self, symbol: Symbol) -> String { + match symbol { + Symbol::LocalFunction(index) => { + format!("wasmer_function_{}_{}", self.prefix, index.index()) + } + Symbol::Section(index) => format!("wasmer_section_{}_{}", self.prefix, index.index()), + Symbol::FunctionCallTrampoline(index) => { + format!( + "wasmer_trampoline_function_call_{}_{}", + self.prefix, + index.index() + ) + } + Symbol::DynamicFunctionTrampoline(index) => { + format!( + "wasmer_trampoline_dynamic_function_{}_{}", + self.prefix, + index.index() + ) + } + } + } + + fn name_to_symbol(&self, name: &str) -> Option { + if let Some(index) = name.strip_prefix(&format!("wasmer_function_{}_", self.prefix)) { + index + .parse::() + .ok() + .map(|index| Symbol::LocalFunction(LocalFunctionIndex::from_u32(index))) + } else if let Some(index) = name.strip_prefix(&format!("wasmer_section_{}_", self.prefix)) { + index + .parse::() + .ok() + .map(|index| Symbol::Section(SectionIndex::from_u32(index))) + } else if let Some(index) = + name.strip_prefix(&format!("wasmer_trampoline_function_call_{}_", self.prefix)) + { + index + .parse::() + .ok() + .map(|index| Symbol::FunctionCallTrampoline(SignatureIndex::from_u32(index))) + } else if let Some(index) = name.strip_prefix(&format!( + "wasmer_trampoline_dynamic_function_{}_", + self.prefix + )) { + index + .parse::() + .ok() + .map(|index| Symbol::DynamicFunctionTrampoline(FunctionIndex::from_u32(index))) + } else { + None + } + } +} diff --git a/lib/types/src/lib.rs b/lib/types/src/lib.rs index 74b12e3e26d..20f91736d55 100644 --- a/lib/types/src/lib.rs +++ b/lib/types/src/lib.rs @@ -124,6 +124,7 @@ pub use crate::compilation::function::{ }; pub use crate::compilation::module::CompileModuleInfo; pub use crate::compilation::sourceloc::SourceLoc; +pub use crate::compilation::symbols::{Symbol, SymbolRegistry}; pub use crate::compilation::trap::TrapInformation; pub use crate::compilation::unwind::CompiledFunctionUnwindInfo; From 430a7e2e19c25e362139a1aee76d193c0d373745 Mon Sep 17 00:00:00 2001 From: Manos Pitsidianakis Date: Tue, 2 Aug 2022 13:25:02 +0300 Subject: [PATCH 2/4] Make serde optional everywhere with `enable-serde` feature flag --- Cargo.lock | 3 --- lib/api/Cargo.toml | 9 ++++++++- lib/c-api/Cargo.toml | 1 - lib/cli/Cargo.toml | 9 +++++++++ lib/compiler/Cargo.toml | 6 +++++- lib/types/Cargo.toml | 6 +++--- lib/vbus/Cargo.toml | 3 +-- lib/vfs/Cargo.toml | 2 +- lib/vfs/src/host_fs.rs | 10 +++++----- lib/vfs/src/lib.rs | 6 +++--- lib/vm/Cargo.toml | 5 +++-- lib/vnet/Cargo.toml | 3 +-- lib/wasi-experimental-io-devices/Cargo.toml | 2 +- lib/wasi-types/Cargo.toml | 4 ++-- lib/wasi/src/state/mod.rs | 7 +++++++ lib/wasi/src/state/socket.rs | 4 ++-- lib/wasi/src/state/types.rs | 2 +- 17 files changed, 52 insertions(+), 30 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 512d3de9722..ed3b5d8aea1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2887,7 +2887,6 @@ dependencies = [ "lazy_static", "libc", "paste", - "serde", "thiserror", "typetag", "wasmer", @@ -3129,7 +3128,6 @@ name = "wasmer-vbus" version = "3.0.0-alpha.4" dependencies = [ "libc", - "serde", "slab", "thiserror", "tracing", @@ -3178,7 +3176,6 @@ version = "3.0.0-alpha.4" dependencies = [ "bytes", "libc", - "serde", "slab", "thiserror", "tracing", diff --git a/lib/api/Cargo.toml b/lib/api/Cargo.toml index d869a49e76a..aae032f3400 100644 --- a/lib/api/Cargo.toml +++ b/lib/api/Cargo.toml @@ -22,7 +22,7 @@ edition = "2018" # Shared dependencies. [dependencies] # - Mandatory shared dependencies. -indexmap = { version = "1.6", features = ["serde-1"] } +indexmap = { version = "1.6" } cfg-if = "1.0" thiserror = "1.0" more-asserts = "0.2" @@ -113,5 +113,12 @@ wasm-types-polyfill = ["js", "wasmparser"] js-serializable-module = [] +# Optional +enable-serde = [ + "wasmer-vm/enable-serde", + "wasmer-compiler/enable-serde", + "wasmer-types/enable-serde", +] + [package.metadata.docs.rs] features = ["compiler", "core", "cranelift", "engine", "jit", "native", "singlepass", "sys", "sys-default" ] diff --git a/lib/c-api/Cargo.toml b/lib/c-api/Cargo.toml index ecbd90add21..db43f1412c6 100644 --- a/lib/c-api/Cargo.toml +++ b/lib/c-api/Cargo.toml @@ -35,7 +35,6 @@ enumset = "1.0.2" cfg-if = "1.0" lazy_static = "1.4" libc = { version = "^0.2", default-features = false } -serde = { version = "1", optional = true, features = ["derive"] } thiserror = "1" typetag = { version = "0.1", optional = true } paste = "1.0" diff --git a/lib/cli/Cargo.toml b/lib/cli/Cargo.toml index c686fe5455e..5dcc9003988 100644 --- a/lib/cli/Cargo.toml +++ b/lib/cli/Cargo.toml @@ -97,3 +97,12 @@ debug = ["fern", "log", "wasmer-wasi/logging"] disable-all-logging = ["wasmer-wasi/disable-all-logging"] headless = [] headless-minimal = ["headless", "disable-all-logging", "wasi"] + +# Optional +enable-serde = [ + "wasmer/enable-serde", + "wasmer-vm/enable-serde", + "wasmer-compiler/enable-serde", + "wasmer-types/enable-serde", + "wasmer-wasi/enable-serde", +] diff --git a/lib/compiler/Cargo.toml b/lib/compiler/Cargo.toml index 05e37c734c5..94e9b73601e 100644 --- a/lib/compiler/Cargo.toml +++ b/lib/compiler/Cargo.toml @@ -38,12 +38,16 @@ region = { version = "3.0" } winapi = { version = "0.3", features = ["winnt", "impl-default"] } [features] -default = ["std", "enable-serde" ] +default = ["std" ] # This feature is for compiler implementors, it enables using `Compiler` and # `CompilerConfig`, as well as the included wasmparser. # Disable this feature if you just want a headless engine. translator = ["wasmparser"] compiler = ["translator"] +wasmer-artifact-load = [] +wasmer-artifact-create = [] +static-artifact-load = [] +static-artifact-create = [] std = ["wasmer-types/std"] core = ["hashbrown", "wasmer-types/core"] enable-serde = ["serde", "serde_bytes", "wasmer-types/enable-serde"] diff --git a/lib/types/Cargo.toml b/lib/types/Cargo.toml index 391a95d5bf3..8caeac5b412 100644 --- a/lib/types/Cargo.toml +++ b/lib/types/Cargo.toml @@ -15,14 +15,14 @@ serde = { version = "1.0", features = ["derive", "rc"], optional = true, default serde_bytes = { version = "0.11", optional = true } thiserror = "1.0" more-asserts = "0.2" -indexmap = { version = "1.6", features = ["serde-1"] } +indexmap = { version = "1.6" } rkyv = { version = "0.7.38", features = ["indexmap"] } enum-iterator = "0.7.0" target-lexicon = { version = "0.12.2", default-features = false } enumset = "1.0" [features] -default = ["std", "enable-serde"] +default = ["std"] std = [] core = [] -enable-serde = ["serde", "serde/std", "serde_bytes"] +enable-serde = ["serde", "serde/std", "serde_bytes", "indexmap/serde-1"] diff --git a/lib/vbus/Cargo.toml b/lib/vbus/Cargo.toml index a54b3841b5c..40b542b59b5 100644 --- a/lib/vbus/Cargo.toml +++ b/lib/vbus/Cargo.toml @@ -11,11 +11,10 @@ libc = { version = "^0.2", default-features = false, optional = true } thiserror = "1" tracing = { version = "0.1" } typetag = { version = "0.1", optional = true } -serde = { version = "1.0", default-features = false, features = ["derive"], optional = true } slab = { version = "0.4", optional = true } wasmer-vfs = { path = "../vfs", version = "=3.0.0-alpha.4", default-features = false } [features] default = ["mem_fs"] mem_fs = ["wasmer-vfs/mem-fs"] -host_fs = ["wasmer-vfs/host-fs"] \ No newline at end of file +host_fs = ["wasmer-vfs/host-fs"] diff --git a/lib/vfs/Cargo.toml b/lib/vfs/Cargo.toml index 6ea1c3617de..0f8c7f30dbb 100644 --- a/lib/vfs/Cargo.toml +++ b/lib/vfs/Cargo.toml @@ -22,4 +22,4 @@ enable-serde = [ "serde", "typetag" ] -no-time = [] \ No newline at end of file +no-time = [] diff --git a/lib/vfs/src/host_fs.rs b/lib/vfs/src/host_fs.rs index 788d21b3b77..0cc9ddec068 100644 --- a/lib/vfs/src/host_fs.rs +++ b/lib/vfs/src/host_fs.rs @@ -326,7 +326,7 @@ impl File { inner: file, host_path, #[cfg(feature = "enable-serde")] - _flags, + flags: _flags, } } @@ -377,7 +377,7 @@ impl Write for File { } } -#[cfg_attr(feature = "enable-serde", typetag::serde)] +//#[cfg_attr(feature = "enable-serde", typetag::serde)] impl VirtualFile for File { fn last_accessed(&self) -> u64 { self.metadata() @@ -506,7 +506,7 @@ impl Write for Stdout { } } -#[cfg_attr(feature = "enable-serde", typetag::serde)] +//#[cfg_attr(feature = "enable-serde", typetag::serde)] impl VirtualFile for Stdout { fn last_accessed(&self) -> u64 { 0 @@ -602,7 +602,7 @@ impl Write for Stderr { } } -#[cfg_attr(feature = "enable-serde", typetag::serde)] +//#[cfg_attr(feature = "enable-serde", typetag::serde)] impl VirtualFile for Stderr { fn last_accessed(&self) -> u64 { 0 @@ -697,7 +697,7 @@ impl Write for Stdin { } } -#[cfg_attr(feature = "enable-serde", typetag::serde)] +//#[cfg_attr(feature = "enable-serde", typetag::serde)] impl VirtualFile for Stdin { fn last_accessed(&self) -> u64 { 0 diff --git a/lib/vfs/src/lib.rs b/lib/vfs/src/lib.rs index 982b944b3ee..e6c028724a1 100644 --- a/lib/vfs/src/lib.rs +++ b/lib/vfs/src/lib.rs @@ -8,8 +8,8 @@ use thiserror::Error; #[cfg(all(not(feature = "host-fs"), not(feature = "mem-fs")))] compile_error!("At least the `host-fs` or the `mem-fs` feature must be enabled. Please, pick one."); -#[cfg(all(feature = "mem-fs", feature = "enable-serde"))] -compile_error!("`mem-fs` does not support `enable-serde` for the moment."); +//#[cfg(all(feature = "mem-fs", feature = "enable-serde"))] +//compile_warn!("`mem-fs` does not support `enable-serde` for the moment."); #[cfg(feature = "host-fs")] pub mod host_fs; @@ -171,7 +171,7 @@ impl OpenOptions { } /// This trait relies on your file closing when it goes out of scope via `Drop` -#[cfg_attr(feature = "enable-serde", typetag::serde)] +//#[cfg_attr(feature = "enable-serde", typetag::serde)] pub trait VirtualFile: fmt::Debug + Write + Read + Seek + Upcastable { /// the last time the file was accessed in nanoseconds as a UNIX timestamp fn last_accessed(&self) -> u64; diff --git a/lib/vm/Cargo.toml b/lib/vm/Cargo.toml index 21d384bd7d4..47773975da8 100644 --- a/lib/vm/Cargo.toml +++ b/lib/vm/Cargo.toml @@ -14,12 +14,12 @@ edition = "2018" wasmer-types = { path = "../types", version = "=3.0.0-alpha.4" } libc = { version = "^0.2", default-features = false } memoffset = "0.6" -indexmap = { version = "1.6", features = ["serde-1"] } +indexmap = { version = "1.6" } thiserror = "1.0" more-asserts = "0.2" cfg-if = "1.0" backtrace = "0.3" -serde = { version = "1.0", features = ["derive", "rc"] } +serde = { version = "1.0", features = ["derive", "rc"], optional = true } enum-iterator = "0.7.0" scopeguard = "1.1.0" lazy_static = "1.4.0" @@ -40,3 +40,4 @@ maintenance = { status = "actively-developed" } [features] default = [] +enable-serde = ["serde", "indexmap/serde-1", "wasmer-types/enable-serde" ] diff --git a/lib/vnet/Cargo.toml b/lib/vnet/Cargo.toml index 697b316a742..fb922f14754 100644 --- a/lib/vnet/Cargo.toml +++ b/lib/vnet/Cargo.toml @@ -11,7 +11,6 @@ libc = { version = "^0.2", default-features = false, optional = true } thiserror = "1" tracing = { version = "0.1" } typetag = { version = "0.1", optional = true } -serde = { version = "1.0", default-features = false, features = ["derive"], optional = true } slab = { version = "0.4", optional = true } wasmer-vfs = { path = "../vfs", version = "=3.0.0-alpha.4", default-features = false } bytes = "1" @@ -19,4 +18,4 @@ bytes = "1" [features] default = ["mem_fs"] mem_fs = ["wasmer-vfs/mem-fs"] -host_fs = ["wasmer-vfs/host-fs"] \ No newline at end of file +host_fs = ["wasmer-vfs/host-fs"] diff --git a/lib/wasi-experimental-io-devices/Cargo.toml b/lib/wasi-experimental-io-devices/Cargo.toml index f0f40e57e5d..0a1d3f6f396 100644 --- a/lib/wasi-experimental-io-devices/Cargo.toml +++ b/lib/wasi-experimental-io-devices/Cargo.toml @@ -35,4 +35,4 @@ enable-serde = [ # have to enable this feature manually link_external_libs = [ "minifb" -] \ No newline at end of file +] diff --git a/lib/wasi-types/Cargo.toml b/lib/wasi-types/Cargo.toml index 297d7784be6..2a8abbfa0da 100644 --- a/lib/wasi-types/Cargo.toml +++ b/lib/wasi-types/Cargo.toml @@ -13,9 +13,9 @@ edition = "2018" [dependencies] wasmer-types = { path = "../types", version = "=3.0.0-alpha.4" } wasmer-derive = { path = "../derive", version = "=3.0.0-alpha.4" } -serde = { version = "1.0", features = ["derive"], optional=true } +serde = { version = "1.0", features = ["derive"], optional = true } byteorder = "1.3" time = "0.2" [features] -enable-serde = ["serde"] +enable-serde = ["serde", "wasmer-types/serde"] diff --git a/lib/wasi/src/state/mod.rs b/lib/wasi/src/state/mod.rs index 1f34a3f92fa..e81223d3c02 100644 --- a/lib/wasi/src/state/mod.rs +++ b/lib/wasi/src/state/mod.rs @@ -104,6 +104,7 @@ impl InodeVal { pub enum Kind { File { /// The open file, if it's open + #[cfg_attr(feature = "enable-serde", serde(skip))] handle: Option>, /// The path on the host system where the file is located /// This is deprecated and will be removed soon @@ -114,10 +115,12 @@ pub enum Kind { /// TOOD: clarify here? fd: Option, }, + #[cfg_attr(feature = "enable-serde", serde(skip))] Socket { /// Represents a networking socket socket: InodeSocket, }, + #[cfg_attr(feature = "enable-serde", serde(skip))] Pipe { /// Reference to the pipe pipe: WasiPipe, @@ -161,6 +164,7 @@ pub enum Kind { /// Flag that indicates if this is operating is_semaphore: bool, /// Receiver that wakes sleeping threads + #[cfg_attr(feature = "enable-serde", serde(skip))] wakers: Arc>>>, }, } @@ -1801,9 +1805,12 @@ impl WasiState { #[derive(Debug, Default)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub(crate) struct WasiStateThreading { + #[cfg_attr(feature = "enable-serde", serde(skip))] pub threads: HashMap, pub thread_seed: u32, + #[cfg_attr(feature = "enable-serde", serde(skip))] pub processes: HashMap, + #[cfg_attr(feature = "enable-serde", serde(skip))] pub process_reuse: HashMap, WasiBusProcessId>, pub process_seed: u32, } diff --git a/lib/wasi/src/state/socket.rs b/lib/wasi/src/state/socket.rs index ff4e92eca45..58c093c4afc 100644 --- a/lib/wasi/src/state/socket.rs +++ b/lib/wasi/src/state/socket.rs @@ -33,7 +33,7 @@ pub enum InodeHttpSocketType { } #[derive(Debug)] -#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] +//#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub enum InodeSocketKind { PreSocket { family: __wasi_addressfamily_t, @@ -143,7 +143,7 @@ pub struct WasiHttpStatus { } #[derive(Debug)] -#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] +//#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub struct InodeSocket { kind: InodeSocketKind, read_buffer: Option, diff --git a/lib/wasi/src/state/types.rs b/lib/wasi/src/state/types.rs index 7842f68bc5d..f5828c8ee59 100644 --- a/lib/wasi/src/state/types.rs +++ b/lib/wasi/src/state/types.rs @@ -419,7 +419,7 @@ impl Seek for Pipe { } } -#[cfg_attr(feature = "enable-serde", typetag::serde)] +//#[cfg_attr(feature = "enable-serde", typetag::serde)] impl VirtualFile for Pipe { fn last_accessed(&self) -> u64 { 0 From 8b096a504e4c67103334ab887852756757741c38 Mon Sep 17 00:00:00 2001 From: Manos Pitsidianakis Date: Tue, 2 Aug 2022 13:48:29 +0300 Subject: [PATCH 3/4] compiler: add `wasmer-artifact-load`,`wasmer-artifact-create`,`static-artifact-load`,`static-artifact-create` feature flags --- Cargo.lock | 2 +- lib/cli/Cargo.toml | 6 +- lib/cli/src/cli.rs | 6 +- lib/cli/src/commands.rs | 4 +- lib/compiler/Cargo.toml | 3 +- lib/compiler/src/engine/artifact.rs | 151 +++++++++++++++++++++++++++- lib/object/Cargo.toml | 4 - 7 files changed, 162 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ed3b5d8aea1..af57075cc93 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2964,6 +2964,7 @@ dependencies = [ "serde_bytes", "smallvec", "thiserror", + "wasmer-object", "wasmer-types", "wasmer-vm", "wasmparser 0.83.0", @@ -3104,7 +3105,6 @@ version = "3.0.0-alpha.4" dependencies = [ "object 0.28.4", "thiserror", - "wasmer-compiler", "wasmer-types", ] diff --git a/lib/cli/Cargo.toml b/lib/cli/Cargo.toml index 5dcc9003988..ea4e1e365da 100644 --- a/lib/cli/Cargo.toml +++ b/lib/cli/Cargo.toml @@ -37,7 +37,7 @@ wasmer-wasi-experimental-io-devices = { version = "=3.0.0-alpha.4", path = "../w wasmer-wast = { version = "=3.0.0-alpha.4", path = "../../tests/lib/wast", optional = true } wasmer-cache = { version = "=3.0.0-alpha.4", path = "../cache", optional = true } wasmer-types = { version = "=3.0.0-alpha.4", path = "../types" } -wasmer-object = { version = "=3.0.0-alpha.4", path = "../object" } +wasmer-object = { version = "=3.0.0-alpha.4", path = "../object", optional = true } wasmer-vfs = { version = "=3.0.0-alpha.4", path = "../vfs", default-features = false, features = ["host-fs"] } atty = "0.2" colored = "2.0" @@ -66,6 +66,7 @@ default = [ "wasi", "emscripten", "compiler", + "wasmer-artifact-create", ] cache = ["wasmer-cache"] cache-blake3-pure = ["wasmer-cache/blake3-pure"] @@ -77,6 +78,9 @@ compiler = [ "wasmer-compiler/translator", "wasmer-compiler/compiler", ] +wasmer-artifact-create = ["compiler", "wasmer-compiler/wasmer-artifact-create", "wasmer-object"] +static-artifact-create = ["compiler", "wasmer-compiler/static-artifact-create", "wasmer-object"] + experimental-io-devices = [ "wasmer-wasi-experimental-io-devices", "wasi" diff --git a/lib/cli/src/cli.rs b/lib/cli/src/cli.rs index ad938a9ae8d..3587dba5a93 100644 --- a/lib/cli/src/cli.rs +++ b/lib/cli/src/cli.rs @@ -4,7 +4,7 @@ use crate::commands::Binfmt; #[cfg(feature = "compiler")] use crate::commands::Compile; -#[cfg(feature = "compiler")] +#[cfg(any(feature = "static-artifact-create", feature = "wasmer-artifact-create"))] use crate::commands::CreateExe; #[cfg(feature = "wast")] use crate::commands::Wast; @@ -47,7 +47,7 @@ enum WasmerCLIOptions { Compile(Compile), /// Compile a WebAssembly binary into a native executable - #[cfg(feature = "compiler")] + #[cfg(any(feature = "static-artifact-create", feature = "wasmer-artifact-create"))] #[structopt(name = "create-exe")] CreateExe(CreateExe), @@ -84,7 +84,7 @@ impl WasmerCLIOptions { Self::Validate(validate) => validate.execute(), #[cfg(feature = "compiler")] Self::Compile(compile) => compile.execute(), - #[cfg(feature = "compiler")] + #[cfg(any(feature = "static-artifact-create", feature = "wasmer-artifact-create"))] Self::CreateExe(create_exe) => create_exe.execute(), Self::Config(config) => config.execute(), Self::Inspect(inspect) => inspect.execute(), diff --git a/lib/cli/src/commands.rs b/lib/cli/src/commands.rs index 3313370f2b2..01066a5018c 100644 --- a/lib/cli/src/commands.rs +++ b/lib/cli/src/commands.rs @@ -5,7 +5,7 @@ mod cache; #[cfg(feature = "compiler")] mod compile; mod config; -#[cfg(feature = "compiler")] +#[cfg(any(feature = "static-artifact-create", feature = "wasmer-artifact-create"))] mod create_exe; mod inspect; mod run; @@ -18,7 +18,7 @@ mod wast; pub use binfmt::*; #[cfg(feature = "compiler")] pub use compile::*; -#[cfg(feature = "compiler")] +#[cfg(any(feature = "static-artifact-create", feature = "wasmer-artifact-create"))] pub use create_exe::*; #[cfg(feature = "wast")] pub use wast::*; diff --git a/lib/compiler/Cargo.toml b/lib/compiler/Cargo.toml index 94e9b73601e..75624058775 100644 --- a/lib/compiler/Cargo.toml +++ b/lib/compiler/Cargo.toml @@ -12,6 +12,7 @@ edition = "2018" [dependencies] wasmer-types = { path = "../types", version = "=3.0.0-alpha.4", default-features = false } +wasmer-object = { path = "../object", version = "=3.0.0-alpha.4", optional = true } wasmparser = { version = "0.83", optional = true, default-features = false } enumset = "1.0.2" hashbrown = { version = "0.11", optional = true } @@ -47,7 +48,7 @@ compiler = ["translator"] wasmer-artifact-load = [] wasmer-artifact-create = [] static-artifact-load = [] -static-artifact-create = [] +static-artifact-create = ["wasmer-object"] std = ["wasmer-types/std"] core = ["hashbrown", "wasmer-types/core"] enable-serde = ["serde", "serde_bytes", "wasmer-types/enable-serde"] diff --git a/lib/compiler/src/engine/artifact.rs b/lib/compiler/src/engine/artifact.rs index 876b8957792..46ba30f560a 100644 --- a/lib/compiler/src/engine/artifact.rs +++ b/lib/compiler/src/engine/artifact.rs @@ -5,18 +5,24 @@ use crate::engine::link::link_module; use crate::ArtifactBuild; use crate::ArtifactCreate; use crate::Features; -#[cfg(feature = "compiler")] -use crate::ModuleEnvironment; +#[cfg(feature = "static-artifact-create")] +use crate::{FunctionBodyData, Compiler, ModuleTranslationState}; use crate::{ register_frame_info, resolve_imports, FunctionExtent, GlobalFrameInfoRegistration, InstantiationError, RuntimeError, Tunables, }; use crate::{Engine, EngineInner}; +use crate::ModuleEnvironment; use enumset::EnumSet; use std::sync::Arc; use std::sync::Mutex; +#[cfg(feature = "static-artifact-create")] +use wasmer_object::{emit_compilation, emit_data, get_object_for_target, Object}; use wasmer_types::entity::{BoxedSlice, PrimaryMap}; use wasmer_types::MetadataHeader; +#[cfg(feature = "static-artifact-create")] +#[cfg(feature = "compiler")] +use wasmer_types::{compilation::symbols::ModuleMetadata, CompileModuleInfo, Target}; use wasmer_types::{ CompileError, CpuFeature, DataInitializer, DeserializeError, FunctionIndex, LocalFunctionIndex, MemoryIndex, ModuleInfo, OwnedDataInitializer, SerializableModule, SerializeError, @@ -378,4 +384,145 @@ impl Artifact { .finish_instantiation(trap_handler, &data_initializers) .map_err(|trap| InstantiationError::Start(RuntimeError::from_trap(trap))) } + + #[cfg(feature = "static-artifact-create")] + /// Generate a compilation + fn generate_metadata<'data>( + &self, + data: &'data [u8], + compiler: &dyn Compiler, + tunables: &dyn Tunables, + ) -> Result< + ( + CompileModuleInfo, + PrimaryMap>, + Vec>, + Option, + ), + CompileError, + > { + let environ = ModuleEnvironment::new(); + let translation = environ.translate(data).map_err(CompileError::Wasm)?; + let features = self.features().clone(); + + // We try to apply the middleware first + use crate::translator::ModuleMiddlewareChain; + let mut module = translation.module; + let middlewares = compiler.get_middlewares(); + middlewares.apply_on_module_info(&mut module); + + let memory_styles: PrimaryMap = module + .memories + .values() + .map(|memory_type| tunables.memory_style(memory_type)) + .collect(); + let table_styles: PrimaryMap = module + .tables + .values() + .map(|table_type| tunables.table_style(table_type)) + .collect(); + + let compile_info = CompileModuleInfo { + module, + features, + memory_styles, + table_styles, + }; + Ok(( + compile_info, + translation.function_body_inputs, + translation.data_initializers, + translation.module_translation_state, + )) + } + + /// Compile a module into an object file, which can be statically linked against. + /// + /// The `prefixer` returns the a String to prefix each of the + /// functions in the static object generated by the + /// so we can assure no collisions. + #[cfg(feature = "static-artifact-create")] + pub fn generate_object<'data>( + &self, + compiler: &dyn Compiler, + data: &[u8], + prefixer: Option String + Send>>, + target: &'data Target, + tunables: &dyn Tunables, + ) -> Result, CompileError> { + fn to_compile_error(err: impl std::error::Error) -> CompileError { + CompileError::Codegen(format!("{}", err)) + } + + #[allow(dead_code)] + const WASMER_METADATA_SYMBOL: &[u8] = b"WASMER_METADATA"; + + let (compile_info, function_body_inputs, data_initializers, module_translation) = + self.generate_metadata(data, compiler, tunables)?; + + let data_initializers = data_initializers + .iter() + .map(OwnedDataInitializer::new) + .collect::>() + .into_boxed_slice(); + + let target_triple = target.triple(); + + // TODO: we currently supply all-zero function body lengths. + // We don't know the lengths until they're compiled, yet we have to + // supply the metadata as an input to the compile. + let function_body_lengths = function_body_inputs + .keys() + .map(|_function_body| 0u64) + .collect::>(); + + let mut metadata = ModuleMetadata { + compile_info, + prefix: prefixer.as_ref().map(|p| p(data)).unwrap_or_default(), + data_initializers, + function_body_lengths, + cpu_features: target.cpu_features().as_u64(), + }; + + /* + In the C file we need: + - imports + - exports + + to construct an api::Module which is a Store (can be passed in via argument) and an + Arc which means this struct which includes: + - CompileModuleInfo + - Features + - ModuleInfo + - MemoryIndex -> MemoryStyle + - TableIndex -> TableStyle + - LocalFunctionIndex -> FunctionBodyPtr // finished functions + - FunctionIndex -> FunctionBodyPtr // finished dynamic function trampolines + - SignatureIndex -> VMSharedSignatureindextureIndex // signatures + */ + + let serialized_data = metadata.serialize().map_err(to_compile_error)?; + let mut metadata_binary = vec![]; + metadata_binary.extend(MetadataHeader::new(serialized_data.len()).into_bytes()); + metadata_binary.extend(serialized_data); + + let (_compile_info, symbol_registry) = metadata.split(); + + let compilation = compiler.compile_module( + &target, + &metadata.compile_info, + module_translation.as_ref().unwrap(), + function_body_inputs, + )?; + + let mut obj = get_object_for_target(&target_triple).map_err(to_compile_error)?; + + emit_data(&mut obj, b"WASMER_MODULE_METADATA", &metadata_binary, 1) + .map_err(to_compile_error)?; + + emit_compilation(&mut obj, compilation, &symbol_registry, &target_triple) + .map_err(to_compile_error)?; + + Ok(obj) + } } diff --git a/lib/object/Cargo.toml b/lib/object/Cargo.toml index 6dd1f05e9bb..f4da47654e8 100644 --- a/lib/object/Cargo.toml +++ b/lib/object/Cargo.toml @@ -12,9 +12,5 @@ edition = "2018" [dependencies] wasmer-types = { path = "../types", version = "=3.0.0-alpha.4" } -wasmer-compiler = { path = "../compiler", version = "=3.0.0-alpha.4", default-features = false, features = [ - "std", - "translator", -] } object = { version = "0.28.3", default-features = false, features = ["write"] } thiserror = "1.0" From 83a97f5533d9cc5ac204dc7b27cb06c58d40c92a Mon Sep 17 00:00:00 2001 From: Manos Pitsidianakis Date: Wed, 13 Jul 2022 23:25:19 +0300 Subject: [PATCH 4/4] cli: add create-obj command lib/compiler: read static object --- Cargo.toml | 4 + Makefile | 4 +- fuzz/Cargo.toml | 4 + lib/api/Cargo.toml | 5 + lib/c-api/Cargo.toml | 4 + lib/cli/Cargo.toml | 25 +- lib/cli/src/c_gen/mod.rs | 550 ++++++++++++++++++ lib/cli/src/c_gen/staticlib_header.rs | 298 ++++++++++ lib/cli/src/cli.rs | 9 + lib/cli/src/commands.rs | 27 + lib/cli/src/commands/create_exe.rs | 140 ++++- lib/cli/src/commands/create_obj.rs | 193 ++++++ .../commands/wasmer_static_create_exe_main.c | 192 ++++++ lib/cli/src/lib.rs | 1 + lib/compiler/src/engine/artifact.rs | 347 +++++++++-- lib/types/src/compilation/symbols.rs | 16 +- lib/types/src/error.rs | 2 +- 17 files changed, 1734 insertions(+), 87 deletions(-) create mode 100644 lib/cli/src/c_gen/mod.rs create mode 100644 lib/cli/src/c_gen/staticlib_header.rs create mode 100644 lib/cli/src/commands/create_obj.rs create mode 100644 lib/cli/src/commands/wasmer_static_create_exe_main.c diff --git a/Cargo.toml b/Cargo.toml index 47b3265dd04..a138288584c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -106,6 +106,10 @@ singlepass = ["wasmer-compiler-singlepass", "compiler"] cranelift = ["wasmer-compiler-cranelift", "compiler"] llvm = ["wasmer-compiler-llvm", "compiler"] middlewares = ["wasmer-middlewares"] +wasmer-artifact-load = ["wasmer-compiler/wasmer-artifact-load"] +wasmer-artifact-create = ["wasmer-compiler/wasmer-artifact-create"] +static-artifact-load = ["wasmer-compiler/static-artifact-load"] +static-artifact-create = ["wasmer-compiler/static-artifact-create"] # Testing features test-singlepass = ["singlepass"] diff --git a/Makefile b/Makefile index e99f00317c2..ed45ee807ee 100644 --- a/Makefile +++ b/Makefile @@ -254,13 +254,13 @@ space := $() $() comma := , # Define the compiler Cargo features for all crates. -compiler_features := --features $(subst $(space),$(comma),$(compilers)) +compiler_features := --features $(subst $(space),$(comma),$(compilers)),wasmer-artifact-create,static-artifact-create,wasmer-artifact-load,static-artifact-load capi_compilers_engines_exclude := # Define the compiler Cargo features for the C API. It always excludes # LLVM for the moment because it causes the linker to fail since LLVM is not statically linked. # TODO: Reenable LLVM in C-API -capi_compiler_features := --features $(subst $(space),$(comma),$(filter-out llvm, $(compilers))) +capi_compiler_features := --features $(subst $(space),$(comma),$(filter-out llvm, $(compilers))),wasmer-artifact-create,static-artifact-create,wasmer-artifact-load,static-artifact-load capi_compilers_engines_exclude += llvm-universal # We exclude singlepass-universal because it doesn't support multivalue (required in wasm-c-api tests) diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 7b2ce822057..d4a2e6a4a3d 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -25,6 +25,10 @@ cranelift = [ "wasmer-compiler-cranelift" ] llvm = [ "wasmer-compiler-llvm" ] singlepass = [ "wasmer-compiler-singlepass" ] universal = [ "wasmer-compiler" ] +wasmer-artifact-load = ["wasmer-compiler/wasmer-artifact-load"] +wasmer-artifact-create = ["wasmer-compiler/wasmer-artifact-create"] +static-artifact-load = ["wasmer-compiler/static-artifact-load"] +static-artifact-create = ["wasmer-compiler/static-artifact-create"] [[bin]] name = "equivalence_universal" diff --git a/lib/api/Cargo.toml b/lib/api/Cargo.toml index aae032f3400..737c7d01baf 100644 --- a/lib/api/Cargo.toml +++ b/lib/api/Cargo.toml @@ -120,5 +120,10 @@ enable-serde = [ "wasmer-types/enable-serde", ] +wasmer-artifact-load = ["wasmer-compiler/wasmer-artifact-load"] +wasmer-artifact-create = ["wasmer-compiler/wasmer-artifact-create"] +static-artifact-load = ["wasmer-compiler/static-artifact-load"] +static-artifact-create = ["wasmer-compiler/static-artifact-create"] + [package.metadata.docs.rs] features = ["compiler", "core", "cranelift", "engine", "jit", "native", "singlepass", "sys", "sys-default" ] diff --git a/lib/c-api/Cargo.toml b/lib/c-api/Cargo.toml index db43f1412c6..2a4a78e62e2 100644 --- a/lib/c-api/Cargo.toml +++ b/lib/c-api/Cargo.toml @@ -79,6 +79,10 @@ llvm = [ "wasmer-compiler-llvm", "compiler", ] +wasmer-artifact-load = ["wasmer-compiler/wasmer-artifact-load"] +wasmer-artifact-create = ["wasmer-compiler/wasmer-artifact-create"] +static-artifact-load = ["wasmer-compiler/static-artifact-load"] +static-artifact-create = ["wasmer-compiler/static-artifact-create"] # Deprecated features. jit = ["compiler"] diff --git a/lib/cli/Cargo.toml b/lib/cli/Cargo.toml index ea4e1e365da..3b206ba90ab 100644 --- a/lib/cli/Cargo.toml +++ b/lib/cli/Cargo.toml @@ -67,6 +67,7 @@ default = [ "emscripten", "compiler", "wasmer-artifact-create", + "static-artifact-create", ] cache = ["wasmer-cache"] cache-blake3-pure = ["wasmer-cache/blake3-pure"] @@ -78,8 +79,28 @@ compiler = [ "wasmer-compiler/translator", "wasmer-compiler/compiler", ] -wasmer-artifact-create = ["compiler", "wasmer-compiler/wasmer-artifact-create", "wasmer-object"] -static-artifact-create = ["compiler", "wasmer-compiler/static-artifact-create", "wasmer-object"] +wasmer-artifact-create = ["compiler", + "wasmer/wasmer-artifact-load", + "wasmer/wasmer-artifact-create", + "wasmer-compiler/wasmer-artifact-load", + "wasmer-compiler/wasmer-artifact-create", + "wasmer-object", + ] +static-artifact-create = ["compiler", + "wasmer/static-artifact-load", + "wasmer/static-artifact-create", + "wasmer-compiler/static-artifact-load", + "wasmer-compiler/static-artifact-create", + "wasmer-object", + ] +wasmer-artifact-load = ["compiler", + "wasmer/wasmer-artifact-load", + "wasmer-compiler/wasmer-artifact-load", + ] +static-artifact-load = ["compiler", + "wasmer/static-artifact-load", + "wasmer-compiler/static-artifact-load", + ] experimental-io-devices = [ "wasmer-wasi-experimental-io-devices", diff --git a/lib/cli/src/c_gen/mod.rs b/lib/cli/src/c_gen/mod.rs new file mode 100644 index 00000000000..addc30a8c76 --- /dev/null +++ b/lib/cli/src/c_gen/mod.rs @@ -0,0 +1,550 @@ +//! A convenient little abstraction for building up C expressions and generating +//! simple C code. + +pub mod staticlib_header; + +/// An identifier in C. +pub type CIdent = String; + +/// A Type in the C language. +#[derive(Debug, Clone)] +pub enum CType { + /// C `void` type. + Void, + /// A pointer to some other type. + PointerTo { + /// Whether the pointer is `const`. + is_const: bool, + /// The type that the pointer points to. + inner: Box, + }, + /// C 8 bit unsigned integer type. + U8, + /// C 16 bit unsigned integer type. + U16, + /// C 32 bit unsigned integer type. + U32, + /// C 64 bit unsigned integer type. + U64, + /// C pointer sized unsigned integer type. + USize, + /// C 8 bit signed integer type. + I8, + /// C 16 bit signed integer type. + I16, + /// C 32 bit signed integer type. + I32, + /// C 64 bit signed integer type. + I64, + /// C pointer sized signed integer type. + ISize, + /// A function or function pointer. + Function { + /// The arguments the function takes. + arguments: Vec, + /// The return value if it has one + /// + /// None is equivalent to Some(Box(Ctype::Void)). + return_value: Option>, + }, + /// C constant array. + Array { + /// The type of the array. + inner: Box, + }, + /// A user defined type. + TypeDef(String), +} + +impl CType { + /// Convenience function to get a mutable void pointer type. + pub fn void_ptr() -> Self { + CType::PointerTo { + is_const: false, + inner: Box::new(CType::Void), + } + } + + /// Convenience function to get a const void pointer type. + #[allow(dead_code)] + pub fn const_void_ptr() -> Self { + CType::PointerTo { + is_const: true, + inner: Box::new(CType::Void), + } + } + + /// Generate the C source code for a type into the given `String`. + fn generate_c(&self, w: &mut String) { + match &self { + Self::Void => { + w.push_str("void"); + } + Self::PointerTo { is_const, inner } => { + if *is_const { + w.push_str("const "); + } + inner.generate_c(w); + w.push('*'); + } + Self::U8 => { + w.push_str("unsigned char"); + } + Self::U16 => { + w.push_str("unsigned short"); + } + Self::U32 => { + w.push_str("unsigned int"); + } + Self::U64 => { + w.push_str("unsigned long long"); + } + Self::USize => { + w.push_str("unsigned size_t"); + } + Self::I8 => { + w.push_str("char"); + } + Self::I16 => { + w.push_str("short"); + } + Self::I32 => { + w.push_str("int"); + } + Self::I64 => { + w.push_str("long long"); + } + Self::ISize => { + w.push_str("size_t"); + } + Self::Function { + arguments, + return_value, + } => { + // function with no, name, assume it's a function pointer + #[allow(clippy::borrowed_box)] + let ret: CType = return_value + .as_ref() + .map(|i: &Box| (&**i).clone()) + .unwrap_or_default(); + ret.generate_c(w); + w.push(' '); + w.push_str("(*)"); + w.push('('); + match arguments.len() { + l if l > 1 => { + for arg in &arguments[..arguments.len() - 1] { + arg.generate_c(w); + w.push_str(", "); + } + arguments.last().unwrap().generate_c(w); + } + 1 => { + arguments[0].generate_c(w); + } + _ => {} + } + w.push(')'); + } + Self::Array { inner } => { + inner.generate_c(w); + w.push_str("[]"); + } + Self::TypeDef(inner) => { + w.push_str(inner); + } + } + } + + /// Generate the C source code for a type with a nameinto the given `String`. + fn generate_c_with_name(&self, name: &str, w: &mut String) { + match &self { + Self::PointerTo { .. } + | Self::TypeDef { .. } + | Self::Void + | Self::U8 + | Self::U16 + | Self::U32 + | Self::U64 + | Self::USize + | Self::I8 + | Self::I16 + | Self::I32 + | Self::I64 + | Self::ISize => { + self.generate_c(w); + w.push(' '); + w.push_str(name); + } + Self::Function { + arguments, + return_value, + } => { + #[allow(clippy::borrowed_box)] + let ret: CType = return_value + .as_ref() + .map(|i: &Box| (&**i).clone()) + .unwrap_or_default(); + ret.generate_c(w); + w.push(' '); + w.push_str(name); + w.push('('); + match arguments.len() { + l if l > 1 => { + for arg in &arguments[..arguments.len() - 1] { + arg.generate_c(w); + w.push_str(", "); + } + arguments.last().unwrap().generate_c(w); + } + 1 => { + arguments[0].generate_c(w); + } + _ => {} + } + w.push(')'); + } + Self::Array { inner } => { + inner.generate_c(w); + w.push(' '); + w.push_str(name); + w.push_str("[]"); + } + } + } +} + +impl Default for CType { + fn default() -> CType { + CType::Void + } +} + +/// A statement in the C programming language. This may not be exact to what an +/// AST would look like or what the C standard says about the C language, it's +/// simply a structed way to organize data for generating C code. +#[derive(Debug, Clone)] +pub enum CStatement { + /// A declaration of some kind. + Declaration { + /// The name of the thing being declared. + name: CIdent, + /// Whether the thing being declared is `extern`. + is_extern: bool, + /// Whether the thing being declared is `const`. + is_const: bool, + /// The type of the thing being declared. + ctype: CType, + /// The definition of the thing being declared. + /// + /// This is useful for initializing constant arrays, for example. + definition: Option>, + }, + + /// A literal array of CStatements. + LiteralArray { + /// The contents of the array. + items: Vec, + }, + + /// A literal constant value, passed through directly as a string. + LiteralConstant { + /// The raw value acting as a constant. + value: String, + }, + + /// A C-style cast + Cast { + /// The type to cast to. + target_type: CType, + /// The thing being cast. + expression: Box, + }, + + /// Typedef one type to another. + TypeDef { + /// The type of the thing being typedef'd. + source_type: CType, + /// The new name by which this type may be called. + new_name: CIdent, + }, +} + +impl CStatement { + /// Generate C source code for the given CStatement. + fn generate_c(&self, w: &mut String) { + match &self { + Self::Declaration { + name, + is_extern, + is_const, + ctype, + definition, + } => { + if *is_const { + w.push_str("const "); + } + if *is_extern { + w.push_str("extern "); + } + ctype.generate_c_with_name(name, w); + if let Some(def) = definition { + w.push_str(" = "); + def.generate_c(w); + } + w.push(';'); + w.push('\n'); + } + Self::LiteralArray { items } => { + w.push('{'); + if !items.is_empty() { + w.push('\n'); + } + for item in items { + w.push('\t'); + item.generate_c(w); + w.push(','); + w.push('\n'); + } + w.push('}'); + } + Self::LiteralConstant { value } => { + w.push_str(value); + } + Self::Cast { + target_type, + expression, + } => { + w.push('('); + target_type.generate_c(w); + w.push(')'); + w.push(' '); + expression.generate_c(w); + } + Self::TypeDef { + source_type, + new_name, + } => { + w.push_str("typedef "); + // leaky abstraction / hack, doesn't fully solve the problem + if let CType::Function { .. } = source_type { + source_type.generate_c_with_name(&format!("(*{})", new_name), w); + } else { + source_type.generate_c(w); + w.push(' '); + w.push_str(new_name); + } + w.push(';'); + w.push('\n'); + } + } + } +} + +/// Generate C source code from some `CStatements` into a String. +// TODO: add config section +pub fn generate_c(statements: &[CStatement]) -> String { + let mut out = String::new(); + for statement in statements { + statement.generate_c(&mut out); + } + out +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn generate_types() { + macro_rules! assert_c_type { + ($ctype:expr, $expected:expr) => { + let mut w = String::new(); + let ctype = $ctype; + ctype.generate_c(&mut w); + assert_eq!(w, $expected); + }; + } + + assert_c_type!(CType::Void, "void"); + assert_c_type!(CType::void_ptr(), "void*"); + assert_c_type!(CType::const_void_ptr(), "const void*"); + assert_c_type!(CType::U8, "unsigned char"); + assert_c_type!(CType::U16, "unsigned short"); + assert_c_type!(CType::U32, "unsigned int"); + assert_c_type!(CType::U64, "unsigned long long"); + assert_c_type!(CType::USize, "unsigned size_t"); + assert_c_type!(CType::I8, "char"); + assert_c_type!(CType::I16, "short"); + assert_c_type!(CType::I32, "int"); + assert_c_type!(CType::I64, "long long"); + assert_c_type!(CType::ISize, "size_t"); + assert_c_type!(CType::TypeDef("my_type".to_string()), "my_type"); + assert_c_type!( + CType::Function { + arguments: vec![CType::U8, CType::ISize], + return_value: None + }, + "void (*)(unsigned char, size_t)" + ); + assert_c_type!( + CType::Function { + arguments: vec![], + return_value: Some(Box::new(CType::ISize)) + }, + "size_t (*)()" + ); + assert_c_type!( + CType::PointerTo { + is_const: true, + inner: Box::new(CType::PointerTo { + is_const: false, + inner: Box::new(CType::U32) + }) + }, + "const unsigned int**" + ); + // TODO: test more complicated const correctness rules: there are bugs relating to it. + } + + #[test] + fn generate_types_with_names() { + macro_rules! assert_c_type { + ($ctype:expr, $name:literal, $expected:expr) => { + let mut w = String::new(); + let ctype = $ctype; + ctype.generate_c_with_name($name, &mut w); + assert_eq!(w, $expected); + }; + } + + assert_c_type!(CType::Void, "main", "void main"); + assert_c_type!(CType::void_ptr(), "data", "void* data"); + assert_c_type!(CType::const_void_ptr(), "data", "const void* data"); + assert_c_type!(CType::U8, "data", "unsigned char data"); + assert_c_type!(CType::U16, "data", "unsigned short data"); + assert_c_type!(CType::U32, "data", "unsigned int data"); + assert_c_type!(CType::U64, "data", "unsigned long long data"); + assert_c_type!(CType::USize, "data", "unsigned size_t data"); + assert_c_type!(CType::I8, "data", "char data"); + assert_c_type!(CType::I16, "data", "short data"); + assert_c_type!(CType::I32, "data", "int data"); + assert_c_type!(CType::I64, "data", "long long data"); + assert_c_type!(CType::ISize, "data", "size_t data"); + assert_c_type!( + CType::TypeDef("my_type".to_string()), + "data", + "my_type data" + ); + assert_c_type!( + CType::Function { + arguments: vec![CType::U8, CType::ISize], + return_value: None + }, + "my_func", + "void my_func(unsigned char, size_t)" + ); + assert_c_type!( + CType::Function { + arguments: vec![], + return_value: Some(Box::new(CType::ISize)) + }, + "my_func", + "size_t my_func()" + ); + assert_c_type!( + CType::PointerTo { + is_const: true, + inner: Box::new(CType::PointerTo { + is_const: false, + inner: Box::new(CType::U32) + }) + }, + "data", + "const unsigned int** data" + ); + // TODO: test more complicated const correctness rules: there are bugs relating to it. + } + + #[test] + fn generate_expressions_works() { + macro_rules! assert_c_expr { + ($cexpr:expr, $expected:expr) => { + let mut w = String::new(); + let cexpr = $cexpr; + cexpr.generate_c(&mut w); + assert_eq!(w, $expected); + }; + } + + assert_c_expr!( + CStatement::LiteralConstant { + value: "\"Hello, world!\"".to_string() + }, + "\"Hello, world!\"" + ); + assert_c_expr!( + CStatement::TypeDef { + source_type: CType::Function { + arguments: vec![CType::I32, CType::I32], + return_value: None, + }, + new_name: "my_func_ptr".to_string(), + }, + "typedef void (*my_func_ptr)(int, int);\n" + ); + assert_c_expr!( + CStatement::LiteralArray { + items: vec![ + CStatement::LiteralConstant { + value: "1".to_string() + }, + CStatement::LiteralConstant { + value: "2".to_string() + }, + CStatement::LiteralConstant { + value: "3".to_string() + }, + ] + }, + "{\n\t1,\n\t2,\n\t3,\n}" + ); + assert_c_expr!(CStatement::LiteralArray { items: vec![] }, "{}"); + assert_c_expr!( + CStatement::Declaration { + name: "my_array".to_string(), + is_extern: false, + is_const: true, + ctype: CType::Array { + inner: Box::new(CType::I32) + }, + definition: Some(Box::new(CStatement::LiteralArray { + items: vec![ + CStatement::LiteralConstant { + value: "1".to_string() + }, + CStatement::LiteralConstant { + value: "2".to_string() + }, + CStatement::LiteralConstant { + value: "3".to_string() + }, + ] + })) + }, + "const int my_array[] = {\n\t1,\n\t2,\n\t3,\n};\n" + ); + assert_c_expr!( + CStatement::Declaration { + name: "my_array".to_string(), + is_extern: true, + is_const: true, + ctype: CType::Array { + inner: Box::new(CType::I32) + }, + definition: None, + }, + "const extern int my_array[];\n" + ); + } +} diff --git a/lib/cli/src/c_gen/staticlib_header.rs b/lib/cli/src/c_gen/staticlib_header.rs new file mode 100644 index 00000000000..bd203c644b8 --- /dev/null +++ b/lib/cli/src/c_gen/staticlib_header.rs @@ -0,0 +1,298 @@ +//! Generate a header file for the static object file produced. + +use super::{generate_c, CStatement, CType}; +use wasmer_types::ModuleInfo; +use wasmer_types::{Symbol, SymbolRegistry}; + +/// Helper functions to simplify the usage of the static artifact. +const HELPER_FUNCTIONS: &str = r#" +wasm_byte_vec_t generate_serialized_data() { + // We need to pass all the bytes as one big buffer so we have to do all this logic to memcpy + // the various pieces together from the generated header file. + // + // We should provide a `deseralize_vectored` function to avoid requiring this extra work. + + char* byte_ptr = (char*)&WASMER_METADATA[0]; + + size_t num_function_pointers + = sizeof(function_pointers) / sizeof(void*); + size_t num_function_trampolines + = sizeof(function_trampolines) / sizeof(void*); + size_t num_dynamic_function_trampoline_pointers + = sizeof(dynamic_function_trampoline_pointers) / sizeof(void*); + + + size_t buffer_size = module_bytes_len + + sizeof(size_t) + sizeof(function_pointers) + + sizeof(size_t) + sizeof(function_trampolines) + + sizeof(size_t) + sizeof(dynamic_function_trampoline_pointers); + + char* memory_buffer = (char*) malloc(buffer_size); + size_t current_offset = 0; + + memcpy(memory_buffer + current_offset, byte_ptr, module_bytes_len); + current_offset += module_bytes_len; + + memcpy(memory_buffer + current_offset, (void*)&num_function_pointers, sizeof(size_t)); + current_offset += sizeof(size_t); + + memcpy(memory_buffer + current_offset, (void*)&function_pointers[0], sizeof(function_pointers)); + current_offset += sizeof(function_pointers); + + memcpy(memory_buffer + current_offset, (void*)&num_function_trampolines, sizeof(size_t)); + current_offset += sizeof(size_t); + + memcpy(memory_buffer + current_offset, (void*)&function_trampolines[0], sizeof(function_trampolines)); + current_offset += sizeof(function_trampolines); + + memcpy(memory_buffer + current_offset, (void*)&num_dynamic_function_trampoline_pointers, sizeof(size_t)); + current_offset += sizeof(size_t); + + memcpy(memory_buffer + current_offset, (void*)&dynamic_function_trampoline_pointers[0], sizeof(dynamic_function_trampoline_pointers)); + current_offset += sizeof(dynamic_function_trampoline_pointers); + + wasm_byte_vec_t module_byte_vec = { + .size = buffer_size, + .data = memory_buffer, + }; + return module_byte_vec; +} + +wasm_module_t* wasmer_static_module_new(wasm_store_t* store, const char* wasm_name) { + // wasm_name intentionally unused for now: will be used in the future. + wasm_byte_vec_t module_byte_vec = generate_serialized_data(); + wasm_module_t* module = wasm_module_deserialize(store, &module_byte_vec); + free(module_byte_vec.data); + + return module; +} +"#; + +/// Generate the header file that goes with the generated object file. +pub fn generate_header_file( + module_info: &ModuleInfo, + symbol_registry: &dyn SymbolRegistry, + metadata_length: usize, +) -> String { + let mut c_statements = vec![ + CStatement::LiteralConstant { + value: "#include \"wasmer.h\"\n#include \n#include \n\n" + .to_string(), + }, + CStatement::LiteralConstant { + value: "#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n".to_string(), + }, + CStatement::Declaration { + name: "module_bytes_len".to_string(), + is_extern: false, + is_const: true, + ctype: CType::U32, + definition: Some(Box::new(CStatement::LiteralConstant { + value: metadata_length.to_string(), + })), + }, + CStatement::Declaration { + name: "WASMER_METADATA".to_string(), + is_extern: true, + is_const: true, + ctype: CType::Array { + inner: Box::new(CType::U8), + }, + definition: None, + }, + ]; + let function_declarations = module_info + .functions + .iter() + .filter_map(|(f_index, sig_index)| { + Some((module_info.local_func_index(f_index)?, sig_index)) + }) + .map(|(function_local_index, _sig_index)| { + let function_name = + symbol_registry.symbol_to_name(Symbol::LocalFunction(function_local_index)); + // TODO: figure out the signature here too + CStatement::Declaration { + name: function_name, + is_extern: true, + is_const: false, + ctype: CType::Function { + arguments: vec![CType::Void], + return_value: None, + }, + definition: None, + } + }); + c_statements.push(CStatement::LiteralConstant { + value: r#" +// Compiled Wasm function pointers ordered by function index: the order they +// appeared in in the Wasm module. +"# + .to_string(), + }); + c_statements.extend(function_declarations); + + // function pointer array + { + let function_pointer_array_statements = module_info + .functions + .iter() + .filter_map(|(f_index, sig_index)| { + Some((module_info.local_func_index(f_index)?, sig_index)) + }) + .map(|(function_local_index, _sig_index)| { + let function_name = + symbol_registry.symbol_to_name(Symbol::LocalFunction(function_local_index)); + // TODO: figure out the signature here too + + CStatement::Cast { + target_type: CType::void_ptr(), + expression: Box::new(CStatement::LiteralConstant { + value: function_name, + }), + } + }) + .collect::>(); + + c_statements.push(CStatement::Declaration { + name: "function_pointers".to_string(), + is_extern: false, + is_const: true, + ctype: CType::Array { + inner: Box::new(CType::void_ptr()), + }, + definition: Some(Box::new(CStatement::LiteralArray { + items: function_pointer_array_statements, + })), + }); + } + + let func_trampoline_declarations = + module_info + .signatures + .iter() + .map(|(sig_index, _func_type)| { + let function_name = + symbol_registry.symbol_to_name(Symbol::FunctionCallTrampoline(sig_index)); + + CStatement::Declaration { + name: function_name, + is_extern: true, + is_const: false, + ctype: CType::Function { + arguments: vec![CType::void_ptr(), CType::void_ptr(), CType::void_ptr()], + return_value: None, + }, + definition: None, + } + }); + c_statements.push(CStatement::LiteralConstant { + value: r#" +// Trampolines (functions by which we can call into Wasm) ordered by signature. +// There is 1 trampoline per function signature in the order they appear in +// the Wasm module. +"# + .to_string(), + }); + c_statements.extend(func_trampoline_declarations); + + // function trampolines + { + let function_trampoline_statements = module_info + .signatures + .iter() + .map(|(sig_index, _vm_shared_index)| { + let function_name = + symbol_registry.symbol_to_name(Symbol::FunctionCallTrampoline(sig_index)); + CStatement::LiteralConstant { + value: function_name, + } + }) + .collect::>(); + + c_statements.push(CStatement::Declaration { + name: "function_trampolines".to_string(), + is_extern: false, + is_const: true, + ctype: CType::Array { + inner: Box::new(CType::void_ptr()), + }, + definition: Some(Box::new(CStatement::LiteralArray { + items: function_trampoline_statements, + })), + }); + } + + let dyn_func_declarations = module_info + .functions + .keys() + .take(module_info.num_imported_functions) + .map(|func_index| { + let function_name = + symbol_registry.symbol_to_name(Symbol::DynamicFunctionTrampoline(func_index)); + // TODO: figure out the signature here + CStatement::Declaration { + name: function_name, + is_extern: true, + is_const: false, + ctype: CType::Function { + arguments: vec![CType::void_ptr(), CType::void_ptr(), CType::void_ptr()], + return_value: None, + }, + definition: None, + } + }); + c_statements.push(CStatement::LiteralConstant { + value: r#" +// Dynamic trampolines are per-function and are used for each function where +// the type signature is not known statically. In this case, this corresponds to +// the imported functions. +"# + .to_string(), + }); + c_statements.extend(dyn_func_declarations); + + c_statements.push(CStatement::TypeDef { + source_type: CType::Function { + arguments: vec![CType::void_ptr(), CType::void_ptr(), CType::void_ptr()], + return_value: None, + }, + new_name: "dyn_func_trampoline_t".to_string(), + }); + + // dynamic function trampoline pointer array + { + let dynamic_function_trampoline_statements = module_info + .functions + .keys() + .take(module_info.num_imported_functions) + .map(|func_index| { + let function_name = + symbol_registry.symbol_to_name(Symbol::DynamicFunctionTrampoline(func_index)); + CStatement::LiteralConstant { + value: function_name, + } + }) + .collect::>(); + c_statements.push(CStatement::Declaration { + name: "dynamic_function_trampoline_pointers".to_string(), + is_extern: false, + is_const: true, + ctype: CType::Array { + inner: Box::new(CType::TypeDef("dyn_func_trampoline_t".to_string())), + }, + definition: Some(Box::new(CStatement::LiteralArray { + items: dynamic_function_trampoline_statements, + })), + }); + } + + c_statements.push(CStatement::LiteralConstant { + value: HELPER_FUNCTIONS.to_string(), + }); + + c_statements.push(CStatement::LiteralConstant { + value: "\n#ifdef __cplusplus\n}\n#endif\n\n".to_string(), + }); + + generate_c(&c_statements) +} diff --git a/lib/cli/src/cli.rs b/lib/cli/src/cli.rs index 3587dba5a93..d82120a7d2f 100644 --- a/lib/cli/src/cli.rs +++ b/lib/cli/src/cli.rs @@ -6,6 +6,8 @@ use crate::commands::Binfmt; use crate::commands::Compile; #[cfg(any(feature = "static-artifact-create", feature = "wasmer-artifact-create"))] use crate::commands::CreateExe; +#[cfg(feature = "static-artifact-create")] +use crate::commands::CreateObj; #[cfg(feature = "wast")] use crate::commands::Wast; use crate::commands::{Cache, Config, Inspect, Run, SelfUpdate, Validate}; @@ -51,6 +53,11 @@ enum WasmerCLIOptions { #[structopt(name = "create-exe")] CreateExe(CreateExe), + /// Compile a WebAssembly binary into an object file + #[cfg(feature = "static-artifact-create")] + #[structopt(name = "create-obj")] + CreateObj(CreateObj), + /// Get various configuration information needed /// to compile programs which use Wasmer #[structopt(name = "config")] @@ -86,6 +93,8 @@ impl WasmerCLIOptions { Self::Compile(compile) => compile.execute(), #[cfg(any(feature = "static-artifact-create", feature = "wasmer-artifact-create"))] Self::CreateExe(create_exe) => create_exe.execute(), + #[cfg(feature = "static-artifact-create")] + Self::CreateObj(create_obj) => create_obj.execute(), Self::Config(config) => config.execute(), Self::Inspect(inspect) => inspect.execute(), #[cfg(feature = "wast")] diff --git a/lib/cli/src/commands.rs b/lib/cli/src/commands.rs index 01066a5018c..95cb1ad5d96 100644 --- a/lib/cli/src/commands.rs +++ b/lib/cli/src/commands.rs @@ -7,6 +7,8 @@ mod compile; mod config; #[cfg(any(feature = "static-artifact-create", feature = "wasmer-artifact-create"))] mod create_exe; +#[cfg(feature = "static-artifact-create")] +mod create_obj; mod inspect; mod run; mod self_update; @@ -20,6 +22,31 @@ pub use binfmt::*; pub use compile::*; #[cfg(any(feature = "static-artifact-create", feature = "wasmer-artifact-create"))] pub use create_exe::*; +#[cfg(feature = "static-artifact-create")] +pub use create_obj::*; #[cfg(feature = "wast")] pub use wast::*; pub use {cache::*, config::*, inspect::*, run::*, self_update::*, validate::*}; + +/// The kind of object format to emit. +#[derive(Debug, Copy, Clone, structopt::StructOpt)] +#[cfg(any(feature = "static-artifact-create", feature = "wasmer-artifact-create"))] +pub enum ObjectFormat { + /// Serialize the entire module into an object file. + Serialized, + /// Serialize only the module metadata into an object file and emit functions as symbols. + Symbols, +} + +#[cfg(any(feature = "static-artifact-create", feature = "wasmer-artifact-create"))] +impl std::str::FromStr for ObjectFormat { + type Err = &'static str; + + fn from_str(s: &str) -> Result { + match s { + "serialized" => Ok(Self::Serialized), + "symbols" => Ok(Self::Symbols), + _ => Err("must be one of two options: `serialized` or `symbols`."), + } + } +} diff --git a/lib/cli/src/commands/create_exe.rs b/lib/cli/src/commands/create_exe.rs index 1d8f468fe63..cf445016fc0 100644 --- a/lib/cli/src/commands/create_exe.rs +++ b/lib/cli/src/commands/create_exe.rs @@ -1,5 +1,6 @@ //! Create a standalone native executable for a given Wasm file. +use super::ObjectFormat; use crate::store::CompilerOptions; use anyhow::{Context, Result}; use std::env; @@ -14,6 +15,8 @@ use wasmer::*; use wasmer_object::{emit_serialized, get_object_for_target}; const WASMER_MAIN_C_SOURCE: &[u8] = include_bytes!("wasmer_create_exe_main.c"); +#[cfg(feature = "static-artifact-create")] +const WASMER_STATIC_MAIN_C_SOURCE: &[u8] = include_bytes!("wasmer_static_create_exe_main.c"); #[derive(Debug, StructOpt)] /// The options for the `wasmer create-exe` subcommand @@ -30,9 +33,9 @@ pub struct CreateExe { #[structopt(long = "target")] target_triple: Option, - #[structopt(flatten)] - compiler: CompilerOptions, - + /// Object format options + #[structopt(name = "OBJECT_FORMAT", long = "object-format")] + object_format: Option, #[structopt(short = "m", multiple = true, number_of_values = 1)] cpu_features: Vec, @@ -40,6 +43,9 @@ pub struct CreateExe { /// This is useful for fixing linker errors that may occur on some systems. #[structopt(short = "l", multiple = true, number_of_values = 1)] libraries: Vec, + + #[structopt(flatten)] + compiler: CompilerOptions, } impl CreateExe { @@ -61,9 +67,11 @@ impl CreateExe { }) .unwrap_or_default(); let (store, compiler_type) = self.compiler.get_store_for_target(target.clone())?; + let object_format = self.object_format.unwrap_or(ObjectFormat::Symbols); println!("Compiler: {}", compiler_type.to_string()); println!("Target: {}", target.triple()); + println!("Format: {:?}", object_format); let working_dir = tempfile::tempdir()?; let starting_cd = env::current_dir()?; @@ -77,19 +85,63 @@ impl CreateExe { let wasm_module_path = starting_cd.join(&self.path); - let module = - Module::from_file(&store, &wasm_module_path).context("failed to compile Wasm")?; - let bytes = module.serialize()?; - let mut obj = get_object_for_target(target.triple())?; - emit_serialized(&mut obj, &bytes, target.triple())?; - let mut writer = BufWriter::new(File::create(&wasm_object_path)?); - obj.write_stream(&mut writer) - .map_err(|err| anyhow::anyhow!(err.to_string()))?; - writer.flush()?; - drop(writer); - - self.compile_c(wasm_object_path, output_path)?; - + match object_format { + ObjectFormat::Serialized => { + let module = Module::from_file(&store, &wasm_module_path) + .context("failed to compile Wasm")?; + let bytes = module.serialize()?; + let mut obj = get_object_for_target(target.triple())?; + emit_serialized(&mut obj, &bytes, target.triple())?; + let mut writer = BufWriter::new(File::create(&wasm_object_path)?); + obj.write_stream(&mut writer) + .map_err(|err| anyhow::anyhow!(err.to_string()))?; + writer.flush()?; + drop(writer); + + self.compile_c(wasm_object_path, output_path)?; + } + #[cfg(not(feature = "static-artifact-create"))] + ObjectFormat::Symbols => { + return Err(anyhow!("This version of wasmer-cli hasn't been compiled with static artifact support. You need to enable the `static-artifact-create` feature during compilation.")); + } + #[cfg(feature = "static-artifact-create")] + ObjectFormat::Symbols => { + let engine = store.engine(); + let engine_inner = engine.inner(); + let compiler = engine_inner.compiler()?; + let features = engine_inner.features(); + let tunables = store.tunables(); + let data: Vec = fs::read(wasm_module_path)?; + let prefixer: Option String + Send>> = None; + let (module_info, obj, metadata_length, symbol_registry) = + Artifact::generate_object( + compiler, &data, prefixer, &target, tunables, features, + )?; + + let header_file_src = crate::c_gen::staticlib_header::generate_header_file( + &module_info, + &*symbol_registry, + metadata_length, + ); + /* Write object file with functions */ + let object_file_path: std::path::PathBuf = + std::path::Path::new("functions.o").into(); + let mut writer = BufWriter::new(File::create(&object_file_path)?); + obj.write_stream(&mut writer) + .map_err(|err| anyhow::anyhow!(err.to_string()))?; + writer.flush()?; + /* Write down header file that includes pointer arrays and the deserialize function + * */ + let mut writer = BufWriter::new(File::create("static_defs.h")?); + writer.write_all(header_file_src.as_bytes())?; + writer.flush()?; + link( + output_path, + object_file_path, + std::path::Path::new("static_defs.h").into(), + )?; + } + } eprintln!( "✔ Native executable compiled successfully to `{}`.", self.output.display(), @@ -130,6 +182,62 @@ impl CreateExe { } } +#[cfg(feature = "static-artifact-create")] +fn link( + output_path: PathBuf, + object_path: PathBuf, + header_code_path: PathBuf, +) -> anyhow::Result<()> { + let c_src_path = Path::new("wasmer_main.c"); + let mut libwasmer_path = get_libwasmer_path()? + .canonicalize() + .context("Failed to find libwasmer")?; + println!("Using libwasmer: {}", libwasmer_path.display()); + let lib_filename = libwasmer_path + .file_name() + .unwrap() + .to_str() + .unwrap() + .to_string(); + libwasmer_path.pop(); + { + let mut c_src_file = fs::OpenOptions::new() + .create_new(true) + .write(true) + .open(&c_src_path) + .context("Failed to open C source code file")?; + c_src_file.write_all(WASMER_STATIC_MAIN_C_SOURCE)?; + } + + println!( + "link output {:?}", + Command::new("cc") + .arg(&object_path) + .arg(&header_code_path) + .arg(&c_src_path) + .arg(&format!("-L{}", libwasmer_path.display())) + .arg(&format!("-I{}", get_wasmer_include_directory()?.display())) + .arg(&format!("-l:{}", lib_filename)) + // Add libraries required per platform. + // We need userenv, sockets (Ws2_32), advapi32 for some system calls and bcrypt for random numbers. + //#[cfg(windows)] + // .arg("-luserenv") + // .arg("-lWs2_32") + // .arg("-ladvapi32") + // .arg("-lbcrypt") + // On unix we need dlopen-related symbols, libmath for a few things, and pthreads. + //#[cfg(not(windows))] + .arg("-ldl") + .arg("-lm") + .arg("-pthread") + //.arg(&format!("-I{}", header_code_path.display())) + .arg("-o") + .arg(&output_path) + .output()? + ); + Ok(()) +} + fn get_wasmer_dir() -> anyhow::Result { Ok(PathBuf::from( env::var("WASMER_DIR") diff --git a/lib/cli/src/commands/create_obj.rs b/lib/cli/src/commands/create_obj.rs new file mode 100644 index 00000000000..64aff8b84c5 --- /dev/null +++ b/lib/cli/src/commands/create_obj.rs @@ -0,0 +1,193 @@ +#![allow(dead_code)] +//! Create a standalone native executable for a given Wasm file. + +use super::ObjectFormat; +use crate::store::CompilerOptions; +use anyhow::{Context, Result}; +use std::env; +use std::fs; +use std::fs::File; +use std::io::prelude::*; +use std::io::BufWriter; +use std::path::PathBuf; +use std::process::Command; +use structopt::StructOpt; +use wasmer::*; +use wasmer_object::{emit_serialized, get_object_for_target}; + +#[derive(Debug, StructOpt)] +/// The options for the `wasmer create-exe` subcommand +pub struct CreateObj { + /// Input file + #[structopt(name = "FILE", parse(from_os_str))] + path: PathBuf, + + /// Output file + #[structopt(name = "OUTPUT PATH", short = "o", parse(from_os_str))] + output: PathBuf, + + /// Compilation Target triple + #[structopt(long = "target")] + target_triple: Option, + + /// Object format options + #[structopt(name = "OBJECT_FORMAT", long = "object-format")] + object_format: Option, + + #[structopt(short = "m", multiple = true, number_of_values = 1)] + cpu_features: Vec, + + #[structopt(flatten)] + compiler: CompilerOptions, +} + +impl CreateObj { + /// Runs logic for the `create-obj` subcommand + pub fn execute(&self) -> Result<()> { + println!("objectformat: {:?}", &self.object_format); + let target = self + .target_triple + .as_ref() + .map(|target_triple| { + let mut features = self + .cpu_features + .clone() + .into_iter() + .fold(CpuFeature::set(), |a, b| a | b); + // Cranelift requires SSE2, so we have this "hack" for now to facilitate + // usage + features |= CpuFeature::SSE2; + Target::new(target_triple.clone(), features) + }) + .unwrap_or_default(); + let (store, compiler_type) = self.compiler.get_store_for_target(target.clone())?; + let object_format = self.object_format.unwrap_or(ObjectFormat::Symbols); + + println!("Compiler: {}", compiler_type.to_string()); + println!("Target: {}", target.triple()); + println!("Format: {:?}", object_format); + + let starting_cd = env::current_dir()?; + let output_path = starting_cd.join(&self.output); + + let wasm_module_path = starting_cd.join(&self.path); + + match object_format { + ObjectFormat::Serialized => { + let module = Module::from_file(&store, &wasm_module_path) + .context("failed to compile Wasm")?; + let bytes = module.serialize()?; + let mut obj = get_object_for_target(target.triple())?; + emit_serialized(&mut obj, &bytes, target.triple())?; + let mut writer = BufWriter::new(File::create(&output_path)?); + obj.write_stream(&mut writer) + .map_err(|err| anyhow::anyhow!(err.to_string()))?; + writer.flush()?; + } + ObjectFormat::Symbols => { + let engine = store.engine(); + let engine_inner = engine.inner(); + let compiler = engine_inner.compiler()?; + let features = engine_inner.features(); + let tunables = store.tunables(); + let data: Vec = fs::read(wasm_module_path)?; + let prefixer: Option String + Send>> = None; + let (module_info, obj, metadata_length, symbol_registry) = + Artifact::generate_object( + compiler, &data, prefixer, &target, tunables, features, + )?; + + let header_file_src = crate::c_gen::staticlib_header::generate_header_file( + &module_info, + &*symbol_registry, + metadata_length, + ); + let mut writer = BufWriter::new(File::create(&output_path)?); + obj.write_stream(&mut writer) + .map_err(|err| anyhow::anyhow!(err.to_string()))?; + writer.flush()?; + { + let mut writer = BufWriter::new(File::create("/tmp/main_obj.o")?); + obj.write_stream(&mut writer) + .map_err(|err| anyhow::anyhow!(err.to_string()))?; + writer.flush()?; + } + let mut writer = BufWriter::new(File::create("func.c")?); + writer.write_all(header_file_src.as_bytes())?; + writer.flush()?; + { + let mut writer = BufWriter::new(File::create("/tmp/func.c")?); + writer.write_all(header_file_src.as_bytes())?; + writer.flush()?; + } + //link(output_path.clone(), std::path::Path::new("func.c").into())?; + } + } + + eprintln!( + "✔ Object compiled successfully to `{}`.", + self.output.display(), + ); + + Ok(()) + } +} +fn link( + output_path: PathBuf, + object_path: PathBuf, + header_code_path: PathBuf, +) -> anyhow::Result<()> { + let libwasmer_path = get_libwasmer_path()? + .canonicalize() + .context("Failed to find libwasmer")?; + println!( + "link output {:?}", + Command::new("cc") + .arg(&header_code_path) + .arg(&format!("-L{}", libwasmer_path.display())) + //.arg(&format!("-I{}", header_code_path.display())) + .arg("-pie") + .arg("-o") + .arg("header_obj.o") + .output()? + ); + //ld -relocatable a.o b.o -o c.o + + println!( + "link output {:?}", + Command::new("ld") + .arg("-relocatable") + .arg(&object_path) + .arg("header_obj.o") + .arg("-o") + .arg(&output_path) + .output()? + ); + + Ok(()) +} + +/// path to the static libwasmer +fn get_libwasmer_path() -> anyhow::Result { + let mut path = get_wasmer_dir()?; + path.push("lib"); + + // TODO: prefer headless Wasmer if/when it's a separate library. + #[cfg(not(windows))] + path.push("libwasmer.a"); + #[cfg(windows)] + path.push("wasmer.lib"); + + Ok(path) +} +fn get_wasmer_dir() -> anyhow::Result { + Ok(PathBuf::from( + env::var("WASMER_DIR") + .or_else(|e| { + option_env!("WASMER_INSTALL_PREFIX") + .map(str::to_string) + .ok_or(e) + }) + .context("Trying to read env var `WASMER_DIR`")?, + )) +} diff --git a/lib/cli/src/commands/wasmer_static_create_exe_main.c b/lib/cli/src/commands/wasmer_static_create_exe_main.c new file mode 100644 index 00000000000..2276ed3bc75 --- /dev/null +++ b/lib/cli/src/commands/wasmer_static_create_exe_main.c @@ -0,0 +1,192 @@ +#include "wasmer.h" +#include "static_defs.h" +#include +#include +#include + +#define own + +// TODO: make this define templated so that the Rust code can toggle it on/off +#define WASI + +extern wasm_module_t* wasmer_module_new(wasm_store_t* store) asm("wasmer_module_new"); +extern wasm_module_t* wasmer_static_module_new(wasm_store_t* store,const char* wasm_name) asm("wasmer_static_module_new"); + + +static void print_wasmer_error() { + int error_len = wasmer_last_error_length(); + printf("Error len: `%d`\n", error_len); + char *error_str = (char *)malloc(error_len); + wasmer_last_error_message(error_str, error_len); + printf("%s\n", error_str); + free(error_str); +} + +#ifdef WASI +static void pass_mapdir_arg(wasi_config_t *wasi_config, char *mapdir) { + int colon_location = strchr(mapdir, ':') - mapdir; + if (colon_location == 0) { + // error malformed argument + fprintf(stderr, "Expected mapdir argument of the form alias:directory\n"); + exit(-1); + } + + char *alias = (char *)malloc(colon_location + 1); + memcpy(alias, mapdir, colon_location); + alias[colon_location] = '\0'; + + int dir_len = strlen(mapdir) - colon_location; + char *dir = (char *)malloc(dir_len + 1); + memcpy(dir, &mapdir[colon_location + 1], dir_len); + dir[dir_len] = '\0'; + + wasi_config_mapdir(wasi_config, alias, dir); + free(alias); + free(dir); +} + +// We try to parse out `--dir` and `--mapdir` ahead of time and process those +// specially. All other arguments are passed to the guest program. +static void handle_arguments(wasi_config_t *wasi_config, int argc, + char *argv[]) { + for (int i = 1; i < argc; ++i) { + // We probably want special args like `--dir` and `--mapdir` to not be + // passed directly + if (strcmp(argv[i], "--dir") == 0) { + // next arg is a preopen directory + if ((i + 1) < argc) { + i++; + wasi_config_preopen_dir(wasi_config, argv[i]); + } else { + fprintf(stderr, "--dir expects a following argument specifying which " + "directory to preopen\n"); + exit(-1); + } + } else if (strcmp(argv[i], "--mapdir") == 0) { + // next arg is a mapdir + if ((i + 1) < argc) { + i++; + pass_mapdir_arg(wasi_config, argv[i]); + } else { + fprintf(stderr, + "--mapdir expects a following argument specifying which " + "directory to preopen in the form alias:directory\n"); + exit(-1); + } + } else if (strncmp(argv[i], "--dir=", strlen("--dir=")) == 0) { + // this arg is a preopen dir + char *dir = argv[i] + strlen("--dir="); + wasi_config_preopen_dir(wasi_config, dir); + } else if (strncmp(argv[i], "--mapdir=", strlen("--mapdir=")) == 0) { + // this arg is a mapdir + char *mapdir = argv[i] + strlen("--mapdir="); + pass_mapdir_arg(wasi_config, mapdir); + } else { + // guest argument + wasi_config_arg(wasi_config, argv[i]); + } + } +} +#endif + +int main(int argc, char *argv[]) { + wasm_config_t *config = wasm_config_new(); + wasm_engine_t *engine = wasm_engine_new_with_config(config); + wasm_store_t *store = wasm_store_new(engine); + + wasm_module_t *module = wasmer_static_module_new(store, "module"); + + if (!module) { + fprintf(stderr, "Failed to create module\n"); + print_wasmer_error(); + return -1; + } + + // We have now finished the memory buffer book keeping and we have a valid + // Module. + +#ifdef WASI + wasi_config_t *wasi_config = wasi_config_new(argv[0]); + handle_arguments(wasi_config, argc, argv); + + wasi_env_t *wasi_env = wasi_env_new(store, wasi_config); + if (!wasi_env) { + fprintf(stderr, "Error building WASI env!\n"); + print_wasmer_error(); + return 1; + } +#endif + + wasm_importtype_vec_t import_types; + wasm_module_imports(module, &import_types); + + wasm_extern_vec_t imports; + wasm_extern_vec_new_uninitialized(&imports, import_types.size); + wasm_importtype_vec_delete(&import_types); + +#ifdef WASI + bool get_imports_result = wasi_get_imports(store, wasi_env, module, &imports); + + if (!get_imports_result) { + fprintf(stderr, "Error getting WASI imports!\n"); + print_wasmer_error(); + + return 1; + } +#endif + + wasm_instance_t *instance = wasm_instance_new(store, module, &imports, NULL); + + if (!instance) { + fprintf(stderr, "Failed to create instance\n"); + print_wasmer_error(); + return -1; + } + +#ifdef WASI + // Read the exports. + wasm_extern_vec_t exports; + wasm_instance_exports(instance, &exports); + wasm_memory_t* mem = NULL; + for (size_t i = 0; i < exports.size; i++) { + mem = wasm_extern_as_memory(exports.data[i]); + if (mem) { + break; + } + } + + if (!mem) { + fprintf(stderr, "Failed to create instance: Could not find memory in exports\n"); + print_wasmer_error(); + return -1; + } + wasi_env_set_memory(wasi_env, mem); + + own wasm_func_t *start_function = wasi_get_start_function(instance); + if (!start_function) { + fprintf(stderr, "`_start` function not found\n"); + print_wasmer_error(); + return -1; + } + + wasm_val_vec_t args = WASM_EMPTY_VEC; + wasm_val_vec_t results = WASM_EMPTY_VEC; + own wasm_trap_t *trap = wasm_func_call(start_function, &args, &results); + if (trap) { + fprintf(stderr, "Trap is not NULL: TODO:\n"); + return -1; + } +#endif + + // TODO: handle non-WASI start (maybe with invoke?) + +#ifdef WASI + wasi_env_delete(wasi_env); + wasm_extern_vec_delete(&exports); +#endif + wasm_instance_delete(instance); + wasm_module_delete(module); + wasm_store_delete(store); + wasm_engine_delete(engine); + return 0; +} diff --git a/lib/cli/src/lib.rs b/lib/cli/src/lib.rs index 4cb32b02bee..7a80b9fae69 100644 --- a/lib/cli/src/lib.rs +++ b/lib/cli/src/lib.rs @@ -20,6 +20,7 @@ pub mod commands; pub mod common; #[macro_use] pub mod error; +pub mod c_gen; pub mod cli; #[cfg(feature = "debug")] pub mod logging; diff --git a/lib/compiler/src/engine/artifact.rs b/lib/compiler/src/engine/artifact.rs index 46ba30f560a..2c55d1260b0 100644 --- a/lib/compiler/src/engine/artifact.rs +++ b/lib/compiler/src/engine/artifact.rs @@ -5,15 +5,19 @@ use crate::engine::link::link_module; use crate::ArtifactBuild; use crate::ArtifactCreate; use crate::Features; -#[cfg(feature = "static-artifact-create")] -use crate::{FunctionBodyData, Compiler, ModuleTranslationState}; +use crate::ModuleEnvironment; use crate::{ register_frame_info, resolve_imports, FunctionExtent, GlobalFrameInfoRegistration, InstantiationError, RuntimeError, Tunables, }; +#[cfg(feature = "static-artifact-create")] +use crate::{Compiler, FunctionBodyData, ModuleTranslationState}; use crate::{Engine, EngineInner}; -use crate::ModuleEnvironment; use enumset::EnumSet; +#[cfg(feature = "static-artifact-create")] +use std::collections::BTreeMap; +#[cfg(feature = "static-artifact-create")] +use std::mem; use std::sync::Arc; use std::sync::Mutex; #[cfg(feature = "static-artifact-create")] @@ -21,8 +25,11 @@ use wasmer_object::{emit_compilation, emit_data, get_object_for_target, Object}; use wasmer_types::entity::{BoxedSlice, PrimaryMap}; use wasmer_types::MetadataHeader; #[cfg(feature = "static-artifact-create")] -#[cfg(feature = "compiler")] -use wasmer_types::{compilation::symbols::ModuleMetadata, CompileModuleInfo, Target}; +use wasmer_types::{ + compilation::symbols::{ModuleMetadata, ModuleMetadataSymbolRegistry}, + entity::EntityRef, + CompileModuleInfo, Target, +}; use wasmer_types::{ CompileError, CpuFeature, DataInitializer, DeserializeError, FunctionIndex, LocalFunctionIndex, MemoryIndex, ModuleInfo, OwnedDataInitializer, SerializableModule, SerializeError, @@ -32,7 +39,17 @@ use wasmer_vm::{FunctionBodyPtr, MemoryStyle, TableStyle, VMSharedSignatureIndex use wasmer_vm::{InstanceAllocator, InstanceHandle, StoreObjects, TrapHandlerFn, VMExtern}; /// A compiled wasm module, ready to be instantiated. -pub struct Artifact { +pub enum Artifact { + /// The default artifact format. + Universal(UniversalArtifact), + /// Stores functions etc as symbols and data meant to be stored in object files and + /// executables. + #[cfg(feature = "static-artifact-create")] + Static(StaticArtifact), +} + +/// The default artifact format. +pub struct UniversalArtifact { artifact: ArtifactBuild, finished_functions: BoxedSlice, finished_function_call_trampolines: BoxedSlice, @@ -42,6 +59,24 @@ pub struct Artifact { finished_function_lengths: BoxedSlice, } +/// Stores functions etc as symbols and data meant to be stored in object files and +/// executables. +#[cfg(feature = "static-artifact-create")] +pub struct StaticArtifact { + metadata: ModuleMetadata, + _module_bytes: Vec, + finished_functions: BoxedSlice, + finished_function_call_trampolines: BoxedSlice, + finished_dynamic_function_trampolines: BoxedSlice, + signatures: BoxedSlice, + // Length of the serialized metadata + _metadata_length: usize, + _symbol_registry: ModuleMetadataSymbolRegistry, +} + +#[cfg(feature = "static-artifact-create")] +const WASMER_METADATA_SYMBOL: &[u8] = b"WASMER_METADATA"; + impl Artifact { /// Compile a data buffer into a `ArtifactBuild`, which may then be instantiated. #[cfg(feature = "compiler")] @@ -91,6 +126,15 @@ impl Artifact { /// the data. pub unsafe fn deserialize(engine: &Engine, bytes: &[u8]) -> Result { if !ArtifactBuild::is_deserializable(bytes) { + let static_artifact = Self::deserialize_object(engine, bytes); + match static_artifact { + Ok(v) => { + return Ok(v); + } + Err(err) => { + eprintln!("Could not deserialize as static object: {}", err); + } + } return Err(DeserializeError::Incompatible( "The provided bytes are not wasmer-universal".to_string(), )); @@ -177,7 +221,7 @@ impl Artifact { finished_dynamic_function_trampolines.into_boxed_slice(); let signatures = signatures.into_boxed_slice(); - Ok(Self { + Ok(Self::Universal(UniversalArtifact { artifact, finished_functions, finished_function_call_trampolines, @@ -185,7 +229,7 @@ impl Artifact { signatures, frame_info_registration: Mutex::new(None), finished_function_lengths, - }) + })) } /// Check if the provided bytes look like a serialized `ArtifactBuild`. @@ -196,31 +240,59 @@ impl Artifact { impl ArtifactCreate for Artifact { fn create_module_info(&self) -> ModuleInfo { - self.artifact.create_module_info() + match self { + Self::Universal(UniversalArtifact { artifact, .. }) => artifact.create_module_info(), + #[cfg(feature = "static-artifact-create")] + Self::Static(StaticArtifact { metadata, .. }) => metadata.compile_info.module.clone(), + } } fn features(&self) -> &Features { - self.artifact.features() + match self { + Self::Universal(UniversalArtifact { artifact, .. }) => artifact.features(), + #[cfg(feature = "static-artifact-create")] + Self::Static(StaticArtifact { metadata, .. }) => &metadata.compile_info.features, + } } fn cpu_features(&self) -> EnumSet { - self.artifact.cpu_features() + match self { + Self::Universal(_self) => _self.artifact.cpu_features(), + #[cfg(feature = "static-artifact-create")] + Self::Static(_self) => EnumSet::from_u64(_self.metadata.cpu_features), + } } fn data_initializers(&self) -> &[OwnedDataInitializer] { - self.artifact.data_initializers() + match self { + Self::Universal(_self) => _self.artifact.data_initializers(), + #[cfg(feature = "static-artifact-create")] + Self::Static(_self) => &_self.metadata.data_initializers, + } } fn memory_styles(&self) -> &PrimaryMap { - self.artifact.memory_styles() + match self { + Self::Universal(_self) => _self.artifact.memory_styles(), + #[cfg(feature = "static-artifact-create")] + Self::Static(_self) => &_self.metadata.compile_info.memory_styles, + } } fn table_styles(&self) -> &PrimaryMap { - self.artifact.table_styles() + match self { + Self::Universal(UniversalArtifact { artifact, .. }) => artifact.table_styles(), + #[cfg(feature = "static-artifact-create")] + Self::Static(StaticArtifact { metadata, .. }) => &metadata.compile_info.table_styles, + } } fn serialize(&self) -> Result, SerializeError> { - self.artifact.serialize() + match self { + Self::Universal(UniversalArtifact { artifact, .. }) => artifact.serialize(), + #[cfg(feature = "static-artifact-create")] + Self::Static(StaticArtifact { .. }) => todo!(), + } } } @@ -229,39 +301,55 @@ impl Artifact { /// /// This is required to ensure that any traps can be properly symbolicated. pub fn register_frame_info(&self) { - let mut info = self.frame_info_registration.lock().unwrap(); - - if info.is_some() { - return; + match self { + Self::Universal(_self) => { + let mut info = _self.frame_info_registration.lock().unwrap(); + + if info.is_some() { + return; + } + + let finished_function_extents = _self + .finished_functions + .values() + .copied() + .zip(_self.finished_function_lengths.values().copied()) + .map(|(ptr, length)| FunctionExtent { ptr, length }) + .collect::>() + .into_boxed_slice(); + + let frame_infos = _self.artifact.get_frame_info_ref(); + *info = register_frame_info( + _self.artifact.create_module_info(), + &finished_function_extents, + frame_infos.clone(), + ); + } + #[cfg(feature = "static-artifact-create")] + Self::Static(_self) => { + // Do nothing for static artifact + } } - - let finished_function_extents = self - .finished_functions - .values() - .copied() - .zip(self.finished_function_lengths.values().copied()) - .map(|(ptr, length)| FunctionExtent { ptr, length }) - .collect::>() - .into_boxed_slice(); - - let frame_infos = self.artifact.get_frame_info_ref(); - *info = register_frame_info( - self.artifact.create_module_info(), - &finished_function_extents, - frame_infos.clone(), - ); } /// Returns the functions allocated in memory or this `Artifact` /// ready to be run. pub fn finished_functions(&self) -> &BoxedSlice { - &self.finished_functions + match self { + Self::Universal(_self) => &_self.finished_functions, + #[cfg(feature = "static-artifact-create")] + Self::Static(_self) => &_self.finished_functions, + } } /// Returns the function call trampolines allocated in memory of this /// `Artifact`, ready to be run. pub fn finished_function_call_trampolines(&self) -> &BoxedSlice { - &self.finished_function_call_trampolines + match self { + Self::Universal(_self) => &_self.finished_function_call_trampolines, + #[cfg(feature = "static-artifact-create")] + Self::Static(_self) => &_self.finished_function_call_trampolines, + } } /// Returns the dynamic function trampolines allocated in memory @@ -269,12 +357,20 @@ impl Artifact { pub fn finished_dynamic_function_trampolines( &self, ) -> &BoxedSlice { - &self.finished_dynamic_function_trampolines + match self { + Self::Universal(_self) => &_self.finished_dynamic_function_trampolines, + #[cfg(feature = "static-artifact-create")] + Self::Static(_self) => &_self.finished_dynamic_function_trampolines, + } } /// Returns the associated VM signatures for this `Artifact`. pub fn signatures(&self) -> &BoxedSlice { - &self.signatures + match self { + Self::Universal(_self) => &_self.signatures, + #[cfg(feature = "static-artifact-create")] + Self::Static(_self) => &_self.signatures, + } } /// Do preinstantiation logic that is executed before instantiating @@ -385,13 +481,14 @@ impl Artifact { .map_err(|trap| InstantiationError::Start(RuntimeError::from_trap(trap))) } + #[allow(clippy::type_complexity)] #[cfg(feature = "static-artifact-create")] /// Generate a compilation fn generate_metadata<'data>( - &self, data: &'data [u8], compiler: &dyn Compiler, tunables: &dyn Tunables, + features: &Features, ) -> Result< ( CompileModuleInfo, @@ -403,7 +500,6 @@ impl Artifact { > { let environ = ModuleEnvironment::new(); let translation = environ.translate(data).map_err(CompileError::Wasm)?; - let features = self.features().clone(); // We try to apply the middleware first use crate::translator::ModuleMiddlewareChain; @@ -424,7 +520,7 @@ impl Artifact { let compile_info = CompileModuleInfo { module, - features, + features: features.clone(), memory_styles, table_styles, }; @@ -443,22 +539,28 @@ impl Artifact { /// so we can assure no collisions. #[cfg(feature = "static-artifact-create")] pub fn generate_object<'data>( - &self, compiler: &dyn Compiler, data: &[u8], prefixer: Option String + Send>>, target: &'data Target, tunables: &dyn Tunables, - ) -> Result, CompileError> { + features: &Features, + ) -> Result< + ( + ModuleInfo, + Object<'data>, + usize, + Box, + ), + CompileError, + > { fn to_compile_error(err: impl std::error::Error) -> CompileError { CompileError::Codegen(format!("{}", err)) } #[allow(dead_code)] - const WASMER_METADATA_SYMBOL: &[u8] = b"WASMER_METADATA"; - let (compile_info, function_body_inputs, data_initializers, module_translation) = - self.generate_metadata(data, compiler, tunables)?; + Self::generate_metadata(data, compiler, tunables, features)?; let data_initializers = data_initializers .iter() @@ -508,21 +610,150 @@ impl Artifact { let (_compile_info, symbol_registry) = metadata.split(); - let compilation = compiler.compile_module( - &target, - &metadata.compile_info, - module_translation.as_ref().unwrap(), - function_body_inputs, - )?; - - let mut obj = get_object_for_target(&target_triple).map_err(to_compile_error)?; + let compilation: wasmer_types::compilation::function::Compilation = compiler + .compile_module( + target, + &metadata.compile_info, + module_translation.as_ref().unwrap(), + function_body_inputs, + )?; + let mut obj = get_object_for_target(target_triple).map_err(to_compile_error)?; - emit_data(&mut obj, b"WASMER_MODULE_METADATA", &metadata_binary, 1) + emit_data(&mut obj, WASMER_METADATA_SYMBOL, &metadata_binary, 1) .map_err(to_compile_error)?; - emit_compilation(&mut obj, compilation, &symbol_registry, &target_triple) + emit_compilation(&mut obj, compilation, &symbol_registry, target_triple) .map_err(to_compile_error)?; + Ok(( + metadata.compile_info.module, + obj, + metadata_binary.len(), + Box::new(symbol_registry), + )) + } + + /// Deserialize a ArtifactBuild from an object file + /// + /// # Safety + /// The object must be a valid static object generated by wasmer. + #[cfg(not(feature = "static-artifact-load"))] + pub unsafe fn deserialize_object( + _engine: &Engine, + _bytes: &[u8], + ) -> Result { + Err(DeserializeError::Compiler( + CompileError::UnsupportedFeature("static load is not compiled in".to_string()), + )) + } + + /// Deserialize a ArtifactBuild from an object file + /// + /// # Safety + /// The object must be a valid static object generated by wasmer. + #[cfg(feature = "static-artifact-load")] + pub unsafe fn deserialize_object( + engine: &Engine, + bytes: &[u8], + ) -> Result { + let metadata_len = MetadataHeader::parse(bytes)?; + + let metadata_slice = &bytes[MetadataHeader::LEN..][..metadata_len]; + let mut metadata: ModuleMetadata = ModuleMetadata::deserialize(metadata_slice)?; + + const WORD_SIZE: usize = mem::size_of::(); + let mut byte_buffer = [0u8; WORD_SIZE]; + + let mut cur_offset = MetadataHeader::LEN + metadata_len; + byte_buffer[0..WORD_SIZE].clone_from_slice(&bytes[cur_offset..(cur_offset + WORD_SIZE)]); + cur_offset += WORD_SIZE; + + let num_finished_functions = usize::from_ne_bytes(byte_buffer); + let mut finished_functions: PrimaryMap = + PrimaryMap::new(); + + let engine_inner = engine.inner(); + let signature_registry = engine_inner.signatures(); + let mut sig_map: BTreeMap = BTreeMap::new(); + + let num_imported_functions = metadata.compile_info.module.num_imported_functions; + // set up the imported functions first... + for i in 0..num_imported_functions { + let sig_idx = metadata.compile_info.module.functions[FunctionIndex::new(i)]; + let func_type = &metadata.compile_info.module.signatures[sig_idx]; + let vm_shared_idx = signature_registry.register(func_type); + sig_map.insert(sig_idx, vm_shared_idx); + } + // read finished functions in order now... + for i in 0..num_finished_functions { + let local_func_idx = LocalFunctionIndex::new(i); + let func_idx = metadata.compile_info.module.func_index(local_func_idx); + let sig_idx = metadata.compile_info.module.functions[func_idx]; + let func_type = &metadata.compile_info.module.signatures[sig_idx]; + let vm_shared_idx = signature_registry.register(func_type); + sig_map.insert(sig_idx, vm_shared_idx); + + byte_buffer[0..WORD_SIZE] + .clone_from_slice(&bytes[cur_offset..(cur_offset + WORD_SIZE)]); + let fp = FunctionBodyPtr(usize::from_ne_bytes(byte_buffer) as _); + cur_offset += WORD_SIZE; + + // TODO: we can read back the length here if we serialize it. This will improve debug output. + + finished_functions.push(fp); + } + + let mut signatures: PrimaryMap<_, VMSharedSignatureIndex> = PrimaryMap::new(); + for i in 0..(sig_map.len()) { + if let Some(shared_idx) = sig_map.get(&SignatureIndex::new(i)) { + signatures.push(*shared_idx); + } else { + panic!("Invalid data, missing sig idx; TODO: handle this error"); + } + } + + // read trampolines in order + let mut finished_function_call_trampolines = PrimaryMap::new(); + byte_buffer[0..WORD_SIZE].clone_from_slice(&bytes[cur_offset..(cur_offset + WORD_SIZE)]); + cur_offset += WORD_SIZE; + let num_function_trampolines = usize::from_ne_bytes(byte_buffer); + for _ in 0..num_function_trampolines { + byte_buffer[0..WORD_SIZE] + .clone_from_slice(&bytes[cur_offset..(cur_offset + WORD_SIZE)]); + cur_offset += WORD_SIZE; + let trampoline_ptr_bytes = usize::from_ne_bytes(byte_buffer); + let trampoline = mem::transmute::(trampoline_ptr_bytes); + finished_function_call_trampolines.push(trampoline); + // TODO: we can read back the length here if we serialize it. This will improve debug output. + } + + // read dynamic function trampolines in order now... + let mut finished_dynamic_function_trampolines = PrimaryMap::new(); + byte_buffer[0..WORD_SIZE].clone_from_slice(&bytes[cur_offset..(cur_offset + WORD_SIZE)]); + cur_offset += WORD_SIZE; + let num_dynamic_trampoline_functions = usize::from_ne_bytes(byte_buffer); + for _i in 0..num_dynamic_trampoline_functions { + byte_buffer[0..WORD_SIZE] + .clone_from_slice(&bytes[cur_offset..(cur_offset + WORD_SIZE)]); + let fp = FunctionBodyPtr(usize::from_ne_bytes(byte_buffer) as _); + cur_offset += WORD_SIZE; + + // TODO: we can read back the length here if we serialize it. This will improve debug output. + + finished_dynamic_function_trampolines.push(fp); + } - Ok(obj) + let (_compile_info, _symbol_registry) = metadata.split(); + Ok(Self::Static(StaticArtifact { + metadata, + _module_bytes: bytes.to_vec(), + finished_functions: finished_functions.into_boxed_slice(), + finished_function_call_trampolines: finished_function_call_trampolines + .into_boxed_slice(), + finished_dynamic_function_trampolines: finished_dynamic_function_trampolines + .into_boxed_slice(), + signatures: signatures.into_boxed_slice(), + _metadata_length: metadata_len, + _symbol_registry, + })) } } diff --git a/lib/types/src/compilation/symbols.rs b/lib/types/src/compilation/symbols.rs index 7201b60f5c6..6971d4f6a11 100644 --- a/lib/types/src/compilation/symbols.rs +++ b/lib/types/src/compilation/symbols.rs @@ -57,26 +57,26 @@ pub trait SymbolRegistry: Send + Sync { #[derive(Debug, RkyvSerialize, RkyvDeserialize, Archive)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub struct ModuleMetadata { - /// FIXME + /// Compile info pub compile_info: CompileModuleInfo, - /// FIXME + /// Prefix for function etc symbols pub prefix: String, - /// FIXME + /// Data initializers pub data_initializers: Box<[OwnedDataInitializer]>, /// The function body lengths (used to find function by address) pub function_body_lengths: PrimaryMap, - /// FIXME + /// CPU features used (See [`CpuFeature`]) pub cpu_features: u64, } -/// FIXME +/// A simple metadata registry pub struct ModuleMetadataSymbolRegistry { - /// FIXME + /// Symbol prefix stirng pub prefix: String, } impl ModuleMetadata { - /// FIXME + /// Get mutable ref to compile info and a copy of the registry pub fn split(&mut self) -> (&mut CompileModuleInfo, ModuleMetadataSymbolRegistry) { let compile_info = &mut self.compile_info; let symbol_registry = ModuleMetadataSymbolRegistry { @@ -85,7 +85,7 @@ impl ModuleMetadata { (compile_info, symbol_registry) } - /// FIXME + /// Returns symbol registry. pub fn get_symbol_registry(&self) -> ModuleMetadataSymbolRegistry { ModuleMetadataSymbolRegistry { prefix: self.prefix.clone(), diff --git a/lib/types/src/error.rs b/lib/types/src/error.rs index 115cb1ae6c1..53555d6239c 100644 --- a/lib/types/src/error.rs +++ b/lib/types/src/error.rs @@ -34,7 +34,7 @@ pub enum DeserializeError { /// The binary was valid, but we got an error when /// trying to allocate the required resources. #[error(transparent)] - Compiler(CompileError), + Compiler(#[from] CompileError), } /// An ImportError.