From 16465ee43085cfbb9b06a19be6e2cab1a70deaa6 Mon Sep 17 00:00:00 2001 From: Manos Pitsidianakis Date: Wed, 13 Jul 2022 23:25:19 +0300 Subject: [PATCH] cli: add create-obj command lib/compiler: read static object --- Cargo.toml | 4 + Makefile | 4 +- lib/api/Cargo.toml | 5 + lib/c-api/Cargo.toml | 4 + lib/cli/Cargo.toml | 25 +- lib/cli/src/c_gen/mod.rs | 548 ++++++++++++++++++ lib/cli/src/c_gen/staticlib_header.rs | 298 ++++++++++ lib/cli/src/cli.rs | 9 + lib/cli/src/commands.rs | 27 + lib/cli/src/commands/create_exe.rs | 140 ++++- lib/cli/src/commands/create_obj.rs | 193 ++++++ .../commands/wasmer_static_create_exe_main.c | 192 ++++++ lib/cli/src/lib.rs | 1 + lib/compiler/src/engine/artifact.rs | 342 +++++++++-- lib/types/src/compilation/symbols.rs | 16 +- lib/types/src/error.rs | 2 +- 16 files changed, 1725 insertions(+), 85 deletions(-) create mode 100644 lib/cli/src/c_gen/mod.rs create mode 100644 lib/cli/src/c_gen/staticlib_header.rs create mode 100644 lib/cli/src/commands/create_obj.rs create mode 100644 lib/cli/src/commands/wasmer_static_create_exe_main.c diff --git a/Cargo.toml b/Cargo.toml index 47b3265dd04..a138288584c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -106,6 +106,10 @@ singlepass = ["wasmer-compiler-singlepass", "compiler"] cranelift = ["wasmer-compiler-cranelift", "compiler"] llvm = ["wasmer-compiler-llvm", "compiler"] middlewares = ["wasmer-middlewares"] +wasmer-artifact-load = ["wasmer-compiler/wasmer-artifact-load"] +wasmer-artifact-create = ["wasmer-compiler/wasmer-artifact-create"] +static-artifact-load = ["wasmer-compiler/static-artifact-load"] +static-artifact-create = ["wasmer-compiler/static-artifact-create"] # Testing features test-singlepass = ["singlepass"] diff --git a/Makefile b/Makefile index e99f00317c2..ed45ee807ee 100644 --- a/Makefile +++ b/Makefile @@ -254,13 +254,13 @@ space := $() $() comma := , # Define the compiler Cargo features for all crates. -compiler_features := --features $(subst $(space),$(comma),$(compilers)) +compiler_features := --features $(subst $(space),$(comma),$(compilers)),wasmer-artifact-create,static-artifact-create,wasmer-artifact-load,static-artifact-load capi_compilers_engines_exclude := # Define the compiler Cargo features for the C API. It always excludes # LLVM for the moment because it causes the linker to fail since LLVM is not statically linked. # TODO: Reenable LLVM in C-API -capi_compiler_features := --features $(subst $(space),$(comma),$(filter-out llvm, $(compilers))) +capi_compiler_features := --features $(subst $(space),$(comma),$(filter-out llvm, $(compilers))),wasmer-artifact-create,static-artifact-create,wasmer-artifact-load,static-artifact-load capi_compilers_engines_exclude += llvm-universal # We exclude singlepass-universal because it doesn't support multivalue (required in wasm-c-api tests) diff --git a/lib/api/Cargo.toml b/lib/api/Cargo.toml index aae032f3400..737c7d01baf 100644 --- a/lib/api/Cargo.toml +++ b/lib/api/Cargo.toml @@ -120,5 +120,10 @@ enable-serde = [ "wasmer-types/enable-serde", ] +wasmer-artifact-load = ["wasmer-compiler/wasmer-artifact-load"] +wasmer-artifact-create = ["wasmer-compiler/wasmer-artifact-create"] +static-artifact-load = ["wasmer-compiler/static-artifact-load"] +static-artifact-create = ["wasmer-compiler/static-artifact-create"] + [package.metadata.docs.rs] features = ["compiler", "core", "cranelift", "engine", "jit", "native", "singlepass", "sys", "sys-default" ] diff --git a/lib/c-api/Cargo.toml b/lib/c-api/Cargo.toml index db43f1412c6..2a4a78e62e2 100644 --- a/lib/c-api/Cargo.toml +++ b/lib/c-api/Cargo.toml @@ -79,6 +79,10 @@ llvm = [ "wasmer-compiler-llvm", "compiler", ] +wasmer-artifact-load = ["wasmer-compiler/wasmer-artifact-load"] +wasmer-artifact-create = ["wasmer-compiler/wasmer-artifact-create"] +static-artifact-load = ["wasmer-compiler/static-artifact-load"] +static-artifact-create = ["wasmer-compiler/static-artifact-create"] # Deprecated features. jit = ["compiler"] diff --git a/lib/cli/Cargo.toml b/lib/cli/Cargo.toml index ea4e1e365da..3b206ba90ab 100644 --- a/lib/cli/Cargo.toml +++ b/lib/cli/Cargo.toml @@ -67,6 +67,7 @@ default = [ "emscripten", "compiler", "wasmer-artifact-create", + "static-artifact-create", ] cache = ["wasmer-cache"] cache-blake3-pure = ["wasmer-cache/blake3-pure"] @@ -78,8 +79,28 @@ compiler = [ "wasmer-compiler/translator", "wasmer-compiler/compiler", ] -wasmer-artifact-create = ["compiler", "wasmer-compiler/wasmer-artifact-create", "wasmer-object"] -static-artifact-create = ["compiler", "wasmer-compiler/static-artifact-create", "wasmer-object"] +wasmer-artifact-create = ["compiler", + "wasmer/wasmer-artifact-load", + "wasmer/wasmer-artifact-create", + "wasmer-compiler/wasmer-artifact-load", + "wasmer-compiler/wasmer-artifact-create", + "wasmer-object", + ] +static-artifact-create = ["compiler", + "wasmer/static-artifact-load", + "wasmer/static-artifact-create", + "wasmer-compiler/static-artifact-load", + "wasmer-compiler/static-artifact-create", + "wasmer-object", + ] +wasmer-artifact-load = ["compiler", + "wasmer/wasmer-artifact-load", + "wasmer-compiler/wasmer-artifact-load", + ] +static-artifact-load = ["compiler", + "wasmer/static-artifact-load", + "wasmer-compiler/static-artifact-load", + ] experimental-io-devices = [ "wasmer-wasi-experimental-io-devices", diff --git a/lib/cli/src/c_gen/mod.rs b/lib/cli/src/c_gen/mod.rs new file mode 100644 index 00000000000..49093072719 --- /dev/null +++ b/lib/cli/src/c_gen/mod.rs @@ -0,0 +1,548 @@ +//! A convenient little abstraction for building up C expressions and generating +//! simple C code. + +pub mod staticlib_header; + +/// An identifier in C. +pub type CIdent = String; + +/// A Type in the C language. +#[derive(Debug, Clone)] +pub enum CType { + /// C `void` type. + Void, + /// A pointer to some other type. + PointerTo { + /// Whether the pointer is `const`. + is_const: bool, + /// The type that the pointer points to. + inner: Box, + }, + /// C 8 bit unsigned integer type. + U8, + /// C 16 bit unsigned integer type. + U16, + /// C 32 bit unsigned integer type. + U32, + /// C 64 bit unsigned integer type. + U64, + /// C pointer sized unsigned integer type. + USize, + /// C 8 bit signed integer type. + I8, + /// C 16 bit signed integer type. + I16, + /// C 32 bit signed integer type. + I32, + /// C 64 bit signed integer type. + I64, + /// C pointer sized signed integer type. + ISize, + /// A function or function pointer. + Function { + /// The arguments the function takes. + arguments: Vec, + /// The return value if it has one + /// + /// None is equivalent to Some(Box(Ctype::Void)). + return_value: Option>, + }, + /// C constant array. + Array { + /// The type of the array. + inner: Box, + }, + /// A user defined type. + TypeDef(String), +} + +impl CType { + /// Convenience function to get a mutable void pointer type. + pub fn void_ptr() -> Self { + CType::PointerTo { + is_const: false, + inner: Box::new(CType::Void), + } + } + + /// Convenience function to get a const void pointer type. + #[allow(dead_code)] + pub fn const_void_ptr() -> Self { + CType::PointerTo { + is_const: true, + inner: Box::new(CType::Void), + } + } + + /// Generate the C source code for a type into the given `String`. + fn generate_c(&self, w: &mut String) { + match &self { + Self::Void => { + w.push_str("void"); + } + Self::PointerTo { is_const, inner } => { + if *is_const { + w.push_str("const "); + } + inner.generate_c(w); + w.push('*'); + } + Self::U8 => { + w.push_str("unsigned char"); + } + Self::U16 => { + w.push_str("unsigned short"); + } + Self::U32 => { + w.push_str("unsigned int"); + } + Self::U64 => { + w.push_str("unsigned long long"); + } + Self::USize => { + w.push_str("unsigned size_t"); + } + Self::I8 => { + w.push_str("char"); + } + Self::I16 => { + w.push_str("short"); + } + Self::I32 => { + w.push_str("int"); + } + Self::I64 => { + w.push_str("long long"); + } + Self::ISize => { + w.push_str("size_t"); + } + Self::Function { + arguments, + return_value, + } => { + // function with no, name, assume it's a function pointer + let ret: CType = return_value + .as_ref() + .map(|i: &Box| (&**i).clone()) + .unwrap_or_default(); + ret.generate_c(w); + w.push(' '); + w.push_str("(*)"); + w.push('('); + match arguments.len() { + l if l > 1 => { + for arg in &arguments[..arguments.len() - 1] { + arg.generate_c(w); + w.push_str(", "); + } + arguments.last().unwrap().generate_c(w); + } + 1 => { + arguments[0].generate_c(w); + } + _ => {} + } + w.push(')'); + } + Self::Array { inner } => { + inner.generate_c(w); + w.push_str("[]"); + } + Self::TypeDef(inner) => { + w.push_str(inner); + } + } + } + + /// Generate the C source code for a type with a nameinto the given `String`. + fn generate_c_with_name(&self, name: &str, w: &mut String) { + match &self { + Self::PointerTo { .. } + | Self::TypeDef { .. } + | Self::Void + | Self::U8 + | Self::U16 + | Self::U32 + | Self::U64 + | Self::USize + | Self::I8 + | Self::I16 + | Self::I32 + | Self::I64 + | Self::ISize => { + self.generate_c(w); + w.push(' '); + w.push_str(name); + } + Self::Function { + arguments, + return_value, + } => { + let ret: CType = return_value + .as_ref() + .map(|i: &Box| (&**i).clone()) + .unwrap_or_default(); + ret.generate_c(w); + w.push(' '); + w.push_str(name); + w.push('('); + match arguments.len() { + l if l > 1 => { + for arg in &arguments[..arguments.len() - 1] { + arg.generate_c(w); + w.push_str(", "); + } + arguments.last().unwrap().generate_c(w); + } + 1 => { + arguments[0].generate_c(w); + } + _ => {} + } + w.push(')'); + } + Self::Array { inner } => { + inner.generate_c(w); + w.push(' '); + w.push_str(name); + w.push_str("[]"); + } + } + } +} + +impl Default for CType { + fn default() -> CType { + CType::Void + } +} + +/// A statement in the C programming language. This may not be exact to what an +/// AST would look like or what the C standard says about the C language, it's +/// simply a structed way to organize data for generating C code. +#[derive(Debug, Clone)] +pub enum CStatement { + /// A declaration of some kind. + Declaration { + /// The name of the thing being declared. + name: CIdent, + /// Whether the thing being declared is `extern`. + is_extern: bool, + /// Whether the thing being declared is `const`. + is_const: bool, + /// The type of the thing being declared. + ctype: CType, + /// The definition of the thing being declared. + /// + /// This is useful for initializing constant arrays, for example. + definition: Option>, + }, + + /// A literal array of CStatements. + LiteralArray { + /// The contents of the array. + items: Vec, + }, + + /// A literal constant value, passed through directly as a string. + LiteralConstant { + /// The raw value acting as a constant. + value: String, + }, + + /// A C-style cast + Cast { + /// The type to cast to. + target_type: CType, + /// The thing being cast. + expression: Box, + }, + + /// Typedef one type to another. + TypeDef { + /// The type of the thing being typedef'd. + source_type: CType, + /// The new name by which this type may be called. + new_name: CIdent, + }, +} + +impl CStatement { + /// Generate C source code for the given CStatement. + fn generate_c(&self, w: &mut String) { + match &self { + Self::Declaration { + name, + is_extern, + is_const, + ctype, + definition, + } => { + if *is_const { + w.push_str("const "); + } + if *is_extern { + w.push_str("extern "); + } + ctype.generate_c_with_name(name, w); + if let Some(def) = definition { + w.push_str(" = "); + def.generate_c(w); + } + w.push(';'); + w.push('\n'); + } + Self::LiteralArray { items } => { + w.push('{'); + if !items.is_empty() { + w.push('\n'); + } + for item in items { + w.push('\t'); + item.generate_c(w); + w.push(','); + w.push('\n'); + } + w.push('}'); + } + Self::LiteralConstant { value } => { + w.push_str(value); + } + Self::Cast { + target_type, + expression, + } => { + w.push('('); + target_type.generate_c(w); + w.push(')'); + w.push(' '); + expression.generate_c(w); + } + Self::TypeDef { + source_type, + new_name, + } => { + w.push_str("typedef "); + // leaky abstraction / hack, doesn't fully solve the problem + if let CType::Function { .. } = source_type { + source_type.generate_c_with_name(&format!("(*{})", new_name), w); + } else { + source_type.generate_c(w); + w.push(' '); + w.push_str(new_name); + } + w.push(';'); + w.push('\n'); + } + } + } +} + +/// Generate C source code from some `CStatements` into a String. +// TODO: add config section +pub fn generate_c(statements: &[CStatement]) -> String { + let mut out = String::new(); + for statement in statements { + statement.generate_c(&mut out); + } + out +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn generate_types() { + macro_rules! assert_c_type { + ($ctype:expr, $expected:expr) => { + let mut w = String::new(); + let ctype = $ctype; + ctype.generate_c(&mut w); + assert_eq!(w, $expected); + }; + } + + assert_c_type!(CType::Void, "void"); + assert_c_type!(CType::void_ptr(), "void*"); + assert_c_type!(CType::const_void_ptr(), "const void*"); + assert_c_type!(CType::U8, "unsigned char"); + assert_c_type!(CType::U16, "unsigned short"); + assert_c_type!(CType::U32, "unsigned int"); + assert_c_type!(CType::U64, "unsigned long long"); + assert_c_type!(CType::USize, "unsigned size_t"); + assert_c_type!(CType::I8, "char"); + assert_c_type!(CType::I16, "short"); + assert_c_type!(CType::I32, "int"); + assert_c_type!(CType::I64, "long long"); + assert_c_type!(CType::ISize, "size_t"); + assert_c_type!(CType::TypeDef("my_type".to_string()), "my_type"); + assert_c_type!( + CType::Function { + arguments: vec![CType::U8, CType::ISize], + return_value: None + }, + "void (*)(unsigned char, size_t)" + ); + assert_c_type!( + CType::Function { + arguments: vec![], + return_value: Some(Box::new(CType::ISize)) + }, + "size_t (*)()" + ); + assert_c_type!( + CType::PointerTo { + is_const: true, + inner: Box::new(CType::PointerTo { + is_const: false, + inner: Box::new(CType::U32) + }) + }, + "const unsigned int**" + ); + // TODO: test more complicated const correctness rules: there are bugs relating to it. + } + + #[test] + fn generate_types_with_names() { + macro_rules! assert_c_type { + ($ctype:expr, $name:literal, $expected:expr) => { + let mut w = String::new(); + let ctype = $ctype; + ctype.generate_c_with_name($name, &mut w); + assert_eq!(w, $expected); + }; + } + + assert_c_type!(CType::Void, "main", "void main"); + assert_c_type!(CType::void_ptr(), "data", "void* data"); + assert_c_type!(CType::const_void_ptr(), "data", "const void* data"); + assert_c_type!(CType::U8, "data", "unsigned char data"); + assert_c_type!(CType::U16, "data", "unsigned short data"); + assert_c_type!(CType::U32, "data", "unsigned int data"); + assert_c_type!(CType::U64, "data", "unsigned long long data"); + assert_c_type!(CType::USize, "data", "unsigned size_t data"); + assert_c_type!(CType::I8, "data", "char data"); + assert_c_type!(CType::I16, "data", "short data"); + assert_c_type!(CType::I32, "data", "int data"); + assert_c_type!(CType::I64, "data", "long long data"); + assert_c_type!(CType::ISize, "data", "size_t data"); + assert_c_type!( + CType::TypeDef("my_type".to_string()), + "data", + "my_type data" + ); + assert_c_type!( + CType::Function { + arguments: vec![CType::U8, CType::ISize], + return_value: None + }, + "my_func", + "void my_func(unsigned char, size_t)" + ); + assert_c_type!( + CType::Function { + arguments: vec![], + return_value: Some(Box::new(CType::ISize)) + }, + "my_func", + "size_t my_func()" + ); + assert_c_type!( + CType::PointerTo { + is_const: true, + inner: Box::new(CType::PointerTo { + is_const: false, + inner: Box::new(CType::U32) + }) + }, + "data", + "const unsigned int** data" + ); + // TODO: test more complicated const correctness rules: there are bugs relating to it. + } + + #[test] + fn generate_expressions_works() { + macro_rules! assert_c_expr { + ($cexpr:expr, $expected:expr) => { + let mut w = String::new(); + let cexpr = $cexpr; + cexpr.generate_c(&mut w); + assert_eq!(w, $expected); + }; + } + + assert_c_expr!( + CStatement::LiteralConstant { + value: "\"Hello, world!\"".to_string() + }, + "\"Hello, world!\"" + ); + assert_c_expr!( + CStatement::TypeDef { + source_type: CType::Function { + arguments: vec![CType::I32, CType::I32], + return_value: None, + }, + new_name: "my_func_ptr".to_string(), + }, + "typedef void (*my_func_ptr)(int, int);\n" + ); + assert_c_expr!( + CStatement::LiteralArray { + items: vec![ + CStatement::LiteralConstant { + value: "1".to_string() + }, + CStatement::LiteralConstant { + value: "2".to_string() + }, + CStatement::LiteralConstant { + value: "3".to_string() + }, + ] + }, + "{\n\t1,\n\t2,\n\t3,\n}" + ); + assert_c_expr!(CStatement::LiteralArray { items: vec![] }, "{}"); + assert_c_expr!( + CStatement::Declaration { + name: "my_array".to_string(), + is_extern: false, + is_const: true, + ctype: CType::Array { + inner: Box::new(CType::I32) + }, + definition: Some(Box::new(CStatement::LiteralArray { + items: vec![ + CStatement::LiteralConstant { + value: "1".to_string() + }, + CStatement::LiteralConstant { + value: "2".to_string() + }, + CStatement::LiteralConstant { + value: "3".to_string() + }, + ] + })) + }, + "const int my_array[] = {\n\t1,\n\t2,\n\t3,\n};\n" + ); + assert_c_expr!( + CStatement::Declaration { + name: "my_array".to_string(), + is_extern: true, + is_const: true, + ctype: CType::Array { + inner: Box::new(CType::I32) + }, + definition: None, + }, + "const extern int my_array[];\n" + ); + } +} diff --git a/lib/cli/src/c_gen/staticlib_header.rs b/lib/cli/src/c_gen/staticlib_header.rs new file mode 100644 index 00000000000..bd203c644b8 --- /dev/null +++ b/lib/cli/src/c_gen/staticlib_header.rs @@ -0,0 +1,298 @@ +//! Generate a header file for the static object file produced. + +use super::{generate_c, CStatement, CType}; +use wasmer_types::ModuleInfo; +use wasmer_types::{Symbol, SymbolRegistry}; + +/// Helper functions to simplify the usage of the static artifact. +const HELPER_FUNCTIONS: &str = r#" +wasm_byte_vec_t generate_serialized_data() { + // We need to pass all the bytes as one big buffer so we have to do all this logic to memcpy + // the various pieces together from the generated header file. + // + // We should provide a `deseralize_vectored` function to avoid requiring this extra work. + + char* byte_ptr = (char*)&WASMER_METADATA[0]; + + size_t num_function_pointers + = sizeof(function_pointers) / sizeof(void*); + size_t num_function_trampolines + = sizeof(function_trampolines) / sizeof(void*); + size_t num_dynamic_function_trampoline_pointers + = sizeof(dynamic_function_trampoline_pointers) / sizeof(void*); + + + size_t buffer_size = module_bytes_len + + sizeof(size_t) + sizeof(function_pointers) + + sizeof(size_t) + sizeof(function_trampolines) + + sizeof(size_t) + sizeof(dynamic_function_trampoline_pointers); + + char* memory_buffer = (char*) malloc(buffer_size); + size_t current_offset = 0; + + memcpy(memory_buffer + current_offset, byte_ptr, module_bytes_len); + current_offset += module_bytes_len; + + memcpy(memory_buffer + current_offset, (void*)&num_function_pointers, sizeof(size_t)); + current_offset += sizeof(size_t); + + memcpy(memory_buffer + current_offset, (void*)&function_pointers[0], sizeof(function_pointers)); + current_offset += sizeof(function_pointers); + + memcpy(memory_buffer + current_offset, (void*)&num_function_trampolines, sizeof(size_t)); + current_offset += sizeof(size_t); + + memcpy(memory_buffer + current_offset, (void*)&function_trampolines[0], sizeof(function_trampolines)); + current_offset += sizeof(function_trampolines); + + memcpy(memory_buffer + current_offset, (void*)&num_dynamic_function_trampoline_pointers, sizeof(size_t)); + current_offset += sizeof(size_t); + + memcpy(memory_buffer + current_offset, (void*)&dynamic_function_trampoline_pointers[0], sizeof(dynamic_function_trampoline_pointers)); + current_offset += sizeof(dynamic_function_trampoline_pointers); + + wasm_byte_vec_t module_byte_vec = { + .size = buffer_size, + .data = memory_buffer, + }; + return module_byte_vec; +} + +wasm_module_t* wasmer_static_module_new(wasm_store_t* store, const char* wasm_name) { + // wasm_name intentionally unused for now: will be used in the future. + wasm_byte_vec_t module_byte_vec = generate_serialized_data(); + wasm_module_t* module = wasm_module_deserialize(store, &module_byte_vec); + free(module_byte_vec.data); + + return module; +} +"#; + +/// Generate the header file that goes with the generated object file. +pub fn generate_header_file( + module_info: &ModuleInfo, + symbol_registry: &dyn SymbolRegistry, + metadata_length: usize, +) -> String { + let mut c_statements = vec![ + CStatement::LiteralConstant { + value: "#include \"wasmer.h\"\n#include \n#include \n\n" + .to_string(), + }, + CStatement::LiteralConstant { + value: "#ifdef __cplusplus\nextern \"C\" {\n#endif\n\n".to_string(), + }, + CStatement::Declaration { + name: "module_bytes_len".to_string(), + is_extern: false, + is_const: true, + ctype: CType::U32, + definition: Some(Box::new(CStatement::LiteralConstant { + value: metadata_length.to_string(), + })), + }, + CStatement::Declaration { + name: "WASMER_METADATA".to_string(), + is_extern: true, + is_const: true, + ctype: CType::Array { + inner: Box::new(CType::U8), + }, + definition: None, + }, + ]; + let function_declarations = module_info + .functions + .iter() + .filter_map(|(f_index, sig_index)| { + Some((module_info.local_func_index(f_index)?, sig_index)) + }) + .map(|(function_local_index, _sig_index)| { + let function_name = + symbol_registry.symbol_to_name(Symbol::LocalFunction(function_local_index)); + // TODO: figure out the signature here too + CStatement::Declaration { + name: function_name, + is_extern: true, + is_const: false, + ctype: CType::Function { + arguments: vec![CType::Void], + return_value: None, + }, + definition: None, + } + }); + c_statements.push(CStatement::LiteralConstant { + value: r#" +// Compiled Wasm function pointers ordered by function index: the order they +// appeared in in the Wasm module. +"# + .to_string(), + }); + c_statements.extend(function_declarations); + + // function pointer array + { + let function_pointer_array_statements = module_info + .functions + .iter() + .filter_map(|(f_index, sig_index)| { + Some((module_info.local_func_index(f_index)?, sig_index)) + }) + .map(|(function_local_index, _sig_index)| { + let function_name = + symbol_registry.symbol_to_name(Symbol::LocalFunction(function_local_index)); + // TODO: figure out the signature here too + + CStatement::Cast { + target_type: CType::void_ptr(), + expression: Box::new(CStatement::LiteralConstant { + value: function_name, + }), + } + }) + .collect::>(); + + c_statements.push(CStatement::Declaration { + name: "function_pointers".to_string(), + is_extern: false, + is_const: true, + ctype: CType::Array { + inner: Box::new(CType::void_ptr()), + }, + definition: Some(Box::new(CStatement::LiteralArray { + items: function_pointer_array_statements, + })), + }); + } + + let func_trampoline_declarations = + module_info + .signatures + .iter() + .map(|(sig_index, _func_type)| { + let function_name = + symbol_registry.symbol_to_name(Symbol::FunctionCallTrampoline(sig_index)); + + CStatement::Declaration { + name: function_name, + is_extern: true, + is_const: false, + ctype: CType::Function { + arguments: vec![CType::void_ptr(), CType::void_ptr(), CType::void_ptr()], + return_value: None, + }, + definition: None, + } + }); + c_statements.push(CStatement::LiteralConstant { + value: r#" +// Trampolines (functions by which we can call into Wasm) ordered by signature. +// There is 1 trampoline per function signature in the order they appear in +// the Wasm module. +"# + .to_string(), + }); + c_statements.extend(func_trampoline_declarations); + + // function trampolines + { + let function_trampoline_statements = module_info + .signatures + .iter() + .map(|(sig_index, _vm_shared_index)| { + let function_name = + symbol_registry.symbol_to_name(Symbol::FunctionCallTrampoline(sig_index)); + CStatement::LiteralConstant { + value: function_name, + } + }) + .collect::>(); + + c_statements.push(CStatement::Declaration { + name: "function_trampolines".to_string(), + is_extern: false, + is_const: true, + ctype: CType::Array { + inner: Box::new(CType::void_ptr()), + }, + definition: Some(Box::new(CStatement::LiteralArray { + items: function_trampoline_statements, + })), + }); + } + + let dyn_func_declarations = module_info + .functions + .keys() + .take(module_info.num_imported_functions) + .map(|func_index| { + let function_name = + symbol_registry.symbol_to_name(Symbol::DynamicFunctionTrampoline(func_index)); + // TODO: figure out the signature here + CStatement::Declaration { + name: function_name, + is_extern: true, + is_const: false, + ctype: CType::Function { + arguments: vec![CType::void_ptr(), CType::void_ptr(), CType::void_ptr()], + return_value: None, + }, + definition: None, + } + }); + c_statements.push(CStatement::LiteralConstant { + value: r#" +// Dynamic trampolines are per-function and are used for each function where +// the type signature is not known statically. In this case, this corresponds to +// the imported functions. +"# + .to_string(), + }); + c_statements.extend(dyn_func_declarations); + + c_statements.push(CStatement::TypeDef { + source_type: CType::Function { + arguments: vec![CType::void_ptr(), CType::void_ptr(), CType::void_ptr()], + return_value: None, + }, + new_name: "dyn_func_trampoline_t".to_string(), + }); + + // dynamic function trampoline pointer array + { + let dynamic_function_trampoline_statements = module_info + .functions + .keys() + .take(module_info.num_imported_functions) + .map(|func_index| { + let function_name = + symbol_registry.symbol_to_name(Symbol::DynamicFunctionTrampoline(func_index)); + CStatement::LiteralConstant { + value: function_name, + } + }) + .collect::>(); + c_statements.push(CStatement::Declaration { + name: "dynamic_function_trampoline_pointers".to_string(), + is_extern: false, + is_const: true, + ctype: CType::Array { + inner: Box::new(CType::TypeDef("dyn_func_trampoline_t".to_string())), + }, + definition: Some(Box::new(CStatement::LiteralArray { + items: dynamic_function_trampoline_statements, + })), + }); + } + + c_statements.push(CStatement::LiteralConstant { + value: HELPER_FUNCTIONS.to_string(), + }); + + c_statements.push(CStatement::LiteralConstant { + value: "\n#ifdef __cplusplus\n}\n#endif\n\n".to_string(), + }); + + generate_c(&c_statements) +} diff --git a/lib/cli/src/cli.rs b/lib/cli/src/cli.rs index 3587dba5a93..d82120a7d2f 100644 --- a/lib/cli/src/cli.rs +++ b/lib/cli/src/cli.rs @@ -6,6 +6,8 @@ use crate::commands::Binfmt; use crate::commands::Compile; #[cfg(any(feature = "static-artifact-create", feature = "wasmer-artifact-create"))] use crate::commands::CreateExe; +#[cfg(feature = "static-artifact-create")] +use crate::commands::CreateObj; #[cfg(feature = "wast")] use crate::commands::Wast; use crate::commands::{Cache, Config, Inspect, Run, SelfUpdate, Validate}; @@ -51,6 +53,11 @@ enum WasmerCLIOptions { #[structopt(name = "create-exe")] CreateExe(CreateExe), + /// Compile a WebAssembly binary into an object file + #[cfg(feature = "static-artifact-create")] + #[structopt(name = "create-obj")] + CreateObj(CreateObj), + /// Get various configuration information needed /// to compile programs which use Wasmer #[structopt(name = "config")] @@ -86,6 +93,8 @@ impl WasmerCLIOptions { Self::Compile(compile) => compile.execute(), #[cfg(any(feature = "static-artifact-create", feature = "wasmer-artifact-create"))] Self::CreateExe(create_exe) => create_exe.execute(), + #[cfg(feature = "static-artifact-create")] + Self::CreateObj(create_obj) => create_obj.execute(), Self::Config(config) => config.execute(), Self::Inspect(inspect) => inspect.execute(), #[cfg(feature = "wast")] diff --git a/lib/cli/src/commands.rs b/lib/cli/src/commands.rs index 01066a5018c..95cb1ad5d96 100644 --- a/lib/cli/src/commands.rs +++ b/lib/cli/src/commands.rs @@ -7,6 +7,8 @@ mod compile; mod config; #[cfg(any(feature = "static-artifact-create", feature = "wasmer-artifact-create"))] mod create_exe; +#[cfg(feature = "static-artifact-create")] +mod create_obj; mod inspect; mod run; mod self_update; @@ -20,6 +22,31 @@ pub use binfmt::*; pub use compile::*; #[cfg(any(feature = "static-artifact-create", feature = "wasmer-artifact-create"))] pub use create_exe::*; +#[cfg(feature = "static-artifact-create")] +pub use create_obj::*; #[cfg(feature = "wast")] pub use wast::*; pub use {cache::*, config::*, inspect::*, run::*, self_update::*, validate::*}; + +/// The kind of object format to emit. +#[derive(Debug, Copy, Clone, structopt::StructOpt)] +#[cfg(any(feature = "static-artifact-create", feature = "wasmer-artifact-create"))] +pub enum ObjectFormat { + /// Serialize the entire module into an object file. + Serialized, + /// Serialize only the module metadata into an object file and emit functions as symbols. + Symbols, +} + +#[cfg(any(feature = "static-artifact-create", feature = "wasmer-artifact-create"))] +impl std::str::FromStr for ObjectFormat { + type Err = &'static str; + + fn from_str(s: &str) -> Result { + match s { + "serialized" => Ok(Self::Serialized), + "symbols" => Ok(Self::Symbols), + _ => Err("must be one of two options: `serialized` or `symbols`."), + } + } +} diff --git a/lib/cli/src/commands/create_exe.rs b/lib/cli/src/commands/create_exe.rs index 1d8f468fe63..a06f16cdee9 100644 --- a/lib/cli/src/commands/create_exe.rs +++ b/lib/cli/src/commands/create_exe.rs @@ -1,5 +1,6 @@ //! Create a standalone native executable for a given Wasm file. +use super::ObjectFormat; use crate::store::CompilerOptions; use anyhow::{Context, Result}; use std::env; @@ -14,6 +15,8 @@ use wasmer::*; use wasmer_object::{emit_serialized, get_object_for_target}; const WASMER_MAIN_C_SOURCE: &[u8] = include_bytes!("wasmer_create_exe_main.c"); +#[cfg(feature = "static-artifact-create")] +const WASMER_STATIC_MAIN_C_SOURCE: &[u8] = include_bytes!("wasmer_static_create_exe_main.c"); #[derive(Debug, StructOpt)] /// The options for the `wasmer create-exe` subcommand @@ -30,9 +33,9 @@ pub struct CreateExe { #[structopt(long = "target")] target_triple: Option, - #[structopt(flatten)] - compiler: CompilerOptions, - + /// Object format options + #[structopt(name = "OBJECT_FORMAT", long = "object-format")] + object_format: Option, #[structopt(short = "m", multiple = true, number_of_values = 1)] cpu_features: Vec, @@ -40,6 +43,9 @@ pub struct CreateExe { /// This is useful for fixing linker errors that may occur on some systems. #[structopt(short = "l", multiple = true, number_of_values = 1)] libraries: Vec, + + #[structopt(flatten)] + compiler: CompilerOptions, } impl CreateExe { @@ -61,9 +67,11 @@ impl CreateExe { }) .unwrap_or_default(); let (store, compiler_type) = self.compiler.get_store_for_target(target.clone())?; + let object_format = self.object_format.unwrap_or(ObjectFormat::Symbols); println!("Compiler: {}", compiler_type.to_string()); println!("Target: {}", target.triple()); + println!("Format: {:?}", object_format); let working_dir = tempfile::tempdir()?; let starting_cd = env::current_dir()?; @@ -77,19 +85,63 @@ impl CreateExe { let wasm_module_path = starting_cd.join(&self.path); - let module = - Module::from_file(&store, &wasm_module_path).context("failed to compile Wasm")?; - let bytes = module.serialize()?; - let mut obj = get_object_for_target(target.triple())?; - emit_serialized(&mut obj, &bytes, target.triple())?; - let mut writer = BufWriter::new(File::create(&wasm_object_path)?); - obj.write_stream(&mut writer) - .map_err(|err| anyhow::anyhow!(err.to_string()))?; - writer.flush()?; - drop(writer); - - self.compile_c(wasm_object_path, output_path)?; - + match object_format { + ObjectFormat::Serialized => { + let module = Module::from_file(&store, &wasm_module_path) + .context("failed to compile Wasm")?; + let bytes = module.serialize()?; + let mut obj = get_object_for_target(target.triple())?; + emit_serialized(&mut obj, &bytes, target.triple())?; + let mut writer = BufWriter::new(File::create(&wasm_object_path)?); + obj.write_stream(&mut writer) + .map_err(|err| anyhow::anyhow!(err.to_string()))?; + writer.flush()?; + drop(writer); + + self.compile_c(wasm_object_path, output_path)?; + } + #[cfg(not(feature = "static-artifact-create"))] + ObjectFormat::Symbols => { + return Err(anyhow!("This version of wasmer-cli hasn't been compiled with static artifact support. You need to enable the `static-artifact-create` feature during compilation.")); + } + #[cfg(feature = "static-artifact-create")] + ObjectFormat::Symbols => { + let engine = store.engine(); + let engine_inner = engine.inner(); + let compiler = engine_inner.compiler()?; + let features = engine_inner.features(); + let tunables = store.tunables(); + let data: Vec = fs::read(wasm_module_path)?; + let prefixer: Option String + Send>> = None; + let (module_info, obj, metadata_length, symbol_registry) = + Artifact::generate_object( + compiler, &data, prefixer, &target, tunables, features, + )?; + + let header_file_src = crate::c_gen::staticlib_header::generate_header_file( + &module_info, + &*symbol_registry, + metadata_length, + ); + /* Write object file with functions */ + let object_file_path: std::path::PathBuf = + std::path::Path::new("functions.o").into(); + let mut writer = BufWriter::new(File::create(&object_file_path)?); + obj.write_stream(&mut writer) + .map_err(|err| anyhow::anyhow!(err.to_string()))?; + writer.flush()?; + /* Write down header file that includes pointer arrays and the deserialize function + * */ + let mut writer = BufWriter::new(File::create("static_defs.h")?); + writer.write_all(header_file_src.as_bytes())?; + writer.flush()?; + link( + output_path.clone(), + object_file_path, + std::path::Path::new("static_defs.h").into(), + )?; + } + } eprintln!( "✔ Native executable compiled successfully to `{}`.", self.output.display(), @@ -130,6 +182,62 @@ impl CreateExe { } } +#[cfg(feature = "static-artifact-create")] +fn link( + output_path: PathBuf, + object_path: PathBuf, + header_code_path: PathBuf, +) -> anyhow::Result<()> { + let c_src_path = Path::new("wasmer_main.c"); + let mut libwasmer_path = get_libwasmer_path()? + .canonicalize() + .context("Failed to find libwasmer")?; + println!("Using libwasmer: {}", libwasmer_path.display()); + let lib_filename = libwasmer_path + .file_name() + .unwrap() + .to_str() + .unwrap() + .to_string(); + libwasmer_path.pop(); + { + let mut c_src_file = fs::OpenOptions::new() + .create_new(true) + .write(true) + .open(&c_src_path) + .context("Failed to open C source code file")?; + c_src_file.write_all(WASMER_STATIC_MAIN_C_SOURCE)?; + } + + println!( + "link output {:?}", + Command::new("cc") + .arg(&object_path) + .arg(&header_code_path) + .arg(&c_src_path) + .arg(&format!("-L{}", libwasmer_path.display())) + .arg(&format!("-I{}", get_wasmer_include_directory()?.display())) + .arg(&format!("-l:{}", lib_filename)) + // Add libraries required per platform. + // We need userenv, sockets (Ws2_32), advapi32 for some system calls and bcrypt for random numbers. + //#[cfg(windows)] + // .arg("-luserenv") + // .arg("-lWs2_32") + // .arg("-ladvapi32") + // .arg("-lbcrypt") + // On unix we need dlopen-related symbols, libmath for a few things, and pthreads. + //#[cfg(not(windows))] + .arg("-ldl") + .arg("-lm") + .arg("-pthread") + //.arg(&format!("-I{}", header_code_path.display())) + .arg("-o") + .arg(&output_path) + .output()? + ); + Ok(()) +} + fn get_wasmer_dir() -> anyhow::Result { Ok(PathBuf::from( env::var("WASMER_DIR") diff --git a/lib/cli/src/commands/create_obj.rs b/lib/cli/src/commands/create_obj.rs new file mode 100644 index 00000000000..64aff8b84c5 --- /dev/null +++ b/lib/cli/src/commands/create_obj.rs @@ -0,0 +1,193 @@ +#![allow(dead_code)] +//! Create a standalone native executable for a given Wasm file. + +use super::ObjectFormat; +use crate::store::CompilerOptions; +use anyhow::{Context, Result}; +use std::env; +use std::fs; +use std::fs::File; +use std::io::prelude::*; +use std::io::BufWriter; +use std::path::PathBuf; +use std::process::Command; +use structopt::StructOpt; +use wasmer::*; +use wasmer_object::{emit_serialized, get_object_for_target}; + +#[derive(Debug, StructOpt)] +/// The options for the `wasmer create-exe` subcommand +pub struct CreateObj { + /// Input file + #[structopt(name = "FILE", parse(from_os_str))] + path: PathBuf, + + /// Output file + #[structopt(name = "OUTPUT PATH", short = "o", parse(from_os_str))] + output: PathBuf, + + /// Compilation Target triple + #[structopt(long = "target")] + target_triple: Option, + + /// Object format options + #[structopt(name = "OBJECT_FORMAT", long = "object-format")] + object_format: Option, + + #[structopt(short = "m", multiple = true, number_of_values = 1)] + cpu_features: Vec, + + #[structopt(flatten)] + compiler: CompilerOptions, +} + +impl CreateObj { + /// Runs logic for the `create-obj` subcommand + pub fn execute(&self) -> Result<()> { + println!("objectformat: {:?}", &self.object_format); + let target = self + .target_triple + .as_ref() + .map(|target_triple| { + let mut features = self + .cpu_features + .clone() + .into_iter() + .fold(CpuFeature::set(), |a, b| a | b); + // Cranelift requires SSE2, so we have this "hack" for now to facilitate + // usage + features |= CpuFeature::SSE2; + Target::new(target_triple.clone(), features) + }) + .unwrap_or_default(); + let (store, compiler_type) = self.compiler.get_store_for_target(target.clone())?; + let object_format = self.object_format.unwrap_or(ObjectFormat::Symbols); + + println!("Compiler: {}", compiler_type.to_string()); + println!("Target: {}", target.triple()); + println!("Format: {:?}", object_format); + + let starting_cd = env::current_dir()?; + let output_path = starting_cd.join(&self.output); + + let wasm_module_path = starting_cd.join(&self.path); + + match object_format { + ObjectFormat::Serialized => { + let module = Module::from_file(&store, &wasm_module_path) + .context("failed to compile Wasm")?; + let bytes = module.serialize()?; + let mut obj = get_object_for_target(target.triple())?; + emit_serialized(&mut obj, &bytes, target.triple())?; + let mut writer = BufWriter::new(File::create(&output_path)?); + obj.write_stream(&mut writer) + .map_err(|err| anyhow::anyhow!(err.to_string()))?; + writer.flush()?; + } + ObjectFormat::Symbols => { + let engine = store.engine(); + let engine_inner = engine.inner(); + let compiler = engine_inner.compiler()?; + let features = engine_inner.features(); + let tunables = store.tunables(); + let data: Vec = fs::read(wasm_module_path)?; + let prefixer: Option String + Send>> = None; + let (module_info, obj, metadata_length, symbol_registry) = + Artifact::generate_object( + compiler, &data, prefixer, &target, tunables, features, + )?; + + let header_file_src = crate::c_gen::staticlib_header::generate_header_file( + &module_info, + &*symbol_registry, + metadata_length, + ); + let mut writer = BufWriter::new(File::create(&output_path)?); + obj.write_stream(&mut writer) + .map_err(|err| anyhow::anyhow!(err.to_string()))?; + writer.flush()?; + { + let mut writer = BufWriter::new(File::create("/tmp/main_obj.o")?); + obj.write_stream(&mut writer) + .map_err(|err| anyhow::anyhow!(err.to_string()))?; + writer.flush()?; + } + let mut writer = BufWriter::new(File::create("func.c")?); + writer.write_all(header_file_src.as_bytes())?; + writer.flush()?; + { + let mut writer = BufWriter::new(File::create("/tmp/func.c")?); + writer.write_all(header_file_src.as_bytes())?; + writer.flush()?; + } + //link(output_path.clone(), std::path::Path::new("func.c").into())?; + } + } + + eprintln!( + "✔ Object compiled successfully to `{}`.", + self.output.display(), + ); + + Ok(()) + } +} +fn link( + output_path: PathBuf, + object_path: PathBuf, + header_code_path: PathBuf, +) -> anyhow::Result<()> { + let libwasmer_path = get_libwasmer_path()? + .canonicalize() + .context("Failed to find libwasmer")?; + println!( + "link output {:?}", + Command::new("cc") + .arg(&header_code_path) + .arg(&format!("-L{}", libwasmer_path.display())) + //.arg(&format!("-I{}", header_code_path.display())) + .arg("-pie") + .arg("-o") + .arg("header_obj.o") + .output()? + ); + //ld -relocatable a.o b.o -o c.o + + println!( + "link output {:?}", + Command::new("ld") + .arg("-relocatable") + .arg(&object_path) + .arg("header_obj.o") + .arg("-o") + .arg(&output_path) + .output()? + ); + + Ok(()) +} + +/// path to the static libwasmer +fn get_libwasmer_path() -> anyhow::Result { + let mut path = get_wasmer_dir()?; + path.push("lib"); + + // TODO: prefer headless Wasmer if/when it's a separate library. + #[cfg(not(windows))] + path.push("libwasmer.a"); + #[cfg(windows)] + path.push("wasmer.lib"); + + Ok(path) +} +fn get_wasmer_dir() -> anyhow::Result { + Ok(PathBuf::from( + env::var("WASMER_DIR") + .or_else(|e| { + option_env!("WASMER_INSTALL_PREFIX") + .map(str::to_string) + .ok_or(e) + }) + .context("Trying to read env var `WASMER_DIR`")?, + )) +} diff --git a/lib/cli/src/commands/wasmer_static_create_exe_main.c b/lib/cli/src/commands/wasmer_static_create_exe_main.c new file mode 100644 index 00000000000..2276ed3bc75 --- /dev/null +++ b/lib/cli/src/commands/wasmer_static_create_exe_main.c @@ -0,0 +1,192 @@ +#include "wasmer.h" +#include "static_defs.h" +#include +#include +#include + +#define own + +// TODO: make this define templated so that the Rust code can toggle it on/off +#define WASI + +extern wasm_module_t* wasmer_module_new(wasm_store_t* store) asm("wasmer_module_new"); +extern wasm_module_t* wasmer_static_module_new(wasm_store_t* store,const char* wasm_name) asm("wasmer_static_module_new"); + + +static void print_wasmer_error() { + int error_len = wasmer_last_error_length(); + printf("Error len: `%d`\n", error_len); + char *error_str = (char *)malloc(error_len); + wasmer_last_error_message(error_str, error_len); + printf("%s\n", error_str); + free(error_str); +} + +#ifdef WASI +static void pass_mapdir_arg(wasi_config_t *wasi_config, char *mapdir) { + int colon_location = strchr(mapdir, ':') - mapdir; + if (colon_location == 0) { + // error malformed argument + fprintf(stderr, "Expected mapdir argument of the form alias:directory\n"); + exit(-1); + } + + char *alias = (char *)malloc(colon_location + 1); + memcpy(alias, mapdir, colon_location); + alias[colon_location] = '\0'; + + int dir_len = strlen(mapdir) - colon_location; + char *dir = (char *)malloc(dir_len + 1); + memcpy(dir, &mapdir[colon_location + 1], dir_len); + dir[dir_len] = '\0'; + + wasi_config_mapdir(wasi_config, alias, dir); + free(alias); + free(dir); +} + +// We try to parse out `--dir` and `--mapdir` ahead of time and process those +// specially. All other arguments are passed to the guest program. +static void handle_arguments(wasi_config_t *wasi_config, int argc, + char *argv[]) { + for (int i = 1; i < argc; ++i) { + // We probably want special args like `--dir` and `--mapdir` to not be + // passed directly + if (strcmp(argv[i], "--dir") == 0) { + // next arg is a preopen directory + if ((i + 1) < argc) { + i++; + wasi_config_preopen_dir(wasi_config, argv[i]); + } else { + fprintf(stderr, "--dir expects a following argument specifying which " + "directory to preopen\n"); + exit(-1); + } + } else if (strcmp(argv[i], "--mapdir") == 0) { + // next arg is a mapdir + if ((i + 1) < argc) { + i++; + pass_mapdir_arg(wasi_config, argv[i]); + } else { + fprintf(stderr, + "--mapdir expects a following argument specifying which " + "directory to preopen in the form alias:directory\n"); + exit(-1); + } + } else if (strncmp(argv[i], "--dir=", strlen("--dir=")) == 0) { + // this arg is a preopen dir + char *dir = argv[i] + strlen("--dir="); + wasi_config_preopen_dir(wasi_config, dir); + } else if (strncmp(argv[i], "--mapdir=", strlen("--mapdir=")) == 0) { + // this arg is a mapdir + char *mapdir = argv[i] + strlen("--mapdir="); + pass_mapdir_arg(wasi_config, mapdir); + } else { + // guest argument + wasi_config_arg(wasi_config, argv[i]); + } + } +} +#endif + +int main(int argc, char *argv[]) { + wasm_config_t *config = wasm_config_new(); + wasm_engine_t *engine = wasm_engine_new_with_config(config); + wasm_store_t *store = wasm_store_new(engine); + + wasm_module_t *module = wasmer_static_module_new(store, "module"); + + if (!module) { + fprintf(stderr, "Failed to create module\n"); + print_wasmer_error(); + return -1; + } + + // We have now finished the memory buffer book keeping and we have a valid + // Module. + +#ifdef WASI + wasi_config_t *wasi_config = wasi_config_new(argv[0]); + handle_arguments(wasi_config, argc, argv); + + wasi_env_t *wasi_env = wasi_env_new(store, wasi_config); + if (!wasi_env) { + fprintf(stderr, "Error building WASI env!\n"); + print_wasmer_error(); + return 1; + } +#endif + + wasm_importtype_vec_t import_types; + wasm_module_imports(module, &import_types); + + wasm_extern_vec_t imports; + wasm_extern_vec_new_uninitialized(&imports, import_types.size); + wasm_importtype_vec_delete(&import_types); + +#ifdef WASI + bool get_imports_result = wasi_get_imports(store, wasi_env, module, &imports); + + if (!get_imports_result) { + fprintf(stderr, "Error getting WASI imports!\n"); + print_wasmer_error(); + + return 1; + } +#endif + + wasm_instance_t *instance = wasm_instance_new(store, module, &imports, NULL); + + if (!instance) { + fprintf(stderr, "Failed to create instance\n"); + print_wasmer_error(); + return -1; + } + +#ifdef WASI + // Read the exports. + wasm_extern_vec_t exports; + wasm_instance_exports(instance, &exports); + wasm_memory_t* mem = NULL; + for (size_t i = 0; i < exports.size; i++) { + mem = wasm_extern_as_memory(exports.data[i]); + if (mem) { + break; + } + } + + if (!mem) { + fprintf(stderr, "Failed to create instance: Could not find memory in exports\n"); + print_wasmer_error(); + return -1; + } + wasi_env_set_memory(wasi_env, mem); + + own wasm_func_t *start_function = wasi_get_start_function(instance); + if (!start_function) { + fprintf(stderr, "`_start` function not found\n"); + print_wasmer_error(); + return -1; + } + + wasm_val_vec_t args = WASM_EMPTY_VEC; + wasm_val_vec_t results = WASM_EMPTY_VEC; + own wasm_trap_t *trap = wasm_func_call(start_function, &args, &results); + if (trap) { + fprintf(stderr, "Trap is not NULL: TODO:\n"); + return -1; + } +#endif + + // TODO: handle non-WASI start (maybe with invoke?) + +#ifdef WASI + wasi_env_delete(wasi_env); + wasm_extern_vec_delete(&exports); +#endif + wasm_instance_delete(instance); + wasm_module_delete(module); + wasm_store_delete(store); + wasm_engine_delete(engine); + return 0; +} diff --git a/lib/cli/src/lib.rs b/lib/cli/src/lib.rs index 4cb32b02bee..7a80b9fae69 100644 --- a/lib/cli/src/lib.rs +++ b/lib/cli/src/lib.rs @@ -20,6 +20,7 @@ pub mod commands; pub mod common; #[macro_use] pub mod error; +pub mod c_gen; pub mod cli; #[cfg(feature = "debug")] pub mod logging; diff --git a/lib/compiler/src/engine/artifact.rs b/lib/compiler/src/engine/artifact.rs index 46ba30f560a..cece7a14c2e 100644 --- a/lib/compiler/src/engine/artifact.rs +++ b/lib/compiler/src/engine/artifact.rs @@ -5,15 +5,19 @@ use crate::engine::link::link_module; use crate::ArtifactBuild; use crate::ArtifactCreate; use crate::Features; -#[cfg(feature = "static-artifact-create")] -use crate::{FunctionBodyData, Compiler, ModuleTranslationState}; +use crate::ModuleEnvironment; use crate::{ register_frame_info, resolve_imports, FunctionExtent, GlobalFrameInfoRegistration, InstantiationError, RuntimeError, Tunables, }; +#[cfg(feature = "static-artifact-create")] +use crate::{Compiler, FunctionBodyData, ModuleTranslationState}; use crate::{Engine, EngineInner}; -use crate::ModuleEnvironment; use enumset::EnumSet; +#[cfg(feature = "static-artifact-create")] +use std::collections::BTreeMap; +#[cfg(feature = "static-artifact-create")] +use std::mem; use std::sync::Arc; use std::sync::Mutex; #[cfg(feature = "static-artifact-create")] @@ -21,8 +25,11 @@ use wasmer_object::{emit_compilation, emit_data, get_object_for_target, Object}; use wasmer_types::entity::{BoxedSlice, PrimaryMap}; use wasmer_types::MetadataHeader; #[cfg(feature = "static-artifact-create")] -#[cfg(feature = "compiler")] -use wasmer_types::{compilation::symbols::ModuleMetadata, CompileModuleInfo, Target}; +use wasmer_types::{ + compilation::symbols::{ModuleMetadata, ModuleMetadataSymbolRegistry}, + entity::EntityRef, + CompileModuleInfo, Target, +}; use wasmer_types::{ CompileError, CpuFeature, DataInitializer, DeserializeError, FunctionIndex, LocalFunctionIndex, MemoryIndex, ModuleInfo, OwnedDataInitializer, SerializableModule, SerializeError, @@ -32,7 +39,17 @@ use wasmer_vm::{FunctionBodyPtr, MemoryStyle, TableStyle, VMSharedSignatureIndex use wasmer_vm::{InstanceAllocator, InstanceHandle, StoreObjects, TrapHandlerFn, VMExtern}; /// A compiled wasm module, ready to be instantiated. -pub struct Artifact { +pub enum Artifact { + /// The default artifact format. + Universal(UniversalArtifact), + /// Stores functions etc as symbols and data meant to be stored in object files and + /// executables. + #[cfg(feature = "static-artifact-create")] + Static(StaticArtifact), +} + +/// The default artifact format. +pub struct UniversalArtifact { artifact: ArtifactBuild, finished_functions: BoxedSlice, finished_function_call_trampolines: BoxedSlice, @@ -42,6 +59,24 @@ pub struct Artifact { finished_function_lengths: BoxedSlice, } +/// Stores functions etc as symbols and data meant to be stored in object files and +/// executables. +#[cfg(feature = "static-artifact-create")] +pub struct StaticArtifact { + metadata: ModuleMetadata, + _module_bytes: Vec, + finished_functions: BoxedSlice, + finished_function_call_trampolines: BoxedSlice, + finished_dynamic_function_trampolines: BoxedSlice, + signatures: BoxedSlice, + // Length of the serialized metadata + _metadata_length: usize, + _symbol_registry: ModuleMetadataSymbolRegistry, +} + +#[cfg(feature = "static-artifact-create")] +const WASMER_METADATA_SYMBOL: &[u8] = b"WASMER_METADATA"; + impl Artifact { /// Compile a data buffer into a `ArtifactBuild`, which may then be instantiated. #[cfg(feature = "compiler")] @@ -91,6 +126,15 @@ impl Artifact { /// the data. pub unsafe fn deserialize(engine: &Engine, bytes: &[u8]) -> Result { if !ArtifactBuild::is_deserializable(bytes) { + let static_artifact = Artifact::deserialize_object(engine, bytes); + match static_artifact { + Ok(v) => { + return Ok(v); + } + Err(err) => { + eprintln!("Could not deserialize as static object: {}", err); + } + } return Err(DeserializeError::Incompatible( "The provided bytes are not wasmer-universal".to_string(), )); @@ -177,7 +221,7 @@ impl Artifact { finished_dynamic_function_trampolines.into_boxed_slice(); let signatures = signatures.into_boxed_slice(); - Ok(Self { + Ok(Self::Universal(UniversalArtifact { artifact, finished_functions, finished_function_call_trampolines, @@ -185,7 +229,7 @@ impl Artifact { signatures, frame_info_registration: Mutex::new(None), finished_function_lengths, - }) + })) } /// Check if the provided bytes look like a serialized `ArtifactBuild`. @@ -196,31 +240,59 @@ impl Artifact { impl ArtifactCreate for Artifact { fn create_module_info(&self) -> ModuleInfo { - self.artifact.create_module_info() + match self { + Self::Universal(UniversalArtifact { artifact, .. }) => artifact.create_module_info(), + #[cfg(feature = "static-artifact-create")] + Self::Static(StaticArtifact { metadata, .. }) => metadata.compile_info.module.clone(), + } } fn features(&self) -> &Features { - self.artifact.features() + match self { + Self::Universal(UniversalArtifact { artifact, .. }) => artifact.features(), + #[cfg(feature = "static-artifact-create")] + Self::Static(StaticArtifact { metadata, .. }) => &metadata.compile_info.features, + } } fn cpu_features(&self) -> EnumSet { - self.artifact.cpu_features() + match self { + Self::Universal(_self) => _self.artifact.cpu_features(), + #[cfg(feature = "static-artifact-create")] + Self::Static(_self) => EnumSet::from_u64(_self.metadata.cpu_features), + } } fn data_initializers(&self) -> &[OwnedDataInitializer] { - self.artifact.data_initializers() + match self { + Self::Universal(_self) => _self.artifact.data_initializers(), + #[cfg(feature = "static-artifact-create")] + Self::Static(_self) => &_self.metadata.data_initializers, + } } fn memory_styles(&self) -> &PrimaryMap { - self.artifact.memory_styles() + match self { + Self::Universal(_self) => _self.artifact.memory_styles(), + #[cfg(feature = "static-artifact-create")] + Self::Static(_self) => &_self.metadata.compile_info.memory_styles, + } } fn table_styles(&self) -> &PrimaryMap { - self.artifact.table_styles() + match self { + Self::Universal(UniversalArtifact { artifact, .. }) => artifact.table_styles(), + #[cfg(feature = "static-artifact-create")] + Self::Static(StaticArtifact { metadata, .. }) => &metadata.compile_info.table_styles, + } } fn serialize(&self) -> Result, SerializeError> { - self.artifact.serialize() + match self { + Self::Universal(UniversalArtifact { artifact, .. }) => artifact.serialize(), + #[cfg(feature = "static-artifact-create")] + Self::Static(StaticArtifact { .. }) => todo!(), + } } } @@ -229,39 +301,55 @@ impl Artifact { /// /// This is required to ensure that any traps can be properly symbolicated. pub fn register_frame_info(&self) { - let mut info = self.frame_info_registration.lock().unwrap(); - - if info.is_some() { - return; + match self { + Self::Universal(_self) => { + let mut info = _self.frame_info_registration.lock().unwrap(); + + if info.is_some() { + return; + } + + let finished_function_extents = _self + .finished_functions + .values() + .copied() + .zip(_self.finished_function_lengths.values().copied()) + .map(|(ptr, length)| FunctionExtent { ptr, length }) + .collect::>() + .into_boxed_slice(); + + let frame_infos = _self.artifact.get_frame_info_ref(); + *info = register_frame_info( + _self.artifact.create_module_info(), + &finished_function_extents, + frame_infos.clone(), + ); + } + #[cfg(feature = "static-artifact-create")] + Self::Static(_self) => { + // Do nothing for static artifact + } } - - let finished_function_extents = self - .finished_functions - .values() - .copied() - .zip(self.finished_function_lengths.values().copied()) - .map(|(ptr, length)| FunctionExtent { ptr, length }) - .collect::>() - .into_boxed_slice(); - - let frame_infos = self.artifact.get_frame_info_ref(); - *info = register_frame_info( - self.artifact.create_module_info(), - &finished_function_extents, - frame_infos.clone(), - ); } /// Returns the functions allocated in memory or this `Artifact` /// ready to be run. pub fn finished_functions(&self) -> &BoxedSlice { - &self.finished_functions + match self { + Self::Universal(_self) => &_self.finished_functions, + #[cfg(feature = "static-artifact-create")] + Self::Static(_self) => &_self.finished_functions, + } } /// Returns the function call trampolines allocated in memory of this /// `Artifact`, ready to be run. pub fn finished_function_call_trampolines(&self) -> &BoxedSlice { - &self.finished_function_call_trampolines + match self { + Self::Universal(_self) => &_self.finished_function_call_trampolines, + #[cfg(feature = "static-artifact-create")] + Self::Static(_self) => &_self.finished_function_call_trampolines, + } } /// Returns the dynamic function trampolines allocated in memory @@ -269,12 +357,20 @@ impl Artifact { pub fn finished_dynamic_function_trampolines( &self, ) -> &BoxedSlice { - &self.finished_dynamic_function_trampolines + match self { + Self::Universal(_self) => &_self.finished_dynamic_function_trampolines, + #[cfg(feature = "static-artifact-create")] + Self::Static(_self) => &_self.finished_dynamic_function_trampolines, + } } /// Returns the associated VM signatures for this `Artifact`. pub fn signatures(&self) -> &BoxedSlice { - &self.signatures + match self { + Self::Universal(_self) => &_self.signatures, + #[cfg(feature = "static-artifact-create")] + Self::Static(_self) => &_self.signatures, + } } /// Do preinstantiation logic that is executed before instantiating @@ -388,10 +484,10 @@ impl Artifact { #[cfg(feature = "static-artifact-create")] /// Generate a compilation fn generate_metadata<'data>( - &self, data: &'data [u8], compiler: &dyn Compiler, tunables: &dyn Tunables, + features: &Features, ) -> Result< ( CompileModuleInfo, @@ -403,7 +499,6 @@ impl Artifact { > { let environ = ModuleEnvironment::new(); let translation = environ.translate(data).map_err(CompileError::Wasm)?; - let features = self.features().clone(); // We try to apply the middleware first use crate::translator::ModuleMiddlewareChain; @@ -424,7 +519,7 @@ impl Artifact { let compile_info = CompileModuleInfo { module, - features, + features: features.clone(), memory_styles, table_styles, }; @@ -443,22 +538,28 @@ impl Artifact { /// so we can assure no collisions. #[cfg(feature = "static-artifact-create")] pub fn generate_object<'data>( - &self, compiler: &dyn Compiler, data: &[u8], prefixer: Option String + Send>>, target: &'data Target, tunables: &dyn Tunables, - ) -> Result, CompileError> { + features: &Features, + ) -> Result< + ( + ModuleInfo, + Object<'data>, + usize, + Box, + ), + CompileError, + > { fn to_compile_error(err: impl std::error::Error) -> CompileError { CompileError::Codegen(format!("{}", err)) } #[allow(dead_code)] - const WASMER_METADATA_SYMBOL: &[u8] = b"WASMER_METADATA"; - let (compile_info, function_body_inputs, data_initializers, module_translation) = - self.generate_metadata(data, compiler, tunables)?; + Self::generate_metadata(data, compiler, tunables, features)?; let data_initializers = data_initializers .iter() @@ -508,21 +609,150 @@ impl Artifact { let (_compile_info, symbol_registry) = metadata.split(); - let compilation = compiler.compile_module( - &target, - &metadata.compile_info, - module_translation.as_ref().unwrap(), - function_body_inputs, - )?; - + let compilation: wasmer_types::compilation::function::Compilation = compiler + .compile_module( + &target, + &metadata.compile_info, + module_translation.as_ref().unwrap(), + function_body_inputs, + )?; let mut obj = get_object_for_target(&target_triple).map_err(to_compile_error)?; - emit_data(&mut obj, b"WASMER_MODULE_METADATA", &metadata_binary, 1) + emit_data(&mut obj, WASMER_METADATA_SYMBOL, &metadata_binary, 1) .map_err(to_compile_error)?; emit_compilation(&mut obj, compilation, &symbol_registry, &target_triple) .map_err(to_compile_error)?; + Ok(( + metadata.compile_info.module, + obj, + metadata_binary.len(), + Box::new(symbol_registry), + )) + } + + /// Deserialize a ArtifactBuild from an object file + /// + /// # Safety + /// The object must be a valid static object generated by wasmer. + #[cfg(not(feature = "static-artifact-load"))] + pub unsafe fn deserialize_object( + _engine: &Engine, + _bytes: &[u8], + ) -> Result { + Err(DeserializeError::Compiler( + CompileError::UnsupportedFeature("static load is not compiled in".to_string()), + )) + } + + /// Deserialize a ArtifactBuild from an object file + /// + /// # Safety + /// The object must be a valid static object generated by wasmer. + #[cfg(feature = "static-artifact-load")] + pub unsafe fn deserialize_object( + engine: &Engine, + bytes: &[u8], + ) -> Result { + let metadata_len = MetadataHeader::parse(bytes)?; + + let metadata_slice = &bytes[MetadataHeader::LEN..][..metadata_len]; + let mut metadata: ModuleMetadata = ModuleMetadata::deserialize(metadata_slice)?; + + const WORD_SIZE: usize = mem::size_of::(); + let mut byte_buffer = [0u8; WORD_SIZE]; + + let mut cur_offset = MetadataHeader::LEN + metadata_len; + byte_buffer[0..WORD_SIZE].clone_from_slice(&bytes[cur_offset..(cur_offset + WORD_SIZE)]); + cur_offset += WORD_SIZE; + + let num_finished_functions = usize::from_ne_bytes(byte_buffer); + let mut finished_functions: PrimaryMap = + PrimaryMap::new(); + + let engine_inner = engine.inner(); + let signature_registry = engine_inner.signatures(); + let mut sig_map: BTreeMap = BTreeMap::new(); + + let num_imported_functions = metadata.compile_info.module.num_imported_functions; + // set up the imported functions first... + for i in 0..num_imported_functions { + let sig_idx = metadata.compile_info.module.functions[FunctionIndex::new(i)]; + let func_type = &metadata.compile_info.module.signatures[sig_idx]; + let vm_shared_idx = signature_registry.register(&func_type); + sig_map.insert(sig_idx, vm_shared_idx); + } + // read finished functions in order now... + for i in 0..num_finished_functions { + let local_func_idx = LocalFunctionIndex::new(i); + let func_idx = metadata.compile_info.module.func_index(local_func_idx); + let sig_idx = metadata.compile_info.module.functions[func_idx]; + let func_type = &metadata.compile_info.module.signatures[sig_idx]; + let vm_shared_idx = signature_registry.register(&func_type); + sig_map.insert(sig_idx, vm_shared_idx); + + byte_buffer[0..WORD_SIZE] + .clone_from_slice(&bytes[cur_offset..(cur_offset + WORD_SIZE)]); + let fp = FunctionBodyPtr(usize::from_ne_bytes(byte_buffer) as _); + cur_offset += WORD_SIZE; + + // TODO: we can read back the length here if we serialize it. This will improve debug output. + + finished_functions.push(fp); + } + + let mut signatures: PrimaryMap<_, VMSharedSignatureIndex> = PrimaryMap::new(); + for i in 0..(sig_map.len()) { + if let Some(shared_idx) = sig_map.get(&SignatureIndex::new(i)) { + signatures.push(*shared_idx); + } else { + panic!("Invalid data, missing sig idx; TODO: handle this error"); + } + } + + // read trampolines in order + let mut finished_function_call_trampolines = PrimaryMap::new(); + byte_buffer[0..WORD_SIZE].clone_from_slice(&bytes[cur_offset..(cur_offset + WORD_SIZE)]); + cur_offset += WORD_SIZE; + let num_function_trampolines = usize::from_ne_bytes(byte_buffer); + for _ in 0..num_function_trampolines { + byte_buffer[0..WORD_SIZE] + .clone_from_slice(&bytes[cur_offset..(cur_offset + WORD_SIZE)]); + cur_offset += WORD_SIZE; + let trampoline_ptr_bytes = usize::from_ne_bytes(byte_buffer); + let trampoline = mem::transmute::(trampoline_ptr_bytes); + finished_function_call_trampolines.push(trampoline); + // TODO: we can read back the length here if we serialize it. This will improve debug output. + } + + // read dynamic function trampolines in order now... + let mut finished_dynamic_function_trampolines = PrimaryMap::new(); + byte_buffer[0..WORD_SIZE].clone_from_slice(&bytes[cur_offset..(cur_offset + WORD_SIZE)]); + cur_offset += WORD_SIZE; + let num_dynamic_trampoline_functions = usize::from_ne_bytes(byte_buffer); + for _i in 0..num_dynamic_trampoline_functions { + byte_buffer[0..WORD_SIZE] + .clone_from_slice(&bytes[cur_offset..(cur_offset + WORD_SIZE)]); + let fp = FunctionBodyPtr(usize::from_ne_bytes(byte_buffer) as _); + cur_offset += WORD_SIZE; + + // TODO: we can read back the length here if we serialize it. This will improve debug output. + + finished_dynamic_function_trampolines.push(fp); + } - Ok(obj) + let (_compile_info, _symbol_registry) = metadata.split(); + Ok(Self::Static(StaticArtifact { + metadata, + _module_bytes: bytes.to_vec(), + finished_functions: finished_functions.into_boxed_slice(), + finished_function_call_trampolines: finished_function_call_trampolines + .into_boxed_slice(), + finished_dynamic_function_trampolines: finished_dynamic_function_trampolines + .into_boxed_slice(), + signatures: signatures.into_boxed_slice(), + _metadata_length: metadata_len, + _symbol_registry, + })) } } diff --git a/lib/types/src/compilation/symbols.rs b/lib/types/src/compilation/symbols.rs index 7201b60f5c6..6971d4f6a11 100644 --- a/lib/types/src/compilation/symbols.rs +++ b/lib/types/src/compilation/symbols.rs @@ -57,26 +57,26 @@ pub trait SymbolRegistry: Send + Sync { #[derive(Debug, RkyvSerialize, RkyvDeserialize, Archive)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub struct ModuleMetadata { - /// FIXME + /// Compile info pub compile_info: CompileModuleInfo, - /// FIXME + /// Prefix for function etc symbols pub prefix: String, - /// FIXME + /// Data initializers pub data_initializers: Box<[OwnedDataInitializer]>, /// The function body lengths (used to find function by address) pub function_body_lengths: PrimaryMap, - /// FIXME + /// CPU features used (See [`CpuFeature`]) pub cpu_features: u64, } -/// FIXME +/// A simple metadata registry pub struct ModuleMetadataSymbolRegistry { - /// FIXME + /// Symbol prefix stirng pub prefix: String, } impl ModuleMetadata { - /// FIXME + /// Get mutable ref to compile info and a copy of the registry pub fn split(&mut self) -> (&mut CompileModuleInfo, ModuleMetadataSymbolRegistry) { let compile_info = &mut self.compile_info; let symbol_registry = ModuleMetadataSymbolRegistry { @@ -85,7 +85,7 @@ impl ModuleMetadata { (compile_info, symbol_registry) } - /// FIXME + /// Returns symbol registry. pub fn get_symbol_registry(&self) -> ModuleMetadataSymbolRegistry { ModuleMetadataSymbolRegistry { prefix: self.prefix.clone(), diff --git a/lib/types/src/error.rs b/lib/types/src/error.rs index 115cb1ae6c1..53555d6239c 100644 --- a/lib/types/src/error.rs +++ b/lib/types/src/error.rs @@ -34,7 +34,7 @@ pub enum DeserializeError { /// The binary was valid, but we got an error when /// trying to allocate the required resources. #[error(transparent)] - Compiler(CompileError), + Compiler(#[from] CompileError), } /// An ImportError.