Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Metering middleware. #1839

Merged
merged 15 commits into from
Dec 1, 2020
Merged
Show file tree
Hide file tree
Changes from 14 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 3 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ wasmer-wasi = { version = "1.0.0-alpha5", path = "lib/wasi", optional = true }
wasmer-wast = { version = "1.0.0-alpha5", path = "tests/lib/wast", optional = true }
wasmer-cache = { version = "1.0.0-alpha5", path = "lib/cache", optional = true }
wasmer-types = { version = "1.0.0-alpha5", path = "lib/wasmer-types" }
wasmer-middlewares = { version = "1.0.0-alpha5", path = "lib/middlewares", optional = true }
cfg-if = "1.0"

[workspace]
Expand Down Expand Up @@ -79,6 +80,7 @@ default = [
"cache",
"wasi",
# "emscripten",
"middlewares",
]
engine = []
jit = [
Expand Down Expand Up @@ -117,6 +119,7 @@ llvm = [
"wasmer-compiler-llvm",
"compiler",
]
middlewares = ["wasmer-middlewares"]

# Testing features
test-singlepass = [
Expand Down
3 changes: 1 addition & 2 deletions lib/api/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -71,8 +71,7 @@ pub use crate::utils::is_wasm;
pub use target_lexicon::{Architecture, CallingConvention, OperatingSystem, Triple, HOST};
#[cfg(feature = "compiler")]
pub use wasmer_compiler::{
wasmparser, CompilerConfig, FunctionMiddleware, FunctionMiddlewareGenerator,
MiddlewareReaderState,
wasmparser, CompilerConfig, FunctionMiddleware, MiddlewareReaderState, ModuleMiddleware,
};
pub use wasmer_compiler::{CpuFeature, Features, Target};
pub use wasmer_engine::{
Expand Down
13 changes: 9 additions & 4 deletions lib/compiler-cranelift/src/compiler.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,13 @@ use cranelift_codegen::{binemit, Context};
#[cfg(feature = "unwind")]
use gimli::write::{Address, EhFrame, FrameTable};
use rayon::prelude::{IntoParallelRefIterator, ParallelIterator};
use std::sync::Arc;
use wasmer_compiler::CompileError;
use wasmer_compiler::{CallingConvention, ModuleTranslationState, Target};
use wasmer_compiler::{
Compilation, CompileModuleInfo, CompiledFunction, CompiledFunctionFrameInfo,
CompiledFunctionUnwindInfo, Compiler, Dwarf, FunctionBody, FunctionBodyData, SectionIndex,
CompiledFunctionUnwindInfo, Compiler, Dwarf, FunctionBody, FunctionBodyData,
ModuleMiddlewareChain, SectionIndex,
};
use wasmer_types::entity::{EntityRef, PrimaryMap};
use wasmer_types::{FunctionIndex, LocalFunctionIndex, SignatureIndex};
Expand Down Expand Up @@ -54,14 +56,17 @@ impl Compiler for CraneliftCompiler {
fn compile_module(
&self,
target: &Target,
compile_info: &CompileModuleInfo,
compile_info: &mut CompileModuleInfo,
module_translation_state: &ModuleTranslationState,
function_body_inputs: PrimaryMap<LocalFunctionIndex, FunctionBodyData<'_>>,
) -> Result<Compilation, CompileError> {
let isa = self.config().isa(target);
let frontend_config = isa.frontend_config();
let memory_styles = &compile_info.memory_styles;
let table_styles = &compile_info.table_styles;
let mut module = (*compile_info.module).clone();
self.config.middlewares.apply_on_module_info(&mut module);
compile_info.module = Arc::new(module);
let module = &compile_info.module;
let signatures = module
.signatures
Expand All @@ -77,7 +82,7 @@ impl Compiler for CraneliftCompiler {
// FDEs will cause some issues in Linux.
None
} else {
use std::sync::{Arc, Mutex};
use std::sync::Mutex;
match target.triple().default_calling_convention() {
Ok(CallingConvention::SystemV) => {
match isa.create_systemv_cie() {
Expand Down Expand Up @@ -125,7 +130,7 @@ impl Compiler for CraneliftCompiler {
)?;

let mut code_buf: Vec<u8> = Vec::new();
let mut reloc_sink = RelocSink::new(module, func_index);
let mut reloc_sink = RelocSink::new(&module, func_index);
let mut trap_sink = TrapSink::new();
let mut stackmap_sink = binemit::NullStackMapSink {};
context
Expand Down
6 changes: 3 additions & 3 deletions lib/compiler-cranelift/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ use cranelift_codegen::isa::{lookup, TargetIsa};
use cranelift_codegen::settings::{self, Configurable};
use std::sync::Arc;
use wasmer_compiler::{
Architecture, Compiler, CompilerConfig, CpuFeature, FunctionMiddlewareGenerator, Target,
Architecture, Compiler, CompilerConfig, CpuFeature, ModuleMiddleware, Target,
};

// Runtime Environment
Expand Down Expand Up @@ -35,7 +35,7 @@ pub struct Cranelift {
enable_pic: bool,
opt_level: OptLevel,
/// The middleware chain.
pub(crate) middlewares: Vec<Arc<dyn FunctionMiddlewareGenerator>>,
pub(crate) middlewares: Vec<Arc<dyn ModuleMiddleware>>,
}

impl Cranelift {
Expand Down Expand Up @@ -199,7 +199,7 @@ impl CompilerConfig for Cranelift {
}

/// Pushes a middleware onto the back of the middleware chain.
fn push_middleware(&mut self, middleware: Arc<dyn FunctionMiddlewareGenerator>) {
fn push_middleware(&mut self, middleware: Arc<dyn ModuleMiddleware>) {
self.middlewares.push(middleware);
}
}
Expand Down
4 changes: 2 additions & 2 deletions lib/compiler-cranelift/src/translator/func_translator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext, Variable};
use tracing::info;
use wasmer_compiler::wasmparser;
use wasmer_compiler::{
to_wasm_error, wasm_unsupported, GenerateMiddlewareChain, MiddlewareBinaryReader,
to_wasm_error, wasm_unsupported, MiddlewareBinaryReader, ModuleMiddlewareChain,
ModuleTranslationState, WasmResult,
};
use wasmer_types::LocalFunctionIndex;
Expand Down Expand Up @@ -75,7 +75,7 @@ impl FuncTranslator {
reader.set_middleware_chain(
config
.middlewares
.generate_middleware_chain(local_function_index),
.generate_function_middleware_chain(local_function_index),
);
self.translate_from_reader(module_translation_state, reader, func, environ)
}
Expand Down
23 changes: 16 additions & 7 deletions lib/compiler-llvm/src/compiler.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,16 @@ use inkwell::module::{Linkage, Module};
use inkwell::targets::FileType;
use inkwell::DLLStorageClass;
use rayon::prelude::{IntoParallelRefIterator, ParallelIterator};
use std::sync::Arc;
use wasmer_compiler::{
Compilation, CompileError, CompileModuleInfo, Compiler, CustomSection, CustomSectionProtection,
Dwarf, FunctionBodyData, ModuleTranslationState, RelocationTarget, SectionBody, SectionIndex,
Symbol, SymbolRegistry, Target,
Dwarf, FunctionBodyData, ModuleMiddlewareChain, ModuleTranslationState, RelocationTarget,
SectionBody, SectionIndex, Symbol, SymbolRegistry, Target,
};
use wasmer_types::entity::{EntityRef, PrimaryMap};
use wasmer_types::{FunctionIndex, LocalFunctionIndex, SignatureIndex};

//use std::sync::{Arc, Mutex};
//use std::sync::Mutex;

/// A compiler that compiles a WebAssembly module with LLVM, translating the Wasm to LLVM IR,
/// optimizing it and then translating to assembly.
Expand Down Expand Up @@ -210,17 +211,21 @@ impl Compiler for LLVMCompiler {
fn experimental_native_compile_module<'data, 'module>(
&self,
target: &Target,
module: &'module CompileModuleInfo,
compile_info: &'module mut CompileModuleInfo,
module_translation: &ModuleTranslationState,
// The list of function bodies
function_body_inputs: &PrimaryMap<LocalFunctionIndex, FunctionBodyData<'data>>,
symbol_registry: &dyn SymbolRegistry,
// The metadata to inject into the wasmer_metadata section of the object file.
wasmer_metadata: &[u8],
) -> Option<Result<Vec<u8>, CompileError>> {
let mut module = (*compile_info.module).clone();
self.config.middlewares.apply_on_module_info(&mut module);
compile_info.module = Arc::new(module);

Some(self.compile_native_object(
target,
module,
compile_info,
module_translation,
function_body_inputs,
symbol_registry,
Expand All @@ -233,13 +238,17 @@ impl Compiler for LLVMCompiler {
fn compile_module<'data, 'module>(
&self,
target: &Target,
compile_info: &'module CompileModuleInfo,
compile_info: &'module mut CompileModuleInfo,
module_translation: &ModuleTranslationState,
function_body_inputs: PrimaryMap<LocalFunctionIndex, FunctionBodyData<'data>>,
) -> Result<Compilation, CompileError> {
//let data = Arc::new(Mutex::new(0));
let memory_styles = &compile_info.memory_styles;
let table_styles = &compile_info.table_styles;

let mut module = (*compile_info.module).clone();
self.config.middlewares.apply_on_module_info(&mut module);
compile_info.module = Arc::new(module);
let module = &compile_info.module;

// TODO: merge constants in sections.
Expand All @@ -260,7 +269,7 @@ impl Compiler for LLVMCompiler {
// TODO: remove (to serialize)
//let _data = data.lock().unwrap();
func_translator.translate(
&module,
module,
module_translation,
i,
input,
Expand Down
6 changes: 3 additions & 3 deletions lib/compiler-llvm/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ use itertools::Itertools;
use std::fmt::Debug;
use std::sync::Arc;
use target_lexicon::Architecture;
use wasmer_compiler::{Compiler, CompilerConfig, FunctionMiddlewareGenerator, Target, Triple};
use wasmer_compiler::{Compiler, CompilerConfig, ModuleMiddleware, Target, Triple};
use wasmer_types::{FunctionType, LocalFunctionIndex};

/// The InkWell ModuleInfo type
Expand Down Expand Up @@ -45,7 +45,7 @@ pub struct LLVM {
is_pic: bool,
pub(crate) callbacks: Option<Arc<dyn LLVMCallbacks>>,
/// The middleware chain.
pub(crate) middlewares: Vec<Arc<dyn FunctionMiddlewareGenerator>>,
pub(crate) middlewares: Vec<Arc<dyn ModuleMiddleware>>,
}

impl LLVM {
Expand Down Expand Up @@ -212,7 +212,7 @@ impl CompilerConfig for LLVM {
}

/// Pushes a middleware onto the back of the middleware chain.
fn push_middleware(&mut self, middleware: Arc<dyn FunctionMiddlewareGenerator>) {
fn push_middleware(&mut self, middleware: Arc<dyn ModuleMiddleware>) {
self.middlewares.push(middleware);
}
}
Expand Down
6 changes: 3 additions & 3 deletions lib/compiler-llvm/src/translator/code.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@ use crate::config::{CompiledKind, LLVM};
use crate::object_file::{load_object_file, CompiledFunction};
use wasmer_compiler::wasmparser::{MemoryImmediate, Operator};
use wasmer_compiler::{
to_wasm_error, wptype_to_type, CompileError, FunctionBodyData, GenerateMiddlewareChain,
MiddlewareBinaryReader, ModuleTranslationState, RelocationTarget, Symbol, SymbolRegistry,
to_wasm_error, wptype_to_type, CompileError, FunctionBodyData, MiddlewareBinaryReader,
ModuleMiddlewareChain, ModuleTranslationState, RelocationTarget, Symbol, SymbolRegistry,
};
use wasmer_types::entity::PrimaryMap;
use wasmer_types::{
Expand Down Expand Up @@ -150,7 +150,7 @@ impl FuncTranslator {
reader.set_middleware_chain(
config
.middlewares
.generate_middleware_chain(*local_func_index),
.generate_function_middleware_chain(*local_func_index),
);

let mut params = vec![];
Expand Down
14 changes: 10 additions & 4 deletions lib/compiler-singlepass/src/compiler.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ use wasmer_compiler::wasmparser::BinaryReaderError;
use wasmer_compiler::TrapInformation;
use wasmer_compiler::{Compilation, CompileError, CompiledFunction, Compiler, SectionIndex};
use wasmer_compiler::{
CompileModuleInfo, CompilerConfig, GenerateMiddlewareChain, MiddlewareBinaryReader,
CompileModuleInfo, CompilerConfig, MiddlewareBinaryReader, ModuleMiddlewareChain,
ModuleTranslationState, Target,
};
use wasmer_compiler::{FunctionBody, FunctionBodyData};
Expand Down Expand Up @@ -47,16 +47,19 @@ impl Compiler for SinglepassCompiler {
fn compile_module(
&self,
_target: &Target,
compile_info: &CompileModuleInfo,
compile_info: &mut CompileModuleInfo,
_module_translation: &ModuleTranslationState,
function_body_inputs: PrimaryMap<LocalFunctionIndex, FunctionBodyData<'_>>,
) -> Result<Compilation, CompileError> {
if compile_info.features.multi_value {
return Err(CompileError::UnsupportedFeature("multivalue".to_string()));
}
let vmoffsets = VMOffsets::new(8, &compile_info.module);
let memory_styles = &compile_info.memory_styles;
let table_styles = &compile_info.table_styles;
let mut module = (*compile_info.module).clone();
self.config.middlewares.apply_on_module_info(&mut module);
compile_info.module = Arc::new(module);
Comment on lines +59 to +61
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This will leave all the old Arc<Module>s unaffected, this should be looked at closely for bugs

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Indeed that was a source of the bug fixed in 363a28c of this PR. I'm trusting that the tests will catch any real problems here. (Also, there are no old Arc's? At least not in this function?)

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

compile_info.module before it gets overwritten is pointing to the old Arc, anything that it's shared with won't get the update

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah I'm not sure about its correctness in context, but it's possible that this could cause problems later

let vmoffsets = VMOffsets::new(8, &compile_info.module);
let module = &compile_info.module;
let import_trampolines: PrimaryMap<SectionIndex, _> = (0..module.num_imported_functions)
.map(FunctionIndex::new)
Expand All @@ -73,7 +76,10 @@ impl Compiler for SinglepassCompiler {
.collect::<Vec<(LocalFunctionIndex, &FunctionBodyData<'_>)>>()
.par_iter()
.map(|(i, input)| {
let middleware_chain = self.config.middlewares.generate_middleware_chain(*i);
let middleware_chain = self
.config
.middlewares
.generate_function_middleware_chain(*i);
let mut reader =
MiddlewareBinaryReader::new_with_offset(input.data, input.module_offset);
reader.set_middleware_chain(middleware_chain);
Expand Down
6 changes: 3 additions & 3 deletions lib/compiler-singlepass/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,15 @@

use crate::compiler::SinglepassCompiler;
use std::sync::Arc;
use wasmer_compiler::{Compiler, CompilerConfig, CpuFeature, FunctionMiddlewareGenerator, Target};
use wasmer_compiler::{Compiler, CompilerConfig, CpuFeature, ModuleMiddleware, Target};
use wasmer_types::Features;

#[derive(Debug, Clone)]
pub struct Singlepass {
pub(crate) enable_nan_canonicalization: bool,
pub(crate) enable_stack_check: bool,
/// The middleware chain.
pub(crate) middlewares: Vec<Arc<dyn FunctionMiddlewareGenerator>>,
pub(crate) middlewares: Vec<Arc<dyn ModuleMiddleware>>,
}

impl Singlepass {
Expand Down Expand Up @@ -66,7 +66,7 @@ impl CompilerConfig for Singlepass {
}

/// Pushes a middleware onto the back of the middleware chain.
fn push_middleware(&mut self, middleware: Arc<dyn FunctionMiddlewareGenerator>) {
fn push_middleware(&mut self, middleware: Arc<dyn ModuleMiddleware>) {
self.middlewares.push(middleware);
}
}
Expand Down
2 changes: 1 addition & 1 deletion lib/compiler/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ pub trait Compiler {
fn compile_module<'data, 'module>(
&self,
target: &Target,
compile_info: &'module CompileModuleInfo,
compile_info: &'module mut CompileModuleInfo,
module_translation: &ModuleTranslationState,
// The list of function bodies
function_body_inputs: PrimaryMap<LocalFunctionIndex, FunctionBodyData<'data>>,
Expand Down
8 changes: 4 additions & 4 deletions lib/compiler/src/compiler.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ use crate::lib::std::boxed::Box;
use crate::lib::std::sync::Arc;
use crate::module::CompileModuleInfo;
use crate::target::Target;
use crate::translator::FunctionMiddlewareGenerator;
use crate::translator::ModuleMiddleware;
use crate::FunctionBodyData;
use crate::ModuleTranslationState;
use crate::SectionIndex;
Expand Down Expand Up @@ -45,7 +45,7 @@ pub trait CompilerConfig {
}

/// Pushes a middleware onto the back of the middleware chain.
fn push_middleware(&mut self, middleware: Arc<dyn FunctionMiddlewareGenerator>);
fn push_middleware(&mut self, middleware: Arc<dyn ModuleMiddleware>);
}

/// An implementation of a Compiler from parsed WebAssembly module to Compiled native code.
Expand Down Expand Up @@ -84,7 +84,7 @@ pub trait Compiler {
fn compile_module<'data, 'module>(
&self,
target: &Target,
module: &'module CompileModuleInfo,
module: &'module mut CompileModuleInfo,
module_translation: &ModuleTranslationState,
// The list of function bodies
function_body_inputs: PrimaryMap<LocalFunctionIndex, FunctionBodyData<'data>>,
Expand All @@ -96,7 +96,7 @@ pub trait Compiler {
fn experimental_native_compile_module<'data, 'module>(
&self,
_target: &Target,
_module: &'module CompileModuleInfo,
_module: &'module mut CompileModuleInfo,
_module_translation: &ModuleTranslationState,
// The list of function bodies
_function_body_inputs: &PrimaryMap<LocalFunctionIndex, FunctionBodyData<'data>>,
Expand Down
4 changes: 2 additions & 2 deletions lib/compiler/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -86,8 +86,8 @@ pub use crate::target::{
#[cfg(feature = "translator")]
pub use crate::translator::{
to_wasm_error, translate_module, wptype_to_type, FunctionBodyData, FunctionMiddleware,
FunctionMiddlewareGenerator, GenerateMiddlewareChain, MiddlewareBinaryReader,
MiddlewareReaderState, ModuleEnvironment, ModuleInfoTranslation, ModuleTranslationState,
MiddlewareBinaryReader, MiddlewareReaderState, ModuleEnvironment, ModuleInfoTranslation,
ModuleMiddleware, ModuleMiddlewareChain, ModuleTranslationState,
};
pub use crate::trap::TrapInformation;
pub use crate::unwind::CompiledFunctionUnwindInfo;
Expand Down
Loading