From 9bd2c4773096b9f6854e49d427404a2815406ec2 Mon Sep 17 00:00:00 2001 From: Syrus Date: Thu, 29 Oct 2020 20:19:16 -0700 Subject: [PATCH 1/6] Upgraded Cranelift to 0.67 --- Cargo.lock | 33 +- lib/api/Cargo.toml | 2 +- lib/compiler-cranelift/Cargo.toml | 11 +- lib/compiler-cranelift/src/compiler.rs | 4 +- lib/compiler-cranelift/src/func_environ.rs | 137 +- lib/compiler-cranelift/src/sink.rs | 1 + .../src/trampoline/dynamic_function.rs | 2 +- .../src/trampoline/function_call.rs | 2 +- lib/compiler-cranelift/src/trampoline/mod.rs | 2 +- .../src/translator/code_translator.rs | 1200 ++++++++++++----- .../src/translator/func_environ.rs | 85 +- .../src/translator/func_state.rs | 43 +- .../src/translator/func_translator.rs | 5 +- lib/compiler-llvm/Cargo.toml | 2 +- lib/compiler-llvm/src/compiler.rs | 6 +- lib/compiler/Cargo.toml | 2 +- lib/engine-jit/src/link.rs | 2 +- lib/engine/Cargo.toml | 2 +- lib/vm/src/trap/trapcode.rs | 31 +- lib/wasmer-types/Cargo.toml | 2 +- tests/ignores.txt | 11 +- tests/lib/test-generator/Cargo.toml | 2 +- 22 files changed, 1123 insertions(+), 464 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 247f9017179..8f950ac2720 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -317,18 +317,18 @@ checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" [[package]] name = "cranelift-bforest" -version = "0.65.0" +version = "0.67.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9413a2c6bdb01ab8acc867421bd7343ddea491d015453f4e56f4f60c816d120" +checksum = "0f065f6889758f817f61a230220d1811ba99a9762af2fb69ae23048314f75ff2" dependencies = [ "cranelift-entity", ] [[package]] name = "cranelift-codegen" -version = "0.65.0" +version = "0.67.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28d389588c2375bb95292e0bc6cbf010e7f30fb4e9734738b576521b737826a" +checksum = "510aa2ab4307644100682b94e449940a0ea15c5887f1d4b9678b8dd5ef31e736" dependencies = [ "byteorder", "cranelift-bforest", @@ -347,9 +347,9 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.65.0" +version = "0.67.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74dd3cf6f107c1df4c2b8aab91ec4181aee7ff17289673fcbec63325e7e40a83" +checksum = "b4cb0c7e87c60d63b35f9670c15479ee4a5e557dd127efab88b2f9b2ca83c9a0" dependencies = [ "cranelift-codegen-shared", "cranelift-entity", @@ -357,24 +357,24 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" -version = "0.65.0" +version = "0.67.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efaf050fab2dbf324544489443ff3cc8c67c9420c8902ec6628bd906bd7393e9" +checksum = "60636227098693e06de8d6d88beea2a7d32ecf8a8030dacdb57c68e06f381826" [[package]] name = "cranelift-entity" -version = "0.65.0" +version = "0.67.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f07eb8aa0a5da94b56339e4e3052c496a3df4354357cd5da8c7b02c6e8f1dc1d" +checksum = "6156db73e0c9f65f80c512988d63ec736be0dee3dd66bf951e3e28aed9dc02d3" dependencies = [ "serde", ] [[package]] name = "cranelift-frontend" -version = "0.65.0" +version = "0.67.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f6fe1d3e968576f4b60d23f40ee90281f8de2cdf23d2110f3b0296ff420555e" +checksum = "e09cd158c9a820a4cc14a34076811da225cce1d31dc6d03c5ef85b91aef560b9" dependencies = [ "cranelift-codegen", "hashbrown 0.7.2", @@ -1568,9 +1568,9 @@ checksum = "d813022b2e00774a48eaf43caaa3c20b45f040ba8cbf398e2e8911a06668dbe6" [[package]] name = "regalloc" -version = "0.0.26" +version = "0.0.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c03092d79e0fd610932d89ed53895a38c0dd3bcd317a0046e69940de32f1d95" +checksum = "2041c2d34f6ff346d6f428974f03d8bf12679b0c816bb640dc5eb1d48848d8d1" dependencies = [ "log", "rustc-hash", @@ -1855,9 +1855,9 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.10.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab0e7238dcc7b40a7be719a25365910f6807bd864f4cce6b2e6b873658e2b19d" +checksum = "4ee5a98e506fb7231a304c3a1bd7c132a55016cf65001e0282480665870dfcb9" [[package]] name = "tempfile" @@ -2263,6 +2263,7 @@ dependencies = [ "more-asserts", "rayon", "serde", + "smallvec", "target-lexicon", "tracing", "wasmer-compiler", diff --git a/lib/api/Cargo.toml b/lib/api/Cargo.toml index a66269bee53..74d90c188b8 100644 --- a/lib/api/Cargo.toml +++ b/lib/api/Cargo.toml @@ -25,7 +25,7 @@ cfg-if = "0.1" wat = { version = "1.0", optional = true } thiserror = "1.0" more-asserts = "0.2" -target-lexicon = { version = "0.10", default-features = false } +target-lexicon = { version = "0.11", default-features = false } [target.'cfg(target_os = "windows")'.dependencies] winapi = "0.3" diff --git a/lib/compiler-cranelift/Cargo.toml b/lib/compiler-cranelift/Cargo.toml index 0633afa467d..9591507b5be 100644 --- a/lib/compiler-cranelift/Cargo.toml +++ b/lib/compiler-cranelift/Cargo.toml @@ -15,18 +15,19 @@ edition = "2018" wasmer-compiler = { path = "../compiler", version = "1.0.0-alpha4", features = ["translator"], default-features = false } wasmer-vm = { path = "../vm", version = "1.0.0-alpha4" } wasmer-types = { path = "../wasmer-types", version = "1.0.0-alpha4", default-features = false, features = ["std"] } -cranelift-codegen = { version = "0.65", default-features = false, features = ["x86", "arm64"] } -cranelift-frontend = { version = "0.65", default-features = false } +cranelift-codegen = { version = "0.67", default-features = false, features = ["x86", "arm64"] } +cranelift-frontend = { version = "0.67", default-features = false } tracing = "0.1" hashbrown = { version = "0.8", optional = true } rayon = "1.3" serde = { version = "1.0", features = ["derive"] } more-asserts = "0.2" gimli = { version = "0.21", optional = true } +smallvec = "1.0.0" [dev-dependencies] -target-lexicon = { version = "0.10", default-features = false } -cranelift-codegen = { version = "0.65", features = ["enable-serde", "all-arch"] } +target-lexicon = { version = "0.11", default-features = false } +cranelift-codegen = { version = "0.67", features = ["enable-serde", "all-arch"] } lazy_static = "1.4" [badges] @@ -35,6 +36,6 @@ maintenance = { status = "actively-developed" } [features] default = ["std", "enable-serde", "unwind"] unwind = ["cranelift-codegen/unwind", "gimli"] -enable-serde = ["wasmer-compiler/enable-serde", "cranelift-codegen/enable-serde", "wasmer-types/enable-serde"] +enable-serde = ["wasmer-compiler/enable-serde", "wasmer-types/enable-serde"] std = ["cranelift-codegen/std", "cranelift-frontend/std", "wasmer-compiler/std", "wasmer-types/std"] core = ["hashbrown", "cranelift-codegen/core", "cranelift-frontend/core"] diff --git a/lib/compiler-cranelift/src/compiler.rs b/lib/compiler-cranelift/src/compiler.rs index 08c7d5803e6..265aa5ac1d3 100644 --- a/lib/compiler-cranelift/src/compiler.rs +++ b/lib/compiler-cranelift/src/compiler.rs @@ -95,7 +95,7 @@ impl Compiler for CraneliftCompiler { }; let functions = function_body_inputs - .into_iter() + .iter() .collect::)>>() .par_iter() .map_init(FuncTranslator::new, |func_translator, (i, input)| { @@ -127,7 +127,7 @@ impl Compiler for CraneliftCompiler { let mut code_buf: Vec = Vec::new(); let mut reloc_sink = RelocSink::new(module, func_index); let mut trap_sink = TrapSink::new(); - let mut stackmap_sink = binemit::NullStackmapSink {}; + let mut stackmap_sink = binemit::NullStackMapSink {}; context .compile_and_emit( &*isa, diff --git a/lib/compiler-cranelift/src/func_environ.rs b/lib/compiler-cranelift/src/func_environ.rs index dc4dfe74ee0..9d9161d2c84 100644 --- a/lib/compiler-cranelift/src/func_environ.rs +++ b/lib/compiler-cranelift/src/func_environ.rs @@ -11,7 +11,9 @@ use cranelift_codegen::ir::immediates::{Offset32, Uimm64}; use cranelift_codegen::ir::types::*; use cranelift_codegen::ir::{AbiParam, ArgumentPurpose, Function, InstBuilder, Signature}; use cranelift_codegen::isa::TargetFrontendConfig; +use cranelift_frontend::FunctionBuilder; use std::convert::TryFrom; +use wasmer_compiler::wasmparser::Type; use wasmer_compiler::{WasmError, WasmResult}; use wasmer_types::entity::EntityRef; use wasmer_types::entity::PrimaryMap; @@ -538,10 +540,11 @@ impl<'module_environment> BaseFuncEnvironment for FuncEnvironment<'module_enviro fn translate_table_grow( &mut self, - _: cranelift_codegen::cursor::FuncCursor<'_>, - _: TableIndex, - _: ir::Value, - _: ir::Value, + mut _pos: cranelift_codegen::cursor::FuncCursor<'_>, + _table_index: TableIndex, + _table: ir::Table, + _delta: ir::Value, + _init_value: ir::Value, ) -> WasmResult { Err(WasmError::Unsupported( "the `table.grow` instruction is not supported yet".into(), @@ -550,9 +553,10 @@ impl<'module_environment> BaseFuncEnvironment for FuncEnvironment<'module_enviro fn translate_table_get( &mut self, - _: cranelift_codegen::cursor::FuncCursor<'_>, - _: TableIndex, - _: ir::Value, + _builder: &mut FunctionBuilder, + _table_index: TableIndex, + _table: ir::Table, + _index: ir::Value, ) -> WasmResult { Err(WasmError::Unsupported( "the `table.get` instruction is not supported yet".into(), @@ -561,10 +565,11 @@ impl<'module_environment> BaseFuncEnvironment for FuncEnvironment<'module_enviro fn translate_table_set( &mut self, - _: cranelift_codegen::cursor::FuncCursor<'_>, - _: TableIndex, - _: ir::Value, - _: ir::Value, + _builder: &mut FunctionBuilder, + _table_index: TableIndex, + _table: ir::Table, + _value: ir::Value, + _index: ir::Value, ) -> WasmResult<()> { Err(WasmError::Unsupported( "the `table.set` instruction is not supported yet".into(), @@ -573,21 +578,56 @@ impl<'module_environment> BaseFuncEnvironment for FuncEnvironment<'module_enviro fn translate_table_fill( &mut self, - _: cranelift_codegen::cursor::FuncCursor<'_>, - _: TableIndex, - _: ir::Value, - _: ir::Value, - _: ir::Value, + mut _pos: cranelift_codegen::cursor::FuncCursor<'_>, + _table_index: TableIndex, + _dst: ir::Value, + _val: ir::Value, + _len: ir::Value, ) -> WasmResult<()> { Err(WasmError::Unsupported( "the `table.fill` instruction is not supported yet".into(), )) } + fn translate_ref_null( + &mut self, + mut pos: cranelift_codegen::cursor::FuncCursor, + ty: Type, + ) -> WasmResult { + Ok(match ty { + Type::FuncRef => pos.ins().iconst(self.pointer_type(), 0), + // Type::ExternRef => pos.ins().null(self.reference_type(ty)), + _ => { + return Err(WasmError::Unsupported( + "`ref.null T` that is not a `funcref`".into(), + )); + } + }) + } + + fn translate_ref_is_null( + &mut self, + mut pos: cranelift_codegen::cursor::FuncCursor, + value: ir::Value, + ) -> WasmResult { + let bool_is_null = match pos.func.dfg.value_type(value) { + // `externref` + ty if ty.is_ref() => pos.ins().is_null(value), + // `funcref` + ty if ty == self.pointer_type() => { + pos.ins() + .icmp_imm(cranelift_codegen::ir::condcodes::IntCC::Equal, value, 0) + } + _ => unreachable!(), + }; + + Ok(pos.ins().bint(ir::types::I32, bool_is_null)) + } + fn translate_ref_func( &mut self, - _: cranelift_codegen::cursor::FuncCursor<'_>, - _: u32, + mut _pos: cranelift_codegen::cursor::FuncCursor<'_>, + _func_index: FunctionIndex, ) -> WasmResult { Err(WasmError::Unsupported( "the `ref.func` instruction is not supported yet".into(), @@ -596,17 +636,17 @@ impl<'module_environment> BaseFuncEnvironment for FuncEnvironment<'module_enviro fn translate_custom_global_get( &mut self, - _: cranelift_codegen::cursor::FuncCursor<'_>, - _: GlobalIndex, + mut _pos: cranelift_codegen::cursor::FuncCursor<'_>, + _index: GlobalIndex, ) -> WasmResult { unreachable!("we don't make any custom globals") } fn translate_custom_global_set( &mut self, - _: cranelift_codegen::cursor::FuncCursor<'_>, - _: GlobalIndex, - _: ir::Value, + mut _pos: cranelift_codegen::cursor::FuncCursor<'_>, + _index: GlobalIndex, + _value: ir::Value, ) -> WasmResult<()> { unreachable!("we don't make any custom globals") } @@ -771,7 +811,7 @@ impl<'module_environment> BaseFuncEnvironment for FuncEnvironment<'module_enviro match self.table_styles[table_index] { TableStyle::CallerChecksSignature => { let sig_id_size = self.offsets.size_of_vmshared_signature_index(); - let sig_id_type = Type::int(u16::from(sig_id_size) * 8).unwrap(); + let sig_id_type = ir::Type::int(u16::from(sig_id_size) * 8).unwrap(); let vmctx = self.vmctx(pos.func); let base = pos.ins().global_value(pointer_type, vmctx); let offset = @@ -897,24 +937,22 @@ impl<'module_environment> BaseFuncEnvironment for FuncEnvironment<'module_enviro fn translate_memory_copy( &mut self, mut pos: FuncCursor, - memory_index: MemoryIndex, - _heap: ir::Heap, + src_index: MemoryIndex, + _src_heap: ir::Heap, + _dst_index: MemoryIndex, + _dst_heap: ir::Heap, dst: ir::Value, src: ir::Value, len: ir::Value, ) -> WasmResult<()> { - let (func_sig, memory_index, func_idx) = - self.get_memory_copy_func(&mut pos.func, memory_index); + let (func_sig, src_index, func_idx) = self.get_memory_copy_func(&mut pos.func, src_index); - let memory_index_arg = pos.ins().iconst(I32, memory_index as i64); + let src_index_arg = pos.ins().iconst(I32, src_index as i64); let (vmctx, func_addr) = self.translate_load_builtin_function_address(&mut pos, func_idx); - pos.ins().call_indirect( - func_sig, - func_addr, - &[vmctx, memory_index_arg, dst, src, len], - ); + pos.ins() + .call_indirect(func_sig, func_addr, &[vmctx, src_index_arg, dst, src, len]); Ok(()) } @@ -981,8 +1019,8 @@ impl<'module_environment> BaseFuncEnvironment for FuncEnvironment<'module_enviro fn translate_table_size( &mut self, - _pos: FuncCursor, - _index: TableIndex, + mut _pos: FuncCursor, + _table_index: TableIndex, _table: ir::Table, ) -> WasmResult { Err(WasmError::Unsupported( @@ -1064,4 +1102,31 @@ impl<'module_environment> BaseFuncEnvironment for FuncEnvironment<'module_enviro Ok(()) } + + fn translate_atomic_wait( + &mut self, + _pos: FuncCursor, + _index: MemoryIndex, + _heap: ir::Heap, + _addr: ir::Value, + _expected: ir::Value, + _timeout: ir::Value, + ) -> WasmResult { + Err(WasmError::Unsupported( + "wasm atomics (fn translate_atomic_wait)".to_string(), + )) + } + + fn translate_atomic_notify( + &mut self, + _pos: FuncCursor, + _index: MemoryIndex, + _heap: ir::Heap, + _addr: ir::Value, + _count: ir::Value, + ) -> WasmResult { + Err(WasmError::Unsupported( + "wasm atomics (fn translate_atomic_notify)".to_string(), + )) + } } diff --git a/lib/compiler-cranelift/src/sink.rs b/lib/compiler-cranelift/src/sink.rs index faa6e2807db..c96ef41af65 100644 --- a/lib/compiler-cranelift/src/sink.rs +++ b/lib/compiler-cranelift/src/sink.rs @@ -124,6 +124,7 @@ fn translate_ir_trapcode(trap: ir::TrapCode) -> TrapCode { match trap { ir::TrapCode::StackOverflow => TrapCode::StackOverflow, ir::TrapCode::HeapOutOfBounds => TrapCode::HeapAccessOutOfBounds, + ir::TrapCode::HeapMisaligned => TrapCode::HeapMisaligned, ir::TrapCode::TableOutOfBounds => TrapCode::TableAccessOutOfBounds, ir::TrapCode::IndirectCallToNull => TrapCode::IndirectCallToNull, ir::TrapCode::BadSignature => TrapCode::BadSignature, diff --git a/lib/compiler-cranelift/src/trampoline/dynamic_function.rs b/lib/compiler-cranelift/src/trampoline/dynamic_function.rs index e350ae16a25..db9d4633b48 100644 --- a/lib/compiler-cranelift/src/trampoline/dynamic_function.rs +++ b/lib/compiler-cranelift/src/trampoline/dynamic_function.rs @@ -110,7 +110,7 @@ pub fn make_trampoline_dynamic_function( let mut code_buf = Vec::new(); let mut reloc_sink = TrampolineRelocSink {}; let mut trap_sink = binemit::NullTrapSink {}; - let mut stackmap_sink = binemit::NullStackmapSink {}; + let mut stackmap_sink = binemit::NullStackMapSink {}; context .compile_and_emit( isa, diff --git a/lib/compiler-cranelift/src/trampoline/function_call.rs b/lib/compiler-cranelift/src/trampoline/function_call.rs index bb39860ce91..876bc8612d5 100644 --- a/lib/compiler-cranelift/src/trampoline/function_call.rs +++ b/lib/compiler-cranelift/src/trampoline/function_call.rs @@ -108,7 +108,7 @@ pub fn make_trampoline_function_call( let mut code_buf = Vec::new(); let mut reloc_sink = TrampolineRelocSink {}; let mut trap_sink = binemit::NullTrapSink {}; - let mut stackmap_sink = binemit::NullStackmapSink {}; + let mut stackmap_sink = binemit::NullStackMapSink {}; context .compile_and_emit( diff --git a/lib/compiler-cranelift/src/trampoline/mod.rs b/lib/compiler-cranelift/src/trampoline/mod.rs index c2a73622863..98b8bbae048 100644 --- a/lib/compiler-cranelift/src/trampoline/mod.rs +++ b/lib/compiler-cranelift/src/trampoline/mod.rs @@ -12,7 +12,7 @@ pub use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext}; pub mod binemit { pub use cranelift_codegen::binemit::NullTrapSink; - pub use cranelift_codegen::binemit::{CodeOffset, NullStackmapSink, TrapSink}; + pub use cranelift_codegen::binemit::{CodeOffset, NullStackMapSink, TrapSink}; use cranelift_codegen::{binemit, ir}; diff --git a/lib/compiler-cranelift/src/translator/code_translator.rs b/lib/compiler-cranelift/src/translator/code_translator.rs index f324ef8b458..c765a492a22 100644 --- a/lib/compiler-cranelift/src/translator/code_translator.rs +++ b/lib/compiler-cranelift/src/translator/code_translator.rs @@ -37,17 +37,19 @@ use cranelift_codegen::ir::condcodes::{FloatCC, IntCC}; use cranelift_codegen::ir::immediates::Offset32; use cranelift_codegen::ir::types::*; use cranelift_codegen::ir::{ - self, ConstantData, InstBuilder, JumpTableData, MemFlags, Value, ValueLabel, + self, AtomicRmwOp, ConstantData, InstBuilder, JumpTableData, MemFlags, Value, ValueLabel, }; use cranelift_codegen::packed_option::ReservedValue; use cranelift_frontend::{FunctionBuilder, Variable}; +use smallvec::SmallVec; use std::vec::Vec; + use wasmer_compiler::wasmparser::{MemoryImmediate, Operator}; use wasmer_compiler::{to_wasm_error, WasmResult}; use wasmer_compiler::{wasm_unsupported, ModuleTranslationState}; use wasmer_types::{FunctionIndex, GlobalIndex, MemoryIndex, SignatureIndex, TableIndex}; -// Clippy warns about "flags: _" but its important to document that the flags field is ignored +// Clippy warns about "align: _" but its important to document that the flags field is ignored #[cfg_attr( feature = "cargo-clippy", allow(clippy::unneeded_field_pattern, clippy::cognitive_complexity) @@ -190,7 +192,7 @@ pub fn translate_operator( let (params, results) = module_translation_state.blocktype_params_results(*ty)?; let loop_body = block_with_params(builder, params, environ)?; let next = block_with_params(builder, results, environ)?; - builder.ins().jump(loop_body, state.peekn(params.len())); + canonicalise_then_jump(builder, loop_body, state.peekn(params.len())); state.push_loop(loop_body, next, params.len(), results.len()); // Pop the initial `Block` actuals and replace them with the `Block`'s @@ -214,25 +216,22 @@ pub fn translate_operator( // destination block following the whole `if...end`. If we do end // up discovering an `else`, then we will allocate a block for it // and go back and patch the jump. - let destination = block_with_params(builder, results, environ)?; - let branch_inst = builder - .ins() - .brz(val, destination, state.peekn(params.len())); + let destination = block_with_params(builder, results.clone(), environ)?; + let branch_inst = + canonicalise_then_brz(builder, val, destination, state.peekn(params.len())); (destination, ElseData::NoElse { branch_inst }) } else { // The `if` type signature is not valid without an `else` block, // so we eagerly allocate the `else` block here. - let destination = block_with_params(builder, results, environ)?; - let else_block = block_with_params(builder, params, environ)?; - builder - .ins() - .brz(val, else_block, state.peekn(params.len())); + let destination = block_with_params(builder, results.clone(), environ)?; + let else_block = block_with_params(builder, params.clone(), environ)?; + canonicalise_then_brz(builder, val, else_block, state.peekn(params.len())); builder.seal_block(else_block); (destination, ElseData::WithElse { else_block }) }; let next_block = builder.create_block(); - builder.ins().jump(next_block, &[]); + canonicalise_then_jump(builder, next_block, &[]); builder.seal_block(next_block); // Only predecessor is the current block. builder.switch_to_block(next_block); @@ -272,8 +271,13 @@ pub fn translate_operator( let (params, _results) = module_translation_state.blocktype_params_results(blocktype)?; debug_assert_eq!(params.len(), num_return_values); - let else_block = block_with_params(builder, params, environ)?; - builder.ins().jump(destination, state.peekn(params.len())); + let else_block = + block_with_params(builder, params.clone(), environ)?; + canonicalise_then_jump( + builder, + destination, + state.peekn(params.len()), + ); state.popn(params.len()); builder.change_jump_destination(branch_inst, else_block); @@ -281,9 +285,11 @@ pub fn translate_operator( else_block } ElseData::WithElse { else_block } => { - builder - .ins() - .jump(destination, state.peekn(num_return_values)); + canonicalise_then_jump( + builder, + destination, + state.peekn(num_return_values), + ); state.popn(num_return_values); else_block } @@ -315,9 +321,7 @@ pub fn translate_operator( if !builder.is_unreachable() || !builder.is_pristine() { let return_count = frame.num_return_values(); let return_args = state.peekn_mut(return_count); - let next_block_types = builder.func.dfg.block_param_types(next_block); - bitcast_arguments(return_args, &next_block_types, builder); - builder.ins().jump(frame.following_code(), return_args); + canonicalise_then_jump(builder, frame.following_code(), return_args); // You might expect that if we just finished an `if` block that // didn't have a corresponding `else` block, then we would clean // up our duplicate set of parameters that we pushed earlier @@ -325,13 +329,16 @@ pub fn translate_operator( // since we truncate the stack back to the original height // below. } + builder.switch_to_block(next_block); builder.seal_block(next_block); + // If it is a loop we also have to seal the body loop block if let ControlStackFrame::Loop { header, .. } = frame { builder.seal_block(header) } - state.stack.truncate(frame.original_stack_size()); + + frame.truncate_value_stack_to_original_size(&mut state.stack); state .stack .extend_from_slice(builder.block_params(next_block)); @@ -370,17 +377,8 @@ pub fn translate_operator( }; (return_count, frame.br_destination()) }; - - // Bitcast any vector arguments to their default type, I8X16, before jumping. let destination_args = state.peekn_mut(return_count); - let destination_types = builder.func.dfg.block_param_types(br_destination); - bitcast_arguments( - destination_args, - &destination_types[..return_count], - builder, - ); - - builder.ins().jump(br_destination, destination_args); + canonicalise_then_jump(builder, br_destination, destination_args); state.popn(return_count); state.reachable = false; } @@ -392,7 +390,7 @@ pub fn translate_operator( .map_err(to_wasm_error)?; let default = depths.pop().unwrap().0; let mut min_depth = default; - for (depth, _) in &*depths { + for (depth, _) in depths.iter() { if *depth < min_depth { min_depth = *depth; } @@ -401,7 +399,7 @@ pub fn translate_operator( let i = state.control_stack.len() - 1 - (min_depth as usize); let min_depth_frame = &state.control_stack[i]; if min_depth_frame.is_loop() { - 0 + min_depth_frame.num_param_values() } else { min_depth_frame.num_return_values() } @@ -410,7 +408,7 @@ pub fn translate_operator( let mut data = JumpTableData::with_capacity(depths.len()); if jump_args_count == 0 { // No jump arguments - for (depth, _) in &*depths { + for (depth, _) in depths.iter() { let block = { let i = state.control_stack.len() - 1 - (*depth as usize); let frame = &mut state.control_stack[i]; @@ -433,7 +431,7 @@ pub fn translate_operator( let return_count = jump_args_count; let mut dest_block_sequence = vec![]; let mut dest_block_map = HashMap::new(); - for (depth, _) in &*depths { + for (depth, _) in depths.iter() { let branch_block = match dest_block_map.entry(*depth as usize) { hash_map::Entry::Occupied(entry) => *entry.get(), hash_map::Entry::Vacant(entry) => { @@ -463,17 +461,8 @@ pub fn translate_operator( frame.set_branched_to_exit(); frame.br_destination() }; - - // Bitcast any vector arguments to their default type, I8X16, before jumping. let destination_args = state.peekn_mut(return_count); - let destination_types = builder.func.dfg.block_param_types(real_dest_block); - bitcast_arguments( - destination_args, - &destination_types[..return_count], - builder, - ); - - builder.ins().jump(real_dest_block, destination_args); + canonicalise_then_jump(builder, real_dest_block, destination_args); } state.popn(return_count); } @@ -495,7 +484,7 @@ pub fn translate_operator( match environ.return_mode() { ReturnMode::NormalReturns => builder.ins().return_(return_args), ReturnMode::FallthroughReturn => { - builder.ins().jump(br_destination, return_args) + canonicalise_then_jump(builder, br_destination, return_args) } }; } @@ -540,7 +529,7 @@ pub fn translate_operator( // `index` is the index of the function's signature and `table_index` is the index of // the table to search the function in. let (sigref, num_args) = state.get_indirect_sig(builder.func, *index, environ)?; - let table = state.get_table(builder.func, *table_index, environ)?; + let table = state.get_or_create_table(builder.func, *table_index, environ)?; let callee = state.pop1(); // Bitcast any vector arguments to their default type, I8X16, before calling. @@ -590,120 +579,78 @@ pub fn translate_operator( * Wasm specifies an integer alignment flag but we drop it in Cranelift. * The memory base address is provided by the environment. ************************************************************************************/ - Operator::I32Load8U { - memarg: MemoryImmediate { offset, .. }, - } => { - translate_load(*offset, ir::Opcode::Uload8, I32, builder, state, environ)?; + Operator::I32Load8U { memarg } => { + translate_load(memarg, ir::Opcode::Uload8, I32, builder, state, environ)?; } - Operator::I32Load16U { - memarg: MemoryImmediate { offset, .. }, - } => { - translate_load(*offset, ir::Opcode::Uload16, I32, builder, state, environ)?; + Operator::I32Load16U { memarg } => { + translate_load(memarg, ir::Opcode::Uload16, I32, builder, state, environ)?; } - Operator::I32Load8S { - memarg: MemoryImmediate { offset, .. }, - } => { - translate_load(*offset, ir::Opcode::Sload8, I32, builder, state, environ)?; + Operator::I32Load8S { memarg } => { + translate_load(memarg, ir::Opcode::Sload8, I32, builder, state, environ)?; } - Operator::I32Load16S { - memarg: MemoryImmediate { offset, .. }, - } => { - translate_load(*offset, ir::Opcode::Sload16, I32, builder, state, environ)?; + Operator::I32Load16S { memarg } => { + translate_load(memarg, ir::Opcode::Sload16, I32, builder, state, environ)?; } - Operator::I64Load8U { - memarg: MemoryImmediate { offset, .. }, - } => { - translate_load(*offset, ir::Opcode::Uload8, I64, builder, state, environ)?; + Operator::I64Load8U { memarg } => { + translate_load(memarg, ir::Opcode::Uload8, I64, builder, state, environ)?; } - Operator::I64Load16U { - memarg: MemoryImmediate { offset, .. }, - } => { - translate_load(*offset, ir::Opcode::Uload16, I64, builder, state, environ)?; + Operator::I64Load16U { memarg } => { + translate_load(memarg, ir::Opcode::Uload16, I64, builder, state, environ)?; } - Operator::I64Load8S { - memarg: MemoryImmediate { offset, .. }, - } => { - translate_load(*offset, ir::Opcode::Sload8, I64, builder, state, environ)?; + Operator::I64Load8S { memarg } => { + translate_load(memarg, ir::Opcode::Sload8, I64, builder, state, environ)?; } - Operator::I64Load16S { - memarg: MemoryImmediate { offset, .. }, - } => { - translate_load(*offset, ir::Opcode::Sload16, I64, builder, state, environ)?; + Operator::I64Load16S { memarg } => { + translate_load(memarg, ir::Opcode::Sload16, I64, builder, state, environ)?; } - Operator::I64Load32S { - memarg: MemoryImmediate { offset, .. }, - } => { - translate_load(*offset, ir::Opcode::Sload32, I64, builder, state, environ)?; + Operator::I64Load32S { memarg } => { + translate_load(memarg, ir::Opcode::Sload32, I64, builder, state, environ)?; } - Operator::I64Load32U { - memarg: MemoryImmediate { offset, .. }, - } => { - translate_load(*offset, ir::Opcode::Uload32, I64, builder, state, environ)?; + Operator::I64Load32U { memarg } => { + translate_load(memarg, ir::Opcode::Uload32, I64, builder, state, environ)?; } - Operator::I32Load { - memarg: MemoryImmediate { offset, .. }, - } => { - translate_load(*offset, ir::Opcode::Load, I32, builder, state, environ)?; + Operator::I32Load { memarg } => { + translate_load(memarg, ir::Opcode::Load, I32, builder, state, environ)?; } - Operator::F32Load { - memarg: MemoryImmediate { offset, .. }, - } => { - translate_load(*offset, ir::Opcode::Load, F32, builder, state, environ)?; + Operator::F32Load { memarg } => { + translate_load(memarg, ir::Opcode::Load, F32, builder, state, environ)?; } - Operator::I64Load { - memarg: MemoryImmediate { offset, .. }, - } => { - translate_load(*offset, ir::Opcode::Load, I64, builder, state, environ)?; + Operator::I64Load { memarg } => { + translate_load(memarg, ir::Opcode::Load, I64, builder, state, environ)?; } - Operator::F64Load { - memarg: MemoryImmediate { offset, .. }, - } => { - translate_load(*offset, ir::Opcode::Load, F64, builder, state, environ)?; + Operator::F64Load { memarg } => { + translate_load(memarg, ir::Opcode::Load, F64, builder, state, environ)?; } - Operator::V128Load { - memarg: MemoryImmediate { offset, .. }, - } => { - translate_load(*offset, ir::Opcode::Load, I8X16, builder, state, environ)?; + Operator::V128Load { memarg } => { + translate_load(memarg, ir::Opcode::Load, I8X16, builder, state, environ)?; } - Operator::V128Load8x8S { - memarg: MemoryImmediate { offset, .. }, - } => { - let (flags, base, offset) = prepare_load(*offset, 8, builder, state, environ)?; + Operator::V128Load8x8S { memarg } => { + let (flags, base, offset) = prepare_load(memarg, 8, builder, state, environ)?; let loaded = builder.ins().sload8x8(flags, base, offset); state.push1(loaded); } - Operator::V128Load8x8U { - memarg: MemoryImmediate { offset, .. }, - } => { - let (flags, base, offset) = prepare_load(*offset, 8, builder, state, environ)?; + Operator::V128Load8x8U { memarg } => { + let (flags, base, offset) = prepare_load(memarg, 8, builder, state, environ)?; let loaded = builder.ins().uload8x8(flags, base, offset); state.push1(loaded); } - Operator::V128Load16x4S { - memarg: MemoryImmediate { offset, .. }, - } => { - let (flags, base, offset) = prepare_load(*offset, 8, builder, state, environ)?; + Operator::V128Load16x4S { memarg } => { + let (flags, base, offset) = prepare_load(memarg, 8, builder, state, environ)?; let loaded = builder.ins().sload16x4(flags, base, offset); state.push1(loaded); } - Operator::V128Load16x4U { - memarg: MemoryImmediate { offset, .. }, - } => { - let (flags, base, offset) = prepare_load(*offset, 8, builder, state, environ)?; + Operator::V128Load16x4U { memarg } => { + let (flags, base, offset) = prepare_load(memarg, 8, builder, state, environ)?; let loaded = builder.ins().uload16x4(flags, base, offset); state.push1(loaded); } - Operator::V128Load32x2S { - memarg: MemoryImmediate { offset, .. }, - } => { - let (flags, base, offset) = prepare_load(*offset, 8, builder, state, environ)?; + Operator::V128Load32x2S { memarg } => { + let (flags, base, offset) = prepare_load(memarg, 8, builder, state, environ)?; let loaded = builder.ins().sload32x2(flags, base, offset); state.push1(loaded); } - Operator::V128Load32x2U { - memarg: MemoryImmediate { offset, .. }, - } => { - let (flags, base, offset) = prepare_load(*offset, 8, builder, state, environ)?; + Operator::V128Load32x2U { memarg } => { + let (flags, base, offset) = prepare_load(memarg, 8, builder, state, environ)?; let loaded = builder.ins().uload32x2(flags, base, offset); state.push1(loaded); } @@ -711,45 +658,23 @@ pub fn translate_operator( * Wasm specifies an integer alignment flag but we drop it in Cranelift. * The memory base address is provided by the environment. ************************************************************************************/ - Operator::I32Store { - memarg: MemoryImmediate { offset, .. }, + Operator::I32Store { memarg } + | Operator::I64Store { memarg } + | Operator::F32Store { memarg } + | Operator::F64Store { memarg } => { + translate_store(memarg, ir::Opcode::Store, builder, state, environ)?; } - | Operator::I64Store { - memarg: MemoryImmediate { offset, .. }, + Operator::I32Store8 { memarg } | Operator::I64Store8 { memarg } => { + translate_store(memarg, ir::Opcode::Istore8, builder, state, environ)?; } - | Operator::F32Store { - memarg: MemoryImmediate { offset, .. }, + Operator::I32Store16 { memarg } | Operator::I64Store16 { memarg } => { + translate_store(memarg, ir::Opcode::Istore16, builder, state, environ)?; } - | Operator::F64Store { - memarg: MemoryImmediate { offset, .. }, - } => { - translate_store(*offset, ir::Opcode::Store, builder, state, environ)?; - } - Operator::I32Store8 { - memarg: MemoryImmediate { offset, .. }, - } - | Operator::I64Store8 { - memarg: MemoryImmediate { offset, .. }, - } => { - translate_store(*offset, ir::Opcode::Istore8, builder, state, environ)?; - } - Operator::I32Store16 { - memarg: MemoryImmediate { offset, .. }, + Operator::I64Store32 { memarg } => { + translate_store(memarg, ir::Opcode::Istore32, builder, state, environ)?; } - | Operator::I64Store16 { - memarg: MemoryImmediate { offset, .. }, - } => { - translate_store(*offset, ir::Opcode::Istore16, builder, state, environ)?; - } - Operator::I64Store32 { - memarg: MemoryImmediate { offset, .. }, - } => { - translate_store(*offset, ir::Opcode::Istore32, builder, state, environ)?; - } - Operator::V128Store { - memarg: MemoryImmediate { offset, .. }, - } => { - translate_store(*offset, ir::Opcode::Store, builder, state, environ)?; + Operator::V128Store { memarg } => { + translate_store(memarg, ir::Opcode::Store, builder, state, environ)?; } /****************************** Nullary Operators ************************************/ Operator::I32Const { value } => state.push1(builder.ins().iconst(I32, i64::from(*value))), @@ -1046,113 +971,282 @@ pub fn translate_operator( Operator::F32Le | Operator::F64Le => { translate_fcmp(FloatCC::LessThanOrEqual, builder, state) } - Operator::RefNull { ty: _ } => state.push1(builder.ins().null(environ.reference_type())), + Operator::RefNull { ty } => state.push1(environ.translate_ref_null(builder.cursor(), *ty)?), Operator::RefIsNull => { - let arg = state.pop1(); - let val = builder.ins().is_null(arg); - let val_int = builder.ins().bint(I32, val); - state.push1(val_int); + let value = state.pop1(); + state.push1(environ.translate_ref_is_null(builder.cursor(), value)?); } Operator::RefFunc { function_index } => { - state.push1(environ.translate_ref_func(builder.cursor(), *function_index)?); - } - Operator::MemoryAtomicNotify { .. } - | Operator::MemoryAtomicWait32 { .. } - | Operator::MemoryAtomicWait64 { .. } - | Operator::I32AtomicLoad { .. } - | Operator::I64AtomicLoad { .. } - | Operator::I32AtomicLoad8U { .. } - | Operator::I32AtomicLoad16U { .. } - | Operator::I64AtomicLoad8U { .. } - | Operator::I64AtomicLoad16U { .. } - | Operator::I64AtomicLoad32U { .. } - | Operator::I32AtomicStore { .. } - | Operator::I64AtomicStore { .. } - | Operator::I32AtomicStore8 { .. } - | Operator::I32AtomicStore16 { .. } - | Operator::I64AtomicStore8 { .. } - | Operator::I64AtomicStore16 { .. } - | Operator::I64AtomicStore32 { .. } - | Operator::I32AtomicRmwAdd { .. } - | Operator::I64AtomicRmwAdd { .. } - | Operator::I32AtomicRmw8AddU { .. } - | Operator::I32AtomicRmw16AddU { .. } - | Operator::I64AtomicRmw8AddU { .. } - | Operator::I64AtomicRmw16AddU { .. } - | Operator::I64AtomicRmw32AddU { .. } - | Operator::I32AtomicRmwSub { .. } - | Operator::I64AtomicRmwSub { .. } - | Operator::I32AtomicRmw8SubU { .. } - | Operator::I32AtomicRmw16SubU { .. } - | Operator::I64AtomicRmw8SubU { .. } - | Operator::I64AtomicRmw16SubU { .. } - | Operator::I64AtomicRmw32SubU { .. } - | Operator::I32AtomicRmwAnd { .. } - | Operator::I64AtomicRmwAnd { .. } - | Operator::I32AtomicRmw8AndU { .. } - | Operator::I32AtomicRmw16AndU { .. } - | Operator::I64AtomicRmw8AndU { .. } - | Operator::I64AtomicRmw16AndU { .. } - | Operator::I64AtomicRmw32AndU { .. } - | Operator::I32AtomicRmwOr { .. } - | Operator::I64AtomicRmwOr { .. } - | Operator::I32AtomicRmw8OrU { .. } - | Operator::I32AtomicRmw16OrU { .. } - | Operator::I64AtomicRmw8OrU { .. } - | Operator::I64AtomicRmw16OrU { .. } - | Operator::I64AtomicRmw32OrU { .. } - | Operator::I32AtomicRmwXor { .. } - | Operator::I64AtomicRmwXor { .. } - | Operator::I32AtomicRmw8XorU { .. } - | Operator::I32AtomicRmw16XorU { .. } - | Operator::I64AtomicRmw8XorU { .. } - | Operator::I64AtomicRmw16XorU { .. } - | Operator::I64AtomicRmw32XorU { .. } - | Operator::I32AtomicRmwXchg { .. } - | Operator::I64AtomicRmwXchg { .. } - | Operator::I32AtomicRmw8XchgU { .. } - | Operator::I32AtomicRmw16XchgU { .. } - | Operator::I64AtomicRmw8XchgU { .. } - | Operator::I64AtomicRmw16XchgU { .. } - | Operator::I64AtomicRmw32XchgU { .. } - | Operator::I32AtomicRmwCmpxchg { .. } - | Operator::I64AtomicRmwCmpxchg { .. } - | Operator::I32AtomicRmw8CmpxchgU { .. } - | Operator::I32AtomicRmw16CmpxchgU { .. } - | Operator::I64AtomicRmw8CmpxchgU { .. } - | Operator::I64AtomicRmw16CmpxchgU { .. } - | Operator::I64AtomicRmw32CmpxchgU { .. } - | Operator::AtomicFence { .. } => { - return Err(wasm_unsupported!("proposed thread operator {:?}", op)); - } - Operator::MemoryCopy { .. } => { + let index = FunctionIndex::from_u32(*function_index); + state.push1(environ.translate_ref_func(builder.cursor(), index)?); + } + Operator::MemoryAtomicWait32 { memarg } | Operator::MemoryAtomicWait64 { memarg } => { // The WebAssembly MVP only supports one linear memory and // wasmparser will ensure that the memory indices specified are // zero. - let heap_index = MemoryIndex::from_u32(0); - let heap = state.get_heap(builder.func, 0, environ)?; + let implied_ty = match op { + Operator::MemoryAtomicWait64 { .. } => I64, + Operator::MemoryAtomicWait32 { .. } => I32, + _ => unreachable!(), + }; + let heap_index = MemoryIndex::from_u32(memarg.memory); + let heap = state.get_heap(builder.func, memarg.memory, environ)?; + let timeout = state.pop1(); // 64 (fixed) + let expected = state.pop1(); // 32 or 64 (per the `Ixx` in `IxxAtomicWait`) + let addr = state.pop1(); // 32 (fixed) + assert!(builder.func.dfg.value_type(expected) == implied_ty); + // `fn translate_atomic_wait` can inspect the type of `expected` to figure out what + // code it needs to generate, if it wants. + let res = environ.translate_atomic_wait( + builder.cursor(), + heap_index, + heap, + addr, + expected, + timeout, + )?; + state.push1(res); + } + Operator::MemoryAtomicNotify { memarg } => { + let heap_index = MemoryIndex::from_u32(memarg.memory); + let heap = state.get_heap(builder.func, memarg.memory, environ)?; + let count = state.pop1(); // 32 (fixed) + let addr = state.pop1(); // 32 (fixed) + let res = + environ.translate_atomic_notify(builder.cursor(), heap_index, heap, addr, count)?; + state.push1(res); + } + Operator::I32AtomicLoad { memarg } => { + translate_atomic_load(I32, I32, memarg, builder, state, environ)? + } + Operator::I64AtomicLoad { memarg } => { + translate_atomic_load(I64, I64, memarg, builder, state, environ)? + } + Operator::I32AtomicLoad8U { memarg } => { + translate_atomic_load(I32, I8, memarg, builder, state, environ)? + } + Operator::I32AtomicLoad16U { memarg } => { + translate_atomic_load(I32, I16, memarg, builder, state, environ)? + } + Operator::I64AtomicLoad8U { memarg } => { + translate_atomic_load(I64, I8, memarg, builder, state, environ)? + } + Operator::I64AtomicLoad16U { memarg } => { + translate_atomic_load(I64, I16, memarg, builder, state, environ)? + } + Operator::I64AtomicLoad32U { memarg } => { + translate_atomic_load(I64, I32, memarg, builder, state, environ)? + } + + Operator::I32AtomicStore { memarg } => { + translate_atomic_store(I32, memarg, builder, state, environ)? + } + Operator::I64AtomicStore { memarg } => { + translate_atomic_store(I64, memarg, builder, state, environ)? + } + Operator::I32AtomicStore8 { memarg } => { + translate_atomic_store(I8, memarg, builder, state, environ)? + } + Operator::I32AtomicStore16 { memarg } => { + translate_atomic_store(I16, memarg, builder, state, environ)? + } + Operator::I64AtomicStore8 { memarg } => { + translate_atomic_store(I8, memarg, builder, state, environ)? + } + Operator::I64AtomicStore16 { memarg } => { + translate_atomic_store(I16, memarg, builder, state, environ)? + } + Operator::I64AtomicStore32 { memarg } => { + translate_atomic_store(I32, memarg, builder, state, environ)? + } + + Operator::I32AtomicRmwAdd { memarg } => { + translate_atomic_rmw(I32, I32, AtomicRmwOp::Add, memarg, builder, state, environ)? + } + Operator::I64AtomicRmwAdd { memarg } => { + translate_atomic_rmw(I64, I64, AtomicRmwOp::Add, memarg, builder, state, environ)? + } + Operator::I32AtomicRmw8AddU { memarg } => { + translate_atomic_rmw(I32, I8, AtomicRmwOp::Add, memarg, builder, state, environ)? + } + Operator::I32AtomicRmw16AddU { memarg } => { + translate_atomic_rmw(I32, I16, AtomicRmwOp::Add, memarg, builder, state, environ)? + } + Operator::I64AtomicRmw8AddU { memarg } => { + translate_atomic_rmw(I64, I8, AtomicRmwOp::Add, memarg, builder, state, environ)? + } + Operator::I64AtomicRmw16AddU { memarg } => { + translate_atomic_rmw(I64, I16, AtomicRmwOp::Add, memarg, builder, state, environ)? + } + Operator::I64AtomicRmw32AddU { memarg } => { + translate_atomic_rmw(I64, I32, AtomicRmwOp::Add, memarg, builder, state, environ)? + } + + Operator::I32AtomicRmwSub { memarg } => { + translate_atomic_rmw(I32, I32, AtomicRmwOp::Sub, memarg, builder, state, environ)? + } + Operator::I64AtomicRmwSub { memarg } => { + translate_atomic_rmw(I64, I64, AtomicRmwOp::Sub, memarg, builder, state, environ)? + } + Operator::I32AtomicRmw8SubU { memarg } => { + translate_atomic_rmw(I32, I8, AtomicRmwOp::Sub, memarg, builder, state, environ)? + } + Operator::I32AtomicRmw16SubU { memarg } => { + translate_atomic_rmw(I32, I16, AtomicRmwOp::Sub, memarg, builder, state, environ)? + } + Operator::I64AtomicRmw8SubU { memarg } => { + translate_atomic_rmw(I64, I8, AtomicRmwOp::Sub, memarg, builder, state, environ)? + } + Operator::I64AtomicRmw16SubU { memarg } => { + translate_atomic_rmw(I64, I16, AtomicRmwOp::Sub, memarg, builder, state, environ)? + } + Operator::I64AtomicRmw32SubU { memarg } => { + translate_atomic_rmw(I64, I32, AtomicRmwOp::Sub, memarg, builder, state, environ)? + } + + Operator::I32AtomicRmwAnd { memarg } => { + translate_atomic_rmw(I32, I32, AtomicRmwOp::And, memarg, builder, state, environ)? + } + Operator::I64AtomicRmwAnd { memarg } => { + translate_atomic_rmw(I64, I64, AtomicRmwOp::And, memarg, builder, state, environ)? + } + Operator::I32AtomicRmw8AndU { memarg } => { + translate_atomic_rmw(I32, I8, AtomicRmwOp::And, memarg, builder, state, environ)? + } + Operator::I32AtomicRmw16AndU { memarg } => { + translate_atomic_rmw(I32, I16, AtomicRmwOp::And, memarg, builder, state, environ)? + } + Operator::I64AtomicRmw8AndU { memarg } => { + translate_atomic_rmw(I64, I8, AtomicRmwOp::And, memarg, builder, state, environ)? + } + Operator::I64AtomicRmw16AndU { memarg } => { + translate_atomic_rmw(I64, I16, AtomicRmwOp::And, memarg, builder, state, environ)? + } + Operator::I64AtomicRmw32AndU { memarg } => { + translate_atomic_rmw(I64, I32, AtomicRmwOp::And, memarg, builder, state, environ)? + } + + Operator::I32AtomicRmwOr { memarg } => { + translate_atomic_rmw(I32, I32, AtomicRmwOp::Or, memarg, builder, state, environ)? + } + Operator::I64AtomicRmwOr { memarg } => { + translate_atomic_rmw(I64, I64, AtomicRmwOp::Or, memarg, builder, state, environ)? + } + Operator::I32AtomicRmw8OrU { memarg } => { + translate_atomic_rmw(I32, I8, AtomicRmwOp::Or, memarg, builder, state, environ)? + } + Operator::I32AtomicRmw16OrU { memarg } => { + translate_atomic_rmw(I32, I16, AtomicRmwOp::Or, memarg, builder, state, environ)? + } + Operator::I64AtomicRmw8OrU { memarg } => { + translate_atomic_rmw(I64, I8, AtomicRmwOp::Or, memarg, builder, state, environ)? + } + Operator::I64AtomicRmw16OrU { memarg } => { + translate_atomic_rmw(I64, I16, AtomicRmwOp::Or, memarg, builder, state, environ)? + } + Operator::I64AtomicRmw32OrU { memarg } => { + translate_atomic_rmw(I64, I32, AtomicRmwOp::Or, memarg, builder, state, environ)? + } + + Operator::I32AtomicRmwXor { memarg } => { + translate_atomic_rmw(I32, I32, AtomicRmwOp::Xor, memarg, builder, state, environ)? + } + Operator::I64AtomicRmwXor { memarg } => { + translate_atomic_rmw(I64, I64, AtomicRmwOp::Xor, memarg, builder, state, environ)? + } + Operator::I32AtomicRmw8XorU { memarg } => { + translate_atomic_rmw(I32, I8, AtomicRmwOp::Xor, memarg, builder, state, environ)? + } + Operator::I32AtomicRmw16XorU { memarg } => { + translate_atomic_rmw(I32, I16, AtomicRmwOp::Xor, memarg, builder, state, environ)? + } + Operator::I64AtomicRmw8XorU { memarg } => { + translate_atomic_rmw(I64, I8, AtomicRmwOp::Xor, memarg, builder, state, environ)? + } + Operator::I64AtomicRmw16XorU { memarg } => { + translate_atomic_rmw(I64, I16, AtomicRmwOp::Xor, memarg, builder, state, environ)? + } + Operator::I64AtomicRmw32XorU { memarg } => { + translate_atomic_rmw(I64, I32, AtomicRmwOp::Xor, memarg, builder, state, environ)? + } + + Operator::I32AtomicRmwXchg { memarg } => { + translate_atomic_rmw(I32, I32, AtomicRmwOp::Xchg, memarg, builder, state, environ)? + } + Operator::I64AtomicRmwXchg { memarg } => { + translate_atomic_rmw(I64, I64, AtomicRmwOp::Xchg, memarg, builder, state, environ)? + } + Operator::I32AtomicRmw8XchgU { memarg } => { + translate_atomic_rmw(I32, I8, AtomicRmwOp::Xchg, memarg, builder, state, environ)? + } + Operator::I32AtomicRmw16XchgU { memarg } => { + translate_atomic_rmw(I32, I16, AtomicRmwOp::Xchg, memarg, builder, state, environ)? + } + Operator::I64AtomicRmw8XchgU { memarg } => { + translate_atomic_rmw(I64, I8, AtomicRmwOp::Xchg, memarg, builder, state, environ)? + } + Operator::I64AtomicRmw16XchgU { memarg } => { + translate_atomic_rmw(I64, I16, AtomicRmwOp::Xchg, memarg, builder, state, environ)? + } + Operator::I64AtomicRmw32XchgU { memarg } => { + translate_atomic_rmw(I64, I32, AtomicRmwOp::Xchg, memarg, builder, state, environ)? + } + + Operator::I32AtomicRmwCmpxchg { memarg } => { + translate_atomic_cas(I32, I32, memarg, builder, state, environ)? + } + Operator::I64AtomicRmwCmpxchg { memarg } => { + translate_atomic_cas(I64, I64, memarg, builder, state, environ)? + } + Operator::I32AtomicRmw8CmpxchgU { memarg } => { + translate_atomic_cas(I32, I8, memarg, builder, state, environ)? + } + Operator::I32AtomicRmw16CmpxchgU { memarg } => { + translate_atomic_cas(I32, I16, memarg, builder, state, environ)? + } + Operator::I64AtomicRmw8CmpxchgU { memarg } => { + translate_atomic_cas(I64, I8, memarg, builder, state, environ)? + } + Operator::I64AtomicRmw16CmpxchgU { memarg } => { + translate_atomic_cas(I64, I16, memarg, builder, state, environ)? + } + Operator::I64AtomicRmw32CmpxchgU { memarg } => { + translate_atomic_cas(I64, I32, memarg, builder, state, environ)? + } + + Operator::AtomicFence { .. } => { + builder.ins().fence(); + } + Operator::MemoryCopy { src, dst } => { + let src_index = MemoryIndex::from_u32(*src); + let dst_index = MemoryIndex::from_u32(*dst); + let src_heap = state.get_heap(builder.func, *src, environ)?; + let dst_heap = state.get_heap(builder.func, *dst, environ)?; let len = state.pop1(); - let src = state.pop1(); - let dest = state.pop1(); - environ.translate_memory_copy(builder.cursor(), heap_index, heap, dest, src, len)?; + let src_pos = state.pop1(); + let dst_pos = state.pop1(); + environ.translate_memory_copy( + builder.cursor(), + src_index, + src_heap, + dst_index, + dst_heap, + dst_pos, + src_pos, + len, + )?; } - Operator::MemoryFill { .. } => { - // The WebAssembly MVP only supports one linear memory and - // wasmparser will ensure that the memory index specified is - // zero. - let heap_index = MemoryIndex::from_u32(0); - let heap = state.get_heap(builder.func, 0, environ)?; + Operator::MemoryFill { mem } => { + let heap_index = MemoryIndex::from_u32(*mem); + let heap = state.get_heap(builder.func, *mem, environ)?; let len = state.pop1(); let val = state.pop1(); let dest = state.pop1(); environ.translate_memory_fill(builder.cursor(), heap_index, heap, dest, val, len)?; } - Operator::MemoryInit { segment, mem: _ } => { - // The WebAssembly MVP only supports one linear memory and - // wasmparser will ensure that the memory index specified is - // zero. - let heap_index = MemoryIndex::from_u32(0); - let heap = state.get_heap(builder.func, 0, environ)?; + Operator::MemoryInit { segment, mem } => { + let heap_index = MemoryIndex::from_u32(*mem); + let heap = state.get_heap(builder.func, *mem, environ)?; let len = state.pop1(); let src = state.pop1(); let dest = state.pop1(); @@ -1170,41 +1264,45 @@ pub fn translate_operator( environ.translate_data_drop(builder.cursor(), *segment)?; } Operator::TableSize { table: index } => { - let table = state.get_table(builder.func, *index, environ)?; + let table = state.get_or_create_table(builder.func, *index, environ)?; state.push1(environ.translate_table_size( builder.cursor(), TableIndex::from_u32(*index), table, )?); } - Operator::TableGrow { table } => { - let table_index = TableIndex::from_u32(*table); + Operator::TableGrow { table: index } => { + let table_index = TableIndex::from_u32(*index); + let table = state.get_or_create_table(builder.func, *index, environ)?; let delta = state.pop1(); let init_value = state.pop1(); state.push1(environ.translate_table_grow( builder.cursor(), table_index, + table, delta, init_value, )?); } - Operator::TableGet { table } => { - let table_index = TableIndex::from_u32(*table); + Operator::TableGet { table: index } => { + let table_index = TableIndex::from_u32(*index); + let table = state.get_or_create_table(builder.func, *index, environ)?; let index = state.pop1(); - state.push1(environ.translate_table_get(builder.cursor(), table_index, index)?); + state.push1(environ.translate_table_get(builder, table_index, table, index)?); } - Operator::TableSet { table } => { - let table_index = TableIndex::from_u32(*table); + Operator::TableSet { table: index } => { + let table_index = TableIndex::from_u32(*index); + let table = state.get_or_create_table(builder.func, *index, environ)?; let value = state.pop1(); let index = state.pop1(); - environ.translate_table_set(builder.cursor(), table_index, value, index)?; + environ.translate_table_set(builder, table_index, table, value, index)?; } Operator::TableCopy { dst_table: dst_table_index, src_table: src_table_index, } => { - let dst_table = state.get_table(builder.func, *dst_table_index, environ)?; - let src_table = state.get_table(builder.func, *src_table_index, environ)?; + let dst_table = state.get_or_create_table(builder.func, *dst_table_index, environ)?; + let src_table = state.get_or_create_table(builder.func, *src_table_index, environ)?; let len = state.pop1(); let src = state.pop1(); let dest = state.pop1(); @@ -1230,7 +1328,7 @@ pub fn translate_operator( segment, table: table_index, } => { - let table = state.get_table(builder.func, *table_index, environ)?; + let table = state.get_or_create_table(builder.func, *table_index, environ)?; let len = state.pop1(); let src = state.pop1(); let dest = state.pop1(); @@ -1251,7 +1349,8 @@ pub fn translate_operator( let data = value.bytes().to_vec().into(); let handle = builder.func.dfg.constants.insert(data); let value = builder.ins().vconst(I8X16, handle); - // the v128.const is typed in CLIF as a I8x16 but raw_bitcast to a different type before use + // the v128.const is typed in CLIF as a I8x16 but raw_bitcast to a different type + // before use state.push1(value) } Operator::I8x16Splat | Operator::I16x8Splat => { @@ -1266,23 +1365,14 @@ pub fn translate_operator( let splatted = builder.ins().splat(type_of(op), state.pop1()); state.push1(splatted) } - Operator::V128Load8Splat { - memarg: MemoryImmediate { offset, .. }, - } - | Operator::V128Load16Splat { - memarg: MemoryImmediate { offset, .. }, - } - | Operator::V128Load32Splat { - memarg: MemoryImmediate { offset, .. }, - } - | Operator::V128Load64Splat { - memarg: MemoryImmediate { offset, .. }, - } => { - // TODO: For spec compliance, this is initially implemented as a combination of `load + - // splat` but could be implemented eventually as a single instruction (`load_splat`). - // See https://github.com/bytecodealliance/wasmtime/issues/1175. + Operator::V128Load8Splat { memarg } + | Operator::V128Load16Splat { memarg } + | Operator::V128Load32Splat { memarg } + | Operator::V128Load64Splat { memarg } => { + // TODO: remove this in favor of the single instruction LoadSplat code above + // which is not yet available in Cranelift 0.67 translate_load( - *offset, + memarg, ir::Opcode::Load, type_of(op).lane_type(), builder, @@ -1291,15 +1381,27 @@ pub fn translate_operator( )?; let splatted = builder.ins().splat(type_of(op), state.pop1()); state.push1(splatted) + + // let opcode = ir::Opcode::LoadSplat; + // let result_ty = type_of(op); + // let (flags, base, offset) = prepare_load( + // memarg, + // mem_op_size(opcode, result_ty.lane_type()), + // builder, + // state, + // environ, + // )?; + // let (load, dfg) = builder.ins().Load(opcode, result_ty, flags, offset, base); + // state.push1(dfg.first_result(load)) } Operator::I8x16ExtractLaneS { lane } | Operator::I16x8ExtractLaneS { lane } => { let vector = pop1_with_bitcast(state, type_of(op), builder); - let extracted = builder.ins().extractlane(vector, *lane); + let extracted = builder.ins().extractlane(vector, lane.clone()); state.push1(builder.ins().sextend(I32, extracted)) } Operator::I8x16ExtractLaneU { lane } | Operator::I16x8ExtractLaneU { lane } => { let vector = pop1_with_bitcast(state, type_of(op), builder); - let extracted = builder.ins().extractlane(vector, *lane); + let extracted = builder.ins().extractlane(vector, lane.clone()); state.push1(builder.ins().uextend(I32, extracted)); // On x86, PEXTRB zeroes the upper bits of the destination register of extractlane so // uextend could be elided; for now, uextend is needed for Cranelift's type checks to @@ -1310,7 +1412,7 @@ pub fn translate_operator( | Operator::F32x4ExtractLane { lane } | Operator::F64x2ExtractLane { lane } => { let vector = pop1_with_bitcast(state, type_of(op), builder); - state.push1(builder.ins().extractlane(vector, *lane)) + state.push1(builder.ins().extractlane(vector, lane.clone())) } Operator::I8x16ReplaceLane { lane } | Operator::I16x8ReplaceLane { lane } => { let (vector, replacement) = state.pop2(); @@ -1390,6 +1492,10 @@ pub fn translate_operator( let a = pop1_with_bitcast(state, type_of(op), builder); state.push1(builder.ins().ineg(a)) } + Operator::I8x16Abs | Operator::I16x8Abs | Operator::I32x4Abs => { + let a = pop1_with_bitcast(state, type_of(op), builder); + state.push1(builder.ins().iabs(a)) + } Operator::I16x8Mul | Operator::I32x4Mul | Operator::I64x2Mul => { let (a, b) = pop2_with_bitcast(state, type_of(op), builder); state.push1(builder.ins().imul(a, b)) @@ -1460,6 +1566,11 @@ pub fn translate_operator( let bool_result = builder.ins().vall_true(a); state.push1(builder.ins().bint(I32, bool_result)) } + Operator::I8x16Bitmask | Operator::I16x8Bitmask | Operator::I32x4Bitmask => { + unimplemented!("SIMD Operator {:?} not yet implemented", op); + // let a = pop1_with_bitcast(state, type_of(op), builder); + // state.push1(builder.ins().vhigh_bits(I32, a)); + } Operator::I8x16Eq | Operator::I16x8Eq | Operator::I32x4Eq => { translate_vector_icmp(IntCC::Equal, type_of(op), builder, state) } @@ -1535,6 +1646,21 @@ pub fn translate_operator( let (a, b) = pop2_with_bitcast(state, type_of(op), builder); state.push1(builder.ins().fmin(a, b)) } + Operator::F32x4PMax | Operator::F64x2PMax => { + unimplemented!("SIMD Operator {:?} not yet implemented", op); + // let (a, b) = pop2_with_bitcast(state, type_of(op), builder); + // state.push1(builder.ins().fmax_pseudo(a, b)) + } + Operator::F32x4PMin | Operator::F64x2PMin => { + unimplemented!("SIMD Operator {:?} not yet implemented", op); + // let (a, b) = pop2_with_bitcast(state, type_of(op), builder); + // state.push1(builder.ins().fmin_pseudo(a, b)) + } + Operator::I32x4DotI16x8S + | Operator::V128Load32Zero { .. } + | Operator::V128Load64Zero { .. } => { + unimplemented!("SIMD Operator {:?} not yet implemented", op); + } Operator::F32x4Sqrt | Operator::F64x2Sqrt => { let a = pop1_with_bitcast(state, type_of(op), builder); state.push1(builder.ins().sqrt(a)) @@ -1555,42 +1681,81 @@ pub fn translate_operator( let a = pop1_with_bitcast(state, I32X4, builder); state.push1(builder.ins().fcvt_from_uint(F32X4, a)) } - Operator::I32x4TruncSatF32x4S - | Operator::I32x4TruncSatF32x4U - | Operator::I8x16Abs - | Operator::I16x8Abs - | Operator::I32x4Abs - | Operator::I32x4DotI16x8S - | Operator::I8x16NarrowI16x8S { .. } - | Operator::I8x16NarrowI16x8U { .. } - | Operator::I16x8NarrowI32x4S { .. } - | Operator::I16x8NarrowI32x4U { .. } - | Operator::I16x8WidenLowI8x16S { .. } - | Operator::I16x8WidenHighI8x16S { .. } - | Operator::I16x8WidenLowI8x16U { .. } - | Operator::I16x8WidenHighI8x16U { .. } - | Operator::I32x4WidenLowI16x8S { .. } - | Operator::I32x4WidenHighI16x8S { .. } - | Operator::I32x4WidenLowI16x8U { .. } - | Operator::I32x4WidenHighI16x8U { .. } - | Operator::I8x16Bitmask - | Operator::I16x8Bitmask - | Operator::I32x4Bitmask - | Operator::F32x4Ceil - | Operator::F32x4Floor - | Operator::F32x4Trunc - | Operator::F32x4Nearest - | Operator::F64x2Ceil - | Operator::F64x2Floor - | Operator::F64x2Trunc - | Operator::F64x2Nearest - | Operator::F32x4PMin - | Operator::F32x4PMax - | Operator::F64x2PMin - | Operator::F64x2PMax - | Operator::V128Load32Zero { .. } - | Operator::V128Load64Zero { .. } => { - return Err(wasm_unsupported!("proposed SIMD operator {:?}", op)); + Operator::I32x4TruncSatF32x4S => { + let a = pop1_with_bitcast(state, F32X4, builder); + state.push1(builder.ins().fcvt_to_sint_sat(I32X4, a)) + } + Operator::I32x4TruncSatF32x4U => { + let a = pop1_with_bitcast(state, F32X4, builder); + state.push1(builder.ins().fcvt_to_uint_sat(I32X4, a)) + } + Operator::I8x16NarrowI16x8S => { + let (a, b) = pop2_with_bitcast(state, I16X8, builder); + state.push1(builder.ins().snarrow(a, b)) + } + Operator::I16x8NarrowI32x4S => { + let (a, b) = pop2_with_bitcast(state, I32X4, builder); + state.push1(builder.ins().snarrow(a, b)) + } + Operator::I8x16NarrowI16x8U => { + let (a, b) = pop2_with_bitcast(state, I16X8, builder); + state.push1(builder.ins().unarrow(a, b)) + } + Operator::I16x8NarrowI32x4U => { + let (a, b) = pop2_with_bitcast(state, I32X4, builder); + state.push1(builder.ins().unarrow(a, b)) + } + Operator::I16x8WidenLowI8x16S => { + let a = pop1_with_bitcast(state, I8X16, builder); + state.push1(builder.ins().swiden_low(a)) + } + Operator::I16x8WidenHighI8x16S => { + let a = pop1_with_bitcast(state, I8X16, builder); + state.push1(builder.ins().swiden_high(a)) + } + Operator::I16x8WidenLowI8x16U => { + let a = pop1_with_bitcast(state, I8X16, builder); + state.push1(builder.ins().uwiden_low(a)) + } + Operator::I16x8WidenHighI8x16U => { + let a = pop1_with_bitcast(state, I8X16, builder); + state.push1(builder.ins().uwiden_high(a)) + } + Operator::I32x4WidenLowI16x8S => { + let a = pop1_with_bitcast(state, I16X8, builder); + state.push1(builder.ins().swiden_low(a)) + } + Operator::I32x4WidenHighI16x8S => { + let a = pop1_with_bitcast(state, I16X8, builder); + state.push1(builder.ins().swiden_high(a)) + } + Operator::I32x4WidenLowI16x8U => { + let a = pop1_with_bitcast(state, I16X8, builder); + state.push1(builder.ins().uwiden_low(a)) + } + Operator::I32x4WidenHighI16x8U => { + let a = pop1_with_bitcast(state, I16X8, builder); + state.push1(builder.ins().uwiden_high(a)) + } + + Operator::F32x4Ceil | Operator::F64x2Ceil => { + // This is something of a misuse of `type_of`, because that produces the return type + // of `op`. In this case we want the arg type, but we know it's the same as the + // return type. Same for the 3 cases below. + let arg = pop1_with_bitcast(state, type_of(op), builder); + state.push1(builder.ins().ceil(arg)); + } + Operator::F32x4Floor | Operator::F64x2Floor => { + let arg = pop1_with_bitcast(state, type_of(op), builder); + state.push1(builder.ins().floor(arg)); + } + Operator::F32x4Trunc | Operator::F64x2Trunc => { + let arg = pop1_with_bitcast(state, type_of(op), builder); + state.push1(builder.ins().trunc(arg)); + } + Operator::F32x4Nearest | Operator::F64x2Nearest => { + let arg = pop1_with_bitcast(state, type_of(op), builder); + state.push1(builder.ins().nearest(arg)); } Operator::ReturnCall { .. } | Operator::ReturnCallIndirect { .. } => { @@ -1652,13 +1817,19 @@ fn translate_unreachable_operator( let (params, _results) = module_translation_state.blocktype_params_results(blocktype)?; let else_block = block_with_params(builder, params, environ)?; + let frame = state.control_stack.last().unwrap(); + frame.truncate_value_stack_to_else_params(&mut state.stack); // We change the target of the branch instruction. builder.change_jump_destination(branch_inst, else_block); builder.seal_block(else_block); else_block } - ElseData::WithElse { else_block } => else_block, + ElseData::WithElse { else_block } => { + let frame = state.control_stack.last().unwrap(); + frame.truncate_value_stack_to_else_params(&mut state.stack); + else_block + } }; builder.switch_to_block(else_block); @@ -1677,9 +1848,8 @@ fn translate_unreachable_operator( let control_stack = &mut state.control_stack; let frame = control_stack.pop().unwrap(); - // Now we have to split off the stack the values not used - // by unreachable code that hasn't been translated - stack.truncate(frame.original_stack_size()); + // Pop unused parameters from stack. + frame.truncate_value_stack_to_original_size(stack); let reachable_anyway = match frame { // If it is a loop we also have to seal the body loop block @@ -1812,7 +1982,7 @@ fn get_heap_addr( /// Prepare for a load; factors out common functionality between load and load_extend operations. fn prepare_load( - offset: u32, + memarg: &MemoryImmediate, loaded_bytes: u32, builder: &mut FunctionBuilder, state: &mut FuncTranslationState, @@ -1820,12 +1990,11 @@ fn prepare_load( ) -> WasmResult<(MemFlags, Value, Offset32)> { let addr32 = state.pop1(); - // We don't yet support multiple linear memories. - let heap = state.get_heap(builder.func, 0, environ)?; + let heap = state.get_heap(builder.func, memarg.memory, environ)?; let (base, offset) = get_heap_addr( heap, addr32, - offset, + memarg.offset, loaded_bytes, environ.pointer_type(), builder, @@ -1841,7 +2010,7 @@ fn prepare_load( /// Translate a load instruction. fn translate_load( - offset: u32, + memarg: &MemoryImmediate, opcode: ir::Opcode, result_ty: Type, builder: &mut FunctionBuilder, @@ -1849,7 +2018,7 @@ fn translate_load( environ: &mut FE, ) -> WasmResult<()> { let (flags, base, offset) = prepare_load( - offset, + memarg, mem_op_size(opcode, result_ty), builder, state, @@ -1862,7 +2031,7 @@ fn translate_load( /// Translate a store instruction. fn translate_store( - offset: u32, + memarg: &MemoryImmediate, opcode: ir::Opcode, builder: &mut FunctionBuilder, state: &mut FuncTranslationState, @@ -1871,17 +2040,16 @@ fn translate_store( let (addr32, val) = state.pop2(); let val_ty = builder.func.dfg.value_type(val); - // We don't yet support multiple linear memories. - let heap = state.get_heap(builder.func, 0, environ)?; + let heap = state.get_heap(builder.func, memarg.memory, environ)?; let (base, offset) = get_heap_addr( heap, addr32, - offset, + memarg.offset, mem_op_size(opcode, val_ty), environ.pointer_type(), builder, ); - // See the comments in `translate_load` about the flags. + // See the comments in `prepare_load` about the flags. let flags = MemFlags::new(); builder .ins() @@ -1894,7 +2062,7 @@ fn mem_op_size(opcode: ir::Opcode, ty: Type) -> u32 { ir::Opcode::Istore8 | ir::Opcode::Sload8 | ir::Opcode::Uload8 => 1, ir::Opcode::Istore16 | ir::Opcode::Sload16 | ir::Opcode::Uload16 => 2, ir::Opcode::Istore32 | ir::Opcode::Sload32 | ir::Opcode::Uload32 => 4, - ir::Opcode::Store | ir::Opcode::Load => ty.bytes(), + ir::Opcode::Store | ir::Opcode::Load /* | ir::Opcode::LoadSplat */=> ty.bytes(), _ => panic!("unknown size of mem op for {:?}", opcode), } } @@ -1905,6 +2073,235 @@ fn translate_icmp(cc: IntCC, builder: &mut FunctionBuilder, state: &mut FuncTran state.push1(builder.ins().bint(I32, val)); } +// For an atomic memory operation, emit an alignment check for the linear memory address, +// and then compute the final effective address. +fn finalise_atomic_mem_addr( + linear_mem_addr: Value, + memarg: &MemoryImmediate, + access_ty: Type, + builder: &mut FunctionBuilder, + state: &mut FuncTranslationState, + environ: &mut FE, +) -> WasmResult { + // Check the alignment of `linear_mem_addr`. + let access_ty_bytes = access_ty.bytes(); + let final_lma = builder + .ins() + .iadd_imm(linear_mem_addr, i64::from(memarg.offset)); + if access_ty_bytes != 1 { + assert!(access_ty_bytes == 2 || access_ty_bytes == 4 || access_ty_bytes == 8); + let final_lma_misalignment = builder + .ins() + .band_imm(final_lma, i64::from(access_ty_bytes - 1)); + let f = builder + .ins() + .ifcmp_imm(final_lma_misalignment, i64::from(0)); + builder + .ins() + .trapif(IntCC::NotEqual, f, ir::TrapCode::HeapMisaligned); + } + + // Compute the final effective address. + let heap = state.get_heap(builder.func, memarg.memory, environ)?; + let (base, offset) = get_heap_addr( + heap, + final_lma, + /*offset=*/ 0, + access_ty.bytes(), + environ.pointer_type(), + builder, + ); + + let final_effective_address = builder.ins().iadd_imm(base, i64::from(offset)); + Ok(final_effective_address) +} + +fn translate_atomic_rmw( + widened_ty: Type, + access_ty: Type, + op: AtomicRmwOp, + memarg: &MemoryImmediate, + builder: &mut FunctionBuilder, + state: &mut FuncTranslationState, + environ: &mut FE, +) -> WasmResult<()> { + let (linear_mem_addr, mut arg2) = state.pop2(); + let arg2_ty = builder.func.dfg.value_type(arg2); + + // The operation is performed at type `access_ty`, and the old value is zero-extended + // to type `widened_ty`. + match access_ty { + I8 | I16 | I32 | I64 => {} + _ => { + return Err(wasm_unsupported!( + "atomic_rmw: unsupported access type {:?}", + access_ty + )) + } + }; + let w_ty_ok = match widened_ty { + I32 | I64 => true, + _ => false, + }; + assert!(w_ty_ok && widened_ty.bytes() >= access_ty.bytes()); + + assert!(arg2_ty.bytes() >= access_ty.bytes()); + if arg2_ty.bytes() > access_ty.bytes() { + arg2 = builder.ins().ireduce(access_ty, arg2); + } + + let final_effective_address = + finalise_atomic_mem_addr(linear_mem_addr, memarg, access_ty, builder, state, environ)?; + + // See the comments in `prepare_load` about the flags. + let flags = MemFlags::new(); + let mut res = builder + .ins() + .atomic_rmw(access_ty, flags, op, final_effective_address, arg2); + if access_ty != widened_ty { + res = builder.ins().uextend(widened_ty, res); + } + state.push1(res); + Ok(()) +} + +fn translate_atomic_cas( + widened_ty: Type, + access_ty: Type, + memarg: &MemoryImmediate, + builder: &mut FunctionBuilder, + state: &mut FuncTranslationState, + environ: &mut FE, +) -> WasmResult<()> { + let (linear_mem_addr, mut expected, mut replacement) = state.pop3(); + let expected_ty = builder.func.dfg.value_type(expected); + let replacement_ty = builder.func.dfg.value_type(replacement); + + // The compare-and-swap is performed at type `access_ty`, and the old value is zero-extended + // to type `widened_ty`. + match access_ty { + I8 | I16 | I32 | I64 => {} + _ => { + return Err(wasm_unsupported!( + "atomic_cas: unsupported access type {:?}", + access_ty + )) + } + }; + let w_ty_ok = match widened_ty { + I32 | I64 => true, + _ => false, + }; + assert!(w_ty_ok && widened_ty.bytes() >= access_ty.bytes()); + + assert!(expected_ty.bytes() >= access_ty.bytes()); + if expected_ty.bytes() > access_ty.bytes() { + expected = builder.ins().ireduce(access_ty, expected); + } + assert!(replacement_ty.bytes() >= access_ty.bytes()); + if replacement_ty.bytes() > access_ty.bytes() { + replacement = builder.ins().ireduce(access_ty, replacement); + } + + let final_effective_address = + finalise_atomic_mem_addr(linear_mem_addr, memarg, access_ty, builder, state, environ)?; + + // See the comments in `prepare_load` about the flags. + let flags = MemFlags::new(); + let mut res = builder + .ins() + .atomic_cas(flags, final_effective_address, expected, replacement); + if access_ty != widened_ty { + res = builder.ins().uextend(widened_ty, res); + } + state.push1(res); + Ok(()) +} + +fn translate_atomic_load( + widened_ty: Type, + access_ty: Type, + memarg: &MemoryImmediate, + builder: &mut FunctionBuilder, + state: &mut FuncTranslationState, + environ: &mut FE, +) -> WasmResult<()> { + let linear_mem_addr = state.pop1(); + + // The load is performed at type `access_ty`, and the loaded value is zero extended + // to `widened_ty`. + match access_ty { + I8 | I16 | I32 | I64 => {} + _ => { + return Err(wasm_unsupported!( + "atomic_load: unsupported access type {:?}", + access_ty + )) + } + }; + let w_ty_ok = match widened_ty { + I32 | I64 => true, + _ => false, + }; + assert!(w_ty_ok && widened_ty.bytes() >= access_ty.bytes()); + + let final_effective_address = + finalise_atomic_mem_addr(linear_mem_addr, memarg, access_ty, builder, state, environ)?; + + // See the comments in `prepare_load` about the flags. + let flags = MemFlags::new(); + let mut res = builder + .ins() + .atomic_load(access_ty, flags, final_effective_address); + if access_ty != widened_ty { + res = builder.ins().uextend(widened_ty, res); + } + state.push1(res); + Ok(()) +} + +fn translate_atomic_store( + access_ty: Type, + memarg: &MemoryImmediate, + builder: &mut FunctionBuilder, + state: &mut FuncTranslationState, + environ: &mut FE, +) -> WasmResult<()> { + let (linear_mem_addr, mut data) = state.pop2(); + let data_ty = builder.func.dfg.value_type(data); + + // The operation is performed at type `access_ty`, and the data to be stored may first + // need to be narrowed accordingly. + match access_ty { + I8 | I16 | I32 | I64 => {} + _ => { + return Err(wasm_unsupported!( + "atomic_store: unsupported access type {:?}", + access_ty + )) + } + }; + let d_ty_ok = match data_ty { + I32 | I64 => true, + _ => false, + }; + assert!(d_ty_ok && data_ty.bytes() >= access_ty.bytes()); + + if data_ty.bytes() > access_ty.bytes() { + data = builder.ins().ireduce(access_ty, data); + } + + let final_effective_address = + finalise_atomic_mem_addr(linear_mem_addr, memarg, access_ty, builder, state, environ)?; + + // See the comments in `prepare_load` about the flags. + let flags = MemFlags::new(); + builder + .ins() + .atomic_store(flags, data, final_effective_address); + Ok(()) +} + fn translate_vector_icmp( cc: IntCC, needed_type: Type, @@ -1942,15 +2339,10 @@ fn translate_br_if( ) { let val = state.pop1(); let (br_destination, inputs) = translate_br_if_args(relative_depth, state); - - // Bitcast any vector arguments to their default type, I8X16, before jumping. - let destination_types = builder.func.dfg.block_param_types(br_destination); - bitcast_arguments(inputs, &destination_types[..inputs.len()], builder); - - builder.ins().brnz(val, br_destination, inputs); + canonicalise_then_brnz(builder, val, br_destination, inputs); let next_block = builder.create_block(); - builder.ins().jump(next_block, &[]); + canonicalise_then_jump(builder, next_block, &[]); builder.seal_block(next_block); // The only predecessor is the current block. builder.switch_to_block(next_block); } @@ -2006,6 +2398,7 @@ fn type_of(operator: &Operator) -> Type { | Operator::I8x16GeS | Operator::I8x16GeU | Operator::I8x16Neg + | Operator::I8x16Abs | Operator::I8x16AnyTrue | Operator::I8x16AllTrue | Operator::I8x16Shl @@ -2021,7 +2414,8 @@ fn type_of(operator: &Operator) -> Type { | Operator::I8x16MinU | Operator::I8x16MaxS | Operator::I8x16MaxU - | Operator::I8x16RoundingAverageU => I8X16, + | Operator::I8x16RoundingAverageU + | Operator::I8x16Bitmask => I8X16, Operator::I16x8Splat | Operator::V128Load16Splat { .. } @@ -2039,6 +2433,7 @@ fn type_of(operator: &Operator) -> Type { | Operator::I16x8GeS | Operator::I16x8GeU | Operator::I16x8Neg + | Operator::I16x8Abs | Operator::I16x8AnyTrue | Operator::I16x8AllTrue | Operator::I16x8Shl @@ -2055,7 +2450,8 @@ fn type_of(operator: &Operator) -> Type { | Operator::I16x8MaxS | Operator::I16x8MaxU | Operator::I16x8RoundingAverageU - | Operator::I16x8Mul => I16X8, + | Operator::I16x8Mul + | Operator::I16x8Bitmask => I16X8, Operator::I32x4Splat | Operator::V128Load32Splat { .. } @@ -2072,6 +2468,7 @@ fn type_of(operator: &Operator) -> Type { | Operator::I32x4GeS | Operator::I32x4GeU | Operator::I32x4Neg + | Operator::I32x4Abs | Operator::I32x4AnyTrue | Operator::I32x4AllTrue | Operator::I32x4Shl @@ -2085,7 +2482,8 @@ fn type_of(operator: &Operator) -> Type { | Operator::I32x4MaxS | Operator::I32x4MaxU | Operator::F32x4ConvertI32x4S - | Operator::F32x4ConvertI32x4U => I32X4, + | Operator::F32x4ConvertI32x4U + | Operator::I32x4Bitmask => I32X4, Operator::I64x2Splat | Operator::V128Load64Splat { .. } @@ -2117,8 +2515,14 @@ fn type_of(operator: &Operator) -> Type { | Operator::F32x4Div | Operator::F32x4Min | Operator::F32x4Max + | Operator::F32x4PMin + | Operator::F32x4PMax | Operator::I32x4TruncSatF32x4S - | Operator::I32x4TruncSatF32x4U => F32X4, + | Operator::I32x4TruncSatF32x4U + | Operator::F32x4Ceil + | Operator::F32x4Floor + | Operator::F32x4Trunc + | Operator::F32x4Nearest => F32X4, Operator::F64x2Splat | Operator::F64x2ExtractLane { .. } @@ -2137,7 +2541,13 @@ fn type_of(operator: &Operator) -> Type { | Operator::F64x2Mul | Operator::F64x2Div | Operator::F64x2Min - | Operator::F64x2Max => F64X2, + | Operator::F64x2Max + | Operator::F64x2PMin + | Operator::F64x2PMax + | Operator::F64x2Ceil + | Operator::F64x2Floor + | Operator::F64x2Trunc + | Operator::F64x2Nearest => F64X2, _ => unimplemented!( "Currently only SIMD instructions are mapped to their return type; the \ @@ -2149,7 +2559,7 @@ fn type_of(operator: &Operator) -> Type { /// Some SIMD operations only operate on I8X16 in CLIF; this will convert them to that type by /// adding a raw_bitcast if necessary. -pub fn optionally_bitcast_vector( +fn optionally_bitcast_vector( value: Value, needed_type: Type, builder: &mut FunctionBuilder, @@ -2161,6 +2571,80 @@ pub fn optionally_bitcast_vector( } } +#[inline(always)] +fn is_non_canonical_v128(ty: ir::Type) -> bool { + match ty { + B8X16 | B16X8 | B32X4 | B64X2 | I64X2 | I32X4 | I16X8 | F32X4 | F64X2 => true, + _ => false, + } +} + +/// Cast to I8X16, any vector values in `values` that are of "non-canonical" type (meaning, not +/// I8X16), and return them in a slice. A pre-scan is made to determine whether any casts are +/// actually necessary, and if not, the original slice is returned. Otherwise the cast values +/// are returned in a slice that belongs to the caller-supplied `SmallVec`. +fn canonicalise_v128_values<'a>( + tmp_canonicalised: &'a mut SmallVec<[ir::Value; 16]>, + builder: &mut FunctionBuilder, + values: &'a [ir::Value], +) -> &'a [ir::Value] { + debug_assert!(tmp_canonicalised.is_empty()); + // First figure out if any of the parameters need to be cast. Mostly they don't need to be. + let any_non_canonical = values + .iter() + .any(|v| is_non_canonical_v128(builder.func.dfg.value_type(*v))); + // Hopefully we take this exit most of the time, hence doing no heap allocation. + if !any_non_canonical { + return values; + } + // Otherwise we'll have to cast, and push the resulting `Value`s into `canonicalised`. + for v in values { + tmp_canonicalised.push(if is_non_canonical_v128(builder.func.dfg.value_type(*v)) { + builder.ins().raw_bitcast(I8X16, *v) + } else { + *v + }); + } + tmp_canonicalised.as_slice() +} + +/// Generate a `jump` instruction, but first cast all 128-bit vector values to I8X16 if they +/// don't have that type. This is done in somewhat roundabout way so as to ensure that we +/// almost never have to do any heap allocation. +fn canonicalise_then_jump( + builder: &mut FunctionBuilder, + destination: ir::Block, + params: &[ir::Value], +) -> ir::Inst { + let mut tmp_canonicalised = SmallVec::<[ir::Value; 16]>::new(); + let canonicalised = canonicalise_v128_values(&mut tmp_canonicalised, builder, params); + builder.ins().jump(destination, canonicalised) +} + +/// The same but for a `brz` instruction. +fn canonicalise_then_brz( + builder: &mut FunctionBuilder, + cond: ir::Value, + destination: ir::Block, + params: &[Value], +) -> ir::Inst { + let mut tmp_canonicalised = SmallVec::<[ir::Value; 16]>::new(); + let canonicalised = canonicalise_v128_values(&mut tmp_canonicalised, builder, params); + builder.ins().brz(cond, destination, canonicalised) +} + +/// The same but for a `brnz` instruction. +fn canonicalise_then_brnz( + builder: &mut FunctionBuilder, + cond: ir::Value, + destination: ir::Block, + params: &[Value], +) -> ir::Inst { + let mut tmp_canonicalised = SmallVec::<[ir::Value; 16]>::new(); + let canonicalised = canonicalise_v128_values(&mut tmp_canonicalised, builder, params); + builder.ins().brnz(cond, destination, canonicalised) +} + /// A helper for popping and bitcasting a single value; since SIMD values can lose their type by /// using v128 (i.e. CLIF's I8x16) we must re-type the values using a bitcast to avoid CLIF /// typing issues. diff --git a/lib/compiler-cranelift/src/translator/func_environ.rs b/lib/compiler-cranelift/src/translator/func_environ.rs index cfc07b38890..efdcb06994b 100644 --- a/lib/compiler-cranelift/src/translator/func_environ.rs +++ b/lib/compiler-cranelift/src/translator/func_environ.rs @@ -12,7 +12,7 @@ use cranelift_codegen::ir::immediates::Offset32; use cranelift_codegen::ir::{self, InstBuilder}; use cranelift_codegen::isa::TargetFrontendConfig; use cranelift_frontend::FunctionBuilder; -use wasmer_compiler::wasmparser::Operator; +use wasmer_compiler::wasmparser::{Operator, Type}; use wasmer_compiler::WasmResult; use wasmer_types::{FunctionIndex, GlobalIndex, MemoryIndex, SignatureIndex, TableIndex}; @@ -229,8 +229,10 @@ pub trait FuncEnvironment: TargetEnvironment { fn translate_memory_copy( &mut self, pos: FuncCursor, - index: MemoryIndex, - heap: ir::Heap, + src_index: MemoryIndex, + src_heap: ir::Heap, + dst_index: MemoryIndex, + dst_heap: ir::Heap, dst: ir::Value, src: ir::Value, len: ir::Value, @@ -283,6 +285,7 @@ pub trait FuncEnvironment: TargetEnvironment { &mut self, pos: FuncCursor, table_index: TableIndex, + table: ir::Table, delta: ir::Value, init_value: ir::Value, ) -> WasmResult; @@ -290,16 +293,18 @@ pub trait FuncEnvironment: TargetEnvironment { /// Translate a `table.get` WebAssembly instruction. fn translate_table_get( &mut self, - pos: FuncCursor, + builder: &mut FunctionBuilder, table_index: TableIndex, + table: ir::Table, index: ir::Value, ) -> WasmResult; /// Translate a `table.set` WebAssembly instruction. fn translate_table_set( &mut self, - pos: FuncCursor, + builder: &mut FunctionBuilder, table_index: TableIndex, + table: ir::Table, value: ir::Value, index: ir::Value, ) -> WasmResult<()>; @@ -344,8 +349,44 @@ pub trait FuncEnvironment: TargetEnvironment { /// Translate a `elem.drop` WebAssembly instruction. fn translate_elem_drop(&mut self, pos: FuncCursor, seg_index: u32) -> WasmResult<()>; + /// Translate a `ref.null T` WebAssembly instruction. + /// + /// By default, translates into a null reference type. + /// + /// Override this if you don't use Cranelift reference types for all Wasm + /// reference types (e.g. you use a raw pointer for `funcref`s) or if the + /// null sentinel is not a null reference type pointer for your type. If you + /// override this method, then you should also override + /// `translate_ref_is_null` as well. + fn translate_ref_null(&mut self, pos: FuncCursor, ty: Type) -> WasmResult; + // { + // let _ = ty; + // Ok(pos.ins().null(self.reference_type(ty))) + // } + + /// Translate a `ref.is_null` WebAssembly instruction. + /// + /// By default, assumes that `value` is a Cranelift reference type, and that + /// a null Cranelift reference type is the null value for all Wasm reference + /// types. + /// + /// If you override this method, you probably also want to override + /// `translate_ref_null` as well. + fn translate_ref_is_null( + &mut self, + mut pos: FuncCursor, + value: ir::Value, + ) -> WasmResult { + let is_null = pos.ins().is_null(value); + Ok(pos.ins().bint(ir::types::I32, is_null)) + } + /// Translate a `ref.func` WebAssembly instruction. - fn translate_ref_func(&mut self, pos: FuncCursor, func_index: u32) -> WasmResult; + fn translate_ref_func( + &mut self, + pos: FuncCursor, + func_index: FunctionIndex, + ) -> WasmResult; /// Translate a `global.get` WebAssembly instruction at `pos` for a global /// that is custom. @@ -364,6 +405,38 @@ pub trait FuncEnvironment: TargetEnvironment { val: ir::Value, ) -> WasmResult<()>; + /// Translate an `i32.atomic.wait` or `i64.atomic.wait` WebAssembly instruction. + /// The `index` provided identifies the linear memory containing the value + /// to wait on, and `heap` is the heap reference returned by `make_heap` + /// for the same index. Whether the waited-on value is 32- or 64-bit can be + /// determined by examining the type of `expected`, which must be only I32 or I64. + /// + /// Returns an i32, which is negative if the helper call failed. + fn translate_atomic_wait( + &mut self, + pos: FuncCursor, + index: MemoryIndex, + heap: ir::Heap, + addr: ir::Value, + expected: ir::Value, + timeout: ir::Value, + ) -> WasmResult; + + /// Translate an `atomic.notify` WebAssembly instruction. + /// The `index` provided identifies the linear memory containing the value + /// to wait on, and `heap` is the heap reference returned by `make_heap` + /// for the same index. + /// + /// Returns an i64, which is negative if the helper call failed. + fn translate_atomic_notify( + &mut self, + pos: FuncCursor, + index: MemoryIndex, + heap: ir::Heap, + addr: ir::Value, + count: ir::Value, + ) -> WasmResult; + /// Emit code at the beginning of every wasm loop. /// /// This can be used to insert explicit interrupt or safepoint checking at diff --git a/lib/compiler-cranelift/src/translator/func_state.rs b/lib/compiler-cranelift/src/translator/func_state.rs index a9af4b181e1..1cd8db9bd33 100644 --- a/lib/compiler-cranelift/src/translator/func_state.rs +++ b/lib/compiler-cranelift/src/translator/func_state.rs @@ -132,7 +132,9 @@ impl ControlStackFrame { Self::Loop { header, .. } => header, } } - pub fn original_stack_size(&self) -> usize { + /// Private helper. Use `truncate_value_stack_to_else_params()` or + /// `truncate_value_stack_to_original_size()` to restore value-stack state. + fn original_stack_size(&self) -> usize { match *self { Self::If { original_stack_size, @@ -182,6 +184,33 @@ impl ControlStackFrame { Self::Loop { .. } => {} } } + + /// Pop values from the value stack so that it is left at the + /// input-parameters to an else-block. + pub fn truncate_value_stack_to_else_params(&self, stack: &mut Vec) { + debug_assert!(matches!(self, &ControlStackFrame::If { .. })); + stack.truncate(self.original_stack_size()); + } + + /// Pop values from the value stack so that it is left at the state it was + /// before this control-flow frame. + pub fn truncate_value_stack_to_original_size(&self, stack: &mut Vec) { + // The "If" frame pushes its parameters twice, so they're available to the else block + // (see also `FuncTranslationState::push_if`). + // Yet, the original_stack_size member accounts for them only once, so that the else + // block can see the same number of parameters as the consequent block. As a matter of + // fact, we need to substract an extra number of parameter values for if blocks. + let num_duplicated_params = match self { + &ControlStackFrame::If { + num_param_values, .. + } => { + debug_assert!(num_param_values <= self.original_stack_size()); + num_param_values + } + _ => 0, + }; + stack.truncate(self.original_stack_size() - num_duplicated_params); + } } /// Contains information passed along during a function's translation and that records: @@ -219,6 +248,16 @@ pub struct FuncTranslationState { functions: HashMap, } +// Public methods that are exposed to non-`cranelift_wasm` API consumers. +impl FuncTranslationState { + /// True if the current translation state expresses reachable code, false if it is unreachable. + #[inline] + #[allow(dead_code)] + pub fn reachable(&self) -> bool { + self.reachable + } +} + impl FuncTranslationState { /// Construct a new, empty, `FuncTranslationState` pub(crate) fn new() -> Self { @@ -441,7 +480,7 @@ impl FuncTranslationState { /// Get the `Table` reference that should be used to access table `index`. /// Create the reference if necessary. - pub(crate) fn get_table( + pub(crate) fn get_or_create_table( &mut self, func: &mut ir::Function, index: u32, diff --git a/lib/compiler-cranelift/src/translator/func_translator.rs b/lib/compiler-cranelift/src/translator/func_translator.rs index 173707f0ce4..94dcd8d8064 100644 --- a/lib/compiler-cranelift/src/translator/func_translator.rs +++ b/lib/compiler-cranelift/src/translator/func_translator.rs @@ -176,12 +176,9 @@ fn parse_local_decls( let mut next_local = num_params; let local_count = reader.read_local_count().map_err(to_wasm_error)?; - let mut locals_total = 0; for _ in 0..local_count { builder.set_srcloc(cur_srcloc(reader)); - let (count, ty) = reader - .read_local_decl(&mut locals_total) - .map_err(to_wasm_error)?; + let (count, ty) = reader.read_local_decl().map_err(to_wasm_error)?; declare_locals(builder, count, ty, &mut next_local, environ)?; } diff --git a/lib/compiler-llvm/Cargo.toml b/lib/compiler-llvm/Cargo.toml index 87a7c02950b..0727f94f396 100644 --- a/lib/compiler-llvm/Cargo.toml +++ b/lib/compiler-llvm/Cargo.toml @@ -15,7 +15,7 @@ edition = "2018" wasmer-compiler = { path = "../compiler", version = "1.0.0-alpha4", features = ["translator"] } wasmer-vm = { path = "../vm", version = "1.0.0-alpha4" } wasmer-types = { path = "../wasmer-types", version = "1.0.0-alpha4" } -target-lexicon = { version = "0.10", default-features = false } +target-lexicon = { version = "0.11", default-features = false } smallvec = "1" goblin = "0.2" libc = { version = "^0.2.69", default-features = false } diff --git a/lib/compiler-llvm/src/compiler.rs b/lib/compiler-llvm/src/compiler.rs index 7c1b4f69e2e..9b344b61ead 100644 --- a/lib/compiler-llvm/src/compiler.rs +++ b/lib/compiler-llvm/src/compiler.rs @@ -124,7 +124,7 @@ impl LLVMCompiler { compile_info .module .signatures - .into_iter() + .iter() .collect::>() .par_iter() .map_init( @@ -149,7 +149,7 @@ impl LLVMCompiler { compile_info .module .functions - .into_iter() + .iter() .collect::>() .par_iter() .map_init( @@ -248,7 +248,7 @@ impl Compiler for LLVMCompiler { let mut frame_section_bytes = vec![]; let mut frame_section_relocations = vec![]; let functions = function_body_inputs - .into_iter() + .iter() .collect::)>>() .par_iter() .map_init( diff --git a/lib/compiler/Cargo.toml b/lib/compiler/Cargo.toml index e96816410ee..02e64f3dd3a 100644 --- a/lib/compiler/Cargo.toml +++ b/lib/compiler/Cargo.toml @@ -14,7 +14,7 @@ edition = "2018" wasmer-vm = { path = "../vm", version = "1.0.0-alpha4" } wasmer-types = { path = "../wasmer-types", version = "1.0.0-alpha4", default-features = false } wasmparser = { version = "0.65", optional = true, default-features = false } -target-lexicon = { version = "0.10", default-features = false } +target-lexicon = { version = "0.11", default-features = false } enumset = "1.0" hashbrown = { version = "0.8", optional = true } serde = { version = "1.0", features = ["derive"], optional = true } diff --git a/lib/engine-jit/src/link.rs b/lib/engine-jit/src/link.rs index e9b47a7e7a4..46103f2135a 100644 --- a/lib/engine-jit/src/link.rs +++ b/lib/engine-jit/src/link.rs @@ -81,7 +81,7 @@ pub fn link_module( apply_relocation(body, r, allocated_functions, jt_offsets, allocated_sections); } } - for (i, function_relocs) in function_relocations.into_iter() { + for (i, function_relocs) in function_relocations.iter() { let fatptr: *const [VMFunctionBody] = allocated_functions[i].0; let body = fatptr as *const VMFunctionBody as usize; for r in function_relocs { diff --git a/lib/engine/Cargo.toml b/lib/engine/Cargo.toml index 87c611ad5f3..f8eb02c6738 100644 --- a/lib/engine/Cargo.toml +++ b/lib/engine/Cargo.toml @@ -14,7 +14,7 @@ edition = "2018" wasmer-types = { path = "../wasmer-types", version = "1.0.0-alpha4" } wasmer-compiler = { path = "../compiler", version = "1.0.0-alpha4" } wasmer-vm = { path = "../vm", version = "1.0.0-alpha4" } -target-lexicon = { version = "0.10", default-features = false } +target-lexicon = { version = "0.11", default-features = false } # flexbuffers = { path = "../../../flatbuffers/rust/flexbuffers", version = "0.1.0" } backtrace = "0.3" rustc-demangle = "0.1" diff --git a/lib/vm/src/trap/trapcode.rs b/lib/vm/src/trap/trapcode.rs index 465bf8cec1e..4e520d65e2d 100644 --- a/lib/vm/src/trap/trapcode.rs +++ b/lib/vm/src/trap/trapcode.rs @@ -32,44 +32,47 @@ pub enum TrapCode { /// offset-guard pages. HeapAccessOutOfBounds = 2, + /// A `heap_addr` instruction was misaligned. + HeapMisaligned = 3, + /// Table Elements doesn't fit the table size. /// /// This only can happen during instantiation. - TableSetterOutOfBounds = 3, + TableSetterOutOfBounds = 4, /// A `table_addr` instruction detected an out-of-bounds error. - TableAccessOutOfBounds = 4, + TableAccessOutOfBounds = 5, /// Other bounds checking error. - OutOfBounds = 5, + OutOfBounds = 6, /// Indirect call to a null table entry. - IndirectCallToNull = 6, + IndirectCallToNull = 7, /// Signature mismatch on indirect call. - BadSignature = 7, + BadSignature = 8, /// An integer arithmetic operation caused an overflow. - IntegerOverflow = 8, + IntegerOverflow = 9, /// An integer division by zero. - IntegerDivisionByZero = 9, + IntegerDivisionByZero = 10, /// Failed float-to-int conversion. - BadConversionToInteger = 10, + BadConversionToInteger = 11, /// Code that was supposed to have been unreachable was reached. - UnreachableCodeReached = 11, + UnreachableCodeReached = 12, /// Execution has potentially run too long and may be interrupted. /// This trap is resumable. - Interrupt = 12, + Interrupt = 13, /// An atomic memory access was attempted with an unaligned pointer. - UnalignedAtomic = 13, + UnalignedAtomic = 14, /// A trap indicating that the runtime was unable to allocate sufficient memory. - VMOutOfMemory = 14, + VMOutOfMemory = 15, // /// A user-defined trap code. // User(u16), } @@ -81,6 +84,7 @@ impl TrapCode { Self::StackOverflow => "call stack exhausted", Self::HeapSetterOutOfBounds => "memory out of bounds: data segment does not fit", Self::HeapAccessOutOfBounds => "out of bounds memory access", + Self::HeapMisaligned => "misaligned heap", Self::TableSetterOutOfBounds => "table out of bounds: elements segment does not fit", Self::TableAccessOutOfBounds => "undefined element: out of bounds table access", Self::OutOfBounds => "out of bounds", @@ -104,6 +108,7 @@ impl Display for TrapCode { Self::StackOverflow => "stk_ovf", Self::HeapSetterOutOfBounds => "heap_set_oob", Self::HeapAccessOutOfBounds => "heap_get_oob", + Self::HeapMisaligned => "heap_misaligned", Self::TableSetterOutOfBounds => "table_set_oob", Self::TableAccessOutOfBounds => "table_get_oob", Self::OutOfBounds => "oob", @@ -131,6 +136,7 @@ impl FromStr for TrapCode { "stk_ovf" => Ok(StackOverflow), "heap_set_oob" => Ok(HeapSetterOutOfBounds), "heap_get_oob" => Ok(HeapAccessOutOfBounds), + "heap_misaligned" => Ok(HeapMisaligned), "table_set_oob" => Ok(TableSetterOutOfBounds), "table_get_oob" => Ok(TableAccessOutOfBounds), "oob" => Ok(OutOfBounds), @@ -158,6 +164,7 @@ mod tests { TrapCode::StackOverflow, TrapCode::HeapSetterOutOfBounds, TrapCode::HeapAccessOutOfBounds, + TrapCode::HeapMisaligned, TrapCode::TableSetterOutOfBounds, TrapCode::TableAccessOutOfBounds, TrapCode::OutOfBounds, diff --git a/lib/wasmer-types/Cargo.toml b/lib/wasmer-types/Cargo.toml index b47c8a9e60b..d203067910d 100644 --- a/lib/wasmer-types/Cargo.toml +++ b/lib/wasmer-types/Cargo.toml @@ -13,7 +13,7 @@ edition = "2018" [dependencies] # We use `cranelift-entity` here because it's a lightweight dependency and it contains # some useful data structures -cranelift-entity = "0.65" +cranelift-entity = "0.67" serde = { version = "1.0", features = ["derive"], optional = true, default-features = false } [features] diff --git a/tests/ignores.txt b/tests/ignores.txt index 6fcd597d8ed..2b3fe69d10a 100644 --- a/tests/ignores.txt +++ b/tests/ignores.txt @@ -2,17 +2,8 @@ singlepass::spec::multi_value singlepass::spec::simd -## SIMD in Cranelift 0.65 is not fully supported -cranelift::spec::simd::simd_conversions -cranelift::spec::simd::simd_f32x4 -cranelift::spec::simd::simd_f64x2 +## SIMD in Cranelift 0.67 has a small bug cranelift::spec::simd::simd_f64x2_arith -cranelift::spec::simd::simd_i16x8_arith2 -cranelift::spec::simd::simd_i32x4_arith2 -cranelift::spec::simd::simd_i8x16_arith2 -cranelift::spec::simd::simd_load -cranelift::spec::simd::simd_splat - singlepass on windows # Singlepass is not yet supported on Windows diff --git a/tests/lib/test-generator/Cargo.toml b/tests/lib/test-generator/Cargo.toml index 6dfc8fa7dfd..7ab18cdaa47 100644 --- a/tests/lib/test-generator/Cargo.toml +++ b/tests/lib/test-generator/Cargo.toml @@ -6,7 +6,7 @@ publish = false [dependencies] anyhow = "1.0" -target-lexicon = "0.10" +target-lexicon = "0.11" [features] test-native = [] From b123f92f56668e662f4fe49415b90d27a72378d2 Mon Sep 17 00:00:00 2001 From: Syrus Date: Thu, 29 Oct 2020 20:19:37 -0700 Subject: [PATCH 2/6] Simplified local_decl --- lib/compiler-llvm/src/translator/code.rs | 5 +---- lib/compiler-singlepass/src/compiler.rs | 7 ++----- lib/compiler/src/translator/middleware.rs | 2 +- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/lib/compiler-llvm/src/translator/code.rs b/lib/compiler-llvm/src/translator/code.rs index b20f3f779a2..5ffcd3c8c11 100644 --- a/lib/compiler-llvm/src/translator/code.rs +++ b/lib/compiler-llvm/src/translator/code.rs @@ -184,10 +184,7 @@ impl FuncTranslator { let mut locals = vec![]; let num_locals = reader.read_local_count().map_err(to_wasm_error)?; for _ in 0..num_locals { - let mut counter = 0; - let (count, ty) = reader - .read_local_decl(&mut counter) - .map_err(to_wasm_error)?; + let (count, ty) = reader.read_local_decl().map_err(to_wasm_error)?; let ty = wptype_to_type(ty).map_err(to_compile_error)?; let ty = type_to_llvm(&intrinsics, ty)?; for _ in 0..count { diff --git a/lib/compiler-singlepass/src/compiler.rs b/lib/compiler-singlepass/src/compiler.rs index 93170589d33..448e8f19ca4 100644 --- a/lib/compiler-singlepass/src/compiler.rs +++ b/lib/compiler-singlepass/src/compiler.rs @@ -69,7 +69,7 @@ impl Compiler for SinglepassCompiler { .into_iter() .collect(); let functions = function_body_inputs - .into_iter() + .iter() .collect::)>>() .par_iter() .map(|(i, input)| { @@ -82,10 +82,7 @@ impl Compiler for SinglepassCompiler { let mut locals = vec![]; let num_locals = reader.read_local_count().map_err(to_compile_error)?; for _ in 0..num_locals { - let mut counter = 0; - let (count, ty) = reader - .read_local_decl(&mut counter) - .map_err(to_compile_error)?; + let (count, ty) = reader.read_local_decl().map_err(to_compile_error)?; for _ in 0..count { locals.push(ty); } diff --git a/lib/compiler/src/translator/middleware.rs b/lib/compiler/src/translator/middleware.rs index 1bc329bab32..51dc1fcd3c9 100644 --- a/lib/compiler/src/translator/middleware.rs +++ b/lib/compiler/src/translator/middleware.rs @@ -99,7 +99,7 @@ impl<'a> MiddlewareBinaryReader<'a> { } /// Read a `(count, value_type)` declaration of local variables of the same type. - pub fn read_local_decl(&mut self, locals_total: &mut usize) -> WpResult<(u32, Type)> { + pub fn read_local_decl(&mut self) -> WpResult<(u32, Type)> { let count = self.state.inner.read_var_u32()?; let ty = self.state.inner.read_type()?; Ok((count, ty)) From dff27052faef9c22373ab8350cb7009be8db7cb5 Mon Sep 17 00:00:00 2001 From: Syrus Akbary Date: Thu, 29 Oct 2020 20:40:18 -0700 Subject: [PATCH 3/6] Update lib/compiler-cranelift/src/translator/code_translator.rs Co-authored-by: nlewycky --- lib/compiler-cranelift/src/translator/code_translator.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/compiler-cranelift/src/translator/code_translator.rs b/lib/compiler-cranelift/src/translator/code_translator.rs index c765a492a22..0525defd56c 100644 --- a/lib/compiler-cranelift/src/translator/code_translator.rs +++ b/lib/compiler-cranelift/src/translator/code_translator.rs @@ -49,7 +49,7 @@ use wasmer_compiler::{to_wasm_error, WasmResult}; use wasmer_compiler::{wasm_unsupported, ModuleTranslationState}; use wasmer_types::{FunctionIndex, GlobalIndex, MemoryIndex, SignatureIndex, TableIndex}; -// Clippy warns about "align: _" but its important to document that the flags field is ignored +// Clippy warns about "align: _" but its important to document that the align field is ignored #[cfg_attr( feature = "cargo-clippy", allow(clippy::unneeded_field_pattern, clippy::cognitive_complexity) From 9dcc409720e0a70016b22acdae9f2cbcafa2f217 Mon Sep 17 00:00:00 2001 From: Syrus Date: Thu, 29 Oct 2020 20:42:41 -0700 Subject: [PATCH 4/6] Use matches! macro for legibility --- lib/compiler-cranelift/src/translator/code_translator.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/compiler-cranelift/src/translator/code_translator.rs b/lib/compiler-cranelift/src/translator/code_translator.rs index 0525defd56c..cbe155d8d66 100644 --- a/lib/compiler-cranelift/src/translator/code_translator.rs +++ b/lib/compiler-cranelift/src/translator/code_translator.rs @@ -2573,10 +2573,7 @@ fn optionally_bitcast_vector( #[inline(always)] fn is_non_canonical_v128(ty: ir::Type) -> bool { - match ty { - B8X16 | B16X8 | B32X4 | B64X2 | I64X2 | I32X4 | I16X8 | F32X4 | F64X2 => true, - _ => false, - } + matches!(ty, B8X16 | B16X8 | B32X4 | B64X2 | I64X2 | I32X4 | I16X8 | F32X4 | F64X2) } /// Cast to I8X16, any vector values in `values` that are of "non-canonical" type (meaning, not From eef543a1bceb5834f01c1c9d8b2f5ef8ea05164e Mon Sep 17 00:00:00 2001 From: Syrus Date: Thu, 29 Oct 2020 20:45:06 -0700 Subject: [PATCH 5/6] Fixed code linting --- lib/compiler-cranelift/src/translator/code_translator.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/compiler-cranelift/src/translator/code_translator.rs b/lib/compiler-cranelift/src/translator/code_translator.rs index cbe155d8d66..b52d34e0b01 100644 --- a/lib/compiler-cranelift/src/translator/code_translator.rs +++ b/lib/compiler-cranelift/src/translator/code_translator.rs @@ -2573,7 +2573,10 @@ fn optionally_bitcast_vector( #[inline(always)] fn is_non_canonical_v128(ty: ir::Type) -> bool { - matches!(ty, B8X16 | B16X8 | B32X4 | B64X2 | I64X2 | I32X4 | I16X8 | F32X4 | F64X2) + matches!( + ty, + B8X16 | B16X8 | B32X4 | B64X2 | I64X2 | I32X4 | I16X8 | F32X4 | F64X2 + ) } /// Cast to I8X16, any vector values in `values` that are of "non-canonical" type (meaning, not From e6d8ee51eb50ce6ed26ddbb0a788259e7e507871 Mon Sep 17 00:00:00 2001 From: Syrus Date: Thu, 29 Oct 2020 21:46:51 -0700 Subject: [PATCH 6/6] Fixed tests --- lib/vm/src/trap/trapcode.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/vm/src/trap/trapcode.rs b/lib/vm/src/trap/trapcode.rs index 4e520d65e2d..32121fbd08e 100644 --- a/lib/vm/src/trap/trapcode.rs +++ b/lib/vm/src/trap/trapcode.rs @@ -160,7 +160,7 @@ mod tests { use super::*; // Everything but user-defined codes. - const CODES: [TrapCode; 14] = [ + const CODES: [TrapCode; 15] = [ TrapCode::StackOverflow, TrapCode::HeapSetterOutOfBounds, TrapCode::HeapAccessOutOfBounds,