From 98f35ef84ad7f0e88e6b928e9987b21c8255f663 Mon Sep 17 00:00:00 2001 From: Nick Lewycky Date: Wed, 28 Aug 2019 17:23:26 -0700 Subject: [PATCH 01/13] Initial implementation of atomic load/store and i32 atomic rmw add. --- lib/singlepass-backend/src/codegen_x64.rs | 573 ++++++++++++++++++++-- lib/singlepass-backend/src/emitter_x64.rs | 80 ++- 2 files changed, 612 insertions(+), 41 deletions(-) diff --git a/lib/singlepass-backend/src/codegen_x64.rs b/lib/singlepass-backend/src/codegen_x64.rs index 1bfa5602e84..5c1f90b3bdc 100644 --- a/lib/singlepass-backend/src/codegen_x64.rs +++ b/lib/singlepass-backend/src/codegen_x64.rs @@ -34,7 +34,7 @@ use wasmer_runtime_core::{ }, vm::{self, LocalGlobal, LocalTable, INTERNALS_SIZE}, }; -use wasmparser::{Operator, Type as WpType, TypeOrFuncType as WpTypeOrFuncType}; +use wasmparser::{MemoryImmediate, Operator, Type as WpType, TypeOrFuncType as WpTypeOrFuncType}; lazy_static! { /// Performs a System V call to `target` with [stack_top..stack_base] as the argument list, from right to left. @@ -1465,7 +1465,8 @@ impl X64FunctionCode { a: &mut Assembler, m: &mut Machine, addr: Location, - offset: usize, + memarg: &MemoryImmediate, + check_alignment: bool, value_size: usize, cb: F, ) { @@ -1487,7 +1488,6 @@ impl X64FunctionCode { let tmp_addr = m.acquire_temp_gpr().unwrap(); let tmp_base = m.acquire_temp_gpr().unwrap(); - let tmp_bound = m.acquire_temp_gpr().unwrap(); // Load base into temporary register. a.emit_mov( @@ -1500,6 +1500,8 @@ impl X64FunctionCode { ); if need_check { + let tmp_bound = m.acquire_temp_gpr().unwrap(); + a.emit_mov( Size::S64, Location::Memory( @@ -1513,7 +1515,7 @@ impl X64FunctionCode { a.emit_mov(Size::S32, addr, Location::GPR(tmp_addr)); // This branch is used for emitting "faster" code for the special case of (offset + value_size) not exceeding u32 range. - match (offset as u32).checked_add(value_size as u32) { + match (memarg.offset as u32).checked_add(value_size as u32) { Some(0) => {} Some(x) => { a.emit_add(Size::S64, Location::Imm32(x), Location::GPR(tmp_addr)); @@ -1521,7 +1523,7 @@ impl X64FunctionCode { None => { a.emit_add( Size::S64, - Location::Imm32(offset as u32), + Location::Imm32(memarg.offset as u32), Location::GPR(tmp_addr), ); a.emit_add( @@ -1536,22 +1538,41 @@ impl X64FunctionCode { a.emit_add(Size::S64, Location::GPR(tmp_base), Location::GPR(tmp_addr)); a.emit_cmp(Size::S64, Location::GPR(tmp_bound), Location::GPR(tmp_addr)); a.emit_conditional_trap(Condition::Above); - } - m.release_temp_gpr(tmp_bound); + m.release_temp_gpr(tmp_bound); + } // Calculates the real address, and loads from it. a.emit_mov(Size::S32, addr, Location::GPR(tmp_addr)); - if offset != 0 { + if memarg.offset != 0 { a.emit_add( Size::S64, - Location::Imm32(offset as u32), + Location::Imm32(memarg.offset as u32), Location::GPR(tmp_addr), ); } a.emit_add(Size::S64, Location::GPR(tmp_base), Location::GPR(tmp_addr)); m.release_temp_gpr(tmp_base); + let align = match memarg.flags & 3 { + 0 => 1, + 1 => 2, + 2 => 4, + 3 => 8, + _ => unreachable!("this match is fully covered"), + }; + if check_alignment && align != 1 { + let tmp_aligncheck = m.acquire_temp_gpr().unwrap(); + //let tmp_mask = m.acquire_temp_gpr().unwrap(); + a.emit_mov(Size::S32, Location::GPR(tmp_addr), Location::GPR(tmp_aligncheck)); + //a.emit_mov(Size::S64, Location::Imm64(align - 1), Location::GPR(tmp_mask)); + //a.emit_and(Size::S64, Location::GPR(tmp_mask), Location::GPR(tmp_aligncheck)); + a.emit_and(Size::S64, Location::Imm32(align - 1), Location::GPR(tmp_aligncheck)); + a.emit_conditional_trap(Condition::NotEqual); + //m.release_temp_gpr(tmp_mask); + m.release_temp_gpr(tmp_aligncheck); + } + cb(a, m, tmp_addr); m.release_temp_gpr(tmp_addr); @@ -4147,7 +4168,8 @@ impl FunctionCodeGenerator for X64FunctionCode { a, &mut self.machine, target, - memarg.offset as usize, + memarg, + false, 4, |a, m, addr| { Self::emit_relaxed_binop( @@ -4177,7 +4199,8 @@ impl FunctionCodeGenerator for X64FunctionCode { a, &mut self.machine, target, - memarg.offset as usize, + memarg, + false, 4, |a, m, addr| { Self::emit_relaxed_binop( @@ -4207,7 +4230,8 @@ impl FunctionCodeGenerator for X64FunctionCode { a, &mut self.machine, target, - memarg.offset as usize, + memarg, + false, 1, |a, m, addr| { Self::emit_relaxed_zx_sx( @@ -4238,7 +4262,8 @@ impl FunctionCodeGenerator for X64FunctionCode { a, &mut self.machine, target, - memarg.offset as usize, + memarg, + false, 1, |a, m, addr| { Self::emit_relaxed_zx_sx( @@ -4269,7 +4294,8 @@ impl FunctionCodeGenerator for X64FunctionCode { a, &mut self.machine, target, - memarg.offset as usize, + memarg, + false, 2, |a, m, addr| { Self::emit_relaxed_zx_sx( @@ -4300,7 +4326,8 @@ impl FunctionCodeGenerator for X64FunctionCode { a, &mut self.machine, target, - memarg.offset as usize, + memarg, + false, 2, |a, m, addr| { Self::emit_relaxed_zx_sx( @@ -4327,7 +4354,8 @@ impl FunctionCodeGenerator for X64FunctionCode { a, &mut self.machine, target_addr, - memarg.offset as usize, + memarg, + false, 4, |a, m, addr| { Self::emit_relaxed_binop( @@ -4353,7 +4381,8 @@ impl FunctionCodeGenerator for X64FunctionCode { a, &mut self.machine, target_addr, - memarg.offset as usize, + memarg, + false, 4, |a, m, addr| { Self::emit_relaxed_binop( @@ -4379,7 +4408,8 @@ impl FunctionCodeGenerator for X64FunctionCode { a, &mut self.machine, target_addr, - memarg.offset as usize, + memarg, + false, 1, |a, m, addr| { Self::emit_relaxed_binop( @@ -4405,7 +4435,8 @@ impl FunctionCodeGenerator for X64FunctionCode { a, &mut self.machine, target_addr, - memarg.offset as usize, + memarg, + false, 2, |a, m, addr| { Self::emit_relaxed_binop( @@ -4435,7 +4466,8 @@ impl FunctionCodeGenerator for X64FunctionCode { a, &mut self.machine, target, - memarg.offset as usize, + memarg, + false, 8, |a, m, addr| { Self::emit_relaxed_binop( @@ -4465,7 +4497,8 @@ impl FunctionCodeGenerator for X64FunctionCode { a, &mut self.machine, target, - memarg.offset as usize, + memarg, + false, 8, |a, m, addr| { Self::emit_relaxed_binop( @@ -4495,7 +4528,8 @@ impl FunctionCodeGenerator for X64FunctionCode { a, &mut self.machine, target, - memarg.offset as usize, + memarg, + false, 1, |a, m, addr| { Self::emit_relaxed_zx_sx( @@ -4526,7 +4560,8 @@ impl FunctionCodeGenerator for X64FunctionCode { a, &mut self.machine, target, - memarg.offset as usize, + memarg, + false, 1, |a, m, addr| { Self::emit_relaxed_zx_sx( @@ -4557,7 +4592,8 @@ impl FunctionCodeGenerator for X64FunctionCode { a, &mut self.machine, target, - memarg.offset as usize, + memarg, + false, 2, |a, m, addr| { Self::emit_relaxed_zx_sx( @@ -4588,7 +4624,8 @@ impl FunctionCodeGenerator for X64FunctionCode { a, &mut self.machine, target, - memarg.offset as usize, + memarg, + false, 2, |a, m, addr| { Self::emit_relaxed_zx_sx( @@ -4619,7 +4656,8 @@ impl FunctionCodeGenerator for X64FunctionCode { a, &mut self.machine, target, - memarg.offset as usize, + memarg, + false, 4, |a, m, addr| { match ret { @@ -4660,7 +4698,8 @@ impl FunctionCodeGenerator for X64FunctionCode { a, &mut self.machine, target, - memarg.offset as usize, + memarg, + false, 4, |a, m, addr| { Self::emit_relaxed_zx_sx( @@ -4687,7 +4726,8 @@ impl FunctionCodeGenerator for X64FunctionCode { a, &mut self.machine, target_addr, - memarg.offset as usize, + memarg, + false, 8, |a, m, addr| { Self::emit_relaxed_binop( @@ -4713,7 +4753,8 @@ impl FunctionCodeGenerator for X64FunctionCode { a, &mut self.machine, target_addr, - memarg.offset as usize, + memarg, + false, 8, |a, m, addr| { Self::emit_relaxed_binop( @@ -4739,7 +4780,8 @@ impl FunctionCodeGenerator for X64FunctionCode { a, &mut self.machine, target_addr, - memarg.offset as usize, + memarg, + false, 1, |a, m, addr| { Self::emit_relaxed_binop( @@ -4765,7 +4807,8 @@ impl FunctionCodeGenerator for X64FunctionCode { a, &mut self.machine, target_addr, - memarg.offset as usize, + memarg, + false, 2, |a, m, addr| { Self::emit_relaxed_binop( @@ -4791,7 +4834,8 @@ impl FunctionCodeGenerator for X64FunctionCode { a, &mut self.machine, target_addr, - memarg.offset as usize, + memarg, + false, 4, |a, m, addr| { Self::emit_relaxed_binop( @@ -4980,6 +5024,471 @@ impl FunctionCodeGenerator for X64FunctionCode { } } } + Operator::Fence { flags: _ } => { + // Fence is a nop. + // + // Fence was added to preserve information about fences from + // source languages. If in the future Wasm extends the memory + // model, and if we hadn't recorded what fences used to be there, + // it would lead to data races that weren't present in the + // original source language. + } + Operator::I32AtomicLoad { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + target, + memarg, + true, + 4, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_mov, + Size::S32, + Location::Memory(addr, 0), + ret, + ); + }, + ); + } + Operator::I32AtomicLoad8U { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + target, + memarg, + true, + 1, + |a, m, addr| { + Self::emit_relaxed_zx_sx( + a, + m, + Assembler::emit_movzx, + Size::S8, + Location::Memory(addr, 0), + Size::S32, + ret, + ); + }, + ); + } + Operator::I32AtomicLoad16U { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + target, + memarg, + true, + 2, + |a, m, addr| { + Self::emit_relaxed_zx_sx( + a, + m, + Assembler::emit_movzx, + Size::S16, + Location::Memory(addr, 0), + Size::S32, + ret, + ); + }, + ); + } + Operator::I32AtomicStore { ref memarg } => { + let target_value = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target_addr = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + target_addr, + memarg, + true, + 4, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_xchg, + Size::S32, + target_value, + Location::Memory(addr, 0), + ); + }, + ); + } + Operator::I32AtomicStore8 { ref memarg } => { + let target_value = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target_addr = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + target_addr, + memarg, + true, + 1, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_xchg, + Size::S8, + target_value, + Location::Memory(addr, 0), + ); + }, + ); + } + Operator::I32AtomicStore16 { ref memarg } => { + let target_value = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target_addr = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + target_addr, + memarg, + true, + 2, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_xchg, + Size::S16, + target_value, + Location::Memory(addr, 0), + ); + }, + ); + } + Operator::I64AtomicLoad { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + target, + memarg, + true, + 8, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_mov, + Size::S64, + Location::Memory(addr, 0), + ret, + ); + }, + ); + } + Operator::I64AtomicLoad8U { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + target, + memarg, + true, + 1, + |a, m, addr| { + Self::emit_relaxed_zx_sx( + a, + m, + Assembler::emit_movzx, + Size::S8, + Location::Memory(addr, 0), + Size::S64, + ret, + ); + }, + ); + } + Operator::I64AtomicLoad16U { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + target, + memarg, + true, + 2, + |a, m, addr| { + Self::emit_relaxed_zx_sx( + a, + m, + Assembler::emit_movzx, + Size::S16, + Location::Memory(addr, 0), + Size::S64, + ret, + ); + }, + ); + } + Operator::I64AtomicLoad32U { ref memarg } => { + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + target, + memarg, + true, + 4, + |a, m, addr| { + match ret { + Location::GPR(_) => {} + Location::Memory(base, offset) => { + a.emit_mov( + Size::S32, + Location::Imm32(0), + Location::Memory(base, offset + 4), + ); // clear upper bits + } + _ => unreachable!(), + } + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_mov, + Size::S32, + Location::Memory(addr, 0), + ret, + ); + }, + ); + } + Operator::I64AtomicStore { ref memarg } => { + let target_value = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target_addr = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + target_addr, + memarg, + true, + 8, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_xchg, + Size::S64, + target_value, + Location::Memory(addr, 0), + ); + }, + ); + } + Operator::I64AtomicStore8 { ref memarg } => { + let target_value = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target_addr = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + target_addr, + memarg, + true, + 1, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_xchg, + Size::S8, + target_value, + Location::Memory(addr, 0), + ); + }, + ); + } + Operator::I64AtomicStore16 { ref memarg } => { + let target_value = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target_addr = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + target_addr, + memarg, + true, + 2, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_xchg, + Size::S16, + target_value, + Location::Memory(addr, 0), + ); + }, + ); + } + Operator::I64AtomicStore32 { ref memarg } => { + let target_value = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target_addr = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + target_addr, + memarg, + true, + 4, + |a, m, addr| { + Self::emit_relaxed_binop( + a, + m, + Assembler::emit_xchg, + Size::S32, + target_value, + Location::Memory(addr, 0), + ); + }, + ); + } + Operator::I32AtomicRmwAdd { ref memarg } => { + let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_mov( + Size::S32, + loc, + Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + target, + memarg, + true, + 4, + |a, _m, addr| { + a.emit_lock_xadd(Size::S32, Location::GPR(value), Location::Memory(addr, 0)) + } + ); + a.emit_mov( + Size::S32, + Location::GPR(value), + ret); + self.machine.release_temp_gpr(value); + } _ => { return Err(CodegenError { message: format!("not yet implemented: {:?}", op), diff --git a/lib/singlepass-backend/src/emitter_x64.rs b/lib/singlepass-backend/src/emitter_x64.rs index fa2ad485308..9344197ffd7 100644 --- a/lib/singlepass-backend/src/emitter_x64.rs +++ b/lib/singlepass-backend/src/emitter_x64.rs @@ -94,6 +94,8 @@ pub trait Emitter { fn emit_popcnt(&mut self, sz: Size, src: Location, dst: Location); fn emit_movzx(&mut self, sz_src: Size, src: Location, sz_dst: Size, dst: Location); fn emit_movsx(&mut self, sz_src: Size, src: Location, sz_dst: Size, dst: Location); + fn emit_xchg(&mut self, sz: Size, src: Location, dst: Location); + fn emit_lock_xadd(&mut self, sz: Size, src: Location, dst: Location); fn emit_btc_gpr_imm8_32(&mut self, src: u8, dst: GPR); fn emit_btc_gpr_imm8_64(&mut self, src: u8, dst: GPR); @@ -562,7 +564,7 @@ impl Emitter for Assembler { (Size::S64, Location::Memory(src, disp), Location::GPR(dst)) => { dynasm!(self ; lea Rq(dst as u8), [Rq(src as u8) + disp]); } - _ => unreachable!(), + _ => panic!("LEA {:?} {:?} {:?}", sz, src, dst), } } fn emit_lea_label(&mut self, label: Self::Label, dst: Location) { @@ -570,7 +572,7 @@ impl Emitter for Assembler { Location::GPR(x) => { dynasm!(self ; lea Rq(x as u8), [=>label]); } - _ => unreachable!(), + _ => panic!("LEA label={:?} {:?}", label, dst), } } fn emit_cdq(&mut self) { @@ -602,7 +604,7 @@ impl Emitter for Assembler { match loc { Location::GPR(x) => dynasm!(self ; jmp Rq(x as u8)), Location::Memory(base, disp) => dynasm!(self ; jmp QWORD [Rq(base as u8) + disp]), - _ => unreachable!(), + _ => panic!("JMP {:?}", loc), } } fn emit_conditional_trap(&mut self, condition: Condition) { @@ -634,7 +636,7 @@ impl Emitter for Assembler { Condition::Equal => dynasm!(self ; sete Rb(dst as u8)), Condition::NotEqual => dynasm!(self ; setne Rb(dst as u8)), Condition::Signed => dynasm!(self ; sets Rb(dst as u8)), - _ => unreachable!(), + _ => panic!("SET {:?} {:?}", condition, dst), } } fn emit_push(&mut self, sz: Size, src: Location) { @@ -644,7 +646,7 @@ impl Emitter for Assembler { (Size::S64, Location::Memory(src, disp)) => { dynasm!(self ; push QWORD [Rq(src as u8) + disp]) } - _ => panic!("push {:?} {:?}", sz, src), + _ => panic!("PUSH {:?} {:?}", sz, src), } } fn emit_pop(&mut self, sz: Size, dst: Location) { @@ -653,12 +655,12 @@ impl Emitter for Assembler { (Size::S64, Location::Memory(dst, disp)) => { dynasm!(self ; pop QWORD [Rq(dst as u8) + disp]) } - _ => panic!("pop {:?} {:?}", sz, dst), + _ => panic!("POP {:?} {:?}", sz, dst), } } fn emit_cmp(&mut self, sz: Size, left: Location, right: Location) { binop_all_nofp!(cmp, self, sz, left, right, { - panic!("{:?} {:?} {:?}", sz, left, right); + panic!("CMP {:?} {:?} {:?}", sz, left, right); }); } fn emit_add(&mut self, sz: Size, src: Location, dst: Location) { @@ -743,7 +745,7 @@ impl Emitter for Assembler { (Size::S16, Location::Memory(src, disp), Size::S64, Location::GPR(dst)) => { dynasm!(self ; movzx Rq(dst as u8), WORD [Rq(src as u8) + disp]); } - _ => unreachable!(), + _ => panic!("MOVZX {:?} {:?} {:?} {:?}", sz_src, src, sz_dst, dst), } } fn emit_movsx(&mut self, sz_src: Size, src: Location, sz_dst: Size, dst: Location) { @@ -778,7 +780,67 @@ impl Emitter for Assembler { (Size::S32, Location::Memory(src, disp), Size::S64, Location::GPR(dst)) => { dynasm!(self ; movsx Rq(dst as u8), DWORD [Rq(src as u8) + disp]); } - _ => unreachable!(), + _ => panic!("MOVSX {:?} {:?} {:?} {:?}", sz_src, src, sz_dst, dst), + } + } + + fn emit_xchg(&mut self, sz: Size, src: Location, dst: Location) { + match (sz, src, dst) { + (Size::S8, Location::GPR(src), Location::GPR(dst)) => { + dynasm!(self ; xchg Rb(dst as u8), Rb(src as u8)); + } + (Size::S16, Location::GPR(src), Location::GPR(dst)) => { + dynasm!(self ; xchg Rw(dst as u8), Rw(src as u8)); + } + (Size::S32, Location::GPR(src), Location::GPR(dst)) => { + dynasm!(self ; xchg Rd(dst as u8), Rd(src as u8)); + } + (Size::S64, Location::GPR(src), Location::GPR(dst)) => { + dynasm!(self ; xchg Rq(dst as u8), Rq(src as u8)); + } + (Size::S8, Location::Memory(src, disp), Location::GPR(dst)) => { + dynasm!(self ; xchg Rb(dst as u8), [Rq(src as u8) + disp]); + } + (Size::S8, Location::GPR(src), Location::Memory(dst, disp)) => { + dynasm!(self ; xchg [Rq(dst as u8) + disp], Rb(src as u8)); + } + (Size::S16, Location::Memory(src, disp), Location::GPR(dst)) => { + dynasm!(self ; xchg Rw(dst as u8), [Rq(src as u8) + disp]); + } + (Size::S16, Location::GPR(src), Location::Memory(dst, disp)) => { + dynasm!(self ; xchg [Rq(dst as u8) + disp], Rw(src as u8)); + } + (Size::S32, Location::Memory(src, disp), Location::GPR(dst)) => { + dynasm!(self ; xchg Rd(dst as u8), [Rq(src as u8) + disp]); + } + (Size::S32, Location::GPR(src), Location::Memory(dst, disp)) => { + dynasm!(self ; xchg [Rq(dst as u8) + disp], Rd(src as u8)); + } + (Size::S64, Location::Memory(src, disp), Location::GPR(dst)) => { + dynasm!(self ; xchg Rq(dst as u8), [Rq(src as u8) + disp]); + } + (Size::S64, Location::GPR(src), Location::Memory(dst, disp)) => { + dynasm!(self ; xchg [Rq(dst as u8) + disp], Rq(src as u8)); + } + _ => panic!("XCHG {:?} {:?} {:?}", sz, src, dst), + } + } + + fn emit_lock_xadd(&mut self, sz: Size, src: Location, dst: Location) { + match (sz, src, dst) { + (Size::S8, Location::GPR(src), Location::Memory(dst, disp)) => { + dynasm!(self ; lock xadd [Rq(dst as u8) + disp], Rb(src as u8)); + } + (Size::S16, Location::GPR(src), Location::Memory(dst, disp)) => { + dynasm!(self ; lock xadd [Rq(dst as u8) + disp], Rw(src as u8)); + } + (Size::S32, Location::GPR(src), Location::Memory(dst, disp)) => { + dynasm!(self ; lock xadd [Rq(dst as u8) + disp], Rd(src as u8)); + } + (Size::S64, Location::GPR(src), Location::Memory(dst, disp)) => { + dynasm!(self ; lock xadd [Rq(dst as u8) + disp], Rq(src as u8)); + } + _ => panic!("LOCK XADD {:?} {:?} {:?}", sz, src, dst), } } From efc89e829d7b36d3d59c002422215149d10ae998 Mon Sep 17 00:00:00 2001 From: Nick Lewycky Date: Fri, 30 Aug 2019 18:00:10 -0700 Subject: [PATCH 02/13] Add i32 rmw add and sub. --- lib/singlepass-backend/src/codegen_x64.rs | 182 ++++++++++++++++++++++ lib/singlepass-backend/src/emitter_x64.rs | 14 ++ 2 files changed, 196 insertions(+) diff --git a/lib/singlepass-backend/src/codegen_x64.rs b/lib/singlepass-backend/src/codegen_x64.rs index 5c1f90b3bdc..bf40f6eb516 100644 --- a/lib/singlepass-backend/src/codegen_x64.rs +++ b/lib/singlepass-backend/src/codegen_x64.rs @@ -5489,6 +5489,188 @@ impl FunctionCodeGenerator for X64FunctionCode { ret); self.machine.release_temp_gpr(value); } + Operator::I32AtomicRmw8UAdd { ref memarg } => { + let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_movzx( + Size::S8, + loc, + Size::S32, + Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + target, + memarg, + true, + 4, + |a, _m, addr| { + a.emit_lock_xadd(Size::S8, Location::GPR(value), Location::Memory(addr, 0)) + } + ); + a.emit_mov( + Size::S32, + Location::GPR(value), + ret); + self.machine.release_temp_gpr(value); + } + Operator::I32AtomicRmw16UAdd { ref memarg } => { + let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_movzx( + Size::S16, + loc, + Size::S32, + Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + target, + memarg, + true, + 4, + |a, _m, addr| { + a.emit_lock_xadd(Size::S16, Location::GPR(value), Location::Memory(addr, 0)) + } + ); + a.emit_mov( + Size::S32, + Location::GPR(value), + ret); + self.machine.release_temp_gpr(value); + } + Operator::I32AtomicRmwSub { ref memarg } => { + let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_mov( + Size::S32, + loc, + Location::GPR(value)); + a.emit_neg(Size::S32, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + target, + memarg, + true, + 4, + |a, _m, addr| { + a.emit_lock_xadd(Size::S32, Location::GPR(value), Location::Memory(addr, 0)) + } + ); + a.emit_mov( + Size::S32, + Location::GPR(value), + ret); + self.machine.release_temp_gpr(value); + } + Operator::I32AtomicRmw8USub { ref memarg } => { + let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_movzx( + Size::S8, + loc, + Size::S32, + Location::GPR(value)); + a.emit_neg(Size::S8, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + target, + memarg, + true, + 4, + |a, _m, addr| { + a.emit_lock_xadd(Size::S8, Location::GPR(value), Location::Memory(addr, 0)) + } + ); + a.emit_mov( + Size::S32, + Location::GPR(value), + ret); + self.machine.release_temp_gpr(value); + } + Operator::I32AtomicRmw16USub { ref memarg } => { + let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_movzx( + Size::S16, + loc, + Size::S32, + Location::GPR(value)); + a.emit_neg(Size::S16, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + target, + memarg, + true, + 4, + |a, _m, addr| { + a.emit_lock_xadd(Size::S16, Location::GPR(value), Location::Memory(addr, 0)) + } + ); + a.emit_mov( + Size::S32, + Location::GPR(value), + ret); + self.machine.release_temp_gpr(value); + } _ => { return Err(CodegenError { message: format!("not yet implemented: {:?}", op), diff --git a/lib/singlepass-backend/src/emitter_x64.rs b/lib/singlepass-backend/src/emitter_x64.rs index 9344197ffd7..4bea37d3e2a 100644 --- a/lib/singlepass-backend/src/emitter_x64.rs +++ b/lib/singlepass-backend/src/emitter_x64.rs @@ -78,6 +78,7 @@ pub trait Emitter { fn emit_cmp(&mut self, sz: Size, left: Location, right: Location); fn emit_add(&mut self, sz: Size, src: Location, dst: Location); fn emit_sub(&mut self, sz: Size, src: Location, dst: Location); + fn emit_neg(&mut self, sz: Size, value: Location); fn emit_imul(&mut self, sz: Size, src: Location, dst: Location); fn emit_imul_imm32_gpr64(&mut self, src: u32, dst: GPR); fn emit_div(&mut self, sz: Size, divisor: Location); @@ -669,6 +670,19 @@ impl Emitter for Assembler { fn emit_sub(&mut self, sz: Size, src: Location, dst: Location) { binop_all_nofp!(sub, self, sz, src, dst, { unreachable!() }); } + fn emit_neg(&mut self, sz: Size, value: Location) { + match (sz, value) { + (Size::S8, Location::GPR(value)) => { dynasm!(self ; neg Rb(value as u8)) } + (Size::S8, Location::Memory(value, disp)) => { dynasm!(self ; neg [Rq(value as u8) + disp]) } + (Size::S16, Location::GPR(value)) => { dynasm!(self ; neg Rw(value as u8)) } + (Size::S16, Location::Memory(value, disp)) => { dynasm!(self ; neg [Rq(value as u8) + disp]) } + (Size::S32, Location::GPR(value)) => { dynasm!(self ; neg Rd(value as u8)) } + (Size::S32, Location::Memory(value, disp)) => { dynasm!(self ; neg [Rq(value as u8) + disp]) } + (Size::S64, Location::GPR(value)) => { dynasm!(self ; neg Rq(value as u8)) } + (Size::S64, Location::Memory(value, disp)) => { dynasm!(self ; neg [Rq(value as u8) + disp]) } + _ => panic!("NEG {:?} {:?}", sz, value), + } + } fn emit_imul(&mut self, sz: Size, src: Location, dst: Location) { binop_gpr_gpr!(imul, self, sz, src, dst, { binop_mem_gpr!(imul, self, sz, src, dst, { unreachable!() }) From 81895830f0141f89a7d5fadef72a9965358df895 Mon Sep 17 00:00:00 2001 From: Nick Lewycky Date: Wed, 18 Sep 2019 11:58:21 -0700 Subject: [PATCH 03/13] Add emitter for LOCK CMPXCHG so that we can emit compare-and-swap loops. --- lib/singlepass-backend/src/emitter_x64.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/lib/singlepass-backend/src/emitter_x64.rs b/lib/singlepass-backend/src/emitter_x64.rs index 4bea37d3e2a..83099f25970 100644 --- a/lib/singlepass-backend/src/emitter_x64.rs +++ b/lib/singlepass-backend/src/emitter_x64.rs @@ -97,6 +97,7 @@ pub trait Emitter { fn emit_movsx(&mut self, sz_src: Size, src: Location, sz_dst: Size, dst: Location); fn emit_xchg(&mut self, sz: Size, src: Location, dst: Location); fn emit_lock_xadd(&mut self, sz: Size, src: Location, dst: Location); + fn emit_lock_cmpxchg(&mut self, sz: Size, src: Location, dst: Location); fn emit_btc_gpr_imm8_32(&mut self, src: u8, dst: GPR); fn emit_btc_gpr_imm8_64(&mut self, src: u8, dst: GPR); @@ -858,6 +859,24 @@ impl Emitter for Assembler { } } + fn emit_lock_cmpxchg(&mut self, sz: Size, src: Location, dst: Location) { + match (sz, src, dst) { + (Size::S8, Location::GPR(src), Location::Memory(dst, disp)) => { + dynasm!(self ; lock cmpxchg [Rq(dst as u8) + disp], Rb(src as u8)); + } + (Size::S16, Location::GPR(src), Location::Memory(dst, disp)) => { + dynasm!(self ; lock cmpxchg [Rq(dst as u8) + disp], Rw(src as u8)); + } + (Size::S32, Location::GPR(src), Location::Memory(dst, disp)) => { + dynasm!(self ; lock cmpxchg [Rq(dst as u8) + disp], Rd(src as u8)); + } + (Size::S64, Location::GPR(src), Location::Memory(dst, disp)) => { + dynasm!(self ; lock cmpxchg [Rq(dst as u8) + disp], Rq(src as u8)); + } + _ => panic!("LOCK CMPXCHG {:?} {:?} {:?}", sz, src, dst), + } + } + fn emit_btc_gpr_imm8_32(&mut self, src: u8, dst: GPR) { dynasm!(self ; btc Rd(dst as u8), BYTE src as i8); } From 6937019b6525a103aedfa4accd8696fb313ea9b9 Mon Sep 17 00:00:00 2001 From: Nick Lewycky Date: Fri, 20 Sep 2019 23:39:15 -0700 Subject: [PATCH 04/13] Use a compare-and-swap loop for AND. BUG: This might allocate RAX twice. --- lib/singlepass-backend/src/codegen_x64.rs | 157 +++++++++++++--------- 1 file changed, 96 insertions(+), 61 deletions(-) diff --git a/lib/singlepass-backend/src/codegen_x64.rs b/lib/singlepass-backend/src/codegen_x64.rs index bf40f6eb516..71b3358fd03 100644 --- a/lib/singlepass-backend/src/codegen_x64.rs +++ b/lib/singlepass-backend/src/codegen_x64.rs @@ -1564,10 +1564,18 @@ impl X64FunctionCode { if check_alignment && align != 1 { let tmp_aligncheck = m.acquire_temp_gpr().unwrap(); //let tmp_mask = m.acquire_temp_gpr().unwrap(); - a.emit_mov(Size::S32, Location::GPR(tmp_addr), Location::GPR(tmp_aligncheck)); + a.emit_mov( + Size::S32, + Location::GPR(tmp_addr), + Location::GPR(tmp_aligncheck), + ); //a.emit_mov(Size::S64, Location::Imm64(align - 1), Location::GPR(tmp_mask)); //a.emit_and(Size::S64, Location::GPR(tmp_mask), Location::GPR(tmp_aligncheck)); - a.emit_and(Size::S64, Location::Imm32(align - 1), Location::GPR(tmp_aligncheck)); + a.emit_and( + Size::S64, + Location::Imm32(align - 1), + Location::GPR(tmp_aligncheck), + ); a.emit_conditional_trap(Condition::NotEqual); //m.release_temp_gpr(tmp_mask); m.release_temp_gpr(tmp_aligncheck); @@ -5455,7 +5463,8 @@ impl FunctionCodeGenerator for X64FunctionCode { ); } Operator::I32AtomicRmwAdd { ref memarg } => { - let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); let target = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); let ret = self.machine.acquire_locations( @@ -5466,10 +5475,7 @@ impl FunctionCodeGenerator for X64FunctionCode { self.value_stack.push(ret); let value = self.machine.acquire_temp_gpr().unwrap(); - a.emit_mov( - Size::S32, - loc, - Location::GPR(value)); + a.emit_mov(Size::S32, loc, Location::GPR(value)); Self::emit_memory_op( module_info, &self.config, @@ -5481,16 +5487,14 @@ impl FunctionCodeGenerator for X64FunctionCode { 4, |a, _m, addr| { a.emit_lock_xadd(Size::S32, Location::GPR(value), Location::Memory(addr, 0)) - } + }, ); - a.emit_mov( - Size::S32, - Location::GPR(value), - ret); + a.emit_mov(Size::S32, Location::GPR(value), ret); self.machine.release_temp_gpr(value); } Operator::I32AtomicRmw8UAdd { ref memarg } => { - let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); let target = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); let ret = self.machine.acquire_locations( @@ -5501,11 +5505,7 @@ impl FunctionCodeGenerator for X64FunctionCode { self.value_stack.push(ret); let value = self.machine.acquire_temp_gpr().unwrap(); - a.emit_movzx( - Size::S8, - loc, - Size::S32, - Location::GPR(value)); + a.emit_movzx(Size::S8, loc, Size::S32, Location::GPR(value)); Self::emit_memory_op( module_info, &self.config, @@ -5517,16 +5517,14 @@ impl FunctionCodeGenerator for X64FunctionCode { 4, |a, _m, addr| { a.emit_lock_xadd(Size::S8, Location::GPR(value), Location::Memory(addr, 0)) - } + }, ); - a.emit_mov( - Size::S32, - Location::GPR(value), - ret); + a.emit_mov(Size::S32, Location::GPR(value), ret); self.machine.release_temp_gpr(value); } Operator::I32AtomicRmw16UAdd { ref memarg } => { - let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); let target = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); let ret = self.machine.acquire_locations( @@ -5537,11 +5535,7 @@ impl FunctionCodeGenerator for X64FunctionCode { self.value_stack.push(ret); let value = self.machine.acquire_temp_gpr().unwrap(); - a.emit_movzx( - Size::S16, - loc, - Size::S32, - Location::GPR(value)); + a.emit_movzx(Size::S16, loc, Size::S32, Location::GPR(value)); Self::emit_memory_op( module_info, &self.config, @@ -5553,16 +5547,14 @@ impl FunctionCodeGenerator for X64FunctionCode { 4, |a, _m, addr| { a.emit_lock_xadd(Size::S16, Location::GPR(value), Location::Memory(addr, 0)) - } + }, ); - a.emit_mov( - Size::S32, - Location::GPR(value), - ret); + a.emit_mov(Size::S32, Location::GPR(value), ret); self.machine.release_temp_gpr(value); } Operator::I32AtomicRmwSub { ref memarg } => { - let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); let target = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); let ret = self.machine.acquire_locations( @@ -5573,10 +5565,7 @@ impl FunctionCodeGenerator for X64FunctionCode { self.value_stack.push(ret); let value = self.machine.acquire_temp_gpr().unwrap(); - a.emit_mov( - Size::S32, - loc, - Location::GPR(value)); + a.emit_mov(Size::S32, loc, Location::GPR(value)); a.emit_neg(Size::S32, Location::GPR(value)); Self::emit_memory_op( module_info, @@ -5589,16 +5578,14 @@ impl FunctionCodeGenerator for X64FunctionCode { 4, |a, _m, addr| { a.emit_lock_xadd(Size::S32, Location::GPR(value), Location::Memory(addr, 0)) - } + }, ); - a.emit_mov( - Size::S32, - Location::GPR(value), - ret); + a.emit_mov(Size::S32, Location::GPR(value), ret); self.machine.release_temp_gpr(value); } Operator::I32AtomicRmw8USub { ref memarg } => { - let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); let target = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); let ret = self.machine.acquire_locations( @@ -5626,16 +5613,14 @@ impl FunctionCodeGenerator for X64FunctionCode { 4, |a, _m, addr| { a.emit_lock_xadd(Size::S8, Location::GPR(value), Location::Memory(addr, 0)) - } + }, ); - a.emit_mov( - Size::S32, - Location::GPR(value), - ret); + a.emit_mov(Size::S32, Location::GPR(value), ret); self.machine.release_temp_gpr(value); } Operator::I32AtomicRmw16USub { ref memarg } => { - let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); let target = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); let ret = self.machine.acquire_locations( @@ -5646,11 +5631,7 @@ impl FunctionCodeGenerator for X64FunctionCode { self.value_stack.push(ret); let value = self.machine.acquire_temp_gpr().unwrap(); - a.emit_movzx( - Size::S16, - loc, - Size::S32, - Location::GPR(value)); + a.emit_movzx(Size::S16, loc, Size::S32, Location::GPR(value)); a.emit_neg(Size::S16, Location::GPR(value)); Self::emit_memory_op( module_info, @@ -5663,12 +5644,66 @@ impl FunctionCodeGenerator for X64FunctionCode { 4, |a, _m, addr| { a.emit_lock_xadd(Size::S16, Location::GPR(value), Location::Memory(addr, 0)) - } + }, ); - a.emit_mov( - Size::S32, - Location::GPR(value), - ret); + a.emit_mov(Size::S32, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I32AtomicRmwAnd { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let retry = a.get_label(); + + let value = self.machine.acquire_temp_gpr().unwrap(); + let compare = GPR::RAX; + + a.emit_label(retry); + + a.emit_mov(Size::S32, loc, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + target, + memarg, + true, + 4, + |a, _m, addr| { + a.emit_mov(Size::S32, Location::Memory(addr, 0), Location::GPR(compare)) + }, + ); + a.emit_and(Size::S32, Location::GPR(compare), Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + target, + memarg, + true, + 4, + |a, _m, addr| { + a.emit_lock_cmpxchg( + Size::S32, + Location::GPR(value), + Location::Memory(addr, 0), + ) + }, + ); + + a.emit_jmp(Condition::NotEqual, retry); + + a.emit_mov(Size::S32, Location::GPR(value), ret); self.machine.release_temp_gpr(value); } _ => { From cd1d06f5a5f3dc33bff3b907c91e42d9033ed9a6 Mon Sep 17 00:00:00 2001 From: Nick Lewycky Date: Mon, 23 Sep 2019 14:52:05 -0700 Subject: [PATCH 05/13] Initial working implementation of I32AtomicRmwAnd! Adds the ability to reserve a specific temp-gpr register. Needed for CMPXCHG which always uses RAX. --- lib/singlepass-backend/src/codegen_x64.rs | 33 ++++++++--------------- lib/singlepass-backend/src/machine.rs | 9 ++++++- 2 files changed, 19 insertions(+), 23 deletions(-) diff --git a/lib/singlepass-backend/src/codegen_x64.rs b/lib/singlepass-backend/src/codegen_x64.rs index 71b3358fd03..20af509deac 100644 --- a/lib/singlepass-backend/src/codegen_x64.rs +++ b/lib/singlepass-backend/src/codegen_x64.rs @@ -5661,28 +5661,14 @@ impl FunctionCodeGenerator for X64FunctionCode { )[0]; self.value_stack.push(ret); - let retry = a.get_label(); - - let value = self.machine.acquire_temp_gpr().unwrap(); - let compare = GPR::RAX; + let compare = self.machine.reserve_temp_gpr(GPR::RAX); + let value = if loc == Location::GPR(GPR::R14) { GPR::R13 } else { GPR::R14 }; + a.emit_push(Size::S64, Location::GPR(value)); + a.emit_mov(Size::S32, loc, Location::GPR(value)); + let retry = a.get_label(); a.emit_label(retry); - a.emit_mov(Size::S32, loc, Location::GPR(value)); - Self::emit_memory_op( - module_info, - &self.config, - a, - &mut self.machine, - target, - memarg, - true, - 4, - |a, _m, addr| { - a.emit_mov(Size::S32, Location::Memory(addr, 0), Location::GPR(compare)) - }, - ); - a.emit_and(Size::S32, Location::GPR(compare), Location::GPR(value)); Self::emit_memory_op( module_info, &self.config, @@ -5693,18 +5679,21 @@ impl FunctionCodeGenerator for X64FunctionCode { true, 4, |a, _m, addr| { + a.emit_mov(Size::S32, Location::Memory(addr, 0), Location::GPR(compare)); + a.emit_mov(Size::S32, Location::GPR(compare), ret); + a.emit_and(Size::S32, Location::GPR(compare), Location::GPR(value)); a.emit_lock_cmpxchg( Size::S32, Location::GPR(value), Location::Memory(addr, 0), - ) + ); }, ); a.emit_jmp(Condition::NotEqual, retry); - a.emit_mov(Size::S32, Location::GPR(value), ret); - self.machine.release_temp_gpr(value); + a.emit_pop(Size::S64, Location::GPR(value)); + self.machine.release_temp_gpr(compare); } _ => { return Err(CodegenError { diff --git a/lib/singlepass-backend/src/machine.rs b/lib/singlepass-backend/src/machine.rs index a0793f37983..dfeb010dfc0 100644 --- a/lib/singlepass-backend/src/machine.rs +++ b/lib/singlepass-backend/src/machine.rs @@ -83,7 +83,14 @@ impl Machine { /// Releases a temporary GPR. pub fn release_temp_gpr(&mut self, gpr: GPR) { - assert_eq!(self.used_gprs.remove(&gpr), true); + assert!(self.used_gprs.remove(&gpr)); + } + + /// Specify that a given register is in use. + pub fn reserve_temp_gpr(&mut self, gpr: GPR) -> GPR { + assert!(!self.used_gprs.contains(&gpr)); + self.used_gprs.insert(gpr); + gpr } /// Picks an unused XMM register. From f021d59a0bfd37bd6b0b738ee2f24409e736b9fa Mon Sep 17 00:00:00 2001 From: Nick Lewycky Date: Mon, 23 Sep 2019 16:08:49 -0700 Subject: [PATCH 06/13] Refactor out a compare-and-swap loop function. --- lib/singlepass-backend/src/codegen_x64.rs | 84 +++++++++++++++-------- 1 file changed, 55 insertions(+), 29 deletions(-) diff --git a/lib/singlepass-backend/src/codegen_x64.rs b/lib/singlepass-backend/src/codegen_x64.rs index 20af509deac..c69abff43e5 100644 --- a/lib/singlepass-backend/src/codegen_x64.rs +++ b/lib/singlepass-backend/src/codegen_x64.rs @@ -1586,6 +1586,55 @@ impl X64FunctionCode { m.release_temp_gpr(tmp_addr); } + /// Emits a memory operation. + fn emit_cas_loop_op( + module_info: &ModuleInfo, + config: &CodegenConfig, + a: &mut Assembler, + m: &mut Machine, + loc: Location, + target: Location, + ret: Location, + memarg: &MemoryImmediate, + value_size: usize, + cb: F, + ) { + let compare = m.reserve_temp_gpr(GPR::RAX); + let value = if loc == Location::GPR(GPR::R14) { + GPR::R13 + } else { + GPR::R14 + }; + a.emit_push(Size::S64, Location::GPR(value)); + + a.emit_mov(Size::S32, loc, Location::GPR(value)); + + let retry = a.get_label(); + a.emit_label(retry); + + Self::emit_memory_op( + module_info, + config, + a, + m, + target, + memarg, + true, + value_size, + |a, m, addr| { + a.emit_mov(Size::S32, Location::Memory(addr, 0), Location::GPR(compare)); + a.emit_mov(Size::S32, Location::GPR(compare), ret); + cb(a, m, compare, value); + a.emit_lock_cmpxchg(Size::S32, Location::GPR(value), Location::Memory(addr, 0)); + }, + ); + + a.emit_jmp(Condition::NotEqual, retry); + + a.emit_pop(Size::S64, Location::GPR(value)); + m.release_temp_gpr(compare); + } + // Checks for underflow/overflow/nan before IxxTrunc{U/S}F32. fn emit_f32_int_conv_check( a: &mut Assembler, @@ -5596,11 +5645,7 @@ impl FunctionCodeGenerator for X64FunctionCode { self.value_stack.push(ret); let value = self.machine.acquire_temp_gpr().unwrap(); - a.emit_movzx( - Size::S8, - loc, - Size::S32, - Location::GPR(value)); + a.emit_movzx(Size::S8, loc, Size::S32, Location::GPR(value)); a.emit_neg(Size::S8, Location::GPR(value)); Self::emit_memory_op( module_info, @@ -5661,39 +5706,20 @@ impl FunctionCodeGenerator for X64FunctionCode { )[0]; self.value_stack.push(ret); - let compare = self.machine.reserve_temp_gpr(GPR::RAX); - let value = if loc == Location::GPR(GPR::R14) { GPR::R13 } else { GPR::R14 }; - a.emit_push(Size::S64, Location::GPR(value)); - a.emit_mov(Size::S32, loc, Location::GPR(value)); - - let retry = a.get_label(); - a.emit_label(retry); - - Self::emit_memory_op( + Self::emit_cas_loop_op( module_info, &self.config, a, &mut self.machine, + loc, target, + ret, memarg, - true, 4, - |a, _m, addr| { - a.emit_mov(Size::S32, Location::Memory(addr, 0), Location::GPR(compare)); - a.emit_mov(Size::S32, Location::GPR(compare), ret); - a.emit_and(Size::S32, Location::GPR(compare), Location::GPR(value)); - a.emit_lock_cmpxchg( - Size::S32, - Location::GPR(value), - Location::Memory(addr, 0), - ); + |a, _m, src, dst| { + a.emit_and(Size::S32, Location::GPR(src), Location::GPR(dst)); }, ); - - a.emit_jmp(Condition::NotEqual, retry); - - a.emit_pop(Size::S64, Location::GPR(value)); - self.machine.release_temp_gpr(compare); } _ => { return Err(CodegenError { From bc7e017188e6550946873e53e30b0da3ebc10d3b Mon Sep 17 00:00:00 2001 From: Nick Lewycky Date: Mon, 23 Sep 2019 22:01:08 -0700 Subject: [PATCH 07/13] Add atomic.rmw operations, excluding xchg and cmpxchg. Sizes are now ordered, to facilitate an assertion that one size is less (smaller) than another. panic! error messages are provided for remaining emitter functions. --- lib/singlepass-backend/src/codegen_x64.rs | 850 +++++++++++++++++++++- lib/singlepass-backend/src/emitter_x64.rs | 90 ++- 2 files changed, 905 insertions(+), 35 deletions(-) diff --git a/lib/singlepass-backend/src/codegen_x64.rs b/lib/singlepass-backend/src/codegen_x64.rs index c69abff43e5..8a6b1c2654b 100644 --- a/lib/singlepass-backend/src/codegen_x64.rs +++ b/lib/singlepass-backend/src/codegen_x64.rs @@ -1587,7 +1587,7 @@ impl X64FunctionCode { } /// Emits a memory operation. - fn emit_cas_loop_op( + fn emit_compare_and_swap( module_info: &ModuleInfo, config: &CodegenConfig, a: &mut Assembler, @@ -1597,8 +1597,12 @@ impl X64FunctionCode { ret: Location, memarg: &MemoryImmediate, value_size: usize, + memory_sz: Size, + stack_sz: Size, cb: F, ) { + assert!(memory_sz <= stack_sz); + let compare = m.reserve_temp_gpr(GPR::RAX); let value = if loc == Location::GPR(GPR::R14) { GPR::R13 @@ -1607,7 +1611,7 @@ impl X64FunctionCode { }; a.emit_push(Size::S64, Location::GPR(value)); - a.emit_mov(Size::S32, loc, Location::GPR(value)); + a.emit_mov(stack_sz, loc, Location::GPR(value)); let retry = a.get_label(); a.emit_label(retry); @@ -1622,10 +1626,10 @@ impl X64FunctionCode { true, value_size, |a, m, addr| { - a.emit_mov(Size::S32, Location::Memory(addr, 0), Location::GPR(compare)); - a.emit_mov(Size::S32, Location::GPR(compare), ret); + a.emit_mov(memory_sz, Location::Memory(addr, 0), Location::GPR(compare)); + a.emit_mov(stack_sz, Location::GPR(compare), ret); cb(a, m, compare, value); - a.emit_lock_cmpxchg(Size::S32, Location::GPR(value), Location::Memory(addr, 0)); + a.emit_lock_cmpxchg(memory_sz, Location::GPR(value), Location::Memory(addr, 0)); }, ); @@ -5541,6 +5545,36 @@ impl FunctionCodeGenerator for X64FunctionCode { a.emit_mov(Size::S32, Location::GPR(value), ret); self.machine.release_temp_gpr(value); } + Operator::I64AtomicRmwAdd { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_mov(Size::S64, loc, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + target, + memarg, + true, + 8, + |a, _m, addr| { + a.emit_lock_xadd(Size::S64, Location::GPR(value), Location::Memory(addr, 0)) + }, + ); + a.emit_mov(Size::S64, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } Operator::I32AtomicRmw8UAdd { ref memarg } => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); @@ -5563,7 +5597,7 @@ impl FunctionCodeGenerator for X64FunctionCode { target, memarg, true, - 4, + 1, |a, _m, addr| { a.emit_lock_xadd(Size::S8, Location::GPR(value), Location::Memory(addr, 0)) }, @@ -5593,7 +5627,7 @@ impl FunctionCodeGenerator for X64FunctionCode { target, memarg, true, - 4, + 2, |a, _m, addr| { a.emit_lock_xadd(Size::S16, Location::GPR(value), Location::Memory(addr, 0)) }, @@ -5601,6 +5635,96 @@ impl FunctionCodeGenerator for X64FunctionCode { a.emit_mov(Size::S32, Location::GPR(value), ret); self.machine.release_temp_gpr(value); } + Operator::I64AtomicRmw8UAdd { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_movzx(Size::S8, loc, Size::S64, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + target, + memarg, + true, + 1, + |a, _m, addr| { + a.emit_lock_xadd(Size::S8, Location::GPR(value), Location::Memory(addr, 0)) + }, + ); + a.emit_mov(Size::S64, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I64AtomicRmw16UAdd { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_movzx(Size::S16, loc, Size::S64, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + target, + memarg, + true, + 2, + |a, _m, addr| { + a.emit_lock_xadd(Size::S16, Location::GPR(value), Location::Memory(addr, 0)) + }, + ); + a.emit_mov(Size::S64, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I64AtomicRmw32UAdd { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_mov(Size::S32, loc, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + target, + memarg, + true, + 4, + |a, _m, addr| { + a.emit_lock_xadd(Size::S32, Location::GPR(value), Location::Memory(addr, 0)) + }, + ); + a.emit_mov(Size::S64, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } Operator::I32AtomicRmwSub { ref memarg } => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); @@ -5632,6 +5756,37 @@ impl FunctionCodeGenerator for X64FunctionCode { a.emit_mov(Size::S32, Location::GPR(value), ret); self.machine.release_temp_gpr(value); } + Operator::I64AtomicRmwSub { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_mov(Size::S64, loc, Location::GPR(value)); + a.emit_neg(Size::S64, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + target, + memarg, + true, + 8, + |a, _m, addr| { + a.emit_lock_xadd(Size::S64, Location::GPR(value), Location::Memory(addr, 0)) + }, + ); + a.emit_mov(Size::S64, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } Operator::I32AtomicRmw8USub { ref memarg } => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); @@ -5655,7 +5810,7 @@ impl FunctionCodeGenerator for X64FunctionCode { target, memarg, true, - 4, + 1, |a, _m, addr| { a.emit_lock_xadd(Size::S8, Location::GPR(value), Location::Memory(addr, 0)) }, @@ -5686,7 +5841,7 @@ impl FunctionCodeGenerator for X64FunctionCode { target, memarg, true, - 4, + 2, |a, _m, addr| { a.emit_lock_xadd(Size::S16, Location::GPR(value), Location::Memory(addr, 0)) }, @@ -5694,6 +5849,99 @@ impl FunctionCodeGenerator for X64FunctionCode { a.emit_mov(Size::S32, Location::GPR(value), ret); self.machine.release_temp_gpr(value); } + Operator::I64AtomicRmw8USub { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_movzx(Size::S8, loc, Size::S64, Location::GPR(value)); + a.emit_neg(Size::S8, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + target, + memarg, + true, + 1, + |a, _m, addr| { + a.emit_lock_xadd(Size::S8, Location::GPR(value), Location::Memory(addr, 0)) + }, + ); + a.emit_mov(Size::S64, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I64AtomicRmw16USub { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_movzx(Size::S16, loc, Size::S64, Location::GPR(value)); + a.emit_neg(Size::S16, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + target, + memarg, + true, + 2, + |a, _m, addr| { + a.emit_lock_xadd(Size::S16, Location::GPR(value), Location::Memory(addr, 0)) + }, + ); + a.emit_mov(Size::S64, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I64AtomicRmw32USub { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_mov(Size::S32, loc, Location::GPR(value)); + a.emit_neg(Size::S32, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + target, + memarg, + true, + 2, + |a, _m, addr| { + a.emit_lock_xadd(Size::S32, Location::GPR(value), Location::Memory(addr, 0)) + }, + ); + a.emit_mov(Size::S64, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } Operator::I32AtomicRmwAnd { ref memarg } => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); @@ -5706,7 +5954,7 @@ impl FunctionCodeGenerator for X64FunctionCode { )[0]; self.value_stack.push(ret); - Self::emit_cas_loop_op( + Self::emit_compare_and_swap( module_info, &self.config, a, @@ -5716,11 +5964,593 @@ impl FunctionCodeGenerator for X64FunctionCode { ret, memarg, 4, + Size::S32, + Size::S32, |a, _m, src, dst| { a.emit_and(Size::S32, Location::GPR(src), Location::GPR(dst)); }, ); } + Operator::I64AtomicRmwAnd { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + loc, + target, + ret, + memarg, + 8, + Size::S64, + Size::S64, + |a, _m, src, dst| { + a.emit_and(Size::S64, Location::GPR(src), Location::GPR(dst)); + }, + ); + } + Operator::I32AtomicRmw8UAnd { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + loc, + target, + ret, + memarg, + 1, + Size::S8, + Size::S32, + |a, _m, src, dst| { + a.emit_and(Size::S32, Location::GPR(src), Location::GPR(dst)); + }, + ); + } + Operator::I32AtomicRmw16UAnd { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + loc, + target, + ret, + memarg, + 1, + Size::S16, + Size::S32, + |a, _m, src, dst| { + a.emit_and(Size::S32, Location::GPR(src), Location::GPR(dst)); + }, + ); + } + Operator::I64AtomicRmw8UAnd { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + loc, + target, + ret, + memarg, + 1, + Size::S8, + Size::S64, + |a, _m, src, dst| { + a.emit_and(Size::S64, Location::GPR(src), Location::GPR(dst)); + }, + ); + } + Operator::I64AtomicRmw16UAnd { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + loc, + target, + ret, + memarg, + 1, + Size::S16, + Size::S64, + |a, _m, src, dst| { + a.emit_and(Size::S64, Location::GPR(src), Location::GPR(dst)); + }, + ); + } + Operator::I64AtomicRmw32UAnd { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + loc, + target, + ret, + memarg, + 1, + Size::S32, + Size::S64, + |a, _m, src, dst| { + a.emit_and(Size::S64, Location::GPR(src), Location::GPR(dst)); + }, + ); + } + Operator::I32AtomicRmwOr { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + loc, + target, + ret, + memarg, + 4, + Size::S32, + Size::S32, + |a, _m, src, dst| { + a.emit_or(Size::S32, Location::GPR(src), Location::GPR(dst)); + }, + ); + } + Operator::I64AtomicRmwOr { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + loc, + target, + ret, + memarg, + 8, + Size::S64, + Size::S64, + |a, _m, src, dst| { + a.emit_or(Size::S64, Location::GPR(src), Location::GPR(dst)); + }, + ); + } + Operator::I32AtomicRmw8UOr { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + loc, + target, + ret, + memarg, + 1, + Size::S8, + Size::S32, + |a, _m, src, dst| { + a.emit_or(Size::S32, Location::GPR(src), Location::GPR(dst)); + }, + ); + } + Operator::I32AtomicRmw16UOr { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + loc, + target, + ret, + memarg, + 1, + Size::S16, + Size::S32, + |a, _m, src, dst| { + a.emit_or(Size::S32, Location::GPR(src), Location::GPR(dst)); + }, + ); + } + Operator::I64AtomicRmw8UOr { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + loc, + target, + ret, + memarg, + 1, + Size::S8, + Size::S64, + |a, _m, src, dst| { + a.emit_or(Size::S64, Location::GPR(src), Location::GPR(dst)); + }, + ); + } + Operator::I64AtomicRmw16UOr { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + loc, + target, + ret, + memarg, + 1, + Size::S16, + Size::S64, + |a, _m, src, dst| { + a.emit_or(Size::S64, Location::GPR(src), Location::GPR(dst)); + }, + ); + } + Operator::I64AtomicRmw32UOr { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + loc, + target, + ret, + memarg, + 1, + Size::S32, + Size::S64, + |a, _m, src, dst| { + a.emit_or(Size::S64, Location::GPR(src), Location::GPR(dst)); + }, + ); + } + Operator::I32AtomicRmwXor { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + loc, + target, + ret, + memarg, + 4, + Size::S32, + Size::S32, + |a, _m, src, dst| { + a.emit_xor(Size::S32, Location::GPR(src), Location::GPR(dst)); + }, + ); + } + Operator::I64AtomicRmwXor { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + loc, + target, + ret, + memarg, + 8, + Size::S64, + Size::S64, + |a, _m, src, dst| { + a.emit_xor(Size::S64, Location::GPR(src), Location::GPR(dst)); + }, + ); + } + Operator::I32AtomicRmw8UXor { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + loc, + target, + ret, + memarg, + 1, + Size::S8, + Size::S32, + |a, _m, src, dst| { + a.emit_xor(Size::S32, Location::GPR(src), Location::GPR(dst)); + }, + ); + } + Operator::I32AtomicRmw16UXor { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + loc, + target, + ret, + memarg, + 1, + Size::S16, + Size::S32, + |a, _m, src, dst| { + a.emit_xor(Size::S32, Location::GPR(src), Location::GPR(dst)); + }, + ); + } + Operator::I64AtomicRmw8UXor { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + loc, + target, + ret, + memarg, + 1, + Size::S8, + Size::S64, + |a, _m, src, dst| { + a.emit_xor(Size::S64, Location::GPR(src), Location::GPR(dst)); + }, + ); + } + Operator::I64AtomicRmw16UXor { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + loc, + target, + ret, + memarg, + 1, + Size::S16, + Size::S64, + |a, _m, src, dst| { + a.emit_xor(Size::S64, Location::GPR(src), Location::GPR(dst)); + }, + ); + } + Operator::I64AtomicRmw32UXor { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + Self::emit_compare_and_swap( + module_info, + &self.config, + a, + &mut self.machine, + loc, + target, + ret, + memarg, + 1, + Size::S32, + Size::S64, + |a, _m, src, dst| { + a.emit_xor(Size::S64, Location::GPR(src), Location::GPR(dst)); + }, + ); + } _ => { return Err(CodegenError { message: format!("not yet implemented: {:?}", op), diff --git a/lib/singlepass-backend/src/emitter_x64.rs b/lib/singlepass-backend/src/emitter_x64.rs index 83099f25970..5544d08c537 100644 --- a/lib/singlepass-backend/src/emitter_x64.rs +++ b/lib/singlepass-backend/src/emitter_x64.rs @@ -28,7 +28,7 @@ pub enum Condition { Signed, } -#[derive(Copy, Clone, Debug, Eq, PartialEq)] +#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd)] pub enum Size { S8, S16, @@ -584,7 +584,9 @@ impl Emitter for Assembler { dynasm!(self ; cqo); } fn emit_xor(&mut self, sz: Size, src: Location, dst: Location) { - binop_all_nofp!(xor, self, sz, src, dst, { unreachable!() }); + binop_all_nofp!(xor, self, sz, src, dst, { + panic!("XOR {:?} {:?} {:?}", sz, src, dst) + }); } fn emit_jmp(&mut self, condition: Condition, label: Self::Label) { match condition { @@ -666,72 +668,110 @@ impl Emitter for Assembler { }); } fn emit_add(&mut self, sz: Size, src: Location, dst: Location) { - binop_all_nofp!(add, self, sz, src, dst, { unreachable!() }); + binop_all_nofp!(add, self, sz, src, dst, { + panic!("ADD {:?} {:?} {:?}", sz, src, dst) + }); } fn emit_sub(&mut self, sz: Size, src: Location, dst: Location) { - binop_all_nofp!(sub, self, sz, src, dst, { unreachable!() }); + binop_all_nofp!(sub, self, sz, src, dst, { + panic!("SUB {:?} {:?} {:?}", sz, src, dst) + }); } fn emit_neg(&mut self, sz: Size, value: Location) { match (sz, value) { - (Size::S8, Location::GPR(value)) => { dynasm!(self ; neg Rb(value as u8)) } - (Size::S8, Location::Memory(value, disp)) => { dynasm!(self ; neg [Rq(value as u8) + disp]) } - (Size::S16, Location::GPR(value)) => { dynasm!(self ; neg Rw(value as u8)) } - (Size::S16, Location::Memory(value, disp)) => { dynasm!(self ; neg [Rq(value as u8) + disp]) } - (Size::S32, Location::GPR(value)) => { dynasm!(self ; neg Rd(value as u8)) } - (Size::S32, Location::Memory(value, disp)) => { dynasm!(self ; neg [Rq(value as u8) + disp]) } - (Size::S64, Location::GPR(value)) => { dynasm!(self ; neg Rq(value as u8)) } - (Size::S64, Location::Memory(value, disp)) => { dynasm!(self ; neg [Rq(value as u8) + disp]) } + (Size::S8, Location::GPR(value)) => dynasm!(self ; neg Rb(value as u8)), + (Size::S8, Location::Memory(value, disp)) => { + dynasm!(self ; neg [Rq(value as u8) + disp]) + } + (Size::S16, Location::GPR(value)) => dynasm!(self ; neg Rw(value as u8)), + (Size::S16, Location::Memory(value, disp)) => { + dynasm!(self ; neg [Rq(value as u8) + disp]) + } + (Size::S32, Location::GPR(value)) => dynasm!(self ; neg Rd(value as u8)), + (Size::S32, Location::Memory(value, disp)) => { + dynasm!(self ; neg [Rq(value as u8) + disp]) + } + (Size::S64, Location::GPR(value)) => dynasm!(self ; neg Rq(value as u8)), + (Size::S64, Location::Memory(value, disp)) => { + dynasm!(self ; neg [Rq(value as u8) + disp]) + } _ => panic!("NEG {:?} {:?}", sz, value), } } fn emit_imul(&mut self, sz: Size, src: Location, dst: Location) { binop_gpr_gpr!(imul, self, sz, src, dst, { - binop_mem_gpr!(imul, self, sz, src, dst, { unreachable!() }) + binop_mem_gpr!(imul, self, sz, src, dst, { + panic!("IMUL {:?} {:?} {:?}", sz, src, dst) + }) }); } fn emit_imul_imm32_gpr64(&mut self, src: u32, dst: GPR) { dynasm!(self ; imul Rq(dst as u8), Rq(dst as u8), src as i32); } fn emit_div(&mut self, sz: Size, divisor: Location) { - unop_gpr_or_mem!(div, self, sz, divisor, { unreachable!() }); + unop_gpr_or_mem!(div, self, sz, divisor, { + panic!("DIV {:?} {:?}", sz, divisor) + }); } fn emit_idiv(&mut self, sz: Size, divisor: Location) { - unop_gpr_or_mem!(idiv, self, sz, divisor, { unreachable!() }); + unop_gpr_or_mem!(idiv, self, sz, divisor, { + panic!("IDIV {:?} {:?}", sz, divisor) + }); } fn emit_shl(&mut self, sz: Size, src: Location, dst: Location) { - binop_shift!(shl, self, sz, src, dst, { unreachable!() }); + binop_shift!(shl, self, sz, src, dst, { + panic!("SHL {:?} {:?} {:?}", sz, src, dst) + }); } fn emit_shr(&mut self, sz: Size, src: Location, dst: Location) { - binop_shift!(shr, self, sz, src, dst, { unreachable!() }); + binop_shift!(shr, self, sz, src, dst, { + panic!("SHR {:?} {:?} {:?}", sz, src, dst) + }); } fn emit_sar(&mut self, sz: Size, src: Location, dst: Location) { - binop_shift!(sar, self, sz, src, dst, { unreachable!() }); + binop_shift!(sar, self, sz, src, dst, { + panic!("SAR {:?} {:?} {:?}", sz, src, dst) + }); } fn emit_rol(&mut self, sz: Size, src: Location, dst: Location) { - binop_shift!(rol, self, sz, src, dst, { unreachable!() }); + binop_shift!(rol, self, sz, src, dst, { + panic!("ROL {:?} {:?} {:?}", sz, src, dst) + }); } fn emit_ror(&mut self, sz: Size, src: Location, dst: Location) { - binop_shift!(ror, self, sz, src, dst, { unreachable!() }); + binop_shift!(ror, self, sz, src, dst, { + panic!("ROR {:?} {:?} {:?}", sz, src, dst) + }); } fn emit_and(&mut self, sz: Size, src: Location, dst: Location) { - binop_all_nofp!(and, self, sz, src, dst, { unreachable!() }); + binop_all_nofp!(and, self, sz, src, dst, { + panic!("AND {:?} {:?} {:?}", sz, src, dst) + }); } fn emit_or(&mut self, sz: Size, src: Location, dst: Location) { - binop_all_nofp!(or, self, sz, src, dst, { unreachable!() }); + binop_all_nofp!(or, self, sz, src, dst, { + panic!("OR {:?} {:?} {:?}", sz, src, dst) + }); } fn emit_lzcnt(&mut self, sz: Size, src: Location, dst: Location) { binop_gpr_gpr!(lzcnt, self, sz, src, dst, { - binop_mem_gpr!(lzcnt, self, sz, src, dst, { unreachable!() }) + binop_mem_gpr!(lzcnt, self, sz, src, dst, { + panic!("LZCNT {:?} {:?} {:?}", sz, src, dst) + }) }); } fn emit_tzcnt(&mut self, sz: Size, src: Location, dst: Location) { binop_gpr_gpr!(tzcnt, self, sz, src, dst, { - binop_mem_gpr!(tzcnt, self, sz, src, dst, { unreachable!() }) + binop_mem_gpr!(tzcnt, self, sz, src, dst, { + panic!("TZCNT {:?} {:?} {:?}", sz, src, dst) + }) }); } fn emit_popcnt(&mut self, sz: Size, src: Location, dst: Location) { binop_gpr_gpr!(popcnt, self, sz, src, dst, { - binop_mem_gpr!(popcnt, self, sz, src, dst, { unreachable!() }) + binop_mem_gpr!(popcnt, self, sz, src, dst, { + panic!("POPCNT {:?} {:?} {:?}", sz, src, dst) + }) }); } fn emit_movzx(&mut self, sz_src: Size, src: Location, sz_dst: Size, dst: Location) { From ba68cfc2c64e0ae7444af6acd6c29a08f5d4390f Mon Sep 17 00:00:00 2001 From: Nick Lewycky Date: Tue, 24 Sep 2019 15:44:17 -0700 Subject: [PATCH 08/13] Finish atomic operations for singlepass, excluding wait and notify. --- lib/singlepass-backend/src/codegen_x64.rs | 553 ++++++++++++++++++++++ lib/spectests/tests/excludes.txt | 46 +- 2 files changed, 598 insertions(+), 1 deletion(-) diff --git a/lib/singlepass-backend/src/codegen_x64.rs b/lib/singlepass-backend/src/codegen_x64.rs index 8a6b1c2654b..9322531f836 100644 --- a/lib/singlepass-backend/src/codegen_x64.rs +++ b/lib/singlepass-backend/src/codegen_x64.rs @@ -6551,6 +6551,559 @@ impl FunctionCodeGenerator for X64FunctionCode { }, ); } + Operator::I32AtomicRmwXchg { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_mov(Size::S32, loc, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + target, + memarg, + true, + 4, + |a, _m, addr| { + a.emit_xchg(Size::S32, Location::GPR(value), Location::Memory(addr, 0)) + }, + ); + a.emit_mov(Size::S32, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I64AtomicRmwXchg { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_mov(Size::S64, loc, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + target, + memarg, + true, + 8, + |a, _m, addr| { + a.emit_xchg(Size::S64, Location::GPR(value), Location::Memory(addr, 0)) + }, + ); + a.emit_mov(Size::S64, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I32AtomicRmw8UXchg { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_movzx(Size::S8, loc, Size::S32, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + target, + memarg, + true, + 1, + |a, _m, addr| { + a.emit_xchg(Size::S8, Location::GPR(value), Location::Memory(addr, 0)) + }, + ); + a.emit_mov(Size::S32, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I32AtomicRmw16UXchg { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_movzx(Size::S16, loc, Size::S32, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + target, + memarg, + true, + 2, + |a, _m, addr| { + a.emit_xchg(Size::S16, Location::GPR(value), Location::Memory(addr, 0)) + }, + ); + a.emit_mov(Size::S32, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I64AtomicRmw8UXchg { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_movzx(Size::S8, loc, Size::S64, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + target, + memarg, + true, + 1, + |a, _m, addr| { + a.emit_xchg(Size::S8, Location::GPR(value), Location::Memory(addr, 0)) + }, + ); + a.emit_mov(Size::S64, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I64AtomicRmw16UXchg { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_movzx(Size::S16, loc, Size::S64, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + target, + memarg, + true, + 2, + |a, _m, addr| { + a.emit_xchg(Size::S16, Location::GPR(value), Location::Memory(addr, 0)) + }, + ); + a.emit_mov(Size::S64, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I64AtomicRmw32UXchg { ref memarg } => { + let loc = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let value = self.machine.acquire_temp_gpr().unwrap(); + a.emit_mov(Size::S32, loc, Location::GPR(value)); + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + target, + memarg, + true, + 4, + |a, _m, addr| { + a.emit_xchg(Size::S32, Location::GPR(value), Location::Memory(addr, 0)) + }, + ); + a.emit_mov(Size::S64, Location::GPR(value), ret); + self.machine.release_temp_gpr(value); + } + Operator::I32AtomicRmwCmpxchg { ref memarg } => { + let new = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let cmp = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let compare = self.machine.reserve_temp_gpr(GPR::RAX); + let value = if cmp == Location::GPR(GPR::R14) { + if new == Location::GPR(GPR::R13) { + GPR::R12 + } else { + GPR::R13 + } + } else { + GPR::R14 + }; + a.emit_push(Size::S64, Location::GPR(value)); + a.emit_mov(Size::S32, cmp, Location::GPR(compare)); + a.emit_mov(Size::S32, new, Location::GPR(value)); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + target, + memarg, + true, + 4, + |a, _m, addr| { + a.emit_lock_cmpxchg( + Size::S32, + Location::GPR(value), + Location::Memory(addr, 0), + ); + a.emit_mov(Size::S32, Location::GPR(compare), ret); + }, + ); + a.emit_pop(Size::S64, Location::GPR(value)); + self.machine.release_temp_gpr(compare); + } + Operator::I64AtomicRmwCmpxchg { ref memarg } => { + let new = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let cmp = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let compare = self.machine.reserve_temp_gpr(GPR::RAX); + let value = if cmp == Location::GPR(GPR::R14) { + if new == Location::GPR(GPR::R13) { + GPR::R12 + } else { + GPR::R13 + } + } else { + GPR::R14 + }; + a.emit_push(Size::S64, Location::GPR(value)); + a.emit_mov(Size::S64, cmp, Location::GPR(compare)); + a.emit_mov(Size::S64, new, Location::GPR(value)); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + target, + memarg, + true, + 8, + |a, _m, addr| { + a.emit_lock_cmpxchg( + Size::S64, + Location::GPR(value), + Location::Memory(addr, 0), + ); + a.emit_mov(Size::S64, Location::GPR(compare), ret); + }, + ); + a.emit_pop(Size::S64, Location::GPR(value)); + self.machine.release_temp_gpr(compare); + } + Operator::I32AtomicRmw8UCmpxchg { ref memarg } => { + let new = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let cmp = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let compare = self.machine.reserve_temp_gpr(GPR::RAX); + let value = if cmp == Location::GPR(GPR::R14) { + if new == Location::GPR(GPR::R13) { + GPR::R12 + } else { + GPR::R13 + } + } else { + GPR::R14 + }; + a.emit_push(Size::S64, Location::GPR(value)); + a.emit_mov(Size::S32, cmp, Location::GPR(compare)); + a.emit_mov(Size::S32, new, Location::GPR(value)); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + target, + memarg, + true, + 1, + |a, _m, addr| { + a.emit_lock_cmpxchg( + Size::S8, + Location::GPR(value), + Location::Memory(addr, 0), + ); + a.emit_movzx(Size::S8, Location::GPR(compare), Size::S32, ret); + }, + ); + a.emit_pop(Size::S64, Location::GPR(value)); + self.machine.release_temp_gpr(compare); + } + Operator::I32AtomicRmw16UCmpxchg { ref memarg } => { + let new = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let cmp = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let compare = self.machine.reserve_temp_gpr(GPR::RAX); + let value = if cmp == Location::GPR(GPR::R14) { + if new == Location::GPR(GPR::R13) { + GPR::R12 + } else { + GPR::R13 + } + } else { + GPR::R14 + }; + a.emit_push(Size::S64, Location::GPR(value)); + a.emit_mov(Size::S32, cmp, Location::GPR(compare)); + a.emit_mov(Size::S32, new, Location::GPR(value)); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + target, + memarg, + true, + 1, + |a, _m, addr| { + a.emit_lock_cmpxchg( + Size::S16, + Location::GPR(value), + Location::Memory(addr, 0), + ); + a.emit_movzx(Size::S16, Location::GPR(compare), Size::S32, ret); + }, + ); + a.emit_pop(Size::S64, Location::GPR(value)); + self.machine.release_temp_gpr(compare); + } + Operator::I64AtomicRmw8UCmpxchg { ref memarg } => { + let new = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let cmp = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let compare = self.machine.reserve_temp_gpr(GPR::RAX); + let value = if cmp == Location::GPR(GPR::R14) { + if new == Location::GPR(GPR::R13) { + GPR::R12 + } else { + GPR::R13 + } + } else { + GPR::R14 + }; + a.emit_push(Size::S64, Location::GPR(value)); + a.emit_mov(Size::S64, cmp, Location::GPR(compare)); + a.emit_mov(Size::S64, new, Location::GPR(value)); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + target, + memarg, + true, + 1, + |a, _m, addr| { + a.emit_lock_cmpxchg( + Size::S8, + Location::GPR(value), + Location::Memory(addr, 0), + ); + a.emit_movzx(Size::S8, Location::GPR(compare), Size::S64, ret); + }, + ); + a.emit_pop(Size::S64, Location::GPR(value)); + self.machine.release_temp_gpr(compare); + } + Operator::I64AtomicRmw16UCmpxchg { ref memarg } => { + let new = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let cmp = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let compare = self.machine.reserve_temp_gpr(GPR::RAX); + let value = if cmp == Location::GPR(GPR::R14) { + if new == Location::GPR(GPR::R13) { + GPR::R12 + } else { + GPR::R13 + } + } else { + GPR::R14 + }; + a.emit_push(Size::S64, Location::GPR(value)); + a.emit_mov(Size::S64, cmp, Location::GPR(compare)); + a.emit_mov(Size::S64, new, Location::GPR(value)); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + target, + memarg, + true, + 1, + |a, _m, addr| { + a.emit_lock_cmpxchg( + Size::S16, + Location::GPR(value), + Location::Memory(addr, 0), + ); + a.emit_movzx(Size::S16, Location::GPR(compare), Size::S64, ret); + }, + ); + a.emit_pop(Size::S64, Location::GPR(value)); + self.machine.release_temp_gpr(compare); + } + Operator::I64AtomicRmw32UCmpxchg { ref memarg } => { + let new = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let cmp = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let target = + get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; + self.value_stack.push(ret); + + let compare = self.machine.reserve_temp_gpr(GPR::RAX); + let value = if cmp == Location::GPR(GPR::R14) { + if new == Location::GPR(GPR::R13) { + GPR::R12 + } else { + GPR::R13 + } + } else { + GPR::R14 + }; + a.emit_push(Size::S64, Location::GPR(value)); + a.emit_mov(Size::S64, cmp, Location::GPR(compare)); + a.emit_mov(Size::S64, new, Location::GPR(value)); + + Self::emit_memory_op( + module_info, + &self.config, + a, + &mut self.machine, + target, + memarg, + true, + 1, + |a, _m, addr| { + a.emit_lock_cmpxchg( + Size::S32, + Location::GPR(value), + Location::Memory(addr, 0), + ); + a.emit_mov(Size::S32, Location::GPR(compare), ret); + }, + ); + a.emit_pop(Size::S64, Location::GPR(value)); + self.machine.release_temp_gpr(compare); + } _ => { return Err(CodegenError { message: format!("not yet implemented: {:?}", op), diff --git a/lib/spectests/tests/excludes.txt b/lib/spectests/tests/excludes.txt index 19a1c51a8c2..56c6a2d5613 100644 --- a/lib/spectests/tests/excludes.txt +++ b/lib/spectests/tests/excludes.txt @@ -867,7 +867,6 @@ llvm:skip:simd_binaryen.wast:*:unix # Module - caught panic Any # Singlepass -singlepass:skip:atomic.wast:* # Threads not implemented singlepass:skip:simd.wast:* # SIMD not implemented singlepass:skip:simd_binaryen.wast:* # SIMD not implemented @@ -904,6 +903,51 @@ singlepass:fail:address.wast:586 # AssertTrap - expected trap, got Runtime:Error singlepass:fail:address.wast:588 # AssertTrap - expected trap, got [] singlepass:fail:address.wast:589 # AssertTrap - expected trap, got [] singlepass:fail:align.wast:864 # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:atomic.wast:380 # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:atomic.wast:381 # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:atomic.wast:382 # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:atomic.wast:383 # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:atomic.wast:384 # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:atomic.wast:385 # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:atomic.wast:386 # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:atomic.wast:387 # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:atomic.wast:388 # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:atomic.wast:389 # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:atomic.wast:390 # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:atomic.wast:391 # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:atomic.wast:392 # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:atomic.wast:393 # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:atomic.wast:394 # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:atomic.wast:395 # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:atomic.wast:396 # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:atomic.wast:397 # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:atomic.wast:398 # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:atomic.wast:399 # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:atomic.wast:400 # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:atomic.wast:401 # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:atomic.wast:402 # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:atomic.wast:403 # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:atomic.wast:404 # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:atomic.wast:405 # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:atomic.wast:406 # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:atomic.wast:407 # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:atomic.wast:408 # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:atomic.wast:409 # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:atomic.wast:410 # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:atomic.wast:411 # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:atomic.wast:412 # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:atomic.wast:413 # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:atomic.wast:414 # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:atomic.wast:415 # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:atomic.wast:416 # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:atomic.wast:417 # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:atomic.wast:418 # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:atomic.wast:419 # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:atomic.wast:420 # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:atomic.wast:421 # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:atomic.wast:422 # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:atomic.wast:423 # AssertTrap - expected trap, got Runtime:Error unknown error +singlepass:fail:atomic.wast:424 # AssertTrap - expected trap, got Runtime:Error unknown error singlepass:fail:call.wast:289 # AssertTrap - expected trap, got Runtime:Error unknown error singlepass:fail:call_indirect.wast:469 # AssertTrap - expected trap, got Runtime:Error unknown error singlepass:fail:call_indirect.wast:470 # AssertTrap - expected trap, got Runtime:Error unknown error From e401f8ebe054c4637dcd06fca72390a1e2382406 Mon Sep 17 00:00:00 2001 From: Nick Lewycky Date: Tue, 24 Sep 2019 15:47:14 -0700 Subject: [PATCH 09/13] Add changelog entry. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b64cd30c11..e88bd71a415 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ Blocks of changes will separated by version increments. - [#833](https://github.com/wasmerio/wasmer/pull/833) Add doc example of using ImportObject's new `maybe_with_namespace` method - [#832](https://github.com/wasmerio/wasmer/pull/832) Delete unused runtime ABI - [#809](https://github.com/wasmerio/wasmer/pull/809) Fix bugs leading to panics in `LocalBacking`. +- [#831](https://github.com/wasmerio/wasmer/pull/831) Add support for atomic operations, excluding wait and notify, to singlepass. - [#822](https://github.com/wasmerio/wasmer/pull/822) Update Cranelift fork version to `0.43.1` - [#829](https://github.com/wasmerio/wasmer/pull/829) Fix deps on `make bench-*` commands; benchmarks don't compile other backends now - [#807](https://github.com/wasmerio/wasmer/pull/807) Implement Send for `Instance`, breaking change on `ImportObject`, remove method `get_namespace` replaced with `with_namespace` and `maybe_with_namespace` From 11c5e0d71d91979fee465c4cd58176da711e0fa3 Mon Sep 17 00:00:00 2001 From: Nick Lewycky Date: Wed, 25 Sep 2019 10:29:00 -0700 Subject: [PATCH 10/13] Make the panics a bit more descriptive. --- lib/singlepass-backend/src/emitter_x64.rs | 62 +++++++++++------------ 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/lib/singlepass-backend/src/emitter_x64.rs b/lib/singlepass-backend/src/emitter_x64.rs index 5544d08c537..590bbd012f7 100644 --- a/lib/singlepass-backend/src/emitter_x64.rs +++ b/lib/singlepass-backend/src/emitter_x64.rs @@ -553,7 +553,7 @@ impl Emitter for Assembler { dynasm!(self ; movq Rx(dst as u8), Rx(src as u8)); } - _ => panic!("MOV {:?} {:?} {:?}", sz, src, dst), + _ => panic!("singlepass can't emit MOV {:?} {:?} {:?}", sz, src, dst), } }) }); @@ -566,7 +566,7 @@ impl Emitter for Assembler { (Size::S64, Location::Memory(src, disp), Location::GPR(dst)) => { dynasm!(self ; lea Rq(dst as u8), [Rq(src as u8) + disp]); } - _ => panic!("LEA {:?} {:?} {:?}", sz, src, dst), + _ => panic!("singlepass can't emit LEA {:?} {:?} {:?}", sz, src, dst), } } fn emit_lea_label(&mut self, label: Self::Label, dst: Location) { @@ -574,7 +574,7 @@ impl Emitter for Assembler { Location::GPR(x) => { dynasm!(self ; lea Rq(x as u8), [=>label]); } - _ => panic!("LEA label={:?} {:?}", label, dst), + _ => panic!("singlepass can't emit LEA label={:?} {:?}", label, dst), } } fn emit_cdq(&mut self) { @@ -585,7 +585,7 @@ impl Emitter for Assembler { } fn emit_xor(&mut self, sz: Size, src: Location, dst: Location) { binop_all_nofp!(xor, self, sz, src, dst, { - panic!("XOR {:?} {:?} {:?}", sz, src, dst) + panic!("singlepass can't emit XOR {:?} {:?} {:?}", sz, src, dst) }); } fn emit_jmp(&mut self, condition: Condition, label: Self::Label) { @@ -608,7 +608,7 @@ impl Emitter for Assembler { match loc { Location::GPR(x) => dynasm!(self ; jmp Rq(x as u8)), Location::Memory(base, disp) => dynasm!(self ; jmp QWORD [Rq(base as u8) + disp]), - _ => panic!("JMP {:?}", loc), + _ => panic!("singlepass can't emit JMP {:?}", loc), } } fn emit_conditional_trap(&mut self, condition: Condition) { @@ -640,7 +640,7 @@ impl Emitter for Assembler { Condition::Equal => dynasm!(self ; sete Rb(dst as u8)), Condition::NotEqual => dynasm!(self ; setne Rb(dst as u8)), Condition::Signed => dynasm!(self ; sets Rb(dst as u8)), - _ => panic!("SET {:?} {:?}", condition, dst), + _ => panic!("singlepass can't emit SET {:?} {:?}", condition, dst), } } fn emit_push(&mut self, sz: Size, src: Location) { @@ -650,7 +650,7 @@ impl Emitter for Assembler { (Size::S64, Location::Memory(src, disp)) => { dynasm!(self ; push QWORD [Rq(src as u8) + disp]) } - _ => panic!("PUSH {:?} {:?}", sz, src), + _ => panic!("singlepass can't emit PUSH {:?} {:?}", sz, src), } } fn emit_pop(&mut self, sz: Size, dst: Location) { @@ -659,22 +659,22 @@ impl Emitter for Assembler { (Size::S64, Location::Memory(dst, disp)) => { dynasm!(self ; pop QWORD [Rq(dst as u8) + disp]) } - _ => panic!("POP {:?} {:?}", sz, dst), + _ => panic!("singlepass can't emit POP {:?} {:?}", sz, dst), } } fn emit_cmp(&mut self, sz: Size, left: Location, right: Location) { binop_all_nofp!(cmp, self, sz, left, right, { - panic!("CMP {:?} {:?} {:?}", sz, left, right); + panic!("singlepass can't emit CMP {:?} {:?} {:?}", sz, left, right); }); } fn emit_add(&mut self, sz: Size, src: Location, dst: Location) { binop_all_nofp!(add, self, sz, src, dst, { - panic!("ADD {:?} {:?} {:?}", sz, src, dst) + panic!("singlepass can't emit ADD {:?} {:?} {:?}", sz, src, dst) }); } fn emit_sub(&mut self, sz: Size, src: Location, dst: Location) { binop_all_nofp!(sub, self, sz, src, dst, { - panic!("SUB {:?} {:?} {:?}", sz, src, dst) + panic!("singlepass can't emit SUB {:?} {:?} {:?}", sz, src, dst) }); } fn emit_neg(&mut self, sz: Size, value: Location) { @@ -695,13 +695,13 @@ impl Emitter for Assembler { (Size::S64, Location::Memory(value, disp)) => { dynasm!(self ; neg [Rq(value as u8) + disp]) } - _ => panic!("NEG {:?} {:?}", sz, value), + _ => panic!("singlepass can't emit NEG {:?} {:?}", sz, value), } } fn emit_imul(&mut self, sz: Size, src: Location, dst: Location) { binop_gpr_gpr!(imul, self, sz, src, dst, { binop_mem_gpr!(imul, self, sz, src, dst, { - panic!("IMUL {:?} {:?} {:?}", sz, src, dst) + panic!("singlepass can't emit IMUL {:?} {:?} {:?}", sz, src, dst) }) }); } @@ -710,67 +710,67 @@ impl Emitter for Assembler { } fn emit_div(&mut self, sz: Size, divisor: Location) { unop_gpr_or_mem!(div, self, sz, divisor, { - panic!("DIV {:?} {:?}", sz, divisor) + panic!("singlepass can't emit DIV {:?} {:?}", sz, divisor) }); } fn emit_idiv(&mut self, sz: Size, divisor: Location) { unop_gpr_or_mem!(idiv, self, sz, divisor, { - panic!("IDIV {:?} {:?}", sz, divisor) + panic!("singlepass can't emit IDIV {:?} {:?}", sz, divisor) }); } fn emit_shl(&mut self, sz: Size, src: Location, dst: Location) { binop_shift!(shl, self, sz, src, dst, { - panic!("SHL {:?} {:?} {:?}", sz, src, dst) + panic!("singlepass can't emit SHL {:?} {:?} {:?}", sz, src, dst) }); } fn emit_shr(&mut self, sz: Size, src: Location, dst: Location) { binop_shift!(shr, self, sz, src, dst, { - panic!("SHR {:?} {:?} {:?}", sz, src, dst) + panic!("singlepass can't emit SHR {:?} {:?} {:?}", sz, src, dst) }); } fn emit_sar(&mut self, sz: Size, src: Location, dst: Location) { binop_shift!(sar, self, sz, src, dst, { - panic!("SAR {:?} {:?} {:?}", sz, src, dst) + panic!("singlepass can't emit SAR {:?} {:?} {:?}", sz, src, dst) }); } fn emit_rol(&mut self, sz: Size, src: Location, dst: Location) { binop_shift!(rol, self, sz, src, dst, { - panic!("ROL {:?} {:?} {:?}", sz, src, dst) + panic!("singlepass can't emit ROL {:?} {:?} {:?}", sz, src, dst) }); } fn emit_ror(&mut self, sz: Size, src: Location, dst: Location) { binop_shift!(ror, self, sz, src, dst, { - panic!("ROR {:?} {:?} {:?}", sz, src, dst) + panic!("singlepass can't emit ROR {:?} {:?} {:?}", sz, src, dst) }); } fn emit_and(&mut self, sz: Size, src: Location, dst: Location) { binop_all_nofp!(and, self, sz, src, dst, { - panic!("AND {:?} {:?} {:?}", sz, src, dst) + panic!("singlepass can't emit AND {:?} {:?} {:?}", sz, src, dst) }); } fn emit_or(&mut self, sz: Size, src: Location, dst: Location) { binop_all_nofp!(or, self, sz, src, dst, { - panic!("OR {:?} {:?} {:?}", sz, src, dst) + panic!("singlepass can't emit OR {:?} {:?} {:?}", sz, src, dst) }); } fn emit_lzcnt(&mut self, sz: Size, src: Location, dst: Location) { binop_gpr_gpr!(lzcnt, self, sz, src, dst, { binop_mem_gpr!(lzcnt, self, sz, src, dst, { - panic!("LZCNT {:?} {:?} {:?}", sz, src, dst) + panic!("singlepass can't emit LZCNT {:?} {:?} {:?}", sz, src, dst) }) }); } fn emit_tzcnt(&mut self, sz: Size, src: Location, dst: Location) { binop_gpr_gpr!(tzcnt, self, sz, src, dst, { binop_mem_gpr!(tzcnt, self, sz, src, dst, { - panic!("TZCNT {:?} {:?} {:?}", sz, src, dst) + panic!("singlepass can't emit TZCNT {:?} {:?} {:?}", sz, src, dst) }) }); } fn emit_popcnt(&mut self, sz: Size, src: Location, dst: Location) { binop_gpr_gpr!(popcnt, self, sz, src, dst, { binop_mem_gpr!(popcnt, self, sz, src, dst, { - panic!("POPCNT {:?} {:?} {:?}", sz, src, dst) + panic!("singlepass can't emit POPCNT {:?} {:?} {:?}", sz, src, dst) }) }); } @@ -800,7 +800,7 @@ impl Emitter for Assembler { (Size::S16, Location::Memory(src, disp), Size::S64, Location::GPR(dst)) => { dynasm!(self ; movzx Rq(dst as u8), WORD [Rq(src as u8) + disp]); } - _ => panic!("MOVZX {:?} {:?} {:?} {:?}", sz_src, src, sz_dst, dst), + _ => panic!("singlepass can't emit MOVZX {:?} {:?} {:?} {:?}", sz_src, src, sz_dst, dst), } } fn emit_movsx(&mut self, sz_src: Size, src: Location, sz_dst: Size, dst: Location) { @@ -835,7 +835,7 @@ impl Emitter for Assembler { (Size::S32, Location::Memory(src, disp), Size::S64, Location::GPR(dst)) => { dynasm!(self ; movsx Rq(dst as u8), DWORD [Rq(src as u8) + disp]); } - _ => panic!("MOVSX {:?} {:?} {:?} {:?}", sz_src, src, sz_dst, dst), + _ => panic!("singlepass can't emit MOVSX {:?} {:?} {:?} {:?}", sz_src, src, sz_dst, dst), } } @@ -877,7 +877,7 @@ impl Emitter for Assembler { (Size::S64, Location::GPR(src), Location::Memory(dst, disp)) => { dynasm!(self ; xchg [Rq(dst as u8) + disp], Rq(src as u8)); } - _ => panic!("XCHG {:?} {:?} {:?}", sz, src, dst), + _ => panic!("singlepass can't emit XCHG {:?} {:?} {:?}", sz, src, dst), } } @@ -895,7 +895,7 @@ impl Emitter for Assembler { (Size::S64, Location::GPR(src), Location::Memory(dst, disp)) => { dynasm!(self ; lock xadd [Rq(dst as u8) + disp], Rq(src as u8)); } - _ => panic!("LOCK XADD {:?} {:?} {:?}", sz, src, dst), + _ => panic!("singlepass can't emit LOCK XADD {:?} {:?} {:?}", sz, src, dst), } } @@ -913,7 +913,7 @@ impl Emitter for Assembler { (Size::S64, Location::GPR(src), Location::Memory(dst, disp)) => { dynasm!(self ; lock cmpxchg [Rq(dst as u8) + disp], Rq(src as u8)); } - _ => panic!("LOCK CMPXCHG {:?} {:?} {:?}", sz, src, dst), + _ => panic!("singlepass can't emit LOCK CMPXCHG {:?} {:?} {:?}", sz, src, dst), } } @@ -1061,7 +1061,7 @@ impl Emitter for Assembler { match loc { Location::GPR(x) => dynasm!(self ; call Rq(x as u8)), Location::Memory(base, disp) => dynasm!(self ; call QWORD [Rq(base as u8) + disp]), - _ => panic!("CALL {:?}", loc), + _ => panic!("singlepass can't emit CALL {:?}", loc), } } From 83b678bc3657db2ca22c29cc9c624d3acf05e69c Mon Sep 17 00:00:00 2001 From: Nick Lewycky Date: Sat, 28 Sep 2019 22:07:20 -0700 Subject: [PATCH 11/13] Give this function a better name. --- lib/singlepass-backend/src/codegen_x64.rs | 16 ++++++++-------- lib/singlepass-backend/src/machine.rs | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/singlepass-backend/src/codegen_x64.rs b/lib/singlepass-backend/src/codegen_x64.rs index 9322531f836..d8debec1087 100644 --- a/lib/singlepass-backend/src/codegen_x64.rs +++ b/lib/singlepass-backend/src/codegen_x64.rs @@ -1603,7 +1603,7 @@ impl X64FunctionCode { ) { assert!(memory_sz <= stack_sz); - let compare = m.reserve_temp_gpr(GPR::RAX); + let compare = m.reserve_unused_temp_gpr(GPR::RAX); let value = if loc == Location::GPR(GPR::R14) { GPR::R13 } else { @@ -6775,7 +6775,7 @@ impl FunctionCodeGenerator for X64FunctionCode { )[0]; self.value_stack.push(ret); - let compare = self.machine.reserve_temp_gpr(GPR::RAX); + let compare = self.machine.reserve_unused_temp_gpr(GPR::RAX); let value = if cmp == Location::GPR(GPR::R14) { if new == Location::GPR(GPR::R13) { GPR::R12 @@ -6824,7 +6824,7 @@ impl FunctionCodeGenerator for X64FunctionCode { )[0]; self.value_stack.push(ret); - let compare = self.machine.reserve_temp_gpr(GPR::RAX); + let compare = self.machine.reserve_unused_temp_gpr(GPR::RAX); let value = if cmp == Location::GPR(GPR::R14) { if new == Location::GPR(GPR::R13) { GPR::R12 @@ -6873,7 +6873,7 @@ impl FunctionCodeGenerator for X64FunctionCode { )[0]; self.value_stack.push(ret); - let compare = self.machine.reserve_temp_gpr(GPR::RAX); + let compare = self.machine.reserve_unused_temp_gpr(GPR::RAX); let value = if cmp == Location::GPR(GPR::R14) { if new == Location::GPR(GPR::R13) { GPR::R12 @@ -6922,7 +6922,7 @@ impl FunctionCodeGenerator for X64FunctionCode { )[0]; self.value_stack.push(ret); - let compare = self.machine.reserve_temp_gpr(GPR::RAX); + let compare = self.machine.reserve_unused_temp_gpr(GPR::RAX); let value = if cmp == Location::GPR(GPR::R14) { if new == Location::GPR(GPR::R13) { GPR::R12 @@ -6971,7 +6971,7 @@ impl FunctionCodeGenerator for X64FunctionCode { )[0]; self.value_stack.push(ret); - let compare = self.machine.reserve_temp_gpr(GPR::RAX); + let compare = self.machine.reserve_unused_temp_gpr(GPR::RAX); let value = if cmp == Location::GPR(GPR::R14) { if new == Location::GPR(GPR::R13) { GPR::R12 @@ -7020,7 +7020,7 @@ impl FunctionCodeGenerator for X64FunctionCode { )[0]; self.value_stack.push(ret); - let compare = self.machine.reserve_temp_gpr(GPR::RAX); + let compare = self.machine.reserve_unused_temp_gpr(GPR::RAX); let value = if cmp == Location::GPR(GPR::R14) { if new == Location::GPR(GPR::R13) { GPR::R12 @@ -7069,7 +7069,7 @@ impl FunctionCodeGenerator for X64FunctionCode { )[0]; self.value_stack.push(ret); - let compare = self.machine.reserve_temp_gpr(GPR::RAX); + let compare = self.machine.reserve_unused_temp_gpr(GPR::RAX); let value = if cmp == Location::GPR(GPR::R14) { if new == Location::GPR(GPR::R13) { GPR::R12 diff --git a/lib/singlepass-backend/src/machine.rs b/lib/singlepass-backend/src/machine.rs index dfeb010dfc0..727b577d73d 100644 --- a/lib/singlepass-backend/src/machine.rs +++ b/lib/singlepass-backend/src/machine.rs @@ -87,7 +87,7 @@ impl Machine { } /// Specify that a given register is in use. - pub fn reserve_temp_gpr(&mut self, gpr: GPR) -> GPR { + pub fn reserve_unused_temp_gpr(&mut self, gpr: GPR) -> GPR { assert!(!self.used_gprs.contains(&gpr)); self.used_gprs.insert(gpr); gpr From 8e63d54fdb962b25ea4153f5b893c7ffd66eddcd Mon Sep 17 00:00:00 2001 From: Nick Lewycky Date: Sat, 28 Sep 2019 22:08:10 -0700 Subject: [PATCH 12/13] cargo fmt --- lib/singlepass-backend/src/emitter_x64.rs | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/lib/singlepass-backend/src/emitter_x64.rs b/lib/singlepass-backend/src/emitter_x64.rs index 590bbd012f7..2e747f7ec9b 100644 --- a/lib/singlepass-backend/src/emitter_x64.rs +++ b/lib/singlepass-backend/src/emitter_x64.rs @@ -800,7 +800,10 @@ impl Emitter for Assembler { (Size::S16, Location::Memory(src, disp), Size::S64, Location::GPR(dst)) => { dynasm!(self ; movzx Rq(dst as u8), WORD [Rq(src as u8) + disp]); } - _ => panic!("singlepass can't emit MOVZX {:?} {:?} {:?} {:?}", sz_src, src, sz_dst, dst), + _ => panic!( + "singlepass can't emit MOVZX {:?} {:?} {:?} {:?}", + sz_src, src, sz_dst, dst + ), } } fn emit_movsx(&mut self, sz_src: Size, src: Location, sz_dst: Size, dst: Location) { @@ -835,7 +838,10 @@ impl Emitter for Assembler { (Size::S32, Location::Memory(src, disp), Size::S64, Location::GPR(dst)) => { dynasm!(self ; movsx Rq(dst as u8), DWORD [Rq(src as u8) + disp]); } - _ => panic!("singlepass can't emit MOVSX {:?} {:?} {:?} {:?}", sz_src, src, sz_dst, dst), + _ => panic!( + "singlepass can't emit MOVSX {:?} {:?} {:?} {:?}", + sz_src, src, sz_dst, dst + ), } } @@ -895,7 +901,10 @@ impl Emitter for Assembler { (Size::S64, Location::GPR(src), Location::Memory(dst, disp)) => { dynasm!(self ; lock xadd [Rq(dst as u8) + disp], Rq(src as u8)); } - _ => panic!("singlepass can't emit LOCK XADD {:?} {:?} {:?}", sz, src, dst), + _ => panic!( + "singlepass can't emit LOCK XADD {:?} {:?} {:?}", + sz, src, dst + ), } } @@ -913,7 +922,10 @@ impl Emitter for Assembler { (Size::S64, Location::GPR(src), Location::Memory(dst, disp)) => { dynasm!(self ; lock cmpxchg [Rq(dst as u8) + disp], Rq(src as u8)); } - _ => panic!("singlepass can't emit LOCK CMPXCHG {:?} {:?} {:?}", sz, src, dst), + _ => panic!( + "singlepass can't emit LOCK CMPXCHG {:?} {:?} {:?}", + sz, src, dst + ), } } From ab76c2357f1c049a6270ea625fc221f61c5c1bc6 Mon Sep 17 00:00:00 2001 From: Nick Lewycky Date: Tue, 1 Oct 2019 14:08:40 -0700 Subject: [PATCH 13/13] Delete dead (commented out) code. NFC. --- lib/singlepass-backend/src/codegen_x64.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/singlepass-backend/src/codegen_x64.rs b/lib/singlepass-backend/src/codegen_x64.rs index d8debec1087..7e3a9ed5f1b 100644 --- a/lib/singlepass-backend/src/codegen_x64.rs +++ b/lib/singlepass-backend/src/codegen_x64.rs @@ -1563,21 +1563,17 @@ impl X64FunctionCode { }; if check_alignment && align != 1 { let tmp_aligncheck = m.acquire_temp_gpr().unwrap(); - //let tmp_mask = m.acquire_temp_gpr().unwrap(); a.emit_mov( Size::S32, Location::GPR(tmp_addr), Location::GPR(tmp_aligncheck), ); - //a.emit_mov(Size::S64, Location::Imm64(align - 1), Location::GPR(tmp_mask)); - //a.emit_and(Size::S64, Location::GPR(tmp_mask), Location::GPR(tmp_aligncheck)); a.emit_and( Size::S64, Location::Imm32(align - 1), Location::GPR(tmp_aligncheck), ); a.emit_conditional_trap(Condition::NotEqual); - //m.release_temp_gpr(tmp_mask); m.release_temp_gpr(tmp_aligncheck); }