From 2021b199fd16916b2edea600b1b355bd393ec029 Mon Sep 17 00:00:00 2001 From: ptitSeb Date: Fri, 3 Dec 2021 13:26:13 +0100 Subject: [PATCH] improvement(compiler) abstraction of logic and math atomic operators --- lib/compiler-singlepass/src/codegen_x64.rs | 2008 +++++++++--------- lib/compiler-singlepass/src/machine.rs | 578 +++++- lib/compiler-singlepass/src/machine_x64.rs | 2180 ++++++++++++++++++-- 3 files changed, 3596 insertions(+), 1170 deletions(-) diff --git a/lib/compiler-singlepass/src/codegen_x64.rs b/lib/compiler-singlepass/src/codegen_x64.rs index b7c1a181b33..cdca4f36d94 100644 --- a/lib/compiler-singlepass/src/codegen_x64.rs +++ b/lib/compiler-singlepass/src/codegen_x64.rs @@ -569,70 +569,6 @@ impl<'a> FuncGen<'a> { Ok(()) } - /// Emits a memory operation. - fn emit_compare_and_swap( - &mut self, - loc: Location, - target: Location, - ret: Location, - memarg: &MemoryImmediate, - value_size: usize, - memory_sz: Size, - stack_sz: Size, - cb: F, - ) -> Result<(), CodegenError> { - if memory_sz > stack_sz { - return Err(CodegenError { - message: "emit_compare_and_swap: memory size > stack size".to_string(), - }); - } - - let compare = self.machine.reserve_unused_temp_gpr(GPR::RAX); - let value = if loc == Location::GPR(GPR::R14) { - GPR::R13 - } else { - GPR::R14 - }; - self.machine - .specific - .assembler - .emit_push(Size::S64, Location::GPR(value)); - - self.machine - .specific - .move_location(stack_sz, loc, Location::GPR(value)); - - let retry = self.machine.specific.assembler.get_label(); - self.machine.specific.emit_label(retry); - - self.memory_op(target, memarg, true, value_size, |this, addr| { - this.machine.specific.load_address( - memory_sz, - Location::GPR(compare), - Location::Memory(addr, 0), - ); - this.machine - .specific - .move_location(stack_sz, Location::GPR(compare), ret); - cb(this, compare, value); - this.machine.specific.assembler.emit_lock_cmpxchg( - memory_sz, - Location::GPR(value), - Location::Memory(addr, 0), - ); - Ok(()) - })?; - - self.machine.specific.jmp_on_different(retry); - - self.machine - .specific - .assembler - .emit_pop(Size::S64, Location::GPR(value)); - self.machine.release_temp_gpr(compare); - Ok(()) - } - pub fn get_state_diff(&mut self) -> usize { if !self.machine.track_state { return std::usize::MAX; @@ -3748,14 +3684,27 @@ impl<'a> FuncGen<'a> { )[0]; self.value_stack.push(ret); - self.memory_op(target, memarg, true, 4, |this, addr| { - this.machine.specific.emit_relaxed_mov( - Size::S32, - Location::Memory(addr, 0), - ret, - ); - Ok(()) - })?; + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; + + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) + } else { + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i32_atomic_load( + target, + memarg, + ret, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); } Operator::I32AtomicLoad8U { ref memarg } => { let target = self.pop_value_released(); @@ -3765,15 +3714,27 @@ impl<'a> FuncGen<'a> { )[0]; self.value_stack.push(ret); - self.memory_op(target, memarg, true, 1, |this, addr| { - this.machine.specific.emit_relaxed_zero_extension( - Size::S8, - Location::Memory(addr, 0), - Size::S32, - ret, - ); - Ok(()) - })?; + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; + + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) + } else { + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i32_atomic_load_8u( + target, + memarg, + ret, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); } Operator::I32AtomicLoad16U { ref memarg } => { let target = self.pop_value_released(); @@ -3783,54 +3744,102 @@ impl<'a> FuncGen<'a> { )[0]; self.value_stack.push(ret); - self.memory_op(target, memarg, true, 2, |this, addr| { - this.machine.specific.emit_relaxed_zero_extension( - Size::S16, - Location::Memory(addr, 0), - Size::S32, - ret, - ); - Ok(()) - })?; + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; + + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) + } else { + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i32_atomic_load_16u( + target, + memarg, + ret, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); } Operator::I32AtomicStore { ref memarg } => { let target_value = self.pop_value_released(); let target_addr = self.pop_value_released(); + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; - self.memory_op(target_addr, memarg, true, 4, |this, addr| { - this.machine.specific.emit_relaxed_atomic_xchg( - Size::S32, - target_value, - Location::Memory(addr, 0), - ); - Ok(()) - })?; + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) + } else { + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i32_atomic_save( + target_value, + memarg, + target_addr, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); } Operator::I32AtomicStore8 { ref memarg } => { let target_value = self.pop_value_released(); let target_addr = self.pop_value_released(); + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; - self.memory_op(target_addr, memarg, true, 1, |this, addr| { - this.machine.specific.emit_relaxed_atomic_xchg( - Size::S8, - target_value, - Location::Memory(addr, 0), - ); - Ok(()) - })?; + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) + } else { + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i32_atomic_save_8( + target_value, + memarg, + target_addr, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); } Operator::I32AtomicStore16 { ref memarg } => { let target_value = self.pop_value_released(); let target_addr = self.pop_value_released(); + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; - self.memory_op(target_addr, memarg, true, 2, |this, addr| { - this.machine.specific.emit_relaxed_atomic_xchg( - Size::S16, - target_value, - Location::Memory(addr, 0), - ); - Ok(()) - })?; + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) + } else { + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i32_atomic_save_16( + target_value, + memarg, + target_addr, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); } Operator::I64AtomicLoad { ref memarg } => { let target = self.pop_value_released(); @@ -3840,14 +3849,27 @@ impl<'a> FuncGen<'a> { )[0]; self.value_stack.push(ret); - self.memory_op(target, memarg, true, 8, |this, addr| { - this.machine.specific.emit_relaxed_mov( - Size::S64, - Location::Memory(addr, 0), - ret, - ); - Ok(()) - })?; + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; + + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) + } else { + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i64_atomic_load( + target, + memarg, + ret, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); } Operator::I64AtomicLoad8U { ref memarg } => { let target = self.pop_value_released(); @@ -3857,15 +3879,27 @@ impl<'a> FuncGen<'a> { )[0]; self.value_stack.push(ret); - self.memory_op(target, memarg, true, 1, |this, addr| { - this.machine.specific.emit_relaxed_zero_extension( - Size::S8, - Location::Memory(addr, 0), - Size::S64, - ret, - ); - Ok(()) - })?; + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; + + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) + } else { + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i64_atomic_load_8u( + target, + memarg, + ret, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); } Operator::I64AtomicLoad16U { ref memarg } => { let target = self.pop_value_released(); @@ -3875,15 +3909,27 @@ impl<'a> FuncGen<'a> { )[0]; self.value_stack.push(ret); - self.memory_op(target, memarg, true, 2, |this, addr| { - this.machine.specific.emit_relaxed_zero_extension( - Size::S16, - Location::Memory(addr, 0), - Size::S64, - ret, - ); - Ok(()) - })?; + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; + + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) + } else { + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i64_atomic_load_16u( + target, + memarg, + ret, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); } Operator::I64AtomicLoad32U { ref memarg } => { let target = self.pop_value_released(); @@ -3893,82 +3939,127 @@ impl<'a> FuncGen<'a> { )[0]; self.value_stack.push(ret); - self.memory_op(target, memarg, true, 4, |this, addr| { - match ret { - Location::GPR(_) => {} - Location::Memory(base, offset) => { - this.machine.specific.move_location( - Size::S32, - Location::Imm32(0), - Location::Memory(base, offset + 4), - ); // clear upper bits - } - _ => { - return Err(CodegenError { - message: "I64AtomicLoad32U ret: unreachable code".to_string(), - }) - } - } - this.machine.specific.emit_relaxed_zero_extension( - Size::S32, - Location::Memory(addr, 0), - Size::S64, - ret, - ); - Ok(()) - })?; + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; + + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) + } else { + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i64_atomic_load_32u( + target, + memarg, + ret, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); } Operator::I64AtomicStore { ref memarg } => { let target_value = self.pop_value_released(); let target_addr = self.pop_value_released(); + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; - self.memory_op(target_addr, memarg, true, 8, |this, addr| { - this.machine.specific.emit_relaxed_atomic_xchg( - Size::S64, - target_value, - Location::Memory(addr, 0), - ); - Ok(()) - })?; + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) + } else { + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i64_atomic_save( + target_value, + memarg, + target_addr, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); } Operator::I64AtomicStore8 { ref memarg } => { let target_value = self.pop_value_released(); let target_addr = self.pop_value_released(); + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; - self.memory_op(target_addr, memarg, true, 1, |this, addr| { - this.machine.specific.emit_relaxed_atomic_xchg( - Size::S8, - target_value, - Location::Memory(addr, 0), - ); - Ok(()) - })?; + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) + } else { + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i64_atomic_save_8( + target_value, + memarg, + target_addr, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); } Operator::I64AtomicStore16 { ref memarg } => { let target_value = self.pop_value_released(); let target_addr = self.pop_value_released(); + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; - self.memory_op(target_addr, memarg, true, 2, |this, addr| { - this.machine.specific.emit_relaxed_atomic_xchg( - Size::S16, - target_value, - Location::Memory(addr, 0), - ); - Ok(()) - })?; + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) + } else { + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i64_atomic_save_16( + target_value, + memarg, + target_addr, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); } Operator::I64AtomicStore32 { ref memarg } => { let target_value = self.pop_value_released(); let target_addr = self.pop_value_released(); + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; - self.memory_op(target_addr, memarg, true, 4, |this, addr| { - this.machine.specific.emit_relaxed_atomic_xchg( - Size::S32, - target_value, - Location::Memory(addr, 0), - ); - Ok(()) - })?; + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) + } else { + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i64_atomic_save_32( + target_value, + memarg, + target_addr, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); } Operator::I32AtomicRmwAdd { ref memarg } => { let loc = self.pop_value_released(); @@ -3979,44 +4070,30 @@ impl<'a> FuncGen<'a> { )[0]; self.value_stack.push(ret); - if self.machine.specific.has_atomic_xadd() { - let value = self.machine.acquire_temp_gpr().unwrap(); - self.machine - .specific - .move_location(Size::S32, loc, Location::GPR(value)); - self.memory_op(target, memarg, true, 4, |this, addr| { - this.machine.specific.emit_atomic_xadd( - Size::S32, - Location::GPR(value), - Location::Memory(addr, 0), - ); - Ok(()) - })?; - self.machine - .specific - .move_location(Size::S32, Location::GPR(value), ret); - self.machine.release_temp_gpr(value); + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; + + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) } else { - self.emit_compare_and_swap( - loc, - target, - ret, - memarg, - 4, - Size::S32, - Size::S32, - |this, src, dst| { - this.machine.specific.location_add( - Size::S32, - Location::GPR(src), - Location::GPR(dst), - false, - ); - }, - )?; - } - } - Operator::I64AtomicRmwAdd { ref memarg } => { + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i32_atomic_add( + loc, + target, + memarg, + ret, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); + } + Operator::I64AtomicRmwAdd { ref memarg } => { let loc = self.pop_value_released(); let target = self.pop_value_released(); let ret = self.machine.acquire_locations( @@ -4025,42 +4102,28 @@ impl<'a> FuncGen<'a> { )[0]; self.value_stack.push(ret); - if self.machine.specific.has_atomic_xadd() { - let value = self.machine.acquire_temp_gpr().unwrap(); - self.machine - .specific - .move_location(Size::S64, loc, Location::GPR(value)); - self.memory_op(target, memarg, true, 8, |this, addr| { - this.machine.specific.emit_atomic_xadd( - Size::S64, - Location::GPR(value), - Location::Memory(addr, 0), - ); - Ok(()) - })?; - self.machine - .specific - .move_location(Size::S64, Location::GPR(value), ret); - self.machine.release_temp_gpr(value); + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; + + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) } else { - self.emit_compare_and_swap( - loc, - target, - ret, - memarg, - 4, - Size::S64, - Size::S64, - |this, src, dst| { - this.machine.specific.location_add( - Size::S64, - Location::GPR(src), - Location::GPR(dst), - false, - ); - }, - )?; - } + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i64_atomic_add( + loc, + target, + memarg, + ret, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); } Operator::I32AtomicRmw8AddU { ref memarg } => { let loc = self.pop_value_released(); @@ -4071,46 +4134,28 @@ impl<'a> FuncGen<'a> { )[0]; self.value_stack.push(ret); - if self.machine.specific.has_atomic_xadd() { - let value = self.machine.acquire_temp_gpr().unwrap(); - self.machine.specific.move_location_extend( - Size::S8, - false, - loc, - Size::S32, - Location::GPR(value), - ); - self.memory_op(target, memarg, true, 1, |this, addr| { - this.machine.specific.emit_atomic_xadd( - Size::S8, - Location::GPR(value), - Location::Memory(addr, 0), - ); - Ok(()) - })?; - self.machine - .specific - .move_location(Size::S32, Location::GPR(value), ret); - self.machine.release_temp_gpr(value); + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; + + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) } else { - self.emit_compare_and_swap( - loc, - target, - ret, - memarg, - 4, - Size::S8, - Size::S32, - |this, src, dst| { - this.machine.specific.location_add( - Size::S8, - Location::GPR(src), - Location::GPR(dst), - false, - ); - }, - )?; - } + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i32_atomic_add_8u( + loc, + target, + memarg, + ret, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); } Operator::I32AtomicRmw16AddU { ref memarg } => { let loc = self.pop_value_released(); @@ -4121,46 +4166,28 @@ impl<'a> FuncGen<'a> { )[0]; self.value_stack.push(ret); - if self.machine.specific.has_atomic_xadd() { - let value = self.machine.acquire_temp_gpr().unwrap(); - self.machine.specific.move_location_extend( - Size::S16, - false, - loc, - Size::S32, - Location::GPR(value), - ); - self.memory_op(target, memarg, true, 2, |this, addr| { - this.machine.specific.emit_atomic_xadd( - Size::S16, - Location::GPR(value), - Location::Memory(addr, 0), - ); - Ok(()) - })?; - self.machine - .specific - .move_location(Size::S32, Location::GPR(value), ret); - self.machine.release_temp_gpr(value); + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; + + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) } else { - self.emit_compare_and_swap( - loc, - target, - ret, - memarg, - 4, - Size::S16, - Size::S32, - |this, src, dst| { - this.machine.specific.location_add( - Size::S16, - Location::GPR(src), - Location::GPR(dst), - false, - ); - }, - )?; - } + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i32_atomic_add_16u( + loc, + target, + memarg, + ret, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); } Operator::I64AtomicRmw8AddU { ref memarg } => { let loc = self.pop_value_released(); @@ -4171,46 +4198,28 @@ impl<'a> FuncGen<'a> { )[0]; self.value_stack.push(ret); - if self.machine.specific.has_atomic_xadd() { - let value = self.machine.acquire_temp_gpr().unwrap(); - self.machine.specific.move_location_extend( - Size::S8, - false, - loc, - Size::S64, - Location::GPR(value), - ); - self.memory_op(target, memarg, true, 1, |this, addr| { - this.machine.specific.emit_atomic_xadd( - Size::S8, - Location::GPR(value), - Location::Memory(addr, 0), - ); - Ok(()) - })?; - self.machine - .specific - .move_location(Size::S64, Location::GPR(value), ret); - self.machine.release_temp_gpr(value); + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; + + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) } else { - self.emit_compare_and_swap( - loc, - target, - ret, - memarg, - 4, - Size::S8, - Size::S64, - |this, src, dst| { - this.machine.specific.location_add( - Size::S8, - Location::GPR(src), - Location::GPR(dst), - false, - ); - }, - )?; - } + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i64_atomic_add_8u( + loc, + target, + memarg, + ret, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); } Operator::I64AtomicRmw16AddU { ref memarg } => { let loc = self.pop_value_released(); @@ -4221,46 +4230,28 @@ impl<'a> FuncGen<'a> { )[0]; self.value_stack.push(ret); - if self.machine.specific.has_atomic_xadd() { - let value = self.machine.acquire_temp_gpr().unwrap(); - self.machine.specific.move_location_extend( - Size::S16, - false, - loc, - Size::S64, - Location::GPR(value), - ); - self.memory_op(target, memarg, true, 2, |this, addr| { - this.machine.specific.emit_atomic_xadd( - Size::S16, - Location::GPR(value), - Location::Memory(addr, 0), - ); - Ok(()) - })?; - self.machine - .specific - .move_location(Size::S64, Location::GPR(value), ret); - self.machine.release_temp_gpr(value); + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; + + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) } else { - self.emit_compare_and_swap( - loc, - target, - ret, - memarg, - 4, - Size::S16, - Size::S64, - |this, src, dst| { - this.machine.specific.location_add( - Size::S16, - Location::GPR(src), - Location::GPR(dst), - false, - ); - }, - )?; - } + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i64_atomic_add_16u( + loc, + target, + memarg, + ret, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); } Operator::I64AtomicRmw32AddU { ref memarg } => { let loc = self.pop_value_released(); @@ -4271,42 +4262,28 @@ impl<'a> FuncGen<'a> { )[0]; self.value_stack.push(ret); - if self.machine.specific.has_atomic_xadd() { - let value = self.machine.acquire_temp_gpr().unwrap(); - self.machine - .specific - .move_location(Size::S32, loc, Location::GPR(value)); - self.memory_op(target, memarg, true, 4, |this, addr| { - this.machine.specific.emit_atomic_xadd( - Size::S32, - Location::GPR(value), - Location::Memory(addr, 0), - ); - Ok(()) - })?; - self.machine - .specific - .move_location(Size::S64, Location::GPR(value), ret); - self.machine.release_temp_gpr(value); + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; + + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) } else { - self.emit_compare_and_swap( - loc, - target, - ret, - memarg, - 4, - Size::S32, - Size::S64, - |this, src, dst| { - this.machine.specific.location_add( - Size::S32, - Location::GPR(src), - Location::GPR(dst), - false, - ); - }, - )?; - } + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i64_atomic_add_32u( + loc, + target, + memarg, + ret, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); } Operator::I32AtomicRmwSub { ref memarg } => { let loc = self.pop_value_released(); @@ -4317,46 +4294,28 @@ impl<'a> FuncGen<'a> { )[0]; self.value_stack.push(ret); - if self.machine.specific.has_atomic_xadd() { - let value = self.machine.acquire_temp_gpr().unwrap(); - self.machine.specific.location_neg( - Size::S32, - false, - loc, - Size::S32, - Location::GPR(value), - ); - self.memory_op(target, memarg, true, 4, |this, addr| { - this.machine.specific.emit_atomic_xadd( - Size::S32, - Location::GPR(value), - Location::Memory(addr, 0), - ); - Ok(()) - })?; - self.machine - .specific - .move_location(Size::S32, Location::GPR(value), ret); - self.machine.release_temp_gpr(value); + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; + + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) } else { - self.emit_compare_and_swap( - loc, - target, - ret, - memarg, - 4, - Size::S32, - Size::S32, - |this, src, dst| { - this.machine.specific.location_sub( - Size::S32, - Location::GPR(src), - Location::GPR(dst), - false, - ); - }, - )?; - } + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i32_atomic_sub( + loc, + target, + memarg, + ret, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); } Operator::I64AtomicRmwSub { ref memarg } => { let loc = self.pop_value_released(); @@ -4367,46 +4326,28 @@ impl<'a> FuncGen<'a> { )[0]; self.value_stack.push(ret); - if self.machine.specific.has_atomic_xadd() { - let value = self.machine.acquire_temp_gpr().unwrap(); - self.machine.specific.location_neg( - Size::S64, - false, - loc, - Size::S64, - Location::GPR(value), - ); - self.memory_op(target, memarg, true, 8, |this, addr| { - this.machine.specific.emit_atomic_xadd( - Size::S64, - Location::GPR(value), - Location::Memory(addr, 0), - ); - Ok(()) - })?; - self.machine - .specific - .move_location(Size::S64, Location::GPR(value), ret); - self.machine.release_temp_gpr(value); + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; + + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) } else { - self.emit_compare_and_swap( - loc, - target, - ret, - memarg, - 4, - Size::S64, - Size::S64, - |this, src, dst| { - this.machine.specific.location_sub( - Size::S64, - Location::GPR(src), - Location::GPR(dst), - false, - ); - }, - )?; - } + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i64_atomic_sub( + loc, + target, + memarg, + ret, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); } Operator::I32AtomicRmw8SubU { ref memarg } => { let loc = self.pop_value_released(); @@ -4417,46 +4358,28 @@ impl<'a> FuncGen<'a> { )[0]; self.value_stack.push(ret); - if self.machine.specific.has_atomic_xadd() { - let value = self.machine.acquire_temp_gpr().unwrap(); - self.machine.specific.location_neg( - Size::S8, - false, - loc, - Size::S32, - Location::GPR(value), - ); - self.memory_op(target, memarg, true, 1, |this, addr| { - this.machine.specific.emit_atomic_xadd( - Size::S8, - Location::GPR(value), - Location::Memory(addr, 0), - ); - Ok(()) - })?; - self.machine - .specific - .move_location(Size::S32, Location::GPR(value), ret); - self.machine.release_temp_gpr(value); + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; + + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) } else { - self.emit_compare_and_swap( - loc, - target, - ret, - memarg, - 4, - Size::S8, - Size::S64, - |this, src, dst| { - this.machine.specific.location_sub( - Size::S8, - Location::GPR(src), - Location::GPR(dst), - false, - ); - }, - )?; - } + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i32_atomic_sub_8u( + loc, + target, + memarg, + ret, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); } Operator::I32AtomicRmw16SubU { ref memarg } => { let loc = self.pop_value_released(); @@ -4467,46 +4390,28 @@ impl<'a> FuncGen<'a> { )[0]; self.value_stack.push(ret); - if self.machine.specific.has_atomic_xadd() { - let value = self.machine.acquire_temp_gpr().unwrap(); - self.machine.specific.location_neg( - Size::S16, - false, - loc, - Size::S32, - Location::GPR(value), - ); - self.memory_op(target, memarg, true, 2, |this, addr| { - this.machine.specific.emit_atomic_xadd( - Size::S16, - Location::GPR(value), - Location::Memory(addr, 0), - ); - Ok(()) - })?; - self.machine - .specific - .move_location(Size::S32, Location::GPR(value), ret); - self.machine.release_temp_gpr(value); + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; + + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) } else { - self.emit_compare_and_swap( - loc, - target, - ret, - memarg, - 4, - Size::S16, - Size::S32, - |this, src, dst| { - this.machine.specific.location_sub( - Size::S16, - Location::GPR(src), - Location::GPR(dst), - false, - ); - }, - )?; - } + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i32_atomic_sub_16u( + loc, + target, + memarg, + ret, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); } Operator::I64AtomicRmw8SubU { ref memarg } => { let loc = self.pop_value_released(); @@ -4517,46 +4422,28 @@ impl<'a> FuncGen<'a> { )[0]; self.value_stack.push(ret); - if self.machine.specific.has_atomic_xadd() { - let value = self.machine.acquire_temp_gpr().unwrap(); - self.machine.specific.location_neg( - Size::S8, - false, - loc, - Size::S64, - Location::GPR(value), - ); - self.memory_op(target, memarg, true, 1, |this, addr| { - this.machine.specific.emit_atomic_xadd( - Size::S8, - Location::GPR(value), - Location::Memory(addr, 0), - ); - Ok(()) - })?; - self.machine - .specific - .move_location(Size::S64, Location::GPR(value), ret); - self.machine.release_temp_gpr(value); + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; + + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) } else { - self.emit_compare_and_swap( - loc, - target, - ret, - memarg, - 4, - Size::S8, - Size::S64, - |this, src, dst| { - this.machine.specific.location_sub( - Size::S8, - Location::GPR(src), - Location::GPR(dst), - false, - ); - }, - )?; - } + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i64_atomic_sub_8u( + loc, + target, + memarg, + ret, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); } Operator::I64AtomicRmw16SubU { ref memarg } => { let loc = self.pop_value_released(); @@ -4567,46 +4454,28 @@ impl<'a> FuncGen<'a> { )[0]; self.value_stack.push(ret); - if self.machine.specific.has_atomic_xadd() { - let value = self.machine.acquire_temp_gpr().unwrap(); - self.machine.specific.location_neg( - Size::S16, - false, - loc, - Size::S64, - Location::GPR(value), - ); - self.memory_op(target, memarg, true, 2, |this, addr| { - this.machine.specific.emit_atomic_xadd( - Size::S16, - Location::GPR(value), - Location::Memory(addr, 0), - ); - Ok(()) - })?; - self.machine - .specific - .move_location(Size::S64, Location::GPR(value), ret); - self.machine.release_temp_gpr(value); + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; + + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) } else { - self.emit_compare_and_swap( - loc, - target, - ret, - memarg, - 4, - Size::S16, - Size::S64, - |this, src, dst| { - this.machine.specific.location_sub( - Size::S16, - Location::GPR(src), - Location::GPR(dst), - false, - ); - }, - )?; - } + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i64_atomic_sub_16u( + loc, + target, + memarg, + ret, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); } Operator::I64AtomicRmw32SubU { ref memarg } => { let loc = self.pop_value_released(); @@ -4617,46 +4486,28 @@ impl<'a> FuncGen<'a> { )[0]; self.value_stack.push(ret); - if self.machine.specific.has_atomic_xadd() { - let value = self.machine.acquire_temp_gpr().unwrap(); - self.machine.specific.location_neg( - Size::S32, - false, - loc, - Size::S32, - Location::GPR(value), - ); - self.memory_op(target, memarg, true, 2, |this, addr| { - this.machine.specific.emit_atomic_xadd( - Size::S32, - Location::GPR(value), - Location::Memory(addr, 0), - ); - Ok(()) - })?; - self.machine - .specific - .move_location(Size::S64, Location::GPR(value), ret); - self.machine.release_temp_gpr(value); + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; + + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) } else { - self.emit_compare_and_swap( - loc, - target, - ret, - memarg, - 4, - Size::S32, - Size::S64, - |this, src, dst| { - this.machine.specific.location_sub( - Size::S32, - Location::GPR(src), - Location::GPR(dst), - false, - ); - }, - )?; - } + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i64_atomic_sub_32u( + loc, + target, + memarg, + ret, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); } Operator::I32AtomicRmwAnd { ref memarg } => { let loc = self.pop_value_released(); @@ -4667,23 +4518,28 @@ impl<'a> FuncGen<'a> { )[0]; self.value_stack.push(ret); - self.emit_compare_and_swap( + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; + + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) + } else { + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i32_atomic_and( loc, target, - ret, memarg, - 4, - Size::S32, - Size::S32, - |this, src, dst| { - this.machine.specific.location_and( - Size::S32, - Location::GPR(src), - Location::GPR(dst), - false, - ); - }, - )?; + ret, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); } Operator::I64AtomicRmwAnd { ref memarg } => { let loc = self.pop_value_released(); @@ -4694,23 +4550,28 @@ impl<'a> FuncGen<'a> { )[0]; self.value_stack.push(ret); - self.emit_compare_and_swap( + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; + + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) + } else { + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i64_atomic_and( loc, target, - ret, memarg, - 8, - Size::S64, - Size::S64, - |this, src, dst| { - this.machine.specific.location_and( - Size::S64, - Location::GPR(src), - Location::GPR(dst), - false, - ); - }, - )?; + ret, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); } Operator::I32AtomicRmw8AndU { ref memarg } => { let loc = self.pop_value_released(); @@ -4721,23 +4582,28 @@ impl<'a> FuncGen<'a> { )[0]; self.value_stack.push(ret); - self.emit_compare_and_swap( + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; + + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) + } else { + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i32_atomic_and_8u( loc, target, - ret, memarg, - 1, - Size::S8, - Size::S32, - |this, src, dst| { - this.machine.specific.location_and( - Size::S32, - Location::GPR(src), - Location::GPR(dst), - false, - ); - }, - )?; + ret, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); } Operator::I32AtomicRmw16AndU { ref memarg } => { let loc = self.pop_value_released(); @@ -4748,23 +4614,28 @@ impl<'a> FuncGen<'a> { )[0]; self.value_stack.push(ret); - self.emit_compare_and_swap( + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; + + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) + } else { + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i32_atomic_and_16u( loc, target, - ret, memarg, - 1, - Size::S16, - Size::S32, - |this, src, dst| { - this.machine.specific.location_and( - Size::S32, - Location::GPR(src), - Location::GPR(dst), - false, - ); - }, - )?; + ret, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); } Operator::I64AtomicRmw8AndU { ref memarg } => { let loc = self.pop_value_released(); @@ -4775,23 +4646,28 @@ impl<'a> FuncGen<'a> { )[0]; self.value_stack.push(ret); - self.emit_compare_and_swap( + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; + + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) + } else { + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i64_atomic_and_8u( loc, target, - ret, memarg, - 1, - Size::S8, - Size::S64, - |this, src, dst| { - this.machine.specific.location_and( - Size::S64, - Location::GPR(src), - Location::GPR(dst), - false, - ); - }, - )?; + ret, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); } Operator::I64AtomicRmw16AndU { ref memarg } => { let loc = self.pop_value_released(); @@ -4802,23 +4678,28 @@ impl<'a> FuncGen<'a> { )[0]; self.value_stack.push(ret); - self.emit_compare_and_swap( + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; + + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) + } else { + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i64_atomic_and_16u( loc, target, - ret, memarg, - 1, - Size::S16, - Size::S64, - |this, src, dst| { - this.machine.specific.location_and( - Size::S64, - Location::GPR(src), - Location::GPR(dst), - false, - ); - }, - )?; + ret, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); } Operator::I64AtomicRmw32AndU { ref memarg } => { let loc = self.pop_value_released(); @@ -4829,23 +4710,28 @@ impl<'a> FuncGen<'a> { )[0]; self.value_stack.push(ret); - self.emit_compare_and_swap( + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; + + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) + } else { + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i64_atomic_and_32u( loc, target, - ret, memarg, - 1, - Size::S32, - Size::S64, - |this, src, dst| { - this.machine.specific.location_and( - Size::S64, - Location::GPR(src), - Location::GPR(dst), - false, - ); - }, - )?; + ret, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); } Operator::I32AtomicRmwOr { ref memarg } => { let loc = self.pop_value_released(); @@ -4856,23 +4742,28 @@ impl<'a> FuncGen<'a> { )[0]; self.value_stack.push(ret); - self.emit_compare_and_swap( + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; + + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) + } else { + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i32_atomic_or( loc, target, - ret, memarg, - 4, - Size::S32, - Size::S32, - |this, src, dst| { - this.machine.specific.location_or( - Size::S32, - Location::GPR(src), - Location::GPR(dst), - false, - ); - }, - )?; + ret, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); } Operator::I64AtomicRmwOr { ref memarg } => { let loc = self.pop_value_released(); @@ -4883,23 +4774,28 @@ impl<'a> FuncGen<'a> { )[0]; self.value_stack.push(ret); - self.emit_compare_and_swap( + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; + + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) + } else { + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i64_atomic_or( loc, target, - ret, memarg, - 8, - Size::S64, - Size::S64, - |this, src, dst| { - this.machine.specific.location_or( - Size::S64, - Location::GPR(src), - Location::GPR(dst), - false, - ); - }, - )?; + ret, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); } Operator::I32AtomicRmw8OrU { ref memarg } => { let loc = self.pop_value_released(); @@ -4910,23 +4806,28 @@ impl<'a> FuncGen<'a> { )[0]; self.value_stack.push(ret); - self.emit_compare_and_swap( + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; + + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) + } else { + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i32_atomic_or_8u( loc, target, - ret, memarg, - 1, - Size::S8, - Size::S32, - |this, src, dst| { - this.machine.specific.location_or( - Size::S32, - Location::GPR(src), - Location::GPR(dst), - false, - ); - }, - )?; + ret, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); } Operator::I32AtomicRmw16OrU { ref memarg } => { let loc = self.pop_value_released(); @@ -4937,23 +4838,28 @@ impl<'a> FuncGen<'a> { )[0]; self.value_stack.push(ret); - self.emit_compare_and_swap( + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; + + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) + } else { + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i32_atomic_or_16u( loc, target, - ret, memarg, - 1, - Size::S16, - Size::S32, - |this, src, dst| { - this.machine.specific.location_or( - Size::S32, - Location::GPR(src), - Location::GPR(dst), - false, - ); - }, - )?; + ret, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); } Operator::I64AtomicRmw8OrU { ref memarg } => { let loc = self.pop_value_released(); @@ -4964,23 +4870,28 @@ impl<'a> FuncGen<'a> { )[0]; self.value_stack.push(ret); - self.emit_compare_and_swap( + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; + + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) + } else { + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i64_atomic_or_8u( loc, target, - ret, memarg, - 1, - Size::S8, - Size::S64, - |this, src, dst| { - this.machine.specific.location_or( - Size::S64, - Location::GPR(src), - Location::GPR(dst), - false, - ); - }, - )?; + ret, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); } Operator::I64AtomicRmw16OrU { ref memarg } => { let loc = self.pop_value_released(); @@ -4991,23 +4902,28 @@ impl<'a> FuncGen<'a> { )[0]; self.value_stack.push(ret); - self.emit_compare_and_swap( + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; + + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) + } else { + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i64_atomic_or_16u( loc, target, - ret, memarg, - 1, - Size::S16, - Size::S64, - |this, src, dst| { - this.machine.specific.location_or( - Size::S64, - Location::GPR(src), - Location::GPR(dst), - false, - ); - }, - )?; + ret, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); } Operator::I64AtomicRmw32OrU { ref memarg } => { let loc = self.pop_value_released(); @@ -5018,23 +4934,28 @@ impl<'a> FuncGen<'a> { )[0]; self.value_stack.push(ret); - self.emit_compare_and_swap( + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; + + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) + } else { + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i64_atomic_or_32u( loc, target, - ret, memarg, - 1, - Size::S32, - Size::S64, - |this, src, dst| { - this.machine.specific.location_or( - Size::S64, - Location::GPR(src), - Location::GPR(dst), - false, - ); - }, - )?; + ret, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); } Operator::I32AtomicRmwXor { ref memarg } => { let loc = self.pop_value_released(); @@ -5045,23 +4966,28 @@ impl<'a> FuncGen<'a> { )[0]; self.value_stack.push(ret); - self.emit_compare_and_swap( + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; + + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) + } else { + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i32_atomic_xor( loc, target, - ret, memarg, - 4, - Size::S32, - Size::S32, - |this, src, dst| { - this.machine.specific.location_xor( - Size::S32, - Location::GPR(src), - Location::GPR(dst), - false, - ); - }, - )?; + ret, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); } Operator::I64AtomicRmwXor { ref memarg } => { let loc = self.pop_value_released(); @@ -5072,23 +4998,28 @@ impl<'a> FuncGen<'a> { )[0]; self.value_stack.push(ret); - self.emit_compare_and_swap( + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; + + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) + } else { + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i64_atomic_xor( loc, target, - ret, memarg, - 8, - Size::S64, - Size::S64, - |this, src, dst| { - this.machine.specific.location_xor( - Size::S64, - Location::GPR(src), - Location::GPR(dst), - false, - ); - }, - )?; + ret, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); } Operator::I32AtomicRmw8XorU { ref memarg } => { let loc = self.pop_value_released(); @@ -5099,23 +5030,28 @@ impl<'a> FuncGen<'a> { )[0]; self.value_stack.push(ret); - self.emit_compare_and_swap( + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; + + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) + } else { + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i32_atomic_xor_8u( loc, target, - ret, memarg, - 1, - Size::S8, - Size::S32, - |this, src, dst| { - this.machine.specific.location_xor( - Size::S32, - Location::GPR(src), - Location::GPR(dst), - false, - ); - }, - )?; + ret, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); } Operator::I32AtomicRmw16XorU { ref memarg } => { let loc = self.pop_value_released(); @@ -5126,23 +5062,28 @@ impl<'a> FuncGen<'a> { )[0]; self.value_stack.push(ret); - self.emit_compare_and_swap( + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; + + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) + } else { + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i32_atomic_xor_16u( loc, target, - ret, memarg, - 1, - Size::S16, - Size::S32, - |this, src, dst| { - this.machine.specific.location_xor( - Size::S32, - Location::GPR(src), - Location::GPR(dst), - false, - ); - }, - )?; + ret, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); } Operator::I64AtomicRmw8XorU { ref memarg } => { let loc = self.pop_value_released(); @@ -5153,23 +5094,28 @@ impl<'a> FuncGen<'a> { )[0]; self.value_stack.push(ret); - self.emit_compare_and_swap( + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; + + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) + } else { + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i64_atomic_xor_8u( loc, target, - ret, memarg, - 1, - Size::S8, - Size::S64, - |this, src, dst| { - this.machine.specific.location_xor( - Size::S64, - Location::GPR(src), - Location::GPR(dst), - false, - ); - }, - )?; + ret, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); } Operator::I64AtomicRmw16XorU { ref memarg } => { let loc = self.pop_value_released(); @@ -5180,23 +5126,28 @@ impl<'a> FuncGen<'a> { )[0]; self.value_stack.push(ret); - self.emit_compare_and_swap( + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; + + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) + } else { + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i64_atomic_xor_16u( loc, target, - ret, memarg, - 1, - Size::S16, - Size::S64, - |this, src, dst| { - this.machine.specific.location_xor( - Size::S64, - Location::GPR(src), - Location::GPR(dst), - false, - ); - }, - )?; + ret, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); } Operator::I64AtomicRmw32XorU { ref memarg } => { let loc = self.pop_value_released(); @@ -5207,23 +5158,28 @@ impl<'a> FuncGen<'a> { )[0]; self.value_stack.push(ret); - self.emit_compare_and_swap( + let need_check = match self.memory_styles[MemoryIndex::new(0)] { + MemoryStyle::Static { .. } => false, + MemoryStyle::Dynamic { .. } => true, + }; + + let offset = if self.module.num_imported_memories != 0 { + self.vmoffsets + .vmctx_vmmemory_import_definition(MemoryIndex::new(0)) + } else { + self.vmoffsets + .vmctx_vmmemory_definition(LocalMemoryIndex::new(0)) + }; + self.machine.specific.i64_atomic_xor_32u( loc, target, - ret, memarg, - 1, - Size::S32, - Size::S64, - |this, src, dst| { - this.machine.specific.location_xor( - Size::S64, - Location::GPR(src), - Location::GPR(dst), - false, - ); - }, - )?; + ret, + need_check, + self.module.num_imported_memories != 0, + offset as i32, + self.special_labels.heap_access_oob, + ); } Operator::I32AtomicRmwXchg { ref memarg } => { let loc = self.pop_value_released(); diff --git a/lib/compiler-singlepass/src/machine.rs b/lib/compiler-singlepass/src/machine.rs index d667bf089c3..04107b77668 100644 --- a/lib/compiler-singlepass/src/machine.rs +++ b/lib/compiler-singlepass/src/machine.rs @@ -491,6 +491,252 @@ pub trait MachineSpecific { fn i32_rol(&mut self, loc_a: Location, loc_b: Location, ret: Location); /// i32 Roll Right fn i32_ror(&mut self, loc_a: Location, loc_b: Location, ret: Location); + /// i32 atomic load + fn i32_atomic_load( + &mut self, + addr: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i32 atomic load of an unsigned 8bits + fn i32_atomic_load_8u( + &mut self, + addr: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i32 atomic load of an unsigned 16bits + fn i32_atomic_load_16u( + &mut self, + addr: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i32 atomic save + fn i32_atomic_save( + &mut self, + value: Location, + memarg: &MemoryImmediate, + addr: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i32 atomic save of a the lower 8bits + fn i32_atomic_save_8( + &mut self, + value: Location, + memarg: &MemoryImmediate, + addr: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i32 atomic save of a the lower 16bits + fn i32_atomic_save_16( + &mut self, + value: Location, + memarg: &MemoryImmediate, + addr: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i32 atomic Add with i32 + fn i32_atomic_add( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i32 atomic Add with unsigned 8bits + fn i32_atomic_add_8u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i32 atomic Add with unsigned 16bits + fn i32_atomic_add_16u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i32 atomic Sub with i32 + fn i32_atomic_sub( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i32 atomic Sub with unsigned 8bits + fn i32_atomic_sub_8u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i32 atomic Sub with unsigned 16bits + fn i32_atomic_sub_16u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i32 atomic And with i32 + fn i32_atomic_and( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i32 atomic And with unsigned 8bits + fn i32_atomic_and_8u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i32 atomic And with unsigned 16bits + fn i32_atomic_and_16u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i32 atomic Or with i32 + fn i32_atomic_or( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i32 atomic Or with unsigned 8bits + fn i32_atomic_or_8u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i32 atomic Or with unsigned 16bits + fn i32_atomic_or_16u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i32 atomic Xor with i32 + fn i32_atomic_xor( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i32 atomic Xor with unsigned 8bits + fn i32_atomic_xor_8u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i32 atomic Xor with unsigned 16bits + fn i32_atomic_xor_16u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); /// emit a move function address to GPR ready for call, using appropriate relocation fn move_with_reloc( @@ -608,6 +854,334 @@ pub trait MachineSpecific { fn i64_rol(&mut self, loc_a: Location, loc_b: Location, ret: Location); /// i64 Roll Right fn i64_ror(&mut self, loc_a: Location, loc_b: Location, ret: Location); + /// i64 atomic load + fn i64_atomic_load( + &mut self, + addr: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i64 atomic load from unsigned 8bits + fn i64_atomic_load_8u( + &mut self, + addr: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i64 atomic load from unsigned 16bits + fn i64_atomic_load_16u( + &mut self, + addr: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i64 atomic load from unsigned 32bits + fn i64_atomic_load_32u( + &mut self, + addr: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i64 atomic save + fn i64_atomic_save( + &mut self, + value: Location, + memarg: &MemoryImmediate, + addr: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i64 atomic save of a the lower 8bits + fn i64_atomic_save_8( + &mut self, + value: Location, + memarg: &MemoryImmediate, + addr: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i64 atomic save of a the lower 16bits + fn i64_atomic_save_16( + &mut self, + value: Location, + memarg: &MemoryImmediate, + addr: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i64 atomic save of a the lower 32bits + fn i64_atomic_save_32( + &mut self, + value: Location, + memarg: &MemoryImmediate, + addr: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i64 atomic Add with i64 + fn i64_atomic_add( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i64 atomic Add with unsigned 8bits + fn i64_atomic_add_8u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i64 atomic Add with unsigned 16bits + fn i64_atomic_add_16u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i64 atomic Add with unsigned 32bits + fn i64_atomic_add_32u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i64 atomic Sub with i64 + fn i64_atomic_sub( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i64 atomic Sub with unsigned 8bits + fn i64_atomic_sub_8u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i64 atomic Sub with unsigned 16bits + fn i64_atomic_sub_16u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i64 atomic Sub with unsigned 32bits + fn i64_atomic_sub_32u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i64 atomic And with i64 + fn i64_atomic_and( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i64 atomic And with unsigned 8bits + fn i64_atomic_and_8u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i64 atomic And with unsigned 16bits + fn i64_atomic_and_16u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i64 atomic And with unsigned 32bits + fn i64_atomic_and_32u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i64 atomic Or with i64 + fn i64_atomic_or( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i64 atomic Or with unsigned 8bits + fn i64_atomic_or_8u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i64 atomic Or with unsigned 16bits + fn i64_atomic_or_16u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i64 atomic Or with unsigned 32bits + fn i64_atomic_or_32u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i64 atomic Xor with i64 + fn i64_atomic_xor( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i64 atomic Xor with unsigned 8bits + fn i64_atomic_xor_8u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i64 atomic Xor with unsigned 16bits + fn i64_atomic_xor_16u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); + /// i64 atomic Xor with unsigned 32bits + fn i64_atomic_xor_32u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ); /// Convert a F64 from I64, signed or unsigned fn convert_f64_i64(&mut self, loc: Location, signed: bool, ret: Location); @@ -790,10 +1364,6 @@ impl, C: CombinedRegister> Machine R { - self.specific.reserve_unused_temp_gpr(gpr) - } /// Reserve the gpr needed for a cmpxchg operation (if any) pub fn reserve_cmpxchg_temp_gpr(&mut self) { self.specific.reserve_cmpxchg_temp_gpr(); diff --git a/lib/compiler-singlepass/src/machine_x64.rs b/lib/compiler-singlepass/src/machine_x64.rs index 04b281d4eec..c10ef79fd50 100644 --- a/lib/compiler-singlepass/src/machine_x64.rs +++ b/lib/compiler-singlepass/src/machine_x64.rs @@ -327,6 +327,179 @@ impl MachineX86_64 { f(&mut self.assembler, Size::S32, Location::GPR(GPR::RCX), ret); } + fn memory_op( + &mut self, + addr: Location, + memarg: &MemoryImmediate, + check_alignment: bool, + value_size: usize, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + cb: F, + ) { + let tmp_addr = self.pick_temp_gpr().unwrap(); + + // Reusing `tmp_addr` for temporary indirection here, since it's not used before the last reference to `{base,bound}_loc`. + let (base_loc, bound_loc) = if imported_memories { + // Imported memories require one level of indirection. + self.move_location( + Size::S64, + Location::Memory(Machine::get_vmctx_reg(), offset), + Location::GPR(tmp_addr), + ); + (Location::Memory(tmp_addr, 0), Location::Memory(tmp_addr, 8)) + } else { + ( + Location::Memory(Machine::get_vmctx_reg(), offset), + Location::Memory(Machine::get_vmctx_reg(), offset + 8), + ) + }; + + let tmp_base = self.pick_temp_gpr().unwrap(); + self.reserve_gpr(tmp_base); + let tmp_bound = self.pick_temp_gpr().unwrap(); + self.reserve_gpr(tmp_bound); + + // Load base into temporary register. + self.move_location(Size::S64, base_loc, Location::GPR(tmp_base)); + + // Load bound into temporary register, if needed. + if need_check { + self.move_location(Size::S64, bound_loc, Location::GPR(tmp_bound)); + + // Wasm -> Effective. + // Assuming we never underflow - should always be true on Linux/macOS and Windows >=8, + // since the first page from 0x0 to 0x1000 is not accepted by mmap. + + // This `lea` calculates the upper bound allowed for the beginning of the word. + // Since the upper bound of the memory is (exclusively) `tmp_bound + tmp_base`, + // the maximum allowed beginning of word is (inclusively) + // `tmp_bound + tmp_base - value_size`. + self.location_address( + Size::S64, + Location::Memory2(tmp_bound, tmp_base, Multiplier::One, -(value_size as i32)), + Location::GPR(tmp_bound), + ); + } + + // Load effective address. + // `base_loc` and `bound_loc` becomes INVALID after this line, because `tmp_addr` + // might be reused. + self.move_location(Size::S32, addr, Location::GPR(tmp_addr)); + + // Add offset to memory address. + if memarg.offset != 0 { + self.location_add( + Size::S32, + Location::Imm32(memarg.offset), + Location::GPR(tmp_addr), + true, + ); + + // Trap if offset calculation overflowed. + self.jmp_on_overflow(heap_access_oob); + } + + // Wasm linear memory -> real memory + self.location_add( + Size::S64, + Location::GPR(tmp_base), + Location::GPR(tmp_addr), + false, + ); + + if need_check { + // Trap if the end address of the requested area is above that of the linear memory. + self.location_cmp(Size::S64, Location::GPR(tmp_bound), Location::GPR(tmp_addr)); + + // `tmp_bound` is inclusive. So trap only if `tmp_addr > tmp_bound`. + self.jmp_on_above(heap_access_oob); + } + + self.release_gpr(tmp_bound); + self.release_gpr(tmp_base); + + let align = memarg.align; + if check_alignment && align != 1 { + self.location_test( + Size::S32, + Location::Imm32((align - 1).into()), + Location::GPR(tmp_addr), + ); + self.jmp_on_different(heap_access_oob); + } + + let begin = self.get_offset().0; + + cb(self, tmp_addr); + + let end = self.get_offset().0; + self.release_gpr(tmp_addr); + + self.mark_address_range_with_trap_code(TrapCode::HeapAccessOutOfBounds, begin, end); + } + + fn emit_compare_and_swap( + &mut self, + loc: Location, + target: Location, + ret: Location, + memarg: &MemoryImmediate, + value_size: usize, + memory_sz: Size, + stack_sz: Size, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + cb: F, + ) { + if memory_sz > stack_sz { + unreachable!(); + } + + let compare = self.reserve_unused_temp_gpr(GPR::RAX); + let value = if loc == Location::GPR(GPR::R14) { + GPR::R13 + } else { + GPR::R14 + }; + self.assembler.emit_push(Size::S64, Location::GPR(value)); + + self.move_location(stack_sz, loc, Location::GPR(value)); + + let retry = self.assembler.get_label(); + self.emit_label(retry); + + self.memory_op( + target, + memarg, + true, + value_size, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + this.load_address(memory_sz, Location::GPR(compare), Location::Memory(addr, 0)); + this.move_location(stack_sz, Location::GPR(compare), ret); + cb(this, compare, value); + this.assembler.emit_lock_cmpxchg( + memory_sz, + Location::GPR(value), + Location::Memory(addr, 0), + ); + }, + ); + + self.jmp_on_different(retry); + + self.assembler.emit_pop(Size::S64, Location::GPR(value)); + self.release_gpr(compare); + } + // Checks for underflow/overflow/nan. fn emit_f32_int_conv_check( &mut self, @@ -2636,176 +2809,909 @@ impl MachineSpecific for MachineX86_64 { fn i32_ror(&mut self, loc_a: Location, loc_b: Location, ret: Location) { self.emit_shift_i32(Assembler::emit_ror, loc_a, loc_b, ret); } - - fn move_with_reloc( + fn i32_atomic_load( &mut self, - reloc_target: RelocationTarget, - relocations: &mut Vec, + addr: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, ) { - let reloc_at = self.assembler.get_offset().0 + self.assembler.arch_mov64_imm_offset(); - - relocations.push(Relocation { - kind: RelocationKind::Abs8, - reloc_target, - offset: reloc_at as u32, - addend: 0, - }); - - // RAX is preserved on entry to `emit_call_native` callback. - // The Imm64 value is relocated by the JIT linker. - self.assembler.emit_mov( - Size::S64, - Location::Imm64(std::u64::MAX), - Location::GPR(GPR::RAX), + self.memory_op( + addr, + memarg, + true, + 4, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + this.emit_relaxed_mov(Size::S32, Location::Memory(addr, 0), ret); + }, ); } - - fn emit_binop_add64(&mut self, loc_a: Location, loc_b: Location, ret: Location) { - self.emit_binop_i64(Assembler::emit_add, loc_a, loc_b, ret); - } - fn emit_binop_sub64(&mut self, loc_a: Location, loc_b: Location, ret: Location) { - self.emit_binop_i64(Assembler::emit_sub, loc_a, loc_b, ret); - } - fn emit_binop_mul64(&mut self, loc_a: Location, loc_b: Location, ret: Location) { - self.emit_binop_i64(Assembler::emit_imul, loc_a, loc_b, ret); - } - fn emit_binop_udiv64( + fn i32_atomic_load_8u( &mut self, - loc_a: Location, - loc_b: Location, + addr: Location, + memarg: &MemoryImmediate, ret: Location, - integer_division_by_zero: Label, - ) -> usize { - // We assume that RAX and RDX are temporary registers here. - self.assembler - .emit_mov(Size::S64, loc_a, Location::GPR(GPR::RAX)); - self.assembler - .emit_xor(Size::S64, Location::GPR(GPR::RDX), Location::GPR(GPR::RDX)); - let offset = self.emit_relaxed_xdiv( - Assembler::emit_div, - Size::S64, - loc_b, - integer_division_by_zero, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.memory_op( + addr, + memarg, + true, + 1, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + this.emit_relaxed_zero_extension( + Size::S8, + Location::Memory(addr, 0), + Size::S32, + ret, + ); + }, ); - self.assembler - .emit_mov(Size::S64, Location::GPR(GPR::RAX), ret); - offset } - fn emit_binop_sdiv64( + fn i32_atomic_load_16u( &mut self, - loc_a: Location, - loc_b: Location, + addr: Location, + memarg: &MemoryImmediate, ret: Location, - integer_division_by_zero: Label, - ) -> usize { - // We assume that RAX and RDX are temporary registers here. - self.assembler - .emit_mov(Size::S64, loc_a, Location::GPR(GPR::RAX)); - self.assembler.emit_cqo(); - let offset = self.emit_relaxed_xdiv( - Assembler::emit_idiv, - Size::S64, - loc_b, - integer_division_by_zero, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.memory_op( + addr, + memarg, + true, + 2, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + this.emit_relaxed_zero_extension( + Size::S16, + Location::Memory(addr, 0), + Size::S32, + ret, + ); + }, ); - self.assembler - .emit_mov(Size::S64, Location::GPR(GPR::RAX), ret); - offset } - fn emit_binop_urem64( + fn i32_atomic_save( &mut self, - loc_a: Location, - loc_b: Location, - ret: Location, - integer_division_by_zero: Label, - ) -> usize { - // We assume that RAX and RDX are temporary registers here. - self.assembler - .emit_mov(Size::S64, loc_a, Location::GPR(GPR::RAX)); - self.assembler - .emit_xor(Size::S64, Location::GPR(GPR::RDX), Location::GPR(GPR::RDX)); - let offset = self.emit_relaxed_xdiv( - Assembler::emit_div, - Size::S64, - loc_b, - integer_division_by_zero, + value: Location, + memarg: &MemoryImmediate, + target_addr: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.memory_op( + target_addr, + memarg, + true, + 4, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + this.emit_relaxed_atomic_xchg(Size::S32, value, Location::Memory(addr, 0)); + }, ); - self.assembler - .emit_mov(Size::S64, Location::GPR(GPR::RDX), ret); - offset } - fn emit_binop_srem64( + fn i32_atomic_save_8( &mut self, - loc_a: Location, - loc_b: Location, - ret: Location, - integer_division_by_zero: Label, - ) -> usize { - // We assume that RAX and RDX are temporary registers here. - let normal_path = self.assembler.get_label(); - let end = self.assembler.get_label(); - - self.emit_relaxed_cmp(Size::S64, Location::Imm64(0x8000000000000000u64), loc_a); - self.assembler.emit_jmp(Condition::NotEqual, normal_path); - self.emit_relaxed_cmp(Size::S64, Location::Imm64(0xffffffffffffffffu64), loc_b); - self.assembler.emit_jmp(Condition::NotEqual, normal_path); - self.move_location(Size::S64, Location::Imm64(0), ret); - self.assembler.emit_jmp(Condition::None, end); - - self.emit_label(normal_path); - self.assembler - .emit_mov(Size::S64, loc_a, Location::GPR(GPR::RAX)); - self.assembler.emit_cqo(); - let offset = self.emit_relaxed_xdiv( - Assembler::emit_idiv, - Size::S64, - loc_b, - integer_division_by_zero, + value: Location, + memarg: &MemoryImmediate, + target_addr: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.memory_op( + target_addr, + memarg, + true, + 1, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + this.emit_relaxed_atomic_xchg(Size::S8, value, Location::Memory(addr, 0)); + }, ); - self.assembler - .emit_mov(Size::S64, Location::GPR(GPR::RDX), ret); - - self.emit_label(end); - offset } - fn emit_binop_and64(&mut self, loc_a: Location, loc_b: Location, ret: Location) { - self.emit_binop_i64(Assembler::emit_and, loc_a, loc_b, ret); + fn i32_atomic_save_16( + &mut self, + value: Location, + memarg: &MemoryImmediate, + target_addr: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.memory_op( + target_addr, + memarg, + true, + 2, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + this.emit_relaxed_atomic_xchg(Size::S16, value, Location::Memory(addr, 0)); + }, + ); } - fn emit_binop_or64(&mut self, loc_a: Location, loc_b: Location, ret: Location) { - self.emit_binop_i64(Assembler::emit_or, loc_a, loc_b, ret); + // i32 atomic Add with i32 + fn i32_atomic_add( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + if self.has_atomic_xadd() { + let value = self.acquire_temp_gpr().unwrap(); + self.move_location(Size::S32, loc, Location::GPR(value)); + self.memory_op( + target, + memarg, + true, + 4, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + this.emit_atomic_xadd( + Size::S32, + Location::GPR(value), + Location::Memory(addr, 0), + ); + }, + ); + self.move_location(Size::S32, Location::GPR(value), ret); + self.release_gpr(value); + } else { + self.emit_compare_and_swap( + loc, + target, + ret, + memarg, + 4, + Size::S32, + Size::S32, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, src, dst| { + this.location_add(Size::S32, Location::GPR(src), Location::GPR(dst), false); + }, + ); + } } - fn emit_binop_xor64(&mut self, loc_a: Location, loc_b: Location, ret: Location) { - self.emit_binop_i64(Assembler::emit_xor, loc_a, loc_b, ret); + // i32 atomic Add with u8 + fn i32_atomic_add_8u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + if self.has_atomic_xadd() { + let value = self.acquire_temp_gpr().unwrap(); + self.move_location_extend(Size::S8, false, loc, Size::S32, Location::GPR(value)); + self.memory_op( + target, + memarg, + true, + 1, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + this.emit_atomic_xadd( + Size::S8, + Location::GPR(value), + Location::Memory(addr, 0), + ); + }, + ); + self.move_location(Size::S32, Location::GPR(value), ret); + self.release_gpr(value); + } else { + self.emit_compare_and_swap( + loc, + target, + ret, + memarg, + 4, + Size::S8, + Size::S32, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, src, dst| { + this.location_add(Size::S8, Location::GPR(src), Location::GPR(dst), false); + }, + ); + } } - fn i64_cmp_ge_s(&mut self, loc_a: Location, loc_b: Location, ret: Location) { - self.emit_cmpop_i64_dynamic_b(Condition::GreaterEqual, loc_a, loc_b, ret); + // i32 atomic Add with u16 + fn i32_atomic_add_16u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + if self.has_atomic_xadd() { + let value = self.acquire_temp_gpr().unwrap(); + self.move_location_extend(Size::S16, false, loc, Size::S32, Location::GPR(value)); + self.memory_op( + target, + memarg, + true, + 2, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + this.emit_atomic_xadd( + Size::S16, + Location::GPR(value), + Location::Memory(addr, 0), + ); + }, + ); + self.move_location(Size::S32, Location::GPR(value), ret); + self.release_gpr(value); + } else { + self.emit_compare_and_swap( + loc, + target, + ret, + memarg, + 4, + Size::S16, + Size::S32, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, src, dst| { + this.location_add(Size::S16, Location::GPR(src), Location::GPR(dst), false); + }, + ); + } } - fn i64_cmp_gt_s(&mut self, loc_a: Location, loc_b: Location, ret: Location) { - self.emit_cmpop_i64_dynamic_b(Condition::Greater, loc_a, loc_b, ret); + // i32 atomic Sub with i32 + fn i32_atomic_sub( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + if self.has_atomic_xadd() { + let value = self.acquire_temp_gpr().unwrap(); + self.location_neg(Size::S32, false, loc, Size::S32, Location::GPR(value)); + self.memory_op( + target, + memarg, + true, + 4, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + this.emit_atomic_xadd( + Size::S32, + Location::GPR(value), + Location::Memory(addr, 0), + ); + }, + ); + self.move_location(Size::S32, Location::GPR(value), ret); + self.release_gpr(value); + } else { + self.emit_compare_and_swap( + loc, + target, + ret, + memarg, + 4, + Size::S32, + Size::S32, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, src, dst| { + this.location_sub(Size::S32, Location::GPR(src), Location::GPR(dst), false); + }, + ); + } } - fn i64_cmp_le_s(&mut self, loc_a: Location, loc_b: Location, ret: Location) { - self.emit_cmpop_i64_dynamic_b(Condition::LessEqual, loc_a, loc_b, ret); + // i32 atomic Sub with u8 + fn i32_atomic_sub_8u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + if self.has_atomic_xadd() { + let value = self.acquire_temp_gpr().unwrap(); + self.location_neg(Size::S8, false, loc, Size::S32, Location::GPR(value)); + self.memory_op( + target, + memarg, + true, + 1, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + this.emit_atomic_xadd( + Size::S8, + Location::GPR(value), + Location::Memory(addr, 0), + ); + }, + ); + self.move_location(Size::S32, Location::GPR(value), ret); + self.release_gpr(value); + } else { + self.emit_compare_and_swap( + loc, + target, + ret, + memarg, + 4, + Size::S8, + Size::S32, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, src, dst| { + this.location_sub(Size::S8, Location::GPR(src), Location::GPR(dst), false); + }, + ); + } } - fn i64_cmp_lt_s(&mut self, loc_a: Location, loc_b: Location, ret: Location) { - self.emit_cmpop_i64_dynamic_b(Condition::Less, loc_a, loc_b, ret); + // i32 atomic Sub with u16 + fn i32_atomic_sub_16u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + if self.has_atomic_xadd() { + let value = self.acquire_temp_gpr().unwrap(); + self.location_neg(Size::S16, false, loc, Size::S32, Location::GPR(value)); + self.memory_op( + target, + memarg, + true, + 2, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + this.emit_atomic_xadd( + Size::S16, + Location::GPR(value), + Location::Memory(addr, 0), + ); + }, + ); + self.move_location(Size::S32, Location::GPR(value), ret); + self.release_gpr(value); + } else { + self.emit_compare_and_swap( + loc, + target, + ret, + memarg, + 4, + Size::S16, + Size::S32, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, src, dst| { + this.location_sub(Size::S16, Location::GPR(src), Location::GPR(dst), false); + }, + ); + } } - fn i64_cmp_ge_u(&mut self, loc_a: Location, loc_b: Location, ret: Location) { - self.emit_cmpop_i64_dynamic_b(Condition::AboveEqual, loc_a, loc_b, ret); + // i32 atomic And with i32 + fn i32_atomic_and( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.emit_compare_and_swap( + loc, + target, + ret, + memarg, + 4, + Size::S32, + Size::S32, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, src, dst| { + this.location_and(Size::S32, Location::GPR(src), Location::GPR(dst), false); + }, + ); } - fn i64_cmp_gt_u(&mut self, loc_a: Location, loc_b: Location, ret: Location) { - self.emit_cmpop_i64_dynamic_b(Condition::Above, loc_a, loc_b, ret); + // i32 atomic And with u8 + fn i32_atomic_and_8u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.emit_compare_and_swap( + loc, + target, + ret, + memarg, + 1, + Size::S8, + Size::S32, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, src, dst| { + this.location_and(Size::S32, Location::GPR(src), Location::GPR(dst), false); + }, + ); } - fn i64_cmp_le_u(&mut self, loc_a: Location, loc_b: Location, ret: Location) { - self.emit_cmpop_i64_dynamic_b(Condition::BelowEqual, loc_a, loc_b, ret); + // i32 atomic And with u16 + fn i32_atomic_and_16u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.emit_compare_and_swap( + loc, + target, + ret, + memarg, + 2, + Size::S16, + Size::S32, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, src, dst| { + this.location_and(Size::S32, Location::GPR(src), Location::GPR(dst), false); + }, + ); } - fn i64_cmp_lt_u(&mut self, loc_a: Location, loc_b: Location, ret: Location) { - self.emit_cmpop_i64_dynamic_b(Condition::Below, loc_a, loc_b, ret); + // i32 atomic Or with i32 + fn i32_atomic_or( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.emit_compare_and_swap( + loc, + target, + ret, + memarg, + 4, + Size::S32, + Size::S32, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, src, dst| { + this.location_or(Size::S32, Location::GPR(src), Location::GPR(dst), false); + }, + ); } - fn i64_cmp_ne(&mut self, loc_a: Location, loc_b: Location, ret: Location) { - self.emit_cmpop_i64_dynamic_b(Condition::NotEqual, loc_a, loc_b, ret); + // i32 atomic Or with u8 + fn i32_atomic_or_8u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.emit_compare_and_swap( + loc, + target, + ret, + memarg, + 1, + Size::S8, + Size::S32, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, src, dst| { + this.location_or(Size::S32, Location::GPR(src), Location::GPR(dst), false); + }, + ); } - fn i64_cmp_eq(&mut self, loc_a: Location, loc_b: Location, ret: Location) { - self.emit_cmpop_i64_dynamic_b(Condition::Equal, loc_a, loc_b, ret); + // i32 atomic Or with u16 + fn i32_atomic_or_16u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.emit_compare_and_swap( + loc, + target, + ret, + memarg, + 2, + Size::S16, + Size::S32, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, src, dst| { + this.location_or(Size::S32, Location::GPR(src), Location::GPR(dst), false); + }, + ); + } + // i32 atomic Xor with i32 + fn i32_atomic_xor( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.emit_compare_and_swap( + loc, + target, + ret, + memarg, + 4, + Size::S32, + Size::S32, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, src, dst| { + this.location_xor(Size::S32, Location::GPR(src), Location::GPR(dst), false); + }, + ); + } + // i32 atomic Xor with u8 + fn i32_atomic_xor_8u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.emit_compare_and_swap( + loc, + target, + ret, + memarg, + 1, + Size::S8, + Size::S32, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, src, dst| { + this.location_xor(Size::S32, Location::GPR(src), Location::GPR(dst), false); + }, + ); + } + // i32 atomic Xor with u16 + fn i32_atomic_xor_16u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.emit_compare_and_swap( + loc, + target, + ret, + memarg, + 2, + Size::S16, + Size::S32, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, src, dst| { + this.location_xor(Size::S32, Location::GPR(src), Location::GPR(dst), false); + }, + ); + } + + fn move_with_reloc( + &mut self, + reloc_target: RelocationTarget, + relocations: &mut Vec, + ) { + let reloc_at = self.assembler.get_offset().0 + self.assembler.arch_mov64_imm_offset(); + + relocations.push(Relocation { + kind: RelocationKind::Abs8, + reloc_target, + offset: reloc_at as u32, + addend: 0, + }); + + // RAX is preserved on entry to `emit_call_native` callback. + // The Imm64 value is relocated by the JIT linker. + self.assembler.emit_mov( + Size::S64, + Location::Imm64(std::u64::MAX), + Location::GPR(GPR::RAX), + ); + } + + fn emit_binop_add64(&mut self, loc_a: Location, loc_b: Location, ret: Location) { + self.emit_binop_i64(Assembler::emit_add, loc_a, loc_b, ret); + } + fn emit_binop_sub64(&mut self, loc_a: Location, loc_b: Location, ret: Location) { + self.emit_binop_i64(Assembler::emit_sub, loc_a, loc_b, ret); + } + fn emit_binop_mul64(&mut self, loc_a: Location, loc_b: Location, ret: Location) { + self.emit_binop_i64(Assembler::emit_imul, loc_a, loc_b, ret); + } + fn emit_binop_udiv64( + &mut self, + loc_a: Location, + loc_b: Location, + ret: Location, + integer_division_by_zero: Label, + ) -> usize { + // We assume that RAX and RDX are temporary registers here. + self.assembler + .emit_mov(Size::S64, loc_a, Location::GPR(GPR::RAX)); + self.assembler + .emit_xor(Size::S64, Location::GPR(GPR::RDX), Location::GPR(GPR::RDX)); + let offset = self.emit_relaxed_xdiv( + Assembler::emit_div, + Size::S64, + loc_b, + integer_division_by_zero, + ); + self.assembler + .emit_mov(Size::S64, Location::GPR(GPR::RAX), ret); + offset + } + fn emit_binop_sdiv64( + &mut self, + loc_a: Location, + loc_b: Location, + ret: Location, + integer_division_by_zero: Label, + ) -> usize { + // We assume that RAX and RDX are temporary registers here. + self.assembler + .emit_mov(Size::S64, loc_a, Location::GPR(GPR::RAX)); + self.assembler.emit_cqo(); + let offset = self.emit_relaxed_xdiv( + Assembler::emit_idiv, + Size::S64, + loc_b, + integer_division_by_zero, + ); + self.assembler + .emit_mov(Size::S64, Location::GPR(GPR::RAX), ret); + offset + } + fn emit_binop_urem64( + &mut self, + loc_a: Location, + loc_b: Location, + ret: Location, + integer_division_by_zero: Label, + ) -> usize { + // We assume that RAX and RDX are temporary registers here. + self.assembler + .emit_mov(Size::S64, loc_a, Location::GPR(GPR::RAX)); + self.assembler + .emit_xor(Size::S64, Location::GPR(GPR::RDX), Location::GPR(GPR::RDX)); + let offset = self.emit_relaxed_xdiv( + Assembler::emit_div, + Size::S64, + loc_b, + integer_division_by_zero, + ); + self.assembler + .emit_mov(Size::S64, Location::GPR(GPR::RDX), ret); + offset + } + fn emit_binop_srem64( + &mut self, + loc_a: Location, + loc_b: Location, + ret: Location, + integer_division_by_zero: Label, + ) -> usize { + // We assume that RAX and RDX are temporary registers here. + let normal_path = self.assembler.get_label(); + let end = self.assembler.get_label(); + + self.emit_relaxed_cmp(Size::S64, Location::Imm64(0x8000000000000000u64), loc_a); + self.assembler.emit_jmp(Condition::NotEqual, normal_path); + self.emit_relaxed_cmp(Size::S64, Location::Imm64(0xffffffffffffffffu64), loc_b); + self.assembler.emit_jmp(Condition::NotEqual, normal_path); + self.move_location(Size::S64, Location::Imm64(0), ret); + self.assembler.emit_jmp(Condition::None, end); + + self.emit_label(normal_path); + self.assembler + .emit_mov(Size::S64, loc_a, Location::GPR(GPR::RAX)); + self.assembler.emit_cqo(); + let offset = self.emit_relaxed_xdiv( + Assembler::emit_idiv, + Size::S64, + loc_b, + integer_division_by_zero, + ); + self.assembler + .emit_mov(Size::S64, Location::GPR(GPR::RDX), ret); + + self.emit_label(end); + offset + } + fn emit_binop_and64(&mut self, loc_a: Location, loc_b: Location, ret: Location) { + self.emit_binop_i64(Assembler::emit_and, loc_a, loc_b, ret); + } + fn emit_binop_or64(&mut self, loc_a: Location, loc_b: Location, ret: Location) { + self.emit_binop_i64(Assembler::emit_or, loc_a, loc_b, ret); + } + fn emit_binop_xor64(&mut self, loc_a: Location, loc_b: Location, ret: Location) { + self.emit_binop_i64(Assembler::emit_xor, loc_a, loc_b, ret); + } + fn i64_cmp_ge_s(&mut self, loc_a: Location, loc_b: Location, ret: Location) { + self.emit_cmpop_i64_dynamic_b(Condition::GreaterEqual, loc_a, loc_b, ret); + } + fn i64_cmp_gt_s(&mut self, loc_a: Location, loc_b: Location, ret: Location) { + self.emit_cmpop_i64_dynamic_b(Condition::Greater, loc_a, loc_b, ret); + } + fn i64_cmp_le_s(&mut self, loc_a: Location, loc_b: Location, ret: Location) { + self.emit_cmpop_i64_dynamic_b(Condition::LessEqual, loc_a, loc_b, ret); + } + fn i64_cmp_lt_s(&mut self, loc_a: Location, loc_b: Location, ret: Location) { + self.emit_cmpop_i64_dynamic_b(Condition::Less, loc_a, loc_b, ret); + } + fn i64_cmp_ge_u(&mut self, loc_a: Location, loc_b: Location, ret: Location) { + self.emit_cmpop_i64_dynamic_b(Condition::AboveEqual, loc_a, loc_b, ret); + } + fn i64_cmp_gt_u(&mut self, loc_a: Location, loc_b: Location, ret: Location) { + self.emit_cmpop_i64_dynamic_b(Condition::Above, loc_a, loc_b, ret); + } + fn i64_cmp_le_u(&mut self, loc_a: Location, loc_b: Location, ret: Location) { + self.emit_cmpop_i64_dynamic_b(Condition::BelowEqual, loc_a, loc_b, ret); + } + fn i64_cmp_lt_u(&mut self, loc_a: Location, loc_b: Location, ret: Location) { + self.emit_cmpop_i64_dynamic_b(Condition::Below, loc_a, loc_b, ret); + } + fn i64_cmp_ne(&mut self, loc_a: Location, loc_b: Location, ret: Location) { + self.emit_cmpop_i64_dynamic_b(Condition::NotEqual, loc_a, loc_b, ret); + } + fn i64_cmp_eq(&mut self, loc_a: Location, loc_b: Location, ret: Location) { + self.emit_cmpop_i64_dynamic_b(Condition::Equal, loc_a, loc_b, ret); } fn i64_clz(&mut self, loc: Location, ret: Location) { let src = match loc { @@ -2955,6 +3861,1000 @@ impl MachineSpecific for MachineX86_64 { fn i64_ror(&mut self, loc_a: Location, loc_b: Location, ret: Location) { self.emit_shift_i64(Assembler::emit_ror, loc_a, loc_b, ret); } + fn i64_atomic_load( + &mut self, + addr: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.memory_op( + addr, + memarg, + true, + 8, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + this.emit_relaxed_mov(Size::S64, Location::Memory(addr, 0), ret); + }, + ); + } + fn i64_atomic_load_8u( + &mut self, + addr: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.memory_op( + addr, + memarg, + true, + 1, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + this.emit_relaxed_zero_extension( + Size::S8, + Location::Memory(addr, 0), + Size::S64, + ret, + ); + }, + ); + } + fn i64_atomic_load_16u( + &mut self, + addr: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.memory_op( + addr, + memarg, + true, + 2, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + this.emit_relaxed_zero_extension( + Size::S16, + Location::Memory(addr, 0), + Size::S64, + ret, + ); + }, + ); + } + fn i64_atomic_load_32u( + &mut self, + addr: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.memory_op( + addr, + memarg, + true, + 4, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + match ret { + Location::GPR(_) => {} + Location::Memory(base, offset) => { + this.move_location( + Size::S32, + Location::Imm32(0), + Location::Memory(base, offset + 4), + ); // clear upper bits + } + _ => { + unreachable!(); + } + } + this.emit_relaxed_zero_extension( + Size::S32, + Location::Memory(addr, 0), + Size::S64, + ret, + ); + }, + ); + } + fn i64_atomic_save( + &mut self, + value: Location, + memarg: &MemoryImmediate, + target_addr: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.memory_op( + target_addr, + memarg, + true, + 8, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + this.emit_relaxed_atomic_xchg(Size::S64, value, Location::Memory(addr, 0)); + }, + ); + } + fn i64_atomic_save_8( + &mut self, + value: Location, + memarg: &MemoryImmediate, + target_addr: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.memory_op( + target_addr, + memarg, + true, + 1, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + this.emit_relaxed_atomic_xchg(Size::S8, value, Location::Memory(addr, 0)); + }, + ); + } + fn i64_atomic_save_16( + &mut self, + value: Location, + memarg: &MemoryImmediate, + target_addr: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.memory_op( + target_addr, + memarg, + true, + 2, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + this.emit_relaxed_atomic_xchg(Size::S16, value, Location::Memory(addr, 0)); + }, + ); + } + fn i64_atomic_save_32( + &mut self, + value: Location, + memarg: &MemoryImmediate, + target_addr: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.memory_op( + target_addr, + memarg, + true, + 2, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + this.emit_relaxed_atomic_xchg(Size::S32, value, Location::Memory(addr, 0)); + }, + ); + } + // i64 atomic Add with i64 + fn i64_atomic_add( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + if self.has_atomic_xadd() { + let value = self.acquire_temp_gpr().unwrap(); + self.move_location(Size::S64, loc, Location::GPR(value)); + self.memory_op( + target, + memarg, + true, + 8, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + this.emit_atomic_xadd( + Size::S32, + Location::GPR(value), + Location::Memory(addr, 0), + ); + }, + ); + self.move_location(Size::S64, Location::GPR(value), ret); + self.release_gpr(value); + } else { + self.emit_compare_and_swap( + loc, + target, + ret, + memarg, + 4, + Size::S64, + Size::S64, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, src, dst| { + this.location_add(Size::S64, Location::GPR(src), Location::GPR(dst), false); + }, + ); + } + } + // i64 atomic Add with u8 + fn i64_atomic_add_8u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + if self.has_atomic_xadd() { + let value = self.acquire_temp_gpr().unwrap(); + self.move_location_extend(Size::S8, false, loc, Size::S64, Location::GPR(value)); + self.memory_op( + target, + memarg, + true, + 1, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + this.emit_atomic_xadd( + Size::S8, + Location::GPR(value), + Location::Memory(addr, 0), + ); + }, + ); + self.move_location(Size::S64, Location::GPR(value), ret); + self.release_gpr(value); + } else { + self.emit_compare_and_swap( + loc, + target, + ret, + memarg, + 4, + Size::S8, + Size::S64, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, src, dst| { + this.location_add(Size::S8, Location::GPR(src), Location::GPR(dst), false); + }, + ); + } + } + // i64 atomic Add with u16 + fn i64_atomic_add_16u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + if self.has_atomic_xadd() { + let value = self.acquire_temp_gpr().unwrap(); + self.move_location_extend(Size::S16, false, loc, Size::S64, Location::GPR(value)); + self.memory_op( + target, + memarg, + true, + 2, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + this.emit_atomic_xadd( + Size::S16, + Location::GPR(value), + Location::Memory(addr, 0), + ); + }, + ); + self.move_location(Size::S64, Location::GPR(value), ret); + self.release_gpr(value); + } else { + self.emit_compare_and_swap( + loc, + target, + ret, + memarg, + 4, + Size::S16, + Size::S64, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, src, dst| { + this.location_add(Size::S16, Location::GPR(src), Location::GPR(dst), false); + }, + ); + } + } + // i64 atomic Add with u32 + fn i64_atomic_add_32u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + if self.has_atomic_xadd() { + let value = self.acquire_temp_gpr().unwrap(); + self.move_location_extend(Size::S32, false, loc, Size::S64, Location::GPR(value)); + self.memory_op( + target, + memarg, + true, + 4, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + this.emit_atomic_xadd( + Size::S32, + Location::GPR(value), + Location::Memory(addr, 0), + ); + }, + ); + self.move_location(Size::S64, Location::GPR(value), ret); + self.release_gpr(value); + } else { + self.emit_compare_and_swap( + loc, + target, + ret, + memarg, + 4, + Size::S32, + Size::S64, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, src, dst| { + this.location_add(Size::S32, Location::GPR(src), Location::GPR(dst), false); + }, + ); + } + } + // i64 atomic Sub with i64 + fn i64_atomic_sub( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + if self.has_atomic_xadd() { + let value = self.acquire_temp_gpr().unwrap(); + self.move_location(Size::S64, loc, Location::GPR(value)); + let value = self.acquire_temp_gpr().unwrap(); + self.location_neg(Size::S64, false, loc, Size::S64, Location::GPR(value)); + self.memory_op( + target, + memarg, + true, + 8, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + this.emit_atomic_xadd( + Size::S64, + Location::GPR(value), + Location::Memory(addr, 0), + ); + }, + ); + self.move_location(Size::S64, Location::GPR(value), ret); + self.release_gpr(value); + } else { + self.emit_compare_and_swap( + loc, + target, + ret, + memarg, + 4, + Size::S64, + Size::S64, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, src, dst| { + this.location_sub(Size::S64, Location::GPR(src), Location::GPR(dst), false); + }, + ); + } + } + // i64 atomic Sub with u8 + fn i64_atomic_sub_8u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + if self.has_atomic_xadd() { + let value = self.acquire_temp_gpr().unwrap(); + self.location_neg(Size::S8, false, loc, Size::S64, Location::GPR(value)); + self.memory_op( + target, + memarg, + true, + 1, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + this.emit_atomic_xadd( + Size::S8, + Location::GPR(value), + Location::Memory(addr, 0), + ); + }, + ); + self.move_location(Size::S64, Location::GPR(value), ret); + self.release_gpr(value); + } else { + self.emit_compare_and_swap( + loc, + target, + ret, + memarg, + 4, + Size::S8, + Size::S64, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, src, dst| { + this.location_sub(Size::S8, Location::GPR(src), Location::GPR(dst), false); + }, + ); + } + } + // i64 atomic Sub with u16 + fn i64_atomic_sub_16u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + if self.has_atomic_xadd() { + let value = self.acquire_temp_gpr().unwrap(); + self.location_neg(Size::S16, false, loc, Size::S64, Location::GPR(value)); + self.memory_op( + target, + memarg, + true, + 2, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + this.emit_atomic_xadd( + Size::S16, + Location::GPR(value), + Location::Memory(addr, 0), + ); + }, + ); + self.move_location(Size::S64, Location::GPR(value), ret); + self.release_gpr(value); + } else { + self.emit_compare_and_swap( + loc, + target, + ret, + memarg, + 4, + Size::S16, + Size::S64, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, src, dst| { + this.location_sub(Size::S16, Location::GPR(src), Location::GPR(dst), false); + }, + ); + } + } + // i64 atomic Sub with u32 + fn i64_atomic_sub_32u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + if self.has_atomic_xadd() { + let value = self.acquire_temp_gpr().unwrap(); + self.location_neg(Size::S32, false, loc, Size::S64, Location::GPR(value)); + self.memory_op( + target, + memarg, + true, + 4, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, addr| { + this.emit_atomic_xadd( + Size::S32, + Location::GPR(value), + Location::Memory(addr, 0), + ); + }, + ); + self.move_location(Size::S64, Location::GPR(value), ret); + self.release_gpr(value); + } else { + self.emit_compare_and_swap( + loc, + target, + ret, + memarg, + 4, + Size::S32, + Size::S64, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, src, dst| { + this.location_sub(Size::S32, Location::GPR(src), Location::GPR(dst), false); + }, + ); + } + } + // i64 atomic And with i64 + fn i64_atomic_and( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.emit_compare_and_swap( + loc, + target, + ret, + memarg, + 8, + Size::S64, + Size::S64, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, src, dst| { + this.location_and(Size::S64, Location::GPR(src), Location::GPR(dst), false); + }, + ); + } + // i64 atomic And with u8 + fn i64_atomic_and_8u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.emit_compare_and_swap( + loc, + target, + ret, + memarg, + 1, + Size::S8, + Size::S64, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, src, dst| { + this.location_and(Size::S64, Location::GPR(src), Location::GPR(dst), false); + }, + ); + } + // i64 atomic And with u16 + fn i64_atomic_and_16u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.emit_compare_and_swap( + loc, + target, + ret, + memarg, + 2, + Size::S16, + Size::S64, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, src, dst| { + this.location_and(Size::S64, Location::GPR(src), Location::GPR(dst), false); + }, + ); + } + // i64 atomic And with u32 + fn i64_atomic_and_32u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.emit_compare_and_swap( + loc, + target, + ret, + memarg, + 4, + Size::S32, + Size::S64, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, src, dst| { + this.location_and(Size::S64, Location::GPR(src), Location::GPR(dst), false); + }, + ); + } + // i64 atomic Or with i64 + fn i64_atomic_or( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.emit_compare_and_swap( + loc, + target, + ret, + memarg, + 8, + Size::S64, + Size::S64, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, src, dst| { + this.location_or(Size::S64, Location::GPR(src), Location::GPR(dst), false); + }, + ); + } + // i64 atomic Or with u8 + fn i64_atomic_or_8u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.emit_compare_and_swap( + loc, + target, + ret, + memarg, + 1, + Size::S8, + Size::S64, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, src, dst| { + this.location_or(Size::S64, Location::GPR(src), Location::GPR(dst), false); + }, + ); + } + // i64 atomic Or with u16 + fn i64_atomic_or_16u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.emit_compare_and_swap( + loc, + target, + ret, + memarg, + 2, + Size::S16, + Size::S64, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, src, dst| { + this.location_or(Size::S64, Location::GPR(src), Location::GPR(dst), false); + }, + ); + } + // i64 atomic Or with u32 + fn i64_atomic_or_32u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.emit_compare_and_swap( + loc, + target, + ret, + memarg, + 4, + Size::S32, + Size::S64, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, src, dst| { + this.location_or(Size::S64, Location::GPR(src), Location::GPR(dst), false); + }, + ); + } + // i64 atomic xor with i64 + fn i64_atomic_xor( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.emit_compare_and_swap( + loc, + target, + ret, + memarg, + 8, + Size::S64, + Size::S64, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, src, dst| { + this.location_xor(Size::S64, Location::GPR(src), Location::GPR(dst), false); + }, + ); + } + // i64 atomic xor with u8 + fn i64_atomic_xor_8u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.emit_compare_and_swap( + loc, + target, + ret, + memarg, + 1, + Size::S8, + Size::S64, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, src, dst| { + this.location_xor(Size::S64, Location::GPR(src), Location::GPR(dst), false); + }, + ); + } + // i64 atomic xor with u16 + fn i64_atomic_xor_16u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.emit_compare_and_swap( + loc, + target, + ret, + memarg, + 2, + Size::S16, + Size::S64, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, src, dst| { + this.location_xor(Size::S64, Location::GPR(src), Location::GPR(dst), false); + }, + ); + } + // i64 atomic xor with u32 + fn i64_atomic_xor_32u( + &mut self, + loc: Location, + target: Location, + memarg: &MemoryImmediate, + ret: Location, + need_check: bool, + imported_memories: bool, + offset: i32, + heap_access_oob: Label, + ) { + self.emit_compare_and_swap( + loc, + target, + ret, + memarg, + 4, + Size::S32, + Size::S64, + need_check, + imported_memories, + offset, + heap_access_oob, + |this, src, dst| { + this.location_xor(Size::S64, Location::GPR(src), Location::GPR(dst), false); + }, + ); + } fn convert_f64_i64(&mut self, loc: Location, signed: bool, ret: Location) { if self.assembler.arch_has_fconverti() {