diff --git a/.gitmodules b/.gitmodules index 053cf94a7e..f3dfa63e10 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "crates/wasmi/benches/rust"] path = crates/wasmi/benches/rust url = https://github.com/wasmi-labs/rust-benchmarks +[submodule "crates/wasmi/tests/spec/wasmi-tests"] + path = crates/wasmi/tests/spec/wasmi-tests + url = https://github.com/wasmi-labs/wasmi-tests diff --git a/crates/c_api/src/memory.rs b/crates/c_api/src/memory.rs index 48528df379..3e85c1ec17 100644 --- a/crates/c_api/src/memory.rs +++ b/crates/c_api/src/memory.rs @@ -132,7 +132,11 @@ pub unsafe extern "C" fn wasm_memory_data_size(m: &wasm_memory_t) -> usize { #[cfg_attr(not(feature = "prefix-symbols"), no_mangle)] #[cfg_attr(feature = "prefix-symbols", wasmi_c_api_macros::prefix_symbol)] pub unsafe extern "C" fn wasm_memory_size(m: &wasm_memory_t) -> wasm_memory_pages_t { - m.memory().size(m.inner.store.context()) + let size = m.memory().size(m.inner.store.context()); + let Ok(size32) = u32::try_from(size) else { + panic!("linear memory pages out of bounds: {size}") + }; + size32 } /// Grows the [`wasm_memory_t`] by `delta` Wasm pages. @@ -153,5 +157,5 @@ pub unsafe extern "C" fn wasm_memory_grow( ) -> bool { let memory = m.memory(); let mut store = m.inner.store.context_mut(); - memory.grow(&mut store, delta).is_ok() + memory.grow(&mut store, u64::from(delta)).is_ok() } diff --git a/crates/c_api/src/table.rs b/crates/c_api/src/table.rs index 4f1904d5b3..6549b8345e 100644 --- a/crates/c_api/src/table.rs +++ b/crates/c_api/src/table.rs @@ -110,7 +110,7 @@ pub unsafe extern "C" fn wasm_table_get( index: wasm_table_size_t, ) -> Option> { let table = t.table(); - let value = table.get(t.inner.store.context_mut(), index)?; + let value = table.get(t.inner.store.context_mut(), u64::from(index))?; let wasm_ref = match value { wasmi::Val::FuncRef(r) => WasmRef::Func(r), wasmi::Val::ExternRef(r) => WasmRef::Extern(r), @@ -137,7 +137,11 @@ pub unsafe extern "C" fn wasm_table_set( let table = t.table(); let new_value = option_wasm_ref_t_to_ref(new_value, &table.ty(t.inner.store.context())); table - .set(t.inner.store.context_mut(), index, new_value.into()) + .set( + t.inner.store.context_mut(), + u64::from(index), + new_value.into(), + ) .is_ok() } @@ -154,7 +158,8 @@ pub unsafe extern "C" fn wasm_table_set( pub unsafe extern "C" fn wasm_table_size(t: &wasm_table_t) -> wasm_table_size_t { let table = t.table(); let store = t.inner.store.context(); - table.size(store) + let size = table.size(store); + u32::try_from(size).unwrap() } /// Grows the number of cells of the [`wasm_table_t`] by `delta`. /// @@ -176,7 +181,7 @@ pub unsafe extern "C" fn wasm_table_grow( let table = t.table(); let init = option_wasm_ref_t_to_ref(init, &table.ty(t.inner.store.context())); table - .grow(t.inner.store.context_mut(), delta, init.into()) + .grow(t.inner.store.context_mut(), u64::from(delta), init.into()) .is_ok() } diff --git a/crates/c_api/src/types/memory.rs b/crates/c_api/src/types/memory.rs index 63d2a0db0b..3232fdf062 100644 --- a/crates/c_api/src/types/memory.rs +++ b/crates/c_api/src/types/memory.rs @@ -50,8 +50,12 @@ impl wasm_memorytype_t { impl CMemoryType { pub(crate) fn new(ty: MemoryType) -> CMemoryType { - let min: u32 = ty.minimum(); - let max: u32 = ty.maximum().unwrap_or(u32::MAX); + let Ok(min) = u32::try_from(ty.minimum()) else { + panic!("memory minimum size out of bounds: {}", ty.minimum()) + }; + let Ok(max) = u32::try_from(ty.maximum().unwrap_or(u64::from(u32::MAX))) else { + panic!("memory maximum size out of bounds: {:?}", ty.maximum()) + }; CMemoryType { ty, limits: wasm_limits_t { min, max }, diff --git a/crates/c_api/src/types/table.rs b/crates/c_api/src/types/table.rs index b3c0bb1e2a..9f1511c757 100644 --- a/crates/c_api/src/types/table.rs +++ b/crates/c_api/src/types/table.rs @@ -55,8 +55,11 @@ impl CTableType { ty, element: wasm_valtype_t { ty: ty.element() }, limits: wasm_limits_t { - min: ty.minimum(), - max: ty.maximum().unwrap_or(u32::MAX), + min: u32::try_from(ty.minimum()).unwrap(), + max: ty + .maximum() + .map(|max| u32::try_from(max).unwrap()) + .unwrap_or(u32::MAX), }, } } diff --git a/crates/core/src/untyped.rs b/crates/core/src/untyped.rs index d8a5e89640..f3062aef0c 100644 --- a/crates/core/src/untyped.rs +++ b/crates/core/src/untyped.rs @@ -136,11 +136,11 @@ macro_rules! op { /// # Errors /// /// If the resulting effective address overflows. -fn effective_address(address: u32, offset: u32) -> Result { - offset - .checked_add(address) - .map(|address| address as usize) - .ok_or(TrapCode::MemoryOutOfBounds) +fn effective_address(ptr: u64, offset: u64) -> Result { + let Some(address) = ptr.checked_add(offset) else { + return Err(TrapCode::MemoryOutOfBounds); + }; + usize::try_from(address).map_err(|_| TrapCode::MemoryOutOfBounds) } impl UntypedVal { @@ -148,15 +148,15 @@ impl UntypedVal { /// /// # Errors /// - /// - If `address + offset` overflows. - /// - If `address + offset` loads out of bounds from `memory`. - fn load_extend(memory: &[u8], address: Self, offset: u32) -> Result + /// - If `ptr + offset` overflows. + /// - If `ptr + offset` loads out of bounds from `memory`. + fn load_extend(memory: &[u8], ptr: Self, offset: u64) -> Result where T: Into, U: LittleEndianConvert + ExtendInto, { - let raw_address = u32::from(address); - let address = effective_address(raw_address, offset)?; + let ptr = u64::from(ptr); + let address = effective_address(ptr, offset)?; let mut buffer = <::Bytes as Default>::default(); buffer.load_into(memory, address)?; let value: Self = ::from_le_bytes(buffer) @@ -169,153 +169,153 @@ impl UntypedVal { /// /// # Errors /// - /// - If `address + offset` overflows. - /// - If `address + offset` loads out of bounds from `memory`. - fn load(memory: &[u8], address: Self, offset: u32) -> Result + /// - If `ptr + offset` overflows. + /// - If `ptr + offset` loads out of bounds from `memory`. + fn load(memory: &[u8], ptr: Self, offset: u64) -> Result where T: LittleEndianConvert + ExtendInto + Into, { - Self::load_extend::(memory, address, offset) + Self::load_extend::(memory, ptr, offset) } /// Executes a Wasmi `load32` instruction. /// /// # Errors /// - /// - If `address + offset` overflows. - /// - If `address + offset` loads out of bounds from `memory`. - pub fn load32(memory: &[u8], address: Self, offset: u32) -> Result { - Self::load::(memory, address, offset) + /// - If `ptr + offset` overflows. + /// - If `ptr + offset` loads out of bounds from `memory`. + pub fn load32(memory: &[u8], ptr: Self, offset: u64) -> Result { + Self::load::(memory, ptr, offset) } /// Executes a Wasmi `load64` instruction. /// /// # Errors /// - /// - If `address + offset` overflows. - /// - If `address + offset` loads out of bounds from `memory`. - pub fn load64(memory: &[u8], address: Self, offset: u32) -> Result { - Self::load::(memory, address, offset) + /// - If `ptr + offset` overflows. + /// - If `ptr + offset` loads out of bounds from `memory`. + pub fn load64(memory: &[u8], ptr: Self, offset: u64) -> Result { + Self::load::(memory, ptr, offset) } /// Executes the `i32.load8_s` Wasm operation. /// /// # Errors /// - /// - If `address + offset` overflows. - /// - If `address + offset` loads out of bounds from `memory`. - pub fn i32_load8_s(memory: &[u8], address: Self, offset: u32) -> Result { - Self::load_extend::(memory, address, offset) + /// - If `ptr + offset` overflows. + /// - If `ptr + offset` loads out of bounds from `memory`. + pub fn i32_load8_s(memory: &[u8], ptr: Self, offset: u64) -> Result { + Self::load_extend::(memory, ptr, offset) } /// Executes the `i32.load8_u` Wasm operation. /// /// # Errors /// - /// - If `address + offset` overflows. - /// - If `address + offset` loads out of bounds from `memory`. - pub fn i32_load8_u(memory: &[u8], address: Self, offset: u32) -> Result { - Self::load_extend::(memory, address, offset) + /// - If `ptr + offset` overflows. + /// - If `ptr + offset` loads out of bounds from `memory`. + pub fn i32_load8_u(memory: &[u8], ptr: Self, offset: u64) -> Result { + Self::load_extend::(memory, ptr, offset) } /// Executes the `i32.load16_s` Wasm operation. /// /// # Errors /// - /// - If `address + offset` overflows. - /// - If `address + offset` loads out of bounds from `memory`. - pub fn i32_load16_s(memory: &[u8], address: Self, offset: u32) -> Result { - Self::load_extend::(memory, address, offset) + /// - If `ptr + offset` overflows. + /// - If `ptr + offset` loads out of bounds from `memory`. + pub fn i32_load16_s(memory: &[u8], ptr: Self, offset: u64) -> Result { + Self::load_extend::(memory, ptr, offset) } /// Executes the `i32.load16_u` Wasm operation. /// /// # Errors /// - /// - If `address + offset` overflows. - /// - If `address + offset` loads out of bounds from `memory`. - pub fn i32_load16_u(memory: &[u8], address: Self, offset: u32) -> Result { - Self::load_extend::(memory, address, offset) + /// - If `ptr + offset` overflows. + /// - If `ptr + offset` loads out of bounds from `memory`. + pub fn i32_load16_u(memory: &[u8], ptr: Self, offset: u64) -> Result { + Self::load_extend::(memory, ptr, offset) } /// Executes the `i64.load8_s` Wasm operation. /// /// # Errors /// - /// - If `address + offset` overflows. - /// - If `address + offset` loads out of bounds from `memory`. - pub fn i64_load8_s(memory: &[u8], address: Self, offset: u32) -> Result { - Self::load_extend::(memory, address, offset) + /// - If `ptr + offset` overflows. + /// - If `ptr + offset` loads out of bounds from `memory`. + pub fn i64_load8_s(memory: &[u8], ptr: Self, offset: u64) -> Result { + Self::load_extend::(memory, ptr, offset) } /// Executes the `i64.load8_u` Wasm operation. /// /// # Errors /// - /// - If `address + offset` overflows. - /// - If `address + offset` loads out of bounds from `memory`. - pub fn i64_load8_u(memory: &[u8], address: Self, offset: u32) -> Result { - Self::load_extend::(memory, address, offset) + /// - If `ptr + offset` overflows. + /// - If `ptr + offset` loads out of bounds from `memory`. + pub fn i64_load8_u(memory: &[u8], ptr: Self, offset: u64) -> Result { + Self::load_extend::(memory, ptr, offset) } /// Executes the `i64.load16_s` Wasm operation. /// /// # Errors /// - /// - If `address + offset` overflows. - /// - If `address + offset` loads out of bounds from `memory`. - pub fn i64_load16_s(memory: &[u8], address: Self, offset: u32) -> Result { - Self::load_extend::(memory, address, offset) + /// - If `ptr + offset` overflows. + /// - If `ptr + offset` loads out of bounds from `memory`. + pub fn i64_load16_s(memory: &[u8], ptr: Self, offset: u64) -> Result { + Self::load_extend::(memory, ptr, offset) } /// Executes the `i64.load16_u` Wasm operation. /// /// # Errors /// - /// - If `address + offset` overflows. - /// - If `address + offset` loads out of bounds from `memory`. - pub fn i64_load16_u(memory: &[u8], address: Self, offset: u32) -> Result { - Self::load_extend::(memory, address, offset) + /// - If `ptr + offset` overflows. + /// - If `ptr + offset` loads out of bounds from `memory`. + pub fn i64_load16_u(memory: &[u8], ptr: Self, offset: u64) -> Result { + Self::load_extend::(memory, ptr, offset) } /// Executes the `i64.load32_s` Wasm operation. /// /// # Errors /// - /// - If `address + offset` overflows. - /// - If `address + offset` loads out of bounds from `memory`. - pub fn i64_load32_s(memory: &[u8], address: Self, offset: u32) -> Result { - Self::load_extend::(memory, address, offset) + /// - If `ptr + offset` overflows. + /// - If `ptr + offset` loads out of bounds from `memory`. + pub fn i64_load32_s(memory: &[u8], ptr: Self, offset: u64) -> Result { + Self::load_extend::(memory, ptr, offset) } /// Executes the `i64.load32_u` Wasm operation. /// /// # Errors /// - /// - If `address + offset` overflows. - /// - If `address + offset` loads out of bounds from `memory`. - pub fn i64_load32_u(memory: &[u8], address: Self, offset: u32) -> Result { - Self::load_extend::(memory, address, offset) + /// - If `ptr + offset` overflows. + /// - If `ptr + offset` loads out of bounds from `memory`. + pub fn i64_load32_u(memory: &[u8], ptr: Self, offset: u64) -> Result { + Self::load_extend::(memory, ptr, offset) } /// Executes a generic `T.store[N]` Wasm operation. /// /// # Errors /// - /// - If `address + offset` overflows. - /// - If `address + offset` stores out of bounds from `memory`. + /// - If `ptr + offset` overflows. + /// - If `ptr + offset` stores out of bounds from `memory`. fn store_wrap( memory: &mut [u8], - address: Self, - offset: u32, + ptr: Self, + offset: u64, value: Self, ) -> Result<(), TrapCode> where T: From + WrapInto, U: LittleEndianConvert, { - let raw_address = u32::from(address); - let address = effective_address(raw_address, offset)?; + let ptr = u64::from(ptr); + let address = effective_address(ptr, offset)?; let wrapped = T::from(value).wrap_into(); let buffer = ::into_le_bytes(wrapped); buffer.store_from(memory, address)?; @@ -326,118 +326,108 @@ impl UntypedVal { /// /// # Errors /// - /// - If `address + offset` overflows. - /// - If `address + offset` stores out of bounds from `memory`. - fn store(memory: &mut [u8], address: Self, offset: u32, value: Self) -> Result<(), TrapCode> + /// - If `ptr + offset` overflows. + /// - If `ptr + offset` stores out of bounds from `memory`. + fn store(memory: &mut [u8], ptr: Self, offset: u64, value: Self) -> Result<(), TrapCode> where T: From + WrapInto + LittleEndianConvert, { - Self::store_wrap::(memory, address, offset, value) + Self::store_wrap::(memory, ptr, offset, value) } /// Executes a Wasmi `store32` instruction. /// /// # Errors /// - /// - If `address + offset` overflows. - /// - If `address + offset` stores out of bounds from `memory`. - pub fn store32( - memory: &mut [u8], - address: Self, - offset: u32, - value: Self, - ) -> Result<(), TrapCode> { - Self::store::(memory, address, offset, value) + /// - If `ptr + offset` overflows. + /// - If `ptr + offset` stores out of bounds from `memory`. + pub fn store32(memory: &mut [u8], ptr: Self, offset: u64, value: Self) -> Result<(), TrapCode> { + Self::store::(memory, ptr, offset, value) } /// Executes a Wasmi `store64` instruction. /// /// # Errors /// - /// - If `address + offset` overflows. - /// - If `address + offset` stores out of bounds from `memory`. - pub fn store64( - memory: &mut [u8], - address: Self, - offset: u32, - value: Self, - ) -> Result<(), TrapCode> { - Self::store::(memory, address, offset, value) + /// - If `ptr + offset` overflows. + /// - If `ptr + offset` stores out of bounds from `memory`. + pub fn store64(memory: &mut [u8], ptr: Self, offset: u64, value: Self) -> Result<(), TrapCode> { + Self::store::(memory, ptr, offset, value) } /// Executes the `i32.store8` Wasm operation. /// /// # Errors /// - /// - If `address + offset` overflows. - /// - If `address + offset` stores out of bounds from `memory`. + /// - If `ptr + offset` overflows. + /// - If `ptr + offset` stores out of bounds from `memory`. pub fn i32_store8( memory: &mut [u8], - address: Self, - offset: u32, + ptr: Self, + offset: u64, value: Self, ) -> Result<(), TrapCode> { - Self::store_wrap::(memory, address, offset, value) + Self::store_wrap::(memory, ptr, offset, value) } /// Executes the `i32.store16` Wasm operation. /// /// # Errors /// - /// - If `address + offset` overflows. - /// - If `address + offset` stores out of bounds from `memory`. + /// - If `ptr + offset` overflows. + /// - If `ptr + offset` stores out of bounds from `memory`. pub fn i32_store16( memory: &mut [u8], - address: Self, - offset: u32, + ptr: Self, + offset: u64, value: Self, ) -> Result<(), TrapCode> { - Self::store_wrap::(memory, address, offset, value) + Self::store_wrap::(memory, ptr, offset, value) } /// Executes the `i64.store8` Wasm operation. /// /// # Errors /// - /// - If `address + offset` overflows. - /// - If `address + offset` stores out of bounds from `memory`. + /// - If `ptr + offset` overflows. + /// - If `ptr + offset` stores out of bounds from `memory`. pub fn i64_store8( memory: &mut [u8], - address: Self, - offset: u32, + ptr: Self, + offset: u64, value: Self, ) -> Result<(), TrapCode> { - Self::store_wrap::(memory, address, offset, value) + Self::store_wrap::(memory, ptr, offset, value) } /// Executes the `i64.store16` Wasm operation. /// /// # Errors /// - /// - If `address + offset` overflows. - /// - If `address + offset` stores out of bounds from `memory`. + /// - If `ptr + offset` overflows. + /// - If `ptr + offset` stores out of bounds from `memory`. pub fn i64_store16( memory: &mut [u8], - address: Self, - offset: u32, + ptr: Self, + offset: u64, value: Self, ) -> Result<(), TrapCode> { - Self::store_wrap::(memory, address, offset, value) + Self::store_wrap::(memory, ptr, offset, value) } /// Executes the `i64.store32` Wasm operation. /// /// # Errors /// - /// - If `address + offset` overflows. - /// - If `address + offset` stores out of bounds from `memory`. + /// - If `ptr + offset` overflows. + /// - If `ptr + offset` stores out of bounds from `memory`. pub fn i64_store32( memory: &mut [u8], - address: Self, - offset: u32, + ptr: Self, + offset: u64, value: Self, ) -> Result<(), TrapCode> { - Self::store_wrap::(memory, address, offset, value) + Self::store_wrap::(memory, ptr, offset, value) } /// Execute an infallible generic operation on `T` that returns an `R`. diff --git a/crates/ir/src/enum.rs b/crates/ir/src/enum.rs index bd44f4b959..11022366bb 100644 --- a/crates/ir/src/enum.rs +++ b/crates/ir/src/enum.rs @@ -1,4 +1,4 @@ -use crate::{core::TrapCode, for_each_op, index::*, *}; +use crate::{core::TrapCode, for_each_op, index::*, primitive::Offset64Hi, *}; use ::core::num::{NonZeroI32, NonZeroI64, NonZeroU32, NonZeroU64}; macro_rules! define_enum { @@ -218,6 +218,45 @@ impl Instruction { ) -> Self { Self::register_list([reg0.into(), reg1.into(), reg2.into()]) } + + /// Creates a new [`Instruction::RegisterAndImm32`] from the given `reg` and `offset_hi`. + pub fn register_and_offset_hi(reg: impl Into, offset_hi: Offset64Hi) -> Self { + Self::register_and_imm32(reg, offset_hi.0) + } + + /// Returns `Some` [`Reg`] and [`Offset64Hi`] if encoded properly. + /// + /// # Errors + /// + /// Returns back `self` if it was an incorrect [`Instruction`]. + /// This allows for a better error message to inform the user. + pub fn filter_register_and_offset_hi(self) -> Result<(Reg, Offset64Hi), Self> { + if let Instruction::RegisterAndImm32 { reg, imm } = self { + return Ok((reg, Offset64Hi(u32::from(imm)))); + } + Err(self) + } + + /// Creates a new [`Instruction::Imm16AndImm32`] from the given `value` and `offset_hi`. + pub fn imm16_and_offset_hi(value: impl Into, offset_hi: Offset64Hi) -> Self { + Self::imm16_and_imm32(value, offset_hi.0) + } + + /// Returns `Some` [`Reg`] and [`Offset64Hi`] if encoded properly. + /// + /// # Errors + /// + /// Returns back `self` if it was an incorrect [`Instruction`]. + /// This allows for a better error message to inform the user. + pub fn filter_imm16_and_offset_hi(self) -> Result<(T, Offset64Hi), Self> + where + T: From, + { + if let Instruction::Imm16AndImm32 { imm16, imm32 } = self { + return Ok((T::from(imm16), Offset64Hi(u32::from(imm32)))); + } + Err(self) + } } #[test] diff --git a/crates/ir/src/for_each_op.rs b/crates/ir/src/for_each_op.rs index 0d69d5c49d..bbdcfbbd5f 100644 --- a/crates/ir/src/for_each_op.rs +++ b/crates/ir/src/for_each_op.rs @@ -1473,14 +1473,19 @@ macro_rules! for_each_op { /// /// # Encoding /// - /// Followed by an [`Instruction::RegisterAndImm32`] encoding `ptr` and `offset`. + /// Followed by + /// + /// 1. [`Instruction::RegisterAndImm32`]: encoding `ptr` and `offset_hi` + /// 2. Optional [`Instruction::MemoryIndex`]: encoding `memory` index used + /// + /// If [`Instruction::MemoryIndex`] is missing the default memory is used. #[snake_name(load32)] Load32 { @result: Reg, - /// The linear memory index for which the load instruction is executed. - memory: Memory, + /// The lower 32-bit of the 64-bit load offset. + offset_lo: Offset64Lo, }, - /// Load instruction for 32-bit values. + /// Load instruction for 32-bit values and a 32-bit encoded address. /// /// # Note /// @@ -1495,7 +1500,7 @@ macro_rules! for_each_op { Load32At { @result: Reg, /// The `ptr+offset` address of the `load` instruction. - address: u32, + address: Address32, }, /// Load instruction for 32-bit values. /// @@ -1509,7 +1514,7 @@ macro_rules! for_each_op { /// The register storing the pointer of the `load` instruction. ptr: Reg, /// The 16-bit encoded offset of the `load` instruction. - offset: Const16, + offset: Offset16, }, /// Load instruction for 64-bit values. @@ -1520,14 +1525,19 @@ macro_rules! for_each_op { /// /// # Encoding /// - /// Followed by an [`Instruction::RegisterAndImm32`] encoding `ptr` and `offset`. + /// Followed by + /// + /// 1. [`Instruction::RegisterAndImm32`]: encoding `ptr` and `offset_hi` + /// 2. Optional [`Instruction::MemoryIndex`]: encoding `memory` index used + /// + /// If [`Instruction::MemoryIndex`] is missing the default memory is used. #[snake_name(load64)] Load64 { @result: Reg, - /// The linear memory index for which the load instruction is executed. - memory: Memory, + /// The lower 32-bit of the 64-bit load offset. + offset_lo: Offset64Lo, }, - /// Load instruction for 64-bit values. + /// Load instruction for 64-bit values and a 32-bit encoded address. /// /// # Note /// @@ -1542,7 +1552,7 @@ macro_rules! for_each_op { Load64At { @result: Reg, /// The `ptr+offset` address of the `load` instruction. - address: u32, + address: Address32, }, /// Load instruction for 64-bit values. /// @@ -1556,25 +1566,30 @@ macro_rules! for_each_op { /// The register storing the pointer of the `load` instruction. ptr: Reg, /// The 16-bit encoded offset of the `load` instruction. - offset: Const16, + offset: Offset16, }, /// Wasm `i32.load8_s` equivalent Wasmi instruction. /// /// # Encoding /// - /// Followed by an [`Instruction::RegisterAndImm32`] encoding `ptr` and `offset`. + /// Followed by + /// + /// 1. [`Instruction::RegisterAndImm32`]: encoding `ptr` and `offset_hi` + /// 2. Optional [`Instruction::MemoryIndex`]: encoding `memory` index used + /// + /// If [`Instruction::MemoryIndex`] is missing the default memory is used. #[snake_name(i32_load8_s)] I32Load8s { @result: Reg, - /// The linear memory index for which the load instruction is executed. - memory: Memory, + /// The lower 32-bit of the 64-bit load offset. + offset_lo: Offset64Lo, }, /// Wasm `i32.load8_s` equivalent Wasmi instruction. /// /// # Note /// - /// Variant of [`Instruction::I32Load8s`] with a constant load address. + /// Variant of [`Instruction::I32Load8s`] with a 32-bit constant load address. /// /// # Encoding /// @@ -1585,7 +1600,7 @@ macro_rules! for_each_op { I32Load8sAt { @result: Reg, /// The `ptr+offset` address of the `load` instruction. - address: u32, + address: Address32, }, /// Wasm `i32.load8_s` equivalent Wasmi instruction. /// @@ -1599,25 +1614,30 @@ macro_rules! for_each_op { /// The register storing the pointer of the `load` instruction. ptr: Reg, /// The 16-bit encoded offset of the `load` instruction. - offset: Const16, + offset: Offset16, }, /// Wasm `i32.load8_u` equivalent Wasmi instruction. /// /// # Encoding /// - /// Followed by an [`Instruction::RegisterAndImm32`] encoding `ptr` and `offset`. + /// Followed by + /// + /// 1. [`Instruction::RegisterAndImm32`]: encoding `ptr` and `offset_hi` + /// 2. Optional [`Instruction::MemoryIndex`]: encoding `memory` index used + /// + /// If [`Instruction::MemoryIndex`] is missing the default memory is used. #[snake_name(i32_load8_u)] I32Load8u { @result: Reg, - /// The linear memory index for which the load instruction is executed. - memory: Memory, + /// The lower 32-bit of the 64-bit load offset. + offset_lo: Offset64Lo, }, /// Wasm `i32.load8_u` equivalent Wasmi instruction. /// /// # Note /// - /// Variant of [`Instruction::I32Load8u`] with a constant load address. + /// Variant of [`Instruction::I32Load8u`] with a 32-bit constant load address. /// /// # Encoding /// @@ -1628,7 +1648,7 @@ macro_rules! for_each_op { I32Load8uAt { @result: Reg, /// The `ptr+offset` address of the `load` instruction. - address: u32, + address: Address32, }, /// Wasm `i32.load8_u` equivalent Wasmi instruction. /// @@ -1642,25 +1662,30 @@ macro_rules! for_each_op { /// The register storing the pointer of the `load` instruction. ptr: Reg, /// The 16-bit encoded offset of the `load` instruction. - offset: Const16, + offset: Offset16, }, /// Wasm `i32.load16_s` equivalent Wasmi instruction. /// /// # Encoding /// - /// Followed by an [`Instruction::RegisterAndImm32`] encoding `ptr` and `offset`. + /// Followed by + /// + /// 1. [`Instruction::RegisterAndImm32`]: encoding `ptr` and `offset_hi` + /// 2. Optional [`Instruction::MemoryIndex`]: encoding `memory` index used + /// + /// If [`Instruction::MemoryIndex`] is missing the default memory is used. #[snake_name(i32_load16_s)] I32Load16s { @result: Reg, - /// The linear memory index for which the load instruction is executed. - memory: Memory, + /// The lower 32-bit of the 64-bit load offset. + offset_lo: Offset64Lo, }, /// Wasm `i32.load16_s` equivalent Wasmi instruction. /// /// # Note /// - /// Variant of [`Instruction::I32Load16s`] with a constant load address. + /// Variant of [`Instruction::I32Load16s`] with a 32-bit constant load address. /// /// # Encoding /// @@ -1671,7 +1696,7 @@ macro_rules! for_each_op { I32Load16sAt { @result: Reg, /// The `ptr+offset` address of the `load` instruction. - address: u32, + address: Address32, }, /// Wasm `i32.load16_s` equivalent Wasmi instruction. /// @@ -1685,25 +1710,30 @@ macro_rules! for_each_op { /// The register storing the pointer of the `load` instruction. ptr: Reg, /// The 16-bit encoded offset of the `load` instruction. - offset: Const16, + offset: Offset16, }, /// Wasm `i32.load16_u` equivalent Wasmi instruction. /// /// # Encoding /// - /// Followed by an [`Instruction::RegisterAndImm32`] encoding `ptr` and `offset`. + /// Followed by + /// + /// 1. [`Instruction::RegisterAndImm32`]: encoding `ptr` and `offset_hi` + /// 2. Optional [`Instruction::MemoryIndex`]: encoding `memory` index used + /// + /// If [`Instruction::MemoryIndex`] is missing the default memory is used. #[snake_name(i32_load16_u)] I32Load16u { @result: Reg, - /// The linear memory index for which the load instruction is executed. - memory: Memory, + /// The lower 32-bit of the 64-bit load offset. + offset_lo: Offset64Lo, }, /// Wasm `i32.load16_u` equivalent Wasmi instruction. /// /// # Note /// - /// Variant of [`Instruction::I32Load16u`] with a constant load address. + /// Variant of [`Instruction::I32Load16u`] with a 32-bit constant load address. /// /// # Encoding /// @@ -1714,7 +1744,7 @@ macro_rules! for_each_op { I32Load16uAt { @result: Reg, /// The `ptr+offset` address of the `load` instruction. - address: u32, + address: Address32, }, /// Wasm `i32.load16_u` equivalent Wasmi instruction. /// @@ -1728,25 +1758,30 @@ macro_rules! for_each_op { /// The register storing the pointer of the `load` instruction. ptr: Reg, /// The 16-bit encoded offset of the `load` instruction. - offset: Const16, + offset: Offset16, }, /// Wasm `i64.load8_s` equivalent Wasmi instruction. /// /// # Encoding /// - /// Followed by an [`Instruction::RegisterAndImm32`] encoding `ptr` and `offset`. + /// Followed by + /// + /// 1. [`Instruction::RegisterAndImm32`]: encoding `ptr` and `offset_hi` + /// 2. Optional [`Instruction::MemoryIndex`]: encoding `memory` index used + /// + /// If [`Instruction::MemoryIndex`] is missing the default memory is used. #[snake_name(i64_load8_s)] I64Load8s { @result: Reg, - /// The linear memory index for which the load instruction is executed. - memory: Memory, + /// The lower 32-bit of the 64-bit load offset. + offset_lo: Offset64Lo, }, /// Wasm `i64.load8_s` equivalent Wasmi instruction. /// /// # Note /// - /// Variant of [`Instruction::I64Load8s`] with a constant load address. + /// Variant of [`Instruction::I64Load8s`] with a 32-bit constant load address. /// /// # Encoding /// @@ -1757,7 +1792,7 @@ macro_rules! for_each_op { I64Load8sAt { @result: Reg, /// The `ptr+offset` address of the `load` instruction. - address: u32, + address: Address32, }, /// Wasm `i64.load8_s` equivalent Wasmi instruction. /// @@ -1771,25 +1806,30 @@ macro_rules! for_each_op { /// The register storing the pointer of the `load` instruction. ptr: Reg, /// The 16-bit encoded offset of the `load` instruction. - offset: Const16, + offset: Offset16, }, /// Wasm `i64.load8_u` equivalent Wasmi instruction. /// /// # Encoding /// - /// Followed by an [`Instruction::RegisterAndImm32`] encoding `ptr` and `offset`. + /// Followed by + /// + /// 1. [`Instruction::RegisterAndImm32`]: encoding `ptr` and `offset_hi` + /// 2. Optional [`Instruction::MemoryIndex`]: encoding `memory` index used + /// + /// If [`Instruction::MemoryIndex`] is missing the default memory is used. #[snake_name(i64_load8_u)] I64Load8u { @result: Reg, - /// The linear memory index for which the load instruction is executed. - memory: Memory, + /// The lower 32-bit of the 64-bit load offset. + offset_lo: Offset64Lo, }, /// Wasm `i64.load8_u` equivalent Wasmi instruction. /// /// # Note /// - /// Variant of [`Instruction::I64Load8u`] with a constant load address. + /// Variant of [`Instruction::I64Load8u`] with a 32-bit constant load address. /// /// # Encoding /// @@ -1800,7 +1840,7 @@ macro_rules! for_each_op { I64Load8uAt { @result: Reg, /// The `ptr+offset` address of the `load` instruction. - address: u32, + address: Address32, }, /// Wasm `i64.load8_u` equivalent Wasmi instruction. /// @@ -1814,25 +1854,30 @@ macro_rules! for_each_op { /// The register storing the pointer of the `load` instruction. ptr: Reg, /// The 16-bit encoded offset of the `load` instruction. - offset: Const16, + offset: Offset16, }, /// Wasm `i64.load16_s` equivalent Wasmi instruction. /// /// # Encoding /// - /// Followed by an [`Instruction::RegisterAndImm32`] encoding `ptr` and `offset`. + /// Followed by + /// + /// 1. [`Instruction::RegisterAndImm32`]: encoding `ptr` and `offset_hi` + /// 2. Optional [`Instruction::MemoryIndex`]: encoding `memory` index used + /// + /// If [`Instruction::MemoryIndex`] is missing the default memory is used. #[snake_name(i64_load16_s)] I64Load16s { @result: Reg, - /// The linear memory index for which the load instruction is executed. - memory: Memory, + /// The lower 32-bit of the 64-bit load offset. + offset_lo: Offset64Lo, }, /// Wasm `i64.load16_s` equivalent Wasmi instruction. /// /// # Note /// - /// Variant of [`Instruction::I64Load16s`] with a constant load address. + /// Variant of [`Instruction::I64Load16s`] with a 32-bit constant load address. /// /// # Encoding /// @@ -1843,7 +1888,7 @@ macro_rules! for_each_op { I64Load16sAt { @result: Reg, /// The `ptr+offset` address of the `load` instruction. - address: u32, + address: Address32, }, /// Wasm `i64.load16_s` equivalent Wasmi instruction. /// @@ -1857,25 +1902,30 @@ macro_rules! for_each_op { /// The register storing the pointer of the `load` instruction. ptr: Reg, /// The 16-bit encoded offset of the `load` instruction. - offset: Const16, + offset: Offset16, }, /// Wasm `i64.load16_u` equivalent Wasmi instruction. /// /// # Encoding /// - /// Followed by an [`Instruction::RegisterAndImm32`] encoding `ptr` and `offset`. + /// Followed by + /// + /// 1. [`Instruction::RegisterAndImm32`]: encoding `ptr` and `offset_hi` + /// 2. Optional [`Instruction::MemoryIndex`]: encoding `memory` index used + /// + /// If [`Instruction::MemoryIndex`] is missing the default memory is used. #[snake_name(i64_load16_u)] I64Load16u { @result: Reg, - /// The linear memory index for which the load instruction is executed. - memory: Memory, + /// The lower 32-bit of the 64-bit load offset. + offset_lo: Offset64Lo, }, /// Wasm `i64.load16_u` equivalent Wasmi instruction. /// /// # Note /// - /// Variant of [`Instruction::I64Load16u`] with a constant load address. + /// Variant of [`Instruction::I64Load16u`] with a 32-bit constant load address. /// /// # Encoding /// @@ -1886,7 +1936,7 @@ macro_rules! for_each_op { I64Load16uAt { @result: Reg, /// The `ptr+offset` address of the `load` instruction. - address: u32, + address: Address32, }, /// Wasm `i64.load16_u` equivalent Wasmi instruction. /// @@ -1900,25 +1950,30 @@ macro_rules! for_each_op { /// The register storing the pointer of the `load` instruction. ptr: Reg, /// The 16-bit encoded offset of the `load` instruction. - offset: Const16, + offset: Offset16, }, /// Wasm `i64.load32_s` equivalent Wasmi instruction. /// /// # Encoding /// - /// Followed by an [`Instruction::RegisterAndImm32`] encoding `ptr` and `offset`. + /// Followed by + /// + /// 1. [`Instruction::RegisterAndImm32`]: encoding `ptr` and `offset_hi` + /// 2. Optional [`Instruction::MemoryIndex`]: encoding `memory` index used + /// + /// If [`Instruction::MemoryIndex`] is missing the default memory is used. #[snake_name(i64_load32_s)] I64Load32s { @result: Reg, - /// The linear memory index for which the load instruction is executed. - memory: Memory, + /// The lower 32-bit of the 64-bit load offset. + offset_lo: Offset64Lo, }, /// Wasm `i64.load32_s` equivalent Wasmi instruction. /// /// # Note /// - /// Variant of [`Instruction::I64Load32s`] with a constant load address. + /// Variant of [`Instruction::I64Load32s`] with a 32-bit constant load address. /// /// # Encoding /// @@ -1928,8 +1983,8 @@ macro_rules! for_each_op { #[snake_name(i64_load32_s_at)] I64Load32sAt { @result: Reg, - /// The `ptr+offset` address of the `load` instruction. - address: u32, + /// The `ptr+offset` address of the `load` instruction. + address: Address32, }, /// Wasm `i64.load32_s` equivalent Wasmi instruction. /// @@ -1943,25 +1998,30 @@ macro_rules! for_each_op { /// The register storing the pointer of the `load` instruction. ptr: Reg, /// The 16-bit encoded offset of the `load` instruction. - offset: Const16, + offset: Offset16, }, /// Wasm `i64.load32_u` equivalent Wasmi instruction. /// /// # Encoding /// - /// Followed by an [`Instruction::RegisterAndImm32`] encoding `ptr` and `offset`. + /// Followed by + /// + /// 1. [`Instruction::RegisterAndImm32`]: encoding `ptr` and `offset_hi` + /// 2. Optional [`Instruction::MemoryIndex`]: encoding `memory` index used + /// + /// If [`Instruction::MemoryIndex`] is missing the default memory is used. #[snake_name(i64_load32_u)] I64Load32u { @result: Reg, - /// The linear memory index for which the load instruction is executed. - memory: Memory, + /// The lower 32-bit of the 64-bit load offset. + offset_lo: Offset64Lo, }, /// Wasm `i64.load32_u` equivalent Wasmi instruction. /// /// # Note /// - /// Variant of [`Instruction::I64Load32u`] with a constant load address. + /// Variant of [`Instruction::I64Load32u`] with a 32-bit constant load address. /// /// # Encoding /// @@ -1972,7 +2032,7 @@ macro_rules! for_each_op { I64Load32uAt { @result: Reg, /// The `ptr+offset` address of the `load` instruction. - address: u32, + address: Address32, }, /// Wasm `i64.load32_u` equivalent Wasmi instruction. /// @@ -1986,7 +2046,7 @@ macro_rules! for_each_op { /// The register storing the pointer of the `load` instruction. ptr: Reg, /// The 16-bit encoded offset of the `load` instruction. - offset: Const16, + offset: Offset16, }, /// Store instruction for 32-bit values. @@ -1997,13 +2057,18 @@ macro_rules! for_each_op { /// /// # Encoding /// - /// Followed by an [`Instruction::RegisterAndImm32`] encoding `value` and `offset`. + /// Followed by + /// + /// 1. [`Instruction::RegisterAndImm32`]: encoding `value` and `offset_hi` + /// 2. Optional [`Instruction::MemoryIndex`]: encoding `memory` index used + /// + /// If [`Instruction::MemoryIndex`] is missing the default memory is used. #[snake_name(store32)] Store32 { /// The register storing the pointer of the `store` instruction. ptr: Reg, - /// The linear memory index for which the store instruction is executed. - memory: Memory, + /// The lower 32-bit of the 64-bit load offset. + offset_lo: Offset64Lo, }, /// Store instruction for 32-bit values. /// @@ -2016,7 +2081,7 @@ macro_rules! for_each_op { /// The register storing the pointer of the `store` instruction. ptr: Reg, /// The register storing the pointer offset of the `store` instruction. - offset: Const16, + offset: Offset16, /// The value to be stored. value: Reg, }, @@ -2036,7 +2101,7 @@ macro_rules! for_each_op { /// The value to be stored. value: Reg, /// The constant address to store the value. - address: u32, + address: Address32, }, /// Store instruction for 64-bit values. @@ -2047,13 +2112,18 @@ macro_rules! for_each_op { /// /// # Encoding /// - /// Followed by an [`Instruction::RegisterAndImm32`] encoding `value` and `offset`. + /// Followed by + /// + /// 1. [`Instruction::RegisterAndImm32`]: encoding `value` and `offset_hi` + /// 2. Optional [`Instruction::MemoryIndex`]: encoding `memory` index used + /// + /// If [`Instruction::MemoryIndex`] is missing the default memory is used. #[snake_name(store64)] Store64 { /// The register storing the pointer of the `store` instruction. ptr: Reg, - /// The linear memory index for which the store instruction is executed. - memory: Memory, + /// The lower 32-bit of the 64-bit load offset. + offset_lo: Offset64Lo, }, /// Store instruction for 64-bit values. /// @@ -2066,7 +2136,7 @@ macro_rules! for_each_op { /// The register storing the pointer of the `store` instruction. ptr: Reg, /// The register storing the pointer offset of the `store` instruction. - offset: Const16, + offset: Offset16, /// The value to be stored. value: Reg, }, @@ -2086,7 +2156,7 @@ macro_rules! for_each_op { /// The value to be stored. value: Reg, /// The constant address to store the value. - address: u32, + address: Address32, }, /// Wasm `i32.store` equivalent Wasmi instruction. @@ -2097,13 +2167,18 @@ macro_rules! for_each_op { /// /// # Encoding /// - /// Followed by an [`Instruction::Imm16AndImm32`] encoding `value` and `offset`. + /// Followed by + /// + /// 1. [`Instruction::RegisterAndImm32`]: encoding `value` and `offset_hi` + /// 2. Optional [`Instruction::MemoryIndex`]: encoding `memory` index used + /// + /// If [`Instruction::MemoryIndex`] is missing the default memory is used. #[snake_name(i32_store_imm16)] I32StoreImm16 { /// The register storing the pointer of the `store` instruction. ptr: Reg, - /// The linear memory index for which the store instruction is executed. - memory: Memory, + /// The lower 32-bit of the 64-bit load offset. + offset_lo: Offset64Lo, }, /// Wasm `i32.store` equivalent Wasmi instruction. /// @@ -2116,7 +2191,7 @@ macro_rules! for_each_op { /// The register storing the pointer of the `store` instruction. ptr: Reg, /// The register storing the pointer offset of the `store` instruction. - offset: Const16, + offset: Offset16, /// The value to be stored. value: Const16, }, @@ -2136,20 +2211,25 @@ macro_rules! for_each_op { /// The value to be stored. value: Const16, /// The constant address to store the value. - address: u32, + address: Address32, }, /// Wasm `i32.store` equivalent Wasmi instruction. /// /// # Encoding /// - /// Followed by an [`Instruction::RegisterAndImm32`] encoding `value` and `offset`. + /// Followed by + /// + /// 1. [`Instruction::RegisterAndImm32`]: encoding `value` and `offset_hi` + /// 2. Optional [`Instruction::MemoryIndex`]: encoding `memory` index used + /// + /// If [`Instruction::MemoryIndex`] is missing the default memory is used. #[snake_name(i32_store8)] I32Store8 { /// The register storing the pointer of the `store` instruction. ptr: Reg, - /// The linear memory index for which the store instruction is executed. - memory: Memory, + /// The lower 32-bit of the 64-bit load offset. + offset_lo: Offset64Lo, }, /// Wasm `i32.store8` equivalent Wasmi instruction. /// @@ -2159,13 +2239,18 @@ macro_rules! for_each_op { /// /// # Encoding /// - /// Followed by an [`Instruction::Imm16AndImm32`] encoding `value` and `offset`. + /// Followed by + /// + /// 1. [`Instruction::RegisterAndImm32`]: encoding `value` and `offset_hi` + /// 2. Optional [`Instruction::MemoryIndex`]: encoding `memory` index used + /// + /// If [`Instruction::MemoryIndex`] is missing the default memory is used. #[snake_name(i32_store8_imm)] I32Store8Imm { /// The register storing the pointer of the `store` instruction. ptr: Reg, - /// The linear memory index for which the store instruction is executed. - memory: Memory, + /// The lower 32-bit of the 64-bit load offset. + offset_lo: Offset64Lo, }, /// Wasm `i32.store8` equivalent Wasmi instruction. /// @@ -2178,7 +2263,7 @@ macro_rules! for_each_op { /// The register storing the pointer of the `store` instruction. ptr: Reg, /// The register storing the pointer offset of the `store` instruction. - offset: Const16, + offset: Offset16, /// The value to be stored. value: Reg, }, @@ -2193,7 +2278,7 @@ macro_rules! for_each_op { /// The register storing the pointer of the `store` instruction. ptr: Reg, /// The register storing the pointer offset of the `store` instruction. - offset: Const16, + offset: Offset16, /// The value to be stored. value: i8, }, @@ -2213,7 +2298,7 @@ macro_rules! for_each_op { /// The value to be stored. value: Reg, /// The constant address to store the value. - address: u32, + address: Address32, }, /// Wasm `i32.store8` equivalent Wasmi instruction. /// @@ -2231,20 +2316,25 @@ macro_rules! for_each_op { /// The value to be stored. value: i8, /// The constant address to store the value. - address: u32, + address: Address32, }, /// Wasm `i32.store16` equivalent Wasmi instruction. /// /// # Encoding /// - /// Followed by an [`Instruction::RegisterAndImm32`] encoding `value` and `offset`. + /// Followed by + /// + /// 1. [`Instruction::RegisterAndImm32`]: encoding `value` and `offset_hi` + /// 2. Optional [`Instruction::MemoryIndex`]: encoding `memory` index used + /// + /// If [`Instruction::MemoryIndex`] is missing the default memory is used. #[snake_name(i32_store16)] I32Store16 { /// The register storing the pointer of the `store` instruction. ptr: Reg, - /// The linear memory index for which the store instruction is executed. - memory: Memory, + /// The lower 32-bit of the 64-bit load offset. + offset_lo: Offset64Lo, }, /// Wasm `i32.store16` equivalent Wasmi instruction. /// @@ -2254,13 +2344,18 @@ macro_rules! for_each_op { /// /// # Encoding /// - /// Followed by an [`Instruction::Imm16AndImm32`] encoding `value` and `offset`. + /// Followed by + /// + /// 1. [`Instruction::RegisterAndImm32`]: encoding `value` and `offset_hi` + /// 2. Optional [`Instruction::MemoryIndex`]: encoding `memory` index used + /// + /// If [`Instruction::MemoryIndex`] is missing the default memory is used. #[snake_name(i32_store16_imm)] I32Store16Imm { /// The register storing the pointer of the `store` instruction. ptr: Reg, - /// The linear memory index for which the store instruction is executed. - memory: Memory, + /// The lower 32-bit of the 64-bit load offset. + offset_lo: Offset64Lo, }, /// Wasm `i32.store16` equivalent Wasmi instruction. /// @@ -2273,7 +2368,7 @@ macro_rules! for_each_op { /// The register storing the pointer of the `store` instruction. ptr: Reg, /// The register storing the pointer offset of the `store` instruction. - offset: Const16, + offset: Offset16, /// The value to be stored. value: Reg, }, @@ -2288,7 +2383,7 @@ macro_rules! for_each_op { /// The register storing the pointer of the `store` instruction. ptr: Reg, /// The register storing the pointer offset of the `store` instruction. - offset: Const16, + offset: Offset16, /// The value to be stored. value: i16, }, @@ -2308,7 +2403,7 @@ macro_rules! for_each_op { /// The value to be stored. value: Reg, /// The constant address to store the value. - address: u32, + address: Address32, }, /// Wasm `i32.store16` equivalent Wasmi instruction. /// @@ -2326,7 +2421,7 @@ macro_rules! for_each_op { /// The value to be stored. value: i16, /// The constant address to store the value. - address: u32, + address: Address32, }, /// Wasm `i64.store` equivalent Wasmi instruction. @@ -2337,13 +2432,18 @@ macro_rules! for_each_op { /// /// # Encoding /// - /// Followed by an [`Instruction::Imm16AndImm32`] encoding `value` and `offset`. + /// Followed by + /// + /// 1. [`Instruction::RegisterAndImm32`]: encoding `value` and `offset_hi` + /// 2. Optional [`Instruction::MemoryIndex`]: encoding `memory` index used + /// + /// If [`Instruction::MemoryIndex`] is missing the default memory is used. #[snake_name(i64_store_imm16)] I64StoreImm16 { /// The register storing the pointer of the `store` instruction. ptr: Reg, - /// The linear memory index for which the store instruction is executed. - memory: Memory, + /// The lower 32-bit of the 64-bit load offset. + offset_lo: Offset64Lo, }, /// Wasm `i64.store` equivalent Wasmi instruction. /// @@ -2356,7 +2456,7 @@ macro_rules! for_each_op { /// The register storing the pointer of the `store` instruction. ptr: Reg, /// The register storing the pointer offset of the `store` instruction. - offset: Const16, + offset: Offset16, /// The value to be stored. value: Const16, }, @@ -2376,20 +2476,25 @@ macro_rules! for_each_op { /// The value to be stored. value: Const16, /// The constant address to store the value. - address: u32, + address: Address32, }, /// Wasm `i64.store8` equivalent Wasmi instruction. /// /// # Encoding /// - /// Followed by an [`Instruction::RegisterAndImm32`] encoding `value` and `offset`. + /// Followed by + /// + /// 1. [`Instruction::RegisterAndImm32`]: encoding `value` and `offset_hi` + /// 2. Optional [`Instruction::MemoryIndex`]: encoding `memory` index used + /// + /// If [`Instruction::MemoryIndex`] is missing the default memory is used. #[snake_name(i64_store8)] I64Store8 { /// The register storing the pointer of the `store` instruction. ptr: Reg, - /// The linear memory index for which the store instruction is executed. - memory: Memory, + /// The lower 32-bit of the 64-bit load offset. + offset_lo: Offset64Lo, }, /// Wasm `i64.store8` equivalent Wasmi instruction. /// @@ -2399,13 +2504,18 @@ macro_rules! for_each_op { /// /// # Encoding /// - /// Followed by an [`Instruction::Imm16AndImm32`] encoding `value` and `offset`. + /// Followed by + /// + /// 1. [`Instruction::RegisterAndImm32`]: encoding `value` and `offset_hi` + /// 2. Optional [`Instruction::MemoryIndex`]: encoding `memory` index used + /// + /// If [`Instruction::MemoryIndex`] is missing the default memory is used. #[snake_name(i64_store8_imm)] I64Store8Imm { /// The register storing the pointer of the `store` instruction. ptr: Reg, - /// The linear memory index for which the store instruction is executed. - memory: Memory, + /// The lower 32-bit of the 64-bit load offset. + offset_lo: Offset64Lo, }, /// Wasm `i64.store8` equivalent Wasmi instruction. /// @@ -2418,7 +2528,7 @@ macro_rules! for_each_op { /// The register storing the pointer of the `store` instruction. ptr: Reg, /// The register storing the pointer offset of the `store` instruction. - offset: Const16, + offset: Offset16, /// The value to be stored. value: Reg, }, @@ -2433,7 +2543,7 @@ macro_rules! for_each_op { /// The register storing the pointer of the `store` instruction. ptr: Reg, /// The register storing the pointer offset of the `store` instruction. - offset: Const16, + offset: Offset16, /// The value to be stored. value: i8, }, @@ -2453,7 +2563,7 @@ macro_rules! for_each_op { /// The value to be stored. value: Reg, /// The constant address to store the value. - address: u32, + address: Address32, }, /// Wasm `i64.store8` equivalent Wasmi instruction. /// @@ -2471,20 +2581,25 @@ macro_rules! for_each_op { /// The value to be stored. value: i8, /// The constant address to store the value. - address: u32, + address: Address32, }, /// Wasm `i64.store16` equivalent Wasmi instruction. /// /// # Encoding /// - /// Followed by an [`Instruction::RegisterAndImm32`] encoding `value` and `offset`. + /// Followed by + /// + /// 1. [`Instruction::RegisterAndImm32`]: encoding `value` and `offset_hi` + /// 2. Optional [`Instruction::MemoryIndex`]: encoding `memory` index used + /// + /// If [`Instruction::MemoryIndex`] is missing the default memory is used. #[snake_name(i64_store16)] I64Store16 { /// The register storing the pointer of the `store` instruction. ptr: Reg, - /// The linear memory index for which the store instruction is executed. - memory: Memory, + /// The lower 32-bit of the 64-bit load offset. + offset_lo: Offset64Lo, }, /// Wasm `i64.store16` equivalent Wasmi instruction. /// @@ -2494,13 +2609,18 @@ macro_rules! for_each_op { /// /// # Encoding /// - /// Followed by an [`Instruction::Imm16AndImm32`] encoding `value` and `offset`. + /// Followed by + /// + /// 1. [`Instruction::RegisterAndImm32`]: encoding `value` and `offset_hi` + /// 2. Optional [`Instruction::MemoryIndex`]: encoding `memory` index used + /// + /// If [`Instruction::MemoryIndex`] is missing the default memory is used. #[snake_name(i64_store16_imm)] I64Store16Imm { /// The register storing the pointer of the `store` instruction. ptr: Reg, - /// The linear memory index for which the store instruction is executed. - memory: Memory, + /// The lower 32-bit of the 64-bit load offset. + offset_lo: Offset64Lo, }, /// Wasm `i64.store16` equivalent Wasmi instruction. /// @@ -2513,7 +2633,7 @@ macro_rules! for_each_op { /// The register storing the pointer of the `store` instruction. ptr: Reg, /// The register storing the pointer offset of the `store` instruction. - offset: Const16, + offset: Offset16, /// The value to be stored. value: Reg, }, @@ -2528,7 +2648,7 @@ macro_rules! for_each_op { /// The register storing the pointer of the `store` instruction. ptr: Reg, /// The register storing the pointer offset of the `store` instruction. - offset: Const16, + offset: Offset16, /// The value to be stored. value: i16, }, @@ -2548,7 +2668,7 @@ macro_rules! for_each_op { /// The value to be stored. value: Reg, /// The constant address to store the value. - address: u32, + address: Address32, }, /// Wasm `i64.store16` equivalent Wasmi instruction. /// @@ -2566,20 +2686,25 @@ macro_rules! for_each_op { /// The value to be stored. value: i16, /// The constant address to store the value. - address: u32, + address: Address32, }, /// Wasm `i64.store32` equivalent Wasmi instruction. /// /// # Encoding /// - /// Followed by an [`Instruction::RegisterAndImm32`] encoding `value` and `offset`. + /// Followed by + /// + /// 1. [`Instruction::RegisterAndImm32`]: encoding `value` and `offset_hi` + /// 2. Optional [`Instruction::MemoryIndex`]: encoding `memory` index used + /// + /// If [`Instruction::MemoryIndex`] is missing the default memory is used. #[snake_name(i64_store32)] I64Store32 { /// The register storing the pointer of the `store` instruction. ptr: Reg, - /// The linear memory index for which the store instruction is executed. - memory: Memory, + /// The lower 32-bit of the 64-bit load offset. + offset_lo: Offset64Lo, }, /// Wasm `i64.store32` equivalent Wasmi instruction. /// @@ -2589,13 +2714,18 @@ macro_rules! for_each_op { /// /// # Encoding /// - /// Followed by an [`Instruction::Imm16AndImm32`] encoding `value` and `offset`. + /// Followed by + /// + /// 1. [`Instruction::RegisterAndImm32`]: encoding `value` and `offset_hi` + /// 2. Optional [`Instruction::MemoryIndex`]: encoding `memory` index used + /// + /// If [`Instruction::MemoryIndex`] is missing the default memory is used. #[snake_name(i64_store32_imm16)] I64Store32Imm16 { /// The register storing the pointer of the `store` instruction. ptr: Reg, - /// The linear memory index for which the store instruction is executed. - memory: Memory, + /// The lower 32-bit of the 64-bit load offset. + offset_lo: Offset64Lo, }, /// Wasm `i64.store32` equivalent Wasmi instruction. /// @@ -2608,7 +2738,7 @@ macro_rules! for_each_op { /// The register storing the pointer of the `store` instruction. ptr: Reg, /// The register storing the pointer offset of the `store` instruction. - offset: Const16, + offset: Offset16, /// The value to be stored. value: Reg, }, @@ -2623,7 +2753,7 @@ macro_rules! for_each_op { /// The register storing the pointer of the `store` instruction. ptr: Reg, /// The register storing the pointer offset of the `store` instruction. - offset: Const16, + offset: Offset16, /// The value to be stored. value: Const16, }, @@ -2643,7 +2773,7 @@ macro_rules! for_each_op { /// The value to be stored. value: Reg, /// The constant address to store the value. - address: u32, + address: Address32, }, /// Wasm `i64.store32` equivalent Wasmi instruction. /// @@ -2661,7 +2791,7 @@ macro_rules! for_each_op { /// The value to be stored. value: Const16, /// The constant address to store the value. - address: u32, + address: Address32, }, /// Wasm `i32.eq` equivalent Wasmi instruction. @@ -4556,7 +4686,7 @@ macro_rules! for_each_op { TableGetImm { @result: Reg, /// The constant `index` value of the table element to get. - index: u32, + index: Const32, }, /// A Wasm `table.size` instruction. @@ -4589,7 +4719,7 @@ macro_rules! for_each_op { /// The register holding the `value` of the instruction. value: Reg, /// The constant `index` of the instruction. - index: u32, + index: Const32, }, /// Wasm `table.copy ` instruction. @@ -4622,7 +4752,7 @@ macro_rules! for_each_op { #[snake_name(table_copy_to)] TableCopyTo { /// The start index of the `dst` table. - dst: Const16, + dst: Const16, /// The start index of the `src` table. src: Reg, /// The number of copied elements. @@ -4641,7 +4771,7 @@ macro_rules! for_each_op { /// The start index of the `dst` table. dst: Reg, /// The start index of the `src` table. - src: Const16, + src: Const16, /// The number of copied elements. len: Reg, }, @@ -4656,9 +4786,9 @@ macro_rules! for_each_op { #[snake_name(table_copy_from_to)] TableCopyFromTo { /// The start index of the `dst` table. - dst: Const16, + dst: Const16, /// The start index of the `src` table. - src: Const16, + src: Const16, /// The number of copied elements. len: Reg, }, @@ -4681,7 +4811,7 @@ macro_rules! for_each_op { /// The start index of the `src` table. src: Reg, /// The number of copied elements. - len: Const16, + len: Const16, }, /// Variant of [`Instruction::TableCopy`] with a constant 16-bit `len` and `dst`. /// @@ -4698,11 +4828,11 @@ macro_rules! for_each_op { #[snake_name(table_copy_to_exact)] TableCopyToExact { /// The start index of the `dst` table. - dst: Const16, + dst: Const16, /// The start index of the `src` table. src: Reg, /// The number of copied elements. - len: Const16, + len: Const16, }, /// Variant of [`Instruction::TableCopy`] with a constant 16-bit `len` and `src`. /// @@ -4721,9 +4851,9 @@ macro_rules! for_each_op { /// The start index of the `dst` table. dst: Reg, /// The start index of the `src` table. - src: Const16, + src: Const16, /// The number of copied elements. - len: Const16, + len: Const16, }, /// Variant of [`Instruction::TableCopy`] with a constant 16-bit `len` and `src`. /// @@ -4740,11 +4870,11 @@ macro_rules! for_each_op { #[snake_name(table_copy_from_to_exact)] TableCopyFromToExact { /// The start index of the `dst` table. - dst: Const16, + dst: Const16, /// The start index of the `src` table. - src: Const16, + src: Const16, /// The number of copied elements. - len: Const16, + len: Const16, }, /// Wasm `table.init ` instruction. @@ -4777,7 +4907,7 @@ macro_rules! for_each_op { #[snake_name(table_init_to)] TableInitTo { /// The start index of the `dst` table. - dst: Const16, + dst: Const16, /// The start index of the `src` table. src: Reg, /// The number of copied elements. @@ -4811,7 +4941,7 @@ macro_rules! for_each_op { #[snake_name(table_init_from_to)] TableInitFromTo { /// The start index of the `dst` table. - dst: Const16, + dst: Const16, /// The start index of the `src` table. src: Const16, /// The number of copied elements. @@ -4853,7 +4983,7 @@ macro_rules! for_each_op { #[snake_name(table_init_to_exact)] TableInitToExact { /// The start index of the `dst` table. - dst: Const16, + dst: Const16, /// The start index of the `src` table. src: Reg, /// The number of copied elements. @@ -4895,7 +5025,7 @@ macro_rules! for_each_op { #[snake_name(table_init_from_to_exact)] TableInitFromToExact { /// The start index of the `dst` table. - dst: Const16, + dst: Const16, /// The start index of the `src` table. src: Const16, /// The number of copied elements. @@ -4924,7 +5054,7 @@ macro_rules! for_each_op { #[snake_name(table_fill_at)] TableFillAt { /// The start index of the table to fill. - dst: Const16, + dst: Const16, /// The number of elements to fill. len: Reg, /// The value of the filled elements. @@ -4940,7 +5070,7 @@ macro_rules! for_each_op { /// The start index of the table to fill. dst: Reg, /// The number of elements to fill. - len: Const16, + len: Const16, /// The value of the filled elements. value: Reg, }, @@ -4952,9 +5082,9 @@ macro_rules! for_each_op { #[snake_name(table_fill_at_exact)] TableFillAtExact { /// The start index of the table to fill. - dst: Const16, + dst: Const16, /// The number of elements to fill. - len: Const16, + len: Const16, /// The value of the filled elements. value: Reg, }, @@ -4981,7 +5111,7 @@ macro_rules! for_each_op { TableGrowImm { @result: Reg, /// The number of elements to add to the table. - delta: Const16, + delta: Const16, /// The value that is used to fill up the new cells. value: Reg, }, @@ -5058,7 +5188,7 @@ macro_rules! for_each_op { #[snake_name(memory_copy_to)] MemoryCopyTo { /// The start index of the `dst` memory. - dst: Const16, + dst: Const16, /// The start index of the `src` memory. src: Reg, /// The number of copied bytes. @@ -5077,7 +5207,7 @@ macro_rules! for_each_op { /// The start index of the `dst` memory. dst: Reg, /// The start index of the `src` memory. - src: Const16, + src: Const16, /// The number of copied bytes. len: Reg, }, @@ -5092,9 +5222,9 @@ macro_rules! for_each_op { #[snake_name(memory_copy_from_to)] MemoryCopyFromTo { /// The start index of the `dst` memory. - dst: Const16, + dst: Const16, /// The start index of the `src` memory. - src: Const16, + src: Const16, /// The number of copied bytes. len: Reg, }, @@ -5117,7 +5247,7 @@ macro_rules! for_each_op { /// The start index of the `src` memory. src: Reg, /// The number of copied bytes. - len: Const16, + len: Const16, }, /// Variant of [`Instruction::MemoryCopy`] with a constant 16-bit `len` and `dst`. /// @@ -5134,11 +5264,11 @@ macro_rules! for_each_op { #[snake_name(memory_copy_to_exact)] MemoryCopyToExact { /// The start index of the `dst` memory. - dst: Const16, + dst: Const16, /// The start index of the `src` memory. src: Reg, /// The number of copied bytes. - len: Const16, + len: Const16, }, /// Variant of [`Instruction::MemoryCopy`] with a constant 16-bit `len` and `src`. /// @@ -5157,9 +5287,9 @@ macro_rules! for_each_op { /// The start index of the `dst` memory. dst: Reg, /// The start index of the `src` memory. - src: Const16, + src: Const16, /// The number of copied bytes. - len: Const16, + len: Const16, }, /// Variant of [`Instruction::MemoryCopy`] with a constant 16-bit `len` and `src`. /// @@ -5176,11 +5306,11 @@ macro_rules! for_each_op { #[snake_name(memory_copy_from_to_exact)] MemoryCopyFromToExact { /// The start index of the `dst` memory. - dst: Const16, + dst: Const16, /// The start index of the `src` memory. - src: Const16, + src: Const16, /// The number of copied bytes. - len: Const16, + len: Const16, }, /// Wasm `memory.fill` instruction. @@ -5207,7 +5337,7 @@ macro_rules! for_each_op { #[snake_name(memory_fill_at)] MemoryFillAt { /// The start index of the memory to fill. - dst: Const16, + dst: Const16, /// The byte value used to fill the memory. value: Reg, /// The number of bytes to fill. @@ -5239,7 +5369,7 @@ macro_rules! for_each_op { /// The byte value used to fill the memory. value: Reg, /// The number of bytes to fill. - len: Const16, + len: Const16, }, /// Variant of [`Instruction::MemoryFill`] with constant `dst` index and `value`. /// @@ -5249,7 +5379,7 @@ macro_rules! for_each_op { #[snake_name(memory_fill_at_imm)] MemoryFillAtImm { /// The start index of the memory to fill. - dst: Const16, + dst: Const16, /// The byte value used to fill the memory. value: u8, /// The number of bytes to fill. @@ -5263,11 +5393,11 @@ macro_rules! for_each_op { #[snake_name(memory_fill_at_exact)] MemoryFillAtExact { /// The start index of the memory to fill. - dst: Const16, + dst: Const16, /// The byte value used to fill the memory. value: Reg, /// The number of bytes to fill. - len: Const16, + len: Const16, }, /// Variant of [`Instruction::MemoryFill`] with constant fill `value` and `len`. /// @@ -5281,7 +5411,7 @@ macro_rules! for_each_op { /// The byte value used to fill the memory. value: u8, /// The number of bytes to fill. - len: Const16, + len: Const16, }, /// Variant of [`Instruction::MemoryFill`] with constant `dst` index, fill `value` and `len`. /// @@ -5291,11 +5421,11 @@ macro_rules! for_each_op { #[snake_name(memory_fill_at_imm_exact)] MemoryFillAtImmExact { /// The start index of the memory to fill. - dst: Const16, + dst: Const16, /// The byte value used to fill the memory. value: u8, /// The number of bytes to fill. - len: Const16, + len: Const16, }, /// Wasm `memory.init ` instruction. @@ -5328,7 +5458,7 @@ macro_rules! for_each_op { #[snake_name(memory_init_to)] MemoryInitTo { /// The start index of the `dst` memory. - dst: Const16, + dst: Const16, /// The start index of the `src` data segment. src: Reg, /// The number of initialized bytes. @@ -5362,7 +5492,7 @@ macro_rules! for_each_op { #[snake_name(memory_init_from_to)] MemoryInitFromTo { /// The start index of the `dst` memory. - dst: Const16, + dst: Const16, /// The start index of the `src` data segment. src: Const16, /// The number of initialized bytes. @@ -5396,7 +5526,7 @@ macro_rules! for_each_op { #[snake_name(memory_init_to_exact)] MemoryInitToExact { /// The start index of the `dst` memory. - dst: Const16, + dst: Const16, /// The start index of the `src` data segment. src: Reg, /// The number of initialized bytes. @@ -5430,7 +5560,7 @@ macro_rules! for_each_op { #[snake_name(memory_init_from_to_exact)] MemoryInitFromToExact { /// The start index of the `dst` memory. - dst: Const16, + dst: Const16, /// The start index of the `src` data segment. src: Const16, /// The number of initialized bytes. @@ -5618,7 +5748,7 @@ macro_rules! for_each_op { #[snake_name(call_indirect_params_imm16)] CallIndirectParamsImm16 { /// The index of the called function in the table. - index: Const16, + index: Const16, /// The table which holds the called function at the index. table: Table, }, diff --git a/crates/ir/src/immeditate.rs b/crates/ir/src/immeditate.rs index 5a9428cb58..49d129cf07 100644 --- a/crates/ir/src/immeditate.rs +++ b/crates/ir/src/immeditate.rs @@ -18,6 +18,13 @@ pub struct Const16 { marker: PhantomData T>, } +impl Const16 { + /// Casts the `Const16` to a `Const16` value. + pub fn cast(const16: Const16) -> Self { + Self::new(const16.inner) + } +} + impl Const16 { /// Returns `true` if the [`Const16`] is equal to zero. pub fn is_zero(&self) -> bool { @@ -248,6 +255,13 @@ impl Const32 { } } +impl Const32 { + /// Casts the `Const16` to a `Const16` value. + pub fn cast(const32: Const32) -> Self { + Self::new(const32.inner) + } +} + impl Clone for Const32 { fn clone(&self) -> Self { *self diff --git a/crates/ir/src/lib.rs b/crates/ir/src/lib.rs index d1b2185d30..173c9b6c41 100644 --- a/crates/ir/src/lib.rs +++ b/crates/ir/src/lib.rs @@ -24,12 +24,17 @@ pub use self::{ immeditate::{AnyConst16, AnyConst32, Const16, Const32}, index::Reg, primitive::{ + Address32, BlockFuel, BranchOffset, BranchOffset16, Comparator, ComparatorAndOffset, IntoShiftAmount, + Offset16, + Offset64, + Offset64Hi, + Offset64Lo, ShiftAmount, Sign, }, diff --git a/crates/ir/src/primitive.rs b/crates/ir/src/primitive.rs index 19362864e8..f2e7185d87 100644 --- a/crates/ir/src/primitive.rs +++ b/crates/ir/src/primitive.rs @@ -1,4 +1,4 @@ -use crate::{core::UntypedVal, Const16, Error}; +use crate::{core::UntypedVal, immeditate::OutOfBoundsConst, Const16, Const32, Error}; use core::marker::PhantomData; /// The sign of a value. @@ -403,3 +403,112 @@ impl_shift_amount! { (i32, 32), (i64, 64), } + +/// A 64-bit offset in Wasmi bytecode. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[repr(transparent)] +pub struct Offset64(u64); + +/// The high 32 bits of an [`Offset64`]. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[repr(transparent)] +pub struct Offset64Hi(pub(crate) u32); + +/// The low 32 bits of an [`Offset64`]. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[repr(transparent)] +pub struct Offset64Lo(pub(crate) u32); + +impl Offset64 { + /// Creates a new [`Offset64`] lo-hi pair from the given `offset`. + pub fn split(offset: u64) -> (Offset64Hi, Offset64Lo) { + let offset_lo = (offset & 0xFFFF_FFFF) as u32; + let offset_hi = (offset >> 32) as u32; + (Offset64Hi(offset_hi), Offset64Lo(offset_lo)) + } + + /// Combines the given [`Offset64`] lo-hi pair into an [`Offset64`]. + pub fn combine(hi: Offset64Hi, lo: Offset64Lo) -> Self { + let hi = hi.0 as u64; + let lo = lo.0 as u64; + Self((hi << 32) | lo) + } +} + +#[test] +fn test_offset64_split_combine() { + let test_values = [ + 0, + 1, + 1 << 1, + u64::MAX, + u64::MAX - 1, + 42, + 77, + u64::MAX >> 1, + 0xFFFF_FFFF_0000_0000, + 0x0000_0000_FFFF_FFFF, + 0xF0F0_F0F0_0F0F_0F0F, + ]; + for value in test_values { + let (hi, lo) = Offset64::split(value); + let combined = u64::from(Offset64::combine(hi, lo)); + assert_eq!(combined, value); + } +} + +impl From for Offset64 { + fn from(offset: u64) -> Self { + Self(offset) + } +} + +impl From for u64 { + fn from(offset: Offset64) -> Self { + offset.0 + } +} + +/// A 16-bit encoded load or store address offset. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[repr(transparent)] +pub struct Offset16(Const16); + +impl TryFrom for Offset16 { + type Error = OutOfBoundsConst; + + fn try_from(address: u64) -> Result { + >::try_from(address).map(Self) + } +} + +impl From for Offset64 { + fn from(offset: Offset16) -> Self { + Offset64(u64::from(offset.0)) + } +} + +/// A 32-bit memory address used for some load and store instructions. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[repr(transparent)] +pub struct Address32(Const32); + +impl TryFrom for Address32 { + type Error = OutOfBoundsConst; + + fn try_from(address: u64) -> Result { + >::try_from(address).map(Self) + } +} + +impl From for Address32 { + fn from(address: u32) -> Self { + Self(>::from(address)) + } +} + +impl From for u64 { + fn from(address: Address32) -> Self { + u64::from(address.0) + } +} diff --git a/crates/ir/src/visit_regs.rs b/crates/ir/src/visit_regs.rs index 8f39cef5a4..af48b02ef6 100644 --- a/crates/ir/src/visit_regs.rs +++ b/crates/ir/src/visit_regs.rs @@ -92,6 +92,11 @@ impl_host_visitor_for!( Const32, Sign, ShiftAmount, + Offset16, + Offset64, + Offset64Lo, + Offset64Hi, + Address32, ); /// Type-wrapper to signal that the wrapped [`Reg`], [`RegSpan`] (etc.) is a result. diff --git a/crates/wasmi/src/engine/config.rs b/crates/wasmi/src/engine/config.rs index 81b459c60b..dd02ac1bd8 100644 --- a/crates/wasmi/src/engine/config.rs +++ b/crates/wasmi/src/engine/config.rs @@ -182,6 +182,7 @@ impl Config { features.set(WasmFeatures::EXTENDED_CONST, true); features.set(WasmFeatures::FLOATS, true); features.set(WasmFeatures::CUSTOM_PAGE_SIZES, false); + features.set(WasmFeatures::MEMORY64, true); features } @@ -334,6 +335,18 @@ impl Config { self } + /// Enable or disable the [`memory64`] Wasm proposal for the [`Config`]. + /// + /// # Note + /// + /// Disabled by default. + /// + /// [`memory64`]: https://github.com/WebAssembly/memory64 + pub fn wasm_memory64(&mut self, enable: bool) -> &mut Self { + self.features.set(WasmFeatures::MEMORY64, enable); + self + } + /// Enable or disable Wasm floating point (`f32` and `f64`) instructions and types. /// /// Enabled by default. diff --git a/crates/wasmi/src/engine/executor/instrs.rs b/crates/wasmi/src/engine/executor/instrs.rs index cb966d45de..930236717b 100644 --- a/crates/wasmi/src/engine/executor/instrs.rs +++ b/crates/wasmi/src/engine/executor/instrs.rs @@ -9,7 +9,7 @@ use crate::{ DedupFuncType, EngineFunc, }, - ir::{index, BlockFuel, Const16, Instruction, Reg, ShiftAmount}, + ir::{index, BlockFuel, Const16, Instruction, Offset64Hi, Reg, ShiftAmount}, memory::DataSegment, store::StoreInner, table::ElementSegment, @@ -473,8 +473,8 @@ impl<'engine> Executor<'engine> { Instr::GlobalSetI64Imm16 { global, input } => { self.execute_global_set_i64imm16(&mut store.inner, global, input) } - Instr::Load32 { result, memory } => { - self.execute_load32(&store.inner, result, memory)? + Instr::Load32 { result, offset_lo } => { + self.execute_load32(&store.inner, result, offset_lo)? } Instr::Load32At { result, address } => { self.execute_load32_at(&store.inner, result, address)? @@ -484,8 +484,8 @@ impl<'engine> Executor<'engine> { ptr, offset, } => self.execute_load32_offset16(result, ptr, offset)?, - Instr::Load64 { result, memory } => { - self.execute_load64(&store.inner, result, memory)? + Instr::Load64 { result, offset_lo } => { + self.execute_load64(&store.inner, result, offset_lo)? } Instr::Load64At { result, address } => { self.execute_load64_at(&store.inner, result, address)? @@ -495,8 +495,8 @@ impl<'engine> Executor<'engine> { ptr, offset, } => self.execute_load64_offset16(result, ptr, offset)?, - Instr::I32Load8s { result, memory } => { - self.execute_i32_load8_s(&store.inner, result, memory)? + Instr::I32Load8s { result, offset_lo } => { + self.execute_i32_load8_s(&store.inner, result, offset_lo)? } Instr::I32Load8sAt { result, address } => { self.execute_i32_load8_s_at(&store.inner, result, address)? @@ -506,8 +506,8 @@ impl<'engine> Executor<'engine> { ptr, offset, } => self.execute_i32_load8_s_offset16(result, ptr, offset)?, - Instr::I32Load8u { result, memory } => { - self.execute_i32_load8_u(&store.inner, result, memory)? + Instr::I32Load8u { result, offset_lo } => { + self.execute_i32_load8_u(&store.inner, result, offset_lo)? } Instr::I32Load8uAt { result, address } => { self.execute_i32_load8_u_at(&store.inner, result, address)? @@ -517,8 +517,8 @@ impl<'engine> Executor<'engine> { ptr, offset, } => self.execute_i32_load8_u_offset16(result, ptr, offset)?, - Instr::I32Load16s { result, memory } => { - self.execute_i32_load16_s(&store.inner, result, memory)? + Instr::I32Load16s { result, offset_lo } => { + self.execute_i32_load16_s(&store.inner, result, offset_lo)? } Instr::I32Load16sAt { result, address } => { self.execute_i32_load16_s_at(&store.inner, result, address)? @@ -528,8 +528,8 @@ impl<'engine> Executor<'engine> { ptr, offset, } => self.execute_i32_load16_s_offset16(result, ptr, offset)?, - Instr::I32Load16u { result, memory } => { - self.execute_i32_load16_u(&store.inner, result, memory)? + Instr::I32Load16u { result, offset_lo } => { + self.execute_i32_load16_u(&store.inner, result, offset_lo)? } Instr::I32Load16uAt { result, address } => { self.execute_i32_load16_u_at(&store.inner, result, address)? @@ -539,8 +539,8 @@ impl<'engine> Executor<'engine> { ptr, offset, } => self.execute_i32_load16_u_offset16(result, ptr, offset)?, - Instr::I64Load8s { result, memory } => { - self.execute_i64_load8_s(&store.inner, result, memory)? + Instr::I64Load8s { result, offset_lo } => { + self.execute_i64_load8_s(&store.inner, result, offset_lo)? } Instr::I64Load8sAt { result, address } => { self.execute_i64_load8_s_at(&store.inner, result, address)? @@ -550,8 +550,8 @@ impl<'engine> Executor<'engine> { ptr, offset, } => self.execute_i64_load8_s_offset16(result, ptr, offset)?, - Instr::I64Load8u { result, memory } => { - self.execute_i64_load8_u(&store.inner, result, memory)? + Instr::I64Load8u { result, offset_lo } => { + self.execute_i64_load8_u(&store.inner, result, offset_lo)? } Instr::I64Load8uAt { result, address } => { self.execute_i64_load8_u_at(&store.inner, result, address)? @@ -561,8 +561,8 @@ impl<'engine> Executor<'engine> { ptr, offset, } => self.execute_i64_load8_u_offset16(result, ptr, offset)?, - Instr::I64Load16s { result, memory } => { - self.execute_i64_load16_s(&store.inner, result, memory)? + Instr::I64Load16s { result, offset_lo } => { + self.execute_i64_load16_s(&store.inner, result, offset_lo)? } Instr::I64Load16sAt { result, address } => { self.execute_i64_load16_s_at(&store.inner, result, address)? @@ -572,8 +572,8 @@ impl<'engine> Executor<'engine> { ptr, offset, } => self.execute_i64_load16_s_offset16(result, ptr, offset)?, - Instr::I64Load16u { result, memory } => { - self.execute_i64_load16_u(&store.inner, result, memory)? + Instr::I64Load16u { result, offset_lo } => { + self.execute_i64_load16_u(&store.inner, result, offset_lo)? } Instr::I64Load16uAt { result, address } => { self.execute_i64_load16_u_at(&store.inner, result, address)? @@ -583,8 +583,8 @@ impl<'engine> Executor<'engine> { ptr, offset, } => self.execute_i64_load16_u_offset16(result, ptr, offset)?, - Instr::I64Load32s { result, memory } => { - self.execute_i64_load32_s(&store.inner, result, memory)? + Instr::I64Load32s { result, offset_lo } => { + self.execute_i64_load32_s(&store.inner, result, offset_lo)? } Instr::I64Load32sAt { result, address } => { self.execute_i64_load32_s_at(&store.inner, result, address)? @@ -594,8 +594,8 @@ impl<'engine> Executor<'engine> { ptr, offset, } => self.execute_i64_load32_s_offset16(result, ptr, offset)?, - Instr::I64Load32u { result, memory } => { - self.execute_i64_load32_u(&store.inner, result, memory)? + Instr::I64Load32u { result, offset_lo } => { + self.execute_i64_load32_u(&store.inner, result, offset_lo)? } Instr::I64Load32uAt { result, address } => { self.execute_i64_load32_u_at(&store.inner, result, address)? @@ -605,8 +605,8 @@ impl<'engine> Executor<'engine> { ptr, offset, } => self.execute_i64_load32_u_offset16(result, ptr, offset)?, - Instr::Store32 { ptr, memory } => { - self.execute_store32(&mut store.inner, ptr, memory)? + Instr::Store32 { ptr, offset_lo } => { + self.execute_store32(&mut store.inner, ptr, offset_lo)? } Instr::Store32Offset16 { ptr, offset, value } => { self.execute_store32_offset16(ptr, offset, value)? @@ -614,8 +614,8 @@ impl<'engine> Executor<'engine> { Instr::Store32At { address, value } => { self.execute_store32_at(&mut store.inner, address, value)? } - Instr::Store64 { ptr, memory } => { - self.execute_store64(&mut store.inner, ptr, memory)? + Instr::Store64 { ptr, offset_lo } => { + self.execute_store64(&mut store.inner, ptr, offset_lo)? } Instr::Store64Offset16 { ptr, offset, value } => { self.execute_store64_offset16(ptr, offset, value)? @@ -623,8 +623,8 @@ impl<'engine> Executor<'engine> { Instr::Store64At { address, value } => { self.execute_store64_at(&mut store.inner, address, value)? } - Instr::I32StoreImm16 { ptr, memory } => { - self.execute_i32_store_imm16(&mut store.inner, ptr, memory)? + Instr::I32StoreImm16 { ptr, offset_lo } => { + self.execute_i32_store_imm16(&mut store.inner, ptr, offset_lo)? } Instr::I32StoreOffset16Imm16 { ptr, offset, value } => { self.execute_i32_store_offset16_imm16(ptr, offset, value)? @@ -632,11 +632,11 @@ impl<'engine> Executor<'engine> { Instr::I32StoreAtImm16 { address, value } => { self.execute_i32_store_at_imm16(&mut store.inner, address, value)? } - Instr::I32Store8 { ptr, memory } => { - self.execute_i32_store8(&mut store.inner, ptr, memory)? + Instr::I32Store8 { ptr, offset_lo } => { + self.execute_i32_store8(&mut store.inner, ptr, offset_lo)? } - Instr::I32Store8Imm { ptr, memory } => { - self.execute_i32_store8_imm(&mut store.inner, ptr, memory)? + Instr::I32Store8Imm { ptr, offset_lo } => { + self.execute_i32_store8_imm(&mut store.inner, ptr, offset_lo)? } Instr::I32Store8Offset16 { ptr, offset, value } => { self.execute_i32_store8_offset16(ptr, offset, value)? @@ -650,11 +650,11 @@ impl<'engine> Executor<'engine> { Instr::I32Store8AtImm { address, value } => { self.execute_i32_store8_at_imm(&mut store.inner, address, value)? } - Instr::I32Store16 { ptr, memory } => { - self.execute_i32_store16(&mut store.inner, ptr, memory)? + Instr::I32Store16 { ptr, offset_lo } => { + self.execute_i32_store16(&mut store.inner, ptr, offset_lo)? } - Instr::I32Store16Imm { ptr, memory } => { - self.execute_i32_store16_imm(&mut store.inner, ptr, memory)? + Instr::I32Store16Imm { ptr, offset_lo } => { + self.execute_i32_store16_imm(&mut store.inner, ptr, offset_lo)? } Instr::I32Store16Offset16 { ptr, offset, value } => { self.execute_i32_store16_offset16(ptr, offset, value)? @@ -668,8 +668,8 @@ impl<'engine> Executor<'engine> { Instr::I32Store16AtImm { address, value } => { self.execute_i32_store16_at_imm(&mut store.inner, address, value)? } - Instr::I64StoreImm16 { ptr, memory } => { - self.execute_i64_store_imm16(&mut store.inner, ptr, memory)? + Instr::I64StoreImm16 { ptr, offset_lo } => { + self.execute_i64_store_imm16(&mut store.inner, ptr, offset_lo)? } Instr::I64StoreOffset16Imm16 { ptr, offset, value } => { self.execute_i64_store_offset16_imm16(ptr, offset, value)? @@ -677,11 +677,11 @@ impl<'engine> Executor<'engine> { Instr::I64StoreAtImm16 { address, value } => { self.execute_i64_store_at_imm16(&mut store.inner, address, value)? } - Instr::I64Store8 { ptr, memory } => { - self.execute_i64_store8(&mut store.inner, ptr, memory)? + Instr::I64Store8 { ptr, offset_lo } => { + self.execute_i64_store8(&mut store.inner, ptr, offset_lo)? } - Instr::I64Store8Imm { ptr, memory } => { - self.execute_i64_store8_imm(&mut store.inner, ptr, memory)? + Instr::I64Store8Imm { ptr, offset_lo } => { + self.execute_i64_store8_imm(&mut store.inner, ptr, offset_lo)? } Instr::I64Store8Offset16 { ptr, offset, value } => { self.execute_i64_store8_offset16(ptr, offset, value)? @@ -695,11 +695,11 @@ impl<'engine> Executor<'engine> { Instr::I64Store8AtImm { address, value } => { self.execute_i64_store8_at_imm(&mut store.inner, address, value)? } - Instr::I64Store16 { ptr, memory } => { - self.execute_i64_store16(&mut store.inner, ptr, memory)? + Instr::I64Store16 { ptr, offset_lo } => { + self.execute_i64_store16(&mut store.inner, ptr, offset_lo)? } - Instr::I64Store16Imm { ptr, memory } => { - self.execute_i64_store16_imm(&mut store.inner, ptr, memory)? + Instr::I64Store16Imm { ptr, offset_lo } => { + self.execute_i64_store16_imm(&mut store.inner, ptr, offset_lo)? } Instr::I64Store16Offset16 { ptr, offset, value } => { self.execute_i64_store16_offset16(ptr, offset, value)? @@ -713,11 +713,11 @@ impl<'engine> Executor<'engine> { Instr::I64Store16AtImm { address, value } => { self.execute_i64_store16_at_imm(&mut store.inner, address, value)? } - Instr::I64Store32 { ptr, memory } => { - self.execute_i64_store32(&mut store.inner, ptr, memory)? + Instr::I64Store32 { ptr, offset_lo } => { + self.execute_i64_store32(&mut store.inner, ptr, offset_lo)? } - Instr::I64Store32Imm16 { ptr, memory } => { - self.execute_i64_store32_imm16(&mut store.inner, ptr, memory)? + Instr::I64Store32Imm16 { ptr, offset_lo } => { + self.execute_i64_store32_imm16(&mut store.inner, ptr, offset_lo)? } Instr::I64Store32Offset16 { ptr, offset, value } => { self.execute_i64_store32_offset16(ptr, offset, value)? @@ -1633,18 +1633,32 @@ impl Executor<'_> { /// - Returns the default [`index::Memory`] if the parameter is missing. /// - Bumps `self.ip` if a [`Instruction::MemoryIndex`] parameter was found. #[inline(always)] - fn fetch_optional_memory(&mut self) -> index::Memory { + fn fetch_optional_memory(&mut self, delta: usize) -> index::Memory { let mut addr: InstructionPtr = self.ip; - addr.add(1); + addr.add(delta); match *addr.get() { Instruction::MemoryIndex { index } => { hint::cold(); - self.ip = addr; + self.ip.add(1); index } _ => index::Memory::from(0), } } + + /// Fetches the [`Reg`] and [`Offset64Hi`] parameters for a load or store [`Instruction`]. + unsafe fn fetch_reg_and_offset_hi(&self) -> (Reg, Offset64Hi) { + let mut addr: InstructionPtr = self.ip; + addr.add(1); + match addr.get().filter_register_and_offset_hi() { + Ok(value) => value, + Err(instr) => unsafe { + unreachable_unchecked!( + "expected an `Instruction::RegisterAndImm32` but found: {instr:?}" + ) + }, + } + } } impl Executor<'_> { diff --git a/crates/wasmi/src/engine/executor/instrs/call.rs b/crates/wasmi/src/engine/executor/instrs/call.rs index 01a1b6488f..771996e682 100644 --- a/crates/wasmi/src/engine/executor/instrs/call.rs +++ b/crates/wasmi/src/engine/executor/instrs/call.rs @@ -177,11 +177,11 @@ impl Executor<'_> { /// - This is required for some instructions that do not fit into /// a single instruction word and store a [`index::Table`] value in /// another instruction word. - fn pull_call_indirect_params(&mut self) -> (u32, index::Table) { + fn pull_call_indirect_params(&mut self) -> (u64, index::Table) { self.ip.add(1); match *self.ip.get() { Instruction::CallIndirectParams { index, table } => { - let index = u32::from(self.get_register(index)); + let index: u64 = self.get_register_as(index); (index, table) } unexpected => { @@ -206,11 +206,11 @@ impl Executor<'_> { /// - This is required for some instructions that do not fit into /// a single instruction word and store a [`index::Table`] value in /// another instruction word. - fn pull_call_indirect_params_imm16(&mut self) -> (u32, index::Table) { + fn pull_call_indirect_params_imm16(&mut self) -> (u64, index::Table) { self.ip.add(1); match *self.ip.get() { Instruction::CallIndirectParamsImm16 { index, table } => { - let index = u32::from(index); + let index: u64 = index.into(); (index, table) } unexpected => { @@ -733,7 +733,7 @@ impl Executor<'_> { store: &mut Store, results: Option, func_type: index::FuncType, - index: u32, + index: u64, table: index::Table, ) -> Result { let table = self.get_table(table); diff --git a/crates/wasmi/src/engine/executor/instrs/load.rs b/crates/wasmi/src/engine/executor/instrs/load.rs index 06f09c2c78..432f061b22 100644 --- a/crates/wasmi/src/engine/executor/instrs/load.rs +++ b/crates/wasmi/src/engine/executor/instrs/load.rs @@ -1,32 +1,22 @@ use super::Executor; use crate::{ core::{TrapCode, UntypedVal}, - engine::{executor::instr_ptr::InstructionPtr, utils::unreachable_unchecked}, - ir::{index::Memory, Const16, Instruction, Reg}, + ir::{index::Memory, Address32, Offset16, Offset64, Offset64Hi, Offset64Lo, Reg}, store::StoreInner, Error, }; +#[cfg(doc)] +use crate::ir::Instruction; + /// The function signature of Wasm load operations. -type WasmLoadOp = - fn(memory: &[u8], address: UntypedVal, offset: u32) -> Result; +type WasmLoadOp = fn(memory: &[u8], ptr: UntypedVal, offset: u64) -> Result; impl Executor<'_> { - /// Returns the `ptr` and `offset` parameters for a `load` [`Instruction`]. - fn fetch_ptr_and_offset(&self) -> (Reg, u32) { - let mut addr: InstructionPtr = self.ip; - addr.add(1); - match *addr.get() { - Instruction::RegisterAndImm32 { reg, imm } => (reg, u32::from(imm)), - instr => { - // Safety: Wasmi translation guarantees that `Instruction::RegisterAndImm32` exists. - unsafe { - unreachable_unchecked!( - "expected an `Instruction::RegisterAndImm32` but found: {instr:?}" - ) - } - } - } + /// Returns the register `value` and `offset` parameters for a `load` [`Instruction`]. + fn fetch_ptr_and_offset_hi(&self) -> (Reg, Offset64Hi) { + // Safety: Wasmi translation guarantees that `Instruction::RegisterAndImm32` exists. + unsafe { self.fetch_reg_and_offset_hi() } } /// Fetches the bytes of the default memory at index 0. @@ -86,11 +76,11 @@ impl Executor<'_> { memory: Memory, result: Reg, address: UntypedVal, - offset: u32, + offset: Offset64, load_extend: WasmLoadOp, ) -> Result<(), Error> { let memory = self.fetch_memory_bytes(memory, store); - let loaded_value = load_extend(memory, address, offset)?; + let loaded_value = load_extend(memory, address, u64::from(offset))?; self.set_register(result, loaded_value); Ok(()) } @@ -112,11 +102,11 @@ impl Executor<'_> { &mut self, result: Reg, address: UntypedVal, - offset: u32, + offset: Offset64, load_extend: WasmLoadOp, ) -> Result<(), Error> { let memory = self.fetch_default_memory_bytes(); - let loaded_value = load_extend(memory, address, offset)?; + let loaded_value = load_extend(memory, address, u64::from(offset))?; self.set_register(result, loaded_value); Ok(()) } @@ -126,11 +116,13 @@ impl Executor<'_> { &mut self, store: &StoreInner, result: Reg, - memory: Memory, + offset_lo: Offset64Lo, load_extend: WasmLoadOp, ) -> Result<(), Error> { - let (ptr, offset) = self.fetch_ptr_and_offset(); + let (ptr, offset_hi) = self.fetch_ptr_and_offset_hi(); + let memory = self.fetch_optional_memory(2); let address = self.get_register(ptr); + let offset = Offset64::combine(offset_hi, offset_lo); self.execute_load_extend(store, memory, result, address, offset, load_extend)?; self.try_next_instr_at(2) } @@ -140,17 +132,16 @@ impl Executor<'_> { &mut self, store: &StoreInner, result: Reg, - address: u32, + address: Address32, load_extend: WasmLoadOp, ) -> Result<(), Error> { - let memory = self.fetch_optional_memory(); - let offset = address; + let memory = self.fetch_optional_memory(1); self.execute_load_extend( store, memory, result, - UntypedVal::from(0u32), - offset, + UntypedVal::from(u64::from(address)), + Offset64::from(0), load_extend, )?; self.try_next_instr() @@ -161,11 +152,11 @@ impl Executor<'_> { &mut self, result: Reg, ptr: Reg, - offset: Const16, + offset: Offset16, load_extend: WasmLoadOp, ) -> Result<(), Error> { - let offset = u32::from(offset); let address = self.get_register(ptr); + let offset = Offset64::from(offset); self.execute_load_extend_mem0(result, address, offset, load_extend)?; self.try_next_instr() } @@ -182,17 +173,17 @@ macro_rules! impl_execute_load { ),* $(,)? ) => { $( #[doc = concat!("Executes an [`Instruction::", stringify!($var_load), "`].")] - pub fn $fn_load(&mut self, store: &StoreInner, result: Reg, memory: Memory) -> Result<(), Error> { - self.execute_load_impl(store, result, memory, $impl_fn) + pub fn $fn_load(&mut self, store: &StoreInner, result: Reg, offset_lo: Offset64Lo) -> Result<(), Error> { + self.execute_load_impl(store, result, offset_lo, $impl_fn) } #[doc = concat!("Executes an [`Instruction::", stringify!($var_load_at), "`].")] - pub fn $fn_load_at(&mut self, store: &StoreInner, result: Reg, address: u32) -> Result<(), Error> { + pub fn $fn_load_at(&mut self, store: &StoreInner, result: Reg, address: Address32) -> Result<(), Error> { self.execute_load_at_impl(store, result, address, $impl_fn) } #[doc = concat!("Executes an [`Instruction::", stringify!($var_load_off16), "`].")] - pub fn $fn_load_off16(&mut self, result: Reg, ptr: Reg, offset: Const16) -> Result<(), Error> { + pub fn $fn_load_off16(&mut self, result: Reg, ptr: Reg, offset: Offset16) -> Result<(), Error> { self.execute_load_offset16_impl(result, ptr, offset, $impl_fn) } )* diff --git a/crates/wasmi/src/engine/executor/instrs/memory.rs b/crates/wasmi/src/engine/executor/instrs/memory.rs index d757830526..dd0449ea16 100644 --- a/crates/wasmi/src/engine/executor/instrs/memory.rs +++ b/crates/wasmi/src/engine/executor/instrs/memory.rs @@ -76,7 +76,7 @@ impl Executor<'_> { result: Reg, delta: Reg, ) -> Result<(), Error> { - let delta: u32 = self.get_register_as(delta); + let delta: u64 = self.get_register_as(delta); let (store, mut resource_limiter) = store.store_inner_and_resource_limiter_ref(); self.execute_memory_grow_impl(store, result, delta, &mut resource_limiter) } @@ -89,6 +89,7 @@ impl Executor<'_> { delta: u32, ) -> Result<(), Error> { let (store, mut resource_limiter) = store.store_inner_and_resource_limiter_ref(); + let delta = u64::from(delta); self.execute_memory_grow_impl(store, result, delta, &mut resource_limiter) } @@ -98,7 +99,7 @@ impl Executor<'_> { &mut self, store: &'store mut StoreInner, result: Reg, - delta: u32, + delta: u64, resource_limiter: &mut ResourceLimiterRef<'store>, ) -> Result<(), Error> { let memory = self.fetch_memory_index(1); @@ -120,7 +121,10 @@ impl Executor<'_> { unsafe { self.cache.update_memory(store) }; return_value } - Err(EntityGrowError::InvalidGrow) => EntityGrowError::ERROR_CODE, + Err(EntityGrowError::InvalidGrow) => match memory.ty().is_64() { + true => EntityGrowError::ERROR_CODE_64, + false => EntityGrowError::ERROR_CODE_32, + }, Err(EntityGrowError::TrapCode(trap_code)) => return Err(Error::from(trap_code)), }; self.set_register(result, return_value); @@ -135,9 +139,9 @@ impl Executor<'_> { src: Reg, len: Reg, ) -> Result<(), Error> { - let dst: u32 = self.get_register_as(dst); - let src: u32 = self.get_register_as(src); - let len: u32 = self.get_register_as(len); + let dst: u64 = self.get_register_as(dst); + let src: u64 = self.get_register_as(src); + let len: u64 = self.get_register_as(len); self.execute_memory_copy_impl(store, dst, src, len) } @@ -145,13 +149,13 @@ impl Executor<'_> { pub fn execute_memory_copy_to( &mut self, store: &mut StoreInner, - dst: Const16, + dst: Const16, src: Reg, len: Reg, ) -> Result<(), Error> { - let dst: u32 = dst.into(); - let src: u32 = self.get_register_as(src); - let len: u32 = self.get_register_as(len); + let dst: u64 = dst.into(); + let src: u64 = self.get_register_as(src); + let len: u64 = self.get_register_as(len); self.execute_memory_copy_impl(store, dst, src, len) } @@ -160,12 +164,12 @@ impl Executor<'_> { &mut self, store: &mut StoreInner, dst: Reg, - src: Const16, + src: Const16, len: Reg, ) -> Result<(), Error> { - let dst: u32 = self.get_register_as(dst); - let src: u32 = src.into(); - let len: u32 = self.get_register_as(len); + let dst: u64 = self.get_register_as(dst); + let src: u64 = src.into(); + let len: u64 = self.get_register_as(len); self.execute_memory_copy_impl(store, dst, src, len) } @@ -173,13 +177,13 @@ impl Executor<'_> { pub fn execute_memory_copy_from_to( &mut self, store: &mut StoreInner, - dst: Const16, - src: Const16, + dst: Const16, + src: Const16, len: Reg, ) -> Result<(), Error> { - let dst: u32 = dst.into(); - let src: u32 = src.into(); - let len: u32 = self.get_register_as(len); + let dst: u64 = dst.into(); + let src: u64 = src.into(); + let len: u64 = self.get_register_as(len); self.execute_memory_copy_impl(store, dst, src, len) } @@ -189,11 +193,11 @@ impl Executor<'_> { store: &mut StoreInner, dst: Reg, src: Reg, - len: Const16, + len: Const16, ) -> Result<(), Error> { - let dst: u32 = self.get_register_as(dst); - let src: u32 = self.get_register_as(src); - let len: u32 = len.into(); + let dst: u64 = self.get_register_as(dst); + let src: u64 = self.get_register_as(src); + let len: u64 = len.into(); self.execute_memory_copy_impl(store, dst, src, len) } @@ -201,13 +205,13 @@ impl Executor<'_> { pub fn execute_memory_copy_to_exact( &mut self, store: &mut StoreInner, - dst: Const16, + dst: Const16, src: Reg, - len: Const16, + len: Const16, ) -> Result<(), Error> { - let dst: u32 = dst.into(); - let src: u32 = self.get_register_as(src); - let len: u32 = len.into(); + let dst: u64 = dst.into(); + let src: u64 = self.get_register_as(src); + let len: u64 = len.into(); self.execute_memory_copy_impl(store, dst, src, len) } @@ -216,12 +220,12 @@ impl Executor<'_> { &mut self, store: &mut StoreInner, dst: Reg, - src: Const16, - len: Const16, + src: Const16, + len: Const16, ) -> Result<(), Error> { - let dst: u32 = self.get_register_as(dst); - let src: u32 = src.into(); - let len: u32 = len.into(); + let dst: u64 = self.get_register_as(dst); + let src: u64 = src.into(); + let len: u64 = len.into(); self.execute_memory_copy_impl(store, dst, src, len) } @@ -229,13 +233,13 @@ impl Executor<'_> { pub fn execute_memory_copy_from_to_exact( &mut self, store: &mut StoreInner, - dst: Const16, - src: Const16, - len: Const16, + dst: Const16, + src: Const16, + len: Const16, ) -> Result<(), Error> { - let dst: u32 = dst.into(); - let src: u32 = src.into(); - let len: u32 = len.into(); + let dst: u64 = dst.into(); + let src: u64 = src.into(); + let len: u64 = len.into(); self.execute_memory_copy_impl(store, dst, src, len) } @@ -244,14 +248,21 @@ impl Executor<'_> { fn execute_memory_copy_impl( &mut self, store: &mut StoreInner, - dst_index: u32, - src_index: u32, - len: u32, + dst_index: u64, + src_index: u64, + len: u64, ) -> Result<(), Error> { + let Ok(dst_index) = usize::try_from(dst_index) else { + return Err(Error::from(TrapCode::MemoryOutOfBounds)); + }; + let Ok(src_index) = usize::try_from(src_index) else { + return Err(Error::from(TrapCode::MemoryOutOfBounds)); + }; + let Ok(len) = usize::try_from(len) else { + return Err(Error::from(TrapCode::MemoryOutOfBounds)); + }; let dst_memory = self.fetch_memory_index(1); let src_memory = self.fetch_memory_index(2); - let src_index = src_index as usize; - let dst_index = dst_index as usize; if src_memory == dst_memory { return self .execute_memory_copy_within_impl(store, src_memory, dst_index, src_index, len); @@ -264,14 +275,14 @@ impl Executor<'_> { let src_bytes = src_memory .data() .get(src_index..) - .and_then(|memory| memory.get(..len as usize)) + .and_then(|memory| memory.get(..len)) .ok_or(TrapCode::MemoryOutOfBounds)?; let dst_bytes = dst_memory .data_mut() .get_mut(dst_index..) - .and_then(|memory| memory.get_mut(..len as usize)) + .and_then(|memory| memory.get_mut(..len)) .ok_or(TrapCode::MemoryOutOfBounds)?; - fuel.consume_fuel_if(|costs| costs.fuel_for_bytes(u64::from(len)))?; + fuel.consume_fuel_if(|costs| costs.fuel_for_bytes(len as u64))?; dst_bytes.copy_from_slice(src_bytes); self.try_next_instr_at(3) } @@ -283,7 +294,7 @@ impl Executor<'_> { memory: Memory, dst_index: usize, src_index: usize, - len: u32, + len: usize, ) -> Result<(), Error> { let memory = self.get_memory(memory); let (memory, fuel) = store.resolve_memory_and_fuel_mut(&memory); @@ -291,14 +302,14 @@ impl Executor<'_> { // These accesses just perform the bounds checks required by the Wasm spec. bytes .get(src_index..) - .and_then(|memory| memory.get(..len as usize)) + .and_then(|memory| memory.get(..len)) .ok_or(TrapCode::MemoryOutOfBounds)?; bytes .get(dst_index..) - .and_then(|memory| memory.get(..len as usize)) + .and_then(|memory| memory.get(..len)) .ok_or(TrapCode::MemoryOutOfBounds)?; - fuel.consume_fuel_if(|costs| costs.fuel_for_bytes(u64::from(len)))?; - bytes.copy_within(src_index..src_index.wrapping_add(len as usize), dst_index); + fuel.consume_fuel_if(|costs| costs.fuel_for_bytes(len as u64))?; + bytes.copy_within(src_index..src_index.wrapping_add(len), dst_index); self.try_next_instr_at(3) } @@ -310,9 +321,9 @@ impl Executor<'_> { value: Reg, len: Reg, ) -> Result<(), Error> { - let dst: u32 = self.get_register_as(dst); + let dst: u64 = self.get_register_as(dst); let value: u8 = self.get_register_as(value); - let len: u32 = self.get_register_as(len); + let len: u64 = self.get_register_as(len); self.execute_memory_fill_impl(store, dst, value, len) } @@ -320,13 +331,13 @@ impl Executor<'_> { pub fn execute_memory_fill_at( &mut self, store: &mut StoreInner, - dst: Const16, + dst: Const16, value: Reg, len: Reg, ) -> Result<(), Error> { - let dst: u32 = dst.into(); + let dst: u64 = dst.into(); let value: u8 = self.get_register_as(value); - let len: u32 = self.get_register_as(len); + let len: u64 = self.get_register_as(len); self.execute_memory_fill_impl(store, dst, value, len) } @@ -338,8 +349,8 @@ impl Executor<'_> { value: u8, len: Reg, ) -> Result<(), Error> { - let dst: u32 = self.get_register_as(dst); - let len: u32 = self.get_register_as(len); + let dst: u64 = self.get_register_as(dst); + let len: u64 = self.get_register_as(len); self.execute_memory_fill_impl(store, dst, value, len) } @@ -347,12 +358,12 @@ impl Executor<'_> { pub fn execute_memory_fill_at_imm( &mut self, store: &mut StoreInner, - dst: Const16, + dst: Const16, value: u8, len: Reg, ) -> Result<(), Error> { - let dst: u32 = dst.into(); - let len: u32 = self.get_register_as(len); + let dst: u64 = dst.into(); + let len: u64 = self.get_register_as(len); self.execute_memory_fill_impl(store, dst, value, len) } @@ -362,11 +373,11 @@ impl Executor<'_> { store: &mut StoreInner, dst: Reg, value: Reg, - len: Const16, + len: Const16, ) -> Result<(), Error> { - let dst: u32 = self.get_register_as(dst); + let dst: u64 = self.get_register_as(dst); let value: u8 = self.get_register_as(value); - let len: u32 = len.into(); + let len: u64 = len.into(); self.execute_memory_fill_impl(store, dst, value, len) } @@ -374,13 +385,13 @@ impl Executor<'_> { pub fn execute_memory_fill_at_exact( &mut self, store: &mut StoreInner, - dst: Const16, + dst: Const16, value: Reg, - len: Const16, + len: Const16, ) -> Result<(), Error> { - let dst: u32 = dst.into(); + let dst: u64 = dst.into(); let value: u8 = self.get_register_as(value); - let len: u32 = len.into(); + let len: u64 = len.into(); self.execute_memory_fill_impl(store, dst, value, len) } @@ -390,10 +401,10 @@ impl Executor<'_> { store: &mut StoreInner, dst: Reg, value: u8, - len: Const16, + len: Const16, ) -> Result<(), Error> { - let dst: u32 = self.get_register_as(dst); - let len: u32 = len.into(); + let dst: u64 = self.get_register_as(dst); + let len: u64 = len.into(); self.execute_memory_fill_impl(store, dst, value, len) } @@ -401,12 +412,12 @@ impl Executor<'_> { pub fn execute_memory_fill_at_imm_exact( &mut self, store: &mut StoreInner, - dst: Const16, + dst: Const16, value: u8, - len: Const16, + len: Const16, ) -> Result<(), Error> { - let dst: u32 = dst.into(); - let len: u32 = len.into(); + let dst: u64 = dst.into(); + let len: u64 = len.into(); self.execute_memory_fill_impl(store, dst, value, len) } @@ -415,13 +426,17 @@ impl Executor<'_> { fn execute_memory_fill_impl( &mut self, store: &mut StoreInner, - dst: u32, + dst: u64, value: u8, - len: u32, + len: u64, ) -> Result<(), Error> { + let Ok(dst) = usize::try_from(dst) else { + return Err(Error::from(TrapCode::MemoryOutOfBounds)); + }; + let Ok(len) = usize::try_from(len) else { + return Err(Error::from(TrapCode::MemoryOutOfBounds)); + }; let memory = self.fetch_memory_index(1); - let dst = dst as usize; - let len = len as usize; let memory = self.get_memory(memory); let (memory, fuel) = store.resolve_memory_and_fuel_mut(&memory); let slice = memory @@ -442,7 +457,7 @@ impl Executor<'_> { src: Reg, len: Reg, ) -> Result<(), Error> { - let dst: u32 = self.get_register_as(dst); + let dst: u64 = self.get_register_as(dst); let src: u32 = self.get_register_as(src); let len: u32 = self.get_register_as(len); self.execute_memory_init_impl(store, dst, src, len) @@ -452,11 +467,11 @@ impl Executor<'_> { pub fn execute_memory_init_to( &mut self, store: &mut StoreInner, - dst: Const16, + dst: Const16, src: Reg, len: Reg, ) -> Result<(), Error> { - let dst: u32 = dst.into(); + let dst: u64 = dst.into(); let src: u32 = self.get_register_as(src); let len: u32 = self.get_register_as(len); self.execute_memory_init_impl(store, dst, src, len) @@ -470,7 +485,7 @@ impl Executor<'_> { src: Const16, len: Reg, ) -> Result<(), Error> { - let dst: u32 = self.get_register_as(dst); + let dst: u64 = self.get_register_as(dst); let src: u32 = src.into(); let len: u32 = self.get_register_as(len); self.execute_memory_init_impl(store, dst, src, len) @@ -480,11 +495,11 @@ impl Executor<'_> { pub fn execute_memory_init_from_to( &mut self, store: &mut StoreInner, - dst: Const16, + dst: Const16, src: Const16, len: Reg, ) -> Result<(), Error> { - let dst: u32 = dst.into(); + let dst: u64 = dst.into(); let src: u32 = src.into(); let len: u32 = self.get_register_as(len); self.execute_memory_init_impl(store, dst, src, len) @@ -498,7 +513,7 @@ impl Executor<'_> { src: Reg, len: Const16, ) -> Result<(), Error> { - let dst: u32 = self.get_register_as(dst); + let dst: u64 = self.get_register_as(dst); let src: u32 = self.get_register_as(src); let len: u32 = len.into(); self.execute_memory_init_impl(store, dst, src, len) @@ -508,11 +523,11 @@ impl Executor<'_> { pub fn execute_memory_init_to_exact( &mut self, store: &mut StoreInner, - dst: Const16, + dst: Const16, src: Reg, len: Const16, ) -> Result<(), Error> { - let dst: u32 = dst.into(); + let dst: u64 = dst.into(); let src: u32 = self.get_register_as(src); let len: u32 = len.into(); self.execute_memory_init_impl(store, dst, src, len) @@ -526,7 +541,7 @@ impl Executor<'_> { src: Const16, len: Const16, ) -> Result<(), Error> { - let dst: u32 = self.get_register_as(dst); + let dst: u64 = self.get_register_as(dst); let src: u32 = src.into(); let len: u32 = len.into(); self.execute_memory_init_impl(store, dst, src, len) @@ -536,11 +551,11 @@ impl Executor<'_> { pub fn execute_memory_init_from_to_exact( &mut self, store: &mut StoreInner, - dst: Const16, + dst: Const16, src: Const16, len: Const16, ) -> Result<(), Error> { - let dst: u32 = dst.into(); + let dst: u64 = dst.into(); let src: u32 = src.into(); let len: u32 = len.into(); self.execute_memory_init_impl(store, dst, src, len) @@ -551,13 +566,19 @@ impl Executor<'_> { fn execute_memory_init_impl( &mut self, store: &mut StoreInner, - dst: u32, + dst: u64, src: u32, len: u32, ) -> Result<(), Error> { - let dst_index = dst as usize; - let src_index = src as usize; - let len = len as usize; + let Ok(dst_index) = usize::try_from(dst) else { + return Err(Error::from(TrapCode::MemoryOutOfBounds)); + }; + let Ok(src_index) = usize::try_from(src) else { + return Err(Error::from(TrapCode::MemoryOutOfBounds)); + }; + let Ok(len) = usize::try_from(len) else { + return Err(Error::from(TrapCode::MemoryOutOfBounds)); + }; let memory_index: Memory = self.fetch_memory_index(1); let data_index: Data = self.fetch_data_segment_index(2); let (memory, data, fuel) = store.resolve_memory_init_params( diff --git a/crates/wasmi/src/engine/executor/instrs/store.rs b/crates/wasmi/src/engine/executor/instrs/store.rs index 96dcd56814..54adca2a6f 100644 --- a/crates/wasmi/src/engine/executor/instrs/store.rs +++ b/crates/wasmi/src/engine/executor/instrs/store.rs @@ -2,54 +2,53 @@ use super::{Executor, InstructionPtr}; use crate::{ core::{TrapCode, UntypedVal}, engine::utils::unreachable_unchecked, - ir::{index::Memory, AnyConst16, Const16, Instruction, Reg}, + ir::{ + index::Memory, + Address32, + AnyConst16, + Const16, + Offset16, + Offset64, + Offset64Hi, + Offset64Lo, + Reg, + }, store::StoreInner, Error, }; +#[cfg(doc)] +use crate::ir::Instruction; + /// The function signature of Wasm store operations. type WasmStoreOp = fn( memory: &mut [u8], address: UntypedVal, - offset: u32, + offset: u64, value: UntypedVal, ) -> Result<(), TrapCode>; impl Executor<'_> { /// Returns the register `value` and `offset` parameters for a `load` [`Instruction`]. - fn fetch_value_and_offset(&self) -> (Reg, u32) { - let mut addr: InstructionPtr = self.ip; - addr.add(1); - match *addr.get() { - Instruction::RegisterAndImm32 { reg, imm } => (reg, u32::from(imm)), - unexpected => { - // Safety: Wasmi translation guarantees that [`Instruction::RegisterAndImm32`] exists. - unsafe { - unreachable_unchecked!( - "expected `Instruction::RegisterAndImm32` but found {unexpected:?}" - ) - } - } - } + fn fetch_value_and_offset_hi(&self) -> (Reg, Offset64Hi) { + // Safety: Wasmi translation guarantees that `Instruction::RegisterAndImm32` exists. + unsafe { self.fetch_reg_and_offset_hi() } } - /// Returns the immediate `value` and `offset` parameters for a `load` [`Instruction`]. - fn fetch_value_and_offset_imm(&self) -> (T, u32) + /// Returns the immediate `value` and `offset_hi` parameters for a `load` [`Instruction`]. + fn fetch_value_and_offset_imm(&self) -> (T, Offset64Hi) where T: From, { let mut addr: InstructionPtr = self.ip; addr.add(1); - match *addr.get() { - Instruction::Imm16AndImm32 { imm16, imm32 } => (T::from(imm16), u32::from(imm32)), - unexpected => { - // Safety: Wasmi translation guarantees that [`Instruction::Imm16AndImm32`] exists. - unsafe { - unreachable_unchecked!( - "expected `Instruction::Imm16AndImm32` but found {unexpected:?}" - ) - } - } + match addr.get().filter_imm16_and_offset_hi::() { + Ok(value) => value, + Err(instr) => unsafe { + unreachable_unchecked!( + "expected an `Instruction::RegisterAndImm32` but found: {instr:?}" + ) + }, } } @@ -109,12 +108,12 @@ impl Executor<'_> { store: &mut StoreInner, memory: Memory, address: UntypedVal, - offset: u32, + offset: Offset64, value: UntypedVal, store_wrap: WasmStoreOp, ) -> Result<(), Error> { let memory = self.fetch_memory_bytes_mut(memory, store); - store_wrap(memory, address, offset, value)?; + store_wrap(memory, address, u64::from(offset), value)?; Ok(()) } @@ -131,12 +130,12 @@ impl Executor<'_> { fn execute_store_wrap_mem0( &mut self, address: UntypedVal, - offset: u32, + offset: Offset64, value: UntypedVal, store_wrap: WasmStoreOp, ) -> Result<(), Error> { let memory = self.fetch_default_memory_bytes_mut(); - store_wrap(memory, address, offset, value)?; + store_wrap(memory, address, u64::from(offset), value)?; Ok(()) } @@ -144,10 +143,12 @@ impl Executor<'_> { &mut self, store: &mut StoreInner, ptr: Reg, - memory: Memory, + offset_lo: Offset64Lo, store_op: WasmStoreOp, ) -> Result<(), Error> { - let (value, offset) = self.fetch_value_and_offset(); + let (value, offset_hi) = self.fetch_value_and_offset_hi(); + let memory = self.fetch_optional_memory(2); + let offset = Offset64::combine(offset_hi, offset_lo); self.execute_store_wrap( store, memory, @@ -163,13 +164,15 @@ impl Executor<'_> { &mut self, store: &mut StoreInner, ptr: Reg, - memory: Memory, + offset_lo: Offset64Lo, store_op: WasmStoreOp, ) -> Result<(), Error> where T: From + Into, { - let (value, offset) = self.fetch_value_and_offset_imm::(); + let (value, offset_hi) = self.fetch_value_and_offset_imm::(); + let memory = self.fetch_optional_memory(2); + let offset = Offset64::combine(offset_hi, offset_lo); self.execute_store_wrap( store, memory, @@ -184,13 +187,13 @@ impl Executor<'_> { fn execute_store_offset16( &mut self, ptr: Reg, - offset: Const16, + offset: Offset16, value: Reg, store_op: WasmStoreOp, ) -> Result<(), Error> { self.execute_store_wrap_mem0( self.get_register(ptr), - u32::from(offset), + Offset64::from(offset), self.get_register(value), store_op, )?; @@ -200,7 +203,7 @@ impl Executor<'_> { fn execute_store_offset16_imm16( &mut self, ptr: Reg, - offset: Const16, + offset: Offset16, value: V, store_op: WasmStoreOp, ) -> Result<(), Error> @@ -209,7 +212,7 @@ impl Executor<'_> { { self.execute_store_wrap_mem0( self.get_register(ptr), - u32::from(offset), + Offset64::from(offset), T::from(value).into(), store_op, )?; @@ -219,16 +222,16 @@ impl Executor<'_> { fn execute_store_at( &mut self, store: &mut StoreInner, - address: u32, + address: Address32, value: Reg, store_op: WasmStoreOp, ) -> Result<(), Error> { - let memory = self.fetch_optional_memory(); + let memory = self.fetch_optional_memory(1); self.execute_store_wrap( store, memory, - UntypedVal::from(0u32), - address, + UntypedVal::from(u64::from(address)), + Offset64::from(0), self.get_register(value), store_op, )?; @@ -238,19 +241,19 @@ impl Executor<'_> { fn execute_store_at_imm16( &mut self, store: &mut StoreInner, - address: u32, + address: Address32, value: V, store_op: WasmStoreOp, ) -> Result<(), Error> where T: From + Into, { - let memory = self.fetch_optional_memory(); + let memory = self.fetch_optional_memory(1); self.execute_store_wrap( store, memory, - UntypedVal::from(0u32), - address, + UntypedVal::from(u64::from(address)), + Offset64::from(0), T::from(value).into(), store_op, )?; @@ -270,15 +273,15 @@ macro_rules! impl_execute_istore { ),* $(,)? ) => { $( #[doc = concat!("Executes an [`Instruction::", stringify!($var_store_imm), "`].")] - pub fn $fn_store_imm(&mut self, store: &mut StoreInner, ptr: Reg, memory: Memory) -> Result<(), Error> { - self.execute_store_imm::<$to_ty>(store, ptr, memory, $impl_fn) + pub fn $fn_store_imm(&mut self, store: &mut StoreInner, ptr: Reg, offset_lo: Offset64Lo) -> Result<(), Error> { + self.execute_store_imm::<$to_ty>(store, ptr, offset_lo, $impl_fn) } #[doc = concat!("Executes an [`Instruction::", stringify!($var_store_off16_imm16), "`].")] pub fn $fn_store_off16_imm16( &mut self, ptr: Reg, - offset: Const16, + offset: Offset16, value: $from_ty, ) -> Result<(), Error> { self.execute_store_offset16_imm16::<$to_ty, _>(ptr, offset, value, $impl_fn) @@ -288,7 +291,7 @@ macro_rules! impl_execute_istore { pub fn $fn_store_at_imm16( &mut self, store: &mut StoreInner, - address: u32, + address: Address32, value: $from_ty, ) -> Result<(), Error> { self.execute_store_at_imm16::<$to_ty, _>(store, address, value, $impl_fn) @@ -340,22 +343,22 @@ macro_rules! impl_execute_istore_trunc { } #[doc = concat!("Executes an [`Instruction::", stringify!($var_store), "`].")] - pub fn $fn_store(&mut self, store: &mut StoreInner, ptr: Reg, memory: Memory) -> Result<(), Error> { - self.execute_store(store, ptr, memory, $impl_fn) + pub fn $fn_store(&mut self, store: &mut StoreInner, ptr: Reg, offset_lo: Offset64Lo) -> Result<(), Error> { + self.execute_store(store, ptr, offset_lo, $impl_fn) } #[doc = concat!("Executes an [`Instruction::", stringify!($var_store_off16), "`].")] pub fn $fn_store_off16( &mut self, ptr: Reg, - offset: Const16, + offset: Offset16, value: Reg, ) -> Result<(), Error> { self.execute_store_offset16(ptr, offset, value, $impl_fn) } #[doc = concat!("Executes an [`Instruction::", stringify!($var_store_at), "`].")] - pub fn $fn_store_at(&mut self, store: &mut StoreInner, address: u32, value: Reg) -> Result<(), Error> { + pub fn $fn_store_at(&mut self, store: &mut StoreInner, address: Address32, value: Reg) -> Result<(), Error> { self.execute_store_at(store, address, value, $impl_fn) } )* @@ -427,22 +430,22 @@ macro_rules! impl_execute_store { ),* $(,)? ) => { $( #[doc = concat!("Executes an [`Instruction::", stringify!($var_store), "`].")] - pub fn $fn_store(&mut self, store: &mut StoreInner, ptr: Reg, memory: Memory) -> Result<(), Error> { - self.execute_store(store, ptr, memory, $impl_fn) + pub fn $fn_store(&mut self, store: &mut StoreInner, ptr: Reg, offset_lo: Offset64Lo) -> Result<(), Error> { + self.execute_store(store, ptr, offset_lo, $impl_fn) } #[doc = concat!("Executes an [`Instruction::", stringify!($var_store_off16), "`].")] pub fn $fn_store_off16( &mut self, ptr: Reg, - offset: Const16, + offset: Offset16, value: Reg, ) -> Result<(), Error> { self.execute_store_offset16(ptr, offset, value, $impl_fn) } #[doc = concat!("Executes an [`Instruction::", stringify!($var_store_at), "`].")] - pub fn $fn_store_at(&mut self, store: &mut StoreInner,address: u32, value: Reg) -> Result<(), Error> { + pub fn $fn_store_at(&mut self, store: &mut StoreInner,address: Address32, value: Reg) -> Result<(), Error> { self.execute_store_at(store, address, value, $impl_fn) } )* diff --git a/crates/wasmi/src/engine/executor/instrs/table.rs b/crates/wasmi/src/engine/executor/instrs/table.rs index e9b307b662..94edc7172e 100644 --- a/crates/wasmi/src/engine/executor/instrs/table.rs +++ b/crates/wasmi/src/engine/executor/instrs/table.rs @@ -6,6 +6,7 @@ use crate::{ ir::{ index::{Elem, Table}, Const16, + Const32, Instruction, Reg, }, @@ -57,7 +58,7 @@ impl Executor<'_> { result: Reg, index: Reg, ) -> Result<(), Error> { - let index: u32 = self.get_register_as(index); + let index: u64 = self.get_register_as(index); self.execute_table_get_impl(store, result, index) } @@ -66,8 +67,9 @@ impl Executor<'_> { &mut self, store: &StoreInner, result: Reg, - index: u32, + index: Const32, ) -> Result<(), Error> { + let index: u64 = index.into(); self.execute_table_get_impl(store, result, index) } @@ -76,7 +78,7 @@ impl Executor<'_> { &mut self, store: &StoreInner, result: Reg, - index: u32, + index: u64, ) -> Result<(), Error> { let table_index = self.fetch_table_index(1); let table = self.get_table(table_index); @@ -108,7 +110,7 @@ impl Executor<'_> { index: Reg, value: Reg, ) -> Result<(), Error> { - let index: u32 = self.get_register_as(index); + let index: u64 = self.get_register_as(index); self.execute_table_set_impl(store, index, value) } @@ -116,9 +118,10 @@ impl Executor<'_> { pub fn execute_table_set_at( &mut self, store: &mut StoreInner, - index: u32, + index: Const32, value: Reg, ) -> Result<(), Error> { + let index: u64 = index.into(); self.execute_table_set_impl(store, index, value) } @@ -126,7 +129,7 @@ impl Executor<'_> { fn execute_table_set_impl( &mut self, store: &mut StoreInner, - index: u32, + index: u64, value: Reg, ) -> Result<(), Error> { let table_index = self.fetch_table_index(1); @@ -147,9 +150,9 @@ impl Executor<'_> { src: Reg, len: Reg, ) -> Result<(), Error> { - let dst: u32 = self.get_register_as(dst); - let src: u32 = self.get_register_as(src); - let len: u32 = self.get_register_as(len); + let dst: u64 = self.get_register_as(dst); + let src: u64 = self.get_register_as(src); + let len: u64 = self.get_register_as(len); self.execute_table_copy_impl(store, dst, src, len) } @@ -157,13 +160,13 @@ impl Executor<'_> { pub fn execute_table_copy_to( &mut self, store: &mut StoreInner, - dst: Const16, + dst: Const16, src: Reg, len: Reg, ) -> Result<(), Error> { - let dst: u32 = dst.into(); - let src: u32 = self.get_register_as(src); - let len: u32 = self.get_register_as(len); + let dst: u64 = dst.into(); + let src: u64 = self.get_register_as(src); + let len: u64 = self.get_register_as(len); self.execute_table_copy_impl(store, dst, src, len) } @@ -172,12 +175,12 @@ impl Executor<'_> { &mut self, store: &mut StoreInner, dst: Reg, - src: Const16, + src: Const16, len: Reg, ) -> Result<(), Error> { - let dst: u32 = self.get_register_as(dst); - let src: u32 = src.into(); - let len: u32 = self.get_register_as(len); + let dst: u64 = self.get_register_as(dst); + let src: u64 = src.into(); + let len: u64 = self.get_register_as(len); self.execute_table_copy_impl(store, dst, src, len) } @@ -185,13 +188,13 @@ impl Executor<'_> { pub fn execute_table_copy_from_to( &mut self, store: &mut StoreInner, - dst: Const16, - src: Const16, + dst: Const16, + src: Const16, len: Reg, ) -> Result<(), Error> { - let dst: u32 = dst.into(); - let src: u32 = src.into(); - let len: u32 = self.get_register_as(len); + let dst: u64 = dst.into(); + let src: u64 = src.into(); + let len: u64 = self.get_register_as(len); self.execute_table_copy_impl(store, dst, src, len) } @@ -201,11 +204,11 @@ impl Executor<'_> { store: &mut StoreInner, dst: Reg, src: Reg, - len: Const16, + len: Const16, ) -> Result<(), Error> { - let dst: u32 = self.get_register_as(dst); - let src: u32 = self.get_register_as(src); - let len: u32 = len.into(); + let dst: u64 = self.get_register_as(dst); + let src: u64 = self.get_register_as(src); + let len: u64 = len.into(); self.execute_table_copy_impl(store, dst, src, len) } @@ -213,13 +216,13 @@ impl Executor<'_> { pub fn execute_table_copy_to_exact( &mut self, store: &mut StoreInner, - dst: Const16, + dst: Const16, src: Reg, - len: Const16, + len: Const16, ) -> Result<(), Error> { - let dst: u32 = dst.into(); - let src: u32 = self.get_register_as(src); - let len: u32 = len.into(); + let dst: u64 = dst.into(); + let src: u64 = self.get_register_as(src); + let len: u64 = len.into(); self.execute_table_copy_impl(store, dst, src, len) } @@ -228,12 +231,12 @@ impl Executor<'_> { &mut self, store: &mut StoreInner, dst: Reg, - src: Const16, - len: Const16, + src: Const16, + len: Const16, ) -> Result<(), Error> { - let dst: u32 = self.get_register_as(dst); - let src: u32 = src.into(); - let len: u32 = len.into(); + let dst: u64 = self.get_register_as(dst); + let src: u64 = src.into(); + let len: u64 = len.into(); self.execute_table_copy_impl(store, dst, src, len) } @@ -241,13 +244,13 @@ impl Executor<'_> { pub fn execute_table_copy_from_to_exact( &mut self, store: &mut StoreInner, - dst: Const16, - src: Const16, - len: Const16, + dst: Const16, + src: Const16, + len: Const16, ) -> Result<(), Error> { - let dst: u32 = dst.into(); - let src: u32 = src.into(); - let len: u32 = len.into(); + let dst: u64 = dst.into(); + let src: u64 = src.into(); + let len: u64 = len.into(); self.execute_table_copy_impl(store, dst, src, len) } @@ -256,9 +259,9 @@ impl Executor<'_> { fn execute_table_copy_impl( &mut self, store: &mut StoreInner, - dst_index: u32, - src_index: u32, - len: u32, + dst_index: u64, + src_index: u64, + len: u64, ) -> Result<(), Error> { let dst_table_index = self.fetch_table_index(1); let src_table_index = self.fetch_table_index(2); @@ -287,7 +290,7 @@ impl Executor<'_> { src: Reg, len: Reg, ) -> Result<(), Error> { - let dst: u32 = self.get_register_as(dst); + let dst: u64 = self.get_register_as(dst); let src: u32 = self.get_register_as(src); let len: u32 = self.get_register_as(len); self.execute_table_init_impl(store, dst, src, len) @@ -297,11 +300,11 @@ impl Executor<'_> { pub fn execute_table_init_to( &mut self, store: &mut StoreInner, - dst: Const16, + dst: Const16, src: Reg, len: Reg, ) -> Result<(), Error> { - let dst: u32 = dst.into(); + let dst: u64 = dst.into(); let src: u32 = self.get_register_as(src); let len: u32 = self.get_register_as(len); self.execute_table_init_impl(store, dst, src, len) @@ -315,7 +318,7 @@ impl Executor<'_> { src: Const16, len: Reg, ) -> Result<(), Error> { - let dst: u32 = self.get_register_as(dst); + let dst: u64 = self.get_register_as(dst); let src: u32 = src.into(); let len: u32 = self.get_register_as(len); self.execute_table_init_impl(store, dst, src, len) @@ -325,11 +328,11 @@ impl Executor<'_> { pub fn execute_table_init_from_to( &mut self, store: &mut StoreInner, - dst: Const16, + dst: Const16, src: Const16, len: Reg, ) -> Result<(), Error> { - let dst: u32 = dst.into(); + let dst: u64 = dst.into(); let src: u32 = src.into(); let len: u32 = self.get_register_as(len); self.execute_table_init_impl(store, dst, src, len) @@ -343,7 +346,7 @@ impl Executor<'_> { src: Reg, len: Const16, ) -> Result<(), Error> { - let dst: u32 = self.get_register_as(dst); + let dst: u64 = self.get_register_as(dst); let src: u32 = self.get_register_as(src); let len: u32 = len.into(); self.execute_table_init_impl(store, dst, src, len) @@ -353,11 +356,11 @@ impl Executor<'_> { pub fn execute_table_init_to_exact( &mut self, store: &mut StoreInner, - dst: Const16, + dst: Const16, src: Reg, len: Const16, ) -> Result<(), Error> { - let dst: u32 = dst.into(); + let dst: u64 = dst.into(); let src: u32 = self.get_register_as(src); let len: u32 = len.into(); self.execute_table_init_impl(store, dst, src, len) @@ -371,7 +374,7 @@ impl Executor<'_> { src: Const16, len: Const16, ) -> Result<(), Error> { - let dst: u32 = self.get_register_as(dst); + let dst: u64 = self.get_register_as(dst); let src: u32 = src.into(); let len: u32 = len.into(); self.execute_table_init_impl(store, dst, src, len) @@ -381,11 +384,11 @@ impl Executor<'_> { pub fn execute_table_init_from_to_exact( &mut self, store: &mut StoreInner, - dst: Const16, + dst: Const16, src: Const16, len: Const16, ) -> Result<(), Error> { - let dst: u32 = dst.into(); + let dst: u64 = dst.into(); let src: u32 = src.into(); let len: u32 = len.into(); self.execute_table_init_impl(store, dst, src, len) @@ -396,7 +399,7 @@ impl Executor<'_> { fn execute_table_init_impl( &mut self, store: &mut StoreInner, - dst_index: u32, + dst_index: u64, src_index: u32, len: u32, ) -> Result<(), Error> { @@ -418,8 +421,8 @@ impl Executor<'_> { len: Reg, value: Reg, ) -> Result<(), Error> { - let dst: u32 = self.get_register_as(dst); - let len: u32 = self.get_register_as(len); + let dst: u64 = self.get_register_as(dst); + let len: u64 = self.get_register_as(len); self.execute_table_fill_impl(store, dst, len, value) } @@ -427,12 +430,12 @@ impl Executor<'_> { pub fn execute_table_fill_at( &mut self, store: &mut StoreInner, - dst: Const16, + dst: Const16, len: Reg, value: Reg, ) -> Result<(), Error> { - let dst: u32 = dst.into(); - let len: u32 = self.get_register_as(len); + let dst: u64 = dst.into(); + let len: u64 = self.get_register_as(len); self.execute_table_fill_impl(store, dst, len, value) } @@ -441,11 +444,11 @@ impl Executor<'_> { &mut self, store: &mut StoreInner, dst: Reg, - len: Const16, + len: Const16, value: Reg, ) -> Result<(), Error> { - let dst: u32 = self.get_register_as(dst); - let len: u32 = len.into(); + let dst: u64 = self.get_register_as(dst); + let len: u64 = len.into(); self.execute_table_fill_impl(store, dst, len, value) } @@ -453,12 +456,12 @@ impl Executor<'_> { pub fn execute_table_fill_at_exact( &mut self, store: &mut StoreInner, - dst: Const16, - len: Const16, + dst: Const16, + len: Const16, value: Reg, ) -> Result<(), Error> { - let dst: u32 = dst.into(); - let len: u32 = len.into(); + let dst: u64 = dst.into(); + let len: u64 = len.into(); self.execute_table_fill_impl(store, dst, len, value) } @@ -467,8 +470,8 @@ impl Executor<'_> { fn execute_table_fill_impl( &mut self, store: &mut StoreInner, - dst: u32, - len: u32, + dst: u64, + len: u64, value: Reg, ) -> Result<(), Error> { let table_index = self.fetch_table_index(1); @@ -487,7 +490,7 @@ impl Executor<'_> { delta: Reg, value: Reg, ) -> Result<(), Error> { - let delta: u32 = self.get_register_as(delta); + let delta: u64 = self.get_register_as(delta); let (store, mut resource_limiter) = store.store_inner_and_resource_limiter_ref(); self.execute_table_grow_impl(store, result, delta, value, &mut resource_limiter) } @@ -497,10 +500,10 @@ impl Executor<'_> { &mut self, store: &mut Store, result: Reg, - delta: Const16, + delta: Const16, value: Reg, ) -> Result<(), Error> { - let delta: u32 = delta.into(); + let delta: u64 = delta.into(); let (store, mut resource_limiter) = store.store_inner_and_resource_limiter_ref(); self.execute_table_grow_impl(store, result, delta, value, &mut resource_limiter) } @@ -511,7 +514,7 @@ impl Executor<'_> { &mut self, store: &'store mut StoreInner, result: Reg, - delta: u32, + delta: u64, value: Reg, resource_limiter: &mut ResourceLimiterRef<'store>, ) -> Result<(), Error> { @@ -527,7 +530,10 @@ impl Executor<'_> { let return_value = table.grow_untyped(delta, value, Some(fuel), resource_limiter); let return_value = match return_value { Ok(return_value) => return_value, - Err(EntityGrowError::InvalidGrow) => EntityGrowError::ERROR_CODE, + Err(EntityGrowError::InvalidGrow) => match table.ty().is_64() { + true => EntityGrowError::ERROR_CODE_64, + false => EntityGrowError::ERROR_CODE_32, + }, Err(EntityGrowError::TrapCode(trap_code)) => return Err(Error::from(trap_code)), }; self.set_register(result, return_value); diff --git a/crates/wasmi/src/engine/translator/mod.rs b/crates/wasmi/src/engine/translator/mod.rs index 0187260e73..dd8e8c54f3 100644 --- a/crates/wasmi/src/engine/translator/mod.rs +++ b/crates/wasmi/src/engine/translator/mod.rs @@ -46,6 +46,7 @@ use crate::{ engine::{config::FuelCosts, BlockType, EngineFunc}, ir::{ index, + Address32, AnyConst16, BoundedRegSpan, BranchOffset, @@ -53,12 +54,15 @@ use crate::{ Const32, Instruction, IntoShiftAmount, + Offset16, + Offset64, + Offset64Lo, Reg, RegSpan, ShiftAmount, Sign, }, - module::{FuncIdx, FuncTypeIdx, ModuleHeader}, + module::{FuncIdx, FuncTypeIdx, MemoryIdx, ModuleHeader, TableIdx}, Engine, Error, ExternRef, @@ -1899,20 +1903,29 @@ impl FuncTranslator { /// # Panics /// /// If the [`MemArg`] offset is not 32-bit. - fn decode_memarg(memarg: MemArg) -> (index::Memory, u32) { + fn decode_memarg(memarg: MemArg) -> (index::Memory, u64) { let memory = index::Memory::from(memarg.memory); - let offset = u32::try_from(memarg.offset).unwrap_or_else(|_| { - panic!( - "encountered 64-bit memory load/store offset: {}", - memarg.offset - ) - }); - (memory, offset) + (memory, memarg.offset) } /// Returns the effective address `ptr+offset` if it is valid. - fn effective_address(ptr: u32, offset: u32) -> Option { - ptr.checked_add(offset) + fn effective_address(&self, mem: index::Memory, ptr: TypedVal, offset: u64) -> Option { + let memory_type = *self + .module + .get_type_of_memory(MemoryIdx::from(u32::from(mem))); + let ptr = match memory_type.is_64() { + true => u64::from(ptr), + false => u64::from(u32::from(ptr)), + }; + let Some(address) = ptr.checked_add(offset) else { + // Case: address overflows any legal memory index. + return None; + }; + if !memory_type.is_64() && address >= 1 << 32 { + // Case: address overflows the 32-bit memory index. + return None; + } + Some(address) } /// Translates a Wasm `load` instruction to Wasmi bytecode. @@ -1932,40 +1945,53 @@ impl FuncTranslator { fn translate_load( &mut self, memarg: MemArg, - make_instr: fn(result: Reg, memory: index::Memory) -> Instruction, - make_instr_offset16: fn(result: Reg, ptr: Reg, offset: Const16) -> Instruction, - make_instr_at: fn(result: Reg, address: u32) -> Instruction, + make_instr: fn(result: Reg, offset_lo: Offset64Lo) -> Instruction, + make_instr_offset16: fn(result: Reg, ptr: Reg, offset: Offset16) -> Instruction, + make_instr_at: fn(result: Reg, address: Address32) -> Instruction, ) -> Result<(), Error> { bail_unreachable!(self); let (memory, offset) = Self::decode_memarg(memarg); let ptr = self.alloc.stack.pop(); - let ptr = match ptr { - Provider::Register(ptr) => ptr, + let (ptr, offset) = match ptr { + Provider::Register(ptr) => (ptr, offset), Provider::Const(ptr) => { - let Some(address) = Self::effective_address(u32::from(ptr), offset) else { + let Some(address) = self.effective_address(memory, ptr, offset) else { return self.translate_trap(TrapCode::MemoryOutOfBounds); }; - let result = self.alloc.stack.push_dynamic()?; - self.push_fueled_instr(make_instr_at(result, address), FuelCosts::load)?; - if !memory.is_default() { - self.alloc - .instr_encoder - .append_instr(Instruction::memory_index(memory))?; + if let Ok(address) = Address32::try_from(address) { + let result = self.alloc.stack.push_dynamic()?; + self.push_fueled_instr(make_instr_at(result, address), FuelCosts::load)?; + if !memory.is_default() { + self.alloc + .instr_encoder + .append_instr(Instruction::memory_index(memory))?; + } + return Ok(()); } - return Ok(()); + // Case: we cannot use specialized encoding and thus have to fall back + // to the general case where `ptr` is zero and `offset` stores the + // `ptr+offset` address value. + let zero_ptr = self.alloc.stack.alloc_const(0_u64)?; + (zero_ptr, address) } }; let result = self.alloc.stack.push_dynamic()?; if memory.is_default() { - if let Ok(offset) = >::try_from(offset) { + if let Ok(offset) = Offset16::try_from(offset) { self.push_fueled_instr(make_instr_offset16(result, ptr, offset), FuelCosts::load)?; return Ok(()); } } - self.push_fueled_instr(make_instr(result, memory), FuelCosts::load)?; + let (offset_hi, offset_lo) = Offset64::split(offset); + self.push_fueled_instr(make_instr(result, offset_lo), FuelCosts::load)?; self.alloc .instr_encoder - .append_instr(Instruction::register_and_imm32(ptr, offset))?; + .append_instr(Instruction::register_and_offset_hi(ptr, offset_hi))?; + if !memory.is_default() { + self.alloc + .instr_encoder + .append_instr(Instruction::memory_index(memory))?; + } Ok(()) } @@ -1978,12 +2004,12 @@ impl FuncTranslator { fn translate_istore( &mut self, memarg: MemArg, - make_instr: fn(ptr: Reg, memory: index::Memory) -> Instruction, - make_instr_imm: fn(ptr: Reg, memory: index::Memory) -> Instruction, - make_instr_offset16: fn(ptr: Reg, offset: u16, value: Reg) -> Instruction, - make_instr_offset16_imm: fn(ptr: Reg, offset: u16, value: Field) -> Instruction, - make_instr_at: fn(value: Reg, address: u32) -> Instruction, - make_instr_at_imm: fn(value: Field, address: u32) -> Instruction, + make_instr: fn(ptr: Reg, offset_lo: Offset64Lo) -> Instruction, + make_instr_imm: fn(ptr: Reg, offset_lo: Offset64Lo) -> Instruction, + make_instr_offset16: fn(ptr: Reg, offset: Offset16, value: Reg) -> Instruction, + make_instr_offset16_imm: fn(ptr: Reg, offset: Offset16, value: Field) -> Instruction, + make_instr_at: fn(value: Reg, address: Address32) -> Instruction, + make_instr_at_imm: fn(value: Field, address: Address32) -> Instruction, ) -> Result<(), Error> where Src: Copy + From, @@ -2016,12 +2042,12 @@ impl FuncTranslator { fn translate_istore_wrap( &mut self, memarg: MemArg, - make_instr: fn(ptr: Reg, memory: index::Memory) -> Instruction, - make_instr_imm: fn(ptr: Reg, memory: index::Memory) -> Instruction, - make_instr_offset16: fn(ptr: Reg, offset: u16, value: Reg) -> Instruction, - make_instr_offset16_imm: fn(ptr: Reg, offset: u16, value: Field) -> Instruction, - make_instr_at: fn(value: Reg, address: u32) -> Instruction, - make_instr_at_imm: fn(value: Field, address: u32) -> Instruction, + make_instr: fn(ptr: Reg, offset_lo: Offset64Lo) -> Instruction, + make_instr_imm: fn(ptr: Reg, offset_lo: Offset64Lo) -> Instruction, + make_instr_offset16: fn(ptr: Reg, offset: Offset16, value: Reg) -> Instruction, + make_instr_offset16_imm: fn(ptr: Reg, offset: Offset16, value: Field) -> Instruction, + make_instr_at: fn(value: Reg, address: Address32) -> Instruction, + make_instr_at_imm: fn(value: Field, address: Address32) -> Instruction, ) -> Result<(), Error> where Src: Copy + Wrap + From, @@ -2030,17 +2056,26 @@ impl FuncTranslator { bail_unreachable!(self); let (memory, offset) = Self::decode_memarg(memarg); let (ptr, value) = self.alloc.stack.pop2(); - let ptr = match ptr { - Provider::Register(ptr) => ptr, + let (ptr, offset) = match ptr { + Provider::Register(ptr) => (ptr, offset), Provider::Const(ptr) => { - return self.translate_istore_wrap_at::( - memory, - u32::from(ptr), - offset, - value, - make_instr_at, - make_instr_at_imm, - ) + let Some(address) = self.effective_address(memory, ptr, offset) else { + return self.translate_trap(TrapCode::MemoryOutOfBounds); + }; + if let Ok(address) = Address32::try_from(address) { + return self.translate_istore_wrap_at::( + memory, + address, + value, + make_instr_at, + make_instr_at_imm, + ); + } + // Case: we cannot use specialized encoding and thus have to fall back + // to the general case where `ptr` is zero and `offset` stores the + // `ptr+offset` address value. + let zero_ptr = self.alloc.stack.alloc_const(0_u64)?; + (zero_ptr, address) } }; if memory.is_default() { @@ -2054,24 +2089,33 @@ impl FuncTranslator { return Ok(()); } } + let (offset_hi, offset_lo) = Offset64::split(offset); let (instr, param) = match value { TypedProvider::Register(value) => ( - make_instr(ptr, memory), - Instruction::register_and_imm32(value, offset), + make_instr(ptr, offset_lo), + Instruction::register_and_offset_hi(value, offset_hi), ), TypedProvider::Const(value) => match Field::try_from(Src::from(value).wrap()).ok() { Some(value) => ( - make_instr_imm(ptr, memory), - Instruction::imm16_and_imm32(value, offset), + make_instr_imm(ptr, offset_lo), + Instruction::imm16_and_offset_hi(value, offset_hi), ), None => ( - make_instr(ptr, memory), - Instruction::register_and_imm32(self.alloc.stack.alloc_const(value)?, offset), + make_instr(ptr, offset_lo), + Instruction::register_and_offset_hi( + self.alloc.stack.alloc_const(value)?, + offset_hi, + ), ), }, }; self.push_fueled_instr(instr, FuelCosts::store)?; self.alloc.instr_encoder.append_instr(param)?; + if !memory.is_default() { + self.alloc + .instr_encoder + .append_instr(Instruction::memory_index(memory))?; + } Ok(()) } @@ -2083,19 +2127,15 @@ impl FuncTranslator { fn translate_istore_wrap_at( &mut self, memory: index::Memory, - ptr: u32, - offset: u32, + address: Address32, value: TypedProvider, - make_instr_at: fn(value: Reg, address: u32) -> Instruction, - make_instr_at_imm: fn(value: Field, address: u32) -> Instruction, + make_instr_at: fn(value: Reg, address: Address32) -> Instruction, + make_instr_at_imm: fn(value: Field, address: Address32) -> Instruction, ) -> Result<(), Error> where Src: Copy + From + Wrap, Field: TryFrom, { - let Some(address) = Self::effective_address(ptr, offset) else { - return self.translate_trap(TrapCode::MemoryOutOfBounds); - }; match value { Provider::Register(value) => { self.push_fueled_instr(make_instr_at(value, address), FuelCosts::store)?; @@ -2127,16 +2167,16 @@ impl FuncTranslator { fn translate_istore_wrap_mem0( &mut self, ptr: Reg, - offset: u32, + offset: u64, value: TypedProvider, - make_instr_offset16: fn(Reg, u16, Reg) -> Instruction, - make_instr_offset16_imm: fn(Reg, u16, Field) -> Instruction, + make_instr_offset16: fn(Reg, Offset16, Reg) -> Instruction, + make_instr_offset16_imm: fn(Reg, Offset16, Field) -> Instruction, ) -> Result, Error> where Src: Copy + From + Wrap, Field: TryFrom, { - let Ok(offset16) = u16::try_from(offset) else { + let Ok(offset16) = Offset16::try_from(offset) else { return Ok(None); }; let instr = match value { @@ -2175,36 +2215,43 @@ impl FuncTranslator { fn translate_fstore( &mut self, memarg: MemArg, - make_instr: fn(ptr: Reg, memory: index::Memory) -> Instruction, - make_instr_offset16: fn(ptr: Reg, offset: u16, value: Reg) -> Instruction, - make_instr_at: fn(value: Reg, address: u32) -> Instruction, + make_instr: fn(ptr: Reg, offset_lo: Offset64Lo) -> Instruction, + make_instr_offset16: fn(ptr: Reg, offset: Offset16, value: Reg) -> Instruction, + make_instr_at: fn(value: Reg, address: Address32) -> Instruction, ) -> Result<(), Error> { bail_unreachable!(self); let (memory, offset) = Self::decode_memarg(memarg); let (ptr, value) = self.alloc.stack.pop2(); - let ptr = match ptr { - Provider::Register(ptr) => ptr, + let (ptr, offset) = match ptr { + Provider::Register(ptr) => (ptr, offset), Provider::Const(ptr) => { - return self.translate_fstore_at( - memory, - u32::from(ptr), - offset, - value, - make_instr_at, - ) + let Some(address) = self.effective_address(memory, ptr, offset) else { + return self.translate_trap(TrapCode::MemoryOutOfBounds); + }; + if let Ok(address) = Address32::try_from(address) { + return self.translate_fstore_at(memory, address, value, make_instr_at); + } + let zero_ptr = self.alloc.stack.alloc_const(0_u64)?; + (zero_ptr, address) } }; + let (offset_hi, offset_lo) = Offset64::split(offset); let value = self.alloc.stack.provider2reg(&value)?; if memory.is_default() { - if let Ok(offset) = u16::try_from(offset) { + if let Ok(offset) = Offset16::try_from(offset) { self.push_fueled_instr(make_instr_offset16(ptr, offset, value), FuelCosts::store)?; return Ok(()); } } - self.push_fueled_instr(make_instr(ptr, memory), FuelCosts::store)?; + self.push_fueled_instr(make_instr(ptr, offset_lo), FuelCosts::store)?; self.alloc .instr_encoder - .append_instr(Instruction::register_and_imm32(value, offset))?; + .append_instr(Instruction::register_and_offset_hi(value, offset_hi))?; + if !memory.is_default() { + self.alloc + .instr_encoder + .append_instr(Instruction::memory_index(memory))?; + } Ok(()) } @@ -2216,14 +2263,10 @@ impl FuncTranslator { fn translate_fstore_at( &mut self, memory: index::Memory, - ptr: u32, - offset: u32, + address: Address32, value: TypedProvider, - make_instr_at: fn(value: Reg, address: u32) -> Instruction, + make_instr_at: fn(value: Reg, address: Address32) -> Instruction, ) -> Result<(), Error> { - let Some(address) = Self::effective_address(ptr, offset) else { - return self.translate_trap(TrapCode::MemoryOutOfBounds); - }; let value = self.alloc.stack.provider2reg(&value)?; self.push_fueled_instr(make_instr_at(value, address), FuelCosts::store)?; if !memory.is_default() { @@ -2553,21 +2596,11 @@ impl FuncTranslator { index: Provider, table_index: u32, ) -> Result { + let table_type = *self.module.get_type_of_table(TableIdx::from(table_index)); + let index = self.as_index_type_const16(index, table_type.index_ty())?; let instr = match index { - TypedProvider::Const(index) => match >::try_from(u32::from(index)).ok() { - Some(index) => { - // Case: the index is encodable as 16-bit constant value - // which allows us to use an optimized instruction. - Instruction::call_indirect_params_imm16(index, table_index) - } - None => { - // Case: the index is not encodable as 16-bit constant value - // and we need to allocate it as function local constant. - let index = self.alloc.stack.alloc_const(index)?; - Instruction::call_indirect_params(index, table_index) - } - }, - TypedProvider::Register(index) => Instruction::call_indirect_params(index, table_index), + Provider::Register(index) => Instruction::call_indirect_params(index, table_index), + Provider::Const(index) => Instruction::call_indirect_params_imm16(index, table_index), }; Ok(instr) } diff --git a/crates/wasmi/src/engine/translator/tests/op/call/indirect.rs b/crates/wasmi/src/engine/translator/tests/op/call/indirect.rs index 65feea39d4..17e6bc537b 100644 --- a/crates/wasmi/src/engine/translator/tests/op/call/indirect.rs +++ b/crates/wasmi/src/engine/translator/tests/op/call/indirect.rs @@ -29,7 +29,7 @@ fn no_params_reg() { #[test] #[cfg_attr(miri, ignore)] fn no_params_imm16() { - fn test_with(index: u32) { + fn test_with(index: u64) { let wasm = format!( r#" (module @@ -45,7 +45,7 @@ fn no_params_imm16() { TranslationTest::new(&wasm) .expect_func_instrs([ Instruction::call_indirect_0_imm16(RegSpan::new(Reg::from(1)), FuncType::from(0)), - Instruction::call_indirect_params_imm16(u32imm16(index), Table::from(0)), + Instruction::call_indirect_params_imm16(u64imm16(index), Table::from(0)), Instruction::Return, ]) .run(); @@ -53,8 +53,8 @@ fn no_params_imm16() { test_with(0); test_with(1); - test_with(u32::from(u16::MAX) - 1); - test_with(u32::from(u16::MAX)); + test_with(u64::from(u16::MAX) - 1); + test_with(u64::from(u16::MAX)); } #[test] @@ -84,7 +84,7 @@ fn one_reg_param_reg() { #[test] #[cfg_attr(miri, ignore)] fn one_reg_param_imm16() { - fn test_with(index: u32) { + fn test_with(index: u64) { let wasm = format!( r#" (module @@ -101,7 +101,7 @@ fn one_reg_param_imm16() { TranslationTest::new(&wasm) .expect_func_instrs([ Instruction::call_indirect_imm16(RegSpan::new(Reg::from(2)), FuncType::from(0)), - Instruction::call_indirect_params_imm16(u32imm16(index), Table::from(0)), + Instruction::call_indirect_params_imm16(u64imm16(index), Table::from(0)), Instruction::register(1), Instruction::return_reg(Reg::from(2)), ]) @@ -110,8 +110,8 @@ fn one_reg_param_imm16() { test_with(0); test_with(1); - test_with(u32::from(u16::MAX) - 1); - test_with(u32::from(u16::MAX)); + test_with(u64::from(u16::MAX) - 1); + test_with(u64::from(u16::MAX)); } #[test] @@ -179,7 +179,7 @@ fn one_imm_param_reg() { #[test] #[cfg_attr(miri, ignore)] fn one_imm_param_imm16() { - fn test_with(index: u32) { + fn test_with(index: u64) { let wasm = format!( r#" (module @@ -197,7 +197,7 @@ fn one_imm_param_imm16() { .expect_func( ExpectedFunc::new([ Instruction::call_indirect_imm16(RegSpan::new(Reg::from(1)), FuncType::from(0)), - Instruction::call_indirect_params_imm16(u32imm16(index), Table::from(0)), + Instruction::call_indirect_params_imm16(u64imm16(index), Table::from(0)), Instruction::register(-1), Instruction::return_reg(Reg::from(1)), ]) @@ -208,8 +208,8 @@ fn one_imm_param_imm16() { test_with(0); test_with(1); - test_with(u32::from(u16::MAX) - 1); - test_with(u32::from(u16::MAX)); + test_with(u64::from(u16::MAX) - 1); + test_with(u64::from(u16::MAX)); } #[test] @@ -337,7 +337,7 @@ fn two_imm_params_reg() { #[test] #[cfg_attr(miri, ignore)] fn two_reg_params_imm16() { - fn test_with(index: u32) { + fn test_with(index: u64) { let wasm = format!( r#" (module @@ -354,7 +354,7 @@ fn two_reg_params_imm16() { ); let result_reg = Reg::from(2); let results = RegSpan::new(result_reg); - let elem_index = u32imm16(index); + let elem_index = u64imm16(index); TranslationTest::new(&wasm) .expect_func_instrs([ Instruction::call_indirect_imm16(results, FuncType::from(0)), @@ -367,14 +367,14 @@ fn two_reg_params_imm16() { test_with(0); test_with(1); - test_with(u32::from(u16::MAX) - 1); - test_with(u32::from(u16::MAX)); + test_with(u64::from(u16::MAX) - 1); + test_with(u64::from(u16::MAX)); } #[test] #[cfg_attr(miri, ignore)] fn two_reg_params_lhs_imm16() { - fn test_with(index: u32) { + fn test_with(index: u64) { let wasm = format!( r#" (module @@ -391,7 +391,7 @@ fn two_reg_params_lhs_imm16() { ); let result_reg = Reg::from(2); let results = RegSpan::new(result_reg); - let elem_index = u32imm16(index); + let elem_index = u64imm16(index); TranslationTest::new(&wasm) .expect_func_instrs([ Instruction::call_indirect_imm16(results, FuncType::from(0)), @@ -404,14 +404,14 @@ fn two_reg_params_lhs_imm16() { test_with(0); test_with(1); - test_with(u32::from(u16::MAX) - 1); - test_with(u32::from(u16::MAX)); + test_with(u64::from(u16::MAX) - 1); + test_with(u64::from(u16::MAX)); } #[test] #[cfg_attr(miri, ignore)] fn two_imm_params_imm16() { - fn test_with(index: u32) { + fn test_with(index: u64) { let wasm = format!( r#" (module @@ -428,7 +428,7 @@ fn two_imm_params_imm16() { ); let result_reg = Reg::from(0); let results = RegSpan::new(result_reg); - let elem_index = u32imm16(index); + let elem_index = u64imm16(index); TranslationTest::new(&wasm) .expect_func( ExpectedFunc::new([ @@ -444,8 +444,8 @@ fn two_imm_params_imm16() { test_with(0); test_with(1); - test_with(u32::from(u16::MAX) - 1); - test_with(u32::from(u16::MAX)); + test_with(u64::from(u16::MAX) - 1); + test_with(u64::from(u16::MAX)); } #[test] @@ -541,7 +541,7 @@ fn three_imm_params_reg() { #[test] #[cfg_attr(miri, ignore)] fn three_imm_params_imm16() { - fn test_with(index: u32) { + fn test_with(index: u64) { let wasm = format!( r#" (module @@ -559,7 +559,7 @@ fn three_imm_params_imm16() { ); let result_reg = Reg::from(0); let results = RegSpan::new(result_reg); - let elem_index = u32imm16(index); + let elem_index = u64imm16(index); TranslationTest::new(&wasm) .expect_func( ExpectedFunc::new([ @@ -575,8 +575,8 @@ fn three_imm_params_imm16() { test_with(0); test_with(1); - test_with(u32::from(u16::MAX) - 1); - test_with(u32::from(u16::MAX)); + test_with(u64::from(u16::MAX) - 1); + test_with(u64::from(u16::MAX)); } #[test] diff --git a/crates/wasmi/src/engine/translator/tests/op/load.rs b/crates/wasmi/src/engine/translator/tests/op/load.rs index ed21626643..026fb06cba 100644 --- a/crates/wasmi/src/engine/translator/tests/op/load.rs +++ b/crates/wasmi/src/engine/translator/tests/op/load.rs @@ -1,64 +1,43 @@ //! Translation tests for all Wasm `load` instructions. use super::*; -use crate::{core::TrapCode, ir::index::Memory}; - -fn test_load_mem0( - wasm_op: WasmOp, - make_instr: fn(result: Reg, memory: Memory) -> Instruction, - offset: u32, -) { - assert!( - offset > u32::from(u16::MAX), - "offset must not be 16-bit encodable in this testcase" - ); - let result_ty = wasm_op.result_ty(); - let wasm = format!( - r#" - (module - (memory 1) - (func (param $ptr i32) (result {result_ty}) - local.get $ptr - {wasm_op} offset={offset} - ) - ) - "# - ); - TranslationTest::new(&wasm) - .expect_func_instrs([ - make_instr(Reg::from(1), Memory::from(0)), - Instruction::register_and_imm32(Reg::from(0), offset), - Instruction::return_reg(Reg::from(1)), - ]) - .run(); -} +use crate::{ + core::TrapCode, + ir::{Offset16, Offset64, Offset64Lo}, +}; fn test_load( wasm_op: WasmOp, - make_instr: fn(result: Reg, memory: Memory) -> Instruction, - offset: u32, + index_ty: IndexType, + memory_index: MemIdx, + make_instr: fn(result: Reg, offset_lo: Offset64Lo) -> Instruction, + offset: impl Into, ) { + let offset = offset.into(); assert!( - offset > u32::from(u16::MAX), - "offset must not be 16-bit encodable in this testcase" + offset > u64::from(u16::MAX), + "offset must not be 16-bit encodable in this testcase but found: {offset}" ); + let index_ty = index_ty.wat(); let result_ty = wasm_op.result_ty(); let wasm = format!( r#" (module - (memory $mem0 1) - (memory $mem1 1) - (func (param $ptr i32) (result {result_ty}) + (memory $mem0 {index_ty} 1) + (memory $mem1 {index_ty} 1) + (func (param $ptr {index_ty}) (result {result_ty}) local.get $ptr - {wasm_op} $mem1 offset={offset} + {wasm_op} {memory_index} offset={offset} ) ) "# ); + let (offset_hi, offset_lo) = Offset64::split(offset); TranslationTest::new(&wasm) - .expect_func_instrs([ - make_instr(Reg::from(1), Memory::from(1)), - Instruction::register_and_imm32(Reg::from(0), offset), + .expect_func_instrs(iter_filter_opts![ + make_instr(Reg::from(1), offset_lo), + Instruction::register_and_offset_hi(Reg::from(0), offset_hi), + memory_index.instr(), Instruction::return_reg(Reg::from(1)), ]) .run(); @@ -66,15 +45,17 @@ fn test_load( fn test_load_offset16( wasm_op: WasmOp, + index_ty: IndexType, offset: u16, - make_instr_offset16: fn(result: Reg, ptr: Reg, offset: Const16) -> Instruction, + make_instr_offset16: fn(result: Reg, ptr: Reg, offset: Offset16) -> Instruction, ) { let result_ty = wasm_op.result_ty(); + let index_ty = index_ty.wat(); let wasm = format!( r#" (module - (memory 1) - (func (param $ptr i32) (result {result_ty}) + (memory {index_ty} 1) + (func (param $ptr {index_ty}) (result {result_ty}) local.get $ptr {wasm_op} offset={offset} ) @@ -83,56 +64,30 @@ fn test_load_offset16( ); TranslationTest::new(&wasm) .expect_func_instrs([ - make_instr_offset16(Reg::from(1), Reg::from(0), >::from(offset)), + make_instr_offset16(Reg::from(1), Reg::from(0), offset16(offset)), Instruction::return_reg(Reg::from(1)), ]) .run(); } -fn test_load_at_mem0( - wasm_op: WasmOp, - make_instr_at: fn(result: Reg, address: u32) -> Instruction, - ptr: u32, - offset: u32, -) { - let result_ty = wasm_op.result_ty(); - let wasm = format!( - r#" - (module - (memory 1) - (func (result {result_ty}) - i32.const {ptr} - {wasm_op} offset={offset} - ) - ) - "# - ); - let address = ptr - .checked_add(offset) - .expect("ptr+offset must be valid in this testcase"); - TranslationTest::new(&wasm) - .expect_func_instrs([ - make_instr_at(Reg::from(0), address), - Instruction::return_reg(Reg::from(0)), - ]) - .run(); -} - fn test_load_at( wasm_op: WasmOp, + index_ty: IndexType, + memory_index: MemIdx, make_instr_at: fn(result: Reg, address: u32) -> Instruction, - ptr: u32, - offset: u32, + ptr: u64, + offset: u64, ) { let result_ty = wasm_op.result_ty(); + let index_ty = index_ty.wat(); let wasm = format!( r#" (module - (memory $mem0 1) - (memory $mem1 1) + (memory $mem0 {index_ty} 1) + (memory $mem1 {index_ty} 1) (func (result {result_ty}) - i32.const {ptr} - {wasm_op} $mem1 offset={offset} + {index_ty}.const {ptr} + {wasm_op} {memory_index} offset={offset} ) ) "# @@ -140,115 +95,182 @@ fn test_load_at( let address = ptr .checked_add(offset) .expect("ptr+offset must be valid in this testcase"); + let address = u32::try_from(address).expect("ptr+offset must fit into a `u32` value"); TranslationTest::new(&wasm) - .expect_func_instrs([ + .expect_func_instrs(iter_filter_opts![ make_instr_at(Reg::from(0), address), - Instruction::memory_index(1), + memory_index.instr(), Instruction::return_reg(0), ]) .run(); } -fn test_load_at_overflow_mem0(wasm_op: WasmOp, ptr: u32, offset: u32) { +fn test_load_at_overflow( + wasm_op: WasmOp, + index_ty: IndexType, + memory_index: MemIdx, + ptr: u64, + offset: u64, +) { let result_ty = wasm_op.result_ty(); + let index_repr = index_ty.wat(); let wasm = format!( r#" (module - (memory 1) + (memory $mem0 {index_repr} 1) + (memory $mem1 {index_repr} 1) (func (result {result_ty}) - i32.const {ptr} - {wasm_op} offset={offset} + {index_repr}.const {ptr} + {wasm_op} {memory_index} offset={offset} ) ) "# ); - assert!( - ptr.checked_add(offset).is_none(), - "ptr+offset must overflow in this testcase" - ); + assert_overflowing_ptr_offset(index_ty, ptr, offset); TranslationTest::new(&wasm) .expect_func_instrs([Instruction::trap(TrapCode::MemoryOutOfBounds)]) .run(); } -fn test_load_at_overflow(wasm_op: WasmOp, ptr: u32, offset: u32) { +fn test_load_at_fallback( + wasm_op: WasmOp, + memory_index: MemIdx, + make_instr: fn(result: Reg, offset_lo: Offset64Lo) -> Instruction, + ptr: u64, + offset: u64, +) { let result_ty = wasm_op.result_ty(); let wasm = format!( r#" (module - (memory $mem0 1) - (memory $mem1 1) + (memory $mem0 i64 1) + (memory $mem1 i64 1) (func (result {result_ty}) - i32.const {ptr} - {wasm_op} $mem1 offset={offset} + i64.const {ptr} + {wasm_op} {memory_index} offset={offset} ) ) "# ); - assert!( - ptr.checked_add(offset).is_none(), - "ptr+offset must overflow in this testcase" - ); + let Some(address64) = ptr.checked_add(offset) else { + panic!("ptr+offset must be a valid 64-bit result but found: ptr={ptr}, offset={offset}") + }; + if u32::try_from(address64).is_ok() { + panic!("ptr+offset must not fit into a `u32` value but found: ptr={ptr}, offset={offset}") + } + let (offset_hi, offset_lo) = Offset64::split(address64); TranslationTest::new(&wasm) - .expect_func_instrs([Instruction::trap(TrapCode::MemoryOutOfBounds)]) + .expect_func( + ExpectedFunc::new(iter_filter_opts![ + make_instr(Reg::from(0), offset_lo), + Instruction::register_and_offset_hi(Reg::from(-1), offset_hi), + memory_index.instr(), + Instruction::return_reg(Reg::from(0)), + ]) + .consts([0_u64]), + ) .run(); } macro_rules! generate_tests { ( $wasm_op:ident, $make_instr:expr, $make_instr_offset16:expr, $make_instr_at:expr ) => { - #[test] - #[cfg_attr(miri, ignore)] - fn reg_mem0() { - test_load_mem0(WASM_OP, $make_instr, u32::from(u16::MAX) + 1); - test_load_mem0(WASM_OP, $make_instr, u32::MAX - 1); - test_load_mem0(WASM_OP, $make_instr, u32::MAX); - } - #[test] #[cfg_attr(miri, ignore)] fn reg() { - test_load(WASM_OP, $make_instr, u32::from(u16::MAX) + 1); - test_load(WASM_OP, $make_instr, u32::MAX - 1); - test_load(WASM_OP, $make_instr, u32::MAX); + [u32::from(u16::MAX) + 1, u32::MAX - 1, u32::MAX] + .into_iter() + .for_each(|offset| { + test_load(WASM_OP, IndexType::Memory32, MemIdx(0), $make_instr, offset); + test_load(WASM_OP, IndexType::Memory32, MemIdx(1), $make_instr, offset) + }) } #[test] #[cfg_attr(miri, ignore)] - fn offset16() { - test_load_offset16(WASM_OP, 0, $make_instr_offset16); - test_load_offset16(WASM_OP, u16::MAX, $make_instr_offset16); + fn reg_memory64() { + [ + u64::from(u16::MAX) + 1, + u64::from(u32::MAX) - 1, + u64::from(u32::MAX), + u64::from(u32::MAX) + 1, + u64::MAX - 1, + u64::MAX, + ] + .into_iter() + .for_each(|offset| { + test_load(WASM_OP, IndexType::Memory64, MemIdx(0), $make_instr, offset); + test_load(WASM_OP, IndexType::Memory64, MemIdx(1), $make_instr, offset) + }) } #[test] #[cfg_attr(miri, ignore)] - fn at_mem0() { - test_load_at_mem0(WASM_OP, $make_instr_at, 42, 5); - test_load_at_mem0(WASM_OP, $make_instr_at, u32::MAX, 0); - test_load_at_mem0(WASM_OP, $make_instr_at, 0, u32::MAX); + fn offset16() { + [0, 1, u16::MAX - 1, u16::MAX] + .into_iter() + .for_each(|offset| { + for index_ty in [IndexType::Memory32, IndexType::Memory64] { + test_load_offset16(WASM_OP, index_ty, offset, $make_instr_offset16); + } + }) } #[test] #[cfg_attr(miri, ignore)] fn at() { - test_load_at(WASM_OP, $make_instr_at, 42, 5); - test_load_at(WASM_OP, $make_instr_at, u32::MAX, 0); - test_load_at(WASM_OP, $make_instr_at, 0, u32::MAX); + [ + (0, 0), + (42, 5), + (u64::from(u32::MAX), 0), + (0, u64::from(u32::MAX)), + ] + .into_iter() + .for_each(|(ptr, offset)| { + for index_ty in [IndexType::Memory32, IndexType::Memory64] { + test_load_at(WASM_OP, index_ty, MemIdx(0), $make_instr_at, ptr, offset); + test_load_at(WASM_OP, index_ty, MemIdx(1), $make_instr_at, ptr, offset); + } + }) } #[test] #[cfg_attr(miri, ignore)] - fn at_overflow_mem0() { - test_load_at_overflow_mem0(WASM_OP, u32::MAX, 1); - test_load_at_overflow_mem0(WASM_OP, 1, u32::MAX); - test_load_at_overflow_mem0(WASM_OP, u32::MAX, u32::MAX); + fn at_overflow() { + [ + (IndexType::Memory32, u64::from(u32::MAX), 1), + (IndexType::Memory32, 1, u64::from(u32::MAX)), + ( + IndexType::Memory32, + u64::from(u32::MAX), + u64::from(u32::MAX), + ), + (IndexType::Memory64, u64::MAX, 1), + (IndexType::Memory64, 1, u64::MAX), + (IndexType::Memory64, u64::MAX, u64::MAX), + ] + .into_iter() + .for_each(|(index_ty, ptr, offset)| { + test_load_at_overflow(WASM_OP, index_ty, MemIdx(0), ptr, offset); + test_load_at_overflow(WASM_OP, index_ty, MemIdx(1), ptr, offset); + }) } #[test] #[cfg_attr(miri, ignore)] - fn at_overflow() { - test_load_at_overflow(WASM_OP, u32::MAX, 1); - test_load_at_overflow(WASM_OP, 1, u32::MAX); - test_load_at_overflow(WASM_OP, u32::MAX, u32::MAX); + fn at_fallback() { + [ + (u64::from(u32::MAX), 1), + (1, u64::from(u32::MAX)), + (1, u64::MAX - 1), + (u64::MAX - 1, 1), + (0, u64::MAX), + (u64::MAX, 0), + ] + .into_iter() + .for_each(|(ptr, offset)| { + test_load_at_fallback(WASM_OP, MemIdx(0), $make_instr, ptr, offset); + test_load_at_fallback(WASM_OP, MemIdx(1), $make_instr, ptr, offset); + }) } }; } diff --git a/crates/wasmi/src/engine/translator/tests/op/memory/memory_copy.rs b/crates/wasmi/src/engine/translator/tests/op/memory/memory_copy.rs index abfc47fc4b..b200d8800d 100644 --- a/crates/wasmi/src/engine/translator/tests/op/memory/memory_copy.rs +++ b/crates/wasmi/src/engine/translator/tests/op/memory/memory_copy.rs @@ -24,7 +24,7 @@ fn copy() { .run() } -fn testcase_copy_exact(len: u32) -> TranslationTest { +fn testcase_copy_exact(len: u64) -> TranslationTest { let wasm = &format!( r" (module @@ -41,10 +41,10 @@ fn testcase_copy_exact(len: u32) -> TranslationTest { TranslationTest::new(wasm) } -fn test_copy_exact16(len: u32) { +fn test_copy_exact16(len: u64) { testcase_copy_exact(len) .expect_func_instrs([ - Instruction::memory_copy_exact(Reg::from(0), Reg::from(1), u32imm16(len)), + Instruction::memory_copy_exact(Reg::from(0), Reg::from(1), u64imm16(len)), Instruction::memory_index(0), Instruction::memory_index(1), Instruction::Return, @@ -58,10 +58,10 @@ fn copy_exact16() { test_copy_exact16(0); test_copy_exact16(1); test_copy_exact16(42); - test_copy_exact16(u32::from(u16::MAX)); + test_copy_exact16(u64::from(u16::MAX)); } -fn test_copy_exact(len: u32) { +fn test_copy_exact(len: u64) { testcase_copy_exact(len) .expect_func( ExpectedFunc::new([ @@ -78,11 +78,11 @@ fn test_copy_exact(len: u32) { #[test] #[cfg_attr(miri, ignore)] fn copy_exact() { - test_copy_exact(u32::from(u16::MAX) + 1); - test_copy_exact(u32::MAX); + test_copy_exact(u64::from(u16::MAX) + 1); + test_copy_exact(u64::from(u32::MAX)); } -fn testcase_copy_from(src: u32) -> TranslationTest { +fn testcase_copy_from(src: u64) -> TranslationTest { let wasm = &format!( r" (module @@ -99,10 +99,10 @@ fn testcase_copy_from(src: u32) -> TranslationTest { TranslationTest::new(wasm) } -fn test_copy_from16(src: u32) { +fn test_copy_from16(src: u64) { testcase_copy_from(src) .expect_func_instrs([ - Instruction::memory_copy_from(Reg::from(0), u32imm16(src), Reg::from(1)), + Instruction::memory_copy_from(Reg::from(0), u64imm16(src), Reg::from(1)), Instruction::memory_index(0), Instruction::memory_index(1), Instruction::Return, @@ -114,10 +114,10 @@ fn test_copy_from16(src: u32) { #[cfg_attr(miri, ignore)] fn copy_from16() { test_copy_from16(0); - test_copy_from16(u32::from(u16::MAX)); + test_copy_from16(u64::from(u16::MAX)); } -fn test_copy_from(src: u32) { +fn test_copy_from(src: u64) { testcase_copy_from(src) .expect_func( ExpectedFunc::new([ @@ -134,11 +134,11 @@ fn test_copy_from(src: u32) { #[test] #[cfg_attr(miri, ignore)] fn copy_from() { - test_copy_from(u32::from(u16::MAX) + 1); - test_copy_from(u32::MAX); + test_copy_from(u64::from(u16::MAX) + 1); + test_copy_from(u64::from(u32::MAX)); } -fn testcase_copy_to(dst: u32) -> TranslationTest { +fn testcase_copy_to(dst: u64) -> TranslationTest { let wasm = &format!( r" (module @@ -155,10 +155,10 @@ fn testcase_copy_to(dst: u32) -> TranslationTest { TranslationTest::new(wasm) } -fn test_copy_to16(dst: u32) { +fn test_copy_to16(dst: u64) { testcase_copy_to(dst) .expect_func_instrs([ - Instruction::memory_copy_to(u32imm16(dst), Reg::from(0), Reg::from(1)), + Instruction::memory_copy_to(u64imm16(dst), Reg::from(0), Reg::from(1)), Instruction::memory_index(0), Instruction::memory_index(1), Instruction::Return, @@ -170,10 +170,10 @@ fn test_copy_to16(dst: u32) { #[cfg_attr(miri, ignore)] fn copy_to16() { test_copy_to16(0); - test_copy_to16(u32::from(u16::MAX)); + test_copy_to16(u64::from(u16::MAX)); } -fn test_copy_to(dst: u32) { +fn test_copy_to(dst: u64) { testcase_copy_to(dst) .expect_func( ExpectedFunc::new([ @@ -190,11 +190,11 @@ fn test_copy_to(dst: u32) { #[test] #[cfg_attr(miri, ignore)] fn copy_to() { - test_copy_to(u32::from(u16::MAX) + 1); - test_copy_to(u32::MAX); + test_copy_to(u64::from(u16::MAX) + 1); + test_copy_to(u64::from(u32::MAX)); } -fn testcase_copy_from_to(dst: u32, src: u32) -> TranslationTest { +fn testcase_copy_from_to(dst: u64, src: u64) -> TranslationTest { let wasm = &format!( r" (module @@ -211,10 +211,10 @@ fn testcase_copy_from_to(dst: u32, src: u32) -> TranslationTest { TranslationTest::new(wasm) } -fn test_copy_from_to16(dst: u32, src: u32) { +fn test_copy_from_to16(dst: u64, src: u64) { testcase_copy_from_to(dst, src) .expect_func_instrs([ - Instruction::memory_copy_from_to(u32imm16(dst), u32imm16(src), Reg::from(0)), + Instruction::memory_copy_from_to(u64imm16(dst), u64imm16(src), Reg::from(0)), Instruction::memory_index(0), Instruction::memory_index(1), Instruction::Return, @@ -225,7 +225,7 @@ fn test_copy_from_to16(dst: u32, src: u32) { #[test] #[cfg_attr(miri, ignore)] fn copy_from_to16() { - let values = [0, 1, u32::from(u16::MAX) - 1, u32::from(u16::MAX)]; + let values = [0, 1, u64::from(u16::MAX) - 1, u64::from(u16::MAX)]; for dst in values { for src in values { test_copy_from_to16(dst, src); @@ -233,7 +233,7 @@ fn copy_from_to16() { } } -fn test_copy_from_to(dst: u32, src: u32) { +fn test_copy_from_to(dst: u64, src: u64) { testcase_copy_from_to(dst, src) .expect_func( ExpectedFunc::new([ @@ -250,7 +250,11 @@ fn test_copy_from_to(dst: u32, src: u32) { #[test] #[cfg_attr(miri, ignore)] fn copy_from_to() { - let values = [u32::from(u16::MAX) + 1, u32::MAX - 1, u32::MAX]; + let values = [ + u64::from(u16::MAX) + 1, + u64::from(u32::MAX) - 1, + u64::from(u32::MAX), + ]; for dst in values { for src in values { if dst == src { @@ -265,7 +269,7 @@ fn copy_from_to() { } } -fn testcase_copy_to_exact(dst: u32, len: u32) -> TranslationTest { +fn testcase_copy_to_exact(dst: u64, len: u64) -> TranslationTest { let wasm = &format!( r" (module @@ -282,10 +286,10 @@ fn testcase_copy_to_exact(dst: u32, len: u32) -> TranslationTest { TranslationTest::new(wasm) } -fn test_copy_to_exact16(dst: u32, len: u32) { +fn test_copy_to_exact16(dst: u64, len: u64) { testcase_copy_to_exact(dst, len) .expect_func_instrs([ - Instruction::memory_copy_to_exact(u32imm16(dst), Reg::from(0), u32imm16(len)), + Instruction::memory_copy_to_exact(u64imm16(dst), Reg::from(0), u64imm16(len)), Instruction::memory_index(0), Instruction::memory_index(1), Instruction::Return, @@ -296,7 +300,7 @@ fn test_copy_to_exact16(dst: u32, len: u32) { #[test] #[cfg_attr(miri, ignore)] fn copy_to_exact16() { - let values = [0, 1, u32::from(u16::MAX) - 1, u32::from(u16::MAX)]; + let values = [0, 1, u64::from(u16::MAX) - 1, u64::from(u16::MAX)]; for dst in values { for len in values { test_copy_to_exact16(dst, len); @@ -304,7 +308,7 @@ fn copy_to_exact16() { } } -fn test_copy_to_exact(dst: u32, len: u32) { +fn test_copy_to_exact(dst: u64, len: u64) { testcase_copy_to_exact(dst, len) .expect_func( ExpectedFunc::new([ @@ -321,7 +325,11 @@ fn test_copy_to_exact(dst: u32, len: u32) { #[test] #[cfg_attr(miri, ignore)] fn copy_to_exact() { - let values = [u32::from(u16::MAX) + 1, u32::MAX - 1, u32::MAX]; + let values = [ + u64::from(u16::MAX) + 1, + u64::from(u32::MAX) - 1, + u64::from(u32::MAX), + ]; for dst in values { for src in values { if dst == src { @@ -336,7 +344,7 @@ fn copy_to_exact() { } } -fn testcase_copy_from_exact(src: u32, len: u32) -> TranslationTest { +fn testcase_copy_from_exact(src: u64, len: u64) -> TranslationTest { let wasm = &format!( r" (module @@ -353,10 +361,10 @@ fn testcase_copy_from_exact(src: u32, len: u32) -> TranslationTest { TranslationTest::new(wasm) } -fn test_copy_from_exact16(src: u32, len: u32) { +fn test_copy_from_exact16(src: u64, len: u64) { testcase_copy_from_exact(src, len) .expect_func_instrs([ - Instruction::memory_copy_from_exact(Reg::from(0), u32imm16(src), u32imm16(len)), + Instruction::memory_copy_from_exact(Reg::from(0), u64imm16(src), u64imm16(len)), Instruction::memory_index(0), Instruction::memory_index(1), Instruction::Return, @@ -367,7 +375,7 @@ fn test_copy_from_exact16(src: u32, len: u32) { #[test] #[cfg_attr(miri, ignore)] fn copy_from_exact16() { - let values = [0, 1, u32::from(u16::MAX) - 1, u32::from(u16::MAX)]; + let values = [0, 1, u64::from(u16::MAX) - 1, u64::from(u16::MAX)]; for dst in values { for len in values { test_copy_from_exact16(dst, len); @@ -375,7 +383,7 @@ fn copy_from_exact16() { } } -fn test_copy_from_exact(src: u32, len: u32) { +fn test_copy_from_exact(src: u64, len: u64) { testcase_copy_from_exact(src, len) .expect_func( ExpectedFunc::new([ @@ -392,7 +400,11 @@ fn test_copy_from_exact(src: u32, len: u32) { #[test] #[cfg_attr(miri, ignore)] fn copy_from_exact() { - let values = [u32::from(u16::MAX) + 1, u32::MAX - 1, u32::MAX]; + let values = [ + u64::from(u16::MAX) + 1, + u64::from(u32::MAX) - 1, + u64::from(u32::MAX), + ]; for dst in values { for src in values { if dst == src { @@ -407,7 +419,7 @@ fn copy_from_exact() { } } -fn testcase_copy_from_to_exact(dst: u32, src: u32, len: u32) -> TranslationTest { +fn testcase_copy_from_to_exact(dst: u64, src: u64, len: u64) -> TranslationTest { let wasm = &format!( r" (module @@ -424,10 +436,10 @@ fn testcase_copy_from_to_exact(dst: u32, src: u32, len: u32) -> TranslationTest TranslationTest::new(wasm) } -fn test_copy_from_to_exact16(dst: u32, src: u32, len: u32) { +fn test_copy_from_to_exact16(dst: u64, src: u64, len: u64) { testcase_copy_from_to_exact(dst, src, len) .expect_func_instrs([ - Instruction::memory_copy_from_to_exact(u32imm16(dst), u32imm16(src), u32imm16(len)), + Instruction::memory_copy_from_to_exact(u64imm16(dst), u64imm16(src), u64imm16(len)), Instruction::memory_index(0), Instruction::memory_index(1), Instruction::Return, @@ -438,7 +450,7 @@ fn test_copy_from_to_exact16(dst: u32, src: u32, len: u32) { #[test] #[cfg_attr(miri, ignore)] fn copy_from_to_exact16() { - let values = [0, 1, u32::from(u16::MAX) - 1, u32::from(u16::MAX)]; + let values = [0, 1, u64::from(u16::MAX) - 1, u64::from(u16::MAX)]; for dst in values { for src in values { for len in values { @@ -448,7 +460,7 @@ fn copy_from_to_exact16() { } } -fn test_copy_from_to_exact(dst: u32, src: u32, len: u32) { +fn test_copy_from_to_exact(dst: u64, src: u64, len: u64) { testcase_copy_from_to_exact(dst, src, len) .expect_func( ExpectedFunc::new([ @@ -465,7 +477,11 @@ fn test_copy_from_to_exact(dst: u32, src: u32, len: u32) { #[test] #[cfg_attr(miri, ignore)] fn copy_from_to_exact() { - let values = [u32::from(u16::MAX) + 1, u32::MAX - 1, u32::MAX]; + let values = [ + u64::from(u16::MAX) + 1, + u64::from(u32::MAX) - 1, + u64::from(u32::MAX), + ]; for dst in values { for src in values { for len in values { diff --git a/crates/wasmi/src/engine/translator/tests/op/memory/memory_fill.rs b/crates/wasmi/src/engine/translator/tests/op/memory/memory_fill.rs index 85acfe0a97..1264274f60 100644 --- a/crates/wasmi/src/engine/translator/tests/op/memory/memory_fill.rs +++ b/crates/wasmi/src/engine/translator/tests/op/memory/memory_fill.rs @@ -22,7 +22,7 @@ fn fill() { .run() } -fn testcase_fill_exact(len: u32) -> TranslationTest { +fn testcase_fill_exact(len: u64) -> TranslationTest { let wasm = &format!( r" (module @@ -38,10 +38,10 @@ fn testcase_fill_exact(len: u32) -> TranslationTest { TranslationTest::new(wasm) } -fn test_fill_exact16(len: u32) { +fn test_fill_exact16(len: u64) { testcase_fill_exact(len) .expect_func_instrs([ - Instruction::memory_fill_exact(Reg::from(0), Reg::from(1), u32imm16(len)), + Instruction::memory_fill_exact(Reg::from(0), Reg::from(1), u64imm16(len)), Instruction::memory_index(0), Instruction::Return, ]) @@ -54,10 +54,10 @@ fn fill_exact16() { test_fill_exact16(0); test_fill_exact16(1); test_fill_exact16(42); - test_fill_exact16(u32::from(u16::MAX)); + test_fill_exact16(u64::from(u16::MAX)); } -fn test_fill_exact(len: u32) { +fn test_fill_exact(len: u64) { testcase_fill_exact(len) .expect_func( ExpectedFunc::new([ @@ -73,8 +73,8 @@ fn test_fill_exact(len: u32) { #[test] #[cfg_attr(miri, ignore)] fn copy_exact() { - test_fill_exact(u32::from(u16::MAX) + 1); - test_fill_exact(u32::MAX); + test_fill_exact(u64::from(u16::MAX) + 1); + test_fill_exact(u64::from(u32::MAX)); } fn testcase_fill_imm(value: u32) -> TranslationTest { @@ -114,7 +114,7 @@ fn fill_imm() { test_fill_imm(u32::MAX); } -fn testcase_fill_at(dst: u32) -> TranslationTest { +fn testcase_fill_at(dst: u64) -> TranslationTest { let wasm = &format!( r" (module @@ -130,10 +130,10 @@ fn testcase_fill_at(dst: u32) -> TranslationTest { TranslationTest::new(wasm) } -fn test_fill_at16(dst: u32) { +fn test_fill_at16(dst: u64) { testcase_fill_at(dst) .expect_func_instrs([ - Instruction::memory_fill_at(u32imm16(dst), Reg::from(0), Reg::from(1)), + Instruction::memory_fill_at(u64imm16(dst), Reg::from(0), Reg::from(1)), Instruction::memory_index(0), Instruction::Return, ]) @@ -144,10 +144,10 @@ fn test_fill_at16(dst: u32) { #[cfg_attr(miri, ignore)] fn fill_at16() { test_fill_at16(0); - test_fill_at16(u32::from(u16::MAX)); + test_fill_at16(u64::from(u16::MAX)); } -fn test_fill_at(dst: u32) { +fn test_fill_at(dst: u64) { testcase_fill_at(dst) .expect_func( ExpectedFunc::new([ @@ -163,11 +163,11 @@ fn test_fill_at(dst: u32) { #[test] #[cfg_attr(miri, ignore)] fn fill_at() { - test_fill_at(u32::from(u16::MAX) + 1); - test_fill_at(u32::MAX); + test_fill_at(u64::from(u16::MAX) + 1); + test_fill_at(u64::from(u32::MAX)); } -fn testcase_fill_at_imm(dst: u32, value: u32) -> TranslationTest { +fn testcase_fill_at_imm(dst: u64, value: u64) -> TranslationTest { let wasm = &format!( r" (module @@ -183,10 +183,10 @@ fn testcase_fill_at_imm(dst: u32, value: u32) -> TranslationTest { TranslationTest::new(wasm) } -fn test_fill_at16_imm(dst: u32, value: u32) { +fn test_fill_at16_imm(dst: u64, value: u64) { testcase_fill_at_imm(dst, value) .expect_func_instrs([ - Instruction::memory_fill_at_imm(u32imm16(dst), value as u8, Reg::from(0)), + Instruction::memory_fill_at_imm(u64imm16(dst), value as u8, Reg::from(0)), Instruction::memory_index(0), Instruction::Return, ]) @@ -196,16 +196,16 @@ fn test_fill_at16_imm(dst: u32, value: u32) { #[test] #[cfg_attr(miri, ignore)] fn fill_at16_imm() { - let dst_values = [0, 1, u32::from(u16::MAX) - 1, u32::from(u16::MAX)]; + let dst_values = [0, 1, u64::from(u16::MAX) - 1, u64::from(u16::MAX)]; let test_values = [ 0, 1, 42, - u32::from(u16::MAX) - 1, - u32::from(u16::MAX), - u32::from(u16::MAX) + 1, - u32::MAX - 1, - u32::MAX, + u64::from(u16::MAX) - 1, + u64::from(u16::MAX), + u64::from(u16::MAX) + 1, + u64::from(u32::MAX) - 1, + u64::from(u32::MAX), ]; for dst in dst_values { for value in test_values { @@ -214,7 +214,7 @@ fn fill_at16_imm() { } } -fn test_fill_at_imm(dst: u32, value: u32) { +fn test_fill_at_imm(dst: u64, value: u64) { testcase_fill_at_imm(dst, value) .expect_func( ExpectedFunc::new([ @@ -230,7 +230,11 @@ fn test_fill_at_imm(dst: u32, value: u32) { #[test] #[cfg_attr(miri, ignore)] fn fill_at_imm() { - let values = [u32::from(u16::MAX) + 1, u32::MAX - 1, u32::MAX]; + let values = [ + u64::from(u16::MAX) + 1, + u64::from(u32::MAX) - 1, + u64::from(u32::MAX), + ]; for dst in values { for value in values { if dst == value { @@ -245,7 +249,7 @@ fn fill_at_imm() { } } -fn testcase_fill_at_exact(dst: u32, len: u32) -> TranslationTest { +fn testcase_fill_at_exact(dst: u64, len: u64) -> TranslationTest { let wasm = &format!( r" (module @@ -261,10 +265,10 @@ fn testcase_fill_at_exact(dst: u32, len: u32) -> TranslationTest { TranslationTest::new(wasm) } -fn test_fill_at_exact16(dst: u32, len: u32) { +fn test_fill_at_exact16(dst: u64, len: u64) { testcase_fill_at_exact(dst, len) .expect_func_instrs([ - Instruction::memory_fill_at_exact(u32imm16(dst), Reg::from(0), u32imm16(len)), + Instruction::memory_fill_at_exact(u64imm16(dst), Reg::from(0), u64imm16(len)), Instruction::memory_index(0), Instruction::Return, ]) @@ -274,7 +278,7 @@ fn test_fill_at_exact16(dst: u32, len: u32) { #[test] #[cfg_attr(miri, ignore)] fn fill_to_exact16() { - let values = [0, 1, u32::from(u16::MAX) - 1, u32::from(u16::MAX)]; + let values = [0, 1, u64::from(u16::MAX) - 1, u64::from(u16::MAX)]; for dst in values { for len in values { test_fill_at_exact16(dst, len); @@ -282,7 +286,7 @@ fn fill_to_exact16() { } } -fn test_fill_at_exact(dst: u32, len: u32) { +fn test_fill_at_exact(dst: u64, len: u64) { testcase_fill_at_exact(dst, len) .expect_func( ExpectedFunc::new([ @@ -298,7 +302,11 @@ fn test_fill_at_exact(dst: u32, len: u32) { #[test] #[cfg_attr(miri, ignore)] fn fill_at_exact() { - let values = [u32::from(u16::MAX) + 1, u32::MAX - 1, u32::MAX]; + let values = [ + u64::from(u16::MAX) + 1, + u64::from(u32::MAX) - 1, + u64::from(u32::MAX), + ]; for dst in values { for value in values { if dst == value { @@ -313,7 +321,7 @@ fn fill_at_exact() { } } -fn testcase_fill_imm_exact(value: u32, len: u32) -> TranslationTest { +fn testcase_fill_imm_exact(value: u64, len: u64) -> TranslationTest { let wasm = &format!( r" (module @@ -329,10 +337,10 @@ fn testcase_fill_imm_exact(value: u32, len: u32) -> TranslationTest { TranslationTest::new(wasm) } -fn test_fill_imm_exact16(value: u32, len: u32) { +fn test_fill_imm_exact16(value: u64, len: u64) { testcase_fill_imm_exact(value, len) .expect_func_instrs([ - Instruction::memory_fill_imm_exact(Reg::from(0), value as u8, u32imm16(len)), + Instruction::memory_fill_imm_exact(Reg::from(0), value as u8, u64imm16(len)), Instruction::memory_index(0), Instruction::Return, ]) @@ -342,16 +350,16 @@ fn test_fill_imm_exact16(value: u32, len: u32) { #[test] #[cfg_attr(miri, ignore)] fn fill_imm_exact16() { - let len_values = [0, 1, u32::from(u16::MAX) - 1, u32::from(u16::MAX)]; + let len_values = [0, 1, u64::from(u16::MAX) - 1, u64::from(u16::MAX)]; let values = [ 0, 1, 42, - u32::from(u16::MAX) - 1, - u32::from(u16::MAX), - u32::from(u16::MAX) + 1, - u32::MAX - 1, - u32::MAX, + u64::from(u16::MAX) - 1, + u64::from(u16::MAX), + u64::from(u16::MAX) + 1, + u64::from(u32::MAX) - 1, + u64::from(u32::MAX), ]; for value in values { for len in len_values { @@ -360,7 +368,7 @@ fn fill_imm_exact16() { } } -fn test_fill_imm_exact(value: u32, len: u32) { +fn test_fill_imm_exact(value: u64, len: u64) { testcase_fill_imm_exact(value, len) .expect_func( ExpectedFunc::new([ @@ -376,7 +384,11 @@ fn test_fill_imm_exact(value: u32, len: u32) { #[test] #[cfg_attr(miri, ignore)] fn fill_imm_exact() { - let values = [u32::from(u16::MAX) + 1, u32::MAX - 1, u32::MAX]; + let values = [ + u64::from(u16::MAX) + 1, + u64::from(u32::MAX) - 1, + u64::from(u32::MAX), + ]; for dst in values { for value in values { if dst == value { @@ -391,7 +403,7 @@ fn fill_imm_exact() { } } -fn testcase_fill_at_imm_exact(dst: u32, value: u32, len: u32) -> TranslationTest { +fn testcase_fill_at_imm_exact(dst: u64, value: u64, len: u64) -> TranslationTest { let wasm = &format!( r" (module @@ -407,10 +419,10 @@ fn testcase_fill_at_imm_exact(dst: u32, value: u32, len: u32) -> TranslationTest TranslationTest::new(wasm) } -fn test_fill_at_imm_exact16(dst: u32, value: u32, len: u32) { +fn test_fill_at_imm_exact16(dst: u64, value: u64, len: u64) { testcase_fill_at_imm_exact(dst, value, len) .expect_func_instrs([ - Instruction::memory_fill_at_imm_exact(u32imm16(dst), value as u8, u32imm16(len)), + Instruction::memory_fill_at_imm_exact(u64imm16(dst), value as u8, u64imm16(len)), Instruction::memory_index(0), Instruction::Return, ]) @@ -420,7 +432,7 @@ fn test_fill_at_imm_exact16(dst: u32, value: u32, len: u32) { #[test] #[cfg_attr(miri, ignore)] fn fill_at_imm_exact16() { - let values = [0, 1, u32::from(u16::MAX) - 1, u32::from(u16::MAX)]; + let values = [0, 1, u64::from(u16::MAX) - 1, u64::from(u16::MAX)]; for dst in values { for value in values { for len in values { @@ -430,7 +442,7 @@ fn fill_at_imm_exact16() { } } -fn test_fill_at_imm_exact(dst: u32, value: u32, len: u32) { +fn test_fill_at_imm_exact(dst: u64, value: u64, len: u64) { testcase_fill_at_imm_exact(dst, value, len) .expect_func( ExpectedFunc::new([ @@ -446,7 +458,11 @@ fn test_fill_at_imm_exact(dst: u32, value: u32, len: u32) { #[test] #[cfg_attr(miri, ignore)] fn fill_at_imm_exact() { - let values = [u32::from(u16::MAX) + 1, u32::MAX - 1, u32::MAX]; + let values = [ + u64::from(u16::MAX) + 1, + u64::from(u32::MAX) - 1, + u64::from(u32::MAX), + ]; for dst in values { for value in values { for len in values { diff --git a/crates/wasmi/src/engine/translator/tests/op/memory/memory_init.rs b/crates/wasmi/src/engine/translator/tests/op/memory/memory_init.rs index 6d8544b01b..75538a2cd1 100644 --- a/crates/wasmi/src/engine/translator/tests/op/memory/memory_init.rs +++ b/crates/wasmi/src/engine/translator/tests/op/memory/memory_init.rs @@ -138,7 +138,7 @@ fn init_from() { test_copy_from(u32::MAX); } -fn testcase_init_to(dst: u32) -> TranslationTest { +fn testcase_init_to(dst: u64) -> TranslationTest { let wasm = &format!( r" (module @@ -155,10 +155,10 @@ fn testcase_init_to(dst: u32) -> TranslationTest { TranslationTest::new(wasm) } -fn test_copy_to16(dst: u32) { +fn test_copy_to16(dst: u64) { testcase_init_to(dst) .expect_func_instrs([ - Instruction::memory_init_to(u32imm16(dst), Reg::from(0), Reg::from(1)), + Instruction::memory_init_to(u64imm16(dst), Reg::from(0), Reg::from(1)), Instruction::memory_index(0), Instruction::data_index(0), Instruction::Return, @@ -170,10 +170,10 @@ fn test_copy_to16(dst: u32) { #[cfg_attr(miri, ignore)] fn init_to16() { test_copy_to16(0); - test_copy_to16(u32::from(u16::MAX)); + test_copy_to16(u64::from(u16::MAX)); } -fn test_copy_to(dst: u32) { +fn test_copy_to(dst: u64) { testcase_init_to(dst) .expect_func( ExpectedFunc::new([ @@ -190,11 +190,11 @@ fn test_copy_to(dst: u32) { #[test] #[cfg_attr(miri, ignore)] fn init_to() { - test_copy_to(u32::from(u16::MAX) + 1); - test_copy_to(u32::MAX); + test_copy_to(u64::from(u16::MAX) + 1); + test_copy_to(u64::from(u32::MAX)); } -fn testcase_init_from_to(dst: u32, src: u32) -> TranslationTest { +fn testcase_init_from_to(dst: u64, src: u32) -> TranslationTest { let wasm = &format!( r" (module @@ -211,10 +211,10 @@ fn testcase_init_from_to(dst: u32, src: u32) -> TranslationTest { TranslationTest::new(wasm) } -fn test_copy_from_to16(dst: u32, src: u32) { +fn test_copy_from_to16(dst: u64, src: u32) { testcase_init_from_to(dst, src) .expect_func_instrs([ - Instruction::memory_init_from_to(u32imm16(dst), u32imm16(src), Reg::from(0)), + Instruction::memory_init_from_to(u64imm16(dst), u32imm16(src), Reg::from(0)), Instruction::memory_index(0), Instruction::data_index(0), Instruction::Return, @@ -228,12 +228,12 @@ fn init_from_to16() { let values = [0, 1, u32::from(u16::MAX) - 1, u32::from(u16::MAX)]; for dst in values { for src in values { - test_copy_from_to16(dst, src); + test_copy_from_to16(u64::from(dst), src); } } } -fn test_copy_from_to(dst: u32, src: u32) { +fn test_copy_from_to(dst: u64, src: u32) { testcase_init_from_to(dst, src) .expect_func( ExpectedFunc::new([ @@ -242,7 +242,7 @@ fn test_copy_from_to(dst: u32, src: u32) { Instruction::data_index(0), Instruction::Return, ]) - .consts([dst, src]), + .consts([dst, u64::from(src)]), ) .run() } @@ -260,12 +260,12 @@ fn init_from_to() { // Ideally we'd have yet another test for that case. continue; } - test_copy_from_to(dst, src); + test_copy_from_to(u64::from(dst), src); } } } -fn testcase_init_to_exact(dst: u32, len: u32) -> TranslationTest { +fn testcase_init_to_exact(dst: u64, len: u32) -> TranslationTest { let wasm = &format!( r" (module @@ -282,10 +282,10 @@ fn testcase_init_to_exact(dst: u32, len: u32) -> TranslationTest { TranslationTest::new(wasm) } -fn test_copy_to_exact16(dst: u32, len: u32) { +fn test_copy_to_exact16(dst: u64, len: u32) { testcase_init_to_exact(dst, len) .expect_func_instrs([ - Instruction::memory_init_to_exact(u32imm16(dst), Reg::from(0), u32imm16(len)), + Instruction::memory_init_to_exact(u64imm16(dst), Reg::from(0), u32imm16(len)), Instruction::memory_index(0), Instruction::data_index(0), Instruction::Return, @@ -299,12 +299,12 @@ fn init_to_exact16() { let values = [0, 1, u32::from(u16::MAX) - 1, u32::from(u16::MAX)]; for dst in values { for len in values { - test_copy_to_exact16(dst, len); + test_copy_to_exact16(u64::from(dst), len); } } } -fn test_copy_to_exact(dst: u32, len: u32) { +fn test_copy_to_exact(dst: u64, len: u32) { testcase_init_to_exact(dst, len) .expect_func( ExpectedFunc::new([ @@ -313,7 +313,7 @@ fn test_copy_to_exact(dst: u32, len: u32) { Instruction::data_index(0), Instruction::Return, ]) - .consts([dst, len]), + .consts([dst, u64::from(len)]), ) .run() } @@ -331,7 +331,7 @@ fn init_to_exact() { // Ideally we'd have yet another test for that case. continue; } - test_copy_to_exact(dst, src); + test_copy_to_exact(u64::from(dst), src); } } } @@ -407,7 +407,7 @@ fn init_from_exact() { } } -fn testcase_init_from_to_exact(dst: u32, src: u32, len: u32) -> TranslationTest { +fn testcase_init_from_to_exact(dst: u64, src: u32, len: u32) -> TranslationTest { let wasm = &format!( r" (module @@ -424,10 +424,10 @@ fn testcase_init_from_to_exact(dst: u32, src: u32, len: u32) -> TranslationTest TranslationTest::new(wasm) } -fn test_copy_from_to_exact16(dst: u32, src: u32, len: u32) { +fn test_copy_from_to_exact16(dst: u64, src: u32, len: u32) { testcase_init_from_to_exact(dst, src, len) .expect_func_instrs([ - Instruction::memory_init_from_to_exact(u32imm16(dst), u32imm16(src), u32imm16(len)), + Instruction::memory_init_from_to_exact(u64imm16(dst), u32imm16(src), u32imm16(len)), Instruction::memory_index(0), Instruction::data_index(0), Instruction::Return, @@ -442,13 +442,13 @@ fn init_from_to_exact16() { for dst in values { for src in values { for len in values { - test_copy_from_to_exact16(dst, src, len); + test_copy_from_to_exact16(u64::from(dst), src, len); } } } } -fn test_copy_from_to_exact(dst: u32, src: u32, len: u32) { +fn test_copy_from_to_exact(dst: u64, src: u32, len: u32) { testcase_init_from_to_exact(dst, src, len) .expect_func( ExpectedFunc::new([ @@ -457,7 +457,7 @@ fn test_copy_from_to_exact(dst: u32, src: u32, len: u32) { Instruction::data_index(0), Instruction::Return, ]) - .consts([dst, src, len]), + .consts([dst, u64::from(src), u64::from(len)]), ) .run() } @@ -476,7 +476,7 @@ fn init_from_to_exact() { // Ideally we'd have yet another test for that case. continue; } - test_copy_from_to_exact(dst, src, len); + test_copy_from_to_exact(u64::from(dst), src, len); } } } diff --git a/crates/wasmi/src/engine/translator/tests/op/mod.rs b/crates/wasmi/src/engine/translator/tests/op/mod.rs index ec799a73df..a875b1ab3c 100644 --- a/crates/wasmi/src/engine/translator/tests/op/mod.rs +++ b/crates/wasmi/src/engine/translator/tests/op/mod.rs @@ -1,3 +1,13 @@ +/// Macro that turns an iterator over `Option` into an iterator over `T`. +/// +/// - Filters out all the `None` items yielded by the input iterator. +/// - Allows to specify `Some` items as just `T` as convenience. +macro_rules! iter_filter_opts { + [ $($item:expr),* $(,)? ] => {{ + [ $( ::core::option::Option::from($item) ),* ].into_iter().filter_map(|x| x) + }}; +} + mod binary; mod block; mod br; @@ -54,7 +64,8 @@ use super::{ WasmOp, WasmType, }; -use std::format; +use crate::ir::Offset16; +use std::{fmt, format}; /// Creates an [`Const32`] from the given `i32` value. /// @@ -79,6 +90,17 @@ fn u32imm16(value: u32) -> Const16 { .unwrap_or_else(|_| panic!("value must be 16-bit encodable: {}", value)) } +/// Creates an [`Const32`] from the given `u64` value. +/// +/// # Panics +/// +/// If the `value` cannot be converted into `u64` losslessly. +#[track_caller] +fn u64imm16(value: u64) -> Const16 { + >::try_from(value) + .unwrap_or_else(|_| panic!("value must be 16-bit encodable: {}", value)) +} + /// Creates an [`Const32`] from the given `i64` value. /// /// # Panics @@ -140,3 +162,80 @@ fn return_f64imm32_instr(value: f64) -> Instruction { fn return_nez_f64imm32_instr(condition: Reg, value: f64) -> Instruction { Instruction::return_nez_f64imm32(condition, f64imm32(value)) } + +/// Creates an [`Offset16`] from the given `offset`. +fn offset16(offset: u16) -> Offset16 { + Offset16::try_from(u64::from(offset)).unwrap() +} + +/// Adjusts a translation test to use memories with that specified index type. +#[derive(Copy, Clone)] +enum IndexType { + /// The 32-bit index type. + /// + /// This is WebAssembly's default. + Memory32, + /// The 64-bit index type. + /// + /// This got introduced by the Wasm `memory64` proposal. + Memory64, +} + +impl IndexType { + /// Returns the `.wat` string reprensetation for the [`IndexType`] of a `memory` declaration. + fn wat(&self) -> &'static str { + match self { + Self::Memory32 => "i32", + Self::Memory64 => "i64", + } + } +} + +/// Convenience type to create Wat memories with a tagged memory index. +#[derive(Copy, Clone)] +pub struct MemIdx(u32); + +impl fmt::Display for MemIdx { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "$mem{}", self.0) + } +} + +impl MemIdx { + /// Returns `true` if [`MemIdx`] refers to the default Wasm memory index. + fn is_default(&self) -> bool { + self.0 == 0 + } + + /// Returns the `$mem{n}` memory index used by some Wasm memory instructions. + fn instr(&self) -> Option { + match self.0 { + 0 => None, + n => Some(Instruction::memory_index(n)), + } + } +} + +/// Asserts that `ptr+offset` overflow either `i32` or `i64` depending on `index_ty`. +fn assert_overflowing_ptr_offset(index_ty: IndexType, ptr: u64, offset: u64) { + match index_ty { + IndexType::Memory32 => { + let Ok(ptr32) = u32::try_from(ptr) else { + panic!("ptr must be a 32-bit value but found: {ptr}"); + }; + let Ok(offset32) = u32::try_from(offset) else { + panic!("offset must be a 32-bit value but found: {offset}"); + }; + assert!( + ptr32.checked_add(offset32).is_none(), + "ptr+offset must overflow in this testcase (32-bit)" + ); + } + IndexType::Memory64 => { + assert!( + ptr.checked_add(offset).is_none(), + "ptr+offset must overflow in this testcase (64-bit)" + ); + } + } +} diff --git a/crates/wasmi/src/engine/translator/tests/op/return_call/indirect.rs b/crates/wasmi/src/engine/translator/tests/op/return_call/indirect.rs index 70d20f2a55..ef555bb39d 100644 --- a/crates/wasmi/src/engine/translator/tests/op/return_call/indirect.rs +++ b/crates/wasmi/src/engine/translator/tests/op/return_call/indirect.rs @@ -25,7 +25,7 @@ fn no_params_reg() { #[test] #[cfg_attr(miri, ignore)] fn no_params_imm16() { - fn test_with(index: u32) { + fn test_with(index: u64) { let wasm = format!( r#" (module @@ -41,15 +41,15 @@ fn no_params_imm16() { TranslationTest::new(&wasm) .expect_func_instrs([ Instruction::return_call_indirect_0_imm16(FuncType::from(0)), - Instruction::call_indirect_params_imm16(u32imm16(index), Table::from(0)), + Instruction::call_indirect_params_imm16(u64imm16(index), Table::from(0)), ]) .run(); } test_with(0); test_with(1); - test_with(u32::from(u16::MAX) - 1); - test_with(u32::from(u16::MAX)); + test_with(u64::from(u16::MAX) - 1); + test_with(u64::from(u16::MAX)); } #[test] @@ -78,7 +78,7 @@ fn one_reg_param_reg() { #[test] #[cfg_attr(miri, ignore)] fn one_reg_param_imm16() { - fn test_with(index: u32) { + fn test_with(index: u64) { let wasm = format!( r#" (module @@ -95,7 +95,7 @@ fn one_reg_param_imm16() { TranslationTest::new(&wasm) .expect_func_instrs([ Instruction::return_call_indirect_imm16(FuncType::from(0)), - Instruction::call_indirect_params_imm16(u32imm16(index), Table::from(0)), + Instruction::call_indirect_params_imm16(u64imm16(index), Table::from(0)), Instruction::register(1), ]) .run(); @@ -103,8 +103,8 @@ fn one_reg_param_imm16() { test_with(0); test_with(1); - test_with(u32::from(u16::MAX) - 1); - test_with(u32::from(u16::MAX)); + test_with(u64::from(u16::MAX) - 1); + test_with(u64::from(u16::MAX)); } #[test] @@ -170,7 +170,7 @@ fn one_imm_param_reg() { #[test] #[cfg_attr(miri, ignore)] fn one_imm_param_imm16() { - fn test_with(index: u32) { + fn test_with(index: u64) { let wasm = format!( r#" (module @@ -188,7 +188,7 @@ fn one_imm_param_imm16() { .expect_func( ExpectedFunc::new([ Instruction::return_call_indirect_imm16(FuncType::from(0)), - Instruction::call_indirect_params_imm16(u32imm16(index), Table::from(0)), + Instruction::call_indirect_params_imm16(u64imm16(index), Table::from(0)), Instruction::register(-1), ]) .consts([10_i32]), @@ -198,8 +198,8 @@ fn one_imm_param_imm16() { test_with(0); test_with(1); - test_with(u32::from(u16::MAX) - 1); - test_with(u32::from(u16::MAX)); + test_with(u64::from(u16::MAX) - 1); + test_with(u64::from(u16::MAX)); } #[test] @@ -317,7 +317,7 @@ fn two_imm_params_reg() { #[test] #[cfg_attr(miri, ignore)] fn two_reg_params_imm16() { - fn test_with(index: u32) { + fn test_with(index: u64) { let wasm = format!( r#" (module @@ -332,7 +332,7 @@ fn two_reg_params_imm16() { ) "#, ); - let elem_index = u32imm16(index); + let elem_index = u64imm16(index); TranslationTest::new(&wasm) .expect_func_instrs([ Instruction::return_call_indirect_imm16(FuncType::from(0)), @@ -344,14 +344,14 @@ fn two_reg_params_imm16() { test_with(0); test_with(1); - test_with(u32::from(u16::MAX) - 1); - test_with(u32::from(u16::MAX)); + test_with(u64::from(u16::MAX) - 1); + test_with(u64::from(u16::MAX)); } #[test] #[cfg_attr(miri, ignore)] fn two_reg_params_lhs_imm16() { - fn test_with(index: u32) { + fn test_with(index: u64) { let wasm = format!( r#" (module @@ -366,7 +366,7 @@ fn two_reg_params_lhs_imm16() { ) "#, ); - let elem_index = u32imm16(index); + let elem_index = u64imm16(index); TranslationTest::new(&wasm) .expect_func_instrs([ Instruction::return_call_indirect_imm16(FuncType::from(0)), @@ -378,14 +378,14 @@ fn two_reg_params_lhs_imm16() { test_with(0); test_with(1); - test_with(u32::from(u16::MAX) - 1); - test_with(u32::from(u16::MAX)); + test_with(u64::from(u16::MAX) - 1); + test_with(u64::from(u16::MAX)); } #[test] #[cfg_attr(miri, ignore)] fn two_imm_params_imm16() { - fn test_with(index: u32) { + fn test_with(index: u64) { let wasm = format!( r#" (module @@ -400,7 +400,7 @@ fn two_imm_params_imm16() { ) "#, ); - let elem_index = u32imm16(index); + let elem_index = u64imm16(index); TranslationTest::new(&wasm) .expect_func( ExpectedFunc::new([ @@ -415,8 +415,8 @@ fn two_imm_params_imm16() { test_with(0); test_with(1); - test_with(u32::from(u16::MAX) - 1); - test_with(u32::from(u16::MAX)); + test_with(u64::from(u16::MAX) - 1); + test_with(u64::from(u16::MAX)); } #[test] @@ -503,7 +503,7 @@ fn three_imm_params_reg() { #[test] #[cfg_attr(miri, ignore)] fn three_imm_params_imm16() { - fn test_with(index: u32) { + fn test_with(index: u64) { let wasm = format!( r#" (module @@ -519,7 +519,7 @@ fn three_imm_params_imm16() { ) "#, ); - let elem_index = u32imm16(index); + let elem_index = u64imm16(index); TranslationTest::new(&wasm) .expect_func( ExpectedFunc::new([ @@ -534,8 +534,8 @@ fn three_imm_params_imm16() { test_with(0); test_with(1); - test_with(u32::from(u16::MAX) - 1); - test_with(u32::from(u16::MAX)); + test_with(u64::from(u16::MAX) - 1); + test_with(u64::from(u16::MAX)); } #[test] diff --git a/crates/wasmi/src/engine/translator/tests/op/store/f32_store.rs b/crates/wasmi/src/engine/translator/tests/op/store/f32_store.rs index de3aea519c..bc8573d445 100644 --- a/crates/wasmi/src/engine/translator/tests/op/store/f32_store.rs +++ b/crates/wasmi/src/engine/translator/tests/op/store/f32_store.rs @@ -2,6 +2,19 @@ use super::*; const WASM_OP: WasmOp = WasmOp::store(WasmType::F32, "store"); +const DEFAULT_TEST_VALUES: [f32; 10] = [ + 0.0, + 0.5, + -0.5, + 1.0, + -1.0, + 42.25, + -42.25, + f32::INFINITY, + f32::NEG_INFINITY, + f32::NAN, +]; + #[test] #[cfg_attr(miri, ignore)] fn reg() { @@ -11,13 +24,9 @@ fn reg() { #[test] #[cfg_attr(miri, ignore)] fn imm() { - test_store_imm::(WASM_OP, 0.0, Instruction::store32); - test_store_imm::(WASM_OP, 1.0, Instruction::store32); - test_store_imm::(WASM_OP, -1.0, Instruction::store32); - test_store_imm::(WASM_OP, 42.25, Instruction::store32); - test_store_imm::(WASM_OP, f32::INFINITY, Instruction::store32); - test_store_imm::(WASM_OP, f32::NEG_INFINITY, Instruction::store32); - test_store_imm::(WASM_OP, f32::NAN, Instruction::store32); + for value in DEFAULT_TEST_VALUES { + test_store_imm::(WASM_OP, Instruction::store32, value); + } } #[test] @@ -29,12 +38,9 @@ fn offset16() { #[test] #[cfg_attr(miri, ignore)] fn offset16_imm() { - test_store_offset16_imm::(WASM_OP, 0.0, Instruction::store32_offset16); - test_store_offset16_imm::(WASM_OP, 1.0, Instruction::store32_offset16); - test_store_offset16_imm::(WASM_OP, -1.0, Instruction::store32_offset16); - test_store_offset16_imm::(WASM_OP, f32::INFINITY, Instruction::store32_offset16); - test_store_offset16_imm::(WASM_OP, f32::NEG_INFINITY, Instruction::store32_offset16); - test_store_offset16_imm::(WASM_OP, f32::NAN, Instruction::store32_offset16); + for value in DEFAULT_TEST_VALUES { + test_store_offset16_imm::(WASM_OP, value, Instruction::store32_offset16); + } } #[test] @@ -49,25 +55,32 @@ fn at_overflow() { test_store_at_overflow(WASM_OP); } +#[test] +#[cfg_attr(miri, ignore)] +fn at_fallback() { + test_store_at_fallback(WASM_OP, Instruction::store32); +} + #[test] #[cfg_attr(miri, ignore)] fn at_imm() { - test_store_at_imm::(WASM_OP, 0.0, Instruction::store32_at); - test_store_at_imm::(WASM_OP, 1.0, Instruction::store32_at); - test_store_at_imm::(WASM_OP, -1.0, Instruction::store32_at); - test_store_at_imm::(WASM_OP, f32::NEG_INFINITY, Instruction::store32_at); - test_store_at_imm::(WASM_OP, f32::INFINITY, Instruction::store32_at); - test_store_at_imm::(WASM_OP, f32::NAN, Instruction::store32_at); + for value in DEFAULT_TEST_VALUES { + test_store_at_imm::(WASM_OP, value, Instruction::store32_at); + } } #[test] #[cfg_attr(miri, ignore)] fn at_imm_overflow() { - test_store_at_imm_overflow(WASM_OP, 0.0); - test_store_at_imm_overflow(WASM_OP, 1.0); - test_store_at_imm_overflow(WASM_OP, -1.0); - test_store_at_imm_overflow(WASM_OP, 42.25); - test_store_at_imm_overflow(WASM_OP, f32::NEG_INFINITY); - test_store_at_imm_overflow(WASM_OP, f32::INFINITY); - test_store_at_imm_overflow(WASM_OP, f32::NAN); + for value in DEFAULT_TEST_VALUES { + test_store_at_imm_overflow(WASM_OP, value); + } +} + +#[test] +#[cfg_attr(miri, ignore)] +fn at_imm_fallback() { + for value in DEFAULT_TEST_VALUES { + test_store_at_imm_fallback(WASM_OP, Instruction::store32, value); + } } diff --git a/crates/wasmi/src/engine/translator/tests/op/store/f64_store.rs b/crates/wasmi/src/engine/translator/tests/op/store/f64_store.rs index ba786a2604..4dbbf2ad1b 100644 --- a/crates/wasmi/src/engine/translator/tests/op/store/f64_store.rs +++ b/crates/wasmi/src/engine/translator/tests/op/store/f64_store.rs @@ -2,6 +2,19 @@ use super::*; const WASM_OP: WasmOp = WasmOp::store(WasmType::F64, "store"); +const DEFAULT_TEST_VALUES: [f64; 10] = [ + 0.0, + 0.5, + -0.5, + 1.0, + -1.0, + 42.25, + -42.25, + f64::INFINITY, + f64::NEG_INFINITY, + f64::NAN, +]; + #[test] #[cfg_attr(miri, ignore)] fn reg() { @@ -11,13 +24,9 @@ fn reg() { #[test] #[cfg_attr(miri, ignore)] fn imm() { - test_store_imm::(WASM_OP, 0.0, Instruction::store64); - test_store_imm::(WASM_OP, 1.0, Instruction::store64); - test_store_imm::(WASM_OP, -1.0, Instruction::store64); - test_store_imm::(WASM_OP, 42.25, Instruction::store64); - test_store_imm::(WASM_OP, f64::INFINITY, Instruction::store64); - test_store_imm::(WASM_OP, f64::NEG_INFINITY, Instruction::store64); - test_store_imm::(WASM_OP, f64::NAN, Instruction::store64); + for value in DEFAULT_TEST_VALUES { + test_store_imm::(WASM_OP, Instruction::store64, value); + } } #[test] @@ -29,12 +38,9 @@ fn offset16() { #[test] #[cfg_attr(miri, ignore)] fn offset16_imm() { - test_store_offset16_imm::(WASM_OP, 0.0, Instruction::store64_offset16); - test_store_offset16_imm::(WASM_OP, 1.0, Instruction::store64_offset16); - test_store_offset16_imm::(WASM_OP, -1.0, Instruction::store64_offset16); - test_store_offset16_imm::(WASM_OP, f64::INFINITY, Instruction::store64_offset16); - test_store_offset16_imm::(WASM_OP, f64::NEG_INFINITY, Instruction::store64_offset16); - test_store_offset16_imm::(WASM_OP, f64::NAN, Instruction::store64_offset16); + for value in DEFAULT_TEST_VALUES { + test_store_offset16_imm::(WASM_OP, value, Instruction::store64_offset16); + } } #[test] @@ -49,25 +55,32 @@ fn at_overflow() { test_store_at_overflow(WASM_OP); } +#[test] +#[cfg_attr(miri, ignore)] +fn at_fallback() { + test_store_at_fallback(WASM_OP, Instruction::store64); +} + #[test] #[cfg_attr(miri, ignore)] fn at_imm() { - test_store_at_imm::(WASM_OP, 0.0, Instruction::store64_at); - test_store_at_imm::(WASM_OP, 1.0, Instruction::store64_at); - test_store_at_imm::(WASM_OP, -1.0, Instruction::store64_at); - test_store_at_imm::(WASM_OP, f64::NEG_INFINITY, Instruction::store64_at); - test_store_at_imm::(WASM_OP, f64::INFINITY, Instruction::store64_at); - test_store_at_imm::(WASM_OP, f64::NAN, Instruction::store64_at); + for value in DEFAULT_TEST_VALUES { + test_store_at_imm::(WASM_OP, value, Instruction::store64_at); + } } #[test] #[cfg_attr(miri, ignore)] fn at_imm_overflow() { - test_store_at_imm_overflow(WASM_OP, 0.0); - test_store_at_imm_overflow(WASM_OP, 1.0); - test_store_at_imm_overflow(WASM_OP, -1.0); - test_store_at_imm_overflow(WASM_OP, 42.25); - test_store_at_imm_overflow(WASM_OP, f64::NEG_INFINITY); - test_store_at_imm_overflow(WASM_OP, f64::INFINITY); - test_store_at_imm_overflow(WASM_OP, f64::NAN); + for value in DEFAULT_TEST_VALUES { + test_store_at_imm_overflow(WASM_OP, value); + } +} + +#[test] +#[cfg_attr(miri, ignore)] +fn at_imm_fallback() { + for value in DEFAULT_TEST_VALUES { + test_store_at_imm_fallback(WASM_OP, Instruction::store64, value); + } } diff --git a/crates/wasmi/src/engine/translator/tests/op/store/i32_store.rs b/crates/wasmi/src/engine/translator/tests/op/store/i32_store.rs index c260a654fe..46ca630b5e 100644 --- a/crates/wasmi/src/engine/translator/tests/op/store/i32_store.rs +++ b/crates/wasmi/src/engine/translator/tests/op/store/i32_store.rs @@ -20,7 +20,7 @@ fn imm() { i32::MAX - 1, ]; for value in values { - test_store_imm::(WASM_OP, value, Instruction::store32); + test_store_imm::(WASM_OP, Instruction::store32, value); } } @@ -51,32 +51,37 @@ fn offset16() { #[test] #[cfg_attr(miri, ignore)] fn offset16_imm() { - test_store_offset16_imm::( - WASM_OP, + [ i32::from(i16::MIN) - 1, - Instruction::store32_offset16, - ); - test_store_offset16_imm::( - WASM_OP, i32::from(i16::MAX) + 1, - Instruction::store32_offset16, - ); - test_store_offset16_imm::(WASM_OP, i32::MIN + 1, Instruction::store32_offset16); - test_store_offset16_imm::(WASM_OP, i32::MAX - 1, Instruction::store32_offset16); - test_store_offset16_imm::(WASM_OP, i32::MIN, Instruction::store32_offset16); - test_store_offset16_imm::(WASM_OP, i32::MAX, Instruction::store32_offset16); + i32::MIN + 1, + i32::MAX - 1, + i32::MAX, + ] + .into_iter() + .for_each(|value| { + test_store_offset16_imm::(WASM_OP, value, Instruction::store32_offset16); + }) } #[test] #[cfg_attr(miri, ignore)] fn offset16_imm16() { - test_store_offset16_imm16::(WASM_OP, 0, Instruction::i32_store_offset16_imm16); - test_store_offset16_imm16::(WASM_OP, 1, Instruction::i32_store_offset16_imm16); - test_store_offset16_imm16::(WASM_OP, -1, Instruction::i32_store_offset16_imm16); - test_store_offset16_imm16::(WASM_OP, i16::MIN + 1, Instruction::i32_store_offset16_imm16); - test_store_offset16_imm16::(WASM_OP, i16::MAX - 1, Instruction::i32_store_offset16_imm16); - test_store_offset16_imm16::(WASM_OP, i16::MIN, Instruction::i32_store_offset16_imm16); - test_store_offset16_imm16::(WASM_OP, i16::MAX, Instruction::i32_store_offset16_imm16); + [ + 0, + -1, + 1, + -42, + 42, + i16::MIN + 1, + i16::MIN, + i16::MAX - 1, + i16::MAX, + ] + .into_iter() + .for_each(|value| { + test_store_offset16_imm16::(WASM_OP, Instruction::i32_store_offset16_imm16, value); + }) } #[test] @@ -91,21 +96,73 @@ fn at_overflow() { test_store_at_overflow(WASM_OP); } +#[test] +#[cfg_attr(miri, ignore)] +fn at_fallback() { + test_store_at_fallback(WASM_OP, Instruction::store32); +} + #[test] #[cfg_attr(miri, ignore)] fn at_imm() { - test_store_at_imm::(WASM_OP, i32::from(i16::MAX) + 1, Instruction::store32_at); - test_store_at_imm::(WASM_OP, i32::MAX - 1, Instruction::store32_at); - test_store_at_imm::(WASM_OP, i32::MAX, Instruction::store32_at); + [i32::from(i16::MAX) + 1, i32::MAX - 1, i32::MAX] + .into_iter() + .for_each(|value| { + test_store_at_imm::(WASM_OP, value, Instruction::store32_at); + }) } #[test] #[cfg_attr(miri, ignore)] fn imm_at_overflow() { - test_store_at_imm_overflow(WASM_OP, 0); - test_store_at_imm_overflow(WASM_OP, 1); - test_store_at_imm_overflow(WASM_OP, -1); - test_store_at_imm_overflow(WASM_OP, 42); - test_store_at_imm_overflow(WASM_OP, i32::MIN); - test_store_at_imm_overflow(WASM_OP, i32::MAX); + [ + 0, + 1, + -1, + 42, + -42, + i32::MIN, + i32::MIN + 1, + i32::MAX - 1, + i32::MAX, + ] + .into_iter() + .for_each(|value| { + test_store_at_imm_overflow(WASM_OP, value); + }) +} + +#[test] +#[cfg_attr(miri, ignore)] +fn at_imm16_fallback() { + [ + 0, + -1, + 1, + -42, + 42, + i32::from(i16::MIN), + i32::from(i16::MIN) + 1, + i32::from(i16::MAX) - 1, + i32::from(i16::MAX), + ] + .into_iter() + .for_each(|value| { + test_store_at_imm16_fallback::(WASM_OP, Instruction::i32_store_imm16, value); + }) +} + +#[test] +#[cfg_attr(miri, ignore)] +fn at_imm_fallback() { + for value in [ + i32::from(i16::MIN) - 1, + i32::from(i16::MAX) + 1, + i32::MIN, + i32::MIN + 1, + i32::MAX - 1, + i32::MAX, + ] { + test_store_at_imm_fallback::(WASM_OP, Instruction::store32, value); + } } diff --git a/crates/wasmi/src/engine/translator/tests/op/store/i32_store16.rs b/crates/wasmi/src/engine/translator/tests/op/store/i32_store16.rs index 15482f9a11..bf1a06724e 100644 --- a/crates/wasmi/src/engine/translator/tests/op/store/i32_store16.rs +++ b/crates/wasmi/src/engine/translator/tests/op/store/i32_store16.rs @@ -28,7 +28,7 @@ fn imm() { i32::MAX - 1, ]; for value in values { - test_store_wrap_imm::(WASM_OP, value, Instruction::i32_store16_imm); + test_store_wrap_imm::(WASM_OP, Instruction::i32_store16_imm, value); } } @@ -85,6 +85,12 @@ fn at_overflow() { test_store_at_overflow(WASM_OP); } +#[test] +#[cfg_attr(miri, ignore)] +fn at_fallback() { + test_store_at_fallback(WASM_OP, Instruction::i32_store16); +} + #[test] #[cfg_attr(miri, ignore)] fn at_imm() { @@ -103,7 +109,7 @@ fn at_imm() { i32::MAX, ]; for value in values { - test_store_wrap_at_imm::(WASM_OP, value, Instruction::i32_store16_at_imm); + test_store_wrap_at_imm::(WASM_OP, Instruction::i32_store16_at_imm, value); } } @@ -115,3 +121,23 @@ fn imm_at_overflow() { test_store_at_imm_overflow(WASM_OP, value); } } + +#[test] +#[cfg_attr(miri, ignore)] +fn at_imm_fallback() { + [ + 0, + -1, + 1, + -42, + 42, + i32::from(i16::MIN), + i32::from(i16::MIN) + 1, + i32::from(i16::MAX) - 1, + i32::from(i16::MAX), + ] + .into_iter() + .for_each(|value| { + test_store_wrap_at_imm16_fallback::(WASM_OP, Instruction::i32_store16_imm, value); + }) +} diff --git a/crates/wasmi/src/engine/translator/tests/op/store/i32_store8.rs b/crates/wasmi/src/engine/translator/tests/op/store/i32_store8.rs index 3f4f71c3dc..3eb502f0f1 100644 --- a/crates/wasmi/src/engine/translator/tests/op/store/i32_store8.rs +++ b/crates/wasmi/src/engine/translator/tests/op/store/i32_store8.rs @@ -28,7 +28,7 @@ fn imm() { i32::MAX - 1, ]; for value in values { - test_store_wrap_imm::(WASM_OP, value, Instruction::i32_store8_imm); + test_store_wrap_imm::(WASM_OP, Instruction::i32_store8_imm, value); } } @@ -85,6 +85,12 @@ fn at_overflow() { test_store_at_overflow(WASM_OP); } +#[test] +#[cfg_attr(miri, ignore)] +fn at_fallback() { + test_store_at_fallback(WASM_OP, Instruction::i32_store8); +} + #[test] #[cfg_attr(miri, ignore)] fn at_imm() { @@ -103,7 +109,7 @@ fn at_imm() { i32::MAX, ]; for value in values { - test_store_wrap_at_imm::(WASM_OP, value, Instruction::i32_store8_at_imm); + test_store_wrap_at_imm::(WASM_OP, Instruction::i32_store8_at_imm, value); } } @@ -115,3 +121,23 @@ fn imm_at_overflow() { test_store_at_imm_overflow(WASM_OP, value); } } + +#[test] +#[cfg_attr(miri, ignore)] +fn at_imm_fallback() { + [ + 0, + -1, + 1, + -42, + 42, + i32::from(i8::MIN), + i32::from(i8::MIN) + 1, + i32::from(i8::MAX) - 1, + i32::from(i8::MAX), + ] + .into_iter() + .for_each(|value| { + test_store_wrap_at_imm16_fallback::(WASM_OP, Instruction::i32_store8_imm, value); + }) +} diff --git a/crates/wasmi/src/engine/translator/tests/op/store/i64_store.rs b/crates/wasmi/src/engine/translator/tests/op/store/i64_store.rs index 47a2ddbf22..b3885a06b1 100644 --- a/crates/wasmi/src/engine/translator/tests/op/store/i64_store.rs +++ b/crates/wasmi/src/engine/translator/tests/op/store/i64_store.rs @@ -20,7 +20,7 @@ fn imm() { i64::MAX - 1, ]; for value in values { - test_store_imm::(WASM_OP, value, Instruction::store64); + test_store_imm::(WASM_OP, Instruction::store64, value); } } @@ -51,32 +51,38 @@ fn offset16() { #[test] #[cfg_attr(miri, ignore)] fn offset16_imm() { - test_store_offset16_imm::( - WASM_OP, + [ i64::from(i16::MIN) - 1, - Instruction::store64_offset16, - ); - test_store_offset16_imm::( - WASM_OP, i64::from(i16::MAX) + 1, - Instruction::store64_offset16, - ); - test_store_offset16_imm::(WASM_OP, i64::MAX - 1, Instruction::store64_offset16); - test_store_offset16_imm::(WASM_OP, i64::MIN + 1, Instruction::store64_offset16); - test_store_offset16_imm::(WASM_OP, i64::MIN, Instruction::store64_offset16); - test_store_offset16_imm::(WASM_OP, i64::MAX, Instruction::store64_offset16); + i64::MIN, + i64::MIN + 1, + i64::MAX - 1, + i64::MAX, + ] + .into_iter() + .for_each(|value| { + test_store_offset16_imm::(WASM_OP, value, Instruction::store64_offset16); + }) } #[test] #[cfg_attr(miri, ignore)] fn offset16_imm16() { - test_store_offset16_imm16::(WASM_OP, 0, Instruction::i64_store_offset16_imm16); - test_store_offset16_imm16::(WASM_OP, 1, Instruction::i64_store_offset16_imm16); - test_store_offset16_imm16::(WASM_OP, -1, Instruction::i64_store_offset16_imm16); - test_store_offset16_imm16::(WASM_OP, i16::MIN + 1, Instruction::i64_store_offset16_imm16); - test_store_offset16_imm16::(WASM_OP, i16::MAX - 1, Instruction::i64_store_offset16_imm16); - test_store_offset16_imm16::(WASM_OP, i16::MIN, Instruction::i64_store_offset16_imm16); - test_store_offset16_imm16::(WASM_OP, i16::MAX, Instruction::i64_store_offset16_imm16); + [ + 0, + -1, + 1, + -42, + 42, + i16::MIN + 1, + i16::MIN, + i16::MAX - 1, + i16::MAX, + ] + .into_iter() + .for_each(|value| { + test_store_offset16_imm16::(WASM_OP, Instruction::i64_store_offset16_imm16, value); + }) } #[test] @@ -91,6 +97,12 @@ fn at_overflow() { test_store_at_overflow(WASM_OP); } +#[test] +#[cfg_attr(miri, ignore)] +fn at_fallback() { + test_store_at_fallback(WASM_OP, Instruction::store64); +} + #[test] #[cfg_attr(miri, ignore)] fn at_imm() { @@ -102,10 +114,44 @@ fn at_imm() { #[test] #[cfg_attr(miri, ignore)] fn imm_at_overflow() { - test_store_at_imm_overflow(WASM_OP, 0); - test_store_at_imm_overflow(WASM_OP, 1); - test_store_at_imm_overflow(WASM_OP, -1); - test_store_at_imm_overflow(WASM_OP, 42); - test_store_at_imm_overflow(WASM_OP, i64::MIN); - test_store_at_imm_overflow(WASM_OP, i64::MAX); + [0, 1, -1, 42, i64::MIN, i64::MAX] + .into_iter() + .for_each(|value| { + test_store_at_imm_overflow(WASM_OP, value); + }) +} + +#[test] +#[cfg_attr(miri, ignore)] +fn at_imm16_fallback() { + [ + 0, + -1, + 1, + -42, + 42, + i64::from(i16::MIN), + i64::from(i16::MIN) + 1, + i64::from(i16::MAX) - 1, + i64::from(i16::MAX), + ] + .into_iter() + .for_each(|value| { + test_store_at_imm16_fallback::(WASM_OP, Instruction::i64_store_imm16, value); + }) +} + +#[test] +#[cfg_attr(miri, ignore)] +fn at_imm_fallback() { + for value in [ + i64::from(i16::MIN) - 1, + i64::from(i16::MAX) + 1, + i64::MIN, + i64::MIN + 1, + i64::MAX - 1, + i64::MAX, + ] { + test_store_at_imm_fallback::(WASM_OP, Instruction::store64, value); + } } diff --git a/crates/wasmi/src/engine/translator/tests/op/store/i64_store16.rs b/crates/wasmi/src/engine/translator/tests/op/store/i64_store16.rs index 127761daae..1e318cce07 100644 --- a/crates/wasmi/src/engine/translator/tests/op/store/i64_store16.rs +++ b/crates/wasmi/src/engine/translator/tests/op/store/i64_store16.rs @@ -28,7 +28,7 @@ fn imm() { i64::MAX - 1, ]; for value in values { - test_store_wrap_imm::(WASM_OP, value, Instruction::i64_store16_imm); + test_store_wrap_imm::(WASM_OP, Instruction::i64_store16_imm, value); } } @@ -85,6 +85,12 @@ fn at_overflow() { test_store_at_overflow(WASM_OP); } +#[test] +#[cfg_attr(miri, ignore)] +fn at_fallback() { + test_store_at_fallback(WASM_OP, Instruction::i64_store16); +} + #[test] #[cfg_attr(miri, ignore)] fn at_imm() { @@ -103,7 +109,7 @@ fn at_imm() { i64::MAX, ]; for value in values { - test_store_wrap_at_imm::(WASM_OP, value, Instruction::i64_store16_at_imm); + test_store_wrap_at_imm::(WASM_OP, Instruction::i64_store16_at_imm, value); } } @@ -115,3 +121,22 @@ fn imm_at_overflow() { test_store_at_imm_overflow(WASM_OP, value); } } + +#[test] +#[cfg_attr(miri, ignore)] +fn at_imm_fallback() { + [ + 0, + 1, + -1, + 42, + i64::from(i16::MIN), + i64::from(i16::MAX), + i64::MIN, + i64::MAX, + ] + .into_iter() + .for_each(|value| { + test_store_wrap_at_imm16_fallback::(WASM_OP, Instruction::i64_store16_imm, value); + }); +} diff --git a/crates/wasmi/src/engine/translator/tests/op/store/i64_store32.rs b/crates/wasmi/src/engine/translator/tests/op/store/i64_store32.rs index c61dcc7079..28bf6b24c8 100644 --- a/crates/wasmi/src/engine/translator/tests/op/store/i64_store32.rs +++ b/crates/wasmi/src/engine/translator/tests/op/store/i64_store32.rs @@ -18,7 +18,7 @@ fn imm() { i64::from(i32::MIN) + i64::from(i16::MAX) - 1, ]; for value in values { - test_store_imm::(WASM_OP, value, Instruction::i64_store32); + test_store_imm::(WASM_OP, Instruction::i64_store32, value); } } @@ -40,7 +40,7 @@ fn imm16() { i64::MAX - 1, ]; for value in values { - test_store_wrap_imm::(WASM_OP, value, Instruction::i64_store32_imm16); + test_store_wrap_imm::(WASM_OP, Instruction::i64_store32_imm16, value); } } @@ -117,6 +117,12 @@ fn at_imm() { } } +#[test] +#[cfg_attr(miri, ignore)] +fn at_fallback() { + test_store_at_fallback(WASM_OP, Instruction::i64_store32); +} + #[test] #[cfg_attr(miri, ignore)] fn at_imm16() { @@ -135,7 +141,7 @@ fn at_imm16() { i64::MAX, ]; for value in values { - test_store_wrap_at_imm::(WASM_OP, value, Instruction::i64_store32_at_imm16); + test_store_wrap_at_imm::(WASM_OP, Instruction::i64_store32_at_imm16, value); } } @@ -147,3 +153,46 @@ fn imm_at_overflow() { test_store_at_imm_overflow(WASM_OP, value); } } + +#[test] +#[cfg_attr(miri, ignore)] +fn at_imm16_fallback() { + [ + 0, + 1, + -1, + 1000, + -1000, + i64::from(i16::MIN), + i64::from(i16::MIN) + 1, + i64::from(i16::MAX) - 1, + i64::from(i16::MAX), + i64::MIN, + i64::MIN + 1, + i64::MAX - 1, + i64::MAX, + ] + .into_iter() + .for_each(|value| { + test_store_wrap_at_imm16_fallback::( + WASM_OP, + Instruction::i64_store32_imm16, + value, + ); + }); +} + +#[test] +#[cfg_attr(miri, ignore)] +fn at_imm_fallback() { + for value in [ + i64::from(i16::MIN) - 1, + i64::from(i16::MAX) + 1, + i64::from(i32::MIN), + i64::from(i32::MIN) + 1, + i64::from(i32::MAX) - 1, + i64::from(i32::MAX), + ] { + test_store_at_imm_fallback::(WASM_OP, Instruction::i64_store32, value); + } +} diff --git a/crates/wasmi/src/engine/translator/tests/op/store/i64_store8.rs b/crates/wasmi/src/engine/translator/tests/op/store/i64_store8.rs index d78519deda..0e22925936 100644 --- a/crates/wasmi/src/engine/translator/tests/op/store/i64_store8.rs +++ b/crates/wasmi/src/engine/translator/tests/op/store/i64_store8.rs @@ -28,7 +28,7 @@ fn imm() { i64::MAX - 1, ]; for value in values { - test_store_wrap_imm::(WASM_OP, value, Instruction::i64_store8_imm); + test_store_wrap_imm::(WASM_OP, Instruction::i64_store8_imm, value); } } @@ -85,6 +85,12 @@ fn at_overflow() { test_store_at_overflow(WASM_OP); } +#[test] +#[cfg_attr(miri, ignore)] +fn at_fallback() { + test_store_at_fallback(WASM_OP, Instruction::i64_store8); +} + #[test] #[cfg_attr(miri, ignore)] fn at_imm() { @@ -103,7 +109,7 @@ fn at_imm() { i64::MAX, ]; for value in values { - test_store_wrap_at_imm::(WASM_OP, value, Instruction::i64_store8_at_imm); + test_store_wrap_at_imm::(WASM_OP, Instruction::i64_store8_at_imm, value); } } @@ -115,3 +121,22 @@ fn imm_at_overflow() { test_store_at_imm_overflow(WASM_OP, value); } } + +#[test] +#[cfg_attr(miri, ignore)] +fn at_imm_fallback() { + [ + 0, + 1, + -1, + 42, + i64::from(i8::MIN), + i64::from(i8::MAX), + i64::MIN, + i64::MAX, + ] + .into_iter() + .for_each(|value| { + test_store_wrap_at_imm16_fallback::(WASM_OP, Instruction::i64_store8_imm, value); + }); +} diff --git a/crates/wasmi/src/engine/translator/tests/op/store/mod.rs b/crates/wasmi/src/engine/translator/tests/op/store/mod.rs index 318bb4d38b..7c774d02c0 100644 --- a/crates/wasmi/src/engine/translator/tests/op/store/mod.rs +++ b/crates/wasmi/src/engine/translator/tests/op/store/mod.rs @@ -4,9 +4,8 @@ use super::*; use crate::{ core::UntypedVal, engine::translator::utils::Wrap, - ir::{index::Memory, AnyConst16}, + ir::{AnyConst16, Offset16, Offset64, Offset64Lo}, }; -use std::vec; mod f32_store; mod f64_store; @@ -23,52 +22,85 @@ use core::fmt::Display; fn test_store_for( wasm_op: WasmOp, - offset: u32, - make_instr: fn(ptr: Reg, memory: Memory) -> Instruction, + make_instr: fn(ptr: Reg, offset_lo: Offset64Lo) -> Instruction, + memory_index: MemIdx, + index_ty: IndexType, + offset: u64, ) { assert!( - u16::try_from(offset).is_err(), - "this test requires non-16 bit offsets but found {offset}" + u16::try_from(offset).is_err() || !memory_index.is_default(), + "this test requires non-16 bit offsets or non-default memory \ + but found: offset={offset}, memory={memory_index}" ); let param_ty = wasm_op.param_ty(); + let index_ty = index_ty.wat(); let wasm = format!( r#" (module - (memory 1) - (func (param $ptr i32) (param $value {param_ty}) + (memory $mem0 {index_ty} 1) + (memory $mem1 {index_ty} 1) + (func (param $ptr {index_ty}) (param $value {param_ty}) local.get $ptr local.get $value - {wasm_op} offset={offset} + {wasm_op} {memory_index} offset={offset} ) ) "# ); + let (offset_hi, offset_lo) = Offset64::split(offset); TranslationTest::new(&wasm) - .expect_func_instrs([ - make_instr(Reg::from(0), Memory::from(0)), - Instruction::register_and_imm32(Reg::from(1), offset), + .expect_func_instrs(iter_filter_opts![ + make_instr(Reg::from(0), offset_lo), + Instruction::register_and_offset_hi(Reg::from(1), offset_hi), + memory_index.instr(), Instruction::Return, ]) .run(); } -fn test_store(wasm_op: WasmOp, make_instr: fn(ptr: Reg, memory: Memory) -> Instruction) { - test_store_for(wasm_op, u32::from(u16::MAX) + 1, make_instr); - test_store_for(wasm_op, u32::MAX - 1, make_instr); - test_store_for(wasm_op, u32::MAX, make_instr); +fn test_store(wasm_op: WasmOp, make_instr: fn(ptr: Reg, offset_lo: Offset64Lo) -> Instruction) { + // Case: offsets that cannot be 16-bit encoded: + [ + u64::from(u16::MAX) + 1, + u64::from(u32::MAX) - 1, + u64::from(u32::MAX), + ] + .into_iter() + .for_each(|offset| { + test_store_for(wasm_op, make_instr, MemIdx(0), IndexType::Memory32, offset); + test_store_for(wasm_op, make_instr, MemIdx(1), IndexType::Memory32, offset); + test_store_for(wasm_op, make_instr, MemIdx(0), IndexType::Memory64, offset); + test_store_for(wasm_op, make_instr, MemIdx(1), IndexType::Memory64, offset); + }); + // Case: 64-bit offsets and `memory64`: + [u64::from(u32::MAX) + 1, u64::MAX - 1, u64::MAX] + .into_iter() + .for_each(|offset| { + test_store_for(wasm_op, make_instr, MemIdx(0), IndexType::Memory64, offset); + test_store_for(wasm_op, make_instr, MemIdx(1), IndexType::Memory64, offset); + }); + // Case: 16-bit offsets but non-default memory index: + [0, 1, u64::from(u16::MAX) - 1, u64::from(u16::MAX)] + .into_iter() + .for_each(|offset| { + test_store_for(wasm_op, make_instr, MemIdx(1), IndexType::Memory32, offset); + test_store_for(wasm_op, make_instr, MemIdx(1), IndexType::Memory64, offset); + }) } fn test_store_offset16_for( wasm_op: WasmOp, + make_instr: fn(ptr: Reg, offset: Offset16, value: Reg) -> Instruction, + index_ty: IndexType, offset: u16, - make_instr: fn(ptr: Reg, offset: u16, value: Reg) -> Instruction, ) { let param_ty = wasm_op.param_ty(); + let index_ty = index_ty.wat(); let wasm = format!( r#" (module - (memory 1) - (func (param $ptr i32) (param $value {param_ty}) + (memory {index_ty} 1) + (func (param $ptr {index_ty}) (param $value {param_ty}) local.get $ptr local.get $value {wasm_op} offset={offset} @@ -78,7 +110,7 @@ fn test_store_offset16_for( ); TranslationTest::new(&wasm) .expect_func_instrs([ - make_instr(Reg::from(0), offset, Reg::from(1)), + make_instr(Reg::from(0), offset16(offset), Reg::from(1)), Instruction::Return, ]) .run(); @@ -86,29 +118,34 @@ fn test_store_offset16_for( fn test_store_offset16( wasm_op: WasmOp, - make_instr: fn(ptr: Reg, offset: u16, value: Reg) -> Instruction, + make_instr: fn(ptr: Reg, offset: Offset16, value: Reg) -> Instruction, ) { - test_store_offset16_for(wasm_op, 0, make_instr); - test_store_offset16_for(wasm_op, u16::MAX - 1, make_instr); - test_store_offset16_for(wasm_op, u16::MAX, make_instr); + [0, 1, u16::MAX - 1, u16::MAX] + .into_iter() + .for_each(|offset| { + test_store_offset16_for(wasm_op, make_instr, IndexType::Memory32, offset); + test_store_offset16_for(wasm_op, make_instr, IndexType::Memory64, offset); + }) } fn test_store_offset16_imm_for( wasm_op: WasmOp, + make_instr: fn(ptr: Reg, offset: Offset16, value: Reg) -> Instruction, + index_ty: IndexType, offset: u16, value: T, - make_instr: fn(ptr: Reg, offset: u16, value: Reg) -> Instruction, ) where T: Copy + Into, DisplayWasm: Display, { let param_ty = wasm_op.param_ty(); + let index_ty = index_ty.wat(); let display_value = DisplayWasm::from(value); let wasm = format!( r#" (module - (memory 1) - (func (param $ptr i32) + (memory {index_ty} 1) + (func (param $ptr {index_ty}) local.get $ptr {param_ty}.const {display_value} {wasm_op} offset={offset} @@ -119,7 +156,7 @@ fn test_store_offset16_imm_for( TranslationTest::new(&wasm) .expect_func( ExpectedFunc::new([ - make_instr(Reg::from(0), offset, Reg::from(-1)), + make_instr(Reg::from(0), offset16(offset), Reg::from(-1)), Instruction::Return, ]) .consts([value]), @@ -130,32 +167,37 @@ fn test_store_offset16_imm_for( fn test_store_offset16_imm( wasm_op: WasmOp, value: T, - make_instr: fn(ptr: Reg, offset: u16, value: Reg) -> Instruction, + make_instr: fn(ptr: Reg, offset: Offset16, value: Reg) -> Instruction, ) where T: Copy + Into, DisplayWasm: Display, { - test_store_offset16_imm_for(wasm_op, 0, value, make_instr); - test_store_offset16_imm_for(wasm_op, u16::MAX - 1, value, make_instr); - test_store_offset16_imm_for(wasm_op, u16::MAX, value, make_instr); + [0, 1, u16::MAX - 1, u16::MAX] + .into_iter() + .for_each(|offset| { + test_store_offset16_imm_for(wasm_op, make_instr, IndexType::Memory32, offset, value); + test_store_offset16_imm_for(wasm_op, make_instr, IndexType::Memory64, offset, value); + }) } fn test_store_offset16_imm16_for( wasm_op: WasmOp, + make_instr: fn(ptr: Reg, offset: Offset16, value: T) -> Instruction, + index_ty: IndexType, offset: u16, value: T, - make_instr: fn(ptr: Reg, offset: u16, value: T) -> Instruction, ) where T: Copy, DisplayWasm: Display, { let param_ty = wasm_op.param_ty(); + let index_ty = index_ty.wat(); let display_value = DisplayWasm::from(value); let wasm = format!( r#" (module - (memory 1) - (func (param $ptr i32) + (memory {index_ty} 1) + (func (param $ptr {index_ty}) local.get $ptr {param_ty}.const {display_value} {wasm_op} offset={offset} @@ -164,40 +206,48 @@ fn test_store_offset16_imm16_for( "# ); TranslationTest::new(&wasm) - .expect_func_instrs([make_instr(Reg::from(0), offset, value), Instruction::Return]) + .expect_func_instrs([ + make_instr(Reg::from(0), offset16(offset), value), + Instruction::Return, + ]) .run(); } fn test_store_offset16_imm16( wasm_op: WasmOp, + make_instr: fn(ptr: Reg, offset: Offset16, value: T) -> Instruction, value: T, - make_instr: fn(ptr: Reg, offset: u16, value: T) -> Instruction, ) where T: Copy, DisplayWasm: Display, { - test_store_offset16_imm16_for(wasm_op, 0, value, make_instr); - test_store_offset16_imm16_for(wasm_op, u16::MAX - 1, value, make_instr); - test_store_offset16_imm16_for(wasm_op, u16::MAX, value, make_instr); + [0, 1, u16::MAX - 1, u16::MAX] + .into_iter() + .for_each(|offset| { + test_store_offset16_imm16_for(wasm_op, make_instr, IndexType::Memory32, offset, value); + test_store_offset16_imm16_for(wasm_op, make_instr, IndexType::Memory64, offset, value); + }) } fn test_store_wrap_offset16_imm_for( wasm_op: WasmOp, + make_instr: fn(ptr: Reg, offset: Offset16, value: Field) -> Instruction, + index_ty: IndexType, offset: u16, value: Src, - make_instr: fn(ptr: Reg, offset: u16, value: Field) -> Instruction, ) where Src: Copy + Wrap, Field: TryFrom, DisplayWasm: Display, { let param_ty = wasm_op.param_ty(); + let index_ty = index_ty.wat(); let display_value = DisplayWasm::from(value); let wasm = format!( r#" (module - (memory 1) - (func (param $ptr i32) + (memory {index_ty} 1) + (func (param $ptr {index_ty}) local.get $ptr {param_ty}.const {display_value} {wasm_op} offset={offset} @@ -207,58 +257,82 @@ fn test_store_wrap_offset16_imm_for( ); let value = Field::try_from(value.wrap()).ok().unwrap(); TranslationTest::new(&wasm) - .expect_func_instrs([make_instr(Reg::from(0), offset, value), Instruction::Return]) + .expect_func_instrs([ + make_instr(Reg::from(0), offset16(offset), value), + Instruction::Return, + ]) .run(); } fn test_store_wrap_offset16_imm( wasm_op: WasmOp, value: Src, - make_instr: fn(ptr: Reg, offset: u16, value: Field) -> Instruction, + make_instr: fn(ptr: Reg, offset: Offset16, value: Field) -> Instruction, ) where Src: Copy + Wrap, Field: TryFrom, DisplayWasm: Display, { - let offsets = [0, u16::MAX - 1, u16::MAX]; - for offset in offsets { - test_store_wrap_offset16_imm_for(wasm_op, offset, value, make_instr); - } + [0, 1, u16::MAX - 1, u16::MAX] + .into_iter() + .for_each(|offset| { + test_store_wrap_offset16_imm_for( + wasm_op, + make_instr, + IndexType::Memory32, + offset, + value, + ); + test_store_wrap_offset16_imm_for( + wasm_op, + make_instr, + IndexType::Memory64, + offset, + value, + ); + }) } fn test_store_wrap_imm_for( wasm_op: WasmOp, - offset: u32, + make_instr: fn(ptr: Reg, offset_lo: Offset64Lo) -> Instruction, + index_ty: IndexType, + memory_index: MemIdx, + offset: u64, value: Src, - make_instr: fn(ptr: Reg, memory: Memory) -> Instruction, ) where Src: Copy + Into + Wrap, Field: TryFrom + Into, DisplayWasm: Display, { assert!( - u16::try_from(offset).is_err(), - "this test requires non-16 bit offsets but found {offset}" + u16::try_from(offset).is_err() || !memory_index.is_default(), + "this test requires non-16 bit offsets or non-default memory \ + but found: offset={offset}, memory={memory_index}" ); let param_ty = wasm_op.param_ty(); + let index_ty = index_ty.wat(); let display_value = DisplayWasm::from(value); let wasm = format!( r#" (module - (memory 1) - (func (param $ptr i32) + (memory $mem0 {index_ty} 1) + (memory $mem1 {index_ty} 1) + (func (param $ptr {index_ty}) local.get $ptr {param_ty}.const {display_value} - {wasm_op} offset={offset} + {wasm_op} {memory_index} offset={offset} ) ) "# ); + let (offset_hi, offset_lo) = Offset64::split(offset); let value = Field::try_from(value.wrap()).ok().unwrap(); TranslationTest::new(&wasm) - .expect_func_instrs([ - make_instr(Reg::from(0), Memory::from(0)), - Instruction::imm16_and_imm32(value, offset), + .expect_func_instrs(iter_filter_opts![ + make_instr(Reg::from(0), offset_lo), + Instruction::imm16_and_offset_hi(value, offset_hi), + memory_index.instr(), Instruction::Return, ]) .run(); @@ -266,51 +340,80 @@ fn test_store_wrap_imm_for( fn test_store_wrap_imm( wasm_op: WasmOp, + make_instr: fn(ptr: Reg, offset_lo: Offset64Lo) -> Instruction, value: Src, - make_instr: fn(ptr: Reg, memory: Memory) -> Instruction, ) where Src: Copy + Into + Wrap, Field: TryFrom + Into, DisplayWasm: Display, { - let offsets = [u32::from(u16::MAX) + 1, u32::MAX - 1, u32::MAX]; - for offset in offsets { - test_store_wrap_imm_for::(wasm_op, offset, value, make_instr); + for offset in [ + u64::from(u16::MAX) + 1, + u64::from(u32::MAX) - 1, + u64::from(u32::MAX), + ] { + for mem_idx in [0, 1].map(MemIdx) { + for index_ty in [IndexType::Memory32, IndexType::Memory64] { + test_store_wrap_imm_for::( + wasm_op, make_instr, index_ty, mem_idx, offset, value, + ); + } + } + } + for offset in [u64::from(u32::MAX) + 1, u64::MAX - 1, u64::MAX] { + for mem_idx in [0, 1].map(MemIdx) { + test_store_wrap_imm_for::( + wasm_op, + make_instr, + IndexType::Memory64, + mem_idx, + offset, + value, + ); + } } } fn test_store_imm_for( wasm_op: WasmOp, - offset: u32, + make_instr: fn(ptr: Reg, offset_lo: Offset64Lo) -> Instruction, + index_ty: IndexType, + memory_index: MemIdx, + offset: impl Into, value: T, - make_instr: fn(ptr: Reg, memory: Memory) -> Instruction, ) where T: Copy + Into, DisplayWasm: Display, { + let offset = offset.into(); assert!( - u16::try_from(offset).is_err(), - "this test requires non-16 bit offsets but found {offset}" + u16::try_from(offset).is_err() || !memory_index.is_default(), + "this test requires non-16 bit offsets or non-default memory \ + but found: offset={offset}, memory={memory_index}" ); let param_ty = wasm_op.param_ty(); + let index_ty = index_ty.wat(); let display_value = DisplayWasm::from(value); let wasm = format!( r#" (module - (memory 1) - (func (param $ptr i32) + (memory $mem0 {index_ty} 1) + (memory $mem1 {index_ty} 1) + (func (param $ptr {index_ty}) local.get $ptr {param_ty}.const {display_value} - {wasm_op} offset={offset} + {wasm_op} {memory_index} offset={offset} ) ) "# ); + let (offset_hi, offset_lo) = Offset64::split(offset); TranslationTest::new(&wasm) .expect_func( - ExpectedFunc::new([ - make_instr(Reg::from(0), Memory::from(0)), - Instruction::register_and_imm32(Reg::from(-1), offset), + ExpectedFunc::new(iter_filter_opts![ + make_instr(Reg::from(0), offset_lo), + Instruction::register_and_offset_hi(Reg::from(-1), offset_hi), + memory_index.instr(), Instruction::Return, ]) .consts([value]), @@ -320,50 +423,73 @@ fn test_store_imm_for( fn test_store_imm( wasm_op: WasmOp, + make_instr: fn(ptr: Reg, offset_lo: Offset64Lo) -> Instruction, value: T, - make_instr: fn(ptr: Reg, memory: Memory) -> Instruction, ) where T: Copy + Into, DisplayWasm: Display, { - let offsets = [u32::from(u16::MAX) + 1, u32::MAX - 1, u32::MAX]; - for offset in offsets { - test_store_imm_for::(wasm_op, offset, value, make_instr); + for offset in [u32::from(u16::MAX) + 1, u32::MAX - 1, u32::MAX] { + for mem_idx in [0, 1].map(MemIdx) { + for index_ty in [IndexType::Memory32, IndexType::Memory64] { + test_store_imm_for::(wasm_op, make_instr, index_ty, mem_idx, offset, value); + } + } + } + for offset in [u64::from(u32::MAX) + 1, u64::MAX - 1, u64::MAX] { + for mem_idx in [0, 1].map(MemIdx) { + test_store_imm_for::( + wasm_op, + make_instr, + IndexType::Memory64, + mem_idx, + offset, + value, + ); + } } } fn test_store_imm16_for( wasm_op: WasmOp, - make_instr: fn(ptr: Reg, memory: Memory) -> Instruction, + make_instr: fn(ptr: Reg, offset_lo: Offset64Lo) -> Instruction, + index_ty: IndexType, + memory_index: MemIdx, value: T, - offset: u32, + offset: impl Into, ) where T: Copy + TryInto, DisplayWasm: Display, { + let offset = offset.into(); assert!( - u16::try_from(offset).is_err(), - "this test requires non-16 bit offsets but found {offset}" + u16::try_from(offset).is_err() || !memory_index.is_default(), + "this test requires non-16 bit offsets or non-default memory \ + but found: offset={offset}, memory={memory_index}" ); let param_ty = wasm_op.param_ty(); + let index_ty = index_ty.wat(); let display_value = DisplayWasm::from(value); let wasm = format!( r#" (module - (memory 1) - (func (param $ptr i32) + (memory $mem0 {index_ty} 1) + (memory $mem1 {index_ty} 1) + (func (param $ptr {index_ty}) local.get $ptr {param_ty}.const {display_value} - {wasm_op} offset={offset} + {wasm_op} {memory_index} offset={offset} ) ) "# ); + let (offset_hi, offset_lo) = Offset64::split(offset); let value = value.try_into().ok().unwrap(); TranslationTest::new(&wasm) - .expect_func_instrs([ - make_instr(Reg::from(0), Memory::from(0)), - Instruction::imm16_and_imm32(value, offset), + .expect_func_instrs(iter_filter_opts![ + make_instr(Reg::from(0), offset_lo), + Instruction::imm16_and_offset_hi(value, offset_hi), + memory_index.instr(), Instruction::Return, ]) .run(); @@ -371,71 +497,111 @@ fn test_store_imm16_for( fn test_store_imm16( wasm_op: WasmOp, - make_instr: fn(ptr: Reg, memory: Memory) -> Instruction, + make_instr: fn(ptr: Reg, offset_lo: Offset64Lo) -> Instruction, value: T, ) where T: Copy + TryInto, DisplayWasm: Display, { - test_store_imm16_for(wasm_op, make_instr, value, u32::from(u16::MAX) + 1); - test_store_imm16_for(wasm_op, make_instr, value, u32::MAX - 1); - test_store_imm16_for(wasm_op, make_instr, value, u32::MAX); + for offset in [ + u64::from(u16::MAX) + 1, + u64::from(u32::MAX) - 1, + u64::from(u32::MAX), + ] { + for mem_idx in [0, 1].map(MemIdx) { + for index_ty in [IndexType::Memory32, IndexType::Memory64] { + test_store_imm16_for(wasm_op, make_instr, index_ty, mem_idx, value, offset); + } + } + } + for offset in [u64::from(u32::MAX) + 1, u64::MAX - 1, u64::MAX] { + for mem_idx in [0, 1].map(MemIdx) { + test_store_imm16_for( + wasm_op, + make_instr, + IndexType::Memory64, + mem_idx, + value, + offset, + ); + } + } } fn test_store_at_for( wasm_op: WasmOp, + make_instr: fn(value: Reg, address: u32) -> Instruction, + index_ty: IndexType, + memory_index: MemIdx, ptr: u32, offset: u32, - make_instr: fn(value: Reg, address: u32) -> Instruction, ) { let address = ptr .checked_add(offset) .expect("testcase requires valid ptr+offset address"); let param_ty = wasm_op.param_ty(); + let index_ty = index_ty.wat(); let wasm = format!( r#" (module - (memory 1) + (memory $mem0 {index_ty} 1) + (memory $mem1 {index_ty} 1) (func (param $value {param_ty}) - i32.const {ptr} + {index_ty}.const {ptr} local.get $value - {wasm_op} offset={offset} + {wasm_op} {memory_index} offset={offset} ) ) "# ); TranslationTest::new(&wasm) - .expect_func_instrs([make_instr(Reg::from(0), address), Instruction::Return]) + .expect_func_instrs(iter_filter_opts![ + make_instr(Reg::from(0), address), + memory_index.instr(), + Instruction::Return, + ]) .run(); } fn test_store_at(wasm_op: WasmOp, make_instr: fn(value: Reg, address: u32) -> Instruction) { - test_store_at_for(wasm_op, 0, 0, make_instr); - test_store_at_for(wasm_op, 0, 1, make_instr); - test_store_at_for(wasm_op, 1, 0, make_instr); - test_store_at_for(wasm_op, 1, 1, make_instr); - test_store_at_for(wasm_op, 1000, 1000, make_instr); - test_store_at_for(wasm_op, 1, u32::MAX - 1, make_instr); - test_store_at_for(wasm_op, u32::MAX - 1, 1, make_instr); - test_store_at_for(wasm_op, 0, u32::MAX, make_instr); - test_store_at_for(wasm_op, u32::MAX, 0, make_instr); -} - -fn test_store_at_overflow_for(wasm_op: WasmOp, mem_idx: u32, ptr: u32, offset: u32) { - assert!( - ptr.checked_add(offset).is_none(), - "testcase expects overflowing ptr+offset address" - ); + for (ptr, offset) in [ + (0, 0), + (0, 1), + (1, 0), + (1, 1), + (1000, 1000), + (1, u32::MAX - 1), + (u32::MAX - 1, 1), + (0, u32::MAX), + (u32::MAX, 0), + ] { + for mem_idx in [0, 1].map(MemIdx) { + for index_ty in [IndexType::Memory32, IndexType::Memory64] { + test_store_at_for(wasm_op, make_instr, index_ty, mem_idx, ptr, offset); + } + } + } +} + +fn test_store_at_overflow_for( + wasm_op: WasmOp, + index_ty: IndexType, + memory_index: MemIdx, + ptr: u64, + offset: u64, +) { + assert_overflowing_ptr_offset(index_ty, ptr, offset); let param_ty = wasm_op.param_ty(); + let index_ty = index_ty.wat(); let wasm = format!( r#" (module - (memory $mem0 1) - (memory $mem1 1) + (memory $mem0 {index_ty} 1) + (memory $mem1 {index_ty} 1) (func (param $value {param_ty}) - i32.const {ptr} + {index_ty}.const {ptr} local.get $value - {wasm_op} $mem{mem_idx} offset={offset} + {wasm_op} {memory_index} offset={offset} ) ) "# @@ -446,19 +612,87 @@ fn test_store_at_overflow_for(wasm_op: WasmOp, mem_idx: u32, ptr: u32, offset: u } fn test_store_at_overflow(wasm_op: WasmOp) { - let ptrs_and_offsets = [(1, u32::MAX), (u32::MAX, 1), (u32::MAX, u32::MAX)]; - for (ptr, offset) in ptrs_and_offsets { - test_store_at_overflow_for(wasm_op, 0, ptr, offset); - test_store_at_overflow_for(wasm_op, 1, ptr, offset); - } + [ + (IndexType::Memory32, u64::from(u32::MAX), 1), + (IndexType::Memory32, 1, u64::from(u32::MAX)), + ( + IndexType::Memory32, + u64::from(u32::MAX), + u64::from(u32::MAX), + ), + (IndexType::Memory64, u64::MAX, 1), + (IndexType::Memory64, 1, u64::MAX), + (IndexType::Memory64, u64::MAX, u64::MAX), + ] + .into_iter() + .for_each(|(index_ty, ptr, offset)| { + test_store_at_overflow_for(wasm_op, index_ty, MemIdx(0), ptr, offset); + test_store_at_overflow_for(wasm_op, index_ty, MemIdx(1), ptr, offset); + }) +} + +fn test_store_at_fallback_for( + wasm_op: WasmOp, + make_instr: fn(result: Reg, offset_lo: Offset64Lo) -> Instruction, + memory_index: MemIdx, + ptr: u64, + offset: u64, +) { + let param_ty = wasm_op.param_ty(); + let wasm = format!( + r#" + (module + (memory $mem0 i64 1) + (memory $mem1 i64 1) + (func (param $value {param_ty}) + i64.const {ptr} + local.get $value + {wasm_op} {memory_index} offset={offset} + ) + ) + "# + ); + let address = ptr.checked_add(offset).unwrap(); + let (offset_hi, offset_lo) = Offset64::split(address); + TranslationTest::new(&wasm) + .expect_func( + ExpectedFunc::new(iter_filter_opts![ + make_instr(Reg::from(-1), offset_lo), + Instruction::register_and_offset_hi(Reg::from(0), offset_hi), + memory_index.instr(), + Instruction::Return, + ]) + .consts([0_u64]), + ) + .run(); +} + +fn test_store_at_fallback( + wasm_op: WasmOp, + make_instr: fn(result: Reg, offset_lo: Offset64Lo) -> Instruction, +) { + [ + (u64::from(u32::MAX), 1), + (1, u64::from(u32::MAX)), + (u64::from(u32::MAX), u64::from(u32::MAX)), + (u64::MAX - 1, 1), + (1, u64::MAX - 1), + ] + .into_iter() + .for_each(|(ptr, offset)| { + test_store_at_fallback_for(wasm_op, make_instr, MemIdx(0), ptr, offset); + test_store_at_fallback_for(wasm_op, make_instr, MemIdx(1), ptr, offset); + }) } fn test_store_at_imm_for( wasm_op: WasmOp, + make_instr: fn(value: Reg, address: u32) -> Instruction, + index_ty: IndexType, + memory_index: MemIdx, ptr: u32, offset: u32, value: T, - make_instr: fn(value: Reg, address: u32) -> Instruction, ) where T: Copy + Into, DisplayWasm: Display, @@ -468,22 +702,28 @@ fn test_store_at_imm_for( .expect("testcase requires valid ptr+offset address"); let display_value = DisplayWasm::from(value); let param_ty = wasm_op.param_ty(); + let index_ty = index_ty.wat(); let wasm = format!( r#" (module - (memory 1) + (memory $mem0 {index_ty} 1) + (memory $mem1 {index_ty} 1) (func - i32.const {ptr} + {index_ty}.const {ptr} {param_ty}.const {display_value} - {wasm_op} offset={offset} + {wasm_op} {memory_index} offset={offset} ) ) "# ); TranslationTest::new(&wasm) .expect_func( - ExpectedFunc::new([make_instr(Reg::from(-1), address), Instruction::Return]) - .consts([value]), + ExpectedFunc::new(iter_filter_opts![ + make_instr(Reg::from(-1), address), + memory_index.instr(), + Instruction::Return, + ]) + .consts([value]), ) .run(); } @@ -496,26 +736,37 @@ fn test_store_at_imm( T: Copy + Into, DisplayWasm: Display, { - test_store_at_imm_for(wasm_op, 0, 0, value, make_instr); - test_store_at_imm_for(wasm_op, 0, 1, value, make_instr); - test_store_at_imm_for(wasm_op, 1, 0, value, make_instr); - test_store_at_imm_for(wasm_op, 1, 1, value, make_instr); - test_store_at_imm_for(wasm_op, 1000, 1000, value, make_instr); - test_store_at_imm_for(wasm_op, 1, u32::MAX - 1, value, make_instr); - test_store_at_imm_for(wasm_op, u32::MAX - 1, 1, value, make_instr); - test_store_at_imm_for(wasm_op, 0, u32::MAX, value, make_instr); - test_store_at_imm_for(wasm_op, u32::MAX, 0, value, make_instr); + [ + (0, 0), + (0, 1), + (1, 0), + (1, 1), + (1000, 1000), + (1, u32::MAX - 1), + (u32::MAX - 1, 1), + (0, u32::MAX), + (u32::MAX, 0), + ] + .into_iter() + .for_each(|(ptr, offset)| { + for mem_idx in [0, 1].map(MemIdx) { + for index_ty in [IndexType::Memory32, IndexType::Memory64] { + test_store_at_imm_for(wasm_op, make_instr, index_ty, mem_idx, ptr, offset, value); + } + } + }) } fn test_store_wrap_at_imm_for( wasm_op: WasmOp, - mem_idx: u32, + make_instr: fn(value: Field, address: u32) -> Instruction, + index_ty: IndexType, + memory_index: MemIdx, ptr: u32, offset: u32, value: Src, - make_instr: fn(value: Field, address: u32) -> Instruction, ) where - Src: Copy + Into + Wrap, + Src: Copy + Wrap, Field: TryFrom + Into, DisplayWasm: Display, { @@ -524,32 +775,34 @@ fn test_store_wrap_at_imm_for( .expect("testcase requires valid ptr+offset address"); let display_value = DisplayWasm::from(value); let param_ty = wasm_op.param_ty(); + let index_ty = index_ty.wat(); let wasm = format!( r#" (module - (memory $mem0 1) - (memory $mem1 1) + (memory $mem0 {index_ty} 1) + (memory $mem1 {index_ty} 1) (func - i32.const {ptr} + {index_ty}.const {ptr} {param_ty}.const {display_value} - {wasm_op} $mem{mem_idx} offset={offset} + {wasm_op} {memory_index} offset={offset} ) ) "# ); let value = Field::try_from(value.wrap()).ok().unwrap(); - let mut instrs = vec![make_instr(value, address)]; - if mem_idx != 0 { - instrs.push(Instruction::memory_index(mem_idx)); - } - instrs.push(Instruction::Return); - TranslationTest::new(&wasm).expect_func_instrs(instrs).run(); + TranslationTest::new(&wasm) + .expect_func_instrs(iter_filter_opts![ + make_instr(value, address), + memory_index.instr(), + Instruction::Return, + ]) + .run(); } fn test_store_wrap_at_imm( wasm_op: WasmOp, - value: Src, make_instr: fn(value: Field, address: u32) -> Instruction, + value: Src, ) where Src: Copy + Into + Wrap, Field: TryFrom + Into, @@ -567,17 +820,23 @@ fn test_store_wrap_at_imm( (u32::MAX, 0), ]; for (ptr, offset) in ptrs_and_offsets { - test_store_wrap_at_imm_for::( - wasm_op, 0, ptr, offset, value, make_instr, - ); - test_store_wrap_at_imm_for::( - wasm_op, 1, ptr, offset, value, make_instr, - ); + for mem_idx in [0, 1].map(MemIdx) { + for index_ty in [IndexType::Memory32, IndexType::Memory64] { + test_store_wrap_at_imm_for::( + wasm_op, make_instr, index_ty, mem_idx, ptr, offset, value, + ); + } + } } } -fn test_store_at_imm_overflow_for(wasm_op: WasmOp, mem_idx: u8, ptr: u32, offset: u32, value: T) -where +fn test_store_at_imm_overflow_for( + wasm_op: WasmOp, + memory_index: MemIdx, + ptr: u32, + offset: u32, + value: T, +) where T: Copy, DisplayWasm: Display, { @@ -595,7 +854,7 @@ where (func i32.const {ptr} {param_ty}.const {display_value} - {wasm_op} $mem{mem_idx} offset={offset} + {wasm_op} {memory_index} offset={offset} ) ) "# @@ -612,7 +871,174 @@ where { let ptrs_and_offsets = [(1, u32::MAX), (u32::MAX, 1), (u32::MAX, u32::MAX)]; for (ptr, offset) in ptrs_and_offsets { - test_store_at_imm_overflow_for(wasm_op, 0, ptr, offset, value); - test_store_at_imm_overflow_for(wasm_op, 1, ptr, offset, value); + test_store_at_imm_overflow_for(wasm_op, MemIdx(0), ptr, offset, value); + test_store_at_imm_overflow_for(wasm_op, MemIdx(1), ptr, offset, value); + } +} + +fn test_store_at_imm16_fallback_for( + wasm_op: WasmOp, + make_instr: fn(ptr: Reg, offset_lo: Offset64Lo) -> Instruction, + memory_index: MemIdx, + ptr: u64, + offset: u64, + value: T, +) where + T: Copy + Wrap, + Wrapped: TryInto, + DisplayWasm: Display, +{ + assert!( + u32::try_from(ptr.saturating_add(offset)).is_err(), + "testcase expects overflowing 32-bit ptr+offset address" + ); + let display_value = DisplayWasm::from(value); + let param_ty = wasm_op.param_ty(); + let wasm = format!( + r#" + (module + (memory $mem0 i64 1) + (memory $mem1 i64 1) + (func + i64.const {ptr} + {param_ty}.const {display_value} + {wasm_op} {memory_index} offset={offset} + ) + ) + "# + ); + let address = ptr.checked_add(offset).unwrap(); + let (offset_hi, offset_lo) = Offset64::split(address); + let value = value.wrap().try_into().ok().unwrap(); + TranslationTest::new(&wasm) + .expect_func( + ExpectedFunc::new(iter_filter_opts![ + make_instr(Reg::from(-1), offset_lo), + Instruction::imm16_and_offset_hi(value, offset_hi), + memory_index.instr(), + Instruction::Return, + ]) + .consts([0_u64]), + ) + .run(); +} + +fn test_store_wrap_at_imm16_fallback( + wasm_op: WasmOp, + make_instr: fn(ptr: Reg, offset_lo: Offset64Lo) -> Instruction, + value: T, +) where + T: Copy + Wrap, + Wrapped: TryInto, + DisplayWasm: Display, +{ + let ptrs_and_offsets = [ + (1, u64::from(u32::MAX)), + (u64::from(u32::MAX), 1), + (u64::from(u32::MAX), u64::from(u32::MAX)), + (0, u64::MAX), + (u64::MAX, 0), + (1, u64::MAX - 1), + (u64::MAX - 1, 1), + ]; + for (ptr, offset) in ptrs_and_offsets { + for mem_idx in [0, 1].map(MemIdx) { + test_store_at_imm16_fallback_for::( + wasm_op, make_instr, mem_idx, ptr, offset, value, + ); + } + } +} + +fn test_store_at_imm16_fallback( + wasm_op: WasmOp, + make_instr: fn(ptr: Reg, offset_lo: Offset64Lo) -> Instruction, + value: T, +) where + T: Copy + TryInto, + DisplayWasm: Display, +{ + test_store_wrap_at_imm16_fallback::(wasm_op, make_instr, value) +} + +fn test_store_at_imm_fallback_for( + wasm_op: WasmOp, + make_instr: fn(ptr: Reg, offset_lo: Offset64Lo) -> Instruction, + memory_index: MemIdx, + ptr: u64, + offset: u64, + value: T, +) where + T: Copy + Into, + DisplayWasm: Display, +{ + assert!( + u32::try_from(ptr.saturating_add(offset)).is_err(), + "testcase expects overflowing 32-bit ptr+offset address" + ); + let display_value = DisplayWasm::from(value); + let param_ty = wasm_op.param_ty(); + let wasm = format!( + r#" + (module + (memory $mem0 i64 1) + (memory $mem1 i64 1) + (func + i64.const {ptr} + {param_ty}.const {display_value} + {wasm_op} {memory_index} offset={offset} + ) + ) + "# + ); + let address = ptr.checked_add(offset).unwrap(); + let (offset_hi, offset_lo) = Offset64::split(address); + let (value_reg, value_const) = match value.into() == 0_u64.into() { + true => { + // Case: since this scheme always allocates a 0 as function constant value + // and address is zero the translator only uses a single register to + // represent both. (special case) + (Reg::from(-1), None) + } + false => { + // Case: address is non-zero so the translator uses 2 different registers + // to represent the zero'ed ptr value and the value. (common case) + (Reg::from(-2), Some(value.into())) + } + }; + TranslationTest::new(&wasm) + .expect_func( + ExpectedFunc::new(iter_filter_opts![ + make_instr(Reg::from(-1), offset_lo), + Instruction::register_and_offset_hi(value_reg, offset_hi), + memory_index.instr(), + Instruction::Return, + ]) + .consts(iter_filter_opts![UntypedVal::from(0_u64), value_const]), + ) + .run(); +} + +fn test_store_at_imm_fallback( + wasm_op: WasmOp, + make_instr: fn(ptr: Reg, offset_lo: Offset64Lo) -> Instruction, + value: T, +) where + T: Copy + Into, + DisplayWasm: Display, +{ + let ptrs_and_offsets = [ + (1, u64::from(u32::MAX)), + (u64::from(u32::MAX), 1), + (u64::from(u32::MAX), u64::from(u32::MAX)), + (0, u64::MAX), + (u64::MAX, 0), + (1, u64::MAX - 1), + (u64::MAX - 1, 1), + ]; + for (ptr, offset) in ptrs_and_offsets { + for mem_idx in [0, 1].map(MemIdx) { + test_store_at_imm_fallback_for::(wasm_op, make_instr, mem_idx, ptr, offset, value); + } } } diff --git a/crates/wasmi/src/engine/translator/tests/op/table/table_copy.rs b/crates/wasmi/src/engine/translator/tests/op/table/table_copy.rs index d6ddc2e1ec..9dbeeb3b04 100644 --- a/crates/wasmi/src/engine/translator/tests/op/table/table_copy.rs +++ b/crates/wasmi/src/engine/translator/tests/op/table/table_copy.rs @@ -33,7 +33,7 @@ fn copy() { test_copy(ValType::ExternRef); } -fn testcase_copy_exact(ty: ValType, len: u32) -> TranslationTest { +fn testcase_copy_exact(ty: ValType, len: u64) -> TranslationTest { let display_ty = DisplayValueType::from(ty); let wasm = format!( r" @@ -51,10 +51,10 @@ fn testcase_copy_exact(ty: ValType, len: u32) -> TranslationTest { TranslationTest::new(&wasm) } -fn test_copy_exact16(ty: ValType, len: u32) { +fn test_copy_exact16(ty: ValType, len: u64) { testcase_copy_exact(ty, len) .expect_func_instrs([ - Instruction::table_copy_exact(Reg::from(0), Reg::from(1), u32imm16(len)), + Instruction::table_copy_exact(Reg::from(0), Reg::from(1), u64imm16(len)), Instruction::table_index(0), Instruction::table_index(1), Instruction::Return, @@ -65,17 +65,17 @@ fn test_copy_exact16(ty: ValType, len: u32) { #[test] #[cfg_attr(miri, ignore)] fn copy_exact16() { - fn test_for(len: u32) { + fn test_for(len: u64) { test_copy_exact16(ValType::FuncRef, len); test_copy_exact16(ValType::ExternRef, len); } test_for(0); test_for(1); test_for(42); - test_for(u32::from(u16::MAX)); + test_for(u64::from(u16::MAX)); } -fn test_copy_exact(ty: ValType, len: u32) { +fn test_copy_exact(ty: ValType, len: u64) { testcase_copy_exact(ty, len) .expect_func( ExpectedFunc::new([ @@ -92,15 +92,15 @@ fn test_copy_exact(ty: ValType, len: u32) { #[test] #[cfg_attr(miri, ignore)] fn copy_exact() { - fn test_for(len: u32) { + fn test_for(len: u64) { test_copy_exact(ValType::FuncRef, len); test_copy_exact(ValType::ExternRef, len); } - test_for(u32::from(u16::MAX) + 1); - test_for(u32::MAX); + test_for(u64::from(u16::MAX) + 1); + test_for(u64::from(u32::MAX)); } -fn testcase_copy_from(ty: ValType, src: u32) -> TranslationTest { +fn testcase_copy_from(ty: ValType, src: u64) -> TranslationTest { let display_ty = DisplayValueType::from(ty); let wasm = format!( r" @@ -118,10 +118,10 @@ fn testcase_copy_from(ty: ValType, src: u32) -> TranslationTest { TranslationTest::new(&wasm) } -fn test_copy_from16(ty: ValType, src: u32) { +fn test_copy_from16(ty: ValType, src: u64) { testcase_copy_from(ty, src) .expect_func_instrs([ - Instruction::table_copy_from(Reg::from(0), u32imm16(src), Reg::from(1)), + Instruction::table_copy_from(Reg::from(0), u64imm16(src), Reg::from(1)), Instruction::table_index(0), Instruction::table_index(1), Instruction::Return, @@ -132,15 +132,15 @@ fn test_copy_from16(ty: ValType, src: u32) { #[test] #[cfg_attr(miri, ignore)] fn copy_from16() { - fn test_for(src: u32) { + fn test_for(src: u64) { test_copy_from16(ValType::FuncRef, src); test_copy_from16(ValType::ExternRef, src); } test_for(0); - test_for(u32::from(u16::MAX)); + test_for(u64::from(u16::MAX)); } -fn test_copy_from(ty: ValType, src: u32) { +fn test_copy_from(ty: ValType, src: u64) { testcase_copy_from(ty, src) .expect_func( ExpectedFunc::new([ @@ -157,15 +157,15 @@ fn test_copy_from(ty: ValType, src: u32) { #[test] #[cfg_attr(miri, ignore)] fn copy_from() { - fn test_for(src: u32) { + fn test_for(src: u64) { test_copy_from(ValType::FuncRef, src); test_copy_from(ValType::ExternRef, src); } - test_for(u32::from(u16::MAX) + 1); - test_for(u32::MAX); + test_for(u64::from(u16::MAX) + 1); + test_for(u64::from(u32::MAX)); } -fn testcase_copy_to(ty: ValType, dst: u32) -> TranslationTest { +fn testcase_copy_to(ty: ValType, dst: u64) -> TranslationTest { let display_ty = DisplayValueType::from(ty); let wasm = format!( r" @@ -183,10 +183,10 @@ fn testcase_copy_to(ty: ValType, dst: u32) -> TranslationTest { TranslationTest::new(&wasm) } -fn test_copy_to16(ty: ValType, dst: u32) { +fn test_copy_to16(ty: ValType, dst: u64) { testcase_copy_to(ty, dst) .expect_func_instrs([ - Instruction::table_copy_to(u32imm16(dst), Reg::from(0), Reg::from(1)), + Instruction::table_copy_to(u64imm16(dst), Reg::from(0), Reg::from(1)), Instruction::table_index(0), Instruction::table_index(1), Instruction::Return, @@ -197,15 +197,15 @@ fn test_copy_to16(ty: ValType, dst: u32) { #[test] #[cfg_attr(miri, ignore)] fn copy_to16() { - fn test_for(dst: u32) { + fn test_for(dst: u64) { test_copy_to16(ValType::FuncRef, dst); test_copy_to16(ValType::ExternRef, dst); } test_for(0); - test_for(u32::from(u16::MAX)); + test_for(u64::from(u16::MAX)); } -fn test_copy_to(ty: ValType, dst: u32) { +fn test_copy_to(ty: ValType, dst: u64) { testcase_copy_to(ty, dst) .expect_func( ExpectedFunc::new([ @@ -222,15 +222,15 @@ fn test_copy_to(ty: ValType, dst: u32) { #[test] #[cfg_attr(miri, ignore)] fn copy_to() { - fn test_for(dst: u32) { + fn test_for(dst: u64) { test_copy_to(ValType::FuncRef, dst); test_copy_to(ValType::ExternRef, dst); } - test_for(u32::from(u16::MAX) + 1); - test_for(u32::MAX); + test_for(u64::from(u16::MAX) + 1); + test_for(u64::from(u32::MAX)); } -fn testcase_copy_from_to(ty: ValType, dst: u32, src: u32) -> TranslationTest { +fn testcase_copy_from_to(ty: ValType, dst: u64, src: u64) -> TranslationTest { let display_ty = DisplayValueType::from(ty); let wasm = format!( r" @@ -248,10 +248,10 @@ fn testcase_copy_from_to(ty: ValType, dst: u32, src: u32) -> TranslationTest { TranslationTest::new(&wasm) } -fn test_copy_from_to16(ty: ValType, dst: u32, src: u32) { +fn test_copy_from_to16(ty: ValType, dst: u64, src: u64) { testcase_copy_from_to(ty, dst, src) .expect_func_instrs([ - Instruction::table_copy_from_to(u32imm16(dst), u32imm16(src), Reg::from(0)), + Instruction::table_copy_from_to(u64imm16(dst), u64imm16(src), Reg::from(0)), Instruction::table_index(0), Instruction::table_index(1), Instruction::Return, @@ -262,11 +262,11 @@ fn test_copy_from_to16(ty: ValType, dst: u32, src: u32) { #[test] #[cfg_attr(miri, ignore)] fn copy_from_to16() { - fn test_for(dst: u32, src: u32) { + fn test_for(dst: u64, src: u64) { test_copy_from_to16(ValType::FuncRef, dst, src); test_copy_from_to16(ValType::ExternRef, dst, src); } - let values = [0, 1, u32::from(u16::MAX) - 1, u32::from(u16::MAX)]; + let values = [0, 1, u64::from(u16::MAX) - 1, u64::from(u16::MAX)]; for dst in values { for src in values { test_for(dst, src); @@ -274,7 +274,7 @@ fn copy_from_to16() { } } -fn test_copy_from_to(ty: ValType, dst: u32, src: u32) { +fn test_copy_from_to(ty: ValType, dst: u64, src: u64) { testcase_copy_from_to(ty, dst, src) .expect_func( ExpectedFunc::new([ @@ -291,11 +291,15 @@ fn test_copy_from_to(ty: ValType, dst: u32, src: u32) { #[test] #[cfg_attr(miri, ignore)] fn copy_from_to() { - fn test_for(dst: u32, src: u32) { + fn test_for(dst: u64, src: u64) { test_copy_from_to(ValType::FuncRef, dst, src); test_copy_from_to(ValType::ExternRef, dst, src); } - let values = [u32::from(u16::MAX) + 1, u32::MAX - 1, u32::MAX]; + let values = [ + u64::from(u16::MAX) + 1, + u64::from(u32::MAX) - 1, + u64::from(u32::MAX), + ]; for dst in values { for src in values { if dst == src { @@ -310,7 +314,7 @@ fn copy_from_to() { } } -fn testcase_copy_to_exact(ty: ValType, dst: u32, len: u32) -> TranslationTest { +fn testcase_copy_to_exact(ty: ValType, dst: u64, len: u64) -> TranslationTest { let display_ty = DisplayValueType::from(ty); let wasm = format!( r" @@ -328,10 +332,10 @@ fn testcase_copy_to_exact(ty: ValType, dst: u32, len: u32) -> TranslationTest { TranslationTest::new(&wasm) } -fn test_copy_to_exact16(ty: ValType, dst: u32, len: u32) { +fn test_copy_to_exact16(ty: ValType, dst: u64, len: u64) { testcase_copy_to_exact(ty, dst, len) .expect_func_instrs([ - Instruction::table_copy_to_exact(u32imm16(dst), Reg::from(0), u32imm16(len)), + Instruction::table_copy_to_exact(u64imm16(dst), Reg::from(0), u64imm16(len)), Instruction::table_index(0), Instruction::table_index(1), Instruction::Return, @@ -342,11 +346,11 @@ fn test_copy_to_exact16(ty: ValType, dst: u32, len: u32) { #[test] #[cfg_attr(miri, ignore)] fn copy_to_exact16() { - fn test_for(dst: u32, len: u32) { + fn test_for(dst: u64, len: u64) { test_copy_to_exact16(ValType::FuncRef, dst, len); test_copy_to_exact16(ValType::ExternRef, dst, len); } - let values = [0, 1, u32::from(u16::MAX) - 1, u32::from(u16::MAX)]; + let values = [0, 1, u64::from(u16::MAX) - 1, u64::from(u16::MAX)]; for dst in values { for len in values { test_for(dst, len); @@ -354,7 +358,7 @@ fn copy_to_exact16() { } } -fn test_copy_to_exact(ty: ValType, dst: u32, len: u32) { +fn test_copy_to_exact(ty: ValType, dst: u64, len: u64) { testcase_copy_to_exact(ty, dst, len) .expect_func( ExpectedFunc::new([ @@ -371,11 +375,15 @@ fn test_copy_to_exact(ty: ValType, dst: u32, len: u32) { #[test] #[cfg_attr(miri, ignore)] fn copy_to_exact() { - fn test_for(dst: u32, len: u32) { + fn test_for(dst: u64, len: u64) { test_copy_to_exact(ValType::FuncRef, dst, len); test_copy_to_exact(ValType::ExternRef, dst, len); } - let values = [u32::from(u16::MAX) + 1, u32::MAX - 1, u32::MAX]; + let values = [ + u64::from(u16::MAX) + 1, + u64::from(u32::MAX) - 1, + u64::from(u32::MAX), + ]; for dst in values { for src in values { if dst == src { @@ -390,7 +398,7 @@ fn copy_to_exact() { } } -fn testcase_copy_from_exact(ty: ValType, src: u32, len: u32) -> TranslationTest { +fn testcase_copy_from_exact(ty: ValType, src: u64, len: u64) -> TranslationTest { let display_ty = DisplayValueType::from(ty); let wasm = format!( r" @@ -408,10 +416,10 @@ fn testcase_copy_from_exact(ty: ValType, src: u32, len: u32) -> TranslationTest TranslationTest::new(&wasm) } -fn test_copy_from_exact16(ty: ValType, src: u32, len: u32) { +fn test_copy_from_exact16(ty: ValType, src: u64, len: u64) { testcase_copy_from_exact(ty, src, len) .expect_func_instrs([ - Instruction::table_copy_from_exact(Reg::from(0), u32imm16(src), u32imm16(len)), + Instruction::table_copy_from_exact(Reg::from(0), u64imm16(src), u64imm16(len)), Instruction::table_index(0), Instruction::table_index(1), Instruction::Return, @@ -422,11 +430,11 @@ fn test_copy_from_exact16(ty: ValType, src: u32, len: u32) { #[test] #[cfg_attr(miri, ignore)] fn copy_from_exact16() { - fn test_for(dst: u32, len: u32) { + fn test_for(dst: u64, len: u64) { test_copy_from_exact16(ValType::FuncRef, dst, len); test_copy_from_exact16(ValType::ExternRef, dst, len); } - let values = [0, 1, u32::from(u16::MAX) - 1, u32::from(u16::MAX)]; + let values = [0, 1, u64::from(u16::MAX) - 1, u64::from(u16::MAX)]; for dst in values { for len in values { test_for(dst, len); @@ -434,7 +442,7 @@ fn copy_from_exact16() { } } -fn test_copy_from_exact(ty: ValType, src: u32, len: u32) { +fn test_copy_from_exact(ty: ValType, src: u64, len: u64) { testcase_copy_from_exact(ty, src, len) .expect_func( ExpectedFunc::new([ @@ -451,11 +459,15 @@ fn test_copy_from_exact(ty: ValType, src: u32, len: u32) { #[test] #[cfg_attr(miri, ignore)] fn copy_from_exact() { - fn test_for(src: u32, len: u32) { + fn test_for(src: u64, len: u64) { test_copy_from_exact(ValType::FuncRef, src, len); test_copy_from_exact(ValType::ExternRef, src, len); } - let values = [u32::from(u16::MAX) + 1, u32::MAX - 1, u32::MAX]; + let values = [ + u64::from(u16::MAX) + 1, + u64::from(u32::MAX) - 1, + u64::from(u32::MAX), + ]; for dst in values { for src in values { if dst == src { @@ -470,7 +482,7 @@ fn copy_from_exact() { } } -fn testcase_copy_from_to_exact(ty: ValType, dst: u32, src: u32, len: u32) -> TranslationTest { +fn testcase_copy_from_to_exact(ty: ValType, dst: u64, src: u64, len: u64) -> TranslationTest { let display_ty = DisplayValueType::from(ty); let wasm = format!( r" @@ -488,10 +500,10 @@ fn testcase_copy_from_to_exact(ty: ValType, dst: u32, src: u32, len: u32) -> Tra TranslationTest::new(&wasm) } -fn test_copy_from_to_exact16(ty: ValType, dst: u32, src: u32, len: u32) { +fn test_copy_from_to_exact16(ty: ValType, dst: u64, src: u64, len: u64) { testcase_copy_from_to_exact(ty, dst, src, len) .expect_func_instrs([ - Instruction::table_copy_from_to_exact(u32imm16(dst), u32imm16(src), u32imm16(len)), + Instruction::table_copy_from_to_exact(u64imm16(dst), u64imm16(src), u64imm16(len)), Instruction::table_index(0), Instruction::table_index(1), Instruction::Return, @@ -502,11 +514,11 @@ fn test_copy_from_to_exact16(ty: ValType, dst: u32, src: u32, len: u32) { #[test] #[cfg_attr(miri, ignore)] fn copy_from_to_exact16() { - fn test_for(dst: u32, src: u32, len: u32) { + fn test_for(dst: u64, src: u64, len: u64) { test_copy_from_to_exact16(ValType::FuncRef, dst, src, len); test_copy_from_to_exact16(ValType::ExternRef, dst, src, len); } - let values = [0, 1, u32::from(u16::MAX) - 1, u32::from(u16::MAX)]; + let values = [0, 1, u64::from(u16::MAX) - 1, u64::from(u16::MAX)]; for dst in values { for src in values { for len in values { @@ -516,7 +528,7 @@ fn copy_from_to_exact16() { } } -fn test_copy_from_to_exact(ty: ValType, dst: u32, src: u32, len: u32) { +fn test_copy_from_to_exact(ty: ValType, dst: u64, src: u64, len: u64) { testcase_copy_from_to_exact(ty, dst, src, len) .expect_func( ExpectedFunc::new([ @@ -533,11 +545,15 @@ fn test_copy_from_to_exact(ty: ValType, dst: u32, src: u32, len: u32) { #[test] #[cfg_attr(miri, ignore)] fn copy_from_to_exact() { - fn test_for(dst: u32, src: u32, len: u32) { + fn test_for(dst: u64, src: u64, len: u64) { test_copy_from_to_exact(ValType::FuncRef, dst, src, len); test_copy_from_to_exact(ValType::ExternRef, dst, src, len); } - let values = [u32::from(u16::MAX) + 1, u32::MAX - 1, u32::MAX]; + let values = [ + u64::from(u16::MAX) + 1, + u64::from(u32::MAX) - 1, + u64::from(u32::MAX), + ]; for dst in values { for src in values { for len in values { diff --git a/crates/wasmi/src/engine/translator/tests/op/table/table_fill.rs b/crates/wasmi/src/engine/translator/tests/op/table/table_fill.rs index b7279193fb..95767a6d9c 100644 --- a/crates/wasmi/src/engine/translator/tests/op/table/table_fill.rs +++ b/crates/wasmi/src/engine/translator/tests/op/table/table_fill.rs @@ -31,7 +31,7 @@ fn fill() { test_fill(ValType::ExternRef); } -fn testcase_fill_exact(ty: ValType, len: u32) -> TranslationTest { +fn testcase_fill_exact(ty: ValType, len: u64) -> TranslationTest { let display_ty = DisplayValueType::from(ty); let wasm = format!( r" @@ -48,10 +48,10 @@ fn testcase_fill_exact(ty: ValType, len: u32) -> TranslationTest { TranslationTest::new(&wasm) } -fn test_fill_exact16(ty: ValType, len: u32) { +fn test_fill_exact16(ty: ValType, len: u64) { testcase_fill_exact(ty, len) .expect_func_instrs([ - Instruction::table_fill_exact(Reg::from(0), u32imm16(len), Reg::from(1)), + Instruction::table_fill_exact(Reg::from(0), u64imm16(len), Reg::from(1)), Instruction::table_index(0), Instruction::Return, ]) @@ -61,17 +61,17 @@ fn test_fill_exact16(ty: ValType, len: u32) { #[test] #[cfg_attr(miri, ignore)] fn fill_exact16() { - fn test_for(len: u32) { + fn test_for(len: u64) { test_fill_exact16(ValType::FuncRef, len); test_fill_exact16(ValType::ExternRef, len); } test_for(0); test_for(1); test_for(42); - test_for(u32::from(u16::MAX)); + test_for(u64::from(u16::MAX)); } -fn test_fill_exact(ty: ValType, len: u32) { +fn test_fill_exact(ty: ValType, len: u64) { testcase_fill_exact(ty, len) .expect_func( ExpectedFunc::new([ @@ -87,15 +87,15 @@ fn test_fill_exact(ty: ValType, len: u32) { #[test] #[cfg_attr(miri, ignore)] fn fill_exact() { - fn test_for(len: u32) { + fn test_for(len: u64) { test_fill_exact(ValType::FuncRef, len); test_fill_exact(ValType::ExternRef, len); } - test_for(u32::from(u16::MAX) + 1); - test_for(u32::MAX); + test_for(u64::from(u16::MAX) + 1); + test_for(u64::from(u32::MAX)); } -fn testcase_fill_at(ty: ValType, dst: u32) -> TranslationTest { +fn testcase_fill_at(ty: ValType, dst: u64) -> TranslationTest { let display_ty = DisplayValueType::from(ty); let wasm = format!( r" @@ -112,10 +112,10 @@ fn testcase_fill_at(ty: ValType, dst: u32) -> TranslationTest { TranslationTest::new(&wasm) } -fn test_fill_at16(ty: ValType, dst: u32) { +fn test_fill_at16(ty: ValType, dst: u64) { testcase_fill_at(ty, dst) .expect_func_instrs([ - Instruction::table_fill_at(u32imm16(dst), Reg::from(1), Reg::from(0)), + Instruction::table_fill_at(u64imm16(dst), Reg::from(1), Reg::from(0)), Instruction::table_index(0), Instruction::Return, ]) @@ -125,15 +125,15 @@ fn test_fill_at16(ty: ValType, dst: u32) { #[test] #[cfg_attr(miri, ignore)] fn fill_at16() { - fn test_for(dst: u32) { + fn test_for(dst: u64) { test_fill_at16(ValType::FuncRef, dst); test_fill_at16(ValType::ExternRef, dst); } test_for(0); - test_for(u32::from(u16::MAX)); + test_for(u64::from(u16::MAX)); } -fn test_fill_at(ty: ValType, dst: u32) { +fn test_fill_at(ty: ValType, dst: u64) { testcase_fill_at(ty, dst) .expect_func( ExpectedFunc::new([ @@ -149,15 +149,15 @@ fn test_fill_at(ty: ValType, dst: u32) { #[test] #[cfg_attr(miri, ignore)] fn fill_at() { - fn test_for(dst: u32) { + fn test_for(dst: u64) { test_fill_at(ValType::FuncRef, dst); test_fill_at(ValType::ExternRef, dst); } - test_for(u32::from(u16::MAX) + 1); - test_for(u32::MAX); + test_for(u64::from(u16::MAX) + 1); + test_for(u64::from(u32::MAX)); } -fn testcase_fill_at_exact(ty: ValType, dst: u32, len: u32) -> TranslationTest { +fn testcase_fill_at_exact(ty: ValType, dst: u64, len: u64) -> TranslationTest { let display_ty = DisplayValueType::from(ty); let wasm = format!( r" @@ -174,10 +174,10 @@ fn testcase_fill_at_exact(ty: ValType, dst: u32, len: u32) -> TranslationTest { TranslationTest::new(&wasm) } -fn test_fill_at_exact16(ty: ValType, dst: u32, len: u32) { +fn test_fill_at_exact16(ty: ValType, dst: u64, len: u64) { testcase_fill_at_exact(ty, dst, len) .expect_func_instrs([ - Instruction::table_fill_at_exact(u32imm16(dst), u32imm16(len), Reg::from(0)), + Instruction::table_fill_at_exact(u64imm16(dst), u64imm16(len), Reg::from(0)), Instruction::table_index(0), Instruction::Return, ]) @@ -187,11 +187,11 @@ fn test_fill_at_exact16(ty: ValType, dst: u32, len: u32) { #[test] #[cfg_attr(miri, ignore)] fn fill_at_exact16() { - fn test_for(dst: u32, len: u32) { + fn test_for(dst: u64, len: u64) { test_fill_at_exact16(ValType::FuncRef, dst, len); test_fill_at_exact16(ValType::ExternRef, dst, len); } - let values = [0, 1, u32::from(u16::MAX) - 1, u32::from(u16::MAX)]; + let values = [0, 1, u64::from(u16::MAX) - 1, u64::from(u16::MAX)]; for dst in values { for len in values { test_for(dst, len); @@ -199,7 +199,7 @@ fn fill_at_exact16() { } } -fn test_fill_at_exact(ty: ValType, dst: u32, len: u32) { +fn test_fill_at_exact(ty: ValType, dst: u64, len: u64) { testcase_fill_at_exact(ty, dst, len) .expect_func( ExpectedFunc::new([ @@ -215,11 +215,15 @@ fn test_fill_at_exact(ty: ValType, dst: u32, len: u32) { #[test] #[cfg_attr(miri, ignore)] fn fill_at_exact() { - fn test_for(dst: u32, len: u32) { + fn test_for(dst: u64, len: u64) { test_fill_at_exact(ValType::FuncRef, dst, len); test_fill_at_exact(ValType::ExternRef, dst, len); } - let values = [u32::from(u16::MAX) + 1, u32::MAX - 1, u32::MAX]; + let values = [ + u64::from(u16::MAX) + 1, + u64::from(u32::MAX) - 1, + u64::from(u32::MAX), + ]; for dst in values { for len in values { if dst == len { diff --git a/crates/wasmi/src/engine/translator/tests/op/table/table_grow.rs b/crates/wasmi/src/engine/translator/tests/op/table/table_grow.rs index f127e0d10c..e333613217 100644 --- a/crates/wasmi/src/engine/translator/tests/op/table/table_grow.rs +++ b/crates/wasmi/src/engine/translator/tests/op/table/table_grow.rs @@ -30,7 +30,7 @@ fn reg() { test_reg(ValType::ExternRef); } -fn test_imm16(ty: ValType, delta: u32) { +fn test_imm16(ty: ValType, delta: u64) { let display_ty = DisplayValueType::from(ty); let wasm = format!( r" @@ -45,7 +45,7 @@ fn test_imm16(ty: ValType, delta: u32) { ); TranslationTest::new(&wasm) .expect_func_instrs([ - Instruction::table_grow_imm(Reg::from(1), u32imm16(delta), Reg::from(0)), + Instruction::table_grow_imm(Reg::from(1), u64imm16(delta), Reg::from(0)), Instruction::table_index(0), Instruction::return_reg(Reg::from(1)), ]) @@ -55,14 +55,14 @@ fn test_imm16(ty: ValType, delta: u32) { #[test] #[cfg_attr(miri, ignore)] fn imm16() { - fn test_for(delta: u32) { + fn test_for(delta: u64) { test_imm16(ValType::FuncRef, delta); test_imm16(ValType::ExternRef, delta); } test_for(1); test_for(42); - test_for(u32::from(u16::MAX) - 1); - test_for(u32::from(u16::MAX)); + test_for(u64::from(u16::MAX) - 1); + test_for(u64::from(u16::MAX)); } fn test_imm_zero(ty: ValType) { diff --git a/crates/wasmi/src/engine/translator/tests/op/table/table_init.rs b/crates/wasmi/src/engine/translator/tests/op/table/table_init.rs index a3b508e14a..41f16a2be2 100644 --- a/crates/wasmi/src/engine/translator/tests/op/table/table_init.rs +++ b/crates/wasmi/src/engine/translator/tests/op/table/table_init.rs @@ -1,5 +1,5 @@ use super::*; -use crate::core::ValType; +use crate::core::{UntypedVal, ValType}; fn test_init(ty: ValType) { let display_ty = DisplayValueType::from(ty); @@ -165,7 +165,7 @@ fn init_from() { test_for(u32::MAX); } -fn testcase_init_to(ty: ValType, dst: u32) -> TranslationTest { +fn testcase_init_to(ty: ValType, dst: u64) -> TranslationTest { let display_ty = DisplayValueType::from(ty); let wasm = format!( r" @@ -183,10 +183,10 @@ fn testcase_init_to(ty: ValType, dst: u32) -> TranslationTest { TranslationTest::new(&wasm) } -fn test_init_to16(ty: ValType, dst: u32) { +fn test_init_to16(ty: ValType, dst: u64) { testcase_init_to(ty, dst) .expect_func_instrs([ - Instruction::table_init_to(u32imm16(dst), Reg::from(0), Reg::from(1)), + Instruction::table_init_to(u64imm16(dst), Reg::from(0), Reg::from(1)), Instruction::table_index(0), Instruction::elem_index(0), Instruction::Return, @@ -197,15 +197,15 @@ fn test_init_to16(ty: ValType, dst: u32) { #[test] #[cfg_attr(miri, ignore)] fn init_to16() { - fn test_for(dst: u32) { + fn test_for(dst: u64) { test_init_to16(ValType::FuncRef, dst); test_init_to16(ValType::ExternRef, dst); } test_for(0); - test_for(u32::from(u16::MAX)); + test_for(u64::from(u16::MAX)); } -fn test_init_to(ty: ValType, dst: u32) { +fn test_init_to(ty: ValType, dst: u64) { testcase_init_to(ty, dst) .expect_func( ExpectedFunc::new([ @@ -222,15 +222,15 @@ fn test_init_to(ty: ValType, dst: u32) { #[test] #[cfg_attr(miri, ignore)] fn init_to() { - fn test_for(dst: u32) { + fn test_for(dst: u64) { test_init_to(ValType::FuncRef, dst); test_init_to(ValType::ExternRef, dst); } - test_for(u32::from(u16::MAX) + 1); - test_for(u32::MAX); + test_for(u64::from(u16::MAX) + 1); + test_for(u64::from(u32::MAX)); } -fn testcase_init_from_to(ty: ValType, dst: u32, src: u32) -> TranslationTest { +fn testcase_init_from_to(ty: ValType, dst: u64, src: u32) -> TranslationTest { let display_ty = DisplayValueType::from(ty); let wasm = format!( r" @@ -248,10 +248,10 @@ fn testcase_init_from_to(ty: ValType, dst: u32, src: u32) -> TranslationTest { TranslationTest::new(&wasm) } -fn test_init_from_to16(ty: ValType, dst: u32, src: u32) { +fn test_init_from_to16(ty: ValType, dst: u64, src: u32) { testcase_init_from_to(ty, dst, src) .expect_func_instrs([ - Instruction::table_init_from_to(u32imm16(dst), u32imm16(src), Reg::from(0)), + Instruction::table_init_from_to(u64imm16(dst), u32imm16(src), Reg::from(0)), Instruction::table_index(0), Instruction::elem_index(0), Instruction::Return, @@ -262,19 +262,19 @@ fn test_init_from_to16(ty: ValType, dst: u32, src: u32) { #[test] #[cfg_attr(miri, ignore)] fn init_from_to16() { - fn test_for(dst: u32, src: u32) { + fn test_for(dst: u64, src: u32) { test_init_from_to16(ValType::FuncRef, dst, src); test_init_from_to16(ValType::ExternRef, dst, src); } let values = [0, 1, u32::from(u16::MAX) - 1, u32::from(u16::MAX)]; for dst in values { for src in values { - test_for(dst, src); + test_for(u64::from(dst), src); } } } -fn test_init_from_to(ty: ValType, dst: u32, src: u32) { +fn test_init_from_to(ty: ValType, dst: u64, src: u32) { testcase_init_from_to(ty, dst, src) .expect_func( ExpectedFunc::new([ @@ -283,7 +283,7 @@ fn test_init_from_to(ty: ValType, dst: u32, src: u32) { Instruction::elem_index(0), Instruction::Return, ]) - .consts([dst, src]), + .consts([UntypedVal::from(dst), UntypedVal::from(src)]), ) .run() } @@ -291,7 +291,7 @@ fn test_init_from_to(ty: ValType, dst: u32, src: u32) { #[test] #[cfg_attr(miri, ignore)] fn init_from_to() { - fn test_for(dst: u32, src: u32) { + fn test_for(dst: u64, src: u32) { test_init_from_to(ValType::FuncRef, dst, src); test_init_from_to(ValType::ExternRef, dst, src); } @@ -305,12 +305,12 @@ fn init_from_to() { // Ideally we'd have yet another test for that case. continue; } - test_for(dst, src); + test_for(u64::from(dst), src); } } } -fn testcase_init_to_exact(ty: ValType, dst: u32, len: u32) -> TranslationTest { +fn testcase_init_to_exact(ty: ValType, dst: u64, len: u32) -> TranslationTest { let display_ty = DisplayValueType::from(ty); let wasm = format!( r" @@ -328,10 +328,10 @@ fn testcase_init_to_exact(ty: ValType, dst: u32, len: u32) -> TranslationTest { TranslationTest::new(&wasm) } -fn test_init_to_exact16(ty: ValType, dst: u32, len: u32) { +fn test_init_to_exact16(ty: ValType, dst: u64, len: u32) { testcase_init_to_exact(ty, dst, len) .expect_func_instrs([ - Instruction::table_init_to_exact(u32imm16(dst), Reg::from(0), u32imm16(len)), + Instruction::table_init_to_exact(u64imm16(dst), Reg::from(0), u32imm16(len)), Instruction::table_index(0), Instruction::elem_index(0), Instruction::Return, @@ -342,19 +342,19 @@ fn test_init_to_exact16(ty: ValType, dst: u32, len: u32) { #[test] #[cfg_attr(miri, ignore)] fn init_to_exact16() { - fn test_for(dst: u32, len: u32) { + fn test_for(dst: u64, len: u32) { test_init_to_exact16(ValType::FuncRef, dst, len); test_init_to_exact16(ValType::ExternRef, dst, len); } let values = [0, 1, u32::from(u16::MAX) - 1, u32::from(u16::MAX)]; for dst in values { for len in values { - test_for(dst, len); + test_for(u64::from(dst), len); } } } -fn test_init_to_exact(ty: ValType, dst: u32, len: u32) { +fn test_init_to_exact(ty: ValType, dst: u64, len: u32) { testcase_init_to_exact(ty, dst, len) .expect_func( ExpectedFunc::new([ @@ -363,7 +363,7 @@ fn test_init_to_exact(ty: ValType, dst: u32, len: u32) { Instruction::elem_index(0), Instruction::Return, ]) - .consts([dst, len]), + .consts([UntypedVal::from(dst), UntypedVal::from(len)]), ) .run() } @@ -371,7 +371,7 @@ fn test_init_to_exact(ty: ValType, dst: u32, len: u32) { #[test] #[cfg_attr(miri, ignore)] fn init_to_exact() { - fn test_for(dst: u32, len: u32) { + fn test_for(dst: u64, len: u32) { test_init_to_exact(ValType::FuncRef, dst, len); test_init_to_exact(ValType::ExternRef, dst, len); } @@ -385,7 +385,7 @@ fn init_to_exact() { // Ideally we'd have yet another test for that case. continue; } - test_for(dst, src); + test_for(u64::from(dst), src); } } } @@ -470,7 +470,7 @@ fn init_from_exact() { } } -fn testcase_init_from_to_exact(ty: ValType, dst: u32, src: u32, len: u32) -> TranslationTest { +fn testcase_init_from_to_exact(ty: ValType, dst: u64, src: u32, len: u32) -> TranslationTest { let display_ty = DisplayValueType::from(ty); let wasm = format!( r" @@ -488,10 +488,10 @@ fn testcase_init_from_to_exact(ty: ValType, dst: u32, src: u32, len: u32) -> Tra TranslationTest::new(&wasm) } -fn test_init_from_to_exact16(ty: ValType, dst: u32, src: u32, len: u32) { +fn test_init_from_to_exact16(ty: ValType, dst: u64, src: u32, len: u32) { testcase_init_from_to_exact(ty, dst, src, len) .expect_func_instrs([ - Instruction::table_init_from_to_exact(u32imm16(dst), u32imm16(src), u32imm16(len)), + Instruction::table_init_from_to_exact(u64imm16(dst), u32imm16(src), u32imm16(len)), Instruction::table_index(0), Instruction::elem_index(0), Instruction::Return, @@ -502,7 +502,7 @@ fn test_init_from_to_exact16(ty: ValType, dst: u32, src: u32, len: u32) { #[test] #[cfg_attr(miri, ignore)] fn init_from_to_exact16() { - fn test_for(dst: u32, src: u32, len: u32) { + fn test_for(dst: u64, src: u32, len: u32) { test_init_from_to_exact16(ValType::FuncRef, dst, src, len); test_init_from_to_exact16(ValType::ExternRef, dst, src, len); } @@ -510,13 +510,13 @@ fn init_from_to_exact16() { for dst in values { for src in values { for len in values { - test_for(dst, src, len); + test_for(u64::from(dst), src, len); } } } } -fn test_init_from_to_exact(ty: ValType, dst: u32, src: u32, len: u32) { +fn test_init_from_to_exact(ty: ValType, dst: u64, src: u32, len: u32) { testcase_init_from_to_exact(ty, dst, src, len) .expect_func( ExpectedFunc::new([ @@ -525,7 +525,11 @@ fn test_init_from_to_exact(ty: ValType, dst: u32, src: u32, len: u32) { Instruction::elem_index(0), Instruction::Return, ]) - .consts([dst, src, len]), + .consts([ + UntypedVal::from(dst), + UntypedVal::from(src), + UntypedVal::from(len), + ]), ) .run() } @@ -533,7 +537,7 @@ fn test_init_from_to_exact(ty: ValType, dst: u32, src: u32, len: u32) { #[test] #[cfg_attr(miri, ignore)] fn init_from_to_exact() { - fn test_for(dst: u32, src: u32, len: u32) { + fn test_for(dst: u64, src: u32, len: u32) { test_init_from_to_exact(ValType::FuncRef, dst, src, len); test_init_from_to_exact(ValType::ExternRef, dst, src, len); } @@ -548,7 +552,7 @@ fn init_from_to_exact() { // Ideally we'd have yet another test for that case. continue; } - test_for(dst, src, len); + test_for(u64::from(dst), src, len); } } } diff --git a/crates/wasmi/src/engine/translator/utils.rs b/crates/wasmi/src/engine/translator/utils.rs index c54c8db017..61d147c7b3 100644 --- a/crates/wasmi/src/engine/translator/utils.rs +++ b/crates/wasmi/src/engine/translator/utils.rs @@ -1,7 +1,8 @@ use super::{stack::ValueStack, Provider, TypedProvider, TypedVal}; use crate::{ - ir::{BoundedRegSpan, Const16, Reg, RegSpan, Sign}, + ir::{BoundedRegSpan, Const16, Const32, Reg, RegSpan, Sign}, Error, + IndexType, }; /// A WebAssembly integer. Either `i32` or `i64`. @@ -73,24 +74,94 @@ impl WasmFloat for f64 { } } -impl Provider> { - /// Creates a new `table` or `memory` index [`Provider`] from the general [`TypedProvider`]. +macro_rules! impl_provider_new_const16 { + ($ty:ty) => { + impl Provider> { + /// Creates a new `table` or `memory` index [`Provider`] from the general [`TypedProvider`]. + /// + /// # Note + /// + /// This is a convenience function and used by translation + /// procedures for certain Wasm `table` instructions. + pub fn new(provider: TypedProvider, stack: &mut ValueStack) -> Result { + match provider { + TypedProvider::Const(value) => match Const16::try_from(<$ty>::from(value)).ok() + { + Some(value) => Ok(Self::Const(value)), + None => { + let register = stack.alloc_const(value)?; + Ok(Self::Register(register)) + } + }, + TypedProvider::Register(index) => Ok(Self::Register(index)), + } + } + } + }; +} +impl_provider_new_const16!(u32); +impl_provider_new_const16!(u64); + +impl super::FuncTranslator { + /// Converts the `provider` to a 16-bit index-type constant value. + /// + /// # Note + /// + /// - Turns immediates that cannot be 16-bit encoded into function local constants. + /// - The behavior is different wether `memory64` is enabled or disabled. + pub(super) fn as_index_type_const16( + &mut self, + provider: TypedProvider, + index_type: IndexType, + ) -> Result>, Error> { + let value = match provider { + Provider::Register(reg) => return Ok(Provider::Register(reg)), + Provider::Const(value) => value, + }; + match index_type { + IndexType::I64 => { + if let Ok(value) = Const16::try_from(u64::from(value)) { + return Ok(Provider::Const(value)); + } + } + IndexType::I32 => { + if let Ok(value) = Const16::try_from(u32::from(value)) { + return Ok(Provider::Const(>::cast(value))); + } + } + } + let register = self.alloc.stack.alloc_const(value)?; + Ok(Provider::Register(register)) + } + + /// Converts the `provider` to a 32-bit index-type constant value. /// /// # Note /// - /// This is a convenience function and used by translation - /// procedures for certain Wasm `table` instructions. - pub fn new(provider: TypedProvider, stack: &mut ValueStack) -> Result { - match provider { - TypedProvider::Const(value) => match Const16::try_from(u32::from(value)).ok() { - Some(value) => Ok(Self::Const(value)), - None => { - let register = stack.alloc_const(value)?; - Ok(Self::Register(register)) + /// - Turns immediates that cannot be 32-bit encoded into function local constants. + /// - The behavior is different wether `memory64` is enabled or disabled. + pub(super) fn as_index_type_const32( + &mut self, + provider: TypedProvider, + index_type: IndexType, + ) -> Result>, Error> { + let value = match provider { + Provider::Register(reg) => return Ok(Provider::Register(reg)), + Provider::Const(value) => value, + }; + match index_type { + IndexType::I64 => { + if let Ok(value) = Const32::try_from(u64::from(value)) { + return Ok(Provider::Const(value)); } - }, - TypedProvider::Register(index) => Ok(Self::Register(index)), + } + IndexType::I32 => { + let value = Const32::from(u32::from(value)); + return Ok(Provider::Const(>::cast(value))); + } } + let register = self.alloc.stack.alloc_const(value)?; + Ok(Provider::Register(register)) } } diff --git a/crates/wasmi/src/engine/translator/visit.rs b/crates/wasmi/src/engine/translator/visit.rs index 7362808bb9..6cb9c9db17 100644 --- a/crates/wasmi/src/engine/translator/visit.rs +++ b/crates/wasmi/src/engine/translator/visit.rs @@ -21,8 +21,15 @@ use crate::{ BlockType, FuelCosts, }, - ir::{self, index, index::FuncType, BoundedRegSpan, Const16, Instruction, Reg}, - module::{self, FuncIdx, WasmiValueType}, + ir::{ + self, + index::{self, FuncType}, + BoundedRegSpan, + Const16, + Instruction, + Reg, + }, + module::{self, FuncIdx, MemoryIdx, TableIdx, WasmiValueType}, Error, ExternRef, FuncRef, @@ -3047,8 +3054,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { fn visit_memory_init(&mut self, data_index: u32, mem: u32) -> Self::Output { bail_unreachable!(self); let memory = index::Memory::from(mem); + let memory_type = *self.module.get_type_of_memory(MemoryIdx::from(mem)); let (dst, src, len) = self.alloc.stack.pop3(); - let dst = >>::new(dst, &mut self.alloc.stack)?; + let dst = self.as_index_type_const16(dst, memory_type.index_ty())?; let src = >>::new(src, &mut self.alloc.stack)?; let len = >>::new(len, &mut self.alloc.stack)?; let instr = match (dst, src, len) { @@ -3097,10 +3105,16 @@ impl<'a> VisitOperator<'a> for FuncTranslator { bail_unreachable!(self); let dst_memory = index::Memory::from(dst_mem); let src_memory = index::Memory::from(src_mem); + let (dst, src, len) = self.alloc.stack.pop3(); - let dst = >>::new(dst, &mut self.alloc.stack)?; - let src = >>::new(src, &mut self.alloc.stack)?; - let len = >>::new(len, &mut self.alloc.stack)?; + + let dst_memory_type = *self.module.get_type_of_memory(MemoryIdx::from(dst_mem)); + let src_memory_type = *self.module.get_type_of_memory(MemoryIdx::from(src_mem)); + let min_index_ty = dst_memory_type.index_ty().min(&src_memory_type.index_ty()); + let dst = self.as_index_type_const16(dst, dst_memory_type.index_ty())?; + let src = self.as_index_type_const16(src, src_memory_type.index_ty())?; + let len = self.as_index_type_const16(len, min_index_ty)?; + let instr = match (dst, src, len) { (Provider::Register(dst), Provider::Register(src), Provider::Register(len)) => { Instruction::memory_copy(dst, src, len) @@ -3140,10 +3154,11 @@ impl<'a> VisitOperator<'a> for FuncTranslator { fn visit_memory_fill(&mut self, mem: u32) -> Self::Output { bail_unreachable!(self); let memory = index::Memory::from(mem); + let memory_type = *self.module.get_type_of_memory(MemoryIdx::from(mem)); let (dst, value, len) = self.alloc.stack.pop3(); - let dst = >>::new(dst, &mut self.alloc.stack)?; + let dst = self.as_index_type_const16(dst, memory_type.index_ty())?; let value = value.map_const(|value| u32::from(value) as u8); - let len = >>::new(len, &mut self.alloc.stack)?; + let len = self.as_index_type_const16(len, memory_type.index_ty())?; let instr = match (dst, value, len) { (Provider::Register(dst), Provider::Register(value), Provider::Register(len)) => { Instruction::memory_fill(dst, value, len) @@ -3180,7 +3195,8 @@ impl<'a> VisitOperator<'a> for FuncTranslator { fn visit_table_init(&mut self, elem_index: u32, table: u32) -> Self::Output { bail_unreachable!(self); let (dst, src, len) = self.alloc.stack.pop3(); - let dst = >>::new(dst, &mut self.alloc.stack)?; + let table_type = *self.module.get_type_of_table(TableIdx::from(table)); + let dst = self.as_index_type_const16(dst, table_type.index_ty())?; let src = >>::new(src, &mut self.alloc.stack)?; let len = >>::new(len, &mut self.alloc.stack)?; let instr = match (dst, src, len) { @@ -3228,9 +3244,12 @@ impl<'a> VisitOperator<'a> for FuncTranslator { fn visit_table_copy(&mut self, dst_table: u32, src_table: u32) -> Self::Output { bail_unreachable!(self); let (dst, src, len) = self.alloc.stack.pop3(); - let dst = >>::new(dst, &mut self.alloc.stack)?; - let src = >>::new(src, &mut self.alloc.stack)?; - let len = >>::new(len, &mut self.alloc.stack)?; + let dst_table_type = *self.module.get_type_of_table(TableIdx::from(dst_table)); + let src_table_type = *self.module.get_type_of_table(TableIdx::from(src_table)); + let min_index_ty = dst_table_type.index_ty().min(&src_table_type.index_ty()); + let dst = self.as_index_type_const16(dst, dst_table_type.index_ty())?; + let src = self.as_index_type_const16(src, src_table_type.index_ty())?; + let len = self.as_index_type_const16(len, min_index_ty)?; let instr = match (dst, src, len) { (Provider::Register(dst), Provider::Register(src), Provider::Register(len)) => { Instruction::table_copy(dst, src, len) @@ -3270,8 +3289,9 @@ impl<'a> VisitOperator<'a> for FuncTranslator { fn visit_table_fill(&mut self, table: u32) -> Self::Output { bail_unreachable!(self); let (dst, value, len) = self.alloc.stack.pop3(); - let dst = >>::new(dst, &mut self.alloc.stack)?; - let len = >>::new(len, &mut self.alloc.stack)?; + let table_type = *self.module.get_type_of_table(TableIdx::from(table)); + let dst = self.as_index_type_const16(dst, table_type.index_ty())?; + let len = self.as_index_type_const16(len, table_type.index_ty())?; let value = match value { TypedProvider::Register(value) => value, TypedProvider::Const(value) => self.alloc.stack.alloc_const(value)?, @@ -3299,19 +3319,15 @@ impl<'a> VisitOperator<'a> for FuncTranslator { fn visit_table_get(&mut self, table: u32) -> Self::Output { bail_unreachable!(self); + let table_type = *self.module.get_type_of_table(TableIdx::from(table)); let index = self.alloc.stack.pop(); let result = self.alloc.stack.push_dynamic()?; - match index { - TypedProvider::Register(index) => { - self.push_fueled_instr(Instruction::table_get(result, index), FuelCosts::entity)?; - } - TypedProvider::Const(index) => { - self.push_fueled_instr( - Instruction::table_get_imm(result, u32::from(index)), - FuelCosts::entity, - )?; - } - } + let index = self.as_index_type_const32(index, table_type.index_ty())?; + let instr = match index { + Provider::Register(index) => Instruction::table_get(result, index), + Provider::Const(index) => Instruction::table_get_imm(result, index), + }; + self.push_fueled_instr(instr, FuelCosts::entity)?; self.alloc .instr_encoder .append_instr(Instruction::table_index(table))?; @@ -3320,14 +3336,16 @@ impl<'a> VisitOperator<'a> for FuncTranslator { fn visit_table_set(&mut self, table: u32) -> Self::Output { bail_unreachable!(self); + let table_type = *self.module.get_type_of_table(TableIdx::from(table)); let (index, value) = self.alloc.stack.pop2(); + let index = self.as_index_type_const32(index, table_type.index_ty())?; let value = match value { TypedProvider::Register(value) => value, TypedProvider::Const(value) => self.alloc.stack.alloc_const(value)?, }; let instr = match index { - TypedProvider::Register(index) => Instruction::table_set(index, value), - TypedProvider::Const(index) => Instruction::table_set_at(value, u32::from(index)), + Provider::Register(index) => Instruction::table_set(index, value), + Provider::Const(index) => Instruction::table_set_at(value, index), }; self.push_fueled_instr(instr, FuelCosts::entity)?; self.alloc @@ -3338,9 +3356,11 @@ impl<'a> VisitOperator<'a> for FuncTranslator { fn visit_table_grow(&mut self, table: u32) -> Self::Output { bail_unreachable!(self); + let table_type = *self.module.get_type_of_table(TableIdx::from(table)); let (value, delta) = self.alloc.stack.pop2(); + let delta = self.as_index_type_const16(delta, table_type.index_ty())?; if let Provider::Const(delta) = delta { - if u32::from(delta) == 0 { + if u64::from(delta) == 0 { // Case: growing by 0 elements. // // Since `table.grow` returns the `table.size` before the @@ -3351,7 +3371,6 @@ impl<'a> VisitOperator<'a> for FuncTranslator { return Ok(()); } } - let delta = >>::new(delta, &mut self.alloc.stack)?; let value = match value { TypedProvider::Register(value) => value, TypedProvider::Const(value) => self.alloc.stack.alloc_const(value)?, diff --git a/crates/wasmi/src/error.rs b/crates/wasmi/src/error.rs index 3395f12c38..33b00f026f 100644 --- a/crates/wasmi/src/error.rs +++ b/crates/wasmi/src/error.rs @@ -315,9 +315,15 @@ pub enum EntityGrowError { } impl EntityGrowError { - /// The WebAssembly specification demands to return this value - /// if the `memory.grow` or `table.grow` operations fail. - pub const ERROR_CODE: u32 = u32::MAX; + /// The error code returned by `memory.grow` and `table.grow` upon failure. + /// + /// This is the value used for 32-bit memory or table indices. + pub const ERROR_CODE_32: u64 = u32::MAX as u64; + + /// The error code returned by `memory.grow` and `table.grow` upon failure. + /// + /// This is the value used for 64-bit memory or table indices. + pub const ERROR_CODE_64: u64 = u64::MAX; } impl From for EntityGrowError { diff --git a/crates/wasmi/src/index_ty.rs b/crates/wasmi/src/index_ty.rs new file mode 100644 index 0000000000..7131f049a3 --- /dev/null +++ b/crates/wasmi/src/index_ty.rs @@ -0,0 +1,33 @@ +/// The index type used for addressing memories and tables. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum IndexType { + /// A 32-bit address type. + I32, + /// A 64-bit address type. + I64, +} + +impl IndexType { + /// Returns `true` if `self` is `IndexType::I64`. + pub fn is_64(&self) -> bool { + matches!(self, Self::I64) + } + + /// Returns the maximum size for Wasm memories and tables for `self`. + pub fn max_size(&self) -> u128 { + const WASM32_MAX_SIZE: u128 = 1 << 32; + const WASM64_MAX_SIZE: u128 = 1 << 64; + match self { + Self::I32 => WASM32_MAX_SIZE, + Self::I64 => WASM64_MAX_SIZE, + } + } + + /// Returns the minimum [`IndexType`] between `self` and `other`. + pub fn min(&self, other: &Self) -> Self { + match (self, other) { + (IndexType::I64, IndexType::I64) => IndexType::I64, + _ => IndexType::I32, + } + } +} diff --git a/crates/wasmi/src/lib.rs b/crates/wasmi/src/lib.rs index f78145d85d..3000f8a1d2 100644 --- a/crates/wasmi/src/lib.rs +++ b/crates/wasmi/src/lib.rs @@ -87,6 +87,7 @@ mod error; mod externref; mod func; mod global; +mod index_ty; mod instance; mod limits; mod linker; @@ -175,6 +176,7 @@ pub use self::{ use self::{ func::{FuncEntity, FuncIdx}, global::{GlobalEntity, GlobalIdx}, + index_ty::IndexType, instance::{InstanceEntity, InstanceEntityBuilder, InstanceIdx}, memory::{DataSegmentEntity, DataSegmentIdx, MemoryEntity, MemoryIdx}, store::Stored, diff --git a/crates/wasmi/src/limits.rs b/crates/wasmi/src/limits.rs index bcc565f785..218b69383c 100644 --- a/crates/wasmi/src/limits.rs +++ b/crates/wasmi/src/limits.rs @@ -79,9 +79,9 @@ pub trait ResourceLimiter { /// return values of this function indicates. fn table_growing( &mut self, - current: u32, - desired: u32, - maximum: Option, + current: usize, + desired: usize, + maximum: Option, ) -> Result; /// Notifies the resource limiter that growing a linear memory, permitted by @@ -152,7 +152,7 @@ impl StoreLimitsBuilder { /// they're all allowed to reach up to the `limit` specified. /// /// By default, table elements will not be limited. - pub fn table_elements(mut self, limit: u32) -> Self { + pub fn table_elements(mut self, limit: usize) -> Self { self.0.table_elements = Some(limit); self } @@ -224,7 +224,7 @@ impl Default for StoreLimitsBuilder { #[derive(Clone, Debug)] pub struct StoreLimits { memory_size: Option, - table_elements: Option, + table_elements: Option, instances: usize, tables: usize, memories: usize, @@ -259,17 +259,16 @@ impl ResourceLimiter for StoreLimits { }, }; if !allow && self.trap_on_grow_failure { - Err(MemoryError::OutOfBoundsGrowth) - } else { - Ok(allow) + return Err(MemoryError::ResourceLimiterDeniedAllocation); } + Ok(allow) } fn table_growing( &mut self, - current: u32, - desired: u32, - maximum: Option, + _current: usize, + desired: usize, + maximum: Option, ) -> Result { let allow = match self.table_elements { Some(limit) if desired > limit => false, @@ -279,14 +278,9 @@ impl ResourceLimiter for StoreLimits { }, }; if !allow && self.trap_on_grow_failure { - Err(TableError::GrowOutOfBounds { - maximum: maximum.unwrap_or(u32::MAX), - current, - delta: desired - current, - }) - } else { - Ok(allow) + return Err(TableError::ResourceLimiterDeniedAllocation); } + Ok(allow) } fn instances(&self) -> usize { diff --git a/crates/wasmi/src/memory/buffer.rs b/crates/wasmi/src/memory/buffer.rs index e7a116daa9..13050a3b2c 100644 --- a/crates/wasmi/src/memory/buffer.rs +++ b/crates/wasmi/src/memory/buffer.rs @@ -76,7 +76,7 @@ impl ByteBuffer { pub fn new(size: usize) -> Result { let mut vec = Vec::new(); if vec.try_reserve(size).is_err() { - return Err(MemoryError::OutOfBoundsAllocation); + return Err(MemoryError::OutOfSystemMemory); }; vec.extend(iter::repeat_n(0x00_u8, size)); let (ptr, len, capacity) = vec_into_raw_parts(vec); @@ -134,7 +134,7 @@ impl ByteBuffer { debug_assert!(vec.len() <= new_size); let additional = new_size - vec.len(); if vec.try_reserve(additional).is_err() { - return Err(MemoryError::OutOfBoundsAllocation); + return Err(MemoryError::OutOfSystemMemory); }; vec.resize(new_size, 0x00_u8); (self.ptr, self.len, self.capacity) = vec_into_raw_parts(vec); diff --git a/crates/wasmi/src/memory/error.rs b/crates/wasmi/src/memory/error.rs index 05da3fe83f..a13b943784 100644 --- a/crates/wasmi/src/memory/error.rs +++ b/crates/wasmi/src/memory/error.rs @@ -6,7 +6,7 @@ use core::{fmt, fmt::Display}; #[non_exhaustive] pub enum MemoryError { /// Tried to allocate more virtual memory than technically possible. - OutOfBoundsAllocation, + OutOfSystemMemory, /// Tried to grow linear memory out of its set bounds. OutOfBoundsGrowth, /// Tried to access linear memory out of bounds. @@ -26,6 +26,10 @@ pub enum MemoryError { InvalidStaticBufferSize, /// If a resource limiter denied allocation or growth of a linear memory. ResourceLimiterDeniedAllocation, + // The minimum size of the memory type overflows the system index type. + MinimumSizeOverflow, + // The maximum size of the memory type overflows the system index type. + MaximumSizeOverflow, } #[cfg(feature = "std")] @@ -34,8 +38,11 @@ impl std::error::Error for MemoryError {} impl Display for MemoryError { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self { - Self::OutOfBoundsAllocation => { - write!(f, "out of bounds memory allocation") + Self::OutOfSystemMemory => { + write!( + f, + "tried to allocate more virtual memory than available on the system" + ) } Self::OutOfBoundsGrowth => { write!(f, "out of bounds memory growth") @@ -61,6 +68,18 @@ impl Display for MemoryError { "a resource limiter denied to allocate or grow the linear memory" ) } + Self::MinimumSizeOverflow => { + write!( + f, + "the minimum size of the memory type overflows the system index type" + ) + } + Self::MaximumSizeOverflow => { + write!( + f, + "the maximum size of the memory type overflows the system index type" + ) + } } } } diff --git a/crates/wasmi/src/memory/mod.rs b/crates/wasmi/src/memory/mod.rs index 5492d2b935..71936c9a97 100644 --- a/crates/wasmi/src/memory/mod.rs +++ b/crates/wasmi/src/memory/mod.rs @@ -17,6 +17,7 @@ use crate::{ error::EntityGrowError, store::{Fuel, ResourceLimiterRef}, Error, + IndexType, }; /// A raw index to a linear memory entity. @@ -36,12 +37,94 @@ impl ArenaIndex for MemoryIdx { } } +/// Internal memory type data and details. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct MemoryTypeInner { + /// The initial or minimum amount of pages. + minimum: u64, + /// The optional maximum amount of pages. + maximum: Option, + /// The size of a page log2. + page_size_log2: u8, + /// The index type used to address a linear memory. + index_type: IndexType, +} + +/// A type to indicate that a size calculation has overflown. +#[derive(Debug, Copy, Clone)] +pub struct SizeOverflow; + +impl MemoryTypeInner { + /// Returns the minimum size, in bytes, that the linear memory must have. + /// + /// # Errors + /// + /// If the calculation of the minimum size overflows the maximum size. + /// This means that the linear memory can't be allocated. + /// The caller is responsible to deal with that situation. + fn minimum_byte_size(&self) -> Result { + let min = u128::from(self.minimum); + if min > self.absolute_max() { + return Err(SizeOverflow); + } + Ok(min << self.page_size_log2) + } + + /// Returns the maximum size, in bytes, that the linear memory must have. + /// + /// # Note + /// + /// If the maximum size of a memory type is not specified a concrete + /// maximum value is returned dependent on the index type of the memory type. + /// + /// # Errors + /// + /// If the calculation of the maximum size overflows the index type. + /// This means that the linear memory can't be allocated. + /// The caller is responsible to deal with that situation. + fn maximum_byte_size(&self) -> Result { + match self.maximum { + Some(max) => { + let max = u128::from(max); + if max > self.absolute_max() { + return Err(SizeOverflow); + } + Ok(max << self.page_size_log2) + } + None => Ok(self.max_size_based_on_index_type()), + } + } + + /// Returns the size of the linear memory pages in bytes. + fn page_size(&self) -> u32 { + debug_assert!( + self.page_size_log2 == 16 || self.page_size_log2 == 0, + "invalid `page_size_log2`: {}; must be 16 or 0", + self.page_size_log2 + ); + 1 << self.page_size_log2 + } + + /// Returns the maximum size in bytes allowed by the `index_type` of this memory type. + /// + /// # Note + /// + /// - This does _not_ take into account the page size. + /// - This is based _only_ on the index type used by the memory type. + fn max_size_based_on_index_type(&self) -> u128 { + self.index_type.max_size() + } + + /// Returns the absolute maximum size in pages that a linear memory is allowed to have. + fn absolute_max(&self) -> u128 { + self.max_size_based_on_index_type() >> self.page_size_log2 + } +} + /// The memory type of a linear memory. #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub struct MemoryType { - initial_pages: u32, - maximum_pages: Option, - page_size_log2: u8, + inner: MemoryTypeInner, } /// A builder for [`MemoryType`]s. @@ -50,17 +133,18 @@ pub struct MemoryType { /// Allows to incrementally build-up a [`MemoryType`]. When done, finalize creation /// via a call to [`MemoryTypeBuilder::build`]. pub struct MemoryTypeBuilder { - minimum_pages: u32, - maximum_pages: Option, - page_size_log2: u8, + inner: MemoryTypeInner, } impl Default for MemoryTypeBuilder { fn default() -> Self { Self { - minimum_pages: 0, - maximum_pages: None, - page_size_log2: MemoryType::DEFAULT_PAGE_SIZE_LOG2, + inner: MemoryTypeInner { + minimum: 0, + maximum: None, + page_size_log2: MemoryType::DEFAULT_PAGE_SIZE_LOG2, + index_type: IndexType::I32, + }, } } } @@ -75,11 +159,26 @@ impl MemoryTypeBuilder { Self::default() } + /// Set whether this is a 64-bit memory type or not. + /// + /// By default a memory is a 32-bit, a.k.a. `false`. + /// + /// 64-bit memories are part of the [Wasm `memory64` proposal]. + /// + /// [Wasm `memory64` proposal]: https://github.com/WebAssembly/memory64 + pub fn memory64(&mut self, memory64: bool) -> &mut Self { + self.inner.index_type = match memory64 { + true => IndexType::I64, + false => IndexType::I32, + }; + self + } + /// Sets the minimum number of pages the built [`MemoryType`] supports. /// /// The default minimum is `0`. - pub fn min(&mut self, minimum: u32) -> &mut Self { - self.minimum_pages = minimum; + pub fn min(&mut self, minimum: u64) -> &mut Self { + self.inner.minimum = minimum; self } @@ -88,8 +187,8 @@ impl MemoryTypeBuilder { /// A value of `None` means that there is no maximum number of pages. /// /// The default maximum is `None`. - pub fn max(&mut self, maximum: Option) -> &mut Self { - self.maximum_pages = maximum; + pub fn max(&mut self, maximum: Option) -> &mut Self { + self.inner.maximum = maximum; self } @@ -105,7 +204,7 @@ impl MemoryTypeBuilder { /// /// [`custom-page-sizes proposal`]: https://github.com/WebAssembly/custom-page-sizes pub fn page_size_log2(&mut self, page_size_log2: u8) -> &mut Self { - self.page_size_log2 = page_size_log2; + self.inner.page_size_log2 = page_size_log2; self } @@ -116,11 +215,7 @@ impl MemoryTypeBuilder { /// If the chosen configuration for the constructed [`MemoryType`] is invalid. pub fn build(self) -> Result { self.validate()?; - Ok(MemoryType { - initial_pages: self.minimum_pages, - maximum_pages: self.maximum_pages, - page_size_log2: self.page_size_log2, - }) + Ok(MemoryType { inner: self.inner }) } /// Validates the configured [`MemoryType`] of the [`MemoryTypeBuilder`]. @@ -129,14 +224,7 @@ impl MemoryTypeBuilder { /// /// If the chosen configuration for the constructed [`MemoryType`] is invalid. fn validate(&self) -> Result<(), Error> { - if self - .maximum_pages - .is_some_and(|max| max < self.minimum_pages) - { - // Case: maximum page size cannot be smaller than the minimum page size - return Err(Error::from(MemoryError::InvalidMemoryType)); - } - match self.page_size_log2 { + match self.inner.page_size_log2 { 0 | MemoryType::DEFAULT_PAGE_SIZE_LOG2 => {} _ => { // Case: currently, pages sizes log2 can only be 0 or 16. @@ -144,19 +232,17 @@ impl MemoryTypeBuilder { return Err(Error::from(MemoryError::InvalidMemoryType)); } } - let page_size = 2_u32 - .checked_pow(u32::from(self.page_size_log2)) - .expect("page size must not overflow `u32` value"); - let absolute_max = u64::from(u32::MAX) + 1; - let minimum_byte_size = u64::from(self.minimum_pages) * u64::from(page_size); - if minimum_byte_size > absolute_max { - // Case: the page size and the minimum size invalidly overflows `u32`. + if self.inner.minimum_byte_size().is_err() { + // Case: the minimum size overflows a `absolute_max` return Err(Error::from(MemoryError::InvalidMemoryType)); } - if let Some(maximum_pages) = self.maximum_pages { - let maximum_byte_size = u64::from(maximum_pages) * u64::from(page_size); - if maximum_byte_size > absolute_max { - // Case: the page size and the minimum size invalidly overflows `u32`. + if let Some(max) = self.inner.maximum { + if self.inner.maximum_byte_size().is_err() { + // Case: the maximum size overflows a `absolute_max` + return Err(Error::from(MemoryError::InvalidMemoryType)); + } + if self.inner.minimum > max { + // Case: maximum size must be at least as large as minimum size return Err(Error::from(MemoryError::InvalidMemoryType)); } } @@ -176,6 +262,24 @@ impl MemoryType { /// - If the `minimum` or `maximum` pages are out of bounds. pub fn new(minimum: u32, maximum: Option) -> Result { let mut b = Self::builder(); + b.min(u64::from(minimum)); + b.max(maximum.map(u64::from)); + b.build() + } + + /// Creates a new 64-bit memory type with minimum and optional maximum pages. + /// + /// # Errors + /// + /// - If the `minimum` pages exceeds the `maximum` pages. + /// - If the `minimum` or `maximum` pages are out of bounds. + /// + /// 64-bit memories are part of the [Wasm `memory64` proposal]. + /// + /// [Wasm `memory64` proposal]: https://github.com/WebAssembly/memory64 + pub fn new64(minimum: u64, maximum: Option) -> Result { + let mut b = Self::builder(); + b.memory64(true); b.min(minimum); b.max(maximum); b.build() @@ -186,26 +290,38 @@ impl MemoryType { MemoryTypeBuilder::default() } + /// Returns `true` if this is a 64-bit [`MemoryType`]. + /// + /// 64-bit memories are part of the Wasm `memory64` proposal. + pub fn is_64(&self) -> bool { + self.index_ty().is_64() + } + + /// Returns the [`IndexType`] used by the [`MemoryType`]. + pub(crate) fn index_ty(&self) -> IndexType { + self.inner.index_type + } + /// Returns the minimum pages of the memory type. - pub fn minimum(self) -> u32 { - self.initial_pages + pub fn minimum(self) -> u64 { + self.inner.minimum } /// Returns the maximum pages of the memory type. /// /// Returns `None` if there is no limit set. - pub fn maximum(self) -> Option { - self.maximum_pages + pub fn maximum(self) -> Option { + self.inner.maximum } /// Returns the page size of the [`MemoryType`] in bytes. pub fn page_size(self) -> u32 { - 2_u32.pow(u32::from(self.page_size_log2)) + self.inner.page_size() } /// Returns the page size of the [`MemoryType`] in log2(bytes). pub fn page_size_log2(self) -> u8 { - self.page_size_log2 + self.inner.page_size_log2 } /// Checks if `self` is a subtype of `other`. @@ -240,6 +356,9 @@ impl MemoryType { /// [import subtyping]: /// https://webassembly.github.io/spec/core/valid/types.html#import-subtyping pub(crate) fn is_subtype_of(&self, other: &MemoryType) -> bool { + if self.is_64() != other.is_64() { + return false; + } if self.page_size() != other.page_size() { return false; } @@ -257,10 +376,9 @@ impl MemoryType { /// A linear memory entity. #[derive(Debug)] pub struct MemoryEntity { + /// The size of `bytes` will always be a multiple of a page size. bytes: ByteBuffer, memory_type: MemoryType, - /// Current size of the linear memory in pages. - size: u32, } impl MemoryEntity { @@ -289,37 +407,37 @@ impl MemoryEntity { limiter: &mut ResourceLimiterRef<'_>, make_buffer: impl FnOnce(usize) -> Result, ) -> Result { - let bytes_per_page = memory_type.page_size(); - let minimum_pages = memory_type.minimum(); - let maximum_pages = memory_type.maximum(); - let bytes_per_page64 = u64::from(bytes_per_page); - let minimum_byte_size64 = u64::from(minimum_pages) * bytes_per_page64; - let maximum_byte_size64 = maximum_pages - .map(u64::from) - .map(|max| max * bytes_per_page64); - let absolute_max = u64::from(u32::MAX) + 1; - if minimum_byte_size64 > absolute_max { - return Err(MemoryError::InvalidMemoryType); - } - if let Some(maximum_byte_size64) = maximum_byte_size64 { - if maximum_byte_size64 > absolute_max { - return Err(MemoryError::InvalidMemoryType); - } - } - let Ok(minimum_byte_size) = usize::try_from(minimum_byte_size64) else { - return Err(MemoryError::InvalidMemoryType); + let Ok(min_size) = memory_type.inner.minimum_byte_size() else { + return Err(MemoryError::MinimumSizeOverflow); }; - let Ok(maximum_byte_size) = maximum_byte_size64.map(usize::try_from).transpose() else { - return Err(MemoryError::InvalidMemoryType); + let Ok(min_size) = usize::try_from(min_size) else { + return Err(MemoryError::MinimumSizeOverflow); + }; + let max_size = match memory_type.inner.maximum { + Some(max) => { + let max = u128::from(max); + if max > memory_type.inner.absolute_max() { + return Err(MemoryError::MaximumSizeOverflow); + } + // Note: We have to clip `max_size` at `usize::MAX` since we do not want to + // error if the system limits are overflown here. This is because Wasm + // memories grow lazily and thus creation of memories which have a max + // size that overflows system limits are valid as long as they do not + // grow beyond those limits. + let max_size = + usize::try_from(max << memory_type.inner.page_size_log2).unwrap_or(usize::MAX); + Some(max_size) + } + None => None, }; if let Some(limiter) = limiter.as_resource_limiter() { - if !limiter.memory_growing(0, minimum_byte_size, maximum_byte_size)? { + if !limiter.memory_growing(0, min_size, max_size)? { return Err(MemoryError::ResourceLimiterDeniedAllocation); } } - let bytes = match make_buffer(minimum_byte_size) { + let bytes = match make_buffer(min_size) { Ok(buffer) => buffer, Err(error) => { if let Some(limiter) = limiter.as_resource_limiter() { @@ -328,11 +446,7 @@ impl MemoryEntity { return Err(error); } }; - Ok(Self { - bytes, - memory_type, - size: minimum_pages, - }) + Ok(Self { bytes, memory_type }) } /// Returns the memory type of the linear memory. @@ -350,23 +464,25 @@ impl MemoryEntity { let current_pages = self.size(); let maximum_pages = self.ty().maximum(); let page_size_log2 = self.ty().page_size_log2(); + let is_64 = self.ty().is_64(); let mut b = MemoryType::builder(); b.min(current_pages); b.max(maximum_pages); b.page_size_log2(page_size_log2); + b.memory64(is_64); b.build() .expect("must result in valid memory type due to invariants") } /// Returns the size, in WebAssembly pages, of this Wasm linear memory. - pub fn size(&self) -> u32 { - self.size + pub fn size(&self) -> u64 { + (self.bytes.len() as u64) >> self.memory_type.page_size_log2() } /// Returns the size of this Wasm linear memory in bytes. - fn size_in_bytes(&self) -> u32 { + fn size_in_bytes(&self) -> u64 { let pages = self.size(); - let bytes_per_page = self.memory_type.page_size(); + let bytes_per_page = u64::from(self.memory_type.page_size()); let Some(bytes) = pages.checked_mul(bytes_per_page) else { panic!( "unexpected out of bounds linear memory size: \ @@ -377,9 +493,9 @@ impl MemoryEntity { } /// Returns the maximum size of this Wasm linear memory in bytes if any. - fn max_size_in_bytes(&self) -> Option { + fn max_size_in_bytes(&self) -> Option { let max_pages = self.memory_type.maximum()?; - let bytes_per_page = self.memory_type.page_size(); + let bytes_per_page = u64::from(self.memory_type.page_size()); let Some(max_bytes) = max_pages.checked_mul(bytes_per_page) else { panic!( "unexpected out of bounds linear memory maximum size: \ @@ -399,14 +515,14 @@ impl MemoryEntity { /// - If the `limiter` denies the growth operation. pub fn grow( &mut self, - additional: u32, + additional: u64, fuel: Option<&mut Fuel>, limiter: &mut ResourceLimiterRef<'_>, - ) -> Result { + ) -> Result { fn notify_limiter( limiter: &mut ResourceLimiterRef<'_>, err: EntityGrowError, - ) -> Result { + ) -> Result { if let Some(limiter) = limiter.as_resource_limiter() { limiter.memory_grow_failed(&MemoryError::OutOfBoundsGrowth) } @@ -422,16 +538,21 @@ impl MemoryEntity { let Some(desired_size) = current_size.checked_add(additional) else { return Err(EntityGrowError::InvalidGrow); }; + if u128::from(desired_size) > self.memory_type.inner.absolute_max() { + return Err(EntityGrowError::InvalidGrow); + } if let Some(maximum_size) = self.memory_type.maximum() { if desired_size > maximum_size { return Err(EntityGrowError::InvalidGrow); } } - let bytes_per_page = self.memory_type.page_size(); + let bytes_per_page = u64::from(self.memory_type.page_size()); let Some(desired_byte_size) = desired_size.checked_mul(bytes_per_page) else { return Err(EntityGrowError::InvalidGrow); }; - let desired_byte_size = desired_byte_size as usize; + let Ok(desired_byte_size) = usize::try_from(desired_byte_size) else { + return Err(EntityGrowError::InvalidGrow); + }; // The `ResourceLimiter` gets first look at the request. if let Some(limiter) = limiter.as_resource_limiter() { @@ -448,8 +569,8 @@ impl MemoryEntity { // not charge fuel if there is any other deterministic failure preventing the expensive // growth operation. if let Some(fuel) = fuel { - let additional_bytes = u64::from(additional) - .checked_mul(u64::from(bytes_per_page)) + let additional_bytes = additional + .checked_mul(bytes_per_page) .expect("additional size is within [min, max) page bounds"); if fuel .consume_fuel_if(|costs| costs.fuel_for_bytes(additional_bytes)) @@ -468,7 +589,6 @@ impl MemoryEntity { if self.bytes.grow(desired_byte_size).is_err() { return notify_limiter(limiter, EntityGrowError::InvalidGrow); } - self.size = desired_size; Ok(current_size) } @@ -612,7 +732,7 @@ impl Memory { /// # Panics /// /// Panics if `ctx` does not own this [`Memory`]. - pub fn size(&self, ctx: impl AsContext) -> u32 { + pub fn size(&self, ctx: impl AsContext) -> u64 { ctx.as_context().store.inner.resolve_memory(self).size() } @@ -628,7 +748,7 @@ impl Memory { /// # Panics /// /// Panics if `ctx` does not own this [`Memory`]. - pub fn grow(&self, mut ctx: impl AsContextMut, additional: u32) -> Result { + pub fn grow(&self, mut ctx: impl AsContextMut, additional: u64) -> Result { let (inner, mut limiter) = ctx .as_context_mut() .store diff --git a/crates/wasmi/src/module/instantiate/error.rs b/crates/wasmi/src/module/instantiate/error.rs index bee4a34ad1..1ceac82393 100644 --- a/crates/wasmi/src/module/instantiate/error.rs +++ b/crates/wasmi/src/module/instantiate/error.rs @@ -45,9 +45,9 @@ pub enum InstantiationError { /// The table of the element segment. table: Table, /// The offset to store the `amount` of elements into the table. - offset: u32, + table_index: u64, /// The amount of elements with which the table is initialized at the `offset`. - amount: u32, + len: u32, }, /// Caused when the `start` function was unexpectedly found in the instantiated module. FoundStartFn { @@ -79,8 +79,8 @@ impl Display for InstantiationError { } Self::ElementSegmentDoesNotFit { table, - offset, - amount, + table_index: offset, + len: amount, } => write!( f, "out of bounds table access: {table:?} does not fit {amount} elements starting from offset {offset}", diff --git a/crates/wasmi/src/module/instantiate/mod.rs b/crates/wasmi/src/module/instantiate/mod.rs index a8f155e838..5c898b5f92 100644 --- a/crates/wasmi/src/module/instantiate/mod.rs +++ b/crates/wasmi/src/module/instantiate/mod.rs @@ -292,7 +292,7 @@ impl Module { let element = ElementSegment::new(context.as_context_mut(), segment, get_func, get_global); if let ElementSegmentKind::Active(active) = segment.kind() { - let dst_index = u32::from(Self::eval_init_expr( + let dst_index = u64::from(Self::eval_init_expr( context.as_context(), builder, active.offset(), @@ -304,12 +304,12 @@ impl Module { let len_table = table.size(&context); let len_items = element.size(&context); dst_index - .checked_add(len_items) + .checked_add(u64::from(len_items)) .filter(|&max_index| max_index <= len_table) .ok_or(InstantiationError::ElementSegmentDoesNotFit { table, - offset: dst_index, - amount: len_items, + table_index: dst_index, + len: len_items, })?; let (table, elem) = context .as_context_mut() diff --git a/crates/wasmi/src/module/mod.rs b/crates/wasmi/src/module/mod.rs index 028dbc8ad3..17a49b97b7 100644 --- a/crates/wasmi/src/module/mod.rs +++ b/crates/wasmi/src/module/mod.rs @@ -106,6 +106,16 @@ impl ModuleHeader { &self.inner.globals[global_idx.into_u32() as usize] } + /// Returns the [`MemoryType`] the the indexed Wasm memory. + pub fn get_type_of_memory(&self, memory_idx: MemoryIdx) -> &MemoryType { + &self.inner.memories[memory_idx.into_u32() as usize] + } + + /// Returns the [`TableType`] the the indexed Wasm table. + pub fn get_type_of_table(&self, table_idx: TableIdx) -> &TableType { + &self.inner.tables[table_idx.into_u32() as usize] + } + /// Returns the [`EngineFunc`] for the given [`FuncIdx`]. /// /// Returns `None` if [`FuncIdx`] refers to an imported function. diff --git a/crates/wasmi/src/module/utils.rs b/crates/wasmi/src/module/utils.rs index b6f21df261..a0486d4ad2 100644 --- a/crates/wasmi/src/module/utils.rs +++ b/crates/wasmi/src/module/utils.rs @@ -1,7 +1,6 @@ +use crate::{core::ValType, FuncType, GlobalType, IndexType, MemoryType, Mutability, TableType}; use wasmparser::AbstractHeapType; -use crate::{core::ValType, FuncType, GlobalType, MemoryType, Mutability, TableType}; - impl TableType { /// Creates a new [`TableType`] from the given `wasmparser` primitive. /// @@ -11,20 +10,13 @@ impl TableType { /// routine does not become part of the public API of [`TableType`]. pub(crate) fn from_wasmparser(table_type: wasmparser::TableType) -> Self { let element = WasmiValueType::from(table_type.element_type).into_inner(); - let minimum: u32 = table_type - .initial - .try_into() - .unwrap_or_else(|_err| panic!("out of bounds minimum value: {}", table_type.initial)); - let maximum: Option = match table_type.maximum { - Some(maximum) => { - let maximum = maximum - .try_into() - .unwrap_or_else(|_err| panic!("out of bounds maximum value: {}", maximum)); - Some(maximum) - } - None => None, + let minimum: u64 = table_type.initial; + let maximum: Option = table_type.maximum; + let index_ty = match table_type.table64 { + true => IndexType::I64, + false => IndexType::I32, }; - Self::new(element, minimum, maximum) + Self::new_impl(element, index_ty, minimum, maximum) } } @@ -36,36 +28,22 @@ impl MemoryType { /// We do not use the `From` trait here so that this conversion /// routine does not become part of the public API of [`MemoryType`]. pub(crate) fn from_wasmparser(memory_type: wasmparser::MemoryType) -> Self { - assert!( - !memory_type.memory64, - "wasmi does not support the `memory64` Wasm proposal" - ); assert!( !memory_type.shared, "wasmi does not support the `threads` Wasm proposal" ); - let minimum: u32 = memory_type - .initial - .try_into() - .expect("minimum linear memory pages must be a valid `u32`"); - let maximum: Option = memory_type - .maximum - .map(u32::try_from) - .transpose() - .expect("maximum linear memory pages must be a valid `u32` if any"); - let page_size_log2: Option = memory_type - .page_size_log2 - .map(u8::try_from) - .transpose() - .expect("page size (in log2) must be a valid `u8` if any"); let mut b = Self::builder(); - b.min(minimum); - b.max(maximum); - if let Some(page_size_log2) = page_size_log2 { + b.min(memory_type.initial); + b.max(memory_type.maximum); + b.memory64(memory_type.memory64); + if let Some(page_size_log2) = memory_type.page_size_log2 { + let Ok(page_size_log2) = u8::try_from(page_size_log2) else { + panic!("page size (in log2) must be a valid `u8` if any"); + }; b.page_size_log2(page_size_log2); } b.build() - .expect("encountered invalid wasmparser::MemoryType after validation") + .unwrap_or_else(|err| panic!("received invalid `MemoryType` from `wasmparser`: {err}")) } } diff --git a/crates/wasmi/src/table/error.rs b/crates/wasmi/src/table/error.rs index adbe2a5fdf..0b0a905e18 100644 --- a/crates/wasmi/src/table/error.rs +++ b/crates/wasmi/src/table/error.rs @@ -6,14 +6,22 @@ use core::{fmt, fmt::Display}; #[derive(Debug)] #[non_exhaustive] pub enum TableError { + /// Tried to allocate more virtual memory than technically possible. + OutOfSystemMemory, + /// The minimum size of the table type overflows the system index type. + MinimumSizeOverflow, + /// The maximum size of the table type overflows the system index type. + MaximumSizeOverflow, + /// If a resource limiter denied allocation or growth of a linear memory. + ResourceLimiterDeniedAllocation, /// Occurs when growing a table out of its set bounds. GrowOutOfBounds { /// The maximum allowed table size. - maximum: u32, + maximum: u64, /// The current table size before the growth operation. - current: u32, + current: u64, /// The amount of requested invalid growth. - delta: u32, + delta: u64, }, /// Occurs when operating with a [`Table`](crate::Table) and mismatching element types. ElementTypeMismatch { @@ -25,9 +33,9 @@ pub enum TableError { /// Occurs when accessing the table out of bounds. AccessOutOfBounds { /// The current size of the table. - current: u32, + current: u64, /// The accessed index that is out of bounds. - offset: u32, + index: u64, }, /// Occur when coping elements of tables out of bounds. CopyOutOfBounds, @@ -47,6 +55,21 @@ impl std::error::Error for TableError {} impl Display for TableError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { + Self::OutOfSystemMemory => { + write!( + f, + "tried to allocate more virtual memory than available on the system" + ) + } + Self::MinimumSizeOverflow => { + write!(f, "the minimum table size overflows the system bounds") + } + Self::MaximumSizeOverflow => { + write!(f, "the maximum table size overflows the system bounds") + } + Self::ResourceLimiterDeniedAllocation => { + write!(f, "a resource limiter denied to allocate or grow the table") + } Self::GrowOutOfBounds { maximum, current, @@ -61,7 +84,10 @@ impl Display for TableError { Self::ElementTypeMismatch { expected, actual } => { write!(f, "encountered mismatching table element type, expected {expected:?} but found {actual:?}") } - Self::AccessOutOfBounds { current, offset } => { + Self::AccessOutOfBounds { + current, + index: offset, + } => { write!( f, "out of bounds access of table element {offset} \ diff --git a/crates/wasmi/src/table/mod.rs b/crates/wasmi/src/table/mod.rs index c0e7dbdb24..69bdecf428 100644 --- a/crates/wasmi/src/table/mod.rs +++ b/crates/wasmi/src/table/mod.rs @@ -9,10 +9,11 @@ use crate::{ error::EntityGrowError, store::{Fuel, FuelError, ResourceLimiterRef}, value::WithType, + IndexType, Val, }; -use alloc::{vec, vec::Vec}; -use core::cmp::max; +use alloc::vec::Vec; +use core::{cmp::max, iter}; mod element; mod error; @@ -43,11 +44,13 @@ pub struct TableType { /// The type of values stored in the [`Table`]. element: ValType, /// The minimum number of elements the [`Table`] must have. - min: u32, + min: u64, /// The optional maximum number of elements the [`Table`] can have. /// /// If this is `None` then the [`Table`] is not limited in size. - max: Option, + max: Option, + /// The index type used by the [`Table`]. + index_ty: IndexType, } impl TableType { @@ -57,10 +60,54 @@ impl TableType { /// /// If `min` is greater than `max`. pub fn new(element: ValType, min: u32, max: Option) -> Self { - if let Some(max) = max { - assert!(min <= max); + Self::new_impl(element, IndexType::I32, u64::from(min), max.map(u64::from)) + } + + /// Creates a new [`TableType`] with a 64-bit index type. + /// + /// # Note + /// + /// 64-bit tables are part of the [Wasm `memory64` proposal]. + /// + /// [Wasm `memory64` proposal]: https://github.com/WebAssembly/memory64 + /// + /// # Panics + /// + /// If `min` is greater than `max`. + pub fn new64(element: ValType, min: u64, max: Option) -> Self { + Self::new_impl(element, IndexType::I64, min, max) + } + + /// Convenience constructor to create a new [`TableType`]. + pub(crate) fn new_impl( + element: ValType, + index_ty: IndexType, + min: u64, + max: Option, + ) -> Self { + let absolute_max = index_ty.max_size(); + assert!(u128::from(min) <= absolute_max); + max.inspect(|&max| { + assert!(min <= max && u128::from(max) <= absolute_max); + }); + Self { + element, + min, + max, + index_ty, } - Self { element, min, max } + } + + /// Returns `true` if this is a 64-bit [`TableType`]. + /// + /// 64-bit memories are part of the Wasm `memory64` proposal. + pub fn is_64(&self) -> bool { + self.index_ty.is_64() + } + + /// Returns the [`IndexType`] used by the [`TableType`]. + pub(crate) fn index_ty(&self) -> IndexType { + self.index_ty } /// Returns the [`ValType`] of elements stored in the [`Table`]. @@ -69,14 +116,14 @@ impl TableType { } /// Returns minimum number of elements the [`Table`] must have. - pub fn minimum(&self) -> u32 { + pub fn minimum(&self) -> u64 { self.min } /// The optional maximum number of elements the [`Table`] can have. /// /// If this returns `None` then the [`Table`] is not limited in size. - pub fn maximum(&self) -> Option { + pub fn maximum(&self) -> Option { self.max } @@ -123,6 +170,9 @@ impl TableType { /// [import subtyping]: /// https://webassembly.github.io/spec/core/valid/types.html#import-subtyping pub(crate) fn is_subtype_of(&self, other: &Self) -> bool { + if self.is_64() != other.is_64() { + return false; + } if self.matches_element_type(other.element()).is_err() { return false; } @@ -156,21 +206,26 @@ impl TableEntity { limiter: &mut ResourceLimiterRef<'_>, ) -> Result { ty.matches_element_type(init.ty())?; - + let Ok(min_size) = usize::try_from(ty.minimum()) else { + return Err(TableError::MinimumSizeOverflow); + }; + let Ok(max_size) = ty.maximum().map(usize::try_from).transpose() else { + return Err(TableError::MaximumSizeOverflow); + }; if let Some(limiter) = limiter.as_resource_limiter() { - if !limiter.table_growing(0, ty.minimum(), ty.maximum())? { - // Here there's no meaningful way to map Ok(false) to - // INVALID_GROWTH_ERRCODE, so we just translate it to an - // appropriate Err(...) - return Err(TableError::GrowOutOfBounds { - maximum: ty.maximum().unwrap_or(u32::MAX), - current: 0, - delta: ty.minimum(), - }); + if !limiter.table_growing(0, min_size, max_size)? { + return Err(TableError::ResourceLimiterDeniedAllocation); } } - - let elements = vec![init.into(); ty.minimum() as usize]; + let mut elements = Vec::new(); + if elements.try_reserve(min_size).is_err() { + let error = TableError::OutOfSystemMemory; + if let Some(limiter) = limiter.as_resource_limiter() { + limiter.table_grow_failed(&error) + } + return Err(error); + }; + elements.extend(iter::repeat_n::(init.into(), min_size)); Ok(Self { ty, elements }) } @@ -186,12 +241,21 @@ impl TableEntity { /// This respects the current size of the [`TableEntity`] /// as its minimum size and is useful for import subtyping checks. pub fn dynamic_ty(&self) -> TableType { - TableType::new(self.ty().element(), self.size(), self.ty().maximum()) + TableType::new_impl( + self.ty().element(), + self.ty().index_ty, + self.size(), + self.ty().maximum(), + ) } /// Returns the current size of the [`Table`]. - pub fn size(&self) -> u32 { - self.elements.len() as u32 + pub fn size(&self) -> u64 { + let len = self.elements.len(); + let Ok(len64) = u64::try_from(len) else { + panic!("table.size is out of system bounds: {len}"); + }; + len64 } /// Grows the table by the given amount of elements. @@ -208,11 +272,11 @@ impl TableEntity { /// - If `value` does not match the [`Table`] element type. pub fn grow( &mut self, - delta: u32, + delta: u64, init: Val, fuel: Option<&mut Fuel>, limiter: &mut ResourceLimiterRef<'_>, - ) -> Result { + ) -> Result { self.ty() .matches_element_type(init.ty()) .map_err(|_| EntityGrowError::InvalidGrow)?; @@ -234,50 +298,62 @@ impl TableEntity { /// If the table is grown beyond its maximum limits. pub fn grow_untyped( &mut self, - delta: u32, + delta: u64, init: UntypedVal, fuel: Option<&mut Fuel>, limiter: &mut ResourceLimiterRef<'_>, - ) -> Result { + ) -> Result { + let Ok(delta_size) = usize::try_from(delta) else { + return Err(EntityGrowError::InvalidGrow); + }; + let Some(desired) = self.size().checked_add(delta) else { + return Err(EntityGrowError::InvalidGrow); + }; + // We need to divide the `max_size` (in bytes) by 8 because each table element requires 8 bytes. + let max_size = self.ty.index_ty.max_size() / 8; + if u128::from(desired) > max_size { + return Err(EntityGrowError::InvalidGrow); + } + let current = self.elements.len(); + let Ok(desired) = usize::try_from(desired) else { + return Err(EntityGrowError::InvalidGrow); + }; + let Ok(maximum) = self.ty.maximum().map(usize::try_from).transpose() else { + return Err(EntityGrowError::InvalidGrow); + }; + // ResourceLimiter gets first look at the request. - let current = self.size(); - let desired = current.checked_add(delta); - let maximum = self.ty.maximum(); if let Some(limiter) = limiter.as_resource_limiter() { - match limiter.table_growing(current, desired.unwrap_or(u32::MAX), maximum) { + match limiter.table_growing(current, desired, maximum) { Ok(true) => (), Ok(false) => return Err(EntityGrowError::InvalidGrow), Err(_) => return Err(EntityGrowError::TrapCode(TrapCode::GrowthOperationLimited)), } } - - let maximum = maximum.unwrap_or(u32::MAX); let notify_limiter = - |limiter: &mut ResourceLimiterRef<'_>| -> Result { + |limiter: &mut ResourceLimiterRef<'_>| -> Result { if let Some(limiter) = limiter.as_resource_limiter() { - limiter.table_grow_failed(&TableError::GrowOutOfBounds { - maximum, - current, - delta, - }); + limiter.table_grow_failed(&TableError::OutOfSystemMemory); } Err(EntityGrowError::InvalidGrow) }; - - let Some(desired) = desired else { - return notify_limiter(limiter); - }; - if desired > maximum { - return notify_limiter(limiter); + if let Some(maximum) = maximum { + if desired > maximum { + return notify_limiter(limiter); + } } if let Some(fuel) = fuel { - match fuel.consume_fuel(|costs| costs.fuel_for_copies(u64::from(delta))) { + match fuel.consume_fuel(|costs| costs.fuel_for_copies(delta)) { Ok(_) | Err(FuelError::FuelMeteringDisabled) => {} Err(FuelError::OutOfFuel) => return notify_limiter(limiter), } } - self.elements.resize(desired as usize, init); - Ok(current) + if self.elements.try_reserve(delta_size).is_err() { + return notify_limiter(limiter); + } + let size_before = self.size(); + self.elements.resize(desired, init); + Ok(size_before) } /// Converts the internal [`UntypedVal`] into a [`Val`] for this [`Table`] element type. @@ -288,7 +364,7 @@ impl TableEntity { /// Returns the [`Table`] element value at `index`. /// /// Returns `None` if `index` is out of bounds. - pub fn get(&self, index: u32) -> Option { + pub fn get(&self, index: u64) -> Option { self.get_untyped(index) .map(|untyped| self.make_typed(untyped)) } @@ -301,8 +377,9 @@ impl TableEntity { /// /// This is a more efficient version of [`Table::get`] for /// internal use only. - pub fn get_untyped(&self, index: u32) -> Option { - self.elements.get(index as usize).copied() + pub fn get_untyped(&self, index: u64) -> Option { + let index = usize::try_from(index).ok()?; + self.elements.get(index).copied() } /// Sets the [`Val`] of this [`Table`] at `index`. @@ -311,7 +388,7 @@ impl TableEntity { /// /// - If `index` is out of bounds. /// - If `value` does not match the [`Table`] element type. - pub fn set(&mut self, index: u32, value: Val) -> Result<(), TableError> { + pub fn set(&mut self, index: u64, value: Val) -> Result<(), TableError> { self.ty().matches_element_type(value.ty())?; self.set_untyped(index, value.into()) } @@ -321,15 +398,12 @@ impl TableEntity { /// # Errors /// /// If `index` is out of bounds. - pub fn set_untyped(&mut self, index: u32, value: UntypedVal) -> Result<(), TableError> { + pub fn set_untyped(&mut self, index: u64, value: UntypedVal) -> Result<(), TableError> { let current = self.size(); - let untyped = - self.elements - .get_mut(index as usize) - .ok_or(TableError::AccessOutOfBounds { - current, - offset: index, - })?; + let untyped = self + .elements + .get_mut(index as usize) + .ok_or(TableError::AccessOutOfBounds { current, index })?; *untyped = value; Ok(()) } @@ -347,7 +421,7 @@ impl TableEntity { pub fn init( &mut self, element: &ElementSegmentEntity, - dst_index: u32, + dst_index: u64, src_index: u32, len: u32, fuel: Option<&mut Fuel>, @@ -361,28 +435,34 @@ impl TableEntity { .matches_element_type(element.ty()) .map_err(|_| TrapCode::BadSignature)?; // Convert parameters to indices. - let dst_index = dst_index as usize; - let src_index = src_index as usize; - let len = len as usize; + let Ok(dst_index) = usize::try_from(dst_index) else { + return Err(TrapCode::TableOutOfBounds); + }; + let Ok(src_index) = usize::try_from(src_index) else { + return Err(TrapCode::TableOutOfBounds); + }; + let Ok(len_size) = usize::try_from(len) else { + return Err(TrapCode::TableOutOfBounds); + }; // Perform bounds check before anything else. let dst_items = self .elements .get_mut(dst_index..) - .and_then(|items| items.get_mut(..len)) + .and_then(|items| items.get_mut(..len_size)) .ok_or(TrapCode::TableOutOfBounds)?; let src_items = element .items() .get(src_index..) - .and_then(|items| items.get(..len)) + .and_then(|items| items.get(..len_size)) .ok_or(TrapCode::TableOutOfBounds)?; if len == 0 { // Bail out early if nothing needs to be initialized. // The Wasm spec demands to still perform the bounds check - // so we cannot bail out earlier. + // so we cannot bail out earlie64 return Ok(()); } if let Some(fuel) = fuel { - fuel.consume_fuel_if(|costs| costs.fuel_for_copies(len as u64))?; + fuel.consume_fuel_if(|costs| costs.fuel_for_copies(u64::from(len)))?; } // Perform the actual table initialization. dst_items.copy_from_slice(src_items); @@ -398,29 +478,35 @@ impl TableEntity { /// destination tables. pub fn copy( dst_table: &mut Self, - dst_index: u32, + dst_index: u64, src_table: &Self, - src_index: u32, - len: u32, + src_index: u64, + len: u64, fuel: Option<&mut Fuel>, ) -> Result<(), TrapCode> { // Turn parameters into proper slice indices. - let src_index = src_index as usize; - let dst_index = dst_index as usize; - let len = len as usize; + let Ok(src_index) = usize::try_from(src_index) else { + return Err(TrapCode::TableOutOfBounds); + }; + let Ok(dst_index) = usize::try_from(dst_index) else { + return Err(TrapCode::TableOutOfBounds); + }; + let Ok(len_size) = usize::try_from(len) else { + return Err(TrapCode::TableOutOfBounds); + }; // Perform bounds check before anything else. let dst_items = dst_table .elements .get_mut(dst_index..) - .and_then(|items| items.get_mut(..len)) + .and_then(|items| items.get_mut(..len_size)) .ok_or(TrapCode::TableOutOfBounds)?; let src_items = src_table .elements .get(src_index..) - .and_then(|items| items.get(..len)) + .and_then(|items| items.get(..len_size)) .ok_or(TrapCode::TableOutOfBounds)?; if let Some(fuel) = fuel { - fuel.consume_fuel_if(|costs| costs.fuel_for_copies(len as u64))?; + fuel.consume_fuel_if(|costs| costs.fuel_for_copies(len))?; } // Finally, copy elements in-place for the table. dst_items.copy_from_slice(src_items); @@ -434,9 +520,9 @@ impl TableEntity { /// Returns an error if the range is out of bounds of the table. pub fn copy_within( &mut self, - dst_index: u32, - src_index: u32, - len: u32, + dst_index: u64, + src_index: u64, + len: u64, fuel: Option<&mut Fuel>, ) -> Result<(), TrapCode> { // These accesses just perform the bounds checks required by the Wasm spec. @@ -446,15 +532,21 @@ impl TableEntity { .filter(|&offset| offset <= self.size()) .ok_or(TrapCode::TableOutOfBounds)?; // Turn parameters into proper indices. - let src_index = src_index as usize; - let dst_index = dst_index as usize; - let len = len as usize; + let Ok(src_index) = usize::try_from(src_index) else { + return Err(TrapCode::TableOutOfBounds); + }; + let Ok(dst_index) = usize::try_from(dst_index) else { + return Err(TrapCode::TableOutOfBounds); + }; + let Ok(len_size) = usize::try_from(len) else { + return Err(TrapCode::TableOutOfBounds); + }; if let Some(fuel) = fuel { - fuel.consume_fuel_if(|costs| costs.fuel_for_copies(len as u64))?; + fuel.consume_fuel_if(|costs| costs.fuel_for_copies(len))?; } // Finally, copy elements in-place for the table. self.elements - .copy_within(src_index..src_index.wrapping_add(len), dst_index); + .copy_within(src_index..src_index.wrapping_add(len_size), dst_index); Ok(()) } @@ -473,9 +565,9 @@ impl TableEntity { /// [`Store`]: [`crate::Store`] pub fn fill( &mut self, - dst: u32, + dst: u64, val: Val, - len: u32, + len: u64, fuel: Option<&mut Fuel>, ) -> Result<(), TrapCode> { self.ty() @@ -501,20 +593,24 @@ impl TableEntity { /// [`Store`]: [`crate::Store`] pub fn fill_untyped( &mut self, - dst: u32, + dst: u64, val: UntypedVal, - len: u32, + len: u64, fuel: Option<&mut Fuel>, ) -> Result<(), TrapCode> { - let dst_index = dst as usize; - let len = len as usize; + let Ok(dst_index) = usize::try_from(dst) else { + return Err(TrapCode::TableOutOfBounds); + }; + let Ok(len_size) = usize::try_from(len) else { + return Err(TrapCode::TableOutOfBounds); + }; let dst = self .elements .get_mut(dst_index..) - .and_then(|elements| elements.get_mut(..len)) + .and_then(|elements| elements.get_mut(..len_size)) .ok_or(TrapCode::TableOutOfBounds)?; if let Some(fuel) = fuel { - fuel.consume_fuel_if(|costs| costs.fuel_for_copies(len as u64))?; + fuel.consume_fuel_if(|costs| costs.fuel_for_copies(len))?; } dst.fill(val); Ok(()) @@ -584,7 +680,7 @@ impl Table { /// # Panics /// /// If `ctx` does not own this [`Table`]. - pub fn size(&self, ctx: impl AsContext) -> u32 { + pub fn size(&self, ctx: impl AsContext) -> u64 { ctx.as_context().store.inner.resolve_table(self).size() } @@ -607,16 +703,16 @@ impl Table { pub fn grow( &self, mut ctx: impl AsContextMut, - delta: u32, + delta: u64, init: Val, - ) -> Result { + ) -> Result { let (inner, mut limiter) = ctx .as_context_mut() .store .store_inner_and_resource_limiter_ref(); let table = inner.resolve_table_mut(self); let current = table.size(); - let maximum = table.ty().maximum().unwrap_or(u32::MAX); + let maximum = table.ty().maximum().unwrap_or(u64::MAX); table .grow(delta, init, None, &mut limiter) .map_err(|_| TableError::GrowOutOfBounds { @@ -633,7 +729,7 @@ impl Table { /// # Panics /// /// Panics if `ctx` does not own this [`Table`]. - pub fn get(&self, ctx: impl AsContext, index: u32) -> Option { + pub fn get(&self, ctx: impl AsContext, index: u64) -> Option { ctx.as_context().store.inner.resolve_table(self).get(index) } @@ -650,7 +746,7 @@ impl Table { pub fn set( &self, mut ctx: impl AsContextMut, - index: u32, + index: u64, value: Val, ) -> Result<(), TableError> { ctx.as_context_mut() @@ -685,10 +781,10 @@ impl Table { pub fn copy( mut store: impl AsContextMut, dst_table: &Table, - dst_index: u32, + dst_index: u64, src_table: &Table, - src_index: u32, - len: u32, + src_index: u64, + len: u64, ) -> Result<(), TableError> { if Self::eq(dst_table, src_table) { // The `dst_table` and `src_table` are the same table @@ -733,9 +829,9 @@ impl Table { pub fn fill( &self, mut ctx: impl AsContextMut, - dst: u32, + dst: u64, val: Val, - len: u32, + len: u64, ) -> Result<(), TrapCode> { ctx.as_context_mut() .store diff --git a/crates/wasmi/tests/spec/mod.rs b/crates/wasmi/tests/spec/mod.rs index ce974b3d18..d29b699914 100644 --- a/crates/wasmi/tests/spec/mod.rs +++ b/crates/wasmi/tests/spec/mod.rs @@ -60,7 +60,8 @@ fn mvp_config() -> Config { .wasm_saturating_float_to_int(false) .wasm_sign_extension(false) .wasm_multi_value(false) - .wasm_multi_memory(false); + .wasm_multi_memory(false) + .wasm_memory64(false); config } @@ -284,6 +285,32 @@ macro_rules! expand_tests_cps { fn wasm_custom_page_sizes("proposals/custom-page-sizes/custom-page-sizes"); fn wasm_custom_page_sizes_invalid("proposals/custom-page-sizes/custom-page-sizes-invalid"); fn wasm_custom_page_sizes_memory_max("proposals/custom-page-sizes/memory_max"); + fn wasm_custom_page_sizes_memory_max64("proposals/custom-page-sizes/memory_max_i64"); + } + }; +} + +macro_rules! expand_tests_memory64 { + ( $mac:ident, $( $args:tt )* ) => { + $mac! { + $( $args )* + + fn wasm_address64("memory64/address64"); + fn wasm_align64("memory64/align64"); + fn wasm_call_indirect64("memory64/call_indirect"); + fn wasm_endianness64("memory64/endianness64"); + fn wasm_float_memory64("memory64/float_memory64"); + fn wasm_load64("memory64/load64"); + fn wasm_memory_grow64("memory64/memory_grow64"); + fn wasm_memory_trap64("memory64/memory_trap64"); + fn wasm_memory_redundancy64("memory64/memory_redundancy64"); + fn wasm_memory64("memory64/memory64"); + fn wasm_memory_copy64("memory64/memory_copy"); + fn wasm_memory_fill64("memory64/memory_fill"); + fn wasm_memory_init64("memory64/memory_init"); + fn wasm_imports64("memory64/imports"); + fn wasm_table64("memory64/table"); + fn wasm_table_copy_mixed("memory64/table_copy_mixed"); } }; } @@ -306,13 +333,20 @@ mod blobs { let folder = "testsuite"; } + + expand_tests_memory64! { + include_wasm_blobs, + + let folder = "wasmi-tests"; + } } mod multi_memory { use super::*; fn test_config() -> RunnerConfig { - let config = Config::default(); + let mut config = Config::default(); + config.wasm_memory64(false); let parsing_mode = ParsingMode::Buffered; RunnerConfig { config, @@ -349,6 +383,27 @@ mod custom_page_sizes { } } +mod memory64 { + use super::*; + + fn test_config() -> RunnerConfig { + let mut config = Config::default(); + config.wasm_memory64(true); + let parsing_mode = ParsingMode::Buffered; + RunnerConfig { + config, + parsing_mode, + } + } + + expand_tests_memory64! { + define_spec_tests, + + let config = test_config(); + let runner = process_wast; + } +} + mod fueled { use super::*; diff --git a/crates/wasmi/tests/spec/wasmi-tests b/crates/wasmi/tests/spec/wasmi-tests new file mode 160000 index 0000000000..55f0de1310 --- /dev/null +++ b/crates/wasmi/tests/spec/wasmi-tests @@ -0,0 +1 @@ +Subproject commit 55f0de1310cd7cdf2080e4ec8883a0088887c28f diff --git a/crates/wast/src/lib.rs b/crates/wast/src/lib.rs index 3cf2ddb99f..1cb8542105 100644 --- a/crates/wast/src/lib.rs +++ b/crates/wast/src/lib.rs @@ -95,6 +95,11 @@ impl WastRunner { TableType::new(ValType::FuncRef, 10, Some(20)), Val::default(ValType::FuncRef), )?; + let table64 = Table::new( + &mut *store, + TableType::new64(ValType::FuncRef, 0, None), + Val::default(ValType::FuncRef), + )?; let global_i32 = Global::new(&mut *store, Val::I32(666), Mutability::Const); let global_i64 = Global::new(&mut *store, Val::I64(666), Mutability::Const); let global_f32 = Global::new( @@ -110,6 +115,7 @@ impl WastRunner { self.linker.define("spectest", "memory", default_memory)?; self.linker.define("spectest", "table", default_table)?; + self.linker.define("spectest", "table64", table64)?; self.linker.define("spectest", "global_i32", global_i32)?; self.linker.define("spectest", "global_i64", global_i64)?; self.linker.define("spectest", "global_f32", global_f32)?;