diff --git a/crates/ir/src/enum.rs b/crates/ir/src/enum.rs index cfe3f4bcdf..d8ab65f556 100644 --- a/crates/ir/src/enum.rs +++ b/crates/ir/src/enum.rs @@ -31,8 +31,8 @@ macro_rules! define_enum { /// This type represents all such words and for simplicity we call the type [`Instruction`], still. /// /// Most instructions are composed of a single instruction word. An example of - /// this is [`Instruction::I32Add`]. However, some instructions, like - /// [`Instruction::Select`], are composed of two or more instruction words. + /// this is [`Instruction::I32Add`]. However, some instructions, like the `select` instructions + /// are composed of two or more instruction words. /// /// The Wasmi bytecode translation makes sure that instructions always appear in valid sequences. /// The Wasmi executor relies on the guarantees that the Wasmi translator provides. diff --git a/crates/ir/src/for_each_op.rs b/crates/ir/src/for_each_op.rs index aa773404c5..8238e2fd79 100644 --- a/crates/ir/src/for_each_op.rs +++ b/crates/ir/src/for_each_op.rs @@ -1358,115 +1358,890 @@ macro_rules! for_each_op_grouped { func_type: FuncType, }, - /// A Wasm `select` equivalent Wasmi instruction. + /// A fused `i32.and` and `select` instruction. /// /// # Encoding /// - /// Must be followed by [`Instruction::Register2`] to encode `condition` and `rhs`. - #[snake_name(select)] - Select { + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i32_and)] + SelectI32And { @result: Reg, - /// The register holding the left-hand side value. + /// The left-hand side operand to the branch conditional. + lhs: Reg, + /// The right-hand side operand to the branch conditional. + rhs: Reg, + }, + /// A fused `i32.and` and `select` instruction with 16-bit immediate `rhs` value. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i32_and_imm16)] + SelectI32AndImm16 { + @result: Reg, + /// The left-hand side operand to the conditional operator. + lhs: Reg, + /// The right-hand side operand to the conditional operator. + rhs: Const16, + }, + /// A fused `i32.or` and `select` instruction. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i32_or)] + SelectI32Or { + @result: Reg, + /// The left-hand side operand to the branch conditional. + lhs: Reg, + /// The right-hand side operand to the branch conditional. + rhs: Reg, + }, + /// A fused `i32.or` and `select` instruction with 16-bit immediate `rhs` value. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i32_or_imm16)] + SelectI32OrImm16 { + @result: Reg, + /// The left-hand side operand to the conditional operator. + lhs: Reg, + /// The right-hand side operand to the conditional operator. + rhs: Const16, + }, + /// A fused `i32.xor` and `select` instruction. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i32_xor)] + SelectI32Xor { + @result: Reg, + /// The left-hand side operand to the branch conditional. + lhs: Reg, + /// The right-hand side operand to the branch conditional. + rhs: Reg, + }, + /// A fused `i32.xor` and `select` instruction with 16-bit immediate `rhs` value. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i32_xor_imm16)] + SelectI32XorImm16 { + @result: Reg, + /// The left-hand side operand to the conditional operator. + lhs: Reg, + /// The right-hand side operand to the conditional operator. + rhs: Const16, + }, + /// A fused `i32.eqz(i32.`select`)` and branch instruction. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i32_nand)] + SelectI32Nand { + @result: Reg, + /// The left-hand side operand to the branch conditional. + lhs: Reg, + /// The right-hand side operand to the branch conditional. + rhs: Reg, + }, + /// A fused `i32.eqz(i32.`select`)` and branch instruction with 16-bit immediate `rhs` value. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i32_nand_imm16)] + SelectI32NandImm16 { + @result: Reg, + /// The left-hand side operand to the conditional operator. + lhs: Reg, + /// The right-hand side operand to the conditional operator. + rhs: Const16, + }, + /// A fused `i32.eqz(i32.`select`)` and branch instruction. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i32_nor)] + SelectI32Nor { + @result: Reg, + /// The left-hand side operand to the branch conditional. + lhs: Reg, + /// The right-hand side operand to the branch conditional. + rhs: Reg, + }, + /// A fused `i32.eqz(i32.`select`)` and branch instruction with 16-bit immediate `rhs` value. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i32_nor_imm16)] + SelectI32NorImm16 { + @result: Reg, + /// The left-hand side operand to the conditional operator. + lhs: Reg, + /// The right-hand side operand to the conditional operator. + rhs: Const16, + }, + /// A fused `i32.eqz(i32.`select`)` and branch instruction. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i32_xnor)] + SelectI32Xnor { + @result: Reg, + /// The left-hand side operand to the branch conditional. + lhs: Reg, + /// The right-hand side operand to the branch conditional. + rhs: Reg, + }, + /// A fused `i32.eqz(i32.`select`)` and branch instruction with 16-bit immediate `rhs` value. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i32_xnor_imm16)] + SelectI32XnorImm16 { + @result: Reg, + /// The left-hand side operand to the conditional operator. + lhs: Reg, + /// The right-hand side operand to the conditional operator. + rhs: Const16, + }, + /// A fused `i32.eq` and `select` instruction. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i32_eq)] + SelectI32Eq { + @result: Reg, + /// The left-hand side operand to the branch conditional. + lhs: Reg, + /// The right-hand side operand to the branch conditional. + rhs: Reg, + }, + /// A fused `i32.eq` and `select` instruction with 16-bit immediate `rhs` value. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i32_eq_imm16)] + SelectI32EqImm16 { + @result: Reg, + /// The left-hand side operand to the conditional operator. + lhs: Reg, + /// The right-hand side operand to the conditional operator. + rhs: Const16, + }, + /// A fused `i32.ne` and `select` instruction. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i32_ne)] + SelectI32Ne { + @result: Reg, + /// The left-hand side operand to the branch conditional. + lhs: Reg, + /// The right-hand side operand to the branch conditional. + rhs: Reg, + }, + /// A fused `i32.ne` and `select` instruction with 16-bit immediate `rhs` value. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i32_ne_imm16)] + SelectI32NeImm16 { + @result: Reg, + /// The left-hand side operand to the conditional operator. + lhs: Reg, + /// The right-hand side operand to the conditional operator. + rhs: Const16, + }, + /// A fused `i32.lt_s` and `select` instruction. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i32_lt_s)] + SelectI32LtS { + @result: Reg, + /// The left-hand side operand to the branch conditional. + lhs: Reg, + /// The right-hand side operand to the branch conditional. + rhs: Reg, + }, + /// A fused `i32.lt_s` and `select` instruction with 16-bit immediate `lhs` value. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i32_lt_s_imm16_lhs)] + SelectI32LtSImm16Lhs { + @result: Reg, + /// The right-hand side operand to the conditional operator. + lhs: Const16, + /// The left-hand side operand to the conditional operator. + rhs: Reg, + }, + /// A fused `i32.lt_s` and `select` instruction with 16-bit immediate `rhs` value. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i32_lt_s_imm16_rhs)] + SelectI32LtSImm16Rhs { + @result: Reg, + /// The left-hand side operand to the conditional operator. + lhs: Reg, + /// The right-hand side operand to the conditional operator. + rhs: Const16, + }, + /// A fused `i32.lt_u` and `select` instruction. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i32_lt_u)] + SelectI32LtU { + @result: Reg, + /// The left-hand side operand to the branch conditional. + lhs: Reg, + /// The right-hand side operand to the branch conditional. + rhs: Reg, + }, + /// A fused `i32.lt_u` and `select` instruction with 16-bit immediate `lhs` value. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i32_lt_u_imm16_lhs)] + SelectI32LtUImm16Lhs { + @result: Reg, + /// The right-hand side operand to the conditional operator. + lhs: Const16, + /// The left-hand side operand to the conditional operator. + rhs: Reg, + }, + /// A fused `i32.lt_u` and `select` instruction with 16-bit immediate `rhs` value. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i32_lt_u_imm16_rhs)] + SelectI32LtUImm16Rhs { + @result: Reg, + /// The left-hand side operand to the conditional operator. + lhs: Reg, + /// The right-hand side operand to the conditional operator. + rhs: Const16, + }, + /// A fused `i32.le_s` and `select` instruction. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i32_le_s)] + SelectI32LeS { + @result: Reg, + /// The left-hand side operand to the branch conditional. + lhs: Reg, + /// The right-hand side operand to the branch conditional. + rhs: Reg, + }, + /// A fused `i32.le_s` and `select` instruction with 16-bit immediate `lhs` value. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i32_le_s_imm16_lhs)] + SelectI32LeSImm16Lhs { + @result: Reg, + /// The right-hand side operand to the conditional operator. + lhs: Const16, + /// The left-hand side operand to the conditional operator. + rhs: Reg, + }, + /// A fused `i32.le_s` and `select` instruction with 16-bit immediate `rhs` value. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i32_le_s_imm16_rhs)] + SelectI32LeSImm16Rhs { + @result: Reg, + /// The left-hand side operand to the conditional operator. + lhs: Reg, + /// The right-hand side operand to the conditional operator. + rhs: Const16, + }, + /// A fused `i32.le_u` and `select` instruction. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i32_le_u)] + SelectI32LeU { + @result: Reg, + /// The left-hand side operand to the branch conditional. + lhs: Reg, + /// The right-hand side operand to the branch conditional. + rhs: Reg, + }, + /// A fused `i32.le_u` and `select` instruction with 16-bit immediate `lhs` value. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i32_le_u_imm16_lhs)] + SelectI32LeUImm16Lhs { + @result: Reg, + /// The right-hand side operand to the conditional operator. + lhs: Const16, + /// The left-hand side operand to the conditional operator. + rhs: Reg, + }, + /// A fused `i32.le_u` and `select` instruction with 16-bit immediate `rhs` value. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i32_le_u_imm16_rhs)] + SelectI32LeUImm16Rhs { + @result: Reg, + /// The left-hand side operand to the conditional operator. + lhs: Reg, + /// The right-hand side operand to the conditional operator. + rhs: Const16, + }, + /// A fused `i64.and` and `select` instruction. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i64_and)] + SelectI64And { + @result: Reg, + /// The left-hand side operand to the branch conditional. + lhs: Reg, + /// The right-hand side operand to the branch conditional. + rhs: Reg, + }, + /// A fused `i64.and` and `select` instruction with 16-bit immediate `rhs` value. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i64_and_imm16)] + SelectI64AndImm16 { + @result: Reg, + /// The left-hand side operand to the conditional operator. + lhs: Reg, + /// The right-hand side operand to the conditional operator. + rhs: Const16, + }, + /// A fused `i64.or` and `select` instruction. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i64_or)] + SelectI64Or { + @result: Reg, + /// The left-hand side operand to the branch conditional. + lhs: Reg, + /// The right-hand side operand to the branch conditional. + rhs: Reg, + }, + /// A fused `i64.or` and `select` instruction with 16-bit immediate `rhs` value. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i64_or_imm16)] + SelectI64OrImm16 { + @result: Reg, + /// The left-hand side operand to the conditional operator. + lhs: Reg, + /// The right-hand side operand to the conditional operator. + rhs: Const16, + }, + /// A fused `i64.xor` and `select` instruction. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i64_xor)] + SelectI64Xor { + @result: Reg, + /// The left-hand side operand to the branch conditional. + lhs: Reg, + /// The right-hand side operand to the branch conditional. + rhs: Reg, + }, + /// A fused `i64.xor` and `select` instruction with 16-bit immediate `rhs` value. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i64_xor_imm16)] + SelectI64XorImm16 { + @result: Reg, + /// The left-hand side operand to the conditional operator. + lhs: Reg, + /// The right-hand side operand to the conditional operator. + rhs: Const16, + }, + /// A fused `i64.eqz(i64.`select`)` and branch instruction. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i64_nand)] + SelectI64Nand { + @result: Reg, + /// The left-hand side operand to the branch conditional. + lhs: Reg, + /// The right-hand side operand to the branch conditional. + rhs: Reg, + }, + /// A fused `i64.eqz(i64.`select`)` and branch instruction with 16-bit immediate `rhs` value. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i64_nand_imm16)] + SelectI64NandImm16 { + @result: Reg, + /// The left-hand side operand to the conditional operator. lhs: Reg, + /// The right-hand side operand to the conditional operator. + rhs: Const16, }, - /// A Wasm `select` equivalent Wasmi instruction with 32-bit immediate `rhs` value. + /// A fused `i64.eqz(i64.`select`)` and branch instruction. /// /// # Encoding /// - /// Must be followed by [`Instruction::RegisterAndImm32`] to encode `condition` and `rhs`. - #[snake_name(select_imm32_rhs)] - SelectImm32Rhs { + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i64_nor)] + SelectI64Nor { @result: Reg, - /// The register holding the left-hand side value. + /// The left-hand side operand to the branch conditional. lhs: Reg, + /// The right-hand side operand to the branch conditional. + rhs: Reg, }, - /// A Wasm `select` equivalent Wasmi instruction with 32-bit immediate `lhs` value. + /// A fused `i64.eqz(i64.`select`)` and branch instruction with 16-bit immediate `rhs` value. /// /// # Encoding /// - /// Must be followed by [`Instruction::Register2`] to encode `condition` and `lhs`. - #[snake_name(select_imm32_lhs)] - SelectImm32Lhs { + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i64_nor_imm16)] + SelectI64NorImm16 { @result: Reg, - /// The register holding the left-hand side value. - lhs: AnyConst32, + /// The left-hand side operand to the conditional operator. + lhs: Reg, + /// The right-hand side operand to the conditional operator. + rhs: Const16, }, - /// A Wasm `select` equivalent Wasmi instruction with 32-bit immediate `lhs` and `rhs` values. + /// A fused `i64.eqz(i64.`select`)` and branch instruction. /// /// # Encoding /// - /// Must be followed by [`Instruction::RegisterAndImm32`] to encode `condition` and `rhs`. - #[snake_name(select_imm32)] - SelectImm32 { + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i64_xnor)] + SelectI64Xnor { @result: Reg, - /// The register holding the left-hand side value. - lhs: AnyConst32, + /// The left-hand side operand to the branch conditional. + lhs: Reg, + /// The right-hand side operand to the branch conditional. + rhs: Reg, }, - /// A Wasm `select` equivalent Wasmi instruction with 32-bit encoded `i64` immediate `lhs` value. + /// A fused `i64.eqz(i64.`select`)` and branch instruction with 16-bit immediate `rhs` value. /// /// # Encoding /// - /// Must be followed by [`Instruction::RegisterAndImm32`] to encode `condition` and `rhs`. - #[snake_name(select_i64imm32_rhs)] - SelectI64Imm32Rhs { + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i64_xnor_imm16)] + SelectI64XnorImm16 { @result: Reg, - /// The register holding the left-hand side value. + /// The left-hand side operand to the conditional operator. lhs: Reg, + /// The right-hand side operand to the conditional operator. + rhs: Const16, }, - /// A Wasm `select` equivalent Wasmi instruction with 32-bit encoded `i64` immediate `lhs` value. + /// A fused `i64.eq` and `select` instruction. /// /// # Encoding /// - /// Must be followed by [`Instruction::Register2`] to encode `condition` and `rhs`. - #[snake_name(select_i64imm32_lhs)] - SelectI64Imm32Lhs { + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i64_eq)] + SelectI64Eq { @result: Reg, - /// The register holding the left-hand side value. - lhs: Const32, + /// The left-hand side operand to the branch conditional. + lhs: Reg, + /// The right-hand side operand to the branch conditional. + rhs: Reg, }, - /// A Wasm `select` equivalent Wasmi instruction with 32-bit encoded `i64` immediate `lhs` and `rhs` values. + /// A fused `i64.eq` and `select` instruction with 16-bit immediate `rhs` value. /// /// # Encoding /// - /// Must be followed by [`Instruction::RegisterAndImm32`] to encode `condition` and `rhs`. - #[snake_name(select_i64imm32)] - SelectI64Imm32 { + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i64_eq_imm16)] + SelectI64EqImm16 { @result: Reg, - /// The register holding the left-hand side value. - lhs: Const32, + /// The left-hand side operand to the conditional operator. + lhs: Reg, + /// The right-hand side operand to the conditional operator. + rhs: Const16, }, - /// A Wasm `select` equivalent Wasmi instruction with 32-bit encoded `f64` immediate `rhs` value. + /// A fused `i64.ne` and `select` instruction. /// /// # Encoding /// - /// Must be followed by [`Instruction::RegisterAndImm32`] to encode `condition` and `rhs`. - #[snake_name(select_f64imm32_rhs)] - SelectF64Imm32Rhs { + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i64_ne)] + SelectI64Ne { @result: Reg, - /// The register holding the left-hand side value. + /// The left-hand side operand to the branch conditional. lhs: Reg, + /// The right-hand side operand to the branch conditional. + rhs: Reg, }, - /// A Wasm `select` equivalent Wasmi instruction with 32-bit encoded `f64` immediate `lhs` value. + /// A fused `i64.ne` and `select` instruction with 16-bit immediate `rhs` value. /// /// # Encoding /// - /// Must be followed by [`Instruction::Register2`] to encode `condition` and `rhs`. - #[snake_name(select_f64imm32_lhs)] - SelectF64Imm32Lhs { + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i64_ne_imm16)] + SelectI64NeImm16 { @result: Reg, - /// The register holding the left-hand side value. - lhs: Const32, + /// The left-hand side operand to the conditional operator. + lhs: Reg, + /// The right-hand side operand to the conditional operator. + rhs: Const16, }, - /// A Wasm `select` equivalent Wasmi instruction with 32-bit encoded `f64` immediate `lhs` and `rhs` value. + /// A fused `i64.lt_s` and `select` instruction. /// /// # Encoding /// - /// Must be followed by [`Instruction::RegisterAndImm32`] to encode `condition` and `rhs`. - #[snake_name(select_f64imm32)] - SelectF64Imm32 { + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i64_lt_s)] + SelectI64LtS { @result: Reg, - /// The register holding the left-hand side value. - lhs: Const32, + /// The left-hand side operand to the branch conditional. + lhs: Reg, + /// The right-hand side operand to the branch conditional. + rhs: Reg, + }, + /// A fused `i64.lt_s` and `select` instruction with 16-bit immediate `lhs` value. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i64_lt_s_imm16_lhs)] + SelectI64LtSImm16Lhs { + @result: Reg, + /// The right-hand side operand to the conditional operator. + lhs: Const16, + /// The left-hand side operand to the conditional operator. + rhs: Reg, + }, + /// A fused `i64.lt_s` and `select` instruction with 16-bit immediate `rhs` value. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i64_lt_s_imm16_rhs)] + SelectI64LtSImm16Rhs { + @result: Reg, + /// The left-hand side operand to the conditional operator. + lhs: Reg, + /// The right-hand side operand to the conditional operator. + rhs: Const16, + }, + /// A fused `i64.lt_u` and `select` instruction. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i64_lt_u)] + SelectI64LtU { + @result: Reg, + /// The left-hand side operand to the branch conditional. + lhs: Reg, + /// The right-hand side operand to the branch conditional. + rhs: Reg, + }, + /// A fused `i64.lt_u` and `select` instruction with 16-bit immediate `lhs` value. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i64_lt_u_imm16_lhs)] + SelectI64LtUImm16Lhs { + @result: Reg, + /// The right-hand side operand to the conditional operator. + lhs: Const16, + /// The left-hand side operand to the conditional operator. + rhs: Reg, + }, + /// A fused `i64.lt_u` and `select` instruction with 16-bit immediate `rhs` value. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i64_lt_u_imm16_rhs)] + SelectI64LtUImm16Rhs { + @result: Reg, + /// The left-hand side operand to the conditional operator. + lhs: Reg, + /// The right-hand side operand to the conditional operator. + rhs: Const16, + }, + /// A fused `i64.le_s` and `select` instruction. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i64_le_s)] + SelectI64LeS { + @result: Reg, + /// The left-hand side operand to the branch conditional. + lhs: Reg, + /// The right-hand side operand to the branch conditional. + rhs: Reg, + }, + /// A fused `i64.le_s` and `select` instruction with 16-bit immediate `lhs` value. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i64_le_s_imm16_lhs)] + SelectI64LeSImm16Lhs { + @result: Reg, + /// The right-hand side operand to the conditional operator. + lhs: Const16, + /// The left-hand side operand to the conditional operator. + rhs: Reg, + }, + /// A fused `i64.le_s` and `select` instruction with 16-bit immediate `rhs` value. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i64_le_s_imm16_rhs)] + SelectI64LeSImm16Rhs { + @result: Reg, + /// The left-hand side operand to the conditional operator. + lhs: Reg, + /// The right-hand side operand to the conditional operator. + rhs: Const16, + }, + /// A fused `i64.le_u` and `select` instruction. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i64_le_u)] + SelectI64LeU { + @result: Reg, + /// The left-hand side operand to the branch conditional. + lhs: Reg, + /// The right-hand side operand to the branch conditional. + rhs: Reg, + }, + /// A fused `i64.le_u` and `select` instruction with 16-bit immediate `lhs` value. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i64_le_u_imm16_lhs)] + SelectI64LeUImm16Lhs { + @result: Reg, + /// The right-hand side operand to the conditional operator. + lhs: Const16, + /// The left-hand side operand to the conditional operator. + rhs: Reg, + }, + /// A fused `i64.le_u` and `select` instruction with 16-bit immediate `rhs` value. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_i64_le_u_imm16_rhs)] + SelectI64LeUImm16Rhs { + @result: Reg, + /// The left-hand side operand to the conditional operator. + lhs: Reg, + /// The right-hand side operand to the conditional operator. + rhs: Const16, + }, + /// A fused `f32.eq` and `select` instruction. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_f32_eq)] + SelectF32Eq { + @result: Reg, + /// The left-hand side operand to the branch conditional. + lhs: Reg, + /// The right-hand side operand to the branch conditional. + rhs: Reg, + }, + /// A fused `f32.ne` and `select` instruction. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_f32_ne)] + SelectF32Ne { + @result: Reg, + /// The left-hand side operand to the branch conditional. + lhs: Reg, + /// The right-hand side operand to the branch conditional. + rhs: Reg, + }, + /// A fused `f32.lt` and `select` instruction. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_f32_lt)] + SelectF32Lt { + @result: Reg, + /// The left-hand side operand to the branch conditional. + lhs: Reg, + /// The right-hand side operand to the branch conditional. + rhs: Reg, + }, + /// A fused `f32.le` and `select` instruction. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_f32_le)] + SelectF32Le { + @result: Reg, + /// The left-hand side operand to the branch conditional. + lhs: Reg, + /// The right-hand side operand to the branch conditional. + rhs: Reg, + }, + + /// A fused `f32.not_lt` and `select` instruction. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_f32_not_lt)] + SelectF32NotLt { + @result: Reg, + /// The left-hand side operand to the branch conditional. + lhs: Reg, + /// The right-hand side operand to the branch conditional. + rhs: Reg, + }, + /// A fused `f32.not_le` and `select` instruction. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_f32_not_le)] + SelectF32NotLe { + @result: Reg, + /// The left-hand side operand to the branch conditional. + lhs: Reg, + /// The right-hand side operand to the branch conditional. + rhs: Reg, + }, + /// A fused `f64.eq` and `select` instruction. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_f64_eq)] + SelectF64Eq { + @result: Reg, + /// The left-hand side operand to the branch conditional. + lhs: Reg, + /// The right-hand side operand to the branch conditional. + rhs: Reg, + }, + /// A fused `f64.ne` and `select` instruction. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_f64_ne)] + SelectF64Ne { + @result: Reg, + /// The left-hand side operand to the branch conditional. + lhs: Reg, + /// The right-hand side operand to the branch conditional. + rhs: Reg, + }, + /// A fused `f64.lt` and `select` instruction. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_f64_lt)] + SelectF64Lt { + @result: Reg, + /// The left-hand side operand to the branch conditional. + lhs: Reg, + /// The right-hand side operand to the branch conditional. + rhs: Reg, + }, + /// A fused `f64.le` and `select` instruction. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_f64_le)] + SelectF64Le { + @result: Reg, + /// The left-hand side operand to the branch conditional. + lhs: Reg, + /// The right-hand side operand to the branch conditional. + rhs: Reg, + }, + /// A fused `f64.not_lt` and `select` instruction. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_f64_not_lt)] + SelectF64NotLt { + @result: Reg, + /// The left-hand side operand to the branch conditional. + lhs: Reg, + /// The right-hand side operand to the branch conditional. + rhs: Reg, + }, + /// A fused `f64.not_le` and `select` instruction. + /// + /// # Encoding + /// + /// Followed by [`Instruction::Register2`] encoding `true_val` and `false_val`.` + #[snake_name(select_f64_not_le)] + SelectF64NotLe { + @result: Reg, + /// The left-hand side operand to the branch conditional. + lhs: Reg, + /// The right-hand side operand to the branch conditional. + rhs: Reg, }, /// A Wasm `ref.func` equivalent Wasmi instruction. diff --git a/crates/wasmi/src/engine/executor/instrs.rs b/crates/wasmi/src/engine/executor/instrs.rs index f42b9df263..16acc2d561 100644 --- a/crates/wasmi/src/engine/executor/instrs.rs +++ b/crates/wasmi/src/engine/executor/instrs.rs @@ -448,24 +448,210 @@ impl<'engine> Executor<'engine> { Instr::CallIndirectImm16 { results, func_type } => { self.execute_call_indirect_imm16(store, results, func_type)? } - Instr::Select { result, lhs } => self.execute_select(result, lhs), - Instr::SelectImm32Rhs { result, lhs } => self.execute_select_imm32_rhs(result, lhs), - Instr::SelectImm32Lhs { result, lhs } => self.execute_select_imm32_lhs(result, lhs), - Instr::SelectImm32 { result, lhs } => self.execute_select_imm32(result, lhs), - Instr::SelectI64Imm32Rhs { result, lhs } => { - self.execute_select_i64imm32_rhs(result, lhs) + Instr::SelectI32Eq { result, lhs, rhs } => { + self.execute_select_i32_eq(result, lhs, rhs) } - Instr::SelectI64Imm32Lhs { result, lhs } => { - self.execute_select_i64imm32_lhs(result, lhs) + Instr::SelectI32EqImm16 { result, lhs, rhs } => { + self.execute_select_i32_eq_imm16(result, lhs, rhs) } - Instr::SelectI64Imm32 { result, lhs } => self.execute_select_i64imm32(result, lhs), - Instr::SelectF64Imm32Rhs { result, lhs } => { - self.execute_select_f64imm32_rhs(result, lhs) + Instr::SelectI32Ne { result, lhs, rhs } => { + self.execute_select_i32_ne(result, lhs, rhs) } - Instr::SelectF64Imm32Lhs { result, lhs } => { - self.execute_select_f64imm32_lhs(result, lhs) + Instr::SelectI32NeImm16 { result, lhs, rhs } => { + self.execute_select_i32_ne_imm16(result, lhs, rhs) + } + Instr::SelectI32LtS { result, lhs, rhs } => { + self.execute_select_i32_lt_s(result, lhs, rhs) + } + Instr::SelectI32LtSImm16Rhs { result, lhs, rhs } => { + self.execute_select_i32_lt_s_imm16_rhs(result, lhs, rhs) + } + Instr::SelectI32LtSImm16Lhs { result, lhs, rhs } => { + self.execute_select_i32_lt_s_imm16_lhs(result, lhs, rhs) + } + Instr::SelectI32LtU { result, lhs, rhs } => { + self.execute_select_i32_lt_u(result, lhs, rhs) + } + Instr::SelectI32LtUImm16Rhs { result, lhs, rhs } => { + self.execute_select_i32_lt_u_imm16_rhs(result, lhs, rhs) + } + Instr::SelectI32LtUImm16Lhs { result, lhs, rhs } => { + self.execute_select_i32_lt_u_imm16_lhs(result, lhs, rhs) + } + Instr::SelectI32LeS { result, lhs, rhs } => { + self.execute_select_i32_le_s(result, lhs, rhs) + } + Instr::SelectI32LeSImm16Rhs { result, lhs, rhs } => { + self.execute_select_i32_le_s_imm16_rhs(result, lhs, rhs) + } + Instr::SelectI32LeSImm16Lhs { result, lhs, rhs } => { + self.execute_select_i32_le_s_imm16_lhs(result, lhs, rhs) + } + Instr::SelectI32LeU { result, lhs, rhs } => { + self.execute_select_i32_le_u(result, lhs, rhs) + } + Instr::SelectI32LeUImm16Rhs { result, lhs, rhs } => { + self.execute_select_i32_le_u_imm16_rhs(result, lhs, rhs) + } + Instr::SelectI32LeUImm16Lhs { result, lhs, rhs } => { + self.execute_select_i32_le_u_imm16_lhs(result, lhs, rhs) + } + Instr::SelectI32And { result, lhs, rhs } => { + self.execute_select_i32_and(result, lhs, rhs) + } + Instr::SelectI32AndImm16 { result, lhs, rhs } => { + self.execute_select_i32_and_imm16(result, lhs, rhs) + } + Instr::SelectI32Or { result, lhs, rhs } => { + self.execute_select_i32_or(result, lhs, rhs) + } + Instr::SelectI32OrImm16 { result, lhs, rhs } => { + self.execute_select_i32_or_imm16(result, lhs, rhs) + } + Instr::SelectI32Xor { result, lhs, rhs } => { + self.execute_select_i32_xor(result, lhs, rhs) + } + Instr::SelectI32XorImm16 { result, lhs, rhs } => { + self.execute_select_i32_xor_imm16(result, lhs, rhs) + } + Instr::SelectI32Nand { result, lhs, rhs } => { + self.execute_select_i32_nand(result, lhs, rhs) + } + Instr::SelectI32NandImm16 { result, lhs, rhs } => { + self.execute_select_i32_nand_imm16(result, lhs, rhs) + } + Instr::SelectI32Nor { result, lhs, rhs } => { + self.execute_select_i32_nor(result, lhs, rhs) + } + Instr::SelectI32NorImm16 { result, lhs, rhs } => { + self.execute_select_i32_nor_imm16(result, lhs, rhs) + } + Instr::SelectI32Xnor { result, lhs, rhs } => { + self.execute_select_i32_xnor(result, lhs, rhs) + } + Instr::SelectI32XnorImm16 { result, lhs, rhs } => { + self.execute_select_i32_xnor_imm16(result, lhs, rhs) + } + Instr::SelectI64Eq { result, lhs, rhs } => { + self.execute_select_i64_eq(result, lhs, rhs) + } + Instr::SelectI64EqImm16 { result, lhs, rhs } => { + self.execute_select_i64_eq_imm16(result, lhs, rhs) + } + Instr::SelectI64Ne { result, lhs, rhs } => { + self.execute_select_i64_ne(result, lhs, rhs) + } + Instr::SelectI64NeImm16 { result, lhs, rhs } => { + self.execute_select_i64_ne_imm16(result, lhs, rhs) + } + Instr::SelectI64LtS { result, lhs, rhs } => { + self.execute_select_i64_lt_s(result, lhs, rhs) + } + Instr::SelectI64LtSImm16Rhs { result, lhs, rhs } => { + self.execute_select_i64_lt_s_imm16_rhs(result, lhs, rhs) + } + Instr::SelectI64LtSImm16Lhs { result, lhs, rhs } => { + self.execute_select_i64_lt_s_imm16_lhs(result, lhs, rhs) + } + Instr::SelectI64LtU { result, lhs, rhs } => { + self.execute_select_i64_lt_u(result, lhs, rhs) + } + Instr::SelectI64LtUImm16Rhs { result, lhs, rhs } => { + self.execute_select_i64_lt_u_imm16_rhs(result, lhs, rhs) + } + Instr::SelectI64LtUImm16Lhs { result, lhs, rhs } => { + self.execute_select_i64_lt_u_imm16_lhs(result, lhs, rhs) + } + Instr::SelectI64LeS { result, lhs, rhs } => { + self.execute_select_i64_le_s(result, lhs, rhs) + } + Instr::SelectI64LeSImm16Rhs { result, lhs, rhs } => { + self.execute_select_i64_le_s_imm16_rhs(result, lhs, rhs) + } + Instr::SelectI64LeSImm16Lhs { result, lhs, rhs } => { + self.execute_select_i64_le_s_imm16_lhs(result, lhs, rhs) + } + Instr::SelectI64LeU { result, lhs, rhs } => { + self.execute_select_i64_le_u(result, lhs, rhs) + } + Instr::SelectI64LeUImm16Rhs { result, lhs, rhs } => { + self.execute_select_i64_le_u_imm16_rhs(result, lhs, rhs) + } + Instr::SelectI64LeUImm16Lhs { result, lhs, rhs } => { + self.execute_select_i64_le_u_imm16_lhs(result, lhs, rhs) + } + Instr::SelectI64And { result, lhs, rhs } => { + self.execute_select_i64_and(result, lhs, rhs) + } + Instr::SelectI64AndImm16 { result, lhs, rhs } => { + self.execute_select_i64_and_imm16(result, lhs, rhs) + } + Instr::SelectI64Or { result, lhs, rhs } => { + self.execute_select_i64_or(result, lhs, rhs) + } + Instr::SelectI64OrImm16 { result, lhs, rhs } => { + self.execute_select_i64_or_imm16(result, lhs, rhs) + } + Instr::SelectI64Xor { result, lhs, rhs } => { + self.execute_select_i64_xor(result, lhs, rhs) + } + Instr::SelectI64XorImm16 { result, lhs, rhs } => { + self.execute_select_i64_xor_imm16(result, lhs, rhs) + } + Instr::SelectI64Nand { result, lhs, rhs } => { + self.execute_select_i64_nand(result, lhs, rhs) + } + Instr::SelectI64NandImm16 { result, lhs, rhs } => { + self.execute_select_i64_nand_imm16(result, lhs, rhs) + } + Instr::SelectI64Nor { result, lhs, rhs } => { + self.execute_select_i64_nor(result, lhs, rhs) + } + Instr::SelectI64NorImm16 { result, lhs, rhs } => { + self.execute_select_i64_nor_imm16(result, lhs, rhs) + } + Instr::SelectI64Xnor { result, lhs, rhs } => { + self.execute_select_i64_xnor(result, lhs, rhs) + } + Instr::SelectI64XnorImm16 { result, lhs, rhs } => { + self.execute_select_i64_xnor_imm16(result, lhs, rhs) + } + Instr::SelectF32Eq { result, lhs, rhs } => { + self.execute_select_f32_eq(result, lhs, rhs) + } + Instr::SelectF32Ne { result, lhs, rhs } => { + self.execute_select_f32_ne(result, lhs, rhs) + } + Instr::SelectF32Lt { result, lhs, rhs } => { + self.execute_select_f32_lt(result, lhs, rhs) + } + Instr::SelectF32Le { result, lhs, rhs } => { + self.execute_select_f32_le(result, lhs, rhs) + } + Instr::SelectF32NotLt { result, lhs, rhs } => { + self.execute_select_f32_not_lt(result, lhs, rhs) + } + Instr::SelectF32NotLe { result, lhs, rhs } => { + self.execute_select_f32_not_le(result, lhs, rhs) + } + Instr::SelectF64Eq { result, lhs, rhs } => { + self.execute_select_f64_eq(result, lhs, rhs) + } + Instr::SelectF64Ne { result, lhs, rhs } => { + self.execute_select_f64_ne(result, lhs, rhs) + } + Instr::SelectF64Lt { result, lhs, rhs } => { + self.execute_select_f64_lt(result, lhs, rhs) + } + Instr::SelectF64Le { result, lhs, rhs } => { + self.execute_select_f64_le(result, lhs, rhs) + } + Instr::SelectF64NotLt { result, lhs, rhs } => { + self.execute_select_f64_not_lt(result, lhs, rhs) + } + Instr::SelectF64NotLe { result, lhs, rhs } => { + self.execute_select_f64_not_le(result, lhs, rhs) } - Instr::SelectF64Imm32 { result, lhs } => self.execute_select_f64imm32(result, lhs), Instr::RefFunc { result, func } => self.execute_ref_func(result, func), Instr::GlobalGet { result, global } => { self.execute_global_get(store.inner(), result, global) diff --git a/crates/wasmi/src/engine/executor/instrs/select.rs b/crates/wasmi/src/engine/executor/instrs/select.rs index 4897e7395c..e2ac47432e 100644 --- a/crates/wasmi/src/engine/executor/instrs/select.rs +++ b/crates/wasmi/src/engine/executor/instrs/select.rs @@ -1,8 +1,8 @@ -use super::{Executor, InstructionPtr}; +use super::{Executor, InstructionPtr, UntypedValueCmpExt, UntypedValueExt}; use crate::{ - core::UntypedVal, + core::{wasm, ReadAs, UntypedVal}, engine::utils::unreachable_unchecked, - ir::{AnyConst32, Const32, Instruction, Reg}, + ir::{Const16, Instruction, Reg}, }; impl<'engine> Executor<'engine> { @@ -23,138 +23,190 @@ impl<'engine> Executor<'engine> { } } - /// Fetches a [`Reg`] and a 32-bit immediate value of type `T`. - fn fetch_register_and_imm32(&self) -> (Reg, T) + /// Executes a fused `cmp`+`select` instruction. + #[inline(always)] + fn execute_cmp_select_impl(&mut self, result: Reg, lhs: Reg, rhs: Reg, f: fn(T, T) -> bool) where - T: From, + UntypedVal: ReadAs, { - let mut addr: InstructionPtr = self.ip; - addr.add(1); - match *addr.get() { - Instruction::RegisterAndImm32 { reg, imm } => (reg, T::from(imm)), - unexpected => { - // Safety: Wasmi translation guarantees that [`Instruction::RegisterAndImm32`] exists. - unsafe { - unreachable_unchecked!( - "expected `Instruction::RegisterAndImm32` but found {unexpected:?}" - ) - } - } - } + let (true_val, false_val) = self.fetch_register_2(); + let lhs: T = self.get_register_as(lhs); + let rhs: T = self.get_register_as(rhs); + let selected = self.get_register(match f(lhs, rhs) { + true => true_val, + false => false_val, + }); + self.set_register(result, selected); + self.next_instr_at(2); } - /// Executes a `select` instruction generically. - fn execute_select_impl( + /// Executes a fused `cmp`+`select` instruction with immediate `rhs` parameter. + #[inline(always)] + fn execute_cmp_select_imm_rhs_impl( &mut self, result: Reg, - condition: Reg, - lhs: impl FnOnce(&Self) -> L, - rhs: impl FnOnce(&Self) -> R, + lhs: Reg, + rhs: Const16, + f: fn(T, T) -> bool, ) where - L: Into, - R: Into, + UntypedVal: ReadAs, + T: From>, { - let condition: bool = self.get_register_as(condition); - let selected = match condition { - true => lhs(self).into(), - false => rhs(self).into(), - }; + let (true_val, false_val) = self.fetch_register_2(); + let lhs: T = self.get_register_as(lhs); + let rhs: T = rhs.into(); + let selected = self.get_register(match f(lhs, rhs) { + true => true_val, + false => false_val, + }); self.set_register(result, selected); self.next_instr_at(2); } - /// Executes an [`Instruction::Select`]. - pub fn execute_select(&mut self, result: Reg, lhs: Reg) { - let (condition, rhs) = self.fetch_register_2(); - self.execute_select_impl( - result, - condition, - |this| this.get_register(lhs), - |this| this.get_register(rhs), - ) - } - - /// Executes an [`Instruction::SelectImm32Rhs`]. - pub fn execute_select_imm32_rhs(&mut self, result: Reg, lhs: Reg) { - let (condition, rhs) = self.fetch_register_and_imm32::(); - self.execute_select_impl( - result, - condition, - |this: &Executor<'engine>| this.get_register(lhs), - |_| u32::from(rhs), - ) - } - - /// Executes an [`Instruction::SelectImm32Lhs`]. - pub fn execute_select_imm32_lhs(&mut self, result: Reg, lhs: AnyConst32) { - let (condition, rhs) = self.fetch_register_2(); - self.execute_select_impl( - result, - condition, - |_| u32::from(lhs), - |this| this.get_register(rhs), - ) - } - - /// Executes an [`Instruction::SelectImm32`]. - pub fn execute_select_imm32(&mut self, result: Reg, lhs: AnyConst32) { - let (condition, rhs) = self.fetch_register_and_imm32::(); - self.execute_select_impl(result, condition, |_| u32::from(lhs), |_| u32::from(rhs)) + /// Executes a fused `cmp`+`select` instruction with immediate `lhs` parameter. + #[inline(always)] + fn execute_cmp_select_imm_lhs_impl( + &mut self, + result: Reg, + lhs: Const16, + rhs: Reg, + f: fn(T, T) -> bool, + ) where + UntypedVal: ReadAs, + T: From>, + { + let (true_val, false_val) = self.fetch_register_2(); + let lhs: T = lhs.into(); + let rhs: T = self.get_register_as(rhs); + let selected = self.get_register(match f(lhs, rhs) { + true => true_val, + false => false_val, + }); + self.set_register(result, selected); + self.next_instr_at(2); } +} - /// Executes an [`Instruction::SelectI64Imm32Rhs`]. - pub fn execute_select_i64imm32_rhs(&mut self, result: Reg, lhs: Reg) { - let (condition, rhs) = self.fetch_register_and_imm32::(); - self.execute_select_impl( - result, - condition, - |this| this.get_register(lhs), - |_| i64::from(rhs), - ) - } +macro_rules! impl_cmp_select_for { + ( + $( + (Instruction::$doc_name:ident, $fn_name:ident, $op:expr) + ),* $(,)? + ) => { + $( + #[doc = concat!("Executes an [`Instruction::", stringify!($doc_name), "`].")] + pub fn $fn_name(&mut self, result: Reg, lhs: Reg, rhs: Reg) { + self.execute_cmp_select_impl(result, lhs, rhs, $op) + } + )* + }; +} - /// Executes an [`Instruction::SelectI64Imm32Lhs`]. - pub fn execute_select_i64imm32_lhs(&mut self, result: Reg, lhs: Const32) { - let (condition, rhs) = self.fetch_register_2(); - self.execute_select_impl( - result, - condition, - |_| i64::from(lhs), - |this| this.get_register(rhs), - ) - } +macro_rules! impl_cmp_select_imm_rhs_for { + ( + $( + ($ty:ty, Instruction::$doc_name:ident, $fn_name:ident, $op:expr) + ),* $(,)? + ) => { + $( + #[doc = concat!("Executes an [`Instruction::", stringify!($doc_name), "`].")] + pub fn $fn_name(&mut self, result: Reg, lhs: Reg, rhs: Const16<$ty>) { + self.execute_cmp_select_imm_rhs_impl::<$ty>(result, lhs, rhs, $op) + } + )* + }; +} - /// Executes an [`Instruction::SelectI64Imm32`]. - pub fn execute_select_i64imm32(&mut self, result: Reg, lhs: Const32) { - let (condition, rhs) = self.fetch_register_and_imm32::(); - self.execute_select_impl(result, condition, |_| i64::from(lhs), |_| i64::from(rhs)) - } +macro_rules! impl_cmp_select_imm_lhs_for { + ( + $( + ($ty:ty, Instruction::$doc_name:ident, $fn_name:ident, $op:expr) + ),* $(,)? + ) => { + $( + #[doc = concat!("Executes an [`Instruction::", stringify!($doc_name), "`].")] + pub fn $fn_name(&mut self, result: Reg, lhs: Const16<$ty>, rhs: Reg) { + self.execute_cmp_select_imm_lhs_impl::<$ty>(result, lhs, rhs, $op) + } + )* + }; +} - /// Executes an [`Instruction::SelectF64Imm32Rhs`]. - pub fn execute_select_f64imm32_rhs(&mut self, result: Reg, lhs: Reg) { - let (condition, rhs) = self.fetch_register_and_imm32::(); - self.execute_select_impl( - result, - condition, - |this| this.get_register(lhs), - |_| f64::from(rhs), - ) +impl Executor<'_> { + impl_cmp_select_for! { + (Instruction::SelectI32Eq, execute_select_i32_eq, wasm::i32_eq), + (Instruction::SelectI32Ne, execute_select_i32_ne, wasm::i32_ne), + (Instruction::SelectI32LtS, execute_select_i32_lt_s, wasm::i32_lt_s), + (Instruction::SelectI32LtU, execute_select_i32_lt_u, wasm::i32_lt_u), + (Instruction::SelectI32LeS, execute_select_i32_le_s, wasm::i32_le_s), + (Instruction::SelectI32LeU, execute_select_i32_le_u, wasm::i32_le_u), + (Instruction::SelectI32And, execute_select_i32_and, ::and), + (Instruction::SelectI32Or, execute_select_i32_or, ::or), + (Instruction::SelectI32Xor, execute_select_i32_xor, ::xor), + (Instruction::SelectI32Nand, execute_select_i32_nand, ::nand), + (Instruction::SelectI32Nor, execute_select_i32_nor, ::nor), + (Instruction::SelectI32Xnor, execute_select_i32_xnor, ::xnor), + (Instruction::SelectI64Eq, execute_select_i64_eq, wasm::i64_eq), + (Instruction::SelectI64Ne, execute_select_i64_ne, wasm::i64_ne), + (Instruction::SelectI64LtS, execute_select_i64_lt_s, wasm::i64_lt_s), + (Instruction::SelectI64LtU, execute_select_i64_lt_u, wasm::i64_lt_u), + (Instruction::SelectI64LeS, execute_select_i64_le_s, wasm::i64_le_s), + (Instruction::SelectI64LeU, execute_select_i64_le_u, wasm::i64_le_u), + (Instruction::SelectI64And, execute_select_i64_and, ::and), + (Instruction::SelectI64Or, execute_select_i64_or, ::or), + (Instruction::SelectI64Xor, execute_select_i64_xor, ::xor), + (Instruction::SelectI64Nand, execute_select_i64_nand, ::nand), + (Instruction::SelectI64Nor, execute_select_i64_nor, ::nor), + (Instruction::SelectI64Xnor, execute_select_i64_xnor, ::xnor), + (Instruction::SelectF32Eq, execute_select_f32_eq, wasm::f32_eq), + (Instruction::SelectF32Ne, execute_select_f32_ne, wasm::f32_ne), + (Instruction::SelectF32Lt, execute_select_f32_lt, wasm::f32_lt), + (Instruction::SelectF32Le, execute_select_f32_le, wasm::f32_le), + (Instruction::SelectF32NotLt, execute_select_f32_not_lt, ::not_lt), + (Instruction::SelectF32NotLe, execute_select_f32_not_le, ::not_le), + (Instruction::SelectF64Eq, execute_select_f64_eq, wasm::f64_eq), + (Instruction::SelectF64Ne, execute_select_f64_ne, wasm::f64_ne), + (Instruction::SelectF64Lt, execute_select_f64_lt, wasm::f64_lt), + (Instruction::SelectF64Le, execute_select_f64_le, wasm::f64_le), + (Instruction::SelectF64NotLt, execute_select_f64_not_lt, ::not_lt), + (Instruction::SelectF64NotLe, execute_select_f64_not_le, ::not_le), } - /// Executes an [`Instruction::SelectF64Imm32Lhs`]. - pub fn execute_select_f64imm32_lhs(&mut self, result: Reg, lhs: Const32) { - let (condition, rhs) = self.fetch_register_2(); - self.execute_select_impl( - result, - condition, - |_| f64::from(lhs), - |this| this.get_register(rhs), - ) + impl_cmp_select_imm_rhs_for! { + (i32, Instruction::SelectI32EqImm16, execute_select_i32_eq_imm16, wasm::i32_eq), + (i32, Instruction::SelectI32NeImm16, execute_select_i32_ne_imm16, wasm::i32_ne), + (i32, Instruction::SelectI32LtSImm16Rhs, execute_select_i32_lt_s_imm16_rhs, wasm::i32_lt_s), + (u32, Instruction::SelectI32LtUImm16Rhs, execute_select_i32_lt_u_imm16_rhs, wasm::i32_lt_u), + (i32, Instruction::SelectI32LeSImm16Rhs, execute_select_i32_le_s_imm16_rhs, wasm::i32_le_s), + (u32, Instruction::SelectI32LeUImm16Rhs, execute_select_i32_le_u_imm16_rhs, wasm::i32_le_u), + (i32, Instruction::SelectI32AndImm16, execute_select_i32_and_imm16, UntypedValueExt::and), + (i32, Instruction::SelectI32OrImm16, execute_select_i32_or_imm16, UntypedValueExt::or), + (i32, Instruction::SelectI32XorImm16, execute_select_i32_xor_imm16, UntypedValueExt::xor), + (i32, Instruction::SelectI32NandImm16, execute_select_i32_nand_imm16, UntypedValueExt::nand), + (i32, Instruction::SelectI32NorImm16, execute_select_i32_nor_imm16, UntypedValueExt::nor), + (i32, Instruction::SelectI32XnorImm16, execute_select_i32_xnor_imm16, UntypedValueExt::xnor), + (i64, Instruction::SelectI64EqImm16, execute_select_i64_eq_imm16, wasm::i64_eq), + (i64, Instruction::SelectI64NeImm16, execute_select_i64_ne_imm16, wasm::i64_ne), + (i64, Instruction::SelectI64LtSImm16Rhs, execute_select_i64_lt_s_imm16_rhs, wasm::i64_lt_s), + (u64, Instruction::SelectI64LtUImm16Rhs, execute_select_i64_lt_u_imm16_rhs, wasm::i64_lt_u), + (i64, Instruction::SelectI64LeSImm16Rhs, execute_select_i64_le_s_imm16_rhs, wasm::i64_le_s), + (u64, Instruction::SelectI64LeUImm16Rhs, execute_select_i64_le_u_imm16_rhs, wasm::i64_le_u), + (i64, Instruction::SelectI64AndImm16, execute_select_i64_and_imm16, UntypedValueExt::and), + (i64, Instruction::SelectI64OrImm16, execute_select_i64_or_imm16, UntypedValueExt::or), + (i64, Instruction::SelectI64XorImm16, execute_select_i64_xor_imm16, UntypedValueExt::xor), + (i64, Instruction::SelectI64NandImm16, execute_select_i64_nand_imm16, UntypedValueExt::nand), + (i64, Instruction::SelectI64NorImm16, execute_select_i64_nor_imm16, UntypedValueExt::nor), + (i64, Instruction::SelectI64XnorImm16, execute_select_i64_xnor_imm16, UntypedValueExt::xnor), } - /// Executes an [`Instruction::SelectF64Imm32`]. - pub fn execute_select_f64imm32(&mut self, result: Reg, lhs: Const32) { - let (condition, rhs) = self.fetch_register_and_imm32::(); - self.execute_select_impl(result, condition, |_| f64::from(lhs), |_| f64::from(rhs)) + impl_cmp_select_imm_lhs_for! { + (i32, Instruction::SelectI32LtSImm16Lhs, execute_select_i32_lt_s_imm16_lhs, wasm::i32_lt_s), + (u32, Instruction::SelectI32LtUImm16Lhs, execute_select_i32_lt_u_imm16_lhs, wasm::i32_lt_u), + (i32, Instruction::SelectI32LeSImm16Lhs, execute_select_i32_le_s_imm16_lhs, wasm::i32_le_s), + (u32, Instruction::SelectI32LeUImm16Lhs, execute_select_i32_le_u_imm16_lhs, wasm::i32_le_u), + (i64, Instruction::SelectI64LtSImm16Lhs, execute_select_i64_lt_s_imm16_lhs, wasm::i64_lt_s), + (u64, Instruction::SelectI64LtUImm16Lhs, execute_select_i64_lt_u_imm16_lhs, wasm::i64_lt_u), + (i64, Instruction::SelectI64LeSImm16Lhs, execute_select_i64_le_s_imm16_lhs, wasm::i64_le_s), + (u64, Instruction::SelectI64LeUImm16Lhs, execute_select_i64_le_u_imm16_lhs, wasm::i64_le_u), } } diff --git a/crates/wasmi/src/engine/translator/comparator.rs b/crates/wasmi/src/engine/translator/comparator.rs index 9fac7cb555..f04b1bdab1 100644 --- a/crates/wasmi/src/engine/translator/comparator.rs +++ b/crates/wasmi/src/engine/translator/comparator.rs @@ -1,6 +1,6 @@ use super::ValueStack; use crate::{ - ir::{BranchOffset, BranchOffset16, Comparator, ComparatorAndOffset, Instruction}, + ir::{BranchOffset, BranchOffset16, Comparator, ComparatorAndOffset, Instruction, Reg}, Error, }; @@ -208,6 +208,107 @@ impl LogicalizeCmpInstr for Instruction { } } +pub trait TryIntoCmpSelectInstr: Sized { + fn try_into_cmp_select_instr(&self, result: Reg) -> Option; +} + +impl TryIntoCmpSelectInstr for Instruction { + fn try_into_cmp_select_instr(&self, result: Reg) -> Option { + use Instruction as I; + #[rustfmt::skip] + let fused = match *self { + // i32 + I::I32Eq { lhs, rhs, .. } => I::select_i32_eq(result, lhs, rhs), + I::I32Ne { lhs, rhs, .. } => I::select_i32_ne(result, lhs, rhs), + I::I32LeS { lhs, rhs, .. } => I::select_i32_le_s(result, lhs, rhs), + I::I32LeU { lhs, rhs, .. } => I::select_i32_le_u(result, lhs, rhs), + I::I32LtS { lhs, rhs, .. } => I::select_i32_lt_s(result, lhs, rhs), + I::I32LtU { lhs, rhs, .. } => I::select_i32_lt_u(result, lhs, rhs), + I::I32EqImm16 { lhs, rhs, .. } => I::select_i32_eq_imm16(result, lhs, rhs), + I::I32NeImm16 { lhs, rhs, .. } => I::select_i32_ne_imm16(result, lhs, rhs), + I::I32LeSImm16Lhs { lhs, rhs, .. } => I::select_i32_le_s_imm16_lhs(result, lhs, rhs), + I::I32LeUImm16Lhs { lhs, rhs, .. } => I::select_i32_le_u_imm16_lhs(result, lhs, rhs), + I::I32LtSImm16Lhs { lhs, rhs, .. } => I::select_i32_lt_s_imm16_lhs(result, lhs, rhs), + I::I32LtUImm16Lhs { lhs, rhs, .. } => I::select_i32_lt_u_imm16_lhs(result, lhs, rhs), + I::I32LeSImm16Rhs { lhs, rhs, .. } => I::select_i32_le_s_imm16_rhs(result, lhs, rhs), + I::I32LeUImm16Rhs { lhs, rhs, .. } => I::select_i32_le_u_imm16_rhs(result, lhs, rhs), + I::I32LtSImm16Rhs { lhs, rhs, .. } => I::select_i32_lt_s_imm16_rhs(result, lhs, rhs), + I::I32LtUImm16Rhs { lhs, rhs, .. } => I::select_i32_lt_u_imm16_rhs(result, lhs, rhs), + // i32 (and, or, xor) + I::I32BitAnd { lhs, rhs, .. } => I::select_i32_and(result, lhs, rhs), + I::I32BitOr { lhs, rhs, .. } => I::select_i32_or(result, lhs, rhs), + I::I32BitXor { lhs, rhs, .. } => I::select_i32_xor(result, lhs, rhs), + I::I32And { lhs, rhs, .. } => I::select_i32_and(result, lhs, rhs), + I::I32Or { lhs, rhs, .. } => I::select_i32_or(result, lhs, rhs), + I::I32Xor { lhs, rhs, .. } => I::select_i32_xor(result, lhs, rhs), + I::I32Nand { lhs, rhs, .. } => I::select_i32_nand(result, lhs, rhs), + I::I32Nor { lhs, rhs, .. } => I::select_i32_nor(result, lhs, rhs), + I::I32Xnor { lhs, rhs, .. } => I::select_i32_xnor(result, lhs, rhs), + I::I32BitAndImm16 { lhs, rhs, .. } => I::select_i32_and_imm16(result, lhs, rhs), + I::I32BitOrImm16 { lhs, rhs, .. } => I::select_i32_or_imm16(result, lhs, rhs), + I::I32BitXorImm16 { lhs, rhs, .. } => I::select_i32_xor_imm16(result, lhs, rhs), + I::I32AndImm16 { lhs, rhs, .. } => I::select_i32_and_imm16(result, lhs, rhs), + I::I32OrImm16 { lhs, rhs, .. } => I::select_i32_or_imm16(result, lhs, rhs), + I::I32XorImm16 { lhs, rhs, .. } => I::select_i32_xor_imm16(result, lhs, rhs), + I::I32NandImm16 { lhs, rhs, .. } => I::select_i32_nand_imm16(result, lhs, rhs), + I::I32NorImm16 { lhs, rhs, .. } => I::select_i32_nor_imm16(result, lhs, rhs), + I::I32XnorImm16 { lhs, rhs, .. } => I::select_i32_xnor_imm16(result, lhs, rhs), + // i64 + I::I64Eq { lhs, rhs, .. } => I::select_i64_eq(result, lhs, rhs), + I::I64Ne { lhs, rhs, .. } => I::select_i64_ne(result, lhs, rhs), + I::I64LeS { lhs, rhs, .. } => I::select_i64_le_s(result, lhs, rhs), + I::I64LeU { lhs, rhs, .. } => I::select_i64_le_u(result, lhs, rhs), + I::I64LtS { lhs, rhs, .. } => I::select_i64_lt_s(result, lhs, rhs), + I::I64LtU { lhs, rhs, .. } => I::select_i64_lt_u(result, lhs, rhs), + I::I64EqImm16 { lhs, rhs, .. } => I::select_i64_eq_imm16(result, lhs, rhs), + I::I64NeImm16 { lhs, rhs, .. } => I::select_i64_ne_imm16(result, lhs, rhs), + I::I64LeSImm16Lhs { lhs, rhs, .. } => I::select_i64_le_s_imm16_lhs(result, lhs, rhs), + I::I64LeUImm16Lhs { lhs, rhs, .. } => I::select_i64_le_u_imm16_lhs(result, lhs, rhs), + I::I64LtSImm16Lhs { lhs, rhs, .. } => I::select_i64_lt_s_imm16_lhs(result, lhs, rhs), + I::I64LtUImm16Lhs { lhs, rhs, .. } => I::select_i64_lt_u_imm16_lhs(result, lhs, rhs), + I::I64LeSImm16Rhs { lhs, rhs, .. } => I::select_i64_le_s_imm16_rhs(result, lhs, rhs), + I::I64LeUImm16Rhs { lhs, rhs, .. } => I::select_i64_le_u_imm16_rhs(result, lhs, rhs), + I::I64LtSImm16Rhs { lhs, rhs, .. } => I::select_i64_lt_s_imm16_rhs(result, lhs, rhs), + I::I64LtUImm16Rhs { lhs, rhs, .. } => I::select_i64_lt_u_imm16_rhs(result, lhs, rhs), + // i64 (and, or, xor) + I::I64BitAnd { lhs, rhs, .. } => I::select_i64_and(result, lhs, rhs), + I::I64BitOr { lhs, rhs, .. } => I::select_i64_or(result, lhs, rhs), + I::I64BitXor { lhs, rhs, .. } => I::select_i64_xor(result, lhs, rhs), + I::I64And { lhs, rhs, .. } => I::select_i64_and(result, lhs, rhs), + I::I64Or { lhs, rhs, .. } => I::select_i64_or(result, lhs, rhs), + I::I64Xor { lhs, rhs, .. } => I::select_i64_xor(result, lhs, rhs), + I::I64Nand { lhs, rhs, .. } => I::select_i64_nand(result, lhs, rhs), + I::I64Nor { lhs, rhs, .. } => I::select_i64_nor(result, lhs, rhs), + I::I64Xnor { lhs, rhs, .. } => I::select_i64_xnor(result, lhs, rhs), + I::I64BitAndImm16 { lhs, rhs, .. } => I::select_i64_and_imm16(result, lhs, rhs), + I::I64BitOrImm16 { lhs, rhs, .. } => I::select_i64_or_imm16(result, lhs, rhs), + I::I64BitXorImm16 { lhs, rhs, .. } => I::select_i64_xor_imm16(result, lhs, rhs), + I::I64AndImm16 { lhs, rhs, .. } => I::select_i64_and_imm16(result, lhs, rhs), + I::I64OrImm16 { lhs, rhs, .. } => I::select_i64_or_imm16(result, lhs, rhs), + I::I64XorImm16 { lhs, rhs, .. } => I::select_i64_xor_imm16(result, lhs, rhs), + I::I64NandImm16 { lhs, rhs, .. } => I::select_i64_nand_imm16(result, lhs, rhs), + I::I64NorImm16 { lhs, rhs, .. } => I::select_i64_nor_imm16(result, lhs, rhs), + I::I64XnorImm16 { lhs, rhs, .. } => I::select_i64_xnor_imm16(result, lhs, rhs), + // f32 + I::F32Eq { lhs, rhs, .. } => I::select_f32_eq(result, lhs, rhs), + I::F32Ne { lhs, rhs, .. } => I::select_f32_ne(result, lhs, rhs), + I::F32Lt { lhs, rhs, .. } => I::select_f32_lt(result, lhs, rhs), + I::F32Le { lhs, rhs, .. } => I::select_f32_le(result, lhs, rhs), + I::F32NotLt { lhs, rhs, .. } => I::select_f32_not_lt(result, lhs, rhs), + I::F32NotLe { lhs, rhs, .. } => I::select_f32_not_le(result, lhs, rhs), + // f64 + I::F64Eq { lhs, rhs, .. } => I::select_f64_eq(result, lhs, rhs), + I::F64Ne { lhs, rhs, .. } => I::select_f64_ne(result, lhs, rhs), + I::F64Lt { lhs, rhs, .. } => I::select_f64_lt(result, lhs, rhs), + I::F64Le { lhs, rhs, .. } => I::select_f64_le(result, lhs, rhs), + I::F64NotLt { lhs, rhs, .. } => I::select_f64_not_lt(result, lhs, rhs), + I::F64NotLe { lhs, rhs, .. } => I::select_f64_not_le(result, lhs, rhs), + _ => return None, + }; + Some(fused) + } +} + pub trait TryIntoCmpBranchInstr: Sized { fn try_into_cmp_branch_instr( &self, diff --git a/crates/wasmi/src/engine/translator/instr_encoder.rs b/crates/wasmi/src/engine/translator/instr_encoder.rs index 21b3af8274..67c36b06c1 100644 --- a/crates/wasmi/src/engine/translator/instr_encoder.rs +++ b/crates/wasmi/src/engine/translator/instr_encoder.rs @@ -1,4 +1,5 @@ use super::{ + comparator::TryIntoCmpSelectInstr, relink_result::RelinkResult as _, utils::{FromProviders as _, WasmInteger}, visit_register::VisitInputRegisters as _, @@ -929,6 +930,42 @@ impl InstrEncoder { Ok(true) } + /// Tries to fuse a compare instruction with a Wasm `select` instruction. + /// + /// # Returns + /// + /// - Returns `Some` if fusion was successful. + /// - Returns `None` if fusion could not be applied. + pub fn try_fuse_select( + &mut self, + stack: &mut ValueStack, + select_result: Reg, + select_condition: Reg, + ) -> Option { + let Some(last_instr) = self.last_instr else { + // If there is no last instruction there is no comparison instruction to negate. + return None; + }; + let last_instruction = *self.instrs.get(last_instr); + let Some(last_result) = last_instruction.result() else { + // All negatable instructions have a single result register. + return None; + }; + if matches!(stack.get_register_space(last_result), RegisterSpace::Local) { + // The instruction stores its result into a local variable which + // is an observable side effect which we are not allowed to mutate. + return None; + } + if last_result != select_condition { + // The result of the last instruction and the select's `condition` + // are not equal thus indicating that we cannot fuse the instructions. + return None; + } + let fused_select = last_instruction.try_into_cmp_select_instr(select_result)?; + _ = mem::replace(self.instrs.get_mut(last_instr), fused_select); + Some(fused_select) + } + /// Create an [`Instruction::BranchCmpFallback`]. fn make_branch_cmp_fallback( stack: &mut ValueStack, diff --git a/crates/wasmi/src/engine/translator/mod.rs b/crates/wasmi/src/engine/translator/mod.rs index 5bd4baeba6..14ed0a39cb 100644 --- a/crates/wasmi/src/engine/translator/mod.rs +++ b/crates/wasmi/src/engine/translator/mod.rs @@ -2483,22 +2483,28 @@ impl FuncTranslator { /// /// - This applies constant propagation in case `condition` is a constant value. /// - If both `lhs` and `rhs` are equal registers or constant values `lhs` is forwarded. - /// - Properly chooses the correct `select` instruction encoding and optimizes for - /// cases with 32-bit constant values. - fn translate_select(&mut self, type_hint: Option) -> Result<(), Error> { + /// - Fuses compare instructions with the associated select instructions if possible. + fn translate_select(&mut self, _type_hint: Option) -> Result<(), Error> { bail_unreachable!(self); - let (lhs, rhs, condition) = self.alloc.stack.pop3(); + let (true_val, false_val, condition) = self.alloc.stack.pop3(); + if true_val == false_val { + // Optimization: both `lhs` and `rhs` either are the same register or constant values and + // thus `select` will always yield this same value irrespective of the condition. + // + // TODO: we could technically look through registers representing function local constants and + // check whether they are equal to a given constant in cases where `lhs` and `rhs` are referring + // to a function local register and a constant value or vice versa. + self.alloc.stack.push_provider(true_val)?; + return Ok(()); + } let condition = match condition { - Provider::Register(condition) => { - // TODO: technically we could look through function local constant values here. - condition - } + Provider::Register(condition) => condition, Provider::Const(condition) => { // Optimization: since condition is a constant value we can const-fold the `select` // instruction and simply push the selected value back to the provider stack. let selected = match i32::from(condition) != 0 { - true => lhs, - false => rhs, + true => true_val, + false => false_val, }; if let Provider::Register(reg) = selected { if matches!( @@ -2523,202 +2529,22 @@ impl FuncTranslator { return Ok(()); } }; - if lhs == rhs { - // Optimization: both `lhs` and `rhs` either are the same register or constant values and - // thus `select` will always yield this same value irrespective of the condition. - // - // TODO: we could technically look through registers representing function local constants and - // check whether they are equal to a given constant in cases where `lhs` and `rhs` are referring - // to a function local register and a constant value or vice versa. - self.alloc.stack.push_provider(lhs)?; - return Ok(()); - } - let type_infer = match (lhs, rhs) { - (Provider::Register(lhs), Provider::Register(rhs)) => { - let result = self.alloc.stack.push_dynamic()?; - return self.translate_select_regs(result, condition, lhs, rhs); - } - (Provider::Register(_), Provider::Const(rhs)) => rhs.ty(), - (Provider::Const(lhs), Provider::Register(_)) => lhs.ty(), - (Provider::Const(lhs), Provider::Const(rhs)) => { - debug_assert_eq!(lhs.ty(), rhs.ty()); - lhs.ty() - } - }; - if let Some(type_hint) = type_hint { - assert_eq!(type_hint, type_infer); - } + let true_val = self.alloc.stack.provider2reg(&true_val)?; + let false_val = self.alloc.stack.provider2reg(&false_val)?; let result = self.alloc.stack.push_dynamic()?; - match type_infer { - ValType::I32 | ValType::F32 => self.translate_select_32(result, condition, lhs, rhs), - ValType::I64 => self.translate_select_i64(result, condition, lhs, rhs), - ValType::F64 => self.translate_select_f64(result, condition, lhs, rhs), - ValType::V128 | ValType::FuncRef | ValType::ExternRef => { - self.translate_select_generic(result, condition, lhs, rhs) - } - } - } - - fn translate_select_regs( - &mut self, - result: Reg, - condition: Reg, - lhs: Reg, - rhs: Reg, - ) -> Result<(), Error> { - debug_assert_ne!(lhs, rhs); - self.push_fueled_instr(Instruction::select(result, lhs), FuelCostsProvider::base)?; - self.alloc + if self + .alloc .instr_encoder - .append_instr(Instruction::register2_ext(condition, rhs))?; - Ok(()) - } - - fn translate_select_32( - &mut self, - result: Reg, - condition: Reg, - lhs: Provider, - rhs: Provider, - ) -> Result<(), Error> { - debug_assert_ne!(lhs, rhs); - let (instr, param) = match (lhs, rhs) { - (Provider::Register(_), Provider::Register(_)) => unreachable!(), - (Provider::Register(lhs), Provider::Const(rhs)) => { - debug_assert!(matches!(rhs.ty(), ValType::I32 | ValType::F32)); - ( - Instruction::select_imm32_rhs(result, lhs), - Instruction::register_and_imm32(condition, u32::from(rhs.untyped())), - ) - } - (Provider::Const(lhs), Provider::Register(rhs)) => { - debug_assert!(matches!(lhs.ty(), ValType::I32 | ValType::F32)); - ( - Instruction::select_imm32_lhs(result, u32::from(lhs.untyped())), - Instruction::register2_ext(condition, rhs), - ) - } - (Provider::Const(lhs), Provider::Const(rhs)) => { - debug_assert!(matches!(lhs.ty(), ValType::I32 | ValType::F32)); - debug_assert!(matches!(rhs.ty(), ValType::I32 | ValType::F32)); - ( - Instruction::select_imm32(result, u32::from(lhs.untyped())), - Instruction::register_and_imm32(condition, u32::from(rhs.untyped())), - ) - } - }; - self.push_fueled_instr(instr, FuelCostsProvider::base)?; - self.append_instr(param)?; - Ok(()) - } - - fn translate_select_i64( - &mut self, - result: Reg, - condition: Reg, - lhs: Provider, - rhs: Provider, - ) -> Result<(), Error> { - debug_assert_ne!(lhs, rhs); - let lhs = match lhs { - Provider::Register(lhs) => Provider::Register(lhs), - Provider::Const(lhs) => match >::try_from(i64::from(lhs)) { - Ok(lhs) => Provider::Const(lhs), - Err(_) => Provider::Register(self.alloc.stack.alloc_const(lhs)?), - }, - }; - let rhs = match rhs { - Provider::Register(rhs) => Provider::Register(rhs), - Provider::Const(rhs) => match >::try_from(i64::from(rhs)) { - Ok(rhs) => Provider::Const(rhs), - Err(_) => Provider::Register(self.alloc.stack.alloc_const(rhs)?), - }, - }; - let (instr, param) = match (lhs, rhs) { - (Provider::Register(lhs), Provider::Register(rhs)) => { - return self.translate_select_regs(result, condition, lhs, rhs) - } - (Provider::Register(lhs), Provider::Const(rhs)) => ( - Instruction::select_i64imm32_rhs(result, lhs), - Instruction::register_and_imm32(condition, rhs), - ), - (Provider::Const(lhs), Provider::Register(rhs)) => ( - Instruction::select_i64imm32_lhs(result, lhs), - Instruction::register2_ext(condition, rhs), - ), - (Provider::Const(lhs), Provider::Const(rhs)) => ( - Instruction::select_i64imm32(result, lhs), - Instruction::register_and_imm32(condition, rhs), - ), - }; - self.push_fueled_instr(instr, FuelCostsProvider::base)?; - self.append_instr(param)?; - Ok(()) - } - - fn translate_select_f64( - &mut self, - result: Reg, - condition: Reg, - lhs: Provider, - rhs: Provider, - ) -> Result<(), Error> { - debug_assert_ne!(lhs, rhs); - let lhs = match lhs { - Provider::Register(lhs) => Provider::Register(lhs), - Provider::Const(lhs) => match >::try_from(f64::from(lhs)) { - Ok(lhs) => Provider::Const(lhs), - Err(_) => Provider::Register(self.alloc.stack.alloc_const(lhs)?), - }, - }; - let rhs = match rhs { - Provider::Register(rhs) => Provider::Register(rhs), - Provider::Const(rhs) => match >::try_from(f64::from(rhs)) { - Ok(rhs) => Provider::Const(rhs), - Err(_) => Provider::Register(self.alloc.stack.alloc_const(rhs)?), - }, - }; - let (instr, param) = match (lhs, rhs) { - (Provider::Register(lhs), Provider::Register(rhs)) => { - return self.translate_select_regs(result, condition, lhs, rhs) - } - (Provider::Register(lhs), Provider::Const(rhs)) => ( - Instruction::select_f64imm32_rhs(result, lhs), - Instruction::register_and_imm32(condition, rhs), - ), - (Provider::Const(lhs), Provider::Register(rhs)) => ( - Instruction::select_f64imm32_lhs(result, lhs), - Instruction::register2_ext(condition, rhs), - ), - (Provider::Const(lhs), Provider::Const(rhs)) => ( - Instruction::select_f64imm32(result, lhs), - Instruction::register_and_imm32(condition, rhs), - ), + .try_fuse_select(&mut self.alloc.stack, result, condition) + .is_none() + { + let select_instr = Instruction::select_i32_ne_imm16(result, condition, 0_i16); + self.push_fueled_instr(select_instr, FuelCostsProvider::base)?; }; - self.push_fueled_instr(instr, FuelCostsProvider::base)?; - self.append_instr(param)?; + self.append_instr(Instruction::register2_ext(true_val, false_val))?; Ok(()) } - fn translate_select_generic( - &mut self, - result: Reg, - condition: Reg, - lhs: Provider, - rhs: Provider, - ) -> Result<(), Error> { - debug_assert_ne!(lhs, rhs); - let lhs = match lhs { - Provider::Register(lhs) => lhs, - Provider::Const(lhs) => self.alloc.stack.alloc_const(lhs)?, - }; - let rhs: Reg = match rhs { - Provider::Register(rhs) => rhs, - Provider::Const(rhs) => self.alloc.stack.alloc_const(rhs)?, - }; - self.translate_select_regs(result, condition, lhs, rhs) - } - /// Translates a Wasm `reinterpret` instruction. fn translate_reinterpret(&mut self, ty: ValType) -> Result<(), Error> { bail_unreachable!(self); diff --git a/crates/wasmi/src/engine/translator/tests/mod.rs b/crates/wasmi/src/engine/translator/tests/mod.rs index d01b459b82..6d08d7cc55 100644 --- a/crates/wasmi/src/engine/translator/tests/mod.rs +++ b/crates/wasmi/src/engine/translator/tests/mod.rs @@ -45,6 +45,14 @@ macro_rules! swap_cmp_br_ops { } use swap_cmp_br_ops; +/// Used to swap `lhs` and `rhs` operands of a fused `cmp+select` instruction. +macro_rules! swap_cmp_select_ops { + ($fn_name:path) => { + |result, lhs, rhs| -> Instruction { $fn_name(result, rhs, lhs) } + }; +} +use swap_cmp_select_ops; + /// Asserts that the given `wasm` bytes yield functions with expected instructions. /// /// Uses the given [`Config`] to configure the [`Engine`] that the tests are run on. diff --git a/crates/wasmi/src/engine/translator/tests/op/cmp_br.rs b/crates/wasmi/src/engine/translator/tests/op/cmp_br.rs index 79b1b3839f..905654a071 100644 --- a/crates/wasmi/src/engine/translator/tests/op/cmp_br.rs +++ b/crates/wasmi/src/engine/translator/tests/op/cmp_br.rs @@ -1,154 +1,10 @@ use super::{wasm_type::WasmTy, *}; use crate::{ - core::{UntypedVal, ValType}, + core::UntypedVal, ir::{index::Global, BranchOffset, BranchOffset16, Comparator, ComparatorAndOffset}, }; use std::{fmt, fmt::Debug, string::String}; -#[derive(Debug, Copy, Clone)] -pub enum CmpOp { - // i32 - I32And, - I32Or, - I32Xor, - I32Eq, - I32Ne, - I32LtS, - I32LtU, - I32LeS, - I32LeU, - I32GtS, - I32GtU, - I32GeS, - I32GeU, - // i64 - I64And, - I64Or, - I64Xor, - I64Eq, - I64Ne, - I64LtS, - I64LtU, - I64LeS, - I64LeU, - I64GtS, - I64GtU, - I64GeS, - I64GeU, - // f32 - F32Eq, - F32Ne, - F32Lt, - F32Le, - F32Gt, - F32Ge, - // f64 - F64Eq, - F64Ne, - F64Lt, - F64Le, - F64Gt, - F64Ge, -} - -impl CmpOp { - /// Returns the Wasm parameter type of the [`CmpOp`]. - pub fn param_ty(self) -> ValType { - match self { - CmpOp::I32And - | CmpOp::I32Or - | CmpOp::I32Xor - | CmpOp::I32Eq - | CmpOp::I32Ne - | CmpOp::I32LtS - | CmpOp::I32LtU - | CmpOp::I32LeS - | CmpOp::I32LeU - | CmpOp::I32GtS - | CmpOp::I32GtU - | CmpOp::I32GeS - | CmpOp::I32GeU => ValType::I32, - CmpOp::I64And - | CmpOp::I64Or - | CmpOp::I64Xor - | CmpOp::I64Eq - | CmpOp::I64Ne - | CmpOp::I64LtS - | CmpOp::I64LtU - | CmpOp::I64LeS - | CmpOp::I64LeU - | CmpOp::I64GtS - | CmpOp::I64GtU - | CmpOp::I64GeS - | CmpOp::I64GeU => ValType::I64, - CmpOp::F32Eq - | CmpOp::F32Ne - | CmpOp::F32Lt - | CmpOp::F32Le - | CmpOp::F32Gt - | CmpOp::F32Ge => ValType::F32, - CmpOp::F64Eq - | CmpOp::F64Ne - | CmpOp::F64Lt - | CmpOp::F64Le - | CmpOp::F64Gt - | CmpOp::F64Ge => ValType::F64, - } - } - - /// Returns the Wasm result type of the [`CmpOp`]. - pub fn result_ty(self) -> ValType { - match self { - CmpOp::I64And | CmpOp::I64Or | CmpOp::I64Xor => ValType::I64, - _ => ValType::I32, - } - } - - /// Returns a string representation of the Wasm operator without type annotation. - pub fn op_str(self) -> &'static str { - match self { - CmpOp::I32And => "and", - CmpOp::I32Or => "or", - CmpOp::I32Xor => "xor", - CmpOp::I32Eq => "eq", - CmpOp::I32Ne => "ne", - CmpOp::I32LtS => "lt_s", - CmpOp::I32LtU => "lt_u", - CmpOp::I32LeS => "le_s", - CmpOp::I32LeU => "le_u", - CmpOp::I32GtS => "gt_s", - CmpOp::I32GtU => "gt_u", - CmpOp::I32GeS => "ge_s", - CmpOp::I32GeU => "ge_u", - CmpOp::I64And => "and", - CmpOp::I64Or => "or", - CmpOp::I64Xor => "xor", - CmpOp::I64Eq => "eq", - CmpOp::I64Ne => "ne", - CmpOp::I64LtS => "lt_s", - CmpOp::I64LtU => "lt_u", - CmpOp::I64LeS => "le_s", - CmpOp::I64LeU => "le_u", - CmpOp::I64GtS => "gt_s", - CmpOp::I64GtU => "gt_u", - CmpOp::I64GeS => "ge_s", - CmpOp::I64GeU => "ge_u", - CmpOp::F32Eq => "eq", - CmpOp::F32Ne => "ne", - CmpOp::F32Lt => "lt", - CmpOp::F32Le => "le", - CmpOp::F32Gt => "gt", - CmpOp::F32Ge => "ge", - CmpOp::F64Eq => "eq", - CmpOp::F64Ne => "ne", - CmpOp::F64Lt => "lt", - CmpOp::F64Le => "le", - CmpOp::F64Gt => "gt", - CmpOp::F64Ge => "ge", - } - } -} - #[test] #[cfg_attr(miri, ignore)] fn loop_backward() { diff --git a/crates/wasmi/src/engine/translator/tests/op/local_set.rs b/crates/wasmi/src/engine/translator/tests/op/local_set.rs index d8eb3b429f..34ea2e5c2e 100644 --- a/crates/wasmi/src/engine/translator/tests/op/local_set.rs +++ b/crates/wasmi/src/engine/translator/tests/op/local_set.rs @@ -145,11 +145,14 @@ fn overwrite_select_result_1() { ) )"#; TranslationTest::new(wasm) - .expect_func_instrs([ - Instruction::select_imm32(Reg::from(0), 10_i32), - Instruction::register_and_imm32(Reg::from(0), 20_i32), - Instruction::return_reg(Reg::from(0)), - ]) + .expect_func( + ExpectedFunc::new([ + Instruction::select_i32_ne_imm16(Reg::from(0), Reg::from(0), 0_i16), + Instruction::register2_ext(-1, -2), + Instruction::return_reg(Reg::from(0)), + ]) + .consts([10, 20]), + ) .run() } diff --git a/crates/wasmi/src/engine/translator/tests/op/mod.rs b/crates/wasmi/src/engine/translator/tests/op/mod.rs index f6cdbf4d4d..332b531700 100644 --- a/crates/wasmi/src/engine/translator/tests/op/mod.rs +++ b/crates/wasmi/src/engine/translator/tests/op/mod.rs @@ -1,3 +1,5 @@ +use crate::core::ValType; + /// Macro that turns an iterator over `Option` into an iterator over `T`. /// /// - Filters out all the `None` items yielded by the input iterator. @@ -43,6 +45,7 @@ use super::{ display_wasm::DisplayValueType, driver::ExpectedFunc, swap_cmp_br_ops, + swap_cmp_select_ops, swap_ops, test_binary_consteval, test_binary_reg_imm16_lhs, @@ -237,3 +240,147 @@ fn effective_address32(ptr: u64, offset: u64) -> Address32 { }; addr32 } + +#[derive(Debug, Copy, Clone)] +pub enum CmpOp { + // i32 + I32And, + I32Or, + I32Xor, + I32Eq, + I32Ne, + I32LtS, + I32LtU, + I32LeS, + I32LeU, + I32GtS, + I32GtU, + I32GeS, + I32GeU, + // i64 + I64And, + I64Or, + I64Xor, + I64Eq, + I64Ne, + I64LtS, + I64LtU, + I64LeS, + I64LeU, + I64GtS, + I64GtU, + I64GeS, + I64GeU, + // f32 + F32Eq, + F32Ne, + F32Lt, + F32Le, + F32Gt, + F32Ge, + // f64 + F64Eq, + F64Ne, + F64Lt, + F64Le, + F64Gt, + F64Ge, +} + +impl CmpOp { + /// Returns the Wasm parameter type of the [`CmpOp`]. + pub fn param_ty(self) -> ValType { + match self { + CmpOp::I32And + | CmpOp::I32Or + | CmpOp::I32Xor + | CmpOp::I32Eq + | CmpOp::I32Ne + | CmpOp::I32LtS + | CmpOp::I32LtU + | CmpOp::I32LeS + | CmpOp::I32LeU + | CmpOp::I32GtS + | CmpOp::I32GtU + | CmpOp::I32GeS + | CmpOp::I32GeU => ValType::I32, + CmpOp::I64And + | CmpOp::I64Or + | CmpOp::I64Xor + | CmpOp::I64Eq + | CmpOp::I64Ne + | CmpOp::I64LtS + | CmpOp::I64LtU + | CmpOp::I64LeS + | CmpOp::I64LeU + | CmpOp::I64GtS + | CmpOp::I64GtU + | CmpOp::I64GeS + | CmpOp::I64GeU => ValType::I64, + CmpOp::F32Eq + | CmpOp::F32Ne + | CmpOp::F32Lt + | CmpOp::F32Le + | CmpOp::F32Gt + | CmpOp::F32Ge => ValType::F32, + CmpOp::F64Eq + | CmpOp::F64Ne + | CmpOp::F64Lt + | CmpOp::F64Le + | CmpOp::F64Gt + | CmpOp::F64Ge => ValType::F64, + } + } + + /// Returns the Wasm result type of the [`CmpOp`]. + pub fn result_ty(self) -> ValType { + match self { + CmpOp::I64And | CmpOp::I64Or | CmpOp::I64Xor => ValType::I64, + _ => ValType::I32, + } + } + + /// Returns a string representation of the Wasm operator without type annotation. + pub fn op_str(self) -> &'static str { + match self { + CmpOp::I32And => "and", + CmpOp::I32Or => "or", + CmpOp::I32Xor => "xor", + CmpOp::I32Eq => "eq", + CmpOp::I32Ne => "ne", + CmpOp::I32LtS => "lt_s", + CmpOp::I32LtU => "lt_u", + CmpOp::I32LeS => "le_s", + CmpOp::I32LeU => "le_u", + CmpOp::I32GtS => "gt_s", + CmpOp::I32GtU => "gt_u", + CmpOp::I32GeS => "ge_s", + CmpOp::I32GeU => "ge_u", + CmpOp::I64And => "and", + CmpOp::I64Or => "or", + CmpOp::I64Xor => "xor", + CmpOp::I64Eq => "eq", + CmpOp::I64Ne => "ne", + CmpOp::I64LtS => "lt_s", + CmpOp::I64LtU => "lt_u", + CmpOp::I64LeS => "le_s", + CmpOp::I64LeU => "le_u", + CmpOp::I64GtS => "gt_s", + CmpOp::I64GtU => "gt_u", + CmpOp::I64GeS => "ge_s", + CmpOp::I64GeU => "ge_u", + CmpOp::F32Eq => "eq", + CmpOp::F32Ne => "ne", + CmpOp::F32Lt => "lt", + CmpOp::F32Le => "le", + CmpOp::F32Gt => "gt", + CmpOp::F32Ge => "ge", + CmpOp::F64Eq => "eq", + CmpOp::F64Ne => "ne", + CmpOp::F64Lt => "lt", + CmpOp::F64Le => "le", + CmpOp::F64Gt => "gt", + CmpOp::F64Ge => "ge", + } + } +} diff --git a/crates/wasmi/src/engine/translator/tests/op/select.rs b/crates/wasmi/src/engine/translator/tests/op/select.rs index 1d80cfa753..4cf57fc34d 100644 --- a/crates/wasmi/src/engine/translator/tests/op/select.rs +++ b/crates/wasmi/src/engine/translator/tests/op/select.rs @@ -48,11 +48,11 @@ fn reg() { r#" (module (func (param $condition i32) - (param $lhs {display_ty}) - (param $rhs {display_ty}) + (param $true_val {display_ty}) + (param $false_val {display_ty}) (result {display_ty}) - local.get $lhs - local.get $rhs + local.get $true_val + local.get $false_val local.get $condition {display_select} ) @@ -60,13 +60,13 @@ fn reg() { "#, ); let condition = Reg::from(0); - let lhs = Reg::from(1); - let rhs = Reg::from(2); + let true_val = Reg::from(1); + let false_val = Reg::from(2); let result = Reg::from(3); TranslationTest::new(&wasm) .expect_func_instrs([ - Instruction::select(result, lhs), - Instruction::register2_ext(condition, rhs), + Instruction::select_i32_ne_imm16(result, condition, 0_i16), + Instruction::register2_ext(true_val, false_val), Instruction::return_reg(result), ]) .run(); @@ -262,563 +262,207 @@ fn same_imm() { test_for::(-9.87654321); } -fn test_reg_imm(kind: SelectKind, rhs: T) -> TranslationTest -where - T: WasmTy, - DisplayWasm: Display, -{ - let ty = T::VALUE_TYPE; - let display_ty = DisplayValueType::from(ty); - let display_rhs = DisplayWasm::from(rhs); - let display_select = DisplaySelect::new(kind, ty); - let wasm = format!( - r#" - (module - (func (param $condition i32) (param $lhs {display_ty}) (result {display_ty}) - local.get $lhs - {display_ty}.const {display_rhs} - local.get $condition - {display_select} - ) - ) - "#, - ); - TranslationTest::new(&wasm) -} - -#[test] -#[cfg_attr(miri, ignore)] -fn reg_imm32() { - fn test_for_kind(kind: SelectKind, value: T) - where - T: WasmTy, - DisplayWasm: Display, - AnyConst32: From, - { - let result = Reg::from(2); - let condition = Reg::from(0); - let lhs = Reg::from(1); - let expected = [ - Instruction::select_imm32_rhs(result, lhs), - Instruction::register_and_imm32(condition, value), - Instruction::return_reg(result), - ]; - test_reg_imm(kind, value).expect_func_instrs(expected).run(); - } - - fn test_for(value: T) - where - T: WasmTy, - DisplayWasm: Display, - AnyConst32: From, - { - test_for_kind::(SelectKind::Select, value); - test_for_kind::(SelectKind::TypedSelect, value); - } - - test_for::(0); - test_for::(1); - test_for::(-1); - test_for::(i32::MIN + 1); - test_for::(i32::MIN); - test_for::(i32::MAX - 1); - test_for::(i32::MAX); - - test_for::(0.0); - test_for::(0.25); - test_for::(-0.25); - test_for::(0.3); - test_for::(-0.3); - test_for::(1.0); - test_for::(-1.0); - test_for::(f32::NEG_INFINITY); - test_for::(f32::INFINITY); - test_for::(f32::NAN); - test_for::(f32::EPSILON); -} - -#[test] -#[cfg_attr(miri, ignore)] -fn reg_imm() { - fn test_for_kind(kind: SelectKind, value: T) - where - T: WasmTy, - DisplayWasm: Display, - { - let result = Reg::from(2); - let condition = Reg::from(0); - let lhs = Reg::from(1); - let instrs = [ - Instruction::select(result, lhs), - Instruction::register2_ext(condition, Reg::from(-1)), - Instruction::return_reg(result), - ]; - let expected = ExpectedFunc::new(instrs).consts([value]); - test_reg_imm(kind, value).expect_func(expected).run(); - } - - fn test_for(value: T) - where - T: WasmTy, - DisplayWasm: Display, - { - test_for_kind(SelectKind::Select, value); - test_for_kind(SelectKind::TypedSelect, value); - } - - test_for::(i64::from(i32::MIN) - 1); - test_for::(i64::from(i32::MAX) + 1); - test_for::(i64::MIN + 1); - test_for::(i64::MIN); - test_for::(i64::MAX - 1); - test_for::(i64::MAX); - - test_for::(0.3); - test_for::(-0.3); - test_for::(0.123456789); - test_for::(-0.123456789); - test_for::(9.87654321); - test_for::(-9.87654321); -} - #[test] #[cfg_attr(miri, ignore)] -fn reg_i64imm32() { - fn test_for_kind(kind: SelectKind, value: i64) { - let result = Reg::from(2); - let condition = Reg::from(0); - let lhs = Reg::from(1); - let expected = [ - Instruction::select_i64imm32_rhs(result, lhs), - Instruction::register_and_imm32(condition, i32::try_from(value).unwrap()), - Instruction::return_reg(result), - ]; - test_reg_imm(kind, value).expect_func_instrs(expected).run(); - } - - fn test_for(value: i64) { - test_for_kind(SelectKind::Select, value); - test_for_kind(SelectKind::TypedSelect, value); - } - - test_for(0); - test_for(1); - test_for(-1); - test_for(i64::from(i32::MIN) + 1); - test_for(i64::from(i32::MIN)); - test_for(i64::from(i32::MAX) - 1); - test_for(i64::from(i32::MAX)); -} - -#[test] -#[cfg_attr(miri, ignore)] -fn reg_f64imm32() { - fn test_for_kind(kind: SelectKind, value: f64) { - let result = Reg::from(2); - let condition = Reg::from(0); - let lhs = Reg::from(1); - let expected = [ - Instruction::select_f64imm32_rhs(result, lhs), - Instruction::register_and_imm32(condition, value as f32), - Instruction::return_reg(result), - ]; - test_reg_imm(kind, value).expect_func_instrs(expected).run(); - } - - fn test_for(value: f64) { - test_for_kind(SelectKind::Select, value); - test_for_kind(SelectKind::TypedSelect, value); - } - - test_for(0.0); - test_for(0.25); - test_for(-0.25); - test_for(1.0); - test_for(-1.0); - test_for(f64::NEG_INFINITY); - test_for(f64::INFINITY); - test_for(f64::NAN); - test_for(f64::EPSILON); -} - -fn test_imm_reg(kind: SelectKind, lhs: T) -> TranslationTest -where - T: WasmTy, - DisplayWasm: Display, -{ - let ty = T::VALUE_TYPE; - let display_ty = DisplayValueType::from(ty); - let display_lhs = DisplayWasm::from(lhs); - let display_select = DisplaySelect::new(kind, ty); - let wasm = format!( - r#" - (module - (func (param $condition i32) (param $rhs {display_ty}) (result {display_ty}) - {display_ty}.const {display_lhs} - local.get $rhs - local.get $condition - {display_select} - ) - ) - "#, - ); - TranslationTest::new(&wasm) -} - -#[test] -#[cfg_attr(miri, ignore)] -fn imm32_reg() { - fn test_for_kind(kind: SelectKind, value: T) - where - T: WasmTy, - DisplayWasm: Display, - AnyConst32: From, - { - let result = Reg::from(2); - let condition = Reg::from(0); - let lhs = Reg::from(1); - let expected = [ - Instruction::select_imm32_lhs(result, value), - Instruction::register2_ext(condition, lhs), - Instruction::return_reg(result), - ]; - test_imm_reg(kind, value).expect_func_instrs(expected).run(); - } - - fn test_for(value: T) - where - T: WasmTy, - DisplayWasm: Display, - AnyConst32: From, - { - test_for_kind::(SelectKind::Select, value); - test_for_kind::(SelectKind::TypedSelect, value); - } - - test_for::(0); - test_for::(1); - test_for::(-1); - test_for::(i32::MIN + 1); - test_for::(i32::MIN); - test_for::(i32::MAX - 1); - test_for::(i32::MAX); - - test_for::(0.0); - test_for::(0.25); - test_for::(-0.25); - test_for::(0.3); - test_for::(-0.3); - test_for::(1.0); - test_for::(-1.0); - test_for::(f32::NEG_INFINITY); - test_for::(f32::INFINITY); - test_for::(f32::NAN); - test_for::(f32::EPSILON); -} - -#[test] -#[cfg_attr(miri, ignore)] -fn imm_reg() { - fn test_for_kind(kind: SelectKind, value: T) - where - T: WasmTy, - DisplayWasm: Display, - { - let result = Reg::from(2); - let condition = Reg::from(0); - let lhs = Reg::from(1); - let instrs = [ - Instruction::select(result, Reg::from(-1)), - Instruction::register2_ext(condition, lhs), - Instruction::return_reg(result), - ]; - let expected = ExpectedFunc::new(instrs).consts([value]); - test_imm_reg(kind, value).expect_func(expected).run(); - } - - fn test_for(value: T) - where - T: WasmTy, - DisplayWasm: Display, - { - test_for_kind(SelectKind::Select, value); - test_for_kind(SelectKind::TypedSelect, value); - } - - test_for::(i64::from(i32::MIN) - 1); - test_for::(i64::from(i32::MAX) + 1); - test_for::(i64::MIN + 1); - test_for::(i64::MIN); - test_for::(i64::MAX - 1); - test_for::(i64::MAX); - - test_for::(0.3); - test_for::(-0.3); - test_for::(0.123456789); - test_for::(-0.123456789); - test_for::(9.87654321); - test_for::(-9.87654321); -} - -#[test] -#[cfg_attr(miri, ignore)] -fn i64imm32_reg() { - fn test_for_kind(kind: SelectKind, value: i64) { - let result = Reg::from(2); - let condition = Reg::from(0); - let lhs = Reg::from(1); - let expected = [ - Instruction::select_i64imm32_lhs(result, i32::try_from(value).unwrap()), - Instruction::register2_ext(condition, lhs), - Instruction::return_reg(result), - ]; - test_imm_reg(kind, value).expect_func_instrs(expected).run(); - } - - fn test_for(value: i64) { - test_for_kind(SelectKind::Select, value); - test_for_kind(SelectKind::TypedSelect, value); - } - - test_for(0); - test_for(1); - test_for(-1); - test_for(i64::from(i32::MIN) + 1); - test_for(i64::from(i32::MIN)); - test_for(i64::from(i32::MAX) - 1); - test_for(i64::from(i32::MAX)); -} - -#[test] -#[cfg_attr(miri, ignore)] -fn f64imm32_reg() { - fn test_for_kind(kind: SelectKind, value: f64) { - let result = Reg::from(2); - let condition = Reg::from(0); - let lhs = Reg::from(1); - let expected = [ - Instruction::select_f64imm32_lhs(result, value as f32), - Instruction::register2_ext(condition, lhs), - Instruction::return_reg(result), - ]; - test_imm_reg(kind, value).expect_func_instrs(expected).run(); - } - - fn test_for(value: f64) { - test_for_kind(SelectKind::Select, value); - test_for_kind(SelectKind::TypedSelect, value); - } - - test_for(0.0); - test_for(0.25); - test_for(-0.25); - test_for(1.0); - test_for(-1.0); - test_for(f64::NEG_INFINITY); - test_for(f64::INFINITY); - test_for(f64::NAN); - test_for(f64::EPSILON); -} - -fn test_both_imm(kind: SelectKind, lhs: T, rhs: T) -> TranslationTest -where - T: WasmTy, - DisplayWasm: Display, -{ - let ty = T::VALUE_TYPE; - let display_ty = DisplayValueType::from(ty); - let display_lhs = DisplayWasm::from(lhs); - let display_rhs = DisplayWasm::from(rhs); - let display_select = DisplaySelect::new(kind, ty); - let wasm = format!( - r#" - (module - (func (param $condition i32) (result {display_ty}) - {display_ty}.const {display_lhs} - {display_ty}.const {display_rhs} - local.get $condition - {display_select} +fn test_cmp_select() { + fn run_test( + op: CmpOp, + kind: SelectKind, + result_ty: ValType, + expected: fn(result: Reg, lhs: Reg, rhs: Reg) -> Instruction, + ) { + let cmp_input_ty = op.param_ty(); + let cmp_result_ty = op.result_ty(); + let cmp_input_ty = DisplayValueType::from(cmp_input_ty); + let cmp_result_ty = DisplayValueType::from(cmp_result_ty); + let cmp_op = op.op_str(); + + let select_op = DisplaySelect::new(kind, result_ty); + let result_ty = DisplayValueType::from(result_ty); + let wasm = format!( + r#" + (module + (func + (param $lhs {cmp_input_ty}) (param $rhs {cmp_input_ty}) + (param $true_val {result_ty}) (param $false_val {result_ty}) + (result {result_ty}) + ({select_op} + (local.get $true_val) + (local.get $false_val) + ({cmp_result_ty}.ne + ({cmp_input_ty}.{cmp_op} (local.get $lhs) (local.get $rhs)) + ({cmp_result_ty}.const 0) + ) + ) + ) ) - ) - "#, - ); - TranslationTest::new(&wasm) -} - -#[test] -#[cfg_attr(miri, ignore)] -fn both_imm32() { - fn test_for_kind(kind: SelectKind, lhs: T, rhs: T) - where - T: WasmTy, - DisplayWasm: Display, - AnyConst32: From, - { - let result = Reg::from(1); - let condition = Reg::from(0); - let lhs32 = AnyConst32::from(lhs); - let rhs32 = AnyConst32::from(rhs); - let expected = [ - Instruction::select_imm32(result, lhs32), - Instruction::register_and_imm32(condition, rhs32), - Instruction::return_reg(result), - ]; - test_both_imm(kind, lhs, rhs) - .expect_func_instrs(expected) + "#, + ); + TranslationTest::new(&wasm) + .expect_func_instrs([ + expected(Reg::from(4), Reg::from(0), Reg::from(1)), + Instruction::register2_ext(2, 3), + Instruction::return_reg(4), + ]) .run(); } - fn test_for(lhs: T, rhs: T) - where - T: WasmTy, - DisplayWasm: Display, - AnyConst32: From, - { - test_for_kind(SelectKind::Select, lhs, rhs); - test_for_kind(SelectKind::Select, rhs, lhs); - test_for_kind(SelectKind::TypedSelect, lhs, rhs); - test_for_kind(SelectKind::TypedSelect, rhs, lhs); - } - - test_for::(0, 1); - test_for::(-5, 42); - test_for::(i32::MIN + 1, i32::MAX - 1); - test_for::(i32::MIN, i32::MAX); - - test_for::(0.0, 1.0); - test_for::(0.3, -0.3); - test_for::(f32::NEG_INFINITY, f32::INFINITY); - test_for::(f32::NAN, f32::EPSILON); -} - -#[test] -#[cfg_attr(miri, ignore)] -fn both_imm() { - fn test_for_kind(kind: SelectKind, lhs: T, rhs: T) - where - T: WasmTy, - DisplayWasm: Display, - { - let result = Reg::from(1); - let condition = Reg::from(0); - let lhs_reg = Reg::from(-1); - let rhs_reg = Reg::from(-2); - let instrs = [ - Instruction::select(result, lhs_reg), - Instruction::register2_ext(condition, rhs_reg), - Instruction::return_reg(result), - ]; - test_both_imm(kind, lhs, rhs) - .expect_func(ExpectedFunc::new(instrs).consts([lhs, rhs])) - .run(); + #[rustfmt::skip] + fn test_for_each_cmp(kind: SelectKind, ty: ValType) { + for (op, expected) in [ + (CmpOp::I32And, Instruction::select_i32_and as fn(Reg, Reg, Reg) -> Instruction), + (CmpOp::I32Or, Instruction::select_i32_or), + (CmpOp::I32Xor, Instruction::select_i32_xor), + (CmpOp::I32Eq, Instruction::select_i32_eq), + (CmpOp::I32Ne, Instruction::select_i32_ne), + (CmpOp::I32LtS, Instruction::select_i32_lt_s), + (CmpOp::I32LtU, Instruction::select_i32_lt_u), + (CmpOp::I32LeS, Instruction::select_i32_le_s), + (CmpOp::I32LeU, Instruction::select_i32_le_u), + (CmpOp::I32GtS, swap_cmp_select_ops!(Instruction::select_i32_lt_s)), + (CmpOp::I32GtU, swap_cmp_select_ops!(Instruction::select_i32_lt_u)), + (CmpOp::I32GeS, swap_cmp_select_ops!(Instruction::select_i32_le_s)), + (CmpOp::I32GeU, swap_cmp_select_ops!(Instruction::select_i32_le_u)), + (CmpOp::I64And, Instruction::select_i64_and), + (CmpOp::I64Or, Instruction::select_i64_or), + (CmpOp::I64Xor, Instruction::select_i64_xor), + (CmpOp::I64Eq, Instruction::select_i64_eq), + (CmpOp::I64Ne, Instruction::select_i64_ne), + (CmpOp::I64LtS, Instruction::select_i64_lt_s), + (CmpOp::I64LtU, Instruction::select_i64_lt_u), + (CmpOp::I64LeS, Instruction::select_i64_le_s), + (CmpOp::I64LeU, Instruction::select_i64_le_u), + (CmpOp::I64GtS, swap_cmp_select_ops!(Instruction::select_i64_lt_s)), + (CmpOp::I64GtU, swap_cmp_select_ops!(Instruction::select_i64_lt_u)), + (CmpOp::I64GeS, swap_cmp_select_ops!(Instruction::select_i64_le_s)), + (CmpOp::I64GeU, swap_cmp_select_ops!(Instruction::select_i64_le_u)), + (CmpOp::F32Eq, Instruction::select_f32_eq), + (CmpOp::F32Ne, Instruction::select_f32_ne), + (CmpOp::F32Lt, Instruction::select_f32_lt), + (CmpOp::F32Le, Instruction::select_f32_le), + (CmpOp::F32Gt, swap_cmp_select_ops!(Instruction::select_f32_lt)), + (CmpOp::F32Ge, swap_cmp_select_ops!(Instruction::select_f32_le)), + (CmpOp::F64Eq, Instruction::select_f64_eq), + (CmpOp::F64Ne, Instruction::select_f64_ne), + (CmpOp::F64Lt, Instruction::select_f64_lt), + (CmpOp::F64Le, Instruction::select_f64_le), + (CmpOp::F64Gt, swap_cmp_select_ops!(Instruction::select_f64_lt)), + (CmpOp::F64Ge, swap_cmp_select_ops!(Instruction::select_f64_le)), + ] { + run_test(op, kind, ty, expected) + } } - fn test_for(lhs: T, rhs: T) - where - T: WasmTy, - DisplayWasm: Display, - { - test_for_kind(SelectKind::Select, lhs, rhs); - test_for_kind(SelectKind::Select, rhs, lhs); - test_for_kind(SelectKind::TypedSelect, lhs, rhs); - test_for_kind(SelectKind::TypedSelect, rhs, lhs); + fn test_for(kind: SelectKind) { + for ty in [ValType::I32, ValType::I64, ValType::F32, ValType::F64] { + test_for_each_cmp(kind, ty); + } } - test_for::(i64::from(i32::MIN) - 1, i64::from(i32::MAX) + 1); - test_for::(i64::MIN, i64::MAX); - - test_for::(0.3, -0.3); - test_for::(0.123456789, -0.987654321); + test_for(SelectKind::Select); + test_for(SelectKind::TypedSelect); + test_for_each_cmp(SelectKind::TypedSelect, ValType::FuncRef); + test_for_each_cmp(SelectKind::TypedSelect, ValType::ExternRef); } #[test] #[cfg_attr(miri, ignore)] -fn both_i64imm32() { - fn test_for_kind(kind: SelectKind, lhs: i64, rhs: i64) { - let result = Reg::from(1); - let condition = Reg::from(0); - let lhs32 = i64imm32(lhs); - let rhs32 = i64imm32(rhs); - let expected = [ - Instruction::select_i64imm32(result, lhs32), - Instruction::register_and_imm32(condition, rhs32), - Instruction::return_reg(result), - ]; - test_both_imm(kind, lhs, rhs) - .expect_func_instrs(expected) +fn test_cmp_select_eqz() { + fn run_test( + op: CmpOp, + kind: SelectKind, + result_ty: ValType, + expected: fn(result: Reg, lhs: Reg, rhs: Reg) -> Instruction, + ) { + let cmp_input_ty = op.param_ty(); + let cmp_result_ty = op.result_ty(); + let cmp_input_ty = DisplayValueType::from(cmp_input_ty); + let cmp_result_ty = DisplayValueType::from(cmp_result_ty); + let cmp_op = op.op_str(); + + let select_op = DisplaySelect::new(kind, result_ty); + let result_ty = DisplayValueType::from(result_ty); + let wasm = format!( + r#" + (module + (func + (param $lhs {cmp_input_ty}) (param $rhs {cmp_input_ty}) + (param $true_val {result_ty}) (param $false_val {result_ty}) + (result {result_ty}) + ({select_op} + (local.get $true_val) + (local.get $false_val) + ({cmp_result_ty}.eqz + ({cmp_input_ty}.{cmp_op} (local.get $lhs) (local.get $rhs)) + ) + ) + ) + ) + "#, + ); + TranslationTest::new(&wasm) + .expect_func_instrs([ + expected(Reg::from(4), Reg::from(0), Reg::from(1)), + Instruction::register2_ext(2, 3), + Instruction::return_reg(4), + ]) .run(); } - fn test_for(lhs: i64, rhs: i64) { - test_for_kind(SelectKind::Select, lhs, rhs); - test_for_kind(SelectKind::Select, rhs, lhs); - test_for_kind(SelectKind::TypedSelect, lhs, rhs); - test_for_kind(SelectKind::TypedSelect, rhs, lhs); - } - - test_for(0, 1); - test_for(-5, 42); - test_for(i64::from(i32::MIN) + 1, i64::from(i32::MAX) - 1); - test_for(i64::from(i32::MIN), i64::from(i32::MAX)); -} - -#[test] -#[cfg_attr(miri, ignore)] -fn both_f64imm32() { - fn test_for_kind(kind: SelectKind, lhs: f64, rhs: f64) { - let result = Reg::from(1); - let condition = Reg::from(0); - let lhs32 = f64imm32(lhs); - let rhs32 = f64imm32(rhs); - let expected = [ - Instruction::select_f64imm32(result, lhs32), - Instruction::register_and_imm32(condition, rhs32), - Instruction::return_reg(result), - ]; - test_both_imm(kind, lhs, rhs) - .expect_func_instrs(expected) - .run(); + #[rustfmt::skip] + fn test_for_each_cmp(kind: SelectKind, ty: ValType) { + for (op, expected) in [ + (CmpOp::I32And, Instruction::select_i32_nand as fn(Reg, Reg, Reg) -> Instruction), + (CmpOp::I32Or, Instruction::select_i32_nor), + (CmpOp::I32Xor, Instruction::select_i32_xnor), + (CmpOp::I32Eq, Instruction::select_i32_ne), + (CmpOp::I32Ne, Instruction::select_i32_eq), + (CmpOp::I32LtS, swap_cmp_select_ops!(Instruction::select_i32_le_s)), + (CmpOp::I32LtU, swap_cmp_select_ops!(Instruction::select_i32_le_u)), + (CmpOp::I32LeS, swap_cmp_select_ops!(Instruction::select_i32_lt_s)), + (CmpOp::I32LeU, swap_cmp_select_ops!(Instruction::select_i32_lt_u)), + (CmpOp::I32GtS, Instruction::select_i32_le_s), + (CmpOp::I32GtU, Instruction::select_i32_le_u), + (CmpOp::I32GeS, Instruction::select_i32_lt_s), + (CmpOp::I32GeU, Instruction::select_i32_lt_u), + (CmpOp::I64And, Instruction::select_i64_nand), + (CmpOp::I64Or, Instruction::select_i64_nor), + (CmpOp::I64Xor, Instruction::select_i64_xnor), + (CmpOp::I64Eq, Instruction::select_i64_ne), + (CmpOp::I64Ne, Instruction::select_i64_eq), + (CmpOp::I64LtS, swap_cmp_select_ops!(Instruction::select_i64_le_s)), + (CmpOp::I64LtU, swap_cmp_select_ops!(Instruction::select_i64_le_u)), + (CmpOp::I64LeS, swap_cmp_select_ops!(Instruction::select_i64_lt_s)), + (CmpOp::I64LeU, swap_cmp_select_ops!(Instruction::select_i64_lt_u)), + (CmpOp::I64GtS, Instruction::select_i64_le_s), + (CmpOp::I64GtU, Instruction::select_i64_le_u), + (CmpOp::I64GeS, Instruction::select_i64_lt_s), + (CmpOp::I64GeU, Instruction::select_i64_lt_u), + (CmpOp::F32Eq, Instruction::select_f32_ne), + (CmpOp::F32Ne, Instruction::select_f32_eq), + (CmpOp::F32Lt, Instruction::select_f32_not_lt), + (CmpOp::F32Le, Instruction::select_f32_not_le), + (CmpOp::F32Gt, swap_cmp_select_ops!(Instruction::select_f32_not_lt)), + (CmpOp::F32Ge, swap_cmp_select_ops!(Instruction::select_f32_not_le)), + (CmpOp::F64Eq, Instruction::select_f64_ne), + (CmpOp::F64Ne, Instruction::select_f64_eq), + (CmpOp::F64Lt, Instruction::select_f64_not_lt), + (CmpOp::F64Le, Instruction::select_f64_not_le), + (CmpOp::F64Gt, swap_cmp_select_ops!(Instruction::select_f64_not_lt)), + (CmpOp::F64Ge, swap_cmp_select_ops!(Instruction::select_f64_not_le)), + ] { + run_test(op, kind, ty, expected) + } } - fn test_for(lhs: f64, rhs: f64) { - test_for_kind(SelectKind::Select, lhs, rhs); - test_for_kind(SelectKind::Select, rhs, lhs); - test_for_kind(SelectKind::TypedSelect, lhs, rhs); - test_for_kind(SelectKind::TypedSelect, rhs, lhs); + fn test_for(kind: SelectKind) { + for ty in [ValType::I32, ValType::I64, ValType::F32, ValType::F64] { + test_for_each_cmp(kind, ty); + } } - test_for(0.0, 1.0); - test_for(-5.5, 42.25); - test_for(f64::NEG_INFINITY, f64::INFINITY); - test_for(f64::NAN, f64::EPSILON); -} - -#[test] -#[cfg_attr(miri, ignore)] -fn fuzz_fail_01() { - let wasm = r#" - (module - (func (export "test") (param i32) (result i32) - (i32.popcnt (local.get 0)) ;; case: true (i32.const 0) - (i32.clz (i32.eqz (local.get 0))) ;; case: false (i32.const 31) - (i32.const 0) ;; condition (i32.const 0) - (select) ;; case: true (i32.const 31) - (i32.const 0) ;; case: false (i32.const 0) - (i32.eqz (local.get 0)) ;; condition (i32.const 1) - (select) - ) - ) - "#; - TranslationTest::new(wasm) - .expect_func_instrs([ - Instruction::i32_popcnt(1, 0), - Instruction::i32_eq_imm16(2, 0, 0_i16), - Instruction::i32_clz(2, 2), - Instruction::copy(1, 2), - Instruction::i32_eq_imm16(2, 0, 0_i16), - Instruction::select_imm32_rhs(1, 1), - Instruction::register_and_imm32(2, 0_i32), - Instruction::return_reg(1), - ]) - .run(); + test_for(SelectKind::Select); + test_for(SelectKind::TypedSelect); + test_for_each_cmp(SelectKind::TypedSelect, ValType::FuncRef); + test_for_each_cmp(SelectKind::TypedSelect, ValType::ExternRef); }