Merge branch 'paritytech:master' into master

crates/polkavm-disassembler/src/lib.rs

@@ -545,7 +545,7 @@ mod tests {
             "// Code size = 18 bytes",
             "",
             "      : @0 [export #0: 'main'] (gas: 5)",
-            "     0: u32 [0x20000] = 305419896",
+            "     0: u32 [0x20000] = 0x12345678",
             "     9: s0 = a0 + a1",
             "    12: ecalli 0 // 'hostcall'",
             "    13: a0 = a0 + s0",

crates/polkavm-linker/src/lib.rs

@@ -11,7 +11,7 @@ mod reader_wrapper;
 mod riscv;
 mod utils;
 
-pub use crate::program_from_elf::{program_from_elf, Config, ProgramFromElfError};
+pub use crate::program_from_elf::{program_from_elf, Config, OptLevel, ProgramFromElfError};
 pub use polkavm_common::program::{ProgramBlob, ProgramParseError, ProgramParts};
 
 pub static TARGET_JSON_32_BIT: &str = include_str!("../riscv32emac-unknown-none-polkavm.json");

crates/polkavm-linker/src/program_from_elf.rs

@@ -1198,6 +1198,7 @@ fn extract_memory_config<H>(
     sections_bss: &[SectionIndex],
     sections_min_stack_size: &[SectionIndex],
     base_address_for_section: &mut HashMap<SectionIndex, u64>,
+    mut min_stack_size: u32,
 ) -> Result<MemoryConfig, ProgramFromElfError>
 where
     H: object::read::elf::FileHeader<Endian = object::LittleEndian>,
@@ -1223,7 +1224,6 @@ where
         sections_rw_data.iter().copied().chain(sections_bss.iter().copied()),
     );
 
-    let mut min_stack_size = VM_MIN_PAGE_SIZE * 2;
     for &section_index in sections_min_stack_size {
         let section = elf.section_by_index(section_index);
         let data = section.data();
@@ -5599,12 +5599,12 @@ mod test {
             let mut all_blocks = resolve_basic_block_references(&data_sections_set, &section_to_block, &all_blocks).unwrap();
             let mut reachability_graph =
                 calculate_reachability(&section_to_block, &all_blocks, &data_sections_set, &exports, &relocations).unwrap();
-            if config.optimize {
+            if matches!(config.opt_level, OptLevel::O2) {
                 optimize_program(&config, &elf, &imports, &mut all_blocks, &mut reachability_graph, &mut exports);
             }
             let mut used_blocks = collect_used_blocks(&all_blocks, &reachability_graph);
 
-            if config.optimize {
+            if matches!(config.opt_level, OptLevel::O2) {
                 used_blocks = add_missing_fallthrough_blocks(&mut all_blocks, &mut reachability_graph, used_blocks);
                 merge_consecutive_fallthrough_blocks(&mut all_blocks, &mut reachability_graph, &mut section_to_block, &mut used_blocks);
                 replace_immediates_with_registers(&mut all_blocks, &imports, &used_blocks);
@@ -5688,11 +5688,11 @@ mod test {
             expected_disassembly: &str,
         ) {
             let mut unopt = self.build(Config {
-                optimize: false,
+                opt_level: OptLevel::O0,
                 ..Config::default()
             });
             let mut opt = self.build(Config {
-                optimize: true,
+                opt_level: OptLevel::O2,
                 ..Config::default()
             });
 
@@ -7476,9 +7476,9 @@ fn emit_code(
                                 K::Mul32 => I::mul_imm_32(dst, src1, src2),
                                 K::Mul32AndSignExtend => I::mul_imm_32(dst, src1, src2),
                                 K::Mul64 => I::mul_imm_64(dst, src1, src2),
-                                K::RotateRight32 => I::rotate_right_32_imm(dst, src1, src2),
-                                K::RotateRight32AndSignExtend => I::rotate_right_32_imm(dst, src1, src2),
-                                K::RotateRight64 => I::rotate_right_64_imm(dst, src1, src2),
+                                K::RotateRight32 => I::rotate_right_imm_32(dst, src1, src2),
+                                K::RotateRight32AndSignExtend => I::rotate_right_imm_32(dst, src1, src2),
+                                K::RotateRight64 => I::rotate_right_imm_64(dst, src1, src2),
                             }
                         }
                         (RegImm::Imm(src1), RegImm::Reg(src2)) => {
@@ -7512,9 +7512,9 @@ fn emit_code(
                                 K::ShiftArithmeticRight32AndSignExtend => I::shift_arithmetic_right_imm_alt_32(dst, src2, src1),
                                 K::ShiftArithmeticRight64 => I::shift_arithmetic_right_imm_alt_64(dst, src2, src1),
 
-                                K::RotateRight32 => I::rotate_right_32_imm_alt(dst, src2, src1),
-                                K::RotateRight32AndSignExtend => I::rotate_right_32_imm_alt(dst, src2, src1),
-                                K::RotateRight64 => I::rotate_right_64_imm_alt(dst, src2, src1),
+                                K::RotateRight32 => I::rotate_right_imm_alt_32(dst, src2, src1),
+                                K::RotateRight32AndSignExtend => I::rotate_right_imm_alt_32(dst, src2, src1),
+                                K::RotateRight64 => I::rotate_right_imm_alt_64(dst, src2, src1),
                             }
                         }
                         (RegImm::Imm(src1), RegImm::Imm(src2)) => {
@@ -8752,22 +8752,31 @@ where
     Ok(functions)
 }
 
+#[derive(Copy, Clone, PartialEq, Eq, Debug)]
+pub enum OptLevel {
+    O0,
+    O1,
+    O2,
+}
+
 pub struct Config {
     strip: bool,
-    optimize: bool,
+    opt_level: OptLevel,
     inline_threshold: usize,
     elide_unnecessary_loads: bool,
     dispatch_table: Vec<Vec<u8>>,
+    min_stack_size: u32,
 }
 
 impl Default for Config {
     fn default() -> Self {
         Config {
             strip: false,
-            optimize: true,
+            opt_level: OptLevel::O2,
             inline_threshold: 2,
             elide_unnecessary_loads: true,
             dispatch_table: Vec::new(),
+            min_stack_size: VM_MIN_PAGE_SIZE * 2,
         }
     }
 }
@@ -8779,7 +8788,12 @@ impl Config {
     }
 
     pub fn set_optimize(&mut self, value: bool) -> &mut Self {
-        self.optimize = value;
+        self.opt_level = if value { OptLevel::O2 } else { OptLevel::O0 };
+        self
+    }
+
+    pub fn set_opt_level(&mut self, value: OptLevel) -> &mut Self {
+        self.opt_level = value;
         self
     }
 
@@ -8797,6 +8811,11 @@ impl Config {
         self.dispatch_table = dispatch_table;
         self
     }
+
+    pub fn set_min_stack_size(&mut self, value: u32) -> &mut Self {
+        self.min_stack_size = value;
+        self
+    }
 }
 
 pub fn program_from_elf(config: Config, data: &[u8]) -> Result<Vec<u8>, ProgramFromElfError> {
@@ -9002,9 +9021,15 @@ where
     let mut used_blocks;
 
     let mut regspill_size = 0;
-    if config.optimize {
+    if matches!(config.opt_level, OptLevel::O1 | OptLevel::O2) {
         reachability_graph = calculate_reachability(&section_to_block, &all_blocks, &data_sections_set, &exports, &relocations)?;
-        optimize_program(&config, &elf, &imports, &mut all_blocks, &mut reachability_graph, &mut exports);
+        if matches!(config.opt_level, OptLevel::O2) {
+            optimize_program(&config, &elf, &imports, &mut all_blocks, &mut reachability_graph, &mut exports);
+        } else {
+            for current in (0..all_blocks.len()).map(BlockTarget::from_raw) {
+                perform_nop_elimination(&mut all_blocks, current);
+            }
+        }
         used_blocks = collect_used_blocks(&all_blocks, &reachability_graph);
         spill_fake_registers(
             section_regspill,
@@ -9017,7 +9042,9 @@ where
         );
         used_blocks = add_missing_fallthrough_blocks(&mut all_blocks, &mut reachability_graph, used_blocks);
         merge_consecutive_fallthrough_blocks(&mut all_blocks, &mut reachability_graph, &mut section_to_block, &mut used_blocks);
-        replace_immediates_with_registers(&mut all_blocks, &imports, &used_blocks);
+        if matches!(config.opt_level, OptLevel::O2) {
+            replace_immediates_with_registers(&mut all_blocks, &imports, &used_blocks);
+        }
 
         let expected_reachability_graph =
             calculate_reachability(&section_to_block, &all_blocks, &data_sections_set, &exports, &relocations)?;
@@ -9174,6 +9201,7 @@ where
         &sections_bss,
         &sections_min_stack_size,
         &mut base_address_for_section,
+        config.min_stack_size,
     )?;
 
     log::trace!("Memory configuration: {:#?}", memory_config);
@@ -9188,7 +9216,7 @@ where
         &used_blocks,
         &used_imports,
         &jump_target_for_block,
-        config.optimize,
+        matches!(config.opt_level, OptLevel::O2),
         is_rv64,
     )?;
 

crates/polkavm/src/api.rs

@@ -1359,6 +1359,37 @@ impl RawInstance {
         result
     }
 
+    /// Read-only protects a given memory region.
+    ///
+    /// Is only supported when dynamic paging is enabled.
+    pub fn protect_memory(&mut self, address: u32, length: u32) -> Result<(), MemoryAccessError> {
+        if !self.module.is_dynamic_paging() {
+            return Err(MemoryAccessError::Error(
+                "protecting memory is only possible on modules with dynamic paging".into(),
+            ));
+        }
+
+        if length == 0 {
+            return Ok(());
+        }
+
+        if address < 0x10000 {
+            return Err(MemoryAccessError::OutOfRangeAccess {
+                address,
+                length: u64::from(length),
+            });
+        }
+
+        if u64::from(address) + u64::from(length) > 0x100000000 {
+            return Err(MemoryAccessError::OutOfRangeAccess {
+                address,
+                length: u64::from(length),
+            });
+        }
+
+        access_backend!(self.backend, |mut backend| backend.protect_memory(address, length))
+    }
+
     /// Frees the given page(s).
     ///
     /// `address` must be a multiple of the page size. The value of `length` will be rounded up to the nearest multiple of the page size.

crates/polkavm/src/compiler.rs

@@ -1204,34 +1204,34 @@ where
     }
 
     #[inline(always)]
-    fn rotate_right_32_imm(&mut self, code_offset: u32, args_length: u32, d: RawReg, s: RawReg, c: u32) -> Self::ReturnTy {
+    fn rotate_right_imm_32(&mut self, code_offset: u32, args_length: u32, d: RawReg, s: RawReg, c: u32) -> Self::ReturnTy {
         self.before_instruction(code_offset);
-        self.gas_visitor.rotate_right_32_imm(d, s, c);
-        ArchVisitor(self).rotate_right_32_imm(d, s, c);
+        self.gas_visitor.rotate_right_imm_32(d, s, c);
+        ArchVisitor(self).rotate_right_imm_32(d, s, c);
         self.after_instruction::<CONTINUE_BASIC_BLOCK>(code_offset, args_length);
     }
 
     #[inline(always)]
-    fn rotate_right_32_imm_alt(&mut self, code_offset: u32, args_length: u32, d: RawReg, s: RawReg, c: u32) -> Self::ReturnTy {
+    fn rotate_right_imm_alt_32(&mut self, code_offset: u32, args_length: u32, d: RawReg, s: RawReg, c: u32) -> Self::ReturnTy {
         self.before_instruction(code_offset);
-        self.gas_visitor.rotate_right_32_imm_alt(d, s, c);
-        ArchVisitor(self).rotate_right_32_imm_alt(d, s, c);
+        self.gas_visitor.rotate_right_imm_alt_32(d, s, c);
+        ArchVisitor(self).rotate_right_imm_alt_32(d, s, c);
         self.after_instruction::<CONTINUE_BASIC_BLOCK>(code_offset, args_length);
     }
 
     #[inline(always)]
-    fn rotate_right_64_imm(&mut self, code_offset: u32, args_length: u32, d: RawReg, s: RawReg, c: u32) -> Self::ReturnTy {
+    fn rotate_right_imm_64(&mut self, code_offset: u32, args_length: u32, d: RawReg, s: RawReg, c: u32) -> Self::ReturnTy {
         self.before_instruction(code_offset);
-        self.gas_visitor.rotate_right_64_imm(d, s, c);
-        ArchVisitor(self).rotate_right_64_imm(d, s, c);
+        self.gas_visitor.rotate_right_imm_64(d, s, c);
+        ArchVisitor(self).rotate_right_imm_64(d, s, c);
         self.after_instruction::<CONTINUE_BASIC_BLOCK>(code_offset, args_length);
     }
 
     #[inline(always)]
-    fn rotate_right_64_imm_alt(&mut self, code_offset: u32, args_length: u32, d: RawReg, s: RawReg, c: u32) -> Self::ReturnTy {
+    fn rotate_right_imm_alt_64(&mut self, code_offset: u32, args_length: u32, d: RawReg, s: RawReg, c: u32) -> Self::ReturnTy {
         self.before_instruction(code_offset);
-        self.gas_visitor.rotate_right_64_imm_alt(d, s, c);
-        ArchVisitor(self).rotate_right_64_imm_alt(d, s, c);
+        self.gas_visitor.rotate_right_imm_alt_64(d, s, c);
+        ArchVisitor(self).rotate_right_imm_alt_64(d, s, c);
         self.after_instruction::<CONTINUE_BASIC_BLOCK>(code_offset, args_length);
     }
 

crates/polkavm/src/compiler/amd64.rs

@@ -2041,12 +2041,12 @@ where
     }
 
     #[inline(always)]
-    pub fn rotate_right_32_imm(&mut self, d: RawReg, s: RawReg, c: u32) {
+    pub fn rotate_right_imm_32(&mut self, d: RawReg, s: RawReg, c: u32) {
         self.rotate_right_imm_generic(RegSize::R32, d, s, c);
     }
 
     #[inline(always)]
-    pub fn rotate_right_64_imm(&mut self, d: RawReg, s: RawReg, c: u32) {
+    pub fn rotate_right_imm_64(&mut self, d: RawReg, s: RawReg, c: u32) {
         assert_eq!(B::BITNESS, Bitness::B64);
         self.rotate_right_imm_generic(RegSize::R64, d, s, c);
     }
@@ -2071,12 +2071,12 @@ where
     }
 
     #[inline(always)]
-    pub fn rotate_right_32_imm_alt(&mut self, d: RawReg, s: RawReg, c: u32) {
+    pub fn rotate_right_imm_alt_32(&mut self, d: RawReg, s: RawReg, c: u32) {
         self.rotate_right_imm_alt_generic(RegSize::R32, d, s, c);
     }
 
     #[inline(always)]
-    pub fn rotate_right_64_imm_alt(&mut self, d: RawReg, s: RawReg, c: u32) {
+    pub fn rotate_right_imm_alt_64(&mut self, d: RawReg, s: RawReg, c: u32) {
         assert_eq!(B::BITNESS, Bitness::B64);
         self.rotate_right_imm_alt_generic(RegSize::R64, d, s, c);
     }

crates/polkavm/src/gas.rs

@@ -430,22 +430,22 @@ impl InstructionVisitor for GasVisitor {
     }
 
     #[inline(always)]
-    fn rotate_right_32_imm(&mut self, _d: RawReg, _s: RawReg, _c: u32) -> Self::ReturnTy {
+    fn rotate_right_imm_32(&mut self, _d: RawReg, _s: RawReg, _c: u32) -> Self::ReturnTy {
         self.cost += 1;
     }
 
     #[inline(always)]
-    fn rotate_right_32_imm_alt(&mut self, _d: RawReg, _s: RawReg, _c: u32) -> Self::ReturnTy {
+    fn rotate_right_imm_alt_32(&mut self, _d: RawReg, _s: RawReg, _c: u32) -> Self::ReturnTy {
         self.cost += 1;
     }
 
     #[inline(always)]
-    fn rotate_right_64_imm(&mut self, _d: RawReg, _s: RawReg, _c: u32) -> Self::ReturnTy {
+    fn rotate_right_imm_64(&mut self, _d: RawReg, _s: RawReg, _c: u32) -> Self::ReturnTy {
         self.cost += 1;
     }
 
     #[inline(always)]
-    fn rotate_right_64_imm_alt(&mut self, _d: RawReg, _s: RawReg, _c: u32) -> Self::ReturnTy {
+    fn rotate_right_imm_alt_64(&mut self, _d: RawReg, _s: RawReg, _c: u32) -> Self::ReturnTy {
         self.cost += 1;
     }