From 24cb4808704af0973f37f7f9dc960a26ef27e104 Mon Sep 17 00:00:00 2001 From: Jan Bujak Date: Mon, 10 Nov 2025 11:03:43 +0000 Subject: [PATCH 01/21] spectool: remove the non-functional 'test' subcommand --- tools/spectool/src/main.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tools/spectool/src/main.rs b/tools/spectool/src/main.rs index 719c5a83..17334c73 100644 --- a/tools/spectool/src/main.rs +++ b/tools/spectool/src/main.rs @@ -15,7 +15,6 @@ use std::path::{Path, PathBuf}; #[clap(version)] enum Args { Generate, - Test, } fn main() { @@ -24,7 +23,6 @@ fn main() { let args = Args::parse(); match args { Args::Generate => main_generate(), - Args::Test => main_test(), } } @@ -660,7 +658,3 @@ fn main_generate() { std::process::exit(1); } } - -fn main_test() { - todo!(); -} From 85d5441e90c062520cee2ea7d0ca80c1f12bf1f4 Mon Sep 17 00:00:00 2001 From: Jan Bujak Date: Mon, 10 Nov 2025 11:03:44 +0000 Subject: [PATCH 02/21] Fix guest program compilation on rustc 1.91+ --- ci/jobs/clippy.sh | 2 +- crates/polkavm-linker/src/lib.rs | 142 +++++++++++++++++- .../riscv32emac-unknown-none-polkavm.json | 37 +++++ .../riscv64emac-unknown-none-polkavm.json | 37 +++++ .../riscv32emac-unknown-none-polkavm.json | 0 .../riscv64emac-unknown-none-polkavm.json | 0 crates/polkavm/src/tests.rs | 8 +- guest-programs/.cargo/config.toml | 2 +- guest-programs/build-benchmarks.sh | 4 +- tools/polkatool/src/main.rs | 8 +- 10 files changed, 223 insertions(+), 17 deletions(-) create mode 100644 crates/polkavm-linker/targets/1_91/riscv32emac-unknown-none-polkavm.json create mode 100644 crates/polkavm-linker/targets/1_91/riscv64emac-unknown-none-polkavm.json rename crates/polkavm-linker/{ => targets/legacy}/riscv32emac-unknown-none-polkavm.json (100%) rename crates/polkavm-linker/{ => targets/legacy}/riscv64emac-unknown-none-polkavm.json (100%) diff --git a/ci/jobs/clippy.sh b/ci/jobs/clippy.sh index 0cb858ad..4dabf43e 100755 --- a/ci/jobs/clippy.sh +++ b/ci/jobs/clippy.sh @@ -46,7 +46,7 @@ echo ">> cargo clippy (guests)" cd guest-programs cargo clippy \ -Z build-std=core,alloc \ - --target "$PWD/../crates/polkavm-linker/riscv32emac-unknown-none-polkavm.json" \ + --target "$PWD/../crates/polkavm-linker/targets/legacy/riscv32emac-unknown-none-polkavm.json" \ --all cd ../.. diff --git a/crates/polkavm-linker/src/lib.rs b/crates/polkavm-linker/src/lib.rs index fe87925e..f3c06f20 100644 --- a/crates/polkavm-linker/src/lib.rs +++ b/crates/polkavm-linker/src/lib.rs @@ -15,16 +15,106 @@ pub use crate::program_from_elf::{program_from_elf, Config, OptLevel, ProgramFro pub use polkavm_common::assembler::assemble; pub use polkavm_common::program::{ProgramBlob, ProgramParseError, ProgramParts}; -pub static TARGET_JSON_32_BIT: &str = include_str!("../riscv32emac-unknown-none-polkavm.json"); -pub static TARGET_JSON_64_BIT: &str = include_str!("../riscv64emac-unknown-none-polkavm.json"); +pub static TARGET_JSON_32_BIT_OLD: &str = include_str!("../targets/legacy/riscv32emac-unknown-none-polkavm.json"); +pub static TARGET_JSON_64_BIT_OLD: &str = include_str!("../targets/legacy/riscv64emac-unknown-none-polkavm.json"); + +pub static TARGET_JSON_32_BIT_NEW: &str = include_str!("../targets/1_91/riscv32emac-unknown-none-polkavm.json"); +pub static TARGET_JSON_64_BIT_NEW: &str = include_str!("../targets/1_91/riscv64emac-unknown-none-polkavm.json"); + +struct VersionDetector { + major: u32, + minor: u32, + date: u32, + is_nightly: bool, +} + +const YEAR: u32 = 10000; +const MONTH: u32 = 100; + +impl VersionDetector { + fn new() -> Result { + let rustc = std::env::var("RUSTC").unwrap_or_else(|_| "rustc".to_owned()); + let version = std::process::Command::new(&rustc) + .arg("--version") + .output() + .map_err(|err| format!("failed to detect rustc version: failed to run '{rustc} --version': {err}"))? + .stdout; + + let full_version = String::from_utf8(version) + .map_err(|_| format!("failed to detect rustc version: '{rustc} --version' returned non-valid UTF-8"))?; + let full_version = full_version.trim(); + // Examples of version strings: + // 'rustc 1.86.0 (05f9846f8 2025-03-31)' + // 'rustc 1.90.0-nightly (e9182f195 2025-07-13)' + + (move || -> Option { + let version = full_version.split(' ').nth(1)?; + let is_nightly = full_version.contains("-nightly"); + let date = { + let mut it = full_version[full_version.rfind(' ')? + 1..full_version.len() - 1].split('-'); + let year: u32 = it.next()?.parse().ok()?; + let month: u32 = it.next()?.parse().ok()?; + let day: u32 = it.next()?.parse().ok()?; + year * YEAR + month * MONTH + day + }; + let mut it = version.split('.'); + let major: u32 = it.next()?.parse().ok()?; + let minor: u32 = it.next()?.parse().ok()?; + + Some(Self { + major, + minor, + date, + is_nightly, + }) + })() + .ok_or_else(|| format!("failed to detect rustc version: failed to parse output of '{rustc}' --version: {full_version:?}")) + } + + fn check_feature(&self, req_minor: u32, year: u32, month: u32, day: u32) -> bool { + self.major > 1 + || (self.major == 1 && self.minor >= req_minor && !self.is_nightly) + || (self.major == 1 && self.minor >= (req_minor + 1)) + || (self.major == 1 && self.is_nightly && self.date >= year * YEAR + month * MONTH + day) + } +} + +fn target_json_path_impl( + json_name: &str, + json_contents_old: &str, + json_contents_new: &str, + rustc_version: RustcVersion, +) -> Result { + let rustc_version = match rustc_version { + RustcVersion::Autodetect => VersionDetector::new()?, + RustcVersion::Rustc_1_91 => VersionDetector { + major: 1, + minor: 91, + date: 2025 * YEAR + 10 * MONTH + 30, + is_nightly: false, + }, + RustcVersion::Legacy => VersionDetector { + major: 1, + minor: 90, + date: 2025 * YEAR + 9 * MONTH + 18, + is_nightly: false, + }, + }; + + // https://github.com/rust-lang/rust/pull/144443 + let (json_contents, subdirectory) = if rustc_version.check_feature(91, 2025, 9, 1) { + (json_contents_new, "1_91") + } else { + (json_contents_old, "legacy") + }; -fn target_json_path_impl(json_name: &str, json_contents: &str) -> Result { let version = env!("CARGO_PKG_VERSION"); let cache_path = dirs::cache_dir() .or_else(|| std::env::current_dir().ok()) .unwrap_or_else(|| PathBuf::from("./")) .join(".polkavm-linker") - .join(version); + .join(version) + .join(subdirectory); if let Err(error) = std::fs::create_dir_all(&cache_path) { return Err(format!( @@ -48,10 +138,46 @@ fn target_json_path_impl(json_name: &str, json_contents: &str) -> Result Result { - target_json_path_impl("riscv32emac-unknown-none-polkavm.json", TARGET_JSON_32_BIT) +#[allow(non_camel_case_types)] +#[derive(Clone, Default)] +#[non_exhaustive] +pub enum RustcVersion { + #[default] + Autodetect, + Legacy, + Rustc_1_91, +} + +#[derive(Clone)] +#[non_exhaustive] +pub struct TargetJsonArgs { + pub rustc_version: RustcVersion, + pub is_64_bit: bool, } -pub fn target_json_64_path() -> Result { - target_json_path_impl("riscv64emac-unknown-none-polkavm.json", TARGET_JSON_64_BIT) +impl Default for TargetJsonArgs { + fn default() -> Self { + TargetJsonArgs { + rustc_version: Default::default(), + is_64_bit: true, + } + } +} + +pub fn target_json_path(args: TargetJsonArgs) -> Result { + if args.is_64_bit { + target_json_path_impl( + "riscv64emac-unknown-none-polkavm.json", + TARGET_JSON_64_BIT_OLD, + TARGET_JSON_64_BIT_NEW, + args.rustc_version, + ) + } else { + target_json_path_impl( + "riscv32emac-unknown-none-polkavm.json", + TARGET_JSON_32_BIT_OLD, + TARGET_JSON_32_BIT_NEW, + args.rustc_version, + ) + } } diff --git a/crates/polkavm-linker/targets/1_91/riscv32emac-unknown-none-polkavm.json b/crates/polkavm-linker/targets/1_91/riscv32emac-unknown-none-polkavm.json new file mode 100644 index 00000000..d1ae4c38 --- /dev/null +++ b/crates/polkavm-linker/targets/1_91/riscv32emac-unknown-none-polkavm.json @@ -0,0 +1,37 @@ +{ + "arch": "riscv32", + "cpu": "generic-rv32", + "crt-objects-fallback": "false", + "data-layout": "e-m:e-p:32:32-i64:64-n32-S32", + "eh-frame-header": false, + "emit-debug-gdb-scripts": false, + "features": "+e,+m,+a,+c,+zbb,+auipc-addi-fusion,+ld-add-fusion,+lui-addi-fusion,+xtheadcondmov", + "linker": "rust-lld", + "linker-flavor": "ld.lld", + "llvm-abiname": "ilp32e", + "llvm-target": "riscv32", + "max-atomic-width": 64, + "panic-strategy": "abort", + "relocation-model": "pie", + "target-pointer-width": 32, + "singlethread": true, + "pre-link-args": { + "ld": [ + "--emit-relocs", + "--unique", + "--apply-dynamic-relocs", + "--no-allow-shlib-undefined", + "-Bsymbolic" + ] + }, + "env": "polkavm", + "dynamic-linking": true, + "only-cdylib": true, + "position-independent-executables": true, + "static-position-independent-executables": true, + "relro-level": "full", + "default-visibility": "hidden", + "exe-suffix": "", + "dll-prefix": "", + "dll-suffix": ".elf" +} diff --git a/crates/polkavm-linker/targets/1_91/riscv64emac-unknown-none-polkavm.json b/crates/polkavm-linker/targets/1_91/riscv64emac-unknown-none-polkavm.json new file mode 100644 index 00000000..f3e496e2 --- /dev/null +++ b/crates/polkavm-linker/targets/1_91/riscv64emac-unknown-none-polkavm.json @@ -0,0 +1,37 @@ +{ + "arch": "riscv64", + "cpu": "generic-rv64", + "crt-objects-fallback": "false", + "data-layout": "e-m:e-p:64:64-i64:64-i128:128-n32:64-S64", + "eh-frame-header": false, + "emit-debug-gdb-scripts": false, + "features": "+e,+m,+a,+c,+zbb,+auipc-addi-fusion,+ld-add-fusion,+lui-addi-fusion,+xtheadcondmov", + "linker": "rust-lld", + "linker-flavor": "ld.lld", + "llvm-abiname": "lp64e", + "llvm-target": "riscv64", + "max-atomic-width": 64, + "panic-strategy": "abort", + "relocation-model": "pie", + "target-pointer-width": 64, + "singlethread": true, + "pre-link-args": { + "ld": [ + "--emit-relocs", + "--unique", + "--apply-dynamic-relocs", + "--no-allow-shlib-undefined", + "-Bsymbolic" + ] + }, + "env": "polkavm", + "dynamic-linking": true, + "only-cdylib": true, + "position-independent-executables": true, + "static-position-independent-executables": true, + "relro-level": "full", + "default-visibility": "hidden", + "exe-suffix": "", + "dll-prefix": "", + "dll-suffix": ".elf" +} diff --git a/crates/polkavm-linker/riscv32emac-unknown-none-polkavm.json b/crates/polkavm-linker/targets/legacy/riscv32emac-unknown-none-polkavm.json similarity index 100% rename from crates/polkavm-linker/riscv32emac-unknown-none-polkavm.json rename to crates/polkavm-linker/targets/legacy/riscv32emac-unknown-none-polkavm.json diff --git a/crates/polkavm-linker/riscv64emac-unknown-none-polkavm.json b/crates/polkavm-linker/targets/legacy/riscv64emac-unknown-none-polkavm.json similarity index 100% rename from crates/polkavm-linker/riscv64emac-unknown-none-polkavm.json rename to crates/polkavm-linker/targets/legacy/riscv64emac-unknown-none-polkavm.json diff --git a/crates/polkavm/src/tests.rs b/crates/polkavm/src/tests.rs index 0b53316f..420dc8b2 100644 --- a/crates/polkavm/src/tests.rs +++ b/crates/polkavm/src/tests.rs @@ -40,10 +40,14 @@ fn get_test_program(kind: TestProgram, is_64_bit: bool) -> &'static [u8] { .filter(|(k, _)| !["CARGO", "RUSTC", "RUSTUP"].iter().any(|e| k.contains(e))) .collect(); + let mut args = polkavm_linker::TargetJsonArgs::default(); + args.is_64_bit = is_64_bit; + args.rustc_version = polkavm_linker::RustcVersion::Legacy; + let (target, target_path) = if is_64_bit { - ("riscv64emac-unknown-none-polkavm", polkavm_linker::target_json_64_path().unwrap()) + ("riscv64emac-unknown-none-polkavm", polkavm_linker::target_json_path(args).unwrap()) } else { - ("riscv32emac-unknown-none-polkavm", polkavm_linker::target_json_32_path().unwrap()) + ("riscv32emac-unknown-none-polkavm", polkavm_linker::target_json_path(args).unwrap()) }; let (project, filename, profile, is_bin) = match kind { diff --git a/guest-programs/.cargo/config.toml b/guest-programs/.cargo/config.toml index 92a6d380..5c6efe0c 100644 --- a/guest-programs/.cargo/config.toml +++ b/guest-programs/.cargo/config.toml @@ -1,5 +1,5 @@ [build] -target = "../crates/polkavm-linker/riscv32emac-unknown-none-polkavm.json" +target = "../crates/polkavm-linker/targets/legacy/riscv32emac-unknown-none-polkavm.json" [unstable] build-std = ["core", "alloc"] diff --git a/guest-programs/build-benchmarks.sh b/guest-programs/build-benchmarks.sh index 8cbf7738..2173a6d7 100755 --- a/guest-programs/build-benchmarks.sh +++ b/guest-programs/build-benchmarks.sh @@ -66,7 +66,7 @@ build_polkavm() { RUSTFLAGS="$extra_flags" cargo build \ -Z build-std=core,alloc \ - --target "$PWD/../crates/polkavm-linker/riscv32emac-unknown-none-polkavm.json" \ + --target "$PWD/../crates/polkavm-linker/targets/legacy/riscv32emac-unknown-none-polkavm.json" \ -q --release --bin $1 -p $1 pushd .. @@ -81,7 +81,7 @@ build_polkavm() { RUSTFLAGS="$extra_flags" cargo build \ -Z build-std=core,alloc \ - --target "$PWD/../crates/polkavm-linker/riscv64emac-unknown-none-polkavm.json" \ + --target "$PWD/../crates/polkavm-linker/targets/legacy/riscv64emac-unknown-none-polkavm.json" \ -q --release --bin $1 -p $1 pushd .. diff --git a/tools/polkatool/src/main.rs b/tools/polkatool/src/main.rs index 33a03728..e1974707 100644 --- a/tools/polkatool/src/main.rs +++ b/tools/polkatool/src/main.rs @@ -171,11 +171,13 @@ fn main() { Args::Assemble { input, output } => main_assemble(input, output), Args::Stats { inputs } => main_stats(inputs), Args::GetTargetJsonPath { bitness } => { - let result = match bitness { - Bitness::B32 => polkavm_linker::target_json_32_path(), - Bitness::B64 => polkavm_linker::target_json_64_path(), + let mut args = polkavm_linker::TargetJsonArgs::default(); + args.is_64_bit = match bitness { + Bitness::B32 => false, + Bitness::B64 => true, }; + let result = polkavm_linker::target_json_path(args); result.map(|path| print!("{}", path.to_str().unwrap())) } }; From bd8a5daf5d78b0e960dd9dc1d70b90c2f4f52503 Mon Sep 17 00:00:00 2001 From: Jan Bujak Date: Mon, 10 Nov 2025 11:03:45 +0000 Subject: [PATCH 03/21] Add two extra gas cost model tests --- crates/polkavm-common/src/simulator.rs | 38 ++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/crates/polkavm-common/src/simulator.rs b/crates/polkavm-common/src/simulator.rs index 8cec6c35..0913c97d 100644 --- a/crates/polkavm-common/src/simulator.rs +++ b/crates/polkavm-common/src/simulator.rs @@ -2897,4 +2897,42 @@ mod tests { ", ); } + + #[test] + fn test_xor_and_shift() { + assert_timeline( + test_config(), + " + a1 = a1 ^ 0xffffffffffffffff + a1 = a0 >> a1 + fallthrough + ", + " + DeER.. a1 = a1 ^ 0xffffffffffffffff + D=eER. a1 = a0 >> a1 + .DeeER fallthrough + ", + ) + } + + #[test] + fn test_move_reg_decode_slots() { + assert_timeline( + test_config(), + " + s0 = a1 + a0 = a1 + a1 = t0 + a2 = s1 + trap + ", + " + D..... s0 = a1 + D..... a0 = a1 + D..... a1 = t0 + D..... a2 = s1 + .DeeER trap + ", + ) + } } From da2c4c943a20eba255ee06becb5bf044137c7d95 Mon Sep 17 00:00:00 2001 From: Jan Bujak Date: Mon, 10 Nov 2025 11:03:45 +0000 Subject: [PATCH 04/21] spectool: update to allow for multistep tests --- crates/polkavm-common/src/assembler.rs | 18 +- crates/polkavm-common/src/utils.rs | 17 + tools/spectool/src/main.rs | 695 +++++++++++++++++++++---- 3 files changed, 619 insertions(+), 111 deletions(-) diff --git a/crates/polkavm-common/src/assembler.rs b/crates/polkavm-common/src/assembler.rs index 8f32eb3f..d0be8206 100644 --- a/crates/polkavm-common/src/assembler.rs +++ b/crates/polkavm-common/src/assembler.rs @@ -1,5 +1,5 @@ use crate::program::{Instruction, RawReg, Reg}; -use crate::utils::{parse_imm, parse_immediate, parse_reg, ParsedImmediate}; +use crate::utils::{parse_imm, parse_immediate, parse_reg, parse_slice, ParsedImmediate}; use alloc::borrow::ToOwned; use alloc::collections::BTreeMap; use alloc::format; @@ -257,22 +257,6 @@ pub fn assemble(code: &str) -> Result, String> { continue; } - fn parse_slice(text: &str) -> Option> { - let text = text.trim().replace(' ', ""); - if text.len() % 2 != 0 { - return None; - } - - let mut output = Vec::new(); - for chunk in text.as_bytes().chunks(2) { - let chunk = core::str::from_utf8(chunk).ok()?; - let chunk = u8::from_str_radix(chunk, 16).ok()?; - output.push(chunk); - } - - Some(output) - } - if let Some(line) = line.strip_prefix("%ro_data = ") { let Some(value) = parse_slice(line) else { return Err(format!("cannot parse line {nth_line}")); diff --git a/crates/polkavm-common/src/utils.rs b/crates/polkavm-common/src/utils.rs index ef97160b..bbd05c7b 100644 --- a/crates/polkavm-common/src/utils.rs +++ b/crates/polkavm-common/src/utils.rs @@ -346,6 +346,23 @@ pub fn parse_immediate(text: &str) -> Option { } } +#[cfg(feature = "alloc")] +pub fn parse_slice(text: &str) -> Option> { + let text = text.trim().replace(' ', ""); + if text.len() % 2 != 0 { + return None; + } + + let mut output = Vec::new(); + for chunk in text.as_bytes().chunks(2) { + let chunk = core::str::from_utf8(chunk).ok()?; + let chunk = u8::from_str_radix(chunk, 16).ok()?; + output.push(chunk); + } + + Some(output) +} + pub trait GasVisitorT: ParsingVisitor { fn take_block_cost(&mut self) -> Option; fn is_at_start_of_basic_block(&self) -> bool; diff --git a/tools/spectool/src/main.rs b/tools/spectool/src/main.rs index 17334c73..9c9bbd7b 100644 --- a/tools/spectool/src/main.rs +++ b/tools/spectool/src/main.rs @@ -8,6 +8,7 @@ use core::fmt::Write; use polkavm::{CacheModel, CostModelKind, Engine, InterruptKind, Module, ModuleConfig, ProgramBlob, Reg}; use polkavm_common::assembler::assemble; use polkavm_common::program::{asm, ProgramCounter, ProgramParts, ISA64_V1}; +use polkavm_common::utils::parse_slice; use std::collections::BTreeMap; use std::path::{Path, PathBuf}; @@ -29,10 +30,16 @@ fn main() { struct Testcase { disassembly: String, timelines: Vec<(String, u32, u32)>, + initial_page_map: Vec, + final_page_map: Vec, + initial_memory: Vec, + final_memory: Vec, + initial_state: State, + final_state: State, json: TestcaseJson, } -#[derive(serde::Serialize, serde::Deserialize)] +#[derive(PartialEq, Eq, serde::Serialize, serde::Deserialize)] #[serde(rename_all = "kebab-case")] struct Page { address: u32, @@ -40,30 +47,73 @@ struct Page { is_writable: bool, } -#[derive(PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[derive(Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] #[serde(rename_all = "kebab-case")] struct MemoryChunk { address: u32, contents: Vec, } +#[derive(Clone)] +struct State { + status: &'static str, + page_fault_address: Option, + hostcall: Option, + gas: i64, + pc: ProgramCounter, + regs: Vec, + memory: Vec, +} + +fn are_regs_empty(regs: &[Option; 13]) -> bool { + regs.iter().all(|value| value.is_none()) +} + +#[allow(clippy::large_enum_variant)] +#[derive(Clone, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "kebab-case")] +#[serde(tag = "kind")] +enum TestcaseStep { + Run, + Map { + address: u32, + length: u32, + is_writable: bool, + }, + Write { + address: u32, + contents: Vec, + }, + SetReg { + reg: u32, + value: u64, + }, + Assert { + #[serde(skip_serializing_if = "Option::is_none")] + status: Option, + #[serde(skip_serializing_if = "Option::is_none")] + page_fault_address: Option, + #[serde(skip_serializing_if = "Option::is_none")] + hostcall: Option, + #[serde(skip_serializing_if = "Option::is_none")] + gas: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pc: Option, + #[serde(skip_serializing_if = "are_regs_empty")] + regs: [Option; 13], + #[serde(skip_serializing_if = "Option::is_none")] + memory: Option>, + }, +} + #[derive(serde::Serialize, serde::Deserialize)] #[serde(rename_all = "kebab-case")] struct TestcaseJson { name: String, - initial_regs: [u64; 13], initial_pc: u32, - initial_page_map: Vec, - initial_memory: Vec, initial_gas: i64, program: Vec, - expected_status: String, - expected_regs: Vec, - expected_pc: u32, - expected_memory: Vec, - expected_gas: i64, - #[serde(skip_serializing_if = "Option::is_none")] - expected_page_fault_address: Option, + steps: Vec, block_gas_costs: BTreeMap, } @@ -135,6 +185,88 @@ fn parse_pre_post(line: &str, output: &mut PrePost) { } } +fn parse_u32(text: &str) -> Option { + let text = text.trim(); + if let Some(text) = text.strip_prefix("0x") { + u32::from_str_radix(text, 16).ok() + } else if let Some(text) = text.strip_prefix("0b") { + u32::from_str_radix(text, 2).ok() + } else { + text.parse::().ok() + } +} + +fn parse_u64(text: &str) -> Option { + let text = text.trim(); + if let Some(text) = text.strip_prefix("0x") { + u64::from_str_radix(text, 16).ok() + } else if let Some(text) = text.strip_prefix("0b") { + u64::from_str_radix(text, 2).ok() + } else if let Ok(value) = text.parse::() { + Some(value as u64) + } else { + text.parse::().ok() + } +} + +fn parse_step(line: &str, steps: &mut Vec) { + let line = line.trim(); + if line == "run" { + steps.push(TestcaseStep::Run); + return; + } else if let Some(line) = line.strip_prefix("map ") { + let error = "invalid 'step': failed to parse 'map'"; + let line = line.trim(); + let is_writable = if line.strip_prefix("RO ").is_some() { + false + } else if line.strip_prefix("RW ").is_some() { + true + } else { + panic!("{}", error); + }; + + let line = line[3..].trim(); + let mut xs = line.split(".."); + let start = xs.next().expect(error); + let end = xs.next().expect(error); + assert!(xs.next().is_none(), "{}", error); + let start = parse_u32(start).expect(error); + let end = parse_u32(end).expect(error); + if (start % 4096) != 0 || (end % 4096) != 0 { + panic!("invalid 'step': 'map' has address or length that is not page-aligned"); + } + + steps.push(TestcaseStep::Map { + address: start, + length: end - start, + is_writable, + }); + return; + } else if let Some(line) = line.strip_prefix("write ") { + let error = "invalid 'step': failed to parse 'write'"; + let line = line.trim(); + if let Some(index) = line.find(' ') { + let address = line[..index].trim(); + let contents = line[index + 1..].trim(); + let address = parse_u32(address).expect(error); + let contents = parse_slice(contents).expect(error); + steps.push(TestcaseStep::Write { address, contents }); + return; + } + } else if let Some(index) = line.find("=") { + let lhs = line[..index].trim(); + let rhs = line[index + 1..].trim(); + if let Some(reg) = Reg::ALL.iter().position(|reg| reg.name() == lhs) { + if let Some(value) = parse_u64(rhs) { + steps.push(TestcaseStep::SetReg { reg: reg as u32, value }); + return; + } + } + } + + panic!("invalid 'step': failed to parse line: '{line}'"); +} + fn main_generate() { let mut tests = Vec::new(); @@ -159,6 +291,7 @@ fn main_generate() { blob: Vec, pre: PrePost, post: PrePost, + steps: Vec, expected_status: Option<&'static str>, } @@ -168,6 +301,7 @@ fn main_generate() { let mut pre = PrePost::default(); let mut post = PrePost::default(); + let mut steps = Vec::new(); let input = std::fs::read_to_string(&path).unwrap(); let mut input_lines = Vec::new(); @@ -184,9 +318,19 @@ fn main_generate() { continue; } + if let Some(line) = line.strip_prefix("step:") { + parse_step(line, &mut steps); + input_lines.push(""); // Insert dummy line to not mess up the line count. + continue; + } + input_lines.push(line); } + if steps.is_empty() { + steps.push(TestcaseStep::Run); + } + let input = input_lines.join("\n"); let blob = match assemble(&input) { Ok(blob) => blob, @@ -203,6 +347,7 @@ fn main_generate() { blob, pre, post, + steps, expected_status: None, }); } @@ -247,20 +392,24 @@ fn main_generate() { blob, pre: PrePost::default(), post, + steps: vec![TestcaseStep::Run], expected_status: Some("halt"), }); } - for testcase in testcases { + 'next_testcase: for testcase in testcases { let RawTestcase { name, internal_name, blob, pre, post, + mut steps, expected_status, } = testcase; + assert!(!steps.is_empty()); + let initial_gas = pre.gas.unwrap_or(10000); let initial_regs = pre.regs.map(|value| value.unwrap_or(0)); assert!(pre.pc.is_none(), "'pre: pc = ...' is currently unsupported"); @@ -279,37 +428,56 @@ fn main_generate() { let module = Module::from_blob(&engine, &module_config, blob.clone()).unwrap(); let mut instance = module.instantiate().unwrap(); - let mut initial_page_map = Vec::new(); - let mut initial_memory = Vec::new(); - + let mut initial_steps = Vec::new(); if module.memory_map().ro_data_size() > 0 { - initial_page_map.push(Page { + initial_steps.push(TestcaseStep::Map { address: module.memory_map().ro_data_address(), length: module.memory_map().ro_data_size(), is_writable: false, }); - initial_memory.extend(extract_chunks(module.memory_map().ro_data_address(), blob.ro_data())); + for chunk in extract_chunks(module.memory_map().ro_data_address(), blob.ro_data()) { + initial_steps.push(TestcaseStep::Write { + address: chunk.address, + contents: chunk.contents, + }); + } } if module.memory_map().rw_data_size() > 0 { - initial_page_map.push(Page { + initial_steps.push(TestcaseStep::Map { address: module.memory_map().rw_data_address(), length: module.memory_map().rw_data_size(), is_writable: true, }); - initial_memory.extend(extract_chunks(module.memory_map().rw_data_address(), blob.rw_data())); + for chunk in extract_chunks(module.memory_map().rw_data_address(), blob.rw_data()) { + initial_steps.push(TestcaseStep::Write { + address: chunk.address, + contents: chunk.contents, + }); + } } if module.memory_map().stack_size() > 0 { - initial_page_map.push(Page { + initial_steps.push(TestcaseStep::Map { address: module.memory_map().stack_address_low(), length: module.memory_map().stack_size(), is_writable: true, }); } + for (reg, value) in pre.regs.into_iter().enumerate() { + if let Some(value) = value { + if value != 0 { + initial_steps.push(TestcaseStep::SetReg { reg: reg as u32, value }); + } + } + } + + initial_steps.extend(steps); + steps = initial_steps; + let initial_pc = blob.exports().find(|export| export.symbol() == "main").unwrap().program_counter(); let expected_final_pc = if let Some(export) = blob.exports().find(|export| export.symbol() == "expected_exit") { @@ -345,69 +513,289 @@ fn main_generate() { instance.set_reg(reg, value); } - if module_config.dynamic_paging() { - for page in &initial_page_map { - instance.zero_memory(page.address, page.length).unwrap(); - if !page.is_writable { - instance.protect_memory(page.address, page.length).unwrap(); + let mut final_state = State { + status: "", + page_fault_address: None, + hostcall: None, + gas: instance.gas(), + pc: initial_pc, + regs: initial_regs.to_vec(), + memory: vec![], + }; + + let mut initial_state = final_state.clone(); + let mut initial_page_map = Vec::new(); + let mut final_page_map = Vec::new(); + let mut initial_memory = Vec::new(); + let mut nth_step = 0; + while nth_step < steps.len() { + let step = steps[nth_step].clone(); + match step { + TestcaseStep::Map { + address, + length, + is_writable, + } => { + instance.zero_memory(address, length).unwrap(); + if !is_writable { + instance.protect_memory(address, length).unwrap(); + } + + if final_state.status.is_empty() { + initial_page_map.push(Page { + address, + length, + is_writable, + }); + } + + final_page_map.push(Page { + address, + length, + is_writable, + }); + nth_step += 1; + continue; + } + TestcaseStep::Write { address, contents } => { + instance.write_memory(address, &contents).unwrap(); + nth_step += 1; + continue; + } + TestcaseStep::SetReg { reg, value } => { + instance.set_reg(Reg::ALL[reg as usize], value); + nth_step += 1; + continue; + } + TestcaseStep::Assert { + status, + page_fault_address, + hostcall, + gas, + pc, + regs, + memory, + } => { + let mut found_local_errors = false; + if let Some(status) = status { + if status != final_state.status { + eprintln!( + "Unexpected status for {internal_name}: expected {status}, is {}", + final_state.status + ); + found_local_errors = true; + } + } + + if page_fault_address != final_state.page_fault_address { + eprintln!( + "Unexpected page fault address for {internal_name}: expected {page_fault_address:?}, is {:?}", + final_state.page_fault_address + ); + found_local_errors = true; + } + + if hostcall != final_state.hostcall { + eprintln!( + "Unexpected hostcall for {internal_name}: expected {hostcall:?}, is {:?}", + final_state.hostcall + ); + found_local_errors = true; + } + + if let Some(gas) = gas { + if gas != final_state.gas { + eprintln!("Unexpected gas for {internal_name}: expected {gas}, is {}", final_state.gas); + found_local_errors = true; + } + } + + if let Some(pc) = pc { + if pc != final_state.pc.0 { + eprintln!("Unexpected PC for {internal_name}: expected {pc}, is {}", final_state.pc); + found_local_errors = true; + } + } + + assert_eq!(regs.len(), Reg::ALL.len()); + assert_eq!(final_state.regs.len(), Reg::ALL.len()); + for ((reg, expected_value), actual_value) in Reg::ALL + .iter() + .copied() + .zip(regs.iter().copied()) + .zip(final_state.regs.iter().copied()) + { + let Some(expected_value) = expected_value else { continue }; + if actual_value != expected_value { + eprintln!("Unexpected value of {reg} for {internal_name}: expected {expected_value}, is {actual_value}"); + found_local_errors = true; + } + } + + let mut current_memory = Vec::new(); + for page in &final_page_map { + let contents = instance.read_memory(page.address, page.length).unwrap(); + current_memory.extend(extract_chunks(page.address, &contents)); + } + + if let Some(memory) = memory { + if current_memory != memory { + eprintln!("Memory contents mismatch for {internal_name}"); + found_local_errors = true; + } + } + + if found_local_errors { + found_errors = true; + continue 'next_testcase; + } + + nth_step += 1; + continue; + } + TestcaseStep::Run => { + nth_step += 1; } } - for chunk in &initial_memory { - instance.write_memory(chunk.address, &chunk.contents).unwrap(); + if final_state.status.is_empty() { + for page in &initial_page_map { + let memory = instance.read_memory(page.address, page.length).unwrap(); + initial_memory.extend(extract_chunks(page.address, &memory)); + } + + initial_state = final_state.clone(); } - } - let mut final_pc = initial_pc; - let (final_status, page_fault_address) = loop { - match instance.run().unwrap() { - InterruptKind::Finished => break ("halt", None), - InterruptKind::Trap => break ("panic", None), - InterruptKind::Ecalli(..) => todo!(), - InterruptKind::NotEnoughGas => break ("out-of-gas", None), - InterruptKind::Segfault(segfault) => break ("page-fault", Some(segfault.page_address)), - InterruptKind::Step => { - final_pc = instance.program_counter().unwrap(); - continue; + let (new_status, new_page_fault_address, new_hostcall) = loop { + match instance.run().unwrap() { + InterruptKind::Finished => break ("halt", None, None), + InterruptKind::Trap => break ("panic", None, None), + InterruptKind::Ecalli(hostcall) => break ("ecalli", None, Some(hostcall)), + InterruptKind::NotEnoughGas => break ("out-of-gas", None, None), + InterruptKind::Segfault(segfault) => break ("page-fault", Some(segfault.page_address), None), + InterruptKind::Step => { + final_state.pc = instance.program_counter().unwrap(); + continue; + } } + }; + + final_state.status = new_status; + final_state.page_fault_address = new_page_fault_address; + final_state.hostcall = new_hostcall; + final_state.gas = instance.gas(); + if final_state.status != "halt" { + final_state.pc = instance.program_counter().unwrap(); + } + + for reg in Reg::ALL { + final_state.regs[reg.to_usize()] = instance.reg(reg); + } + + final_state.memory.clear(); + for page in &final_page_map { + let contents = instance.read_memory(page.address, page.length).unwrap(); + final_state.memory.extend(extract_chunks(page.address, &contents)); + } + + if nth_step >= steps.len() || !matches!(steps[nth_step], TestcaseStep::Assert { .. }) { + steps.insert( + nth_step, + TestcaseStep::Assert { + status: None, + page_fault_address: None, + hostcall: None, + gas: None, + pc: None, + regs: [None; 13], + memory: None, + }, + ); + } + + let TestcaseStep::Assert { + ref mut status, + ref mut gas, + ref mut pc, + ref mut regs, + ref mut page_fault_address, + ref mut hostcall, + ref mut memory, + } = steps[nth_step] + else { + unreachable!() + }; + if status.is_none() { + *status = Some(final_state.status.to_owned()); + } + + if gas.is_none() { + *gas = Some(final_state.gas); + } + + if pc.is_none() { + *pc = Some(final_state.pc.0); + } + + for reg in Reg::ALL { + if regs[reg.to_usize()].is_none() { + regs[reg.to_usize()] = Some(final_state.regs[reg.to_usize()]); + } + } + + if final_state.status == "page-fault" { + if page_fault_address.is_none() { + *page_fault_address = final_state.page_fault_address; + } + assert!(page_fault_address.is_some()); + } else { + assert!(page_fault_address.is_none()); } - }; - if final_status != "halt" { - final_pc = instance.program_counter().unwrap(); + if final_state.status == "ecalli" { + if hostcall.is_none() { + *hostcall = final_state.hostcall; + } + assert!(hostcall.is_some()); + } else { + assert!(hostcall.is_none()); + } + + if memory.is_none() { + *memory = Some(final_state.memory.clone()); + } + } + + let mut final_memory = Vec::new(); + for page in &final_page_map { + let memory = instance.read_memory(page.address, page.length).unwrap(); + final_memory.extend(extract_chunks(page.address, &memory)); } if let Some(expected_status) = expected_status { - if final_status != expected_status { - eprintln!("Unexpected final status for {internal_name}: expected {expected_status}, is {final_status}"); + if final_state.status != expected_status { + eprintln!( + "Unexpected final status for {internal_name}: expected {expected_status}, is {}", + final_state.status + ); found_errors = true; continue; } } - if final_pc.0 != expected_final_pc { - eprintln!("Unexpected final program counter for {internal_name}: expected {expected_final_pc}, is {final_pc}"); + if final_state.pc.0 != expected_final_pc { + eprintln!( + "Unexpected final program counter for {internal_name}: expected {expected_final_pc}, is {}", + final_state.pc.0 + ); found_errors = true; continue; } - let mut expected_regs = Vec::new(); - for reg in Reg::ALL { - let value = instance.reg(reg); - expected_regs.push(value); - } - - let mut expected_memory = Vec::new(); - for page in &initial_page_map { - let memory = instance.read_memory(page.address, page.length).unwrap(); - expected_memory.extend(extract_chunks(page.address, &memory)); - } - - let expected_gas = instance.gas(); - let mut found_post_check_errors = false; - for ((final_value, reg), required_value) in expected_regs.iter().zip(Reg::ALL).zip(post.regs.iter()) { + for ((final_value, reg), required_value) in final_state.regs.iter().zip(Reg::ALL).zip(post.regs.iter()) { if let Some(required_value) = required_value { if final_value != required_value { eprintln!("{internal_name}: unexpected {reg}: 0x{final_value:x} (expected: 0x{required_value:x})"); @@ -417,8 +805,8 @@ fn main_generate() { } if let Some(post_gas) = post.gas { - if expected_gas != post_gas { - eprintln!("{internal_name}: unexpected gas: {expected_gas} (expected: {post_gas})"); + if final_state.gas != post_gas { + eprintln!("{internal_name}: unexpected gas: {} (expected: {post_gas})", final_state.gas); found_post_check_errors = true; } } @@ -451,7 +839,18 @@ fn main_generate() { let mut disassembly = Vec::new(); disassembler.disassemble_into(&mut disassembly).unwrap(); - let disassembly = String::from_utf8(disassembly).unwrap(); + let mut disassembly = String::from_utf8(disassembly).unwrap().replace(" // INVALID", ""); + if initial_pc.0 != 0 { + let mut disassembly_new = String::new(); + for line in disassembly.lines() { + if line.trim().starts_with(&format!("{}:", initial_pc.0)) { + disassembly_new.push_str(" // Start execution HERE:\n"); + } + disassembly_new.push_str(line); + disassembly_new.push('\n'); + } + disassembly = disassembly_new; + } let mut block_gas_costs = BTreeMap::new(); let mut timelines = Vec::new(); @@ -481,20 +880,18 @@ fn main_generate() { tests.push(Testcase { disassembly, timelines, + initial_page_map, + final_page_map, + initial_memory, + final_memory, + initial_state, + final_state, json: TestcaseJson { name, - initial_regs, initial_pc: initial_pc.0, - initial_page_map, - initial_memory, initial_gas, program: parts.code_and_jump_table.to_vec(), - expected_status: final_status.to_owned(), - expected_regs, - expected_pc: expected_final_pc, - expected_memory, - expected_gas, - expected_page_fault_address: page_fault_address, + steps, block_gas_costs, }, }); @@ -523,9 +920,97 @@ fn main_generate() { writeln!(&mut index_md, "## {}\n", test.json.name).unwrap(); - if !test.json.initial_page_map.is_empty() { + if test.json.steps.iter().filter(|step| matches!(step, TestcaseStep::Run)).count() > 1 { + writeln!(&mut index_md, "Execution steps:").unwrap(); + let mut started = false; + for step in test.json.steps.iter().skip_while(|step| { + matches!( + step, + TestcaseStep::Write { .. } | TestcaseStep::SetReg { .. } | TestcaseStep::Map { .. } + ) + }) { + match step { + TestcaseStep::Map { + address, + length, + is_writable, + } => { + let access = if *is_writable { "RW" } else { "RO" }; + writeln!( + &mut index_md, + " * Map page: 0x{:x}-0x{:x} (0x{:x} bytes, {access})", + address, + address + length, + length + ) + .unwrap(); + } + TestcaseStep::Write { address, contents } => { + let contents_len = contents.len(); + let contents: Vec<_> = contents.iter().map(|byte| format!("0x{:02x}", byte)).collect(); + let contents = contents.join(", "); + writeln!( + &mut index_md, + " * Write: 0x{:x}-0x{:x} (0x{:x} bytes) = [{}]", + address, + address + contents_len as u32, + contents_len, + contents + ) + .unwrap(); + } + TestcaseStep::SetReg { reg, value } => { + writeln!( + &mut index_md, + " * Set: {} = 0x{:x}", + Reg::ALL[*reg as usize].name_non_abi(), + value + ) + .unwrap(); + } + TestcaseStep::Assert { + status, + page_fault_address, + hostcall, + gas, + pc, + regs: _, + memory: _, + } => { + let status = status.as_ref().unwrap(); + let gas = gas.unwrap(); + let pc = pc.unwrap(); + let status_extra = if let Some(address) = page_fault_address { + format!(" (address = 0x{address:x})") + } else if let Some(hostcall) = hostcall { + format!(" (hostcall = {hostcall})") + } else { + String::new() + }; + + writeln!( + &mut index_md, + " * Execution interrupted: status = '{status}'{status_extra}, gas = {gas}, pc = {pc}", + ) + .unwrap(); + } + TestcaseStep::Run => { + if started { + writeln!(&mut index_md, " * Resume execution",).unwrap(); + } else { + writeln!(&mut index_md, " * Start execution",).unwrap(); + } + started = true; + } + } + } + + writeln!(&mut index_md).unwrap(); + } + + if !test.initial_page_map.is_empty() { writeln!(&mut index_md, "Initial page map:").unwrap(); - for page in &test.json.initial_page_map { + for page in &test.initial_page_map { let access = if page.is_writable { "RW" } else { "RO" }; writeln!( @@ -541,9 +1026,27 @@ fn main_generate() { writeln!(&mut index_md).unwrap(); } - if !test.json.initial_memory.is_empty() { + if test.initial_page_map != test.final_page_map { + writeln!(&mut index_md, "Final page map:").unwrap(); + for page in &test.final_page_map { + let access = if page.is_writable { "RW" } else { "RO" }; + + writeln!( + &mut index_md, + " * {access}: 0x{:x}-0x{:x} (0x{:x} bytes)", + page.address, + page.address + page.length, + page.length + ) + .unwrap(); + } + + writeln!(&mut index_md).unwrap(); + } + + if !test.initial_memory.is_empty() { writeln!(&mut index_md, "Initial non-zero memory chunks:").unwrap(); - for chunk in &test.json.initial_memory { + for chunk in &test.initial_memory { let contents: Vec<_> = chunk.contents.iter().map(|byte| format!("0x{:02x}", byte)).collect(); let contents = contents.join(", "); writeln!( @@ -560,10 +1063,10 @@ fn main_generate() { writeln!(&mut index_md).unwrap(); } - if test.json.initial_regs.iter().any(|value| *value != 0) { + if test.initial_state.regs.iter().any(|value| *value != 0) { writeln!(&mut index_md, "Initial non-zero registers:").unwrap(); for reg in Reg::ALL { - let value = test.json.initial_regs[reg as usize]; + let value = test.initial_state.regs[reg as usize]; if value != 0 { writeln!(&mut index_md, " * {} = 0x{:x}", reg.name_non_abi(), value).unwrap(); } @@ -572,19 +1075,23 @@ fn main_generate() { writeln!(&mut index_md).unwrap(); } + if test.json.initial_pc != 0 { + writeln!(&mut index_md, "Initial program counter: {}\n", test.json.initial_pc).unwrap(); + } + writeln!(&mut index_md, "```\n{}```\n", test.disassembly).unwrap(); if test - .json - .initial_regs + .initial_state + .regs .iter() - .zip(test.json.expected_regs.iter()) + .zip(test.final_state.regs.iter()) .any(|(old_value, new_value)| *old_value != *new_value) { writeln!(&mut index_md, "Registers after execution (only changed registers):").unwrap(); for reg in Reg::ALL { - let value_before = test.json.initial_regs[reg as usize]; - let value_after = test.json.expected_regs[reg as usize]; + let value_before = test.initial_state.regs[reg as usize]; + let value_after = test.final_state.regs[reg as usize]; if value_before != value_after { writeln!( &mut index_md, @@ -600,12 +1107,12 @@ fn main_generate() { writeln!(&mut index_md).unwrap(); } - if !test.json.expected_memory.is_empty() { - if test.json.expected_memory == test.json.initial_memory { + if !test.final_memory.is_empty() { + if test.final_memory == test.initial_memory { writeln!(&mut index_md, "The memory contents after execution should be unchanged.").unwrap(); } else { writeln!(&mut index_md, "Final non-zero memory chunks:").unwrap(); - for chunk in &test.json.expected_memory { + for chunk in &test.final_memory { let contents: Vec<_> = chunk.contents.iter().map(|byte| format!("0x{:02x}", byte)).collect(); let contents = contents.join(", "); writeln!( @@ -624,21 +1131,21 @@ fn main_generate() { } assert_eq!( - test.json.expected_status == "page-fault", - test.json.expected_page_fault_address.is_some() + test.final_state.status == "page-fault", + test.final_state.page_fault_address.is_some() ); - write!(&mut index_md, "Program should end with: {}", test.json.expected_status).unwrap(); + write!(&mut index_md, "Program should end with: {}", test.final_state.status).unwrap(); - if let Some(address) = test.json.expected_page_fault_address { + if let Some(address) = test.final_state.page_fault_address { write!(&mut index_md, " (page address = 0x{:x})", address).unwrap(); } writeln!(&mut index_md, "\n").unwrap(); - writeln!(&mut index_md, "Final value of the program counter: {}\n", test.json.expected_pc).unwrap(); + writeln!(&mut index_md, "Final value of the program counter: {}\n", test.final_state.pc).unwrap(); writeln!( &mut index_md, "Gas consumed: {} -> {}\n", - test.json.initial_gas, test.json.expected_gas + test.json.initial_gas, test.final_state.gas ) .unwrap(); From 5153bd105efb912c25816eaa6776968e90ab4279 Mon Sep 17 00:00:00 2001 From: Jan Bujak Date: Mon, 10 Nov 2025 11:03:46 +0000 Subject: [PATCH 05/21] Extend memory protection facilities --- crates/polkavm-zygote/src/main.rs | 7 ++-- crates/polkavm/src/api.rs | 19 ++++++++- crates/polkavm/src/interpreter.rs | 26 +++++++------ crates/polkavm/src/sandbox.rs | 2 +- crates/polkavm/src/sandbox/generic.rs | 45 ++++++++++++++-------- crates/polkavm/src/sandbox/linux.rs | 30 ++++++++++++--- crates/polkavm/src/sandbox/polkavm-zygote | Bin 33232 -> 33448 bytes crates/polkavm/src/tests.rs | 27 ++++++++++++- crates/polkavm/src/utils.rs | 3 ++ 9 files changed, 119 insertions(+), 40 deletions(-) diff --git a/crates/polkavm-zygote/src/main.rs b/crates/polkavm-zygote/src/main.rs index 7cfd0814..3e220d22 100644 --- a/crates/polkavm-zygote/src/main.rs +++ b/crates/polkavm-zygote/src/main.rs @@ -372,9 +372,10 @@ unsafe extern "C" fn signal_handler(signal: u32, info: &linux_raw::siginfo_t, co ); VMCTX.arg.store(address as u32, Ordering::Relaxed); - if !is_protection_fault { - futex_value = VMCTX_FUTEX_GUEST_PAGEFAULT; - } + VMCTX + .arg2 + .store(u32::from(is_protection_fault && is_write_fault), Ordering::Relaxed); + futex_value = VMCTX_FUTEX_GUEST_PAGEFAULT; } if rip < VM_ADDR_NATIVE_CODE || rip > VM_ADDR_NATIVE_CODE + VMCTX.shm_code_length.load(Ordering::Relaxed) { diff --git a/crates/polkavm/src/api.rs b/crates/polkavm/src/api.rs index c2512147..60d35ff4 100644 --- a/crates/polkavm/src/api.rs +++ b/crates/polkavm/src/api.rs @@ -1670,9 +1670,20 @@ impl RawInstance { /// /// Is only supported when dynamic paging is enabled. pub fn protect_memory(&mut self, address: u32, length: u32) -> Result<(), MemoryAccessError> { + self.change_memory_protection(address, length, true) + } + + /// Removes read-only protection from a given memory region. + /// + /// Is only supported when dynamic paging is enabled. + pub fn unprotect_memory(&mut self, address: u32, length: u32) -> Result<(), MemoryAccessError> { + self.change_memory_protection(address, length, false) + } + + fn change_memory_protection(&mut self, address: u32, length: u32, make_read_only: bool) -> Result<(), MemoryAccessError> { if !self.module.is_dynamic_paging() { return Err(MemoryAccessError::Error( - "protecting memory is only possible on modules with dynamic paging".into(), + "protecting/unprotecting memory is only possible on modules with dynamic paging".into(), )); } @@ -1694,7 +1705,11 @@ impl RawInstance { }); } - access_backend!(self.backend, |mut backend| backend.protect_memory(address, length)) + access_backend!(self.backend, |mut backend| backend.change_memory_protection( + address, + length, + make_read_only + )) } /// Frees the given page(s). diff --git a/crates/polkavm/src/interpreter.rs b/crates/polkavm/src/interpreter.rs index d117f6d7..068940a2 100644 --- a/crates/polkavm/src/interpreter.rs +++ b/crates/polkavm/src/interpreter.rs @@ -670,7 +670,7 @@ impl InterpretedInstance { Ok(()) } - pub fn protect_memory(&mut self, address: u32, length: u32) -> Result<(), MemoryAccessError> { + pub fn change_memory_protection(&mut self, address: u32, length: u32, make_read_only: bool) -> Result<(), MemoryAccessError> { assert!(self.module.is_dynamic_paging()); each_page( @@ -679,7 +679,7 @@ impl InterpretedInstance { length, |page_address, page_offset, _buffer_offset, length| { if let Some(page) = self.dynamic_memory.pages.get_mut(&page_address) { - page.is_read_only = true; + page.is_read_only = make_read_only; Ok(()) } else { Err(MemoryAccessError::OutOfRangeAccess { @@ -1206,7 +1206,7 @@ impl<'a> Visitor<'a> { Some(target) } - fn segfault_impl(&mut self, program_counter: ProgramCounter, page_address: u32) -> Option { + fn segfault_impl(&mut self, program_counter: ProgramCounter, page_address: u32, is_write_protected: bool) -> Option { if page_address < 1024 * 16 { return trap_impl::(self, program_counter); } @@ -1217,6 +1217,7 @@ impl<'a> Visitor<'a> { self.inner.interrupt = InterruptKind::Segfault(Segfault { page_address, page_size: self.inner.module.memory_map().page_size(), + is_write_protected, }); None @@ -1228,7 +1229,7 @@ impl<'a> Visitor<'a> { if self.inner.dynamic_memory.pages.contains_key(&page_address) { trap_impl::(self, program_counter) } else { - self.segfault_impl(program_counter, page_address) + self.segfault_impl(program_counter, page_address, false) } } @@ -1272,7 +1273,7 @@ impl<'a> Visitor<'a> { let offset = cast(address).to_usize() - cast(page_address_lo).to_usize(); T::from_slice(&page[offset..offset + core::mem::size_of::()]) } else { - return self.segfault_impl(program_counter, page_address_lo); + return self.segfault_impl(program_counter, page_address_lo, false); } } else { let mut iter = self.inner.dynamic_memory.pages.range(page_address_lo..=page_address_hi); @@ -1291,7 +1292,7 @@ impl<'a> Visitor<'a> { T::from_slice(buffer) } (None, _) => { - return self.segfault_impl(program_counter, page_address_lo); + return self.segfault_impl(program_counter, page_address_lo, false); } (Some((page_address, _)), _) => { let missing_page_address = if *page_address == page_address_lo { @@ -1300,7 +1301,7 @@ impl<'a> Visitor<'a> { page_address_lo }; - return self.segfault_impl(program_counter, missing_page_address); + return self.segfault_impl(program_counter, missing_page_address, false); } } } @@ -1384,14 +1385,14 @@ impl<'a> Visitor<'a> { ); } - return trap_impl::(self, program_counter); + return self.segfault_impl(program_counter, page_address_lo, true); } let offset = cast(address).to_usize() - cast(page_address_lo).to_usize(); let value = value.as_ref(); page[offset..offset + value.len()].copy_from_slice(value); } else { - return self.segfault_impl(program_counter, page_address_lo); + return self.segfault_impl(program_counter, page_address_lo, false); } } else { let mut iter = self.inner.dynamic_memory.pages.range_mut(page_address_lo..=page_address_hi); @@ -1408,7 +1409,8 @@ impl<'a> Visitor<'a> { ); } - return trap_impl::(self, program_counter); + let address = if lo.is_read_only { page_address_lo } else { page_address_hi }; + return self.segfault_impl(program_counter, address, true); } let value = value.as_ref(); @@ -1419,7 +1421,7 @@ impl<'a> Visitor<'a> { hi[..hi_len].copy_from_slice(&value[lo_len..]); } (None, _) => { - return self.segfault_impl(program_counter, page_address_lo); + return self.segfault_impl(program_counter, page_address_lo, false); } (Some((page_address, _)), _) => { let missing_page_address = if *page_address == page_address_lo { @@ -1428,7 +1430,7 @@ impl<'a> Visitor<'a> { page_address_lo }; - return self.segfault_impl(program_counter, missing_page_address); + return self.segfault_impl(program_counter, missing_page_address, false); } } } diff --git a/crates/polkavm/src/sandbox.rs b/crates/polkavm/src/sandbox.rs index 1858818e..e57456db 100644 --- a/crates/polkavm/src/sandbox.rs +++ b/crates/polkavm/src/sandbox.rs @@ -130,7 +130,7 @@ pub(crate) trait Sandbox: Sized { fn read_memory_into<'slice>(&self, address: u32, slice: &'slice mut [MaybeUninit]) -> Result<&'slice mut [u8], MemoryAccessError>; fn write_memory(&mut self, address: u32, data: &[u8]) -> Result<(), MemoryAccessError>; fn zero_memory(&mut self, address: u32, length: u32) -> Result<(), MemoryAccessError>; - fn protect_memory(&mut self, address: u32, length: u32) -> Result<(), MemoryAccessError>; + fn change_memory_protection(&mut self, address: u32, length: u32, make_read_only: bool) -> Result<(), MemoryAccessError>; fn free_pages(&mut self, address: u32, length: u32) -> Result<(), Self::Error>; fn heap_size(&self) -> u32; fn sbrk(&mut self, size: u32) -> Result, Self::Error>; diff --git a/crates/polkavm/src/sandbox/generic.rs b/crates/polkavm/src/sandbox/generic.rs index 54953438..0dd0fac3 100644 --- a/crates/polkavm/src/sandbox/generic.rs +++ b/crates/polkavm/src/sandbox/generic.rs @@ -1120,22 +1120,27 @@ impl Sandbox { let page_address = fault_address.wrapping_sub(self.memory.as_ptr() as u64 + self.guest_memory_offset as u64) as u32; let page_address = page_address & !(page_size - 1); - let is_trap = if self.dynamic_paging_enabled && page_address >= 0x10000 { + let segfault_kind = if self.dynamic_paging_enabled && page_address >= 0x10000 { let module = self.module.as_ref().unwrap(); let page_start = module.address_to_page(module.round_to_page_size_down(page_address)); - self.page_set.contains((page_start, page_start)) + Some(self.page_set.contains((page_start, page_start))) } else { - true + None }; - if is_trap { - self.vmctx().next_native_program_counter.store(0, Ordering::Relaxed); - Ok(InterruptKind::Trap) - } else { + if let Some(is_write_protected) = segfault_kind { self.vmctx() .next_native_program_counter .store(machine_code_address, Ordering::Relaxed); - Ok(InterruptKind::Segfault(Segfault { page_address, page_size })) + + Ok(InterruptKind::Segfault(Segfault { + page_address, + page_size, + is_write_protected, + })) + } else { + self.vmctx().next_native_program_counter.store(0, Ordering::Relaxed); + Ok(InterruptKind::Trap) } } @@ -1837,24 +1842,34 @@ impl super::Sandbox for Sandbox { Ok(()) } - fn protect_memory(&mut self, address: u32, length: u32) -> Result<(), MemoryAccessError> { + fn change_memory_protection(&mut self, address: u32, length: u32, make_read_only: bool) -> Result<(), MemoryAccessError> { assert!(self.dynamic_paging_enabled); log::trace!( - "Protecting memory: 0x{:x}-0x{:x} ({} bytes)", + "{} memory: 0x{:x}-0x{:x} ({} bytes)", + if make_read_only { "Protecting" } else { "Unprotecting" }, address, address as usize + length as usize, length ); - self.memory - .mprotect(self.guest_memory_offset + address as usize, length as usize, PROT_READ) - .map_err(|e| MemoryAccessError::Error(e.into()))?; - let module = self.module.as_ref().unwrap(); let page_start = module.address_to_page(module.round_to_page_size_down(address)); let page_end = module.address_to_page(module.round_to_page_size_down(address + length - 1)); - self.page_set.insert((page_start, page_end)); + if !self.page_set.contains((page_start, page_end)) { + return Err(MemoryAccessError::OutOfRangeAccess { + address, + length: u64::from(length), + }); + } + + self.memory + .mprotect( + self.guest_memory_offset + address as usize, + length as usize, + if make_read_only { PROT_READ } else { PROT_READ | PROT_WRITE }, + ) + .map_err(|e| MemoryAccessError::Error(e.into()))?; Ok(()) } diff --git a/crates/polkavm/src/sandbox/linux.rs b/crates/polkavm/src/sandbox/linux.rs index 2cd2c892..a7735c49 100644 --- a/crates/polkavm/src/sandbox/linux.rs +++ b/crates/polkavm/src/sandbox/linux.rs @@ -2114,20 +2114,35 @@ impl super::Sandbox for Sandbox { Ok(()) } - fn protect_memory(&mut self, address: u32, length: u32) -> Result<(), MemoryAccessError> { + fn change_memory_protection(&mut self, address: u32, length: u32, make_read_only: bool) -> Result<(), MemoryAccessError> { assert!(self.dynamic_paging_enabled); log::trace!( - "Protecting memory: 0x{:x}-0x{:x} ({} bytes)", + "{} memory: 0x{:x}-0x{:x} ({} bytes)", + if make_read_only { "Protecting" } else { "Unprotecting" }, address, address as usize + length as usize, length ); + let module = self.module.as_ref().unwrap(); + let page_start = module.address_to_page(module.round_to_page_size_down(address)); + let page_end = module.address_to_page(module.round_to_page_size_down(address + length - 1)); + if !self.page_set.contains((page_start, page_end)) { + return Err(MemoryAccessError::OutOfRangeAccess { + address, + length: u64::from(length), + }); + } + let mut arg: linux_raw::uffdio_writeprotect = Default::default(); arg.range.start = u64::from(address); arg.range.len = u64::from(length); - arg.mode = linux_raw::UFFDIO_WRITEPROTECT_MODE_WP; + arg.mode = if make_read_only { + linux_raw::UFFDIO_WRITEPROTECT_MODE_WP + } else { + 0 + }; if let Err(error) = linux_raw::sys_uffdio_writeprotect(self.userfaultfd.borrow(), &mut arg) { return Err(MemoryAccessError::Error(error.into())); @@ -2337,8 +2352,9 @@ impl Sandbox { let machine_code_address = self.vmctx().rip.load(Ordering::Relaxed); let address = self.vmctx().arg.load(Ordering::Relaxed); + let is_write_protected = self.vmctx().arg2.load(Ordering::Relaxed); log::trace!( - "Child #{}: pagefault: rip=0x{machine_code_address:x}, address=0x{address:x}", + "Child #{}: pagefault: rip=0x{machine_code_address:x}, address=0x{address:x}, is_write_protected={is_write_protected}", self.child.pid ); let page_size = get_native_page_size() as u32; @@ -2358,7 +2374,11 @@ impl Sandbox { .map_err(Error::from_str)?; self.is_program_counter_valid = true; - return Ok(Interrupt::Segfault(Segfault { page_address, page_size })); + return Ok(Interrupt::Segfault(Segfault { + page_address, + page_size, + is_write_protected: is_write_protected != 0, + })); } if state != VMCTX_FUTEX_BUSY { diff --git a/crates/polkavm/src/sandbox/polkavm-zygote b/crates/polkavm/src/sandbox/polkavm-zygote index dc6e8ccffb6411ab3168a32e7519d9810f4c7b58..35497941783b0efdcb5a18f6aeea39c15136cc9d 100755 GIT binary patch literal 33448 zcmeHw3w%`7wf6~RB(1_kOBC%@j~X>xfzBkEWP-$-Av18MOdgVe_`1Vn9%LlROy@y@ zwHg}Ia_%^Swpza1+S1$BT3ajC%8h74KqW{K&_1Nsim0_tfEIid;*0tIYwvyLoIJFA z{r$e*@4NTQ`6aW@UVHu5UVH7e_g-h8Gjr~6c@|8{%agQYl5~y4avQSQ$spEhj3`OFL~6vDh-MK5 zLQhGg#n*7UmKt%aBr0%7nBH1CA_Jmm&$VwR0!&XGHgPs#fe8ytSYW~e6Bd}Tz=Q=R zEHGh#2@6bEV8Q|u7Wi*xf%9&VBz5{c<7}Ua=w=3kKwHhkWbm6_G?R$o#OU#vhfj}%6kAq5eM5$u%xkw8$6`8rkv=R#hTu)(fy zJfUGUohh9qhvRZ*ESv~JU_7{@J=l@(CBl&oUR{%W;rt~{a%VWvD*Iq=UsUc$wzmXh za-!9j0Jh`GguKjN;cZ^*X!JHNh;#GC6F&b+IUc?h8ie92;w`>-kSeNgXqMy2XfzT_ z1Or#2inca%i`+uoiHPir`CGlwm_N}L@`c;btDwb$3DgyoFJBQyudP5cU_gV&oH!lg zZ;PP1cp?C&Xp4}8j8hg#CPd*JvV%+?IEthTrQIwwV|S18pT|BQrJt4liKXXDv@ckM z9YET{E0WHWXp65%I#-&((jsXdOQ!+B_xSR_E90ED)QXTy5_Cz*-a&Nw2Yo7S!STH| zo!$%@(NRv-jVTBHjPY>9PXJBcXgtwLFRfg+5E(ok^!p?2Q8^TDBXjGW!M%)ih&J4E z9c3s3E*M{pu4)&ON}61?wGDNR-X>SGx30nI@-A?Dnq7-$8f1%(#z6Fwe|PxWF_?4G zQM%aUu5&kg>l=V|=|!Esl|eZfWlFV1;t5%k04L6ke&*9UhPy~0+1|dIJId%Ha=CcK zzcQE@U9zg(pXid?gYBV!Ueu0(!62*Sv0x$@>yU3r!hQ)~OWUYMKv5q?2AhRiVX`)i zrBEa$W2Ctobb-_hpTfGJ(O`hT!|V2VPFpSp`)TtwxfU)R$&6-^?c06vm17EM)@9Rz zUeZohGtL-#5$_+eXfjR?h?&o5?s%{Rlf%eTnkN!R7UF%?{ZD*7cuUrci{q&xw5Mx5nC-s zBgoCgBAD>TTVgB65V4*H6W-BQ;KpV0gTAOJ5p`=Y77>!X{?=s2N=$=( zEOT?2U(%T#wB(A|x|)k+a%5_Lup^RO(JHU-#mB*ksb(yaTB-BTmC3li4)o*fYh;(R zp$?Y72tv=dqGfaC*?Qynm?U_(gZw`pOm-mgx6({TV$l#_*xH^8X0DbopG9J1_E_*s z$za?tX@1h|JjQZu5nlj{e`H0>hY19uUPK|Z0?fr~kU+e~gBgk|Xc&{NDc&&*JrVT~ z6=oFn5yNY)ESJz!iMVA{dvyV$3Mi7QV?$T;w_LLLx_|kX#SOFMcxyWxW+vjB+#1Df zH@+8*R|EGinwn&vpO(BZO#DfHg@AQ69B+)($Z>VF7lXu0=4lEhFbUEq&%OV+3YLI@%0V$N#wyqly&aNG!h6ZbwoKL;qTyVlO(h zR4&03-w{FZ>S}Z?u5;Blxayl-O^a(4m%F}svCED8GJKTIcQ2&EWv*J+e1~&^t5$J) zoG!;ghr1s1I#->`QCsV3axHKyL12PFb-rhj3(Mj4F4r=5Q?qNC)3p?2{@LtwLv$1S ztY7Nzxauh+an-R;V4JUZH7{$P5AXtBZ&M>=H@X{LuEh-tnp|}rcl{z4&pYZDx?By7 zKs7CCY;-j@G}JljuV3r}+hSK!vtw~{)Adbq$xOB+8Wu7=ST1p(xh%3;*ff(($L-9w z+I@jlVZ`*oL^K?rC9^$_^%>E1N#kP1j-YJGV%L0kbJNTb^m<2~Yo^~9MQ7qK))sCL zCz9=$h!dsJ@Ty3{8;e8|p=2W1C6~}dJTrH`GB`7aNjAK~=cnPz&$XC<$ZZL>4)IR> zf8>*VYtuda{`>yBdDWO%7S=cv*Hk9 zCXxvYOjuyT0uvUPu)u@`CM+;vfe8ytSYW~e6Bd}Tz=Q?=F}VkKl(J-iuGigz+mVsXCPjTqd5YKwv1I5x>>%vnhm!IZD5({NN-8qL zlUgBrQjro*s?*|0-CHHtMXAU~L_gS3lZY!Rmq!Yzc zwvu#@dNSK2D?QH?+~~N;@vjaqQ-{^pr1Ttx(t3r3v2#7BdHb<^A^rD|?&x{T>{N%( zvN+NW*SVpUdCP!#%X8-WyViM%%~X+jWJQHH>eUG4_v~p@($&(TY_|V==#;q&n&pbP zJQ%MSWlC zdF@chVeS|RnU@c|V~!3;A^LDxp9_WNm@i%nj3m7~-+V`Y0ew0P3f(>1Wlz;}$-_$3 zzC>fqiNP6(xo-7+ch76XqXgyfA@lF^3zVMi1#aNp@c=i&owwbiK6k{VdtJ@84yud^--O9{ztXx{A;oUY7dDU)62Sz z8#m{D&9o4(d3oD{u!qv~s?bH2x-h{j zKMZyvub+9or4cm+v_45WNU&03X5)5$+ zIyW>d+vt)?^_)=!WhsgMEgIUtH{G9r< zGYT9*SfM9YT_@0pO3B&W|HXHSk*$Vv5VfIwXw^Pn+u21!oTAg67toQm6=GY=DK<)8UPngtaT>2$t6*p_M<@*2)XDNS@>P*e)-(l)22 z1Dn98u;{06l6Gt|%w~Rmf*Wl*c;{=N>OTj3nal}B*h~Zq5nck}tncO+oU6^<|g!FD~n!j$9Nn z#IHTCO?qL-QGm;Uw+Z4__bMq|tSE>{KREyPOYnI?;$?i6C0}`x8tGQ|9r=v`kD@b@ z1&&`)*^yr%)J{IG^n93?^kG=YXgC6$rD#`y7!pT+H)7Cz4k$h(aG~QYE+qe_R>S`b zamA3i=3UCzT)!QGqPol`_nK^8jU@F+`0$wLBg}ugrzRL z5=kFj)@`M0ILB5BX&im*>P~$X;U7YJXi=Sqelzz3!Ke(bEhbl?Ls8cj442HK`vimSwKz=%dz+XiHkOWd@lH2en1U`QT(CI8&2dy@>yQhI3DP>eaZ6 zqVNuu-+kl?^TpjDW0Kp!JRFg(6?S^jY+}Qe8!LVVy0&mS*f4Dt_MW)?g$;c(kTaXO z=tOOHZ}Klko>ZSb`AX05-x3e>41bjPKKc0!=I3qX+oq>qMJMszuf7VaCHZ+_|7DPq zx#&G)N3g3XQ&LY3kWo^P?LyK=kK^iAehEe=b9GS*aVV*t1H{}rkeChNehws34oKtG z{Phl~M8FL?Fk=87;y@ye0>%XJK@HeX^^(IgMr&4Au2U#S>eErzd&0c-ARMtSU7xM% z-D~cB0U3n}0ITFSZ+-2u_(dLU6l~92R*j2^i_KrZ3D*doyc0ykq;bhk?hwR2O0fl^ zHfk&#<}E$PaY`Zo)kynl@d0Nlfc4z!fWqc2^Vbi-o=*6Q9i?Os3sO29J;Md&J1d|c zI)u7UGG8}fKHFKmoIaZA!$}|I^ifR9+*#DCQR=@ga&HpWjg$_YuH$Ezornn)ZUBLJ zXnv(Ssi2SwHX#H0W+ZfwK<0ExAF(l3R5Zhh519l!2r4)n?)r!McL`K$Y|y78WHUoyyh zf+{l3Hnd^nQBS!C2WY5@@yX@BM!gLM6rljMwh`xwdRuWJnn{BR*xP8FfYR54td*`- z#au)WcjuEW;1#6>Vj< zw4UapwC%3v1zn2_`cQ)5hFv4Y{PP1xZWj2%dq?1H8oqusJ~uD)eHT32@(Q^#b6gz`unop!lOyN3PuTX*2p)1+-Pa!xJi|sFK8bN5zZMLIQ~^P^^t^-CuN_D# zY~*YsQ|R*63B5`LJLe(cI?qADQ!`qm`QCwbrUO_MP|4e@5_9h=q_FOMvd$)mVQZ}} zYl)DxOvnQ3TYJEIKe1BRTVKRF=CTeDgfIEu5s+vOQhk!7e;eG2dP-;9EEw16jC+ak z0Wh*%BAs=IVBI8GdHwX2k^X(-DzR4pn0=^X4BP0XrUIF!*_nLmr2eI8k`9^?qPZ#V zc2M-nw+UJMMdcyjy1^j>Wx50!(?qil1SS1VR?t@sqq2_U(+ltE$Ay@T5W~ZUQ4ud9 z1FD!Kj$$+`Yo4L$hPH)orXAaxpU$khgsQ4e?nlH+n}eA*vGsKF0F>CplrXLQ4cCcA z2sLs5DNPIj4ZVOOs7XcrG_7X}uz^P5GLOFCjTDF-LW0I!j4LW?+nZm?ZCcs~cfhKQWnJix(W%w?oDW&5oJ{`VaCkPgmp@DBy}00$q(L`GoB z@_xRVfgyew2lS9hkyR**Hq&A!+Wa25CBRhFk1b9qBj;)^!E`}u4v-d{qRj)MW?K1R z?fV6ovi%nj_>Cd0%{8FWGDp>Na8iJ2WdnGj08^Gb$pbm?GSFD8y0&;Z2hS2xEoDg8j_Rk`l!{08_UADpEViz-U;M_w%>O zjgVa~`ngr_=a+W#nyIM&3If}C&FQH#*(`8p@gb@P-jP8n2HA05^Bn?A9NbAhe2cX? zJ@qIXPInd$3otzrF2Dsig|$lrm^iqT^lRWp1en$|s(FSEmN|I;b6joW;5Dz|U~m_} zg@uz8;uP-rp%~lb+^E?rz?ALB{Q@cD%z-z;4q`lSB5S2h^#Z(sWved~{Y=?@+C<7s z$$>ZP;64sMTYw+r;N7CDD9gdSv03HdvwS&4ekLdvcKhEJd2smZ?|C!((z=2Lrt!A8evYi1ERtN#2yJj$1)u)8f z3aGnS-DVC3_kPgMgFMuHfN`5Xrdbc!T^#)DXL)x~w*Mlk`9*B3v6`P|oOc#eKteTJ z^uD(9zJ5i3iKCxl%FMkS4DRKi;i{TyZe`r2AyG309O&z1gS=+Saz|OF)$A5E*AORU zDM&?|8##EPsF^tU#Q*tR&0V0;{t;^KX56OhM9onS9@xoirYvvsL2N+rzS;m9eg72T zjT}5*fY)(wtpHQDUj}F9`#JD~pixHyyqSYfiSfOOgU=9P%5rcZ2Oa=z1_$rr;B^Ar z$H5PaHdB_j`N|yl0BG6*&%sFn-p|2N0j4YmAI7c|Z}V}`s3lN)n1gR8Ze%kYe4YSP zmV+PWdv&252viXKNSS$@&;?)Qop}`qhC6yo=)PMCx_uGMnJs#Sw8K`zl$|Ck2-tn7 zeh4IIY6{zF`4vQBr{x8g3F7lq{;sU<)@&npBLyd(w zc5J=iQ3h`TE`b)HO}Kaoqr4ac3~q1xCg3y$iHjgwa1j>b5+08;unKI~;rf?x=$qaB z%n;b@rW>i-UnK=>%QQBAHAN!LrtEKo&B@#{G?T03N(r0zM*)0j0A>4VptuJrIDkmM8X1>@7SkKcX?y5f|u63NqzhF}0 zxI1;*1^7r@Fx*FRt#;+G>Lp=SHqCIW*3r@>ta=xlV92U$0X*XncTO6j($ER)i*a}j zb5>`uf==lYa62?76O(V-d8Z7*(>S1nO)h-DodqVSPX>TbXtdu>Q&;9b0JsBc{^x`h zg#Q&lH$(qXx3m6R|ErPxhmGj(LMQlNs>xzKkuBPU+-dpVF)(I}R^xN5@7+yHtgzC3 zIV&;WJ481PdXCP3yWI<{Q1{E2^bG1!q|MYl^wel|tB*nU;1o=bqrF4xHa0S@W!k+A zUann%=mhNMo8G0-pyp#cu=;&A_rA`1zy;NqL4|&Q16t^J;+YZp(JIgMs~M|b1JjRI z&{5tyLcd>tD_eB+xcW8e`dx3-??>1tGwQcQ*ROq~ewS$aZF+Ky{T_kz!6_7BeVTq} zGyT2|-m&(3j3#Anzb8GVX_lD6YPx&entnn3$%il*$bt%3Jpg-<{$A<2E zLhE6Yz#bdIBQPJZxdm$&rc8S3Vw3jdB)1)CC$PrRR`h@Sa2yytS^%dhlrZCP0|{+jZCqXLVM<9-`1pxiiVy5zbo}PU0tx7 z9YlAh7i1m1JI%ch6E*cCT8uoshZ3J(l%9S?UEX`;DI-g_6^ZM^^llqZU6S4Mqi+|&wiUG5Z(cADeB?O+q5m?pNF~6_ESxoW`9i( ziVdESzPt=#J*we(C=x{X;)S4mZSv7#sL_IqU<{JOnUy;DIc;Xpv`E{IY#HeVZuNfZ zjXL#l3Juf)c=E5rxRLiTZg*hcjxCa=E!&_M&I`$=wB<)Z!geCEJ9up0>L&8SboC&p z{WK9~j$ngs{PDWx=teN0N9f^GF`4}tIKvpt#YSbC-da;qrmZyp(Cy5rh#~kbH1bWu zJvhEanZ=Sn>Xuo5A%2AieJHkrr$K-lp*p`);2v~A3^1wLuhPVm>J{Oax%Yj#j7Z&I zNN&~pvUzQQ+%}aiK;mY5^*9lC8pt}1t#tXuZ`0!OSw%fab#ko=NNJd{5gg1lm%$KJ zAGzjPkb|l!tHD(erq1#b?nYFf;Is-Ow+M4l3NDsE%3yAH~B9Co_$- ztAnRV&Vq-2re9+xopD1j?;w-J$_*~6`6da#l=v~`0gw7l#$*t+e7h!!^yQP@j5_sW zT~y9V@k}A6ymvA?Y2!y|a#;?}HGy(2tu3sD@SM$GcvQ4Sw|NN) zVIeo_Cks6d%fNct(@q<;)^H2)xXqIe6i@f40dh&(Qca}3?qU7hin)=QM=XC^X=8|< zZGwev)MRd@(1BbDl31b9JYxNXOCF>3bG1Rh??<^cWDfsgWW(I*%@T$i^mM%2gh49a8qPhxuhM^=Qpq6={ zD`w~0c+i^+i=6<|Y)(3^+Zbk`GDZM&T22<}eF-y0`F5O!&|gYw`gWb$h-6RS0RGZe zQl$Z^Krwopl7^2CVcr3@D>oAxqAEUk(zXrgOmh5GkUN2EDHEah21s2DIP}KH zv<>@5Cp?<2R&PPzL|$X338_Y`rN2Ov_+=|?YGew@u~BSC2H{blxCU#|B6x|uSQ%EM z-q`!u?9dAzkcjFhA)ar*uEe$!kK<)=?OTHCU#2cc_2|#)TSToh`0o6_j|+In0Jt~@ zV0T`^0>JMuvWr3Sa7Wto27|3F?nVnx)td-+=;jG!`!B^uW)=fOljYEA2^HhKky^b> z7$r=WLV(fOPfw3!K7`#_MA8L7u>iCZASwVC3qUKsx3e1nJpjS=Cp|TvL_dj;<8H9) z9k!LO|KY@G+rY!xH@@e5k9CyjJ&_Wpm5Qa!q_rflNQ^uKt zP!Ed`IwLUGTj0tTeF>ku2Ma|HqSs94V^EQ;Fo3HcWM-w^k}DuSa{zlyV*wfJH37Ie z2Vgz;;~c>FJmQt21{pQXLI^^gAoE^Z1pv3{i0dzkVLlISg=HI1i@~xb(2ZF(^s`Zx zP0dcCE@vt^%bEI$#hD70L(q#L(XulMR?_42%SOr+&)BMOC@QvBJ?Yu6|5TGrJ>^mV zN;S@2KuzJHI>NBSnFZLw5D~WYxCY?e+>p*MXTqqQX)jhUCW&A6k%-B<2(RWrVlf^G z7dkK`T_<1+e;r+s*@0~Up@W&(?iB#c!PQ?CfRcW?;K*E>11Mrctr39# zpn-=PY5@2#;wh@3h7ZEG(HVQ_@n7Tj|CT7~o0*re0V1Y6eSG36>UZ7hDJ4~j?;^s+ zR|^Y&fSyiW{vPUaU=w5*omtnLHQ&3vnr`u#@7W>m?*1SzS>sTjU6h_TOMJ7;o!$2< z`eK=9=3m@Z$Kcg&^Zb|a#jix`teV>ocr? z zXOKKp0UJ2hoMNWvJr_6>1fjNrJrXx`9*RL($SBlnLncnSiczRpL+_HAaODJXj%_r7 z552}I@y$e`MQdmur+ka5e43xXq4WQOQ+~oIxNek&zQ-x6b)Gb*+@Mpw%qdst6q+oc z$>(&+b)51CIM&c2e*O~99qbXSp(;-4V-&Pq8oH8Go@Er)mLg7Bt5ay|K`mbaW#}Y8 zIKQ3FkMQ&5bp9$o@6l1ubIR*FPRiJ207)(+V#@Qkv&t-o{Q`xJv#;2OOfR+ z?!*}zXJTn^rG6${{A1?gvlgYFnnhpL#Dj0VtjRA^n!)%cmlE)jPT<3eV3nnqIYSV&J%~bxJW9Qm@QhBC6usHDge*oCSzfUY4 z9NV*J3BKTo7!A(>zpH*sUtKO%Gmd`x^0LSL$Z_}fzg{K=@o!)vSKbMHZFS4cXKT8j zx74g_!`FyuV!=0wxy_*@t#c{UxiBbhVIi;JTVqH%r<=Dl7n|38vw&KX-LCS&o@eMw z$2V(VwneSxGtCt3O&k*zn6SWv1tu);Kg7b)i&>O$O`Ij@QBt53_-^1x&67;_cx<&BIf_??!=san%)ZNYb!Nv|B=%kN~ z9SE{7h0k={Bj}rjoF8zqv`5en;gX2>e=Y!LBa;V7(NXd}pp*Q~-{Jg8+H}IvE$9{m z?Ied{X*wR&_(gwxK=CAw!=RJ?>xBFoMwO(iP(Rh%CFpc*NXH7$nf-;HbOA%hMnTv7 z?w6b_JtydzAF@~tWs`A_gV{&;#L3 zuKmuf=06GoXR3Dto*5$6uQ6^(S_Aso?2$fBun78kMi<5cO{4!WMn@$$(8OE^e-HF? zK+%4uSF_Ipg0B5;?;gV7cvzzgJJ44<=%6n-ksNJ2YxLgq0psK}Ysi^XoS*$i&RcN+< zMRJZs^4?HyEf+745v*2gkbW`W%YHH6TG?7vS?TlH{1v`3U$C^SCFGL0NN-!DW5q4) zQ7wy~i8kig>n*(aS$(`uAsC~I%KbPii$vr0P_l#lHgv^;j)o3@u-xLukK%hHI4w?A zwiL6ERjh?*ShyYe01*SlP{&efw^vkITS_ZJ7JFrx-8F6-s@VH%y!=HqUV8V8)dFyN zRcXj-ttfA)Dzyiz?19q;R{-p*D)sq-7Js13>a$gpmbs)@GM@0Z;D^vVyvdHva7SQV zUF^OVyzItHFVMkj0pfAIj0e*E7OSICbsMTI( z4V3$<%1X=qrNL0;_|S6vioQP>O161f{ixC&s;ID4*~)|H#b9aaCz!6%UGK%4Na`IP zui~h8dR&VE4ORI~Xhh!_5kU%U5)+oNp|(qgNy zSuK`Is|7X+m08EH-sWjoxX@j{(Ceyq%=fsQWTPs3up;Ph@xk3ID@%Q@PZ%VY+-qg1 zGt2FjEtS?l73}S+@CV25k*and9HlK1i2_kkhSBH`m0Kz+{pIBq{_zzo=dVWbinqi7 zS5;nNEwfpnpg&-5u~dwUD=ov%#xJRJdF$}g_FhjzL!16_VbIpj!HQ>i0(Gsc(`TV7}kX?A!*s*6VHO{Y4rK~*IQf0UMs%(A>CJDH+ z!|8+*HhG(AWJ#`N&EjlnU2XF+39rcTM^{sVWkV9LJMq$cduS=h5%BsGFMs`s7q36b zk@-tKa#?yOiJnOWqqz*dRYk(o;EN@4tL;C0LSYg8YjLg=<#7&ssTb&-blcryEe;WZSMfDk z0Z$7@W+l2$I6{BqS$q~1j4Y#rYV=cfb_qTSPyZth(KlNBrTTT;yHSo9; z7;`KacvgmI1}&dB6Z5+2saFqt_6Q<6-aM*yV? zCQ(nPBeb>N-jCOxUR$lb+Fxl^jHn?&C8+qqYVqO&ZIwaf;iC|r^ZTy7_j$}D5q|ys zbAO+k`Q*$#XYIAtUVH7e)?Ry`lQ}oIJafn7}o?gJr=+kQBiP%hC#4O zOCG*Yww%atbE&F8)VFgwizrt~9Illb9<^}-Raxl^pS6NcluJ0&aphLmr*2;_v07ZX(_NYG70Y|KDlg&L)dxJC6Ut@d1ucaO5t;@)yV1IA-9Og~Nm6Y8>r2Zp3jL zj{9)@8OPH&UdHhrj?ZwMu*hPWh~q*Wm*S|yu@pxL$0{5*RL&G-2`>Yyc-bt< zKnT~G!2{nf;GuRwSH?@9Mn~{)hY|_4gjW)OJFkxl{3!7wxQ&;Uyr@Fa;9I6xgT8o6 zY?jp*2t<@<6i{lrJ>0Uw*U>sX>T3%$hgVtJl$BP0IG|V~zP4q`Ebwb5T4hxz8q;Vr znW>y+4MnXhBcYfAhEZi%tI`(p#X{jW-dv-5-t2{q)|H{ya;p#O_O)Bv;;qd}#2Q=f zi-ET7f|zwlWx2O$fuq6OFgME88;$w=E3DDbwU8hfT^4QjMHOnOuD;0{jkmXlBQYg# z37TkW!LV4H2|E_H`Xc`2-u8$;))MrES}>}BMU@!ZQmj*!MKNm2&GvHxQf-z$efa9N%j* z;0Q--pAPlZ+>m;}PZ*Al@QFa>swcQ#FwUD}v!QfWL21Ii= zeV|Nrpi`o&F;uOB)6_=SoH_Ni4cGIBXdzxGeicP%5K*JIJWZ!MRRyecH z9kmNQ?pk+~x2_(vE~9FtZ-ruww=|de8vTHh8{jpWnR;4us0fG@``fkv8j(j&LYBw3R3Aui8-LbHj!LBv`Y;Ze`HkmOn-*Pm0 z8yxdojX$!sg=6rIIK~>0+!tTfVW6+{g{Y3^AwJ|9*UR8{G`cW>@GPsVEf!gAZ4cvW z7AC=%H`*LoF$9SDREc@BwZN6j_$$74qiKJO;)@8K4PkYy5(x`VUjOoV+X{q1Kc=}^ z%r5E7NLutNHm~Zrj1H#mD{bNUvgOugzUVM?BGe3}q+S~Mv#g|CUmM18`uD9aXMHU+ z0T05+FGtU2S*IJF<335`p*FJrs1k2O;a^UXjM$p=g&Ku^ zgnP}hTBlN|MBb9sUxPtb1!YpT%yq?hTc<8q{DU7XsGnwyE^mdw6eGXU=P0JT;jL)6 z7Px&;XtMhJH06b$;*snM3F~SY-Vm*k;TqIlIEk0k)2PG{1j!4G%2`$oYK9r&Gu;@1 zfR&cP={+PAatKPqU`;TL!1Qoiiz#&c-!n0bXxx*e9E(0$X`PBt-WEoZ>S}N;sCCuV zyXu-;jSJ?;E_Yqi0+$=tOYl`P+dYpem$>G*W;>j7U2|l&$LVs+bGYjOuXWYB9CPNl z8eMZ83z2{zA)W1+@4~cqxy!Z0-Pq(>;&d$nn144p-C*6wzUvk_Jgz!QJzTZy8?;TA zyPB3X%?9yY-fm+9csIBkT&@N6a~oZ?9(Ub*7r%DY&2zcx8$i{#u%W@#R9|1~sJncD z3up^mjZKaPO^uf~TBjDXsn0ZF8A-A#?f8Spwst*hAYYsaABFVYff4aMTE2)MD5_E1MS=8c5I zv0yx=tg=p}fLpAeuTY932&SQBK0moBKiB4d0=LB2d}B$Ak2a3lZDu~Y|2@7*rnSG| z_un`EoRj~4ds zfqqPdvBdJ-#bp2Q86vU*rf zoRNM{pt3?v5I9UZ*(pi)*tMdwE6|^d7RnV%SRY7a-dx{7kFWqjG@}!d)y<@Rs zQKQ`X5r{r@tKZ0->60SUR=oq85Msm)2xPv&lM z7jJW{ebMH;B~~QKiLtW3QcmKuhjueI(-*qh`EXSA-E6v`1i>GLu(Zsl{Ci2k)YoI|+R$KR^ z)4gY+_0+A9L3C7fNLB~s?w!)DG+uJfRynyUN3QxpmgapS7Z1p4KNOHJ-FQ1{ozkXl zxUxob?a%Ia%j)NH=ev7?4ymm#C@t;#L~8G|1nJ9Ve>NDLA)VI+8jI!A+0qTU`Sk6~ zFK~DETRm0J#y^y+UWzr;92uAxo8?wNcXz%!n8iq^4@$4+=F6S^`EJmC;sM=6cTT@Y zeRhAIoYNoNP;-l2PWQ`IuSEXA$5VX;UHbI5T)R7V(0DqU8XG)`GT(u8+Lh=Xgbv*6 z(rFYcolw23eltILX@$IQ;46US^z)7RRDl~Ld{d!wo zC#rKii9g>@@@>jfm>^QuRzxWDY&`E-*^xIhI&&)Pj3?PrG~T1O6cwn4sP_e_yS_}P zW&fI@@sM}2E^j`^CPY^0Nlt0e_z8u7K!Xnshz_aeOiT{yc!Eiya|5`sMj0<=KSP)d zq`FN^em<1RuaApKjfu&$p-gT$E+$`oVd&)0ZlT7_dUw_y7n4mUCSMF?a`$mDxyHog z&LK?ZtU9(of1GtwbF$WW%W@jVW zM##irhc1K80n_KPI9y=jaGlOU8`C+J$L8?8>F%EIBrZw~iWm_PB>vtNP&GbkYkSIph9C;=uelb{FrL*8t@1LH!eUPRJq&5$NveyQp z)I$PPc5xQE7I2&OM)=gKH8Lh%h(-oq*W?t9oCe_Evw#|_e`s)sn#eox{dC%?u9`*y zKSsQnS|um4l#k{9eb$&&PE`N>?-iRx@#n`IrY4E{k70_Phr^v9^@8YL5INife6RbV89x3%ot2(ZSp6=@e1%ZgY#rlQ9Wi=|2(pF zOpUbZfaKceQ9mIvxB9xAu*r!AWR>5XdHs3#J}dSHzGueYdYl*_x!?aPnDhJ<6(_{= z9Zyir{w;F%k@%x>=a)J0MR1BGe`8t6{DIsekq6XjhQa;eG!Vf=)IHSA8k)foxYDU# ze@ae$X%Q^!fxl#$9TL-=8d#78Ur02OoA35W&OvwQS2;Fxd0dh%E9V|4 zbWu+eox2F3yDv6LPCWiHM(p`$g%tzS{01(y#iv`lYV3NFYW=n%zdJd4E-g#@Qmgwz;36RtTN7;!*cfGvuYx%{$SIw5jtjVOtU)=ptacUkp;`v@c9djU zGQ0-`%_<}zoLS(o$Uq?j(f~>hM1cbk8^hBqMrL^oLqfgm#8(4_nyF+?vid;}J&(Kv zDuEH#n1}Wvbi=6*Fl#jwvIx-H?$2Z2sm(bm-8cy%ay_~y4RDJmPG-NO{S1%#jeCG? zX=IrF{lJygHAOWLM+O@3){r&;>FLC!ofcBU9sfwUz*|Cih>ndEB+!={0?H%?rkZVU zb0Bz>?R**je+rOUtZ}fE*>DD_^HyR=A*YAtq%CCD=4x8z@ERCYM$IF74ULbP=6St_ zhRR&?u3mFaR?Sh=km3f@v{W0axXd{9Ij|8~zro^&&Hq0UtG;osh&aguD;Cn^qsg)r zFJ;D0!!vsR7jU-Z6z~xzWA6y?_>hnFAk9HYPx5HRA*;xCeah;4z2~Af^%^ zs2+oA&YPb8?|o_51||>>#E3rBo}ZU*r3h1uL4qe>Jc<7kJdVJ-ugA5)ek>@1pu8VY zEAaASkb}a`Mn-lUtLUkQ8n)obaKLdO<#SY4N`rZZ8sMWd5k4WWd-`P%i#?yKoTXQ!z276yL<=y*o9*XpB>;Mv~Q+^`=(V>m*Z%WYGvDSDHxD2nk$cb|RU~NK$X2 zNxe%Z#hq`WRCUTriEi%*Qof1*Dg*z(i(Gz&zikA3*}(s~8UOYX@FyDhi*4 zEoT1rY4~IhCVigGgk$#b?>o5sr6hlc3E%AAYgl76=_jj$G+@l+Xjz?_GXja*L~>3h zNdY5qh^&~*lqW~YyaUY6c{^9qK(Kbsqjq-hqpqX9dr{1Ca7*9_eJnY{XuAj`L?0^t2hD*6G57zoPKS6Nq&JpCu5t znemb7Yxti5fB$#+kmQpd1)tXpKA#IYt*A|T%zS7qB=B1d{6Cocb+(Srhl+(jeWaE$ zDt)A)M5&DwQ_EX>O==+lM04G&4W^d!&CUIR;fs4oRM57%6-LkmJ*Ep!A#GyWG;Uko zESzF-RJ|J_I1}Tr2#m++<{q-W0#qT@YKHUVQbL0iP-Hpo8&Hi%2(r4H{G5^>p)wr?npQ&R`GVR`elp-86aZ<0pbeyIyo_QLUPu1!Ms&=bA zhfz2Nh>ImFcf(B}BX_$S^%#@K&}h1F1-_+|-HBTU3EIDBtS32st($E?l3TiSp0n@X zO!LqFGlQo4nR4e-Q{7c>NL@b!6UULB0hB9hdD@05M&-4^j?i zdwW@Cz;%?H+2QYJNss&YogYnv_%DE$5Pvt~lSzDv-c0vopgKA(LEzGjaMjJxr zDJrnig_oG5HBa$XOj;Hh7mEpijmN4e{e3|KQ>6Rh-3xjH4+%RDw zV|;%PtaRZOBgl8TA>T^0OesCn-$U566#B!gmd9G6JmT9l@dMV<$KW`eCJvFU^O2fh zZpqDC;*)IV{cBnM62|{B7}Dk)^%V1&c2oNM0~rI!73E1(Q;%vBH&g8NBm;DdI)FRN z$+DwQY6=`fRqQT5zv(=XT8Ir9?e51ojV}w2xT{F?tdg5=CN)jTXPBsetVI_wZ@bl9 zwHI+=GWlc*@E2@kXaW@LENEykmg1KB)mjGtN=%PHor&&Cfikw6CKb7ASJVM*-2=FN zH|dMEdoeF{Y|GFSWvtAgB#oVL$bG*CzBBPi4GOj&+lPm8(hl06?eE$yb^R1@PvTxS zCne{mJ>7d~6Y(?Y##K~-dqGFva8KIdPL4mMN z5ks3gl<899>YxhD$_2eFA>P4Aj088+|1b)T2wEaCtp+I*j%!{|e70@Y^esA*$eTi; z4jB*K`ty=}Y@I~Akr25T(q!Kh;Azi3RPbHse4MeIsJ8&WfB|iYy}2(41M4!r-6{{@ z5`aWrfFg>XX-VCAWRr>MwE(*lce2}(-Mgjkd`yP8wxSldp5!MrMpD-=Fb&lvk`qzm zW&$z^DB|&(KAUM##Bcr3Gi>VHBomYRrbSRMQ~EPoM^O3`NDaBt+4jd`rU#xqO%!P$BFF-6pcV~p^4;Cy%n~|CwrqCtrKJfG&!NNMDTIa7bR?R&htDg zymT#YDKh_&lha@Afw_~r?3)KgNatCY!LmWu)*<9S3;<7?f)=H`*$L-lq-7HdSUaWt zI8{jc#OeJ8In#&*kdzk&tyVz984aAsDeqB6lMWRGiikYv}7@NjMzO&AUB+~wra1`wd(71`m)5O zlP^oeCg;yiG)*p;i?7o)l}$ht@+5Nm>k^*HRu&<3v$}a3p}>HcGr&{8lpi;++ zAUv@6Z%p*nr)<*V5ud7Q_nuDyM^i1dog}7!vaF2Co%8nabG@0tMJ5KAX=WM6O!JA1!1h#JR%rMdHWWA3Ds@NdlfE5TKEKrCM}d|vh!HXpG+~Q zJJtCbFXl{!EcA?B&?L*ny;>)@EXSpVDJBN1U{J@cg^y^5MrgsI6J~0m<6&J3h}Bb3 z2|YXym60AcW$IzJCOy{!Uu}4B2NG%D8kY81G;Tr;9z-Ai^C7)2vU(L(@vh;Q@rNK4n@-r0>X%g^)<%Dy!_SHXGyRUv-m4o+Q&6Ut7g(jnMIRSTc3?IO|J}m+Y}w` z7-o8|eg-0c`${E@IWG;Ybm5=yt#vzJ%#L((;0G{yv3}S~y~av~O^qkAKJ(esj4v7Z zcM5zqFTR0c)A^e=v+E7~*2g)2rqkaue32pz$&Y42TA|tXXvGwnrfQnK?@yY82s=-l z^9b5@93dT zWW*-q%>2eX&j@Ai{V%{dhf&TI7$=j)X`|D;7QnMNW3(28_nR~MrP#F4lvDgz<`_Xa z64cfe3I8!uIm|DahG<&h!NQxpABCu< zSH|H|4?^?lLRvX$CO?r9TfP{`D@PMrBv}sg;E_ZILh2!m`*0*gie`E4x|0QhTJ?a-2 z@x+t>lvvxHj@7vm4;BqVk&bm=vwwl8#y>wy6aW0+PQUad0X@YB+*OBU^+UHb`>*VO zAEtvGAY31EB%RZ99Ju>KTwIBc0vDc5>Kc!<2yG%x-lJOzS*(=jN<6_(wg0(jK!X>E z279PAXYpX?;E8-znf-Da|ASI?HF%^8d&qjZJcBY&9;1l||BkWUk<69o-!HffPPW`g z-Gilz1MBY2!IqWhJ9=r?42gEj1JI{Gcb%NGJ-x@1tUd*t_f;VfIM#hbYb>ezOrm4! zLsa)u^;uNiOOX|j_b?#)@&Up5n{-b9C4%$U=$sy#a`e&$A|Trd@;0yCLgzdAc@Ld$ zW0B4$XOAWO}V5JBn~M6`PqL1;$= zGAyBUn;CyTLC)p1E;|2?xweuZG&eiwpDKvMG=dz&xvr~81bNd8DI~~qX2^*I!2;Ew z`WgxAzWdCOj|p-cgNU}?C&-O}pxr$<-@jUK^mV%Yv3~giU0wmS=jddC4)3GOIe%^% z{8Jr;nC$QK60<&~Z6fA6l+{y7Y-N+(OS zH8lDe)xf9*Ml~?1fl&>NYG70YqZ%01z^DfP2@SMkKZ(}E1=w_>vfe>*gezGI z_=ZfrD+FBool=2rSuNn&@0M!t4LBkGeS#0Cw^&{faNEy0f8L(OauNh3{0#z5TXOU{ zTfl9<5=YY z(WgSd?-qRSD}?_$L9hw%9B`7e_upA>dY_|C>12 z@)y8Q6>p6okWIkfV{p-!3H}j-;|m{%rVr#N0DT7#?RR%+tBgMM4^;%$eusAl2eV(= zA-LwRcM15Z*o0#E!ap_mQ~}rgS%X&yxaQX#qQ6c7*Ze*t`2SGAwfLa%xf<{S>QR;*Yvf+`ZV1Z^`cZmu8eQFrGs}$V@@Oo= z&v>c5<;Bb2xdGOJQk$=;ywYA_E3^Bnru&!0XV@&>AJy4Q#S38ss@<;8FW`IGFW}oN zmNy5R6@PPCc`#63TH^NyT^7#L+Y)YD_T$!e?Gir+ZN6f!tMKAy^6`EIB{FP7RqQ1) zUjB9%FTL=^ZbO4J6?@f8pRJ;-tg5oPxgy{?UgB~P`+PGi?4{-9&C1O3vVg79Wr@V2 zF>f<|eZ0*ZZ(A8^3k=Jy($|bP%y{XoHTW(5XcTY20XMW?UJiwo`I=|iYUQ>3Y8sjJJ4M`)D#)Qc@ALS75Z1V5Pl$ zq$b^UUcCOJ&f)ROjyk8uwSW{?8muY{`kE_!KBcO{SHhdD(1sW9&B0H-M|>;M8}tX>*Cc+~zM=$}7uCM{LSPZ&k8a23ujW zVK0mnKQZqOwYIlVS20SqW+hlvsx$`zcHi(SD)ZFOo9C{Z=XKRNW_w&t5LVcmOUtW# zB~?LNWtA^j=^DY3ZTh=hXfTy#TRGHY3)lmdrD$*1=BiqKFqD>XxE&OM(m=4PqO7@0 zsVwuCSB_wbrTl#)Uh!%c&{g@HZ8m>ZX+_XiQXWu-*K5g4{D%9&T9>yLzccUk)Ymt7 zFRN{sy>KoV_=Dy4px<5+sF>-mv?)Tp*^XLX=V)>-a`no>W!u!_9_YXwGtpLsDVAZZTT6kB<@&(|FEl~$GvFKwmvCJbsLSfbcUOKib% ze|ey+6h1bbX_pg)`864!EVcWp0wtBDN@-QIzpA8aSjrMRQ4V=421#nOmo?+fPo*>M zU{f}NPqBv0uZ94nQUzW3XO@>VSK9o_%;sSmM&x56v)3k2KdVZV=H{8@m6a8hGs`IQ z4b#t(3Z|Li->E=MDgw&PAfj2N&sI6pR?e9WbxM1QIU0fxwg>Gh{!+U=07C?!!|8;{ zGs<*LD~eF{NF2W3Vhn`5tkRR79?`aoU?82a^F;3apBzzM+9T^`nKw7?~)n&_1^~2HIj+6cp&?7Y- z@>weA4-3W8{ibtqXzfoF6mw8&x)&f;G_2xR3(Y{BXZN!fu;J+M74(WeFq}~~{r>@e C+_3fl diff --git a/crates/polkavm/src/tests.rs b/crates/polkavm/src/tests.rs index 420dc8b2..a3ae1562 100644 --- a/crates/polkavm/src/tests.rs +++ b/crates/polkavm/src/tests.rs @@ -1276,15 +1276,38 @@ fn dynamic_paging_protect_memory(mut engine_config: Config) { .collect(); let mut instance = module.instantiate().unwrap(); + #[allow(clippy::match_wildcard_for_single_variants)] + match instance.protect_memory(0x10000, page_size).unwrap_err() { + MemoryAccessError::OutOfRangeAccess { address, length } => { + assert_eq!(address, 0x10000); + assert_eq!(length, u64::from(page_size)); + } + error => panic!("unexpected error: {error}"), + } + instance.set_reg(Reg::RA, crate::RETURN_TO_HOST); instance.set_next_program_counter(offsets[0]); let segfault = expect_segfault(instance.run().unwrap()); + assert_eq!(segfault.page_address, 0x10000); + assert!(!segfault.is_write_protected); assert_eq!(instance.program_counter(), Some(offsets[0])); instance.zero_memory(segfault.page_address, page_size).unwrap(); instance.protect_memory(segfault.page_address, page_size).unwrap(); - assert_eq!(instance.run().unwrap(), InterruptKind::Trap); + + let segfault = expect_segfault(instance.run().unwrap()); + assert_eq!(segfault.page_address, 0x10000); + assert!(segfault.is_write_protected); assert_eq!(instance.program_counter(), Some(offsets[1])); - assert_eq!(instance.next_program_counter(), None); + assert_eq!(instance.next_program_counter(), Some(offsets[1])); + + let segfault = expect_segfault(instance.run().unwrap()); + assert_eq!(segfault.page_address, 0x10000); + assert!(segfault.is_write_protected); + assert_eq!(instance.program_counter(), Some(offsets[1])); + assert_eq!(instance.next_program_counter(), Some(offsets[1])); + + instance.unprotect_memory(segfault.page_address, page_size).unwrap(); + match_interrupt!(instance.run().unwrap(), InterruptKind::Finished); } #[cfg(not(feature = "std"))] diff --git a/crates/polkavm/src/utils.rs b/crates/polkavm/src/utils.rs index 59345865..cd814c38 100644 --- a/crates/polkavm/src/utils.rs +++ b/crates/polkavm/src/utils.rs @@ -103,6 +103,9 @@ pub struct Segfault { /// The size of the page. pub page_size: u32, + + /// If `true` the access was a write to a read-only page which exists. + pub is_write_protected: bool, } #[derive(Clone, PartialEq, Eq, Debug)] From 8df6928629bcbdeded62d62906c5566886eccf8b Mon Sep 17 00:00:00 2001 From: Jan Bujak Date: Mon, 10 Nov 2025 11:03:46 +0000 Subject: [PATCH 06/21] Replace a `todo!` with a proper error --- crates/polkavm/src/sandbox/linux.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/polkavm/src/sandbox/linux.rs b/crates/polkavm/src/sandbox/linux.rs index a7735c49..4cd07dc0 100644 --- a/crates/polkavm/src/sandbox/linux.rs +++ b/crates/polkavm/src/sandbox/linux.rs @@ -2159,7 +2159,9 @@ impl super::Sandbox for Sandbox { } if !self.dynamic_paging_enabled { - todo!(); + Err(Error::from_str( + "freeing of pages when dynamic paging is not enabled is not implemented", + )) } else { unsafe { linux_raw::sys_madvise( From 3b74679e441d101a9ddfa685aa4486467f9a6d12 Mon Sep 17 00:00:00 2001 From: Jan Bujak Date: Mon, 10 Nov 2025 11:03:47 +0000 Subject: [PATCH 07/21] Return `OutOfRangeAccess` when trying to read memory which isn't paged in --- crates/polkavm/src/sandbox/generic.rs | 5 ++++- crates/polkavm/src/sandbox/linux.rs | 5 ++++- crates/polkavm/src/tests.rs | 29 +++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/crates/polkavm/src/sandbox/generic.rs b/crates/polkavm/src/sandbox/generic.rs index 0dd0fac3..1e702af4 100644 --- a/crates/polkavm/src/sandbox/generic.rs +++ b/crates/polkavm/src/sandbox/generic.rs @@ -1739,7 +1739,10 @@ impl super::Sandbox for Sandbox { let page_start = module.address_to_page(module.round_to_page_size_down(address)); let page_end = module.address_to_page(module.round_to_page_size_down(address + slice.len() as u32 - 1)); if !self.page_set.contains((page_start, page_end)) { - return Err(MemoryAccessError::Error("incomplete read".into())); + return Err(MemoryAccessError::OutOfRangeAccess { + address, + length: cast(slice.len()).to_u64(), + }); } } diff --git a/crates/polkavm/src/sandbox/linux.rs b/crates/polkavm/src/sandbox/linux.rs index 4cd07dc0..cf240758 100644 --- a/crates/polkavm/src/sandbox/linux.rs +++ b/crates/polkavm/src/sandbox/linux.rs @@ -1966,7 +1966,10 @@ impl super::Sandbox for Sandbox { let page_start = module.address_to_page(module.round_to_page_size_down(address)); let page_end = module.address_to_page(module.round_to_page_size_down(address + slice.len() as u32 - 1)); if !self.page_set.contains((page_start, page_end)) { - return Err(MemoryAccessError::Error("incomplete read".into())); + return Err(MemoryAccessError::OutOfRangeAccess { + address, + length: cast(slice.len()).to_u64(), + }); } else { let memory: &[core::mem::MaybeUninit] = unsafe { core::slice::from_raw_parts(self.memory_mmap.as_ptr().cast(), self.memory_mmap.len()) }; diff --git a/crates/polkavm/src/tests.rs b/crates/polkavm/src/tests.rs index a3ae1562..a00a99c7 100644 --- a/crates/polkavm/src/tests.rs +++ b/crates/polkavm/src/tests.rs @@ -1592,6 +1592,34 @@ fn dynamic_paging_read_at_bottom_of_address_space(mut engine_config: Config) { assert_eq!(instance.run().unwrap(), InterruptKind::Trap); } +fn dynamic_paging_read_memory_which_is_not_paged_in(mut engine_config: Config) { + engine_config.set_allow_dynamic_paging(true); + + let _ = env_logger::try_init(); + + let engine = Engine::new(&engine_config).unwrap(); + let page_size = get_native_page_size() as u32; + let mut builder = ProgramBlobBuilder::new(); + builder.add_export_by_basic_block(0, b"main"); + builder.set_code(&[asm::ret()], &[]); + + let blob = ProgramBlob::parse(builder.into_vec().unwrap().into()).unwrap(); + let mut module_config = ModuleConfig::new(); + module_config.set_page_size(page_size); + module_config.set_dynamic_paging(true); + let module = Module::from_blob(&engine, &module_config, blob).unwrap(); + let instance = module.instantiate().unwrap(); + + #[allow(clippy::match_wildcard_for_single_variants)] + match instance.read_memory(0x10000, page_size).unwrap_err() { + MemoryAccessError::OutOfRangeAccess { address, length } => { + assert_eq!(address, 0x10000); + assert_eq!(length, u64::from(page_size)); + } + error => panic!("unexpected error: {error}"), + } +} + fn dynamic_paging_write_at_page_boundary_with_no_pages(mut engine_config: Config) { engine_config.set_allow_dynamic_paging(true); @@ -4208,6 +4236,7 @@ run_tests! { dynamic_paging_read_at_top_of_address_space dynamic_paging_read_at_bottom_of_address_space dynamic_paging_read_with_upper_bits_set + dynamic_paging_read_memory_which_is_not_paged_in dynamic_paging_write_at_page_boundary_with_no_pages dynamic_paging_write_at_page_boundary_with_first_page dynamic_paging_write_at_page_boundary_with_second_page From 54fbd81bcb3061746ccdf895861cdbb027eba427 Mon Sep 17 00:00:00 2001 From: Jan Bujak Date: Mon, 10 Nov 2025 11:03:47 +0000 Subject: [PATCH 08/21] Fix `PageSet` removal --- crates/polkavm/src/page_set.rs | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/crates/polkavm/src/page_set.rs b/crates/polkavm/src/page_set.rs index cec2b33f..741d3864 100644 --- a/crates/polkavm/src/page_set.rs +++ b/crates/polkavm/src/page_set.rs @@ -238,7 +238,11 @@ impl PageSet { max: removed_max, }, ) { - SubResult::Disjoint => break, + SubResult::Disjoint => { + if interval.min <= removed_min { + break; + } + }, SubResult::None => { log::trace!(" Remove: {interval:?}"); to_remove.push(interval); @@ -585,4 +589,21 @@ mod tests { assert_eq!(set.to_vec(), vec![]); } } + + #[test] + fn disjoint_removal() { + let _ = env_logger::try_init(); + + let mut set = PageSet::new(); + set.insert((55, 221)); + set.remove((117, 131)); + set.remove((65, 131)); + assert!(!set.contains((85, 88))); + assert!(set.contains((55, 64))); + assert!(!set.contains((54, 64))); + assert!(!set.contains((55, 65))); + assert!(set.contains((132, 221))); + assert!(!set.contains((131, 221))); + assert!(!set.contains((132, 222))); + } } From 4da12b74b26591d1093d34334ce323d438504c62 Mon Sep 17 00:00:00 2001 From: Jan Bujak Date: Mon, 10 Nov 2025 11:03:48 +0000 Subject: [PATCH 09/21] Add few extra asserts in `PageSet` tests --- crates/polkavm/src/page_set.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/polkavm/src/page_set.rs b/crates/polkavm/src/page_set.rs index 741d3864..5b95536d 100644 --- a/crates/polkavm/src/page_set.rs +++ b/crates/polkavm/src/page_set.rs @@ -487,6 +487,7 @@ mod tests { { let mut set = PageSet::new(); set.insert((0, 100)); + assert_eq!(set.to_vec(), vec![(0, 100)]); set.insert((120, 130)); set.insert((140, 140)); set.insert((150, 150)); @@ -598,6 +599,7 @@ mod tests { set.insert((55, 221)); set.remove((117, 131)); set.remove((65, 131)); + assert_eq!(set.to_vec(), vec![(55, 64), (132, 221)]); assert!(!set.contains((85, 88))); assert!(set.contains((55, 64))); assert!(!set.contains((54, 64))); From fcfcfeb375e03e96e403b8c3bc4583b668159b21 Mon Sep 17 00:00:00 2001 From: Jan Bujak Date: Mon, 10 Nov 2025 11:03:48 +0000 Subject: [PATCH 10/21] Expand `PageSet` tests --- crates/polkavm/src/page_set.rs | 64 +++++++++++++++++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/crates/polkavm/src/page_set.rs b/crates/polkavm/src/page_set.rs index 5b95536d..7c960922 100644 --- a/crates/polkavm/src/page_set.rs +++ b/crates/polkavm/src/page_set.rs @@ -598,7 +598,7 @@ mod tests { let mut set = PageSet::new(); set.insert((55, 221)); set.remove((117, 131)); - set.remove((65, 131)); + set.remove((65, 131)); assert_eq!(set.to_vec(), vec![(55, 64), (132, 221)]); assert!(!set.contains((85, 88))); assert!(set.contains((55, 64))); @@ -608,4 +608,66 @@ mod tests { assert!(!set.contains((131, 221))); assert!(!set.contains((132, 222))); } + + #[test] + fn remove_in_the_middle_1() { + let _ = env_logger::try_init(); + + let mut set = PageSet::new(); + set.insert((117, 221)); + set.remove((137, 137)); + assert_eq!(set.to_vec(), vec![(117, 136), (138, 221)]); + assert!(set.contains((181, 181))); + } + + #[test] + fn remove_in_the_middle_2() { + let _ = env_logger::try_init(); + + let mut set = PageSet::new(); + set.insert((65, 221)); + set.remove((85, 147)); + assert_eq!(set.to_vec(), vec![(65, 84), (148, 221)]); + assert!(!set.contains((131, 131))); + assert!(set.contains((150, 151))); + } + + #[test] + fn insert_low() { + let _ = env_logger::try_init(); + + let mut set = PageSet::new(); + set.insert((158, 255)); + set.insert((0, 158)); + assert_eq!(set.to_vec(), vec![(0, 255)]); + assert!(set.contains((0, 255))); + } + + #[test] + fn remove_twice() { + let _ = env_logger::try_init(); + + let mut set = PageSet::new(); + set.insert((255, 255)); + assert_eq!(set.to_vec(), vec![(255, 255)]); + set.remove((121, 255)); + assert_eq!(set.to_vec(), vec![]); + set.remove((121, 221)); + assert_eq!(set.to_vec(), vec![]); + assert!(!set.contains((255, 255))); + } + + #[test] + fn insert_remove_insert() { + let _ = env_logger::try_init(); + + let mut set = PageSet::new(); + set.insert((38, 103)); + assert_eq!(set.to_vec(), vec![(38, 103)]); + set.remove((64, 141)); + assert_eq!(set.to_vec(), vec![(38, 63)]); + set.insert((85, 121)); + assert_eq!(set.to_vec(), vec![(38, 63), (85, 121)]); + assert!(!set.contains((65, 85))); + } } From efca0e1fc3f740780d79d304c7c0f809a7d96b18 Mon Sep 17 00:00:00 2001 From: Jan Bujak Date: Mon, 10 Nov 2025 11:03:49 +0000 Subject: [PATCH 11/21] Add `PageSet` benches --- Cargo.lock | 336 +++++++++++++++++++++++++++++++++++--- Cargo.toml | 1 + benches/Cargo.toml | 17 ++ benches/run.sh | 3 + benches/src/main.rs | 92 +++++++++++ crates/polkavm/src/lib.rs | 2 + 6 files changed, 426 insertions(+), 25 deletions(-) create mode 100644 benches/Cargo.toml create mode 100755 benches/run.sh create mode 100644 benches/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 182ac224..dbae610b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -18,7 +18,7 @@ dependencies = [ "getrandom", "once_cell", "version_check", - "zerocopy", + "zerocopy 0.7.34", ] [[package]] @@ -30,25 +30,32 @@ dependencies = [ "memchr", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + [[package]] name = "anstream" -version = "0.6.4" +version = "0.6.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab91ebe16eb252986481c5b62f6098f3b698a45e34b5b98200cf20dd2484a44" +checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" dependencies = [ "anstyle", "anstyle-parse", "anstyle-query", "anstyle-wincon", "colorchoice", + "is_terminal_polyfill", "utf8parse", ] [[package]] name = "anstyle" -version = "1.0.2" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15c4c2c83f81532e5845a733998b6971faca23490340a418e9b72a3ec9de12ea" +checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" [[package]] name = "anstyle-parse" @@ -70,12 +77,13 @@ dependencies = [ [[package]] name = "anstyle-wincon" -version = "3.0.1" +version = "3.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628" +checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a" dependencies = [ "anstyle", - "windows-sys 0.48.0", + "once_cell_polyfill", + "windows-sys 0.60.2", ] [[package]] @@ -105,6 +113,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "benches" +version = "0.30.0" +dependencies = [ + "criterion", + "oorandom", + "polkavm", +] + [[package]] name = "bindgen" version = "0.69.4" @@ -114,7 +131,7 @@ dependencies = [ "bitflags 2.4.2", "cexpr", "clang-sys", - "itertools", + "itertools 0.12.1", "lazy_static", "lazycell", "log", @@ -186,6 +203,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.1.18" @@ -210,6 +233,33 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "clang-sys" version = "1.6.1" @@ -223,9 +273,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.11" +version = "4.5.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfaff671f6b22ca62406885ece523383b9b64022e341e53e009a62ebc47a45f2" +checksum = "4c26d721170e0295f191a69bd9a1f93efcdb0aff38684b61ab5750468972e5f5" dependencies = [ "clap_builder", "clap_derive", @@ -233,9 +283,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.11" +version = "4.5.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a216b506622bb1d316cd51328dce24e07bdff4a6128a47c7e7fad11878d5adbb" +checksum = "75835f0c7bf681bfd05abe44e965760fea999a5286c6eb2d59883634fd02011a" dependencies = [ "anstream", "anstyle", @@ -245,9 +295,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.4.7" +version = "4.5.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" +checksum = "2a0b5487afeab2deb2ff4e03a807ad1a03ac532ff5a2cee5d86884440c7f7671" dependencies = [ "heck", "proc-macro2", @@ -257,9 +307,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.6.0" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" +checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d" [[package]] name = "cmake" @@ -349,6 +399,70 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "criterion" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1c047a62b0cc3e145fa84415a3191f628e980b194c2755aa12300a4e6cbd928" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "itertools 0.13.0", + "num-traits", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b1bcc0dc7dfae599d84ad0b1a55f80cde8af3725da8313b528da95ef783e338" +dependencies = [ + "cast", + "itertools 0.13.0", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "derive_arbitrary" version = "1.4.2" @@ -608,6 +722,17 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy 0.8.27", +] + [[package]] name = "hashbrown" version = "0.13.2" @@ -625,9 +750,9 @@ checksum = "7dfda62a12f55daeae5015f81b0baea145391cb4520f86c248fc615d72640d12" [[package]] name = "heck" -version = "0.4.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "hello-world-host" @@ -661,6 +786,12 @@ dependencies = [ "png", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + [[package]] name = "itertools" version = "0.12.1" @@ -670,6 +801,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.11" @@ -835,6 +975,12 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + [[package]] name = "oorandom" version = "11.1.5" @@ -897,6 +1043,7 @@ dependencies = [ "pathfinder_geometry", "plotters-backend", "plotters-bitmap", + "plotters-svg", "ttf-parser", "wasm-bindgen", "web-sys", @@ -918,6 +1065,15 @@ dependencies = [ "plotters-backend", ] +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + [[package]] name = "png" version = "0.17.16" @@ -1169,6 +1325,26 @@ dependencies = [ "bitflags 2.4.2", ] +[[package]] +name = "rayon" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.3.5" @@ -1432,9 +1608,9 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "strsim" -version = "0.10.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "syn" @@ -1500,6 +1676,16 @@ dependencies = [ "tikv-jemalloc-sys", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "ttf-parser" version = "0.20.0" @@ -1530,9 +1716,9 @@ checksum = "22049a19f4a68748a168c0fc439f9516686aa045927ff767eca0a85101fb6e73" [[package]] name = "utf8parse" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "version-compare" @@ -1681,6 +1867,12 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + [[package]] name = "windows-sys" version = "0.48.0" @@ -1699,6 +1891,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + [[package]] name = "windows-targets" version = "0.48.1" @@ -1723,13 +1924,30 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm", + "windows_i686_gnullvm 0.52.6", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.0" @@ -1742,6 +1960,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + [[package]] name = "windows_aarch64_msvc" version = "0.48.0" @@ -1754,6 +1978,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + [[package]] name = "windows_i686_gnu" version = "0.48.0" @@ -1766,12 +1996,24 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + [[package]] name = "windows_i686_msvc" version = "0.48.0" @@ -1784,6 +2026,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + [[package]] name = "windows_x86_64_gnu" version = "0.48.0" @@ -1796,6 +2044,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.0" @@ -1808,6 +2062,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + [[package]] name = "windows_x86_64_msvc" version = "0.48.0" @@ -1820,6 +2080,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + [[package]] name = "wio" version = "0.2.2" @@ -1852,7 +2118,16 @@ version = "0.7.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae87e3fcd617500e5d106f0380cf7b77f3c6092aae37191433159dda23cfb087" dependencies = [ - "zerocopy-derive", + "zerocopy-derive 0.7.34", +] + +[[package]] +name = "zerocopy" +version = "0.8.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c" +dependencies = [ + "zerocopy-derive 0.8.27", ] [[package]] @@ -1865,3 +2140,14 @@ dependencies = [ "quote", "syn", ] + +[[package]] +name = "zerocopy-derive" +version = "0.8.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/Cargo.toml b/Cargo.toml index a399b040..7f42b1b9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ members = [ "examples/doom", "examples/quake", "examples/hello-world", + "benches", ] [workspace.package] diff --git a/benches/Cargo.toml b/benches/Cargo.toml new file mode 100644 index 00000000..51fc1987 --- /dev/null +++ b/benches/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "benches" +version.workspace = true +authors.workspace = true +license.workspace = true +edition.workspace = true +rust-version.workspace = true +repository.workspace = true +publish = false + +[dependencies] +polkavm = { workspace = true, features = ["export-internals-for-testing"] } +criterion = "0.7.0" +oorandom = "11.1.5" + +[lints] +workspace = true diff --git a/benches/run.sh b/benches/run.sh new file mode 100755 index 00000000..8fa8c488 --- /dev/null +++ b/benches/run.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +cargo run --release -- --bench "$@" diff --git a/benches/src/main.rs b/benches/src/main.rs new file mode 100644 index 00000000..4119fc22 --- /dev/null +++ b/benches/src/main.rs @@ -0,0 +1,92 @@ +use criterion::{criterion_group, criterion_main, Bencher, Criterion}; +use polkavm::_for_testing::PageSet; +use std::hint::black_box; +use std::time::{Duration, Instant}; + +const SEED: u128 = 9324658635124; +const MAX: u64 = 1048576; + +fn run_insert_test(b: &mut Bencher, max_range: u64) { + b.iter_custom(|iters| { + let mut elapsed = Duration::default(); + for _ in 0..iters { + let mut set = PageSet::new(); + let mut rng = oorandom::Rand64::new(SEED); + let start = Instant::now(); + for n in 0..3000000 { + let min = rng.rand_range(0..max_range); + let max = rng.rand_range(min..max_range); + let min = min as u32; + let max = max as u32; + + if (n % 3) == 0 { + set.insert((min, max)); + } else if (n % 3) == 1 { + if REMOVE { + set.remove((min, max)); + } + } else { + if CHECK { + black_box(set.contains((min, max))); + } + } + } + elapsed += start.elapsed(); + black_box(set); + } + + elapsed + }); +} + +fn pageset_benchmarks(c: &mut Criterion) { + c.bench_function("insert million entries (narrow range)", |b| { + run_insert_test::(b, 4000) + }); + c.bench_function("insert and remove million entries (narrow range)", |b| { + run_insert_test::(b, 4000) + }); + c.bench_function("insert million entries (wide range)", |b| run_insert_test::(b, MAX)); + c.bench_function("insert and remove million entries (wide range)", |b| { + run_insert_test::(b, MAX) + }); + c.bench_function("insert, remove and check million entries (wide range)", |b| { + run_insert_test::(b, MAX) + }); + c.bench_function("clear empty set", |b| { + b.iter_custom(|iters| { + let mut set = PageSet::new(); + let start = Instant::now(); + for _ in 0..iters { + set.clear(); + } + let elapsed = start.elapsed(); + black_box(set); + elapsed + }); + }); + + c.bench_function("clear set with 50 thousand entries", |b| { + b.iter_custom(|iters| { + let mut elapsed = Duration::default(); + for _ in 0..iters { + let mut set = PageSet::new(); + let mut rng = oorandom::Rand64::new(SEED); + for _ in 0..50000 { + let min = rng.rand_range(0..MAX); + let max = rng.rand_range(min..MAX); + set.insert((min as u32, max as u32)); + } + + let start = Instant::now(); + set.clear(); + elapsed += start.elapsed(); + black_box(set); + } + elapsed + }); + }); +} + +criterion_group!(benches, pageset_benchmarks); +criterion_main!(benches); diff --git a/crates/polkavm/src/lib.rs b/crates/polkavm/src/lib.rs index 931e2a30..b5050131 100644 --- a/crates/polkavm/src/lib.rs +++ b/crates/polkavm/src/lib.rs @@ -169,5 +169,7 @@ pub mod _for_testing { crate::sandbox::init_native_page_size(); crate::shm_allocator::ShmAllocator::new() } + + pub use crate::page_set::PageSet; } } From a7cc5c38249729165911cf27ed8954a1c490605c Mon Sep 17 00:00:00 2001 From: Jan Bujak Date: Mon, 10 Nov 2025 11:03:49 +0000 Subject: [PATCH 12/21] Add `PageSet` fuzz test --- fuzz/Cargo.toml | 7 +++ fuzz/fuzz_targets/fuzz_page_set.rs | 93 ++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+) create mode 100644 fuzz/fuzz_targets/fuzz_page_set.rs diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 540a1ec4..c59386ca 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -57,6 +57,13 @@ test = false doc = false bench = false +[[bin]] +name = "fuzz_page_set" +path = "fuzz_targets/fuzz_page_set.rs" +test = false +doc = false +bench = false + [workspace] resolver = "2" members = ["."] diff --git a/fuzz/fuzz_targets/fuzz_page_set.rs b/fuzz/fuzz_targets/fuzz_page_set.rs new file mode 100644 index 00000000..b53f1192 --- /dev/null +++ b/fuzz/fuzz_targets/fuzz_page_set.rs @@ -0,0 +1,93 @@ +#![no_main] + +use arbitrary::Arbitrary; +use libfuzzer_sys::fuzz_target; + +use polkavm::_for_testing::PageSet; + +#[derive(Arbitrary, Debug)] +enum Action { + Insert { min: u8, max: u8 }, + Remove { min: u8, max: u8 }, + Contains { min: u8, max: u8 }, +} + +#[derive(Arbitrary, Debug)] +struct Input { + actions: Vec, +} + +#[derive(Default)] +struct PageSetNaive { + bitmask: [u64; 4], +} + +impl PageSetNaive { + fn insert_one(&mut self, n: u8) { + self.bitmask[usize::from(n >> 6)] |= 1 << (n & 0b00111111); + } + + fn remove_one(&mut self, n: u8) { + self.bitmask[usize::from(n >> 6)] &= !(1 << (n & 0b00111111)); + } + + fn contains_one(&self, n: u8) -> bool { + (self.bitmask[usize::from(n >> 6)] & (1 << (n & 0b00111111))) != 0 + } + + fn insert(&mut self, min: u8, max: u8) { + for n in min..=max { + self.insert_one(n); + } + } + + fn remove(&mut self, min: u8, max: u8) { + for n in min..=max { + self.remove_one(n); + } + } + + fn contains(&self, min: u8, max: u8) -> bool { + for n in min..=max { + if !self.contains_one(n) { + return false; + } + } + true + } +} + +fn swap(min: &mut u8, max: &mut u8) { + if *min > *max { + let max_value = *max; + *max = *min; + *min = max_value; + } +} + +fuzz_target!(|input: Input| { + let mut page_set = PageSet::new(); + let mut naive = PageSetNaive::default(); + for action in input.actions { + match action { + Action::Insert { mut min, mut max } => { + swap(&mut min, &mut max); + page_set.insert((u32::from(min), u32::from(max))); + naive.insert(min, max); + } + Action::Remove { mut min, mut max } => { + swap(&mut min, &mut max); + page_set.remove((u32::from(min), u32::from(max))); + naive.remove(min, max); + } + Action::Contains { mut min, mut max } => { + swap(&mut min, &mut max); + assert_eq!( + page_set.contains((u32::from(min), u32::from(max))), + naive.contains(min, max), + "contains failed for: ({min}, {max})" + ); + } + } + } +}); From b00c6c15a73965fdf184ecc2dc50ecc755bb20e5 Mon Sep 17 00:00:00 2001 From: Jan Bujak Date: Mon, 10 Nov 2025 11:03:50 +0000 Subject: [PATCH 13/21] Silence clippy warning for an internal struct --- crates/polkavm/src/generic_allocator.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/polkavm/src/generic_allocator.rs b/crates/polkavm/src/generic_allocator.rs index 81d41e40..b73ad7e3 100644 --- a/crates/polkavm/src/generic_allocator.rs +++ b/crates/polkavm/src/generic_allocator.rs @@ -122,6 +122,7 @@ fn test_to_bin_index() { assert_eq!(self::u32::to_bin_index::<3, true>(2), 1); } +#[allow(clippy::exhaustive_structs)] #[doc(hidden)] #[cfg_attr(test, derive(PartialEq, Debug))] pub struct AllocatorBinConfig { From b5a6989b7e5aa07a4b214e95b4938d5629f0bf6a Mon Sep 17 00:00:00 2001 From: Jan Bujak Date: Mon, 10 Nov 2025 11:03:50 +0000 Subject: [PATCH 14/21] Bump `picosimd` to 0.9.2 --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- crates/polkavm-zygote/Cargo.lock | 4 ++-- fuzz/Cargo.lock | 4 ++-- guest-programs/Cargo.lock | 4 ++-- tools/benchtool/Cargo.lock | 4 ++-- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dbae610b..6e5f2889 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1020,9 +1020,9 @@ dependencies = [ [[package]] name = "picosimd" -version = "0.9.0" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83bd2c4dd3c2cf5366fd00a2c4e5b2b42b73ef7c7bf024ae6ef45473c6b81fca" +checksum = "af35c838647fef3d6d052e27006ef88ea162336eee33063c50a63f163c18cdeb" [[package]] name = "pkg-config" diff --git a/Cargo.toml b/Cargo.toml index 7f42b1b9..54012f36 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,7 +55,7 @@ libc = "0.2.149" log = "0.4.20" object = { version = "0.36.1", default-features = false } paste = "1.0.15" -picosimd = { version = "0.9.0", features = ["ops"] } +picosimd = { version = "0.9.2", features = ["ops"] } proc-macro2 = "1.0.69" proptest = "1.3.1" quote = "1.0.33" diff --git a/crates/polkavm-zygote/Cargo.lock b/crates/polkavm-zygote/Cargo.lock index 33e95c66..07fb609a 100644 --- a/crates/polkavm-zygote/Cargo.lock +++ b/crates/polkavm-zygote/Cargo.lock @@ -10,9 +10,9 @@ checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" [[package]] name = "picosimd" -version = "0.9.0" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83bd2c4dd3c2cf5366fd00a2c4e5b2b42b73ef7c7bf024ae6ef45473c6b81fca" +checksum = "af35c838647fef3d6d052e27006ef88ea162336eee33063c50a63f163c18cdeb" [[package]] name = "polkavm-assembler" diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index 64b37ce3..7dd9d285 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -222,9 +222,9 @@ checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" [[package]] name = "picosimd" -version = "0.9.0" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83bd2c4dd3c2cf5366fd00a2c4e5b2b42b73ef7c7bf024ae6ef45473c6b81fca" +checksum = "af35c838647fef3d6d052e27006ef88ea162336eee33063c50a63f163c18cdeb" [[package]] name = "polkavm" diff --git a/guest-programs/Cargo.lock b/guest-programs/Cargo.lock index 709fb67b..d937230a 100644 --- a/guest-programs/Cargo.lock +++ b/guest-programs/Cargo.lock @@ -105,9 +105,9 @@ checksum = "7e5daf3ef08b3ef4c9b9de02cd7a1638ee612359fd8da3646192bc8ee57e218c" [[package]] name = "picosimd" -version = "0.9.0" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83bd2c4dd3c2cf5366fd00a2c4e5b2b42b73ef7c7bf024ae6ef45473c6b81fca" +checksum = "af35c838647fef3d6d052e27006ef88ea162336eee33063c50a63f163c18cdeb" [[package]] name = "polkavm-common" diff --git a/tools/benchtool/Cargo.lock b/tools/benchtool/Cargo.lock index 596a49b1..2e231b34 100644 --- a/tools/benchtool/Cargo.lock +++ b/tools/benchtool/Cargo.lock @@ -1750,9 +1750,9 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "picosimd" -version = "0.9.0" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83bd2c4dd3c2cf5366fd00a2c4e5b2b42b73ef7c7bf024ae6ef45473c6b81fca" +checksum = "af35c838647fef3d6d052e27006ef88ea162336eee33063c50a63f163c18cdeb" [[package]] name = "pin-project-lite" From fc1f605b2a4d6f1c814d6a110562c83ff3a9d652 Mon Sep 17 00:00:00 2001 From: Jan Bujak Date: Mon, 10 Nov 2025 11:03:51 +0000 Subject: [PATCH 15/21] Rewrite `PageSet` --- Cargo.lock | 2 + crates/polkavm/Cargo.toml | 2 + crates/polkavm/src/page_set.rs | 984 ++++++++++++++++++++++------- crates/polkavm/src/page_set_sse.rs | 245 +++++++ fuzz/Cargo.lock | 1 + tools/benchtool/Cargo.lock | 1 + 6 files changed, 1024 insertions(+), 211 deletions(-) create mode 100644 crates/polkavm/src/page_set_sse.rs diff --git a/Cargo.lock b/Cargo.lock index 6e5f2889..5dbcec0d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1109,7 +1109,9 @@ dependencies = [ "image", "libc", "log", + "oorandom", "paste", + "picosimd", "polkavm-assembler", "polkavm-common", "polkavm-linker", diff --git a/crates/polkavm/Cargo.toml b/crates/polkavm/Cargo.toml index 8987892c..bc0e6aff 100644 --- a/crates/polkavm/Cargo.toml +++ b/crates/polkavm/Cargo.toml @@ -12,6 +12,7 @@ description = "A fast and secure RISC-V based virtual machine" [dependencies] log = { workspace = true } +picosimd = { workspace = true } polkavm-assembler = { workspace = true, features = ["alloc"] } polkavm-common = { workspace = true, features = ["alloc", "logging", "regmap", "unique-id", "simd"] } schnellru = { workspace = true, optional = true } @@ -29,6 +30,7 @@ polkavm-linker = { workspace = true } image = { workspace = true, features = ["tga"] } ruzstd = { workspace = true } paste = { workspace = true } +oorandom = "11.1.5" [lints] workspace = true diff --git a/crates/polkavm/src/page_set.rs b/crates/polkavm/src/page_set.rs index 7c960922..3ba34082 100644 --- a/crates/polkavm/src/page_set.rs +++ b/crates/polkavm/src/page_set.rs @@ -1,290 +1,852 @@ -// TODO: This is inefficient. REWRITE THIS. +use core::marker::PhantomData; +use polkavm_common::cast::cast; -use alloc::collections::BTreeSet; -use alloc::vec::Vec; -use core::cmp::Ord; +#[inline(always)] +const fn bits() -> usize { + core::mem::size_of::() * 8 +} + +trait RawMask: + Copy + Clone + Sized + core::ops::BitAnd + core::ops::BitOr + core::ops::BitAndAssign + core::ops::BitOrAssign +{ + #[inline(always)] + fn alignment() -> usize { + bits::() + } -#[derive(Copy, Clone, Debug)] -struct Interval { - min: u32, - max: u32, + fn zero() -> Self; + fn full() -> Self; + fn bit(position: usize) -> Self; + fn mask_lo(offset: usize, length: usize) -> Self; + fn mask_hi(length: usize) -> Self; + fn bitandnot_assign(&mut self, rhs: Self); + fn not(self) -> Self; + fn is_equal(self, rhs: Self) -> bool; + fn is_zero(self) -> bool; + + #[cfg(test)] + fn trailing_zeros(self) -> u32; + + #[cfg(test)] + fn leading_zeros(self) -> u32; } -impl From<(u32, u32)> for Interval { - fn from((min, max): (u32, u32)) -> Interval { - Interval { min, max } +impl RawMask for u64 { + #[inline(always)] + fn zero() -> Self { + 0 + } + + #[inline(always)] + fn full() -> Self { + Self::MAX + } + + #[inline(always)] + fn bit(position: usize) -> Self { + 1 << position + } + + #[inline(always)] + fn mask_lo(offset: usize, length: usize) -> Self { + (Self::MAX >> (Self::alignment() - length)) << offset + } + + #[inline(always)] + fn mask_hi(length: usize) -> Self { + Self::MAX >> (Self::alignment() - length) + } + + #[inline(always)] + fn bitandnot_assign(&mut self, rhs: Self) { + *self &= !rhs; + } + + #[inline(always)] + fn not(self) -> Self { + !self + } + + #[inline(always)] + fn is_equal(self, rhs: Self) -> bool { + self == rhs + } + + #[inline(always)] + fn is_zero(self) -> bool { + self == 0 + } + + #[cfg(test)] + fn trailing_zeros(self) -> u32 { + u64::trailing_zeros(self) + } + + #[cfg(test)] + fn leading_zeros(self) -> u32 { + u64::leading_zeros(self) } } -#[derive(Copy, Clone, PartialEq, Eq, Debug)] -enum SubResult { - Disjoint, - None, - One(Interval), - Two(Interval, Interval), +#[derive(Debug)] +struct AlignedRange +where + M: RawMask, +{ + mask_ty: PhantomData, + aligned_start_group: usize, + aligned_start: usize, + aligned_end_group: usize, + aligned_end: usize, + unaligned_lo_start: usize, + unaligned_lo_end: usize, + unaligned_lo_group: usize, + unaligned_lo_length: usize, + unaligned_lo_offset: usize, + unaligned_hi_start: usize, + unaligned_hi_end: usize, + unaligned_hi_group: usize, + unaligned_hi_length: usize, } -impl Interval { - fn merge(self, rhs: Interval) -> Option { - if (self.min >= rhs.min && self.min <= rhs.max) - || (rhs.min >= self.min && rhs.min <= self.max) - || (self.max >= rhs.min && self.max <= rhs.max) - || (self.max < u32::MAX && self.max + 1 == rhs.min) - || (rhs.max < u32::MAX && self.min == rhs.max + 1) - { - Some(Interval::from((self.min.min(rhs.min), self.max.max(rhs.max)))) +impl AlignedRange +where + M: RawMask, +{ + #[inline(always)] + fn unaligned_mask_lo(&self) -> M { + if self.unaligned_lo_length == 0 { + M::zero() } else { - None + M::mask_lo(self.unaligned_lo_offset, self.unaligned_lo_length) } } - fn subtract(existing: Interval, to_remove: Interval) -> SubResult { - if to_remove.max < existing.min || to_remove.min > existing.max { - return SubResult::Disjoint; + #[inline(always)] + fn unaligned_mask_hi(&self) -> M { + if self.unaligned_hi_length == 0 { + M::zero() + } else { + M::mask_hi(self.unaligned_hi_length) } + } - if to_remove.min > existing.max || to_remove.max < existing.min { - SubResult::One(existing) - } else if to_remove.min <= existing.min && to_remove.max >= existing.max { - SubResult::None - } else if to_remove.min <= existing.min && to_remove.max < existing.max { - SubResult::One(Interval::from((to_remove.max + 1, existing.max))) - } else if to_remove.min > existing.min && to_remove.max >= existing.max { - SubResult::One(Interval::from((existing.min, to_remove.min - 1))) - } else { - SubResult::Two( - Interval::from((existing.min, to_remove.min - 1)), - Interval::from((to_remove.max + 1, existing.max)), - ) + #[inline(always)] + fn has_unaligned_lo(&self) -> bool { + self.unaligned_lo_length > 0 + } + + #[inline(always)] + fn has_unaligned_hi(&self) -> bool { + self.unaligned_hi_length > 0 + } +} + +macro_rules! kani_assert { + ($cond:expr) => {{ + #[cfg(any(kani, test))] + assert!($cond); + + #[cfg(not(any(kani, test)))] + if !$cond { + // SAFETY: The condition was proven by Kani to always hold. + unsafe { + core::hint::unreachable_unchecked(); + } + } + }}; +} + +macro_rules! kani_assert_eq { + ($lhs:expr, $rhs:expr) => {{ + #[cfg(any(kani, test))] + assert_eq!($lhs, $rhs); + + #[cfg(not(any(kani, test)))] + if $lhs != $rhs { + // SAFETY: The condition was proven by Kani to always hold. + unsafe { + core::hint::unreachable_unchecked(); + } } + }}; +} + +#[inline(always)] +fn align_range(start: usize, end: usize) -> AlignedRange +where + M: RawMask, +{ + #[inline(always)] + fn opt(cond: bool, value: usize) -> usize { + let mask = if cond { !0 } else { 0 }; + mask & value + } + + let original_start = start; + let start = start.min(end); + + let aligned_end_group = end / M::alignment(); + let aligned_start_group = start.div_ceil(M::alignment()); + let has_aligned = aligned_end_group > aligned_start_group; + let aligned_end_group = opt(has_aligned, aligned_end_group); + let aligned_start_group = opt(has_aligned, aligned_start_group); + let aligned_end = aligned_end_group * M::alignment(); + let aligned_start = aligned_start_group * M::alignment(); + + let has_unaligned_lo = (start % M::alignment()) != 0 || !has_aligned; + let unaligned_lo_start = opt(has_unaligned_lo, start); + let unaligned_lo_group = unaligned_lo_start / M::alignment(); + let unaligned_lo_group_start = unaligned_lo_group * M::alignment(); + let unaligned_lo_end = opt(has_unaligned_lo, (unaligned_lo_group_start.saturating_add(M::alignment())).min(end)); + let unaligned_lo_length = unaligned_lo_end - unaligned_lo_start; + let unaligned_lo_offset = unaligned_lo_start - unaligned_lo_group_start; + + let has_unaligned_hi = (end % M::alignment()) != 0 && end > unaligned_lo_end; + let unaligned_hi_end = opt(has_unaligned_hi, end); + let unaligned_hi_group = unaligned_hi_end / M::alignment(); + let unaligned_hi_start = unaligned_hi_group * M::alignment(); + let unaligned_hi_length = unaligned_hi_end - unaligned_hi_start; + + kani_assert!(unaligned_lo_start <= unaligned_lo_end); + kani_assert!(unaligned_hi_start <= unaligned_hi_end); + kani_assert_eq!(unaligned_lo_end - unaligned_lo_start, unaligned_lo_length); + kani_assert_eq!(unaligned_hi_end - unaligned_hi_start, unaligned_hi_length); + kani_assert_eq!(aligned_start % M::alignment(), 0); + kani_assert_eq!(aligned_end % M::alignment(), 0); + kani_assert_eq!(unaligned_hi_start % M::alignment(), 0); + kani_assert!(aligned_end >= aligned_start); + kani_assert!(unaligned_lo_offset < M::alignment()); + kani_assert!(unaligned_hi_length == 0 || unaligned_hi_start >= unaligned_lo_end); + kani_assert!(unaligned_hi_length == 0 || unaligned_hi_start >= aligned_end); + kani_assert!(aligned_start == aligned_end || unaligned_lo_end <= aligned_start); + + kani_assert!(aligned_start_group < usize::MAX / M::alignment()); + kani_assert!(aligned_end_group <= usize::MAX / M::alignment()); + kani_assert!(unaligned_lo_group <= usize::MAX / M::alignment()); + kani_assert!(unaligned_hi_group <= usize::MAX / M::alignment()); + + kani_assert!(start <= original_start); + kani_assert!(start <= end); + kani_assert!(aligned_start_group <= aligned_end_group); + kani_assert!(aligned_start_group <= (start / M::alignment() + 1)); + kani_assert!(aligned_end_group <= end / M::alignment()); + kani_assert!(unaligned_lo_group <= start / M::alignment()); + kani_assert!(unaligned_hi_group <= end / M::alignment()); + + AlignedRange { + mask_ty: PhantomData, + aligned_start_group, + aligned_start, + aligned_end_group, + aligned_end, + + unaligned_lo_start, + unaligned_lo_end, + unaligned_lo_group, + unaligned_lo_length, + unaligned_lo_offset, + + unaligned_hi_start, + unaligned_hi_end, + unaligned_hi_group, + unaligned_hi_length, } } -#[test] -fn test_interval_merge() { - assert_eq!(Interval::from((1, 2)).merge(Interval::from((2, 4))), Some(Interval::from((1, 4)))); - assert_eq!(Interval::from((2, 4)).merge(Interval::from((1, 2))), Some(Interval::from((1, 4)))); - assert_eq!(Interval::from((1, 2)).merge(Interval::from((3, 4))), Some(Interval::from((1, 4)))); - assert_eq!(Interval::from((3, 4)).merge(Interval::from((1, 2))), Some(Interval::from((1, 4)))); - assert_eq!(Interval::from((1, 2)).merge(Interval::from((4, 5))), None); - assert_eq!(Interval::from((4, 5)).merge(Interval::from((1, 2))), None); - assert_eq!(Interval::from((2, 7)).merge(Interval::from((3, 6))), Some(Interval::from((2, 7)))); - assert_eq!(Interval::from((3, 6)).merge(Interval::from((2, 7))), Some(Interval::from((2, 7)))); +#[cfg(kani)] +#[kani::proof] +fn proof_align_range() { + let start: usize = kani::any(); + let end: usize = kani::any(); + align_range::(start, end); } #[test] -fn test_interval_substract() { - assert_eq!( - Interval::subtract((10, 20).into(), (15, 15).into()), - SubResult::Two((10, 14).into(), (16, 20).into()) - ); - assert_eq!( - Interval::subtract((10, 20).into(), (14, 16).into()), - SubResult::Two((10, 13).into(), (17, 20).into()) - ); - assert_eq!( - Interval::subtract((10, 20).into(), (11, 19).into()), - SubResult::Two((10, 10).into(), (20, 20).into()) - ); - assert_eq!(Interval::subtract((10, 20).into(), (10, 20).into()), SubResult::None); - assert_eq!(Interval::subtract((10, 20).into(), (10, 25).into()), SubResult::None); - assert_eq!(Interval::subtract((10, 20).into(), (5, 20).into()), SubResult::None); - - assert_eq!( - Interval::subtract((10, 20).into(), (15, 20).into()), - SubResult::One((10, 14).into()) - ); - assert_eq!( - Interval::subtract((10, 20).into(), (15, 21).into()), - SubResult::One((10, 14).into()) - ); - assert_eq!( - Interval::subtract((10, 20).into(), (20, 20).into()), - SubResult::One((10, 19).into()) - ); - - assert_eq!( - Interval::subtract((10, 20).into(), (10, 15).into()), - SubResult::One((16, 20).into()) - ); - assert_eq!(Interval::subtract((10, 20).into(), (9, 15).into()), SubResult::One((16, 20).into())); - assert_eq!( - Interval::subtract((10, 20).into(), (10, 10).into()), - SubResult::One((11, 20).into()) - ); - - assert_eq!(Interval::subtract((10, 20).into(), (21, 30).into()), SubResult::Disjoint); - assert_eq!(Interval::subtract((10, 20).into(), (0, 9).into()), SubResult::Disjoint); +fn test_align_range() { + let r = align_range::(0, 64); + assert_eq!(r.aligned_start_group, 0); + assert_eq!(r.aligned_end_group, 1); + assert_eq!(r.aligned_start, 0); + assert_eq!(r.aligned_end, 64); + assert_eq!(r.unaligned_lo_start, r.unaligned_lo_end); + assert_eq!(r.unaligned_lo_group, 0); + assert_eq!(r.unaligned_lo_offset, 0); + assert_eq!(r.unaligned_lo_length, 0); + assert_eq!(r.unaligned_hi_start, r.unaligned_hi_end); + assert_eq!(r.unaligned_hi_group, 0); + assert_eq!(r.unaligned_hi_length, 0); + assert_eq!(r.unaligned_mask_lo(), 0); + assert_eq!(r.unaligned_mask_hi(), 0); + + let r = align_range::(1, 64); + assert_eq!(r.aligned_start_group, r.aligned_end_group); + assert_eq!(r.aligned_start, r.aligned_end); + assert_eq!(r.unaligned_lo_start, 1); + assert_eq!(r.unaligned_lo_end, 64); + assert_eq!(r.unaligned_lo_group, 0); + assert_eq!(r.unaligned_lo_offset, 1); + assert_eq!(r.unaligned_lo_length, 63); + assert_eq!(r.unaligned_hi_start, r.unaligned_hi_end); + assert_eq!(r.unaligned_hi_length, 0); + assert_eq!(r.unaligned_mask_lo(), 0xffffffff_fffffffe); + assert_eq!(r.unaligned_mask_hi(), 0); + + let r = align_range::(1, 128); + assert_eq!(r.aligned_start_group, 1); + assert_eq!(r.aligned_end_group, 2); + assert_eq!(r.aligned_start, 64); + assert_eq!(r.aligned_end, 128); + assert_eq!(r.unaligned_lo_start, 1); + assert_eq!(r.unaligned_lo_end, 64); + assert_eq!(r.unaligned_lo_group, 0); + assert_eq!(r.unaligned_lo_offset, 1); + assert_eq!(r.unaligned_lo_length, 63); + assert_eq!(r.unaligned_hi_start, r.unaligned_hi_end); + assert_eq!(r.unaligned_hi_group, 0); + assert_eq!(r.unaligned_hi_length, 0); + assert_eq!(r.unaligned_mask_lo(), 0xffffffff_fffffffe); + assert_eq!(r.unaligned_mask_hi(), 0); + + let r = align_range::(0, 63); + assert_eq!(r.aligned_start_group, 0); + assert_eq!(r.aligned_end_group, 0); + assert_eq!(r.aligned_start, 0); + assert_eq!(r.aligned_end, 0); + assert_eq!(r.unaligned_lo_start, 0); + assert_eq!(r.unaligned_lo_end, 63); + assert_eq!(r.unaligned_lo_group, 0); + assert_eq!(r.unaligned_lo_offset, 0); + assert_eq!(r.unaligned_lo_length, 63); + assert_eq!(r.unaligned_hi_start, r.unaligned_hi_end); + assert_eq!(r.unaligned_hi_group, 0); + assert_eq!(r.unaligned_hi_length, 0); + assert_eq!(r.unaligned_mask_lo(), 0x7fffffff_ffffffff); + assert_eq!(r.unaligned_mask_hi(), 0); + + let r = align_range::(1, 129); + assert_eq!(r.aligned_start_group, 1); + assert_eq!(r.aligned_end_group, 2); + assert_eq!(r.aligned_start, 64); + assert_eq!(r.aligned_end, 128); + assert_eq!(r.unaligned_lo_start, 1); + assert_eq!(r.unaligned_lo_end, 64); + assert_eq!(r.unaligned_lo_group, 0); + assert_eq!(r.unaligned_lo_offset, 1); + assert_eq!(r.unaligned_lo_length, 63); + assert_eq!(r.unaligned_hi_start, 128); + assert_eq!(r.unaligned_hi_end, 129); + assert_eq!(r.unaligned_hi_group, 2); + assert_eq!(r.unaligned_hi_length, 1); + assert_eq!(r.unaligned_mask_lo(), 0xffffffff_fffffffe); + assert_eq!(r.unaligned_mask_hi(), 0x00000000_00000001); + + let r = align_range::(33, 33); + assert_eq!(r.aligned_start_group, r.aligned_end_group); + assert_eq!(r.aligned_start, r.aligned_end); + assert_eq!(r.unaligned_lo_group, 0); + assert_eq!(r.unaligned_hi_group, 0); + assert_eq!(r.unaligned_lo_length, 0); + assert_eq!(r.unaligned_hi_length, 0); + assert_eq!(r.unaligned_mask_lo(), 0); + assert_eq!(r.unaligned_mask_hi(), 0); + + let r = align_range::(66, 70); + assert_eq!(r.aligned_start_group, r.aligned_end_group); + assert_eq!(r.aligned_start, r.aligned_end); + assert_eq!(r.unaligned_lo_group, 1); + assert_eq!(r.unaligned_lo_offset, 2); + assert_eq!(r.unaligned_lo_length, 4); + assert_eq!(r.unaligned_hi_length, 0); + assert_eq!(r.unaligned_mask_lo(), 0b111100, "unexpected mask: 0b{:b}", r.unaligned_mask_lo()); + assert_eq!(r.unaligned_mask_hi(), 0); + + align_range::(4, 2); +} + +#[derive(Clone)] +#[repr(align(32))] +struct AlignedArray([M; N]); + +impl core::ops::Deref for AlignedArray { + type Target = [M; N]; + + #[inline(always)] + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl core::ops::DerefMut for AlignedArray { + #[inline(always)] + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +#[derive(Clone)] +struct BitSet +where + M: RawMask, +{ + data: Box>, } -impl Ord for Interval { - fn cmp(&self, other: &Self) -> core::cmp::Ordering { - // We deliberately ignore `max` here. - self.min.cmp(&other.min) +impl BitSet +where + M: RawMask, +{ + fn new() -> Self { + let byte_length = const { core::mem::size_of::().checked_mul(N).unwrap() }; + + // SAFETY: We properly initialize everything we allocate before calling `assume_init`. + let data = unsafe { + let mut data: Box>> = Box::new_uninit(); + core::ptr::write_bytes(data.as_mut_ptr().cast::(), 0, byte_length); + data.assume_init() + }; + + Self { data } + } + + fn clear(&mut self) { + self.data.fill(M::zero()); + } + + #[inline(always)] + fn insert_range(&mut self, start: usize, end: usize) { + let r = align_range::(start, end); + assert!( + (r.unaligned_lo_group < self.data.len()) + & (r.unaligned_hi_group < self.data.len()) + & (r.aligned_end_group <= self.data.len()) + & (r.aligned_start_group < self.data.len()) + & (r.aligned_start_group <= r.aligned_end_group) + ); + + self.data[r.aligned_start_group..r.aligned_end_group].fill(M::full()); + self.data[r.unaligned_lo_group] |= r.unaligned_mask_lo(); + self.data[r.unaligned_hi_group] |= r.unaligned_mask_hi(); + } + + #[inline(always)] + fn insert_one(&mut self, index: usize) { + let group = index / M::alignment(); + let offset = index % M::alignment(); + self.data[group] |= M::bit(offset); + } + + #[inline(always)] + fn remove_one(&mut self, index: usize) { + let group = index / M::alignment(); + let offset = index % M::alignment(); + self.data[group].bitandnot_assign(M::bit(offset)); + } + + #[inline(always)] + fn contains_one(&self, index: usize) -> bool { + let group = index / M::alignment(); + let offset = index % M::alignment(); + let mask = M::bit(offset); + !(self.data[group] & mask).is_zero() + } + + #[inline(always)] + fn remove_range(&mut self, start: usize, end: usize) { + let r = align_range::(start, end); + self.data[r.aligned_start_group..r.aligned_end_group].fill(M::zero()); + self.data[r.unaligned_lo_group].bitandnot_assign(r.unaligned_mask_lo()); + self.data[r.unaligned_hi_group].bitandnot_assign(r.unaligned_mask_hi()); + } + + #[inline(always)] + fn contains_range(&self, start: usize, end: usize) -> bool { + if start == end { + return true; + } + + let r = align_range::(start, end); + if r.aligned_end_group > self.data.len() { + return false; + } + + let mut result = M::full(); + for &value in &self.data[r.aligned_start_group..r.aligned_end_group] { + result &= value; + } + + result &= r.unaligned_mask_lo().not() | (self.data[r.unaligned_lo_group] & r.unaligned_mask_lo()); + result &= r.unaligned_mask_hi().not() | (self.data[r.unaligned_hi_group] & r.unaligned_mask_hi()); + result.is_equal(M::full()) + } + + #[cfg(test)] + fn first_non_zero(&self) -> Option { + let index = self.data.iter().position(|&value| !value.is_zero())?; + Some(index * M::alignment() + cast(self.data[index].trailing_zeros()).to_usize()) + } + + #[cfg(test)] + fn last_non_zero(&self) -> Option { + let index = self.data.iter().rev().position(|&value| !value.is_zero())?; + let index = self.data.len() - index - 1; + Some(index * M::alignment() + (M::alignment() - 1 - cast(self.data[index].leading_zeros()).to_usize())) } } -impl PartialOrd for Interval { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) +#[test] +fn test_bit_set_u64() { + let _ = env_logger::try_init(); + + let mut set = BitSet::() }>::new(); + set.insert_range(0, 1); + assert!(set.contains_range(0, 1)); + assert!(!set.contains_range(0, 2)); + assert!(!set.contains_range(1, 2)); + assert_eq!(set.data[0], (1 << 0), "unexpected: 0b{:b}", set.data[0]); + set.insert_range(2, 3); + assert_eq!(set.data[0], (1 << 0) | (1 << 2), "unexpected: 0b{:b}", set.data[0]); + set.insert_range(8, 9); + assert_eq!(set.data[0], (1 << 0) | (1 << 2) | (1 << 8), "unexpected: 0b{:b}", set.data[0]); + + { + #[allow(clippy::undocumented_unsafe_blocks)] + let raw_slice = unsafe { core::slice::from_raw_parts(set.data.as_ptr().cast::(), 256 / 8) }; + assert_eq!(raw_slice[0], 0b00000101); + assert_eq!(raw_slice[1], 0b00000001); } + + set.clear(); + set.insert_one(0); + assert_eq!(set.first_non_zero(), Some(0)); + assert_eq!(set.last_non_zero(), Some(0)); + assert!(set.contains_range(0, 1)); + assert!(!set.contains_range(0, 2)); + assert!(!set.contains_range(1, 2)); + + set.clear(); + set.insert_one(1); + assert_eq!(set.first_non_zero(), Some(1)); + assert_eq!(set.last_non_zero(), Some(1)); + assert!(!set.contains_range(0, 1)); + assert!(!set.contains_range(0, 2)); + assert!(set.contains_range(1, 2)); + assert!(!set.contains_range(1, 3)); + assert!(!set.contains_range(2, 3)); + + set.clear(); + set.insert_one(65); + assert_eq!(set.first_non_zero(), Some(65)); + assert_eq!(set.last_non_zero(), Some(65)); + assert!(set.contains_range(65, 66)); + assert!(!set.contains_range(64, 65)); + assert!(!set.contains_range(64, 66)); + assert!(!set.contains_range(64, 67)); + assert!(!set.contains_range(66, 67)); + + set.clear(); + set.insert_range(50, 51); + assert_eq!(&set.data[..], [0x00040000_00000000, 0, 0, 0]); + assert!(set.contains_range(50, 51)); + assert!(!set.contains_range(50, 52)); + assert!(!set.contains_range(49, 51)); + assert!(!set.contains_range(49, 52)); + assert!(!set.contains_range(0, 64)); + assert!(!set.contains_range(0, 256)); + + set.clear(); + assert_eq!(&set.data[..], [0, 0, 0, 0]); + assert!(set.contains_range(0, 0)); + assert!(set.contains_range(123, 123)); + assert!(!set.contains_range(0, 64)); + assert!(!set.contains_range(0, 128)); + assert!(!set.contains_range(64, 128)); + + set.insert_range(64, 128); + assert_eq!(&set.data[..], [0, u64::MAX, 0, 0]); + assert!(set.contains_range(64, 128)); + assert!(set.contains_range(65, 127)); + assert!(set.contains_range(64, 65)); + assert!(!set.contains_range(63, 65)); + assert!(set.contains_range(127, 128)); + assert!(!set.contains_range(127, 129)); + assert_eq!(set.first_non_zero(), Some(64)); + assert_eq!(set.last_non_zero(), Some(127)); + + set.remove_range(64, 128); + assert_eq!(&set.data[..], [0, 0, 0, 0]); + assert!(!set.contains_range(64, 128)); + + set.insert_range(32, 160); + assert_eq!(set.first_non_zero(), Some(32)); + assert_eq!(set.last_non_zero(), Some(159)); + assert_eq!(&set.data[..], [0xffffffff_00000000, u64::MAX, 0x00000000_ffffffff, 0]); + assert!(set.contains_range(64, 128)); + assert!(set.contains_range(63, 128)); + assert!(set.contains_range(64, 129)); + assert!(set.contains_range(63, 129)); + assert!(set.contains_range(65, 128)); + assert!(set.contains_range(64, 127)); + assert!(set.contains_range(65, 127)); + assert!(set.contains_range(100, 101)); + assert!(set.contains_range(32, 160)); + assert!(!set.contains_range(31, 160)); + assert!(!set.contains_range(32, 161)); + + set.remove_range(50, 51); + assert_eq!(&set.data[..], [0xfffbffff_00000000, u64::MAX, 0x00000000_ffffffff, 0]); + assert!(set.contains_range(51, 160)); + assert!(set.contains_range(51, 159)); + assert!(!set.contains_range(50, 160)); + assert!(!set.contains_range(50, 159)); + assert!(set.contains_range(32, 50)); + assert!(!set.contains_range(32, 51)); + assert!(set.contains_range(33, 50)); + assert!(!set.contains_range(33, 51)); } -impl PartialEq for Interval { - fn eq(&self, other: &Self) -> bool { - self.min == other.min +impl Default for BitSet +where + M: RawMask, +{ + fn default() -> Self { + Self::new() } } -impl Eq for Interval {} +#[cfg(target_arch = "x86_64")] +#[allow(clippy::undocumented_unsafe_blocks)] +mod sse { + use super::RawMask; + #[cfg(test)] + use super::{BitSet, bits}; + + include!("page_set_sse.rs"); +} -#[derive(Clone, Debug)] +#[cfg(target_arch = "x86_64")] +type PageSetGroup = picosimd::amd64::sse::si128; + +#[cfg(not(target_arch = "x86_64"))] +type PageSetGroup = u64; + +const PAGE_COUNT: usize = 2usize.pow(32) / 4096; // 1048576 +const PAGES_PER_GROUP: usize = bits::(); +const GROUP_COUNT: usize = PAGE_COUNT / PAGES_PER_GROUP; // 16386 for 64, 8192 for 128, 4096 for 256 + +#[derive(Clone, Default)] pub struct PageSet { - intervals: BTreeSet, + /// A page bitset, with one bit per every page in a partial group. + pages: BitSet() }>, + /// A group bitset, with one bit for every partially-filled group. + groups_partial: BitSet() }>, + /// A group bitset, with one bit for every filled group. + groups_filled: BitSet() }>, } impl PageSet { pub fn new() -> Self { - PageSet { - intervals: Default::default(), - } + Self::default() } - pub fn insert(&mut self, (insert_min, insert_max): (u32, u32)) { - log::trace!("Insert: ({insert_min}, {insert_max})"); - #[cfg(test)] - log::trace!(" Existing: {:?}", self.to_vec()); + #[inline] + pub fn insert(&mut self, (min, max): (u32, u32)) { + let min = cast(min).to_usize(); + let max = cast(max).to_usize(); + self.insert_exclusive((min, max + 1)); + } - let mut to_insert = Interval { - min: insert_min, - max: insert_max, - }; - let mut iter = self.intervals.range( - Interval { min: 0, max: 0 }..=Interval { - min: insert_max.saturating_add(1), - max: 0, - }, - ); - let mut to_remove = Vec::new(); - while let Some(&interval) = iter.next_back() { - log::trace!(" Check: {interval:?}"); - if let Some(new_interval) = to_insert.merge(interval) { - log::trace!(" Add: {new_interval:?}"); - log::trace!(" Remove: {interval:?}"); - to_remove.push(interval); - to_insert = new_interval; + #[inline(never)] + pub fn insert_exclusive(&mut self, (start, end): (usize, usize)) { + let r = align_range::(start, end); + self.groups_filled.insert_range(r.aligned_start_group, r.aligned_end_group); + + if r.has_unaligned_lo() { + if !self.groups_partial.contains_one(r.unaligned_lo_group) { + self.pages.data[r.unaligned_lo_group] = ::zero(); + } + + self.pages.insert_range(r.unaligned_lo_start, r.unaligned_lo_end); + if RawMask::is_equal(self.pages.data[r.unaligned_lo_group], PageSetGroup::full()) { + self.groups_partial.remove_one(r.unaligned_lo_group); + self.groups_filled.insert_one(r.unaligned_lo_group); } else { - break; + self.groups_partial.insert_one(r.unaligned_lo_group); } } - for interval in to_remove { - self.intervals.remove(&interval); + if r.has_unaligned_hi() { + if !self.groups_partial.contains_one(r.unaligned_hi_group) { + self.pages.data[r.unaligned_hi_group] = ::zero(); + } + + self.pages.insert_range(r.unaligned_hi_start, r.unaligned_hi_end); + if RawMask::is_equal(self.pages.data[r.unaligned_hi_group], PageSetGroup::full()) { + self.groups_partial.remove_one(r.unaligned_hi_group); + self.groups_filled.insert_one(r.unaligned_hi_group); + } else { + self.groups_partial.insert_one(r.unaligned_hi_group); + } } + } - self.intervals.insert(to_insert); + #[inline] + pub fn remove(&mut self, (min, max): (u32, u32)) { + let min = cast(min).to_usize(); + let max = cast(max).to_usize(); + self.remove_exclusive((min, max + 1)); } - pub fn contains(&self, (min, max): (u32, u32)) -> bool { - let mut iter = self.intervals.range(Interval { min: 0, max: 0 }..=Interval { min, max: 0 }); - if let Some(i) = iter.next_back() { - if min >= i.min && max <= i.max { - return true; + #[inline(never)] + pub fn remove_exclusive(&mut self, (start, end): (usize, usize)) { + let r = align_range::(start, end); + + self.groups_filled.remove_range(r.aligned_start_group, r.aligned_end_group); + self.groups_partial.remove_range(r.aligned_start_group, r.aligned_end_group); + self.pages.remove_range(r.unaligned_lo_start, r.unaligned_lo_end); + self.pages.remove_range(r.unaligned_hi_start, r.unaligned_hi_end); + if r.has_unaligned_lo() { + if self.groups_filled.contains_one(r.unaligned_lo_group) { + self.groups_filled.remove_one(r.unaligned_lo_group); + self.groups_partial.insert_one(r.unaligned_lo_group); + self.pages + .insert_range(r.unaligned_lo_group * PAGES_PER_GROUP, r.unaligned_lo_start); + self.pages + .insert_range(r.unaligned_lo_end, (r.unaligned_lo_group + 1) * PAGES_PER_GROUP); + } else if RawMask::is_zero(self.pages.data[r.unaligned_lo_group]) { + self.groups_partial.remove_one(r.unaligned_lo_group); } + } - if i.max < min { - return false; + if r.has_unaligned_hi() { + if self.groups_filled.contains_one(r.unaligned_hi_group) { + self.groups_filled.remove_one(r.unaligned_hi_group); + self.groups_partial.insert_one(r.unaligned_hi_group); + self.pages + .insert_range(r.unaligned_hi_end, (r.unaligned_hi_group + 1) * PAGES_PER_GROUP); + } else if RawMask::is_zero(self.pages.data[r.unaligned_hi_group]) { + self.groups_partial.remove_one(r.unaligned_hi_group); } } + } + + #[inline] + pub fn contains(&self, (min, max): (u32, u32)) -> bool { + let min = cast(min).to_usize(); + let max = cast(max).to_usize(); + self.contains_exclusive((min, max + 1)) + } + + fn contains_partial_all(&self, group: usize, mask: PageSetGroup) -> bool { + self.groups_filled.contains_one(group) + || (self.groups_partial.contains_one(group) && RawMask::is_equal(self.pages.data[group] & mask, mask)) + } + + fn contains_partial_any(&self, group: usize, mask: PageSetGroup) -> bool { + self.groups_filled.contains_one(group) + || (self.groups_partial.contains_one(group) && !RawMask::is_zero(self.pages.data[group] & mask)) + } + + #[inline(never)] + pub fn contains_exclusive(&self, (start, end): (usize, usize)) -> bool { + let r = align_range::(start, end); + + if !self.groups_filled.contains_range(r.aligned_start_group, r.aligned_end_group) { + return false; + } + + if r.has_unaligned_lo() && !self.contains_partial_all(r.unaligned_lo_group, r.unaligned_mask_lo()) { + return false; + } + + if r.has_unaligned_hi() && !self.contains_partial_all(r.unaligned_hi_group, r.unaligned_mask_hi()) { + return false; + } - false + true } + #[inline(always)] pub fn is_whole_region_empty(&self, (min, max): (u32, u32)) -> bool { - let mut iter = self.intervals.range(Interval { min: 0, max: 0 }..=Interval { min: max, max: 0 }); - while let Some(i) = iter.next_back() { - if i.max < min { - return true; - } + let min = cast(min).to_usize(); + let max = cast(max).to_usize(); + self.is_whole_region_empty_exclusive((min, max + 1)) + } - if (i.min >= min && i.max <= max) || (i.max >= min && i.min <= max) { - return false; - } + #[inline(never)] + pub fn is_whole_region_empty_exclusive(&self, (start, end): (usize, usize)) -> bool { + let r = align_range::(start, end); + + if r.aligned_start != r.aligned_end && self.groups_filled.contains_range(r.aligned_start_group, r.aligned_end_group) { + return false; + } + + if r.has_unaligned_lo() && self.contains_partial_any(r.unaligned_lo_group, r.unaligned_mask_lo()) { + return false; + } + + if r.has_unaligned_hi() && self.contains_partial_any(r.unaligned_hi_group, r.unaligned_mask_hi()) { + return false; } true } - pub fn remove(&mut self, (removed_min, removed_max): (u32, u32)) { - let mut iter = self.intervals.range( - Interval { min: 0, max: 0 }..=Interval { - min: removed_max.saturating_add(1), - max: 0, - }, - ); + pub fn clear(&mut self) { + self.groups_filled.clear(); + self.groups_partial.clear(); + } - log::trace!("Remove: ({removed_min}, {removed_max})"); - #[cfg(test)] - log::trace!(" Existing: {:?}", self.to_vec()); - - let mut to_add = Vec::new(); - let mut to_remove = Vec::new(); - while let Some(&interval) = iter.next_back() { - log::trace!(" Check: {interval:?}"); - match Interval::subtract( - interval, - Interval { - min: removed_min, - max: removed_max, - }, - ) { - SubResult::Disjoint => { - if interval.min <= removed_min { - break; + #[cfg(test)] + fn to_vec(&self) -> Vec<(u32, u32)> { + // This is horribly inefficient, but it's only for tests so that's fine. + + let mut all = Vec::new(); + if let Some(start) = self.groups_filled.first_non_zero() { + for group_index in start..=self.groups_filled.last_non_zero().unwrap() { + if self.groups_filled.contains_one(group_index) { + for page_index in group_index * PageSetGroup::alignment()..(group_index + 1) * PageSetGroup::alignment() { + let page_index = cast(page_index).assert_always_fits_in_u32(); + all.push(page_index); } - }, - SubResult::None => { - log::trace!(" Remove: {interval:?}"); - to_remove.push(interval); - } - SubResult::One(i) => { - log::trace!(" Add: {i:?}"); - log::trace!(" Remove: {interval:?}"); - to_remove.push(interval); - to_add.push(i); - } - SubResult::Two(i1, i2) => { - log::trace!(" Add: {i1:?}"); - log::trace!(" Add: {i2:?}"); - log::trace!(" Remove: {interval:?}"); - to_remove.push(interval); - to_add.push(i1); - to_add.push(i2); } } } - for interval in to_remove { - self.intervals.remove(&interval); + if let Some(start) = self.groups_partial.first_non_zero() { + for group_index in start..=self.groups_partial.last_non_zero().unwrap() { + if self.groups_partial.contains_one(group_index) { + for page_index in group_index * PageSetGroup::alignment()..(group_index + 1) * PageSetGroup::alignment() { + if self.pages.contains_one(page_index) { + let page_index = cast(page_index).assert_always_fits_in_u32(); + all.push(page_index); + } + } + } + } } - for interval in to_add { - self.intervals.insert(interval); + all.sort_unstable(); + all.dedup(); + + if all.is_empty() { + return Vec::new(); } - } - pub fn clear(&mut self) { - self.intervals.clear(); - } + let mut first = all[0]; + let mut last = all[0]; + let mut out = Vec::new(); + for index in all.into_iter().skip(1) { + if last + 1 != index { + out.push((first, last)); + first = index; + } - #[allow(dead_code)] - pub fn iter(&'_ self) -> impl ExactSizeIterator + '_ { - self.intervals.iter().map(|interval| (interval.min, interval.max)) - } + last = index; + } - #[allow(dead_code)] - fn to_vec(&self) -> Vec<(u32, u32)> { - self.iter().collect() + out.push((first, last)); + out } } diff --git a/crates/polkavm/src/page_set_sse.rs b/crates/polkavm/src/page_set_sse.rs new file mode 100644 index 00000000..d5aebf3c --- /dev/null +++ b/crates/polkavm/src/page_set_sse.rs @@ -0,0 +1,245 @@ +use picosimd::amd64::sse::si128; + +const _: () = { + if core::mem::align_of::() != 16 { + panic!("incorrect u128 alignment"); + } +}; + +static LOOKUP_ARRAY_SHR: [u128; 128] = { + let mut output = [u128::MAX; 128]; + let mut n = 0; + while n < 128 { + output[n] >>= n; + n += 1; + } + output +}; + +static LOOKUP_ARRAY_SHL: [u128; 128] = { + let mut output = [u128::MAX; 128]; + let mut n = 0; + while n < 128 { + output[n] <<= n; + n += 1; + } + output +}; + +static LOOKUP_ARRAY_BIT: [u128; 128] = { + let mut output = [1; 128]; + let mut n = 0; + while n < 128 { + output[n] <<= n; + n += 1; + } + output +}; + +#[inline(always)] +fn lookup_shr(count: usize) -> si128 { + assert!(count < 128); + unsafe { + si128::load_aligned(LOOKUP_ARRAY_SHR.as_ptr().cast::().add(count * core::mem::size_of::())) + } +} + +#[inline(always)] +fn lookup_shl(count: usize) -> si128 { + assert!(count < 128); + unsafe { + si128::load_aligned(LOOKUP_ARRAY_SHL.as_ptr().cast::().add(count * core::mem::size_of::())) + } +} + +#[inline(always)] +fn lookup_bit(position: usize) -> si128 { + assert!(position < 128); + unsafe { + si128::load_aligned(LOOKUP_ARRAY_BIT.as_ptr().cast::().add(position * core::mem::size_of::())) + } +} + +impl RawMask for si128 { + #[inline(always)] + fn zero() -> Self { + unsafe { + si128::zero() + } + } + + #[inline(always)] + fn full() -> Self { + unsafe { + si128::negative_one() + } + } + + #[inline(always)] + fn bit(position: usize) -> Self { + lookup_bit(position) + } + + #[inline(always)] + fn mask_lo(offset: usize, length: usize) -> Self { + lookup_shl(offset) & lookup_shr(Self::alignment() - (offset + length)) + } + + #[inline(always)] + fn mask_hi(length: usize) -> Self { + lookup_shr(Self::alignment() - length) + } + + #[inline(always)] + fn bitandnot_assign(&mut self, rhs: Self) { + unsafe { + *self = self.and_not(rhs); + } + } + + #[inline(always)] + fn not(self) -> Self { + unsafe { + Self::full().and_not(self) + } + } + + #[inline(always)] + fn is_equal(self, rhs: Self) -> bool { + unsafe { + self.is_equal_slow(rhs) + } + } + + #[inline(always)] + fn is_zero(self) -> bool { + unsafe { + self.is_equal_slow(si128::zero()) + } + } + + #[cfg(test)] + fn trailing_zeros(self) -> u32 { + let mut output = 0; + for x in unsafe { self.as_i64x2().to_array() } { + let x = x.trailing_zeros(); + output += x; + + if x != 64 { + break; + } + } + + output + } + + #[cfg(test)] + fn leading_zeros(self) -> u32 { + let mut output = 0; + for x in unsafe { self.as_i64x2().to_array().into_iter().rev() } { + let x = x.leading_zeros(); + output += x; + + if x != 64 { + break; + } + } + + output + } +} + +#[test] +fn test_bit_set_si128() { + let _ = env_logger::try_init(); + + let mut set = BitSet::()}>::new(); + set.insert_range(0, 1); + assert!(set.contains_range(0, 1)); + assert!(!set.contains_range(0, 2)); + assert!(!set.contains_range(1, 2)); + set.insert_range(2, 3); + set.insert_range(8, 9); + + set.clear(); + set.insert_one(0); + assert_eq!(set.first_non_zero(), Some(0)); + assert_eq!(set.last_non_zero(), Some(0)); + assert!(set.contains_range(0, 1)); + assert!(!set.contains_range(0, 2)); + assert!(!set.contains_range(1, 2)); + + set.clear(); + set.insert_one(1); + assert_eq!(set.first_non_zero(), Some(1)); + assert_eq!(set.last_non_zero(), Some(1)); + assert!(!set.contains_range(0, 1)); + assert!(!set.contains_range(0, 2)); + assert!(set.contains_range(1, 2)); + assert!(!set.contains_range(1, 3)); + assert!(!set.contains_range(2, 3)); + + set.clear(); + set.insert_one(65); + assert_eq!(set.first_non_zero(), Some(65)); + assert_eq!(set.last_non_zero(), Some(65)); + assert!(set.contains_range(65, 66)); + assert!(!set.contains_range(64, 65)); + assert!(!set.contains_range(64, 66)); + assert!(!set.contains_range(64, 67)); + assert!(!set.contains_range(66, 67)); + + set.clear(); + set.insert_range(50, 51); + assert!(set.contains_range(50, 51)); + assert!(!set.contains_range(50, 52)); + assert!(!set.contains_range(49, 51)); + assert!(!set.contains_range(49, 52)); + assert!(!set.contains_range(0, 64)); + assert!(!set.contains_range(0, 256)); + + set.clear(); + assert!(set.contains_range(0, 0)); + assert!(set.contains_range(123, 123)); + assert!(!set.contains_range(0, 64)); + assert!(!set.contains_range(0, 128)); + assert!(!set.contains_range(64, 128)); + + set.insert_range(64, 128); + assert!(set.contains_range(64, 128)); + assert!(set.contains_range(65, 127)); + assert!(set.contains_range(64, 65)); + assert!(!set.contains_range(63, 65)); + assert!(set.contains_range(127, 128)); + assert!(!set.contains_range(127, 129)); + assert_eq!(set.first_non_zero(), Some(64)); + assert_eq!(set.last_non_zero(), Some(127)); + + set.remove_range(64, 128); + assert!(!set.contains_range(64, 128)); + + set.insert_range(32, 160); + assert_eq!(set.first_non_zero(), Some(32)); + assert_eq!(set.last_non_zero(), Some(159)); + assert!(set.contains_range(64, 128)); + assert!(set.contains_range(63, 128)); + assert!(set.contains_range(64, 129)); + assert!(set.contains_range(63, 129)); + assert!(set.contains_range(65, 128)); + assert!(set.contains_range(64, 127)); + assert!(set.contains_range(65, 127)); + assert!(set.contains_range(100, 101)); + assert!(set.contains_range(32, 160)); + assert!(!set.contains_range(31, 160)); + assert!(!set.contains_range(32, 161)); + + set.remove_range(50, 51); + assert!(set.contains_range(51, 160)); + assert!(set.contains_range(51, 159)); + assert!(!set.contains_range(50, 160)); + assert!(!set.contains_range(50, 159)); + assert!(set.contains_range(32, 50)); + assert!(!set.contains_range(32, 51)); + assert!(set.contains_range(33, 50)); + assert!(!set.contains_range(33, 51)); +} diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index 7dd9d285..8c7fedd6 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -232,6 +232,7 @@ version = "0.30.0" dependencies = [ "libc", "log", + "picosimd", "polkavm-assembler", "polkavm-common", "polkavm-linux-raw", diff --git a/tools/benchtool/Cargo.lock b/tools/benchtool/Cargo.lock index 2e231b34..292c916f 100644 --- a/tools/benchtool/Cargo.lock +++ b/tools/benchtool/Cargo.lock @@ -1778,6 +1778,7 @@ version = "0.30.0" dependencies = [ "libc", "log", + "picosimd", "polkavm-assembler", "polkavm-common", "polkavm-linux-raw", From 0c93f979ef8c210fc45af8fbb5e66ccc432d10b5 Mon Sep 17 00:00:00 2001 From: Jan Bujak Date: Mon, 10 Nov 2025 11:03:51 +0000 Subject: [PATCH 16/21] Add `#[track_caller]` to `expect_segfault` in tests --- crates/polkavm/src/tests.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/polkavm/src/tests.rs b/crates/polkavm/src/tests.rs index a00a99c7..c0d122a8 100644 --- a/crates/polkavm/src/tests.rs +++ b/crates/polkavm/src/tests.rs @@ -818,6 +818,7 @@ fn zero_memory(engine_config: Config) { assert_eq!(instance.reg(A0), 0x12340000); } +#[track_caller] fn expect_segfault(interrupt: InterruptKind) -> Segfault { match interrupt { InterruptKind::Segfault(segfault) => segfault, From badaa678573ac22ef65a23b5327dbce1f65a971e Mon Sep 17 00:00:00 2001 From: Jan Bujak Date: Mon, 10 Nov 2025 11:03:51 +0000 Subject: [PATCH 17/21] Handle `offset == u32::MAX` case for the bounded instruction iterator --- crates/polkavm-common/src/program.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/polkavm-common/src/program.rs b/crates/polkavm-common/src/program.rs index b605a83b..b4c1f057 100644 --- a/crates/polkavm-common/src/program.rs +++ b/crates/polkavm-common/src/program.rs @@ -3956,7 +3956,11 @@ where (offset, None) } else if is_bounded { is_done = true; - (core::cmp::min(offset + 1, code.len() as u32), Some(offset)) + if offset == u32::MAX { + (u32::MAX, None) + } else { + (core::cmp::min(offset + 1, code.len() as u32), Some(offset)) + } } else { let next_offset = find_next_offset_unbounded(bitmask, code.len() as u32, offset); debug_assert!( From 795de024109ec6dc5663f8e2d63a917822fd083a Mon Sep 17 00:00:00 2001 From: Jan Bujak Date: Mon, 10 Nov 2025 11:03:52 +0000 Subject: [PATCH 18/21] Fix a `cfg` typo in tests --- crates/polkavm-disassembler/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/polkavm-disassembler/src/lib.rs b/crates/polkavm-disassembler/src/lib.rs index 95412954..ad491cb2 100644 --- a/crates/polkavm-disassembler/src/lib.rs +++ b/crates/polkavm-disassembler/src/lib.rs @@ -525,9 +525,9 @@ mod tests { for format in [ DisassemblyFormat::Guest, DisassemblyFormat::DiffFriendly, - #[cfg(target_arg = "x86_84")] + #[cfg(all(target_arch = "x86_64", target_os = "linux"))] DisassemblyFormat::GuestAndNative, - #[cfg(target_arg = "x86_84")] + #[cfg(all(target_arch = "x86_64", target_os = "linux"))] DisassemblyFormat::Native, ] { assert!(!disassemble_with_gas(blob, format).is_empty()); From 8f377158e22458132b82ffb88cc830bac04675f1 Mon Sep 17 00:00:00 2001 From: Jan Bujak Date: Mon, 10 Nov 2025 11:03:52 +0000 Subject: [PATCH 19/21] Rewrite memory management --- crates/polkavm/src/api.rs | 137 +++++++++++---- crates/polkavm/src/interpreter.rs | 60 ++++++- crates/polkavm/src/lib.rs | 2 +- crates/polkavm/src/linker.rs | 38 +++-- crates/polkavm/src/page_set.rs | 8 +- crates/polkavm/src/sandbox.rs | 8 +- crates/polkavm/src/sandbox/generic.rs | 173 +++++++++++++------ crates/polkavm/src/sandbox/linux.rs | 213 ++++++++++++++++++------ crates/polkavm/src/tests.rs | 230 +++++++++++++++++++++----- tools/spectool/src/main.rs | 30 +++- 10 files changed, 695 insertions(+), 204 deletions(-) diff --git a/crates/polkavm/src/api.rs b/crates/polkavm/src/api.rs index 60d35ff4..31359f39 100644 --- a/crates/polkavm/src/api.rs +++ b/crates/polkavm/src/api.rs @@ -27,6 +27,12 @@ use crate::{Gas, ProgramCounter}; #[cfg(feature = "module-cache")] use crate::module_cache::{ModuleCache, ModuleKey}; +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub enum MemoryProtection { + Read, + ReadWrite, +} + if_compiler_is_supported! { { use crate::sandbox::{Sandbox, SandboxInstance}; @@ -1357,10 +1363,10 @@ impl RawInstance { .into_result("failed to reset the instance's memory")) } - /// Returns whether a given chunk of memory is accessible through [`read_memory_into`](Self::read_memory_into)/[`write_memory`](Self::write_memory). + /// Returns whether a given chunk of memory is accessible. /// /// If `size` is zero then this will always return `true`. - pub fn is_memory_accessible(&self, address: u32, size: u32, is_writable: bool) -> bool { + pub fn is_memory_accessible(&self, address: u32, size: u32, minimum_protection: MemoryProtection) -> bool { if size == 0 { return true; } @@ -1369,7 +1375,11 @@ impl RawInstance { return false; } - let upper_limit = if is_writable { self.get_write_upper_limit() } else { 0x100000000 }; + let upper_limit = match minimum_protection { + MemoryProtection::Read => 0x100000000, + MemoryProtection::ReadWrite => self.get_write_upper_limit(), + }; + if u64::from(address) + cast(size).to_u64() > upper_limit { return false; } @@ -1398,17 +1408,23 @@ impl RawInstance { return true; } - if !is_writable && is_within(map.ro_data_range(), address, size) { + if matches!(minimum_protection, MemoryProtection::Read) && is_within(map.ro_data_range(), address, size) { return true; } false } else { - access_backend!(self.backend, |backend| backend.is_memory_accessible(address, size, is_writable)) + access_backend!(self.backend, |backend| backend.is_memory_accessible( + address, + size, + minimum_protection + )) } } /// Reads the VM's memory. + /// + /// The whole memory region must be readable. pub fn read_memory_into<'slice, B>(&self, address: u32, buffer: &'slice mut B) -> Result<&'slice mut [u8], MemoryAccessError> where B: ?Sized + AsUninitSliceMut, @@ -1459,16 +1475,15 @@ impl RawInstance { } } - #[cfg(debug_assertions)] - { - let is_accessible = self.is_memory_accessible(address, cast(length).assert_always_fits_in_u32(), false); - if is_accessible != result.is_ok() { + if cfg!(debug_assertions) { + let is_inaccessible = !self.is_memory_accessible(address, cast(length).assert_always_fits_in_u32(), MemoryProtection::Read); + if is_inaccessible != matches!(result, Err(MemoryAccessError::OutOfRangeAccess { .. })) { panic!( - "'read_memory_into' doesn't match with 'is_memory_accessible' for 0x{:x}-0x{:x} (read_memory_into = {}, is_memory_accessible = {})", + "'read_memory_into' doesn't match with 'is_memory_accessible' for 0x{:x}-0x{:x} (read_memory_into = {:?}, is_memory_accessible = {})", address, cast(address).to_usize() + length, - result.is_ok(), - is_accessible, + result.map(|_| ()), + !is_inaccessible, ); } } @@ -1487,8 +1502,7 @@ impl RawInstance { /// Writes into the VM's memory. /// - /// When dynamic paging is enabled calling this can be used to resolve a segfault. It can also - /// be used to preemptively initialize pages for which no segfault is currently triggered. + /// The whole memory region must be writable. pub fn write_memory(&mut self, address: u32, data: &[u8]) -> Result<(), MemoryAccessError> { if data.is_empty() { return Ok(()); @@ -1519,16 +1533,16 @@ impl RawInstance { } } - #[cfg(debug_assertions)] - { - let is_accessible = self.is_memory_accessible(address, cast(data.len()).assert_always_fits_in_u32(), true); - if is_accessible != result.is_ok() { + if cfg!(debug_assertions) { + let is_inaccessible = + !self.is_memory_accessible(address, cast(data.len()).assert_always_fits_in_u32(), MemoryProtection::ReadWrite); + if is_inaccessible != matches!(result, Err(MemoryAccessError::OutOfRangeAccess { .. })) { panic!( - "'write_memory' doesn't match with 'is_memory_accessible' for 0x{:x}-0x{:x} (write_memory = {}, is_memory_accessible = {})", + "'write_memory' doesn't match with 'is_memory_accessible' for 0x{:x}-0x{:x} (write_memory = {:?}, is_memory_accessible = {})", address, cast(address).to_usize() + data.len(), - result.is_ok(), - is_accessible, + result, + !is_inaccessible, ); } } @@ -1537,6 +1551,8 @@ impl RawInstance { } /// Reads the VM's memory. + /// + /// The whole memory region must be readable. pub fn read_memory(&self, address: u32, length: u32) -> Result, MemoryAccessError> { let mut buffer = Vec::new(); buffer.reserve_exact(cast(length).to_usize()); @@ -1626,14 +1642,51 @@ impl RawInstance { self.write_memory(address, &[value]) } + /// Fills the given memory region with zeros and changes memory protection flags. Similar to [`RawInstance::zero_memory`], but can only be called when dynamic paging is enabled. + /// + /// `address` must be a multiple of the page size. The value of `length` will be rounded up to the nearest multiple of the page size. + /// If `length` is zero then this call has no effect. + /// + /// Can be used to resolve a segfault. It can also be used to preemptively initialize pages for which no segfault is currently triggered. + pub fn zero_memory_with_memory_protection( + &mut self, + address: u32, + length: u32, + memory_protection: MemoryProtection, + ) -> Result<(), MemoryAccessError> { + if !self.module.is_dynamic_paging() { + return Err(MemoryAccessError::Error( + "'zero_memory_with_memory_protection' is only possible on modules with dynamic paging".into(), + )); + } + + if length == 0 { + return Ok(()); + } + + if !self.module.is_multiple_of_page_size(address) { + return Err(MemoryAccessError::Error("address not a multiple of page size".into())); + } + + self.zero_memory_impl(address, length, Some(memory_protection)) + } + /// Fills the given memory region with zeros. /// + /// The whole memory region must be writable. + /// /// `address` must be greater or equal to 0x10000 and `address + length` cannot be greater than 0x100000000. /// If `length` is zero then this call has no effect and will always succeed. - /// - /// When dynamic paging is enabled calling this can be used to resolve a segfault. It can also - /// be used to preemptively initialize pages for which no segfault is currently triggered. pub fn zero_memory(&mut self, address: u32, length: u32) -> Result<(), MemoryAccessError> { + self.zero_memory_impl(address, length, None) + } + + fn zero_memory_impl( + &mut self, + address: u32, + length: u32, + memory_protection: Option, + ) -> Result<(), MemoryAccessError> { if length == 0 { return Ok(()); } @@ -1652,9 +1705,15 @@ impl RawInstance { }); } - let result = access_backend!(self.backend, |mut backend| backend.zero_memory(address, length)); + let length = if memory_protection.is_none() { + length + } else { + self.module().round_to_page_size_up(length) + }; + + let result = access_backend!(self.backend, |mut backend| backend.zero_memory(address, length, memory_protection)); if let Some(ref mut crosscheck) = self.crosscheck_instance { - let expected_result = crosscheck.zero_memory(address, length); + let expected_result = crosscheck.zero_memory(address, length, memory_protection); let expected_success = expected_result.is_ok(); let success = result.is_ok(); if success != expected_success { @@ -1663,6 +1722,19 @@ impl RawInstance { } } + if cfg!(debug_assertions) && memory_protection.is_none() { + let is_inaccessible = !self.is_memory_accessible(address, length, MemoryProtection::ReadWrite); + if is_inaccessible != matches!(result, Err(MemoryAccessError::OutOfRangeAccess { .. })) { + panic!( + "'zero_memory' doesn't match with 'is_memory_accessible' for 0x{:x}-0x{:x} (zero_memory = {:?}, is_memory_accessible = {})", + address, + cast(address).to_usize() + cast(length).to_usize(), + result, + !is_inaccessible, + ); + } + } + result } @@ -1670,17 +1742,17 @@ impl RawInstance { /// /// Is only supported when dynamic paging is enabled. pub fn protect_memory(&mut self, address: u32, length: u32) -> Result<(), MemoryAccessError> { - self.change_memory_protection(address, length, true) + self.change_memory_protection(address, length, MemoryProtection::Read) } /// Removes read-only protection from a given memory region. /// /// Is only supported when dynamic paging is enabled. pub fn unprotect_memory(&mut self, address: u32, length: u32) -> Result<(), MemoryAccessError> { - self.change_memory_protection(address, length, false) + self.change_memory_protection(address, length, MemoryProtection::ReadWrite) } - fn change_memory_protection(&mut self, address: u32, length: u32, make_read_only: bool) -> Result<(), MemoryAccessError> { + fn change_memory_protection(&mut self, address: u32, length: u32, protection: MemoryProtection) -> Result<(), MemoryAccessError> { if !self.module.is_dynamic_paging() { return Err(MemoryAccessError::Error( "protecting/unprotecting memory is only possible on modules with dynamic paging".into(), @@ -1705,11 +1777,8 @@ impl RawInstance { }); } - access_backend!(self.backend, |mut backend| backend.change_memory_protection( - address, - length, - make_read_only - )) + access_backend!(self.backend, |mut backend| backend + .change_memory_protection(address, length, protection)) } /// Frees the given page(s). diff --git a/crates/polkavm/src/interpreter.rs b/crates/polkavm/src/interpreter.rs index 068940a2..a7c6025c 100644 --- a/crates/polkavm/src/interpreter.rs +++ b/crates/polkavm/src/interpreter.rs @@ -1,7 +1,7 @@ #![allow(unknown_lints)] // Because of `non_local_definitions` on older rustc versions. #![allow(non_local_definitions)] #![deny(clippy::as_conversions)] -use crate::api::{MemoryAccessError, Module, RegValue, SetCacheSizeLimitArgs}; +use crate::api::{MemoryAccessError, MemoryProtection, Module, RegValue, SetCacheSizeLimitArgs}; use crate::error::Error; use crate::gas::{CostModelKind, GasVisitor}; use crate::utils::{FlatMap, InterruptKind, Segfault}; @@ -539,15 +539,24 @@ impl InterpretedInstance { None } - pub fn is_memory_accessible(&self, address: u32, size: u32, _is_writable: bool) -> bool { + pub fn is_memory_accessible(&self, address: u32, size: u32, minimum_protection: MemoryProtection) -> bool { assert!(self.module.is_dynamic_paging()); // TODO: This is very slow. let result = each_page(&self.module, address, size, |page_address, _, _, _| { - if !self.dynamic_memory.pages.contains_key(&page_address) { - Err(()) + if let Some(page) = self.dynamic_memory.pages.get(&page_address) { + match minimum_protection { + MemoryProtection::ReadWrite => { + if page.is_read_only { + Err(()) + } else { + Ok(()) + } + } + MemoryProtection::Read => Ok(()), + } } else { - Ok(()) + Err(()) } }); @@ -617,6 +626,13 @@ impl InterpretedInstance { slice.copy_from_slice(data); } else { + if !self.is_memory_accessible(address, cast(data.len()).assert_always_fits_in_u32(), MemoryProtection::ReadWrite) { + return Err(MemoryAccessError::OutOfRangeAccess { + address, + length: cast(data.len()).to_u64(), + }); + } + let dynamic_memory = &mut self.dynamic_memory; let page_size = self.module.memory_map().page_size(); each_page::<()>( @@ -635,8 +651,9 @@ impl InterpretedInstance { Ok(()) } - pub fn zero_memory(&mut self, address: u32, length: u32) -> Result<(), MemoryAccessError> { + pub fn zero_memory(&mut self, address: u32, length: u32, memory_protection: Option) -> Result<(), MemoryAccessError> { if !self.module.is_dynamic_paging() { + debug_assert!(memory_protection.is_none()); let Some(slice) = self.basic_memory.get_memory_slice_mut::(&self.module, address, length) else { return Err(MemoryAccessError::OutOfRangeAccess { address, @@ -646,6 +663,21 @@ impl InterpretedInstance { slice.fill(0); } else { + if memory_protection.is_some() { + debug_assert!(self.module.is_multiple_of_page_size(address)); + debug_assert!(self.module.is_multiple_of_page_size(length)); + } else if !self.is_memory_accessible(address, length, MemoryProtection::ReadWrite) { + return Err(MemoryAccessError::OutOfRangeAccess { + address, + length: u64::from(length), + }); + } + + let is_read_only = memory_protection.map(|prot| match prot { + MemoryProtection::Read => true, + MemoryProtection::ReadWrite => false, + }); + let dynamic_memory = &mut self.dynamic_memory; let page_size = self.module.memory_map().page_size(); each_page::<()>( @@ -656,10 +688,17 @@ impl InterpretedInstance { Entry::Occupied(mut entry) => { let page = entry.get_mut(); page[page_offset..page_offset + length].fill(0); + if let Some(is_read_only) = is_read_only { + page.is_read_only = is_read_only; + } Ok(()) } Entry::Vacant(entry) => { - entry.insert(Page::empty(page_size)); + let mut page = Page::empty(page_size); + if let Some(is_read_only) = is_read_only { + page.is_read_only = is_read_only; + } + entry.insert(page); Ok(()) } }, @@ -670,7 +709,7 @@ impl InterpretedInstance { Ok(()) } - pub fn change_memory_protection(&mut self, address: u32, length: u32, make_read_only: bool) -> Result<(), MemoryAccessError> { + pub fn change_memory_protection(&mut self, address: u32, length: u32, protection: MemoryProtection) -> Result<(), MemoryAccessError> { assert!(self.module.is_dynamic_paging()); each_page( @@ -679,7 +718,10 @@ impl InterpretedInstance { length, |page_address, page_offset, _buffer_offset, length| { if let Some(page) = self.dynamic_memory.pages.get_mut(&page_address) { - page.is_read_only = make_read_only; + page.is_read_only = match protection { + MemoryProtection::Read => true, + MemoryProtection::ReadWrite => false, + }; Ok(()) } else { Err(MemoryAccessError::OutOfRangeAccess { diff --git a/crates/polkavm/src/lib.rs b/crates/polkavm/src/lib.rs index b5050131..25db670d 100644 --- a/crates/polkavm/src/lib.rs +++ b/crates/polkavm/src/lib.rs @@ -139,7 +139,7 @@ pub mod program { pub type Gas = i64; -pub use crate::api::{Engine, MemoryAccessError, Module, RawInstance, RegValue, SetCacheSizeLimitArgs}; +pub use crate::api::{Engine, MemoryAccessError, MemoryProtection, Module, RawInstance, RegValue, SetCacheSizeLimitArgs}; pub use crate::config::{BackendKind, Config, CustomCodegen, GasMeteringKind, ModuleConfig, SandboxKind}; pub use crate::error::Error; pub use crate::gas::{Cost, CostModel, CostModelKind, CostModelRef}; diff --git a/crates/polkavm/src/linker.rs b/crates/polkavm/src/linker.rs index d3b4cb61..909f6abd 100644 --- a/crates/polkavm/src/linker.rs +++ b/crates/polkavm/src/linker.rs @@ -1,4 +1,4 @@ -use crate::api::RegValue; +use crate::api::{MemoryProtection, RegValue}; use crate::error::bail; use crate::program::ProgramSymbol; use crate::{Error, InterruptKind, Module, ProgramCounter, RawInstance, Reg}; @@ -860,7 +860,7 @@ impl Instance { && segfault.page_address + segfault.page_size <= module.memory_map().stack_address_high() { self.instance - .zero_memory(segfault.page_address, segfault.page_size) + .zero_memory_with_memory_protection(segfault.page_address, segfault.page_size, MemoryProtection::ReadWrite) .map_err(|error| { CallError::Error(Error::from_display(format!( "failed to zero memory when handling a segfault at 0x{:x}: {error}", @@ -872,13 +872,29 @@ impl Instance { } macro_rules! handle { - ($range:ident, $data:ident) => {{ + ($range:ident, $data:ident, $protection:ident) => {{ if segfault.page_address >= module.memory_map().$range().start && segfault.page_address + segfault.page_size <= module.memory_map().$range().end { let data_offset = (segfault.page_address - module.memory_map().$range().start) as usize; let data = module.blob().$data(); - if let Some(chunk_length) = data.len().checked_sub(data_offset) { + let chunk_length = data.len().checked_sub(data_offset); + let initial_protection = if chunk_length.is_some() { + MemoryProtection::ReadWrite + } else { + MemoryProtection::$protection + }; + + self.instance + .zero_memory_with_memory_protection(segfault.page_address, segfault.page_size, initial_protection) + .map_err(|error| { + CallError::Error(Error::from_display(format!( + "failed to zero memory when handling a segfault at 0x{:x}: {error}", + segfault.page_address + ))) + })?; + + if let Some(chunk_length) = chunk_length { let chunk_length = core::cmp::min(chunk_length, segfault.page_size as usize); self.instance .write_memory(segfault.page_address, &data[data_offset..data_offset + chunk_length]) @@ -888,24 +904,26 @@ impl Instance { segfault.page_address ))) })?; - } else { + }; + + if MemoryProtection::$protection == MemoryProtection::Read && initial_protection != MemoryProtection::Read { self.instance - .zero_memory(segfault.page_address, segfault.page_size) + .protect_memory(segfault.page_address, segfault.page_size) .map_err(|error| { CallError::Error(Error::from_display(format!( - "failed to zero memory when handling a segfault at 0x{:x}: {error}", + "failed to protect memory when handling a segfault at 0x{:x}: {error}", segfault.page_address ))) })?; - }; + } continue; } }}; } - handle!(ro_data_range, ro_data); - handle!(rw_data_range, rw_data); + handle!(ro_data_range, ro_data, Read); + handle!(rw_data_range, rw_data, ReadWrite); log::debug!("Unexpected segfault: 0x{:x}", segfault.page_address); break Err(CallError::Trap); diff --git a/crates/polkavm/src/page_set.rs b/crates/polkavm/src/page_set.rs index 3ba34082..ad08df5d 100644 --- a/crates/polkavm/src/page_set.rs +++ b/crates/polkavm/src/page_set.rs @@ -619,7 +619,7 @@ where mod sse { use super::RawMask; #[cfg(test)] - use super::{BitSet, bits}; + use super::{bits, BitSet}; include!("page_set_sse.rs"); } @@ -747,6 +747,12 @@ impl PageSet { || (self.groups_partial.contains_one(group) && !RawMask::is_zero(self.pages.data[group] & mask)) } + #[allow(dead_code)] + pub fn contains_one(&self, entry: u32) -> bool { + // TODO: Add a more efficient implementation. + self.contains((entry, entry)) + } + #[inline(never)] pub fn contains_exclusive(&self, (start, end): (usize, usize)) -> bool { let r = align_range::(start, end); diff --git a/crates/polkavm/src/sandbox.rs b/crates/polkavm/src/sandbox.rs index e57456db..e588f24d 100644 --- a/crates/polkavm/src/sandbox.rs +++ b/crates/polkavm/src/sandbox.rs @@ -4,7 +4,7 @@ use core::sync::atomic::{AtomicUsize, Ordering}; use polkavm_common::zygote::AddressTable; -use crate::api::{EngineState, Module}; +use crate::api::{EngineState, MemoryProtection, Module}; use crate::compiler::CompiledModule; use crate::config::{Config, SandboxKind}; use crate::error::Error; @@ -125,12 +125,12 @@ pub(crate) trait Sandbox: Sized { fn set_next_program_counter(&mut self, pc: ProgramCounter); fn accessible_aux_size(&self) -> u32; fn set_accessible_aux_size(&mut self, size: u32) -> Result<(), Self::Error>; - fn is_memory_accessible(&self, address: u32, size: u32, is_writable: bool) -> bool; + fn is_memory_accessible(&self, address: u32, size: u32, minimum_protection: MemoryProtection) -> bool; fn reset_memory(&mut self) -> Result<(), Self::Error>; fn read_memory_into<'slice>(&self, address: u32, slice: &'slice mut [MaybeUninit]) -> Result<&'slice mut [u8], MemoryAccessError>; fn write_memory(&mut self, address: u32, data: &[u8]) -> Result<(), MemoryAccessError>; - fn zero_memory(&mut self, address: u32, length: u32) -> Result<(), MemoryAccessError>; - fn change_memory_protection(&mut self, address: u32, length: u32, make_read_only: bool) -> Result<(), MemoryAccessError>; + fn zero_memory(&mut self, address: u32, length: u32, memory_protection: Option) -> Result<(), MemoryAccessError>; + fn change_memory_protection(&mut self, address: u32, length: u32, protection: MemoryProtection) -> Result<(), MemoryAccessError>; fn free_pages(&mut self, address: u32, length: u32) -> Result<(), Self::Error>; fn heap_size(&self) -> u32; fn sbrk(&mut self, size: u32) -> Result, Self::Error>; diff --git a/crates/polkavm/src/sandbox/generic.rs b/crates/polkavm/src/sandbox/generic.rs index 1e702af4..a372997d 100644 --- a/crates/polkavm/src/sandbox/generic.rs +++ b/crates/polkavm/src/sandbox/generic.rs @@ -18,7 +18,7 @@ use core::sync::atomic::{AtomicI64, AtomicU32, AtomicU64, AtomicUsize, Ordering} use std::sync::Arc; use super::{get_native_page_size, OffsetTable, SandboxInit, SandboxKind, WorkerCache, WorkerCacheKind}; -use crate::api::{CompiledModuleKind, MemoryAccessError, Module}; +use crate::api::{CompiledModuleKind, MemoryAccessError, MemoryProtection, Module}; use crate::compiler::CompiledModule; use crate::config::Config; use crate::config::GasMeteringKind; @@ -869,7 +869,8 @@ pub struct Sandbox { next_program_counter: Option, next_program_counter_changed: bool, - page_set: PageSet, + page_set_present: PageSet, + page_set_writable: PageSet, dynamic_paging_enabled: bool, aux_data_address: u32, @@ -989,7 +990,13 @@ impl Sandbox { let module = self.module.as_ref().unwrap(); let page_start = module.address_to_page(module.round_to_page_size_down(address)); let page_end = module.address_to_page(module.round_to_page_size_down(address + length - 1)); - if self.page_set.contains((page_start, page_end)) { + let page_set = if is_writable { + &self.page_set_writable + } else { + &self.page_set_present + }; + + if page_set.contains((page_start, page_end)) { return Ok(()); } else { return Err(()); @@ -1123,7 +1130,7 @@ impl Sandbox { let segfault_kind = if self.dynamic_paging_enabled && page_address >= 0x10000 { let module = self.module.as_ref().unwrap(); let page_start = module.address_to_page(module.round_to_page_size_down(page_address)); - Some(self.page_set.contains((page_start, page_start))) + Some(self.page_set_present.contains_one(page_start) && !self.page_set_writable.contains_one(page_start)) } else { None }; @@ -1167,6 +1174,39 @@ impl Sandbox { Ok(()) } + + fn mprotect_guest_memory(&mut self, address: u32, length: u32, protection: MemoryProtection) -> Result<(), MemoryAccessError> { + assert!(self.dynamic_paging_enabled); + + self.memory + .mprotect( + self.guest_memory_offset + cast(address).to_usize(), + cast(length).to_usize(), + match protection { + MemoryProtection::Read => PROT_READ, + MemoryProtection::ReadWrite => PROT_READ | PROT_WRITE, + }, + ) + .map_err(|e| MemoryAccessError::Error(e.into())) + } + + fn madvise_remove(&mut self, address: u32, length: u32) -> Result<(), Error> { + assert!(self.dynamic_paging_enabled); + + self.memory.madvise( + self.guest_memory_offset + cast(address).to_usize(), + cast(length).to_usize(), + MADV_DONTNEED, + )?; + + self.memory.madvise( + self.guest_memory_offset + cast(address).to_usize(), + cast(length).to_usize(), + MADV_FREE, + )?; + + Ok(()) + } } impl super::SandboxAddressSpace for Mmap { @@ -1391,7 +1431,8 @@ impl super::Sandbox for Sandbox { is_program_counter_valid: false, next_program_counter: None, next_program_counter_changed: true, - page_set: PageSet::new(), + page_set_present: PageSet::new(), + page_set_writable: PageSet::new(), dynamic_paging_enabled: false, aux_data_address: 0, aux_data_full_length: 0, @@ -1492,7 +1533,6 @@ impl super::Sandbox for Sandbox { } self.module = None; - self.page_set.clear(); Ok(()) } @@ -1705,9 +1745,17 @@ impl super::Sandbox for Sandbox { Ok(()) } - fn is_memory_accessible(&self, address: u32, size: u32, is_writable: bool) -> bool { + fn is_memory_accessible(&self, address: u32, size: u32, minimum_protection: MemoryProtection) -> bool { assert!(self.dynamic_paging_enabled); - self.bound_check_access(address, size, is_writable).is_ok() + self.bound_check_access( + address, + size, + match minimum_protection { + MemoryProtection::Read => false, + MemoryProtection::ReadWrite => true, + }, + ) + .is_ok() } fn reset_memory(&mut self) -> Result<(), Error> { @@ -1738,7 +1786,7 @@ impl super::Sandbox for Sandbox { let module = self.module.as_ref().unwrap(); let page_start = module.address_to_page(module.round_to_page_size_down(address)); let page_end = module.address_to_page(module.round_to_page_size_down(address + slice.len() as u32 - 1)); - if !self.page_set.contains((page_start, page_end)) { + if !self.page_set_present.contains((page_start, page_end)) { return Err(MemoryAccessError::OutOfRangeAccess { address, length: cast(slice.len()).to_u64(), @@ -1776,20 +1824,13 @@ impl super::Sandbox for Sandbox { let module = self.module.as_ref().unwrap(); let page_start = module.address_to_page(module.round_to_page_size_down(address)); let page_end = module.address_to_page(module.round_to_page_size_down(address + data.len() as u32 - 1)); - let page_size = get_native_page_size() as u32; - let page_count = page_end - page_start + 1; - if !self.page_set.contains((page_start, page_end)) { - self.memory - .mprotect( - self.guest_memory_offset + (page_start * page_size) as usize, - (page_count * page_size) as usize, - PROT_READ | PROT_WRITE, - ) - .map_err(|e| MemoryAccessError::Error(e.into()))?; + if !self.page_set_writable.contains((page_start, page_end)) { + return Err(MemoryAccessError::OutOfRangeAccess { + address, + length: cast(data.len()).to_u64(), + }); } - - self.page_set.insert((page_start, page_end)); } let Some(slice) = self.get_memory_slice_mut(address, data.len() as u32) else { @@ -1803,7 +1844,7 @@ impl super::Sandbox for Sandbox { Ok(()) } - fn zero_memory(&mut self, address: u32, length: u32) -> Result<(), MemoryAccessError> { + fn zero_memory(&mut self, address: u32, length: u32, memory_protection: Option) -> Result<(), MemoryAccessError> { log::trace!("Zeroing memory: 0x{:x}-0x{:x} ({} bytes)", address, address + length, length); if length == 0 { @@ -1816,22 +1857,58 @@ impl super::Sandbox for Sandbox { if self.dynamic_paging_enabled { let module = self.module.as_ref().unwrap(); + + if memory_protection.is_some() { + debug_assert!(module.is_multiple_of_page_size(address)); + debug_assert!(module.is_multiple_of_page_size(length)); + } + let page_start = module.address_to_page(module.round_to_page_size_down(address)); let page_end = module.address_to_page(module.round_to_page_size_down(address + length - 1)); - let page_size = get_native_page_size() as u32; - let page_count = page_end - page_start + 1; - if !self.page_set.contains((page_start, page_end)) { - self.memory - .mprotect( - self.guest_memory_offset + (page_start * page_size) as usize, - (page_count * page_size) as usize, - PROT_READ | PROT_WRITE, - ) - .map_err(|e| MemoryAccessError::Error(e.into()))?; - } + match memory_protection { + None => { + if !self.page_set_writable.contains((page_start, page_end)) { + return Err(MemoryAccessError::OutOfRangeAccess { + address, + length: u64::from(length), + }); + } + } + Some(MemoryProtection::Read) => { + if !self.page_set_present.is_whole_region_empty((page_start, page_end)) { + self.madvise_remove(address, length) + .map_err(|error| MemoryAccessError::Error(error.into()))?; + self.page_set_writable.remove((page_start, page_end)); + } + + if let Err(error) = self.mprotect_guest_memory(address, length, MemoryProtection::Read) { + self.page_set_present.remove((page_start, page_end)); + return Err(error); + } - self.page_set.insert((page_start, page_end)); + self.page_set_present.insert((page_start, page_end)); + return Ok(()); + } + Some(MemoryProtection::ReadWrite) => { + if !self.page_set_present.is_whole_region_empty((page_start, page_end)) { + self.madvise_remove(address, length) + .map_err(|error| MemoryAccessError::Error(error.into()))?; + } + + if let Err(error) = self.mprotect_guest_memory(address, length, MemoryProtection::ReadWrite) { + self.page_set_present.remove((page_start, page_end)); + self.page_set_writable.remove((page_start, page_end)); + return Err(error); + } + + self.page_set_present.insert((page_start, page_end)); + self.page_set_writable.insert((page_start, page_end)); + return Ok(()); + } + } + } else { + debug_assert!(memory_protection.is_none()); } let Some(slice) = self.get_memory_slice_mut(address, length as u32) else { @@ -1845,12 +1922,15 @@ impl super::Sandbox for Sandbox { Ok(()) } - fn change_memory_protection(&mut self, address: u32, length: u32, make_read_only: bool) -> Result<(), MemoryAccessError> { + fn change_memory_protection(&mut self, address: u32, length: u32, protection: MemoryProtection) -> Result<(), MemoryAccessError> { assert!(self.dynamic_paging_enabled); log::trace!( "{} memory: 0x{:x}-0x{:x} ({} bytes)", - if make_read_only { "Protecting" } else { "Unprotecting" }, + match protection { + MemoryProtection::Read => "Protecting", + MemoryProtection::ReadWrite => "Unprotecting", + }, address, address as usize + length as usize, length @@ -1859,20 +1939,19 @@ impl super::Sandbox for Sandbox { let module = self.module.as_ref().unwrap(); let page_start = module.address_to_page(module.round_to_page_size_down(address)); let page_end = module.address_to_page(module.round_to_page_size_down(address + length - 1)); - if !self.page_set.contains((page_start, page_end)) { + if !self.page_set_present.contains((page_start, page_end)) { return Err(MemoryAccessError::OutOfRangeAccess { address, length: u64::from(length), }); } - self.memory - .mprotect( - self.guest_memory_offset + address as usize, - length as usize, - if make_read_only { PROT_READ } else { PROT_READ | PROT_WRITE }, - ) - .map_err(|e| MemoryAccessError::Error(e.into()))?; + self.mprotect_guest_memory(address, length, protection)?; + + match protection { + MemoryProtection::Read => self.page_set_writable.remove((page_start, page_end)), + MemoryProtection::ReadWrite => self.page_set_writable.insert((page_start, page_end)), + } Ok(()) } @@ -1887,7 +1966,8 @@ impl super::Sandbox for Sandbox { .madvise(self.guest_memory_offset + address as usize, length as usize, MADV_FREE)?; if address <= 0x10000 && length >= 0xffff0000 { - self.page_set.clear(); + self.page_set_present.clear(); + self.page_set_writable.clear(); self.memory.mprotect(self.guest_memory_offset, 0x100000000 as usize, 0)?; } else { let module = self.module.as_ref().unwrap(); @@ -1895,7 +1975,8 @@ impl super::Sandbox for Sandbox { let page_end = module.address_to_page(module.round_to_page_size_down(address + length - 1)); let page_size = get_native_page_size() as u32; let page_count = page_end - page_start + 1; - self.page_set.remove((page_start, page_end)); + self.page_set_present.remove((page_start, page_end)); + self.page_set_writable.remove((page_start, page_end)); self.memory.mprotect( self.guest_memory_offset + (page_start * page_size) as usize, (page_count * page_size) as usize, diff --git a/crates/polkavm/src/sandbox/linux.rs b/crates/polkavm/src/sandbox/linux.rs index cf240758..03b83912 100644 --- a/crates/polkavm/src/sandbox/linux.rs +++ b/crates/polkavm/src/sandbox/linux.rs @@ -25,7 +25,7 @@ use std::sync::Arc; use std::time::Instant; use super::{get_native_page_size, OffsetTable, SandboxInit, SandboxKind, WorkerCache, WorkerCacheKind}; -use crate::api::{CompiledModuleKind, MemoryAccessError, Module}; +use crate::api::{CompiledModuleKind, MemoryAccessError, MemoryProtection, Module}; use crate::compiler::CompiledModule; use crate::config::Config; use crate::config::GasMeteringKind; @@ -1079,7 +1079,8 @@ pub struct Sandbox { is_program_counter_valid: bool, next_program_counter: Option, next_program_counter_changed: bool, - page_set: PageSet, + page_set_present: PageSet, + page_set_writable: PageSet, dynamic_paging_enabled: bool, aux_data_address: u32, aux_data_length: u32, @@ -1605,7 +1606,8 @@ impl super::Sandbox for Sandbox { is_program_counter_valid: false, next_program_counter: None, next_program_counter_changed: true, - page_set: PageSet::new(), + page_set_present: PageSet::new(), + page_set_writable: PageSet::new(), dynamic_paging_enabled: false, aux_data_address: 0, aux_data_length: 0, @@ -1755,8 +1757,6 @@ impl super::Sandbox for Sandbox { } self.module = None; - self.page_set.clear(); - self.vmctx().jump_into.store(ZYGOTE_TABLES.1.ext_recycle, Ordering::Relaxed); self.wake_oneshot_and_expect_idle() } @@ -1921,14 +1921,17 @@ impl super::Sandbox for Sandbox { self.wake_oneshot_and_expect_idle() } - fn is_memory_accessible(&self, address: u32, size: u32, _is_writable: bool) -> bool { + fn is_memory_accessible(&self, address: u32, size: u32, minimum_protection: MemoryProtection) -> bool { assert!(self.dynamic_paging_enabled); debug_assert_ne!(size, 0); let module = self.module.as_ref().unwrap(); let page_start = module.address_to_page(module.round_to_page_size_down(address)); let page_end = module.address_to_page(module.round_to_page_size_down(address + size - 1)); - self.page_set.contains((page_start, page_end)) + match minimum_protection { + MemoryProtection::Read => self.page_set_present.contains((page_start, page_end)), + MemoryProtection::ReadWrite => self.page_set_writable.contains((page_start, page_end)), + } } fn reset_memory(&mut self) -> Result<(), Error> { @@ -1956,6 +1959,31 @@ impl super::Sandbox for Sandbox { if !self.dynamic_paging_enabled { let length = slice.len(); + + let module = self.module.as_ref().unwrap(); + let memory_map = module.memory_map(); + let is_ok = if address >= memory_map.aux_data_address() { + let aux_data_end = module.memory_map().aux_data_address() + self.aux_data_length; + let address_end = cast(address).to_usize() + length; + address_end <= cast(aux_data_end).to_usize() + } else if address >= memory_map.stack_address_low() { + u64::from(address) + cast(length).to_u64() <= u64::from(memory_map.stack_range().end) + } else if address >= memory_map.rw_data_address() { + let end = unsafe { *self.vmctx().heap_info.heap_threshold.get() }; + u64::from(address) + cast(length).to_u64() <= end + } else if address >= memory_map.ro_data_address() { + u64::from(address) + cast(length).to_u64() <= u64::from(memory_map.ro_data_range().end) + } else { + false + }; + + if !is_ok { + return Err(MemoryAccessError::OutOfRangeAccess { + address, + length: cast(length).to_u64(), + }); + } + match linux_raw::vm_read_memory(self.child.pid, [slice], [(address as usize, length)]) { Ok(actual_length) if actual_length == length => unsafe { Ok(slice_assume_init_mut(slice)) }, Ok(_) => Err(MemoryAccessError::Error("incomplete read".into())), @@ -1965,7 +1993,7 @@ impl super::Sandbox for Sandbox { let module = self.module.as_ref().unwrap(); let page_start = module.address_to_page(module.round_to_page_size_down(address)); let page_end = module.address_to_page(module.round_to_page_size_down(address + slice.len() as u32 - 1)); - if !self.page_set.contains((page_start, page_end)) { + if !self.page_set_present.contains((page_start, page_end)) { return Err(MemoryAccessError::OutOfRangeAccess { address, length: cast(slice.len()).to_u64(), @@ -2030,20 +2058,31 @@ impl super::Sandbox for Sandbox { } else { let page_start = module.address_to_page(module.round_to_page_size_down(address)); let page_end = module.address_to_page(module.round_to_page_size_down(address + data.len() as u32 - 1)); - self.page_set.insert((page_start, page_end)); + if !self.page_set_writable.contains((page_start, page_end)) { + return Err(MemoryAccessError::OutOfRangeAccess { + address, + length: cast(data.len()).to_u64(), + }); + } + self.memory_mmap.as_slice_mut()[address as usize..address as usize + data.len()].copy_from_slice(data); Ok(()) } } - fn zero_memory(&mut self, address: u32, length: u32) -> Result<(), MemoryAccessError> { + fn zero_memory(&mut self, address: u32, length: u32, memory_protection: Option) -> Result<(), MemoryAccessError> { debug_assert_ne!(length, 0); log::trace!( - "Zeroing memory: 0x{:x}-0x{:x} ({} bytes)", + "Zeroing memory: 0x{:x}-0x{:x} ({} bytes{})", address, address as usize + length as usize, - length + length, + match memory_protection { + None => "", + Some(MemoryProtection::Read) => ", R", + Some(MemoryProtection::ReadWrite) => ", RW", + }, ); if length == 0 { @@ -2052,6 +2091,8 @@ impl super::Sandbox for Sandbox { let module = self.module.as_ref().unwrap(); if !self.dynamic_paging_enabled { + debug_assert!(memory_protection.is_none()); + let memory_map = module.memory_map(); let is_ok = if address >= memory_map.aux_data_address() { let aux_data_end = module.memory_map().aux_data_address() + self.aux_data_length; @@ -2089,40 +2130,69 @@ impl super::Sandbox for Sandbox { } else { let page_start = module.address_to_page(module.round_to_page_size_down(address)); let page_end = module.address_to_page(module.round_to_page_size_down(address + length - 1)); - if module.is_multiple_of_page_size(address) - && module.is_multiple_of_page_size(length) - && self.page_set.is_whole_region_empty((page_start, page_end)) - { - let mut arg: linux_raw::uffdio_zeropage = Default::default(); - arg.range.start = u64::from(address); - arg.range.len = u64::from(length); - arg.mode = linux_raw::UFFDIO_ZEROPAGE_MODE_DONTWAKE; - log::trace!( - "sys_uffdio_zeropage: 0x{:x}..0x{:x}", - arg.range.start, - arg.range.start + arg.range.len - ); + if memory_protection.is_some() { + debug_assert!(module.is_multiple_of_page_size(address)); + debug_assert!(module.is_multiple_of_page_size(length)); + } + + match memory_protection { + None => { + if !self.page_set_writable.contains((page_start, page_end)) { + return Err(MemoryAccessError::OutOfRangeAccess { + address, + length: u64::from(length), + }); + } - if let Err(error) = linux_raw::sys_uffdio_zeropage(self.userfaultfd.borrow(), &mut arg) { - return Err(MemoryAccessError::Error(error.into())); + self.memory_mmap.as_slice_mut()[address as usize..address as usize + length as usize].fill(0); + return Ok(()); } - } else { - self.memory_mmap.as_slice_mut()[address as usize..address as usize + length as usize].fill(0); - } + Some(MemoryProtection::Read) => { + if !self.page_set_present.is_whole_region_empty((page_start, page_end)) { + self.madvise_remove(address, length) + .map_err(|error| MemoryAccessError::Error(error.into()))?; + self.page_set_writable.remove((page_start, page_end)); + } + + if let Err(error) = self.uffdio_zeropage(address, length) { + self.page_set_present.remove((page_start, page_end)); + return Err(error); + } + + self.page_set_present.insert((page_start, page_end)); + self.uffdio_writeprotect(address, length, MemoryProtection::Read)?; + } + Some(MemoryProtection::ReadWrite) => { + if !self.page_set_present.is_whole_region_empty((page_start, page_end)) { + self.madvise_remove(address, length) + .map_err(|error| MemoryAccessError::Error(error.into()))?; + } - self.page_set.insert((page_start, page_end)); + if let Err(error) = self.uffdio_zeropage(address, length) { + self.page_set_present.remove((page_start, page_end)); + self.page_set_writable.remove((page_start, page_end)); + return Err(error); + } + + self.page_set_present.insert((page_start, page_end)); + self.page_set_writable.insert((page_start, page_end)); + } + } } Ok(()) } - fn change_memory_protection(&mut self, address: u32, length: u32, make_read_only: bool) -> Result<(), MemoryAccessError> { + fn change_memory_protection(&mut self, address: u32, length: u32, protection: MemoryProtection) -> Result<(), MemoryAccessError> { assert!(self.dynamic_paging_enabled); log::trace!( "{} memory: 0x{:x}-0x{:x} ({} bytes)", - if make_read_only { "Protecting" } else { "Unprotecting" }, + match protection { + MemoryProtection::Read => "Protecting", + MemoryProtection::ReadWrite => "Unprotecting", + }, address, address as usize + length as usize, length @@ -2131,24 +2201,18 @@ impl super::Sandbox for Sandbox { let module = self.module.as_ref().unwrap(); let page_start = module.address_to_page(module.round_to_page_size_down(address)); let page_end = module.address_to_page(module.round_to_page_size_down(address + length - 1)); - if !self.page_set.contains((page_start, page_end)) { + if !self.page_set_present.contains((page_start, page_end)) { return Err(MemoryAccessError::OutOfRangeAccess { address, length: u64::from(length), }); } - let mut arg: linux_raw::uffdio_writeprotect = Default::default(); - arg.range.start = u64::from(address); - arg.range.len = u64::from(length); - arg.mode = if make_read_only { - linux_raw::UFFDIO_WRITEPROTECT_MODE_WP - } else { - 0 - }; + self.uffdio_writeprotect(address, length, protection)?; - if let Err(error) = linux_raw::sys_uffdio_writeprotect(self.userfaultfd.borrow(), &mut arg) { - return Err(MemoryAccessError::Error(error.into())); + match protection { + MemoryProtection::Read => self.page_set_writable.remove((page_start, page_end)), + MemoryProtection::ReadWrite => self.page_set_writable.insert((page_start, page_end)), } Ok(()) @@ -2166,21 +2230,17 @@ impl super::Sandbox for Sandbox { "freeing of pages when dynamic paging is not enabled is not implemented", )) } else { - unsafe { - linux_raw::sys_madvise( - self.memory_mmap.as_mut_ptr().add(address as usize), - length as usize, - linux_raw::MADV_REMOVE, - )?; - } + self.madvise_remove(address, length)?; if address <= 0x10000 && length >= 0xffff0000 { - self.page_set.clear(); + self.page_set_present.clear(); + self.page_set_writable.clear(); } else { let module = self.module.as_ref().unwrap(); let page_start = module.address_to_page(module.round_to_page_size_down(address)); let page_end = module.address_to_page(module.round_to_page_size_down(address + length - 1)); - self.page_set.remove((page_start, page_end)); + self.page_set_present.remove((page_start, page_end)); + self.page_set_writable.remove((page_start, page_end)); } Ok(()) @@ -2445,4 +2505,51 @@ impl Sandbox { Err(Error::from(format!("worker process unexpectedly quit: {status}"))) } } + + fn uffdio_zeropage(&self, address: u32, length: u32) -> Result<(), MemoryAccessError> { + let mut arg: linux_raw::uffdio_zeropage = Default::default(); + arg.range.start = u64::from(address); + arg.range.len = u64::from(length); + arg.mode = linux_raw::UFFDIO_ZEROPAGE_MODE_DONTWAKE; + + log::trace!( + "sys_uffdio_zeropage: 0x{:x}..0x{:x}", + arg.range.start, + arg.range.start + arg.range.len + ); + + if let Err(error) = linux_raw::sys_uffdio_zeropage(self.userfaultfd.borrow(), &mut arg) { + return Err(MemoryAccessError::Error(error.into())); + } + + Ok(()) + } + + fn uffdio_writeprotect(&self, address: u32, length: u32, protection: MemoryProtection) -> Result<(), MemoryAccessError> { + let mut arg: linux_raw::uffdio_writeprotect = Default::default(); + arg.range.start = u64::from(address); + arg.range.len = u64::from(length); + arg.mode = match protection { + MemoryProtection::Read => linux_raw::UFFDIO_WRITEPROTECT_MODE_WP, + MemoryProtection::ReadWrite => 0, + }; + + if let Err(error) = linux_raw::sys_uffdio_writeprotect(self.userfaultfd.borrow(), &mut arg) { + return Err(MemoryAccessError::Error(error.into())); + } + + Ok(()) + } + + fn madvise_remove(&mut self, address: u32, length: u32) -> Result<(), Error> { + unsafe { + linux_raw::sys_madvise( + self.memory_mmap.as_mut_ptr().add(address as usize), + length as usize, + linux_raw::MADV_REMOVE, + )?; + } + + Ok(()) + } } diff --git a/crates/polkavm/src/tests.rs b/crates/polkavm/src/tests.rs index c0d122a8..37ef930b 100644 --- a/crates/polkavm/src/tests.rs +++ b/crates/polkavm/src/tests.rs @@ -1,7 +1,7 @@ use crate::mutex::Mutex; use crate::{ - BackendKind, CallError, Caller, Config, Engine, GasMeteringKind, InterruptKind, Linker, MemoryAccessError, Module, ModuleConfig, - ProgramBlob, ProgramCounter, Reg, Segfault, SetCacheSizeLimitArgs, + BackendKind, CallError, Caller, Config, Engine, GasMeteringKind, InterruptKind, Linker, MemoryAccessError, MemoryProtection, Module, + ModuleConfig, ProgramBlob, ProgramCounter, Reg, Segfault, SetCacheSizeLimitArgs, }; use alloc::collections::BTreeMap; use alloc::format; @@ -116,6 +116,18 @@ fn get_native_page_size() -> usize { } } +#[track_caller] +fn assert_out_of_range_access(result: Result, expected_address: u32, expected_length: u32) { + match result { + Ok(_) => panic!("expected Err(MemoryAccessError::OutOfRangeAccess), got Ok"), + Err(MemoryAccessError::OutOfRangeAccess { address, length }) + if address == expected_address && length == u64::from(expected_length) => {} + Err(error) => panic!( + "expected Err(MemoryAccessError::OutOfRangeAccess {{ address: {expected_address}, length: {expected_length} }}), got {error:?}" + ), + } +} + macro_rules! run_tests { ($($test_name:ident)+) => { if_compiler_is_supported! { @@ -807,6 +819,11 @@ fn zero_memory(engine_config: Config) { .collect(); let mut instance = module.instantiate().unwrap(); + assert_out_of_range_access( + instance.zero_memory(memory_map.ro_data_address(), 1), + memory_map.ro_data_address(), + 1, + ); instance.set_next_program_counter(offsets[0]); instance.set_reg(Reg::RA, crate::RETURN_TO_HOST); match_interrupt!(instance.run().unwrap(), InterruptKind::Ecalli(..)); @@ -1183,10 +1200,21 @@ fn dynamic_paging_basic(mut engine_config: Config) { assert_eq!(segfault.page_address, 0x10000); assert_eq!(segfault.page_size, page_size); + // Both normal 'zero_memory' and 'write_memory' cannot resolve pagefaults. + assert_out_of_range_access( + instance.zero_memory(segfault.page_address, page_size), + segfault.page_address, + page_size, + ); + assert_out_of_range_access(instance.write_memory(segfault.page_address, &[0, 0]), segfault.page_address, 2); + assert_out_of_range_access(instance.read_u8(segfault.page_address), segfault.page_address, 1); + // Now handle it. - instance.zero_memory(segfault.page_address, page_size).unwrap(); - assert!(instance.is_memory_accessible(0x10000, 0x4, false)); - assert!(!instance.is_memory_accessible(0x10000 + page_size, 0x4, false)); + instance + .zero_memory_with_memory_protection(segfault.page_address, page_size, MemoryProtection::ReadWrite) + .unwrap(); + assert!(instance.is_memory_accessible(0x10000, 0x4, MemoryProtection::Read)); + assert!(!instance.is_memory_accessible(0x10000 + page_size, 0x4, MemoryProtection::Read)); let segfault = expect_segfault(instance.run().unwrap()); assert_eq!(segfault.page_address, 0x10000 + page_size); @@ -1197,7 +1225,9 @@ fn dynamic_paging_basic(mut engine_config: Config) { assert_eq!(instance.reg(Reg::A1), 0); assert_eq!(instance.reg(Reg::A2), 0x12); assert_eq!(instance.reg(Reg::T0), 0x5678); - instance.zero_memory(segfault.page_address, page_size).unwrap(); + instance + .zero_memory_with_memory_protection(segfault.page_address, page_size, MemoryProtection::ReadWrite) + .unwrap(); match_interrupt!(instance.run().unwrap(), InterruptKind::Finished); assert_eq!(instance.reg(Reg::A2), 0); @@ -1206,6 +1236,29 @@ fn dynamic_paging_basic(mut engine_config: Config) { // Running the program again produces no more segfaults, since everything is faulted already. instance.set_next_program_counter(offsets[0]); match_interrupt!(instance.run().unwrap(), InterruptKind::Finished); + + // Clear the first page and make it read-only. + instance + .zero_memory_with_memory_protection(0x10000, page_size, MemoryProtection::Read) + .unwrap(); + + // Cannot write to the page anymore, but can read it. + assert_out_of_range_access(instance.zero_memory(0x10000, page_size), 0x10000, page_size); + assert_out_of_range_access(instance.zero_memory(0x10000, 1), 0x10000, 1); + assert_out_of_range_access(instance.write_memory(0x10000, &[0]), 0x10000, 1); + assert_eq!(instance.read_u8(0x10000).unwrap(), 0); + + // The program cannot store anything there either. + instance.set_next_program_counter(offsets[0]); + let segfault = expect_segfault(instance.run().unwrap()); + assert_eq!(segfault.page_address, 0x10000); + assert_eq!(segfault.page_size, page_size); + assert_eq!(instance.program_counter(), Some(offsets[1])); + assert_eq!(instance.next_program_counter(), Some(offsets[1])); + + // But it can read. + instance.set_next_program_counter(offsets[2]); + match_interrupt!(instance.run().unwrap(), InterruptKind::Finished); } fn dynamic_paging_freeing_pages(mut engine_config: Config) { @@ -1234,7 +1287,9 @@ fn dynamic_paging_freeing_pages(mut engine_config: Config) { instance.set_reg(Reg::RA, crate::RETURN_TO_HOST); instance.set_next_program_counter(offsets[0]); let segfault = expect_segfault(instance.run().unwrap()); - instance.zero_memory(segfault.page_address, page_size).unwrap(); + instance + .zero_memory_with_memory_protection(segfault.page_address, page_size, MemoryProtection::ReadWrite) + .unwrap(); match_interrupt!(instance.run().unwrap(), InterruptKind::Finished); instance.set_next_program_counter(offsets[0]); @@ -1292,7 +1347,9 @@ fn dynamic_paging_protect_memory(mut engine_config: Config) { assert_eq!(segfault.page_address, 0x10000); assert!(!segfault.is_write_protected); assert_eq!(instance.program_counter(), Some(offsets[0])); - instance.zero_memory(segfault.page_address, page_size).unwrap(); + instance + .zero_memory_with_memory_protection(segfault.page_address, page_size, MemoryProtection::ReadWrite) + .unwrap(); instance.protect_memory(segfault.page_address, page_size).unwrap(); let segfault = expect_segfault(instance.run().unwrap()); @@ -1347,7 +1404,9 @@ fn dynamic_paging_stress_test(mut engine_config: Config) { instance.set_reg(Reg::RA, crate::RETURN_TO_HOST); instance.set_next_program_counter(offsets[0]); let segfault = expect_segfault(instance.run().unwrap()); - instance.zero_memory(segfault.page_address, page_size).unwrap(); + instance + .zero_memory_with_memory_protection(segfault.page_address, page_size, MemoryProtection::ReadWrite) + .unwrap(); match_interrupt!(instance.run().unwrap(), InterruptKind::Finished); }); threads.push(thread); @@ -1393,7 +1452,9 @@ fn dynamic_paging_initialize_multiple_pages(mut engine_config: Config) { instance.set_next_program_counter(offsets[0]); let segfault = expect_segfault(instance.run().unwrap()); assert_eq!(segfault.page_address, 0x10000); - instance.zero_memory(0x10000, page_size * 2).unwrap(); + instance + .zero_memory_with_memory_protection(0x10000, page_size * 2, MemoryProtection::ReadWrite) + .unwrap(); // We've zeroed two pages, so we don't get a segfault anymore. match_interrupt!(instance.run().unwrap(), InterruptKind::Finished); } @@ -1430,7 +1491,9 @@ fn dynamic_paging_preinitialize_pages(mut engine_config: Config) { let mut instance = module.instantiate().unwrap(); instance.set_reg(Reg::RA, crate::RETURN_TO_HOST); instance.set_next_program_counter(offsets[0]); - instance.zero_memory(0x10000, page_size * 2).unwrap(); + instance + .zero_memory_with_memory_protection(0x10000, page_size * 2, MemoryProtection::ReadWrite) + .unwrap(); match_interrupt!(instance.run().unwrap(), InterruptKind::Finished); } @@ -1494,10 +1557,19 @@ fn dynamic_paging_read_at_page_boundary(mut engine_config: Config) { instance.set_next_program_counter(offsets[0]); let segfault = expect_segfault(instance.run().unwrap()); assert_eq!(segfault.page_address, 0x10000); + instance + .zero_memory_with_memory_protection(0x10000, page_size * 2, MemoryProtection::ReadWrite) + .unwrap(); instance.write_memory(0x10fff, &[0xaa, 0xbb]).unwrap(); match_interrupt!(instance.run().unwrap(), InterruptKind::Finished); assert_eq!(instance.reg(Reg::A0), 0x00bbaa00); + + instance.set_reg(Reg::A0, 0); + instance.protect_memory(0x10000, page_size * 2).unwrap(); + instance.set_next_program_counter(offsets[0]); + match_interrupt!(instance.run().unwrap(), InterruptKind::Finished); + assert_eq!(instance.reg(Reg::A0), 0x00bbaa00); } fn dynamic_paging_read_at_top_of_address_space(mut engine_config: Config) { @@ -1648,12 +1720,16 @@ fn dynamic_paging_write_at_page_boundary_with_no_pages(mut engine_config: Config instance.set_next_program_counter(offsets[0]); let segfault = expect_segfault(instance.run().unwrap()); assert_eq!(segfault.page_address, 0x10000); - instance.zero_memory(0x10000, page_size).unwrap(); + instance + .zero_memory_with_memory_protection(0x10000, page_size, MemoryProtection::ReadWrite) + .unwrap(); let segfault = expect_segfault(instance.run().unwrap()); assert_eq!(segfault.page_address, 0x11000); assert_eq!(instance.read_memory(0x10ffe, 2).unwrap(), vec![0, 0]); - instance.zero_memory(0x11000, page_size).unwrap(); + instance + .zero_memory_with_memory_protection(0x11000, page_size, MemoryProtection::ReadWrite) + .unwrap(); match_interrupt!(instance.run().unwrap(), InterruptKind::Finished); assert_eq!(instance.read_memory(0x10ffe, 2).unwrap(), vec![0x78, 0x56]); @@ -1684,12 +1760,16 @@ fn dynamic_paging_write_at_page_boundary_with_first_page(mut engine_config: Conf let mut instance = module.instantiate().unwrap(); instance.set_reg(Reg::RA, crate::RETURN_TO_HOST); instance.set_next_program_counter(offsets[0]); - instance.zero_memory(0x10000, page_size).unwrap(); + instance + .zero_memory_with_memory_protection(0x10000, page_size, MemoryProtection::ReadWrite) + .unwrap(); let segfault = expect_segfault(instance.run().unwrap()); assert_eq!(segfault.page_address, 0x11000); assert_eq!(instance.read_memory(0x10ffe, 2).unwrap(), vec![0, 0]); - instance.zero_memory(0x11000, page_size).unwrap(); + instance + .zero_memory_with_memory_protection(0x11000, page_size, MemoryProtection::ReadWrite) + .unwrap(); match_interrupt!(instance.run().unwrap(), InterruptKind::Finished); assert_eq!(instance.read_memory(0x10ffe, 2).unwrap(), vec![0x78, 0x56]); @@ -1720,12 +1800,16 @@ fn dynamic_paging_write_at_page_boundary_with_second_page(mut engine_config: Con let mut instance = module.instantiate().unwrap(); instance.set_reg(Reg::RA, crate::RETURN_TO_HOST); instance.set_next_program_counter(offsets[0]); - instance.zero_memory(0x11000, page_size).unwrap(); + instance + .zero_memory_with_memory_protection(0x11000, page_size, MemoryProtection::ReadWrite) + .unwrap(); let segfault = expect_segfault(instance.run().unwrap()); assert_eq!(segfault.page_address, 0x10000); assert_eq!(instance.read_memory(0x11000, 2).unwrap(), vec![0, 0]); - instance.zero_memory(0x10000, page_size).unwrap(); + instance + .zero_memory_with_memory_protection(0x10000, page_size, MemoryProtection::ReadWrite) + .unwrap(); match_interrupt!(instance.run().unwrap(), InterruptKind::Finished); assert_eq!(instance.read_memory(0x11000, 2).unwrap(), vec![0x34, 0x12]); @@ -1760,7 +1844,9 @@ fn dynamic_paging_change_written_value_and_address_during_segfault(mut engine_co instance.set_reg(Reg::A1, 0x10001); let segfault = expect_segfault(instance.run().unwrap()); assert_eq!(segfault.page_address, 0x10000); - instance.zero_memory(0x10000, page_size).unwrap(); + instance + .zero_memory_with_memory_protection(0x10000, page_size, MemoryProtection::ReadWrite) + .unwrap(); instance.set_reg(Reg::A0, 0x55667788); instance.set_reg(Reg::A1, 0x10002); match_interrupt!(instance.run().unwrap(), InterruptKind::Finished); @@ -1790,7 +1876,9 @@ fn dynamic_paging_cancel_segfault_by_changing_address(mut engine_config: Config) .collect(); let mut instance = module.instantiate().unwrap(); - instance.zero_memory(0x11000, page_size).unwrap(); + instance + .zero_memory_with_memory_protection(0x11000, page_size, MemoryProtection::ReadWrite) + .unwrap(); instance.set_reg(Reg::RA, crate::RETURN_TO_HOST); instance.set_next_program_counter(offsets[0]); instance.set_reg(Reg::A0, 0x10000); @@ -1833,7 +1921,7 @@ fn dynamic_paging_worker_recycle_turn_dynamic_paging_on_and_off(mut engine_confi if !is_dynamic { assert_eq!(instance.read_u32(0x20000).unwrap(), 0); } else { - assert!(instance.read_u32(0x20000).is_err()); + assert_out_of_range_access(instance.read_u32(0x20000), 0x20000, 4); } instance.set_reg(Reg::RA, crate::RETURN_TO_HOST); @@ -1843,13 +1931,17 @@ fn dynamic_paging_worker_recycle_turn_dynamic_paging_on_and_off(mut engine_confi assert_eq!(segfault.page_address, 0x20000); assert_eq!(segfault.page_size, page_size); let segfault = expect_segfault(instance.run().unwrap()); - instance.zero_memory(segfault.page_address + 4, page_size).unwrap(); + assert_out_of_range_access(instance.read_u32(0x21000), 0x21000, 4); + instance + .zero_memory_with_memory_protection(segfault.page_address, page_size + 4, MemoryProtection::ReadWrite) + .unwrap(); match_interrupt!(instance.run().unwrap(), InterruptKind::Finished); assert_eq!(instance.read_u32(0x20000).unwrap(), 0x12345678); + assert_eq!(instance.read_u32(0x21000).unwrap(), 0); instance.set_next_program_counter(ProgramCounter(0)); match_interrupt!(instance.run().unwrap(), InterruptKind::Finished); } else { - assert!(instance.read_u32(0x21000).is_err()); + assert_out_of_range_access(instance.read_u32(0x21000), 0x21000, 4); assert_eq!(instance.read_u32(0x20000).unwrap(), 0); match_interrupt!(instance.run().unwrap(), InterruptKind::Finished); assert_eq!(instance.read_u32(0x20000).unwrap(), 0x12345678); @@ -1949,7 +2041,9 @@ fn dynamic_paging_change_program_counter_during_segfault(mut engine_config: Conf instance.set_next_program_counter(offsets[2]); let segfault = expect_segfault(instance.run().unwrap()); assert_eq!(segfault.page_address, 0x11000); - instance.zero_memory(segfault.page_address, page_size).unwrap(); + instance + .zero_memory_with_memory_protection(segfault.page_address, page_size, MemoryProtection::ReadWrite) + .unwrap(); match_interrupt!(instance.run().unwrap(), InterruptKind::Finished); assert_eq!(instance.read_u32(0x11000).unwrap(), 2); } @@ -2195,7 +2289,9 @@ fn dynamic_paging_parallel_page_fault_stress_test(mut engine_config: Config) { break; } assert_eq!(segfault.page_address, address); - instance.zero_memory(segfault.page_address, segfault.page_size).unwrap(); + instance + .zero_memory_with_memory_protection(segfault.page_address, segfault.page_size, MemoryProtection::ReadWrite) + .unwrap(); address += segfault.page_size; } flag.disarm(); @@ -2786,9 +2882,17 @@ fn aux_data_accessible_area(config: Config) { instance.set_next_program_counter(offsets[0]); match_interrupt!(instance.run().unwrap(), InterruptKind::Finished); - assert!(instance.is_memory_accessible(module.memory_map().aux_data_address() + page_size - 3, 4, false)); - assert!(instance.is_memory_accessible(module.memory_map().aux_data_address() + page_size * 2 - 4, 4, false)); - assert!(!instance.is_memory_accessible(module.memory_map().aux_data_address() + page_size * 2 - 3, 4, false)); + assert!(instance.is_memory_accessible(module.memory_map().aux_data_address() + page_size - 3, 4, MemoryProtection::Read)); + assert!(instance.is_memory_accessible( + module.memory_map().aux_data_address() + page_size * 2 - 4, + 4, + MemoryProtection::Read + )); + assert!(!instance.is_memory_accessible( + module.memory_map().aux_data_address() + page_size * 2 - 3, + 4, + MemoryProtection::Read + )); assert!(instance.read_u32(module.memory_map().aux_data_address() + page_size - 3).is_ok()); assert!(instance .read_u32(module.memory_map().aux_data_address() + page_size * 2 - 4) @@ -2797,10 +2901,26 @@ fn aux_data_accessible_area(config: Config) { .read_u32(module.memory_map().aux_data_address() + page_size * 2 - 3) .is_err()); - assert!(instance.is_memory_accessible(module.memory_map().aux_data_address() + page_size - 3, 4, true)); - assert!(instance.is_memory_accessible(module.memory_map().aux_data_address() + page_size * 2 - 4, 4, true)); - assert!(!instance.is_memory_accessible(module.memory_map().aux_data_address() + page_size * 2 - 3, 4, true)); - assert!(!instance.is_memory_accessible(module.memory_map().aux_data_address() + page_size * 2, 4, true)); + assert!(instance.is_memory_accessible( + module.memory_map().aux_data_address() + page_size - 3, + 4, + MemoryProtection::ReadWrite + )); + assert!(instance.is_memory_accessible( + module.memory_map().aux_data_address() + page_size * 2 - 4, + 4, + MemoryProtection::ReadWrite + )); + assert!(!instance.is_memory_accessible( + module.memory_map().aux_data_address() + page_size * 2 - 3, + 4, + MemoryProtection::ReadWrite + )); + assert!(!instance.is_memory_accessible( + module.memory_map().aux_data_address() + page_size * 2, + 4, + MemoryProtection::ReadWrite + )); assert!(instance .write_u32(module.memory_map().aux_data_address() + page_size - 3, 0) .is_ok()); @@ -2829,9 +2949,17 @@ fn aux_data_accessible_area(config: Config) { instance.set_host_side_aux_write_protect(true).unwrap(); // Still readable as before. - assert!(instance.is_memory_accessible(module.memory_map().aux_data_address() + page_size - 3, 4, false)); - assert!(instance.is_memory_accessible(module.memory_map().aux_data_address() + page_size * 2 - 4, 4, false)); - assert!(!instance.is_memory_accessible(module.memory_map().aux_data_address() + page_size * 2 - 3, 4, false)); + assert!(instance.is_memory_accessible(module.memory_map().aux_data_address() + page_size - 3, 4, MemoryProtection::Read)); + assert!(instance.is_memory_accessible( + module.memory_map().aux_data_address() + page_size * 2 - 4, + 4, + MemoryProtection::Read + )); + assert!(!instance.is_memory_accessible( + module.memory_map().aux_data_address() + page_size * 2 - 3, + 4, + MemoryProtection::Read + )); assert!(instance.read_u32(module.memory_map().aux_data_address() + page_size - 3).is_ok()); assert!(instance .read_u32(module.memory_map().aux_data_address() + page_size * 2 - 4) @@ -2841,10 +2969,26 @@ fn aux_data_accessible_area(config: Config) { .is_err()); // Not writable anymore. - assert!(!instance.is_memory_accessible(module.memory_map().aux_data_address() + page_size - 3, 4, true)); - assert!(!instance.is_memory_accessible(module.memory_map().aux_data_address() + page_size * 2 - 4, 4, true)); - assert!(!instance.is_memory_accessible(module.memory_map().aux_data_address() + page_size * 2 - 3, 4, true)); - assert!(!instance.is_memory_accessible(module.memory_map().aux_data_address() + page_size * 2, 4, true)); + assert!(!instance.is_memory_accessible( + module.memory_map().aux_data_address() + page_size - 3, + 4, + MemoryProtection::ReadWrite + )); + assert!(!instance.is_memory_accessible( + module.memory_map().aux_data_address() + page_size * 2 - 4, + 4, + MemoryProtection::ReadWrite + )); + assert!(!instance.is_memory_accessible( + module.memory_map().aux_data_address() + page_size * 2 - 3, + 4, + MemoryProtection::ReadWrite + )); + assert!(!instance.is_memory_accessible( + module.memory_map().aux_data_address() + page_size * 2, + 4, + MemoryProtection::ReadWrite + )); assert!(instance .write_u32(module.memory_map().aux_data_address() + page_size - 3, 0) .is_err()); @@ -3974,7 +4118,9 @@ fn memset_with_dynamic_paging(mut config: Config) { assert_eq!(instance.program_counter(), Some(offsets[0])); assert_eq!(instance.next_program_counter(), Some(offsets[0])); assert_eq!(instance.gas(), 98); - instance.zero_memory(segfault.page_address, segfault.page_size).unwrap(); + instance + .zero_memory_with_memory_protection(segfault.page_address, segfault.page_size, MemoryProtection::ReadWrite) + .unwrap(); assert!(matches!(instance.run().unwrap(), InterruptKind::Finished)); assert_eq!(instance.reg(Reg::A0), u64::from(memory_map.rw_data_range().start + 3)); // Pointer is at the end. assert_eq!(instance.reg(Reg::A1), 0x1234567a); // Value is unchanged. @@ -3986,7 +4132,9 @@ fn memset_with_dynamic_paging(mut config: Config) { assert_eq!(instance.gas(), 95); let mut instance = module.instantiate().unwrap(); - instance.zero_memory(memory_map.rw_data_range().start, page_size).unwrap(); + instance + .zero_memory_with_memory_protection(memory_map.rw_data_range().start, page_size, MemoryProtection::ReadWrite) + .unwrap(); instance.set_gas(100); instance.prepare_call_typed(offsets[0], (memory_map.rw_data_range().start + page_size - 1, 0x1234567a, 4)); let segfault = expect_segfault(instance.run().unwrap()); @@ -4006,7 +4154,7 @@ fn memset_with_dynamic_paging(mut config: Config) { instance.set_reg(Reg::A1, 0x1234567b); instance.set_reg(Reg::A2, 2); instance - .zero_memory(memory_map.rw_data_range().start + page_size, page_size) + .zero_memory_with_memory_protection(memory_map.rw_data_range().start + page_size, page_size, MemoryProtection::ReadWrite) .unwrap(); assert!(matches!(instance.run().unwrap(), InterruptKind::Finished)); assert_eq!(instance.reg(Reg::A0), u64::from(memory_map.rw_data_range().start + page_size + 3)); // Pointer is incremented. diff --git a/tools/spectool/src/main.rs b/tools/spectool/src/main.rs index 9c9bbd7b..e2233fd3 100644 --- a/tools/spectool/src/main.rs +++ b/tools/spectool/src/main.rs @@ -5,7 +5,7 @@ use clap::Parser; use core::fmt::Write; -use polkavm::{CacheModel, CostModelKind, Engine, InterruptKind, Module, ModuleConfig, ProgramBlob, Reg}; +use polkavm::{CacheModel, CostModelKind, Engine, InterruptKind, MemoryProtection, Module, ModuleConfig, ProgramBlob, Reg}; use polkavm_common::assembler::assemble; use polkavm_common::program::{asm, ProgramCounter, ProgramParts, ISA64_V1}; use polkavm_common::utils::parse_slice; @@ -536,10 +536,17 @@ fn main_generate() { length, is_writable, } => { - instance.zero_memory(address, length).unwrap(); - if !is_writable { - instance.protect_memory(address, length).unwrap(); - } + instance + .zero_memory_with_memory_protection( + address, + length, + if is_writable { + MemoryProtection::ReadWrite + } else { + MemoryProtection::Read + }, + ) + .unwrap(); if final_state.status.is_empty() { initial_page_map.push(Page { @@ -558,7 +565,20 @@ fn main_generate() { continue; } TestcaseStep::Write { address, contents } => { + let mut pages_made_writable = Vec::new(); + for address in ((address / 4096 * 4096)..(address + contents.len() as u32).next_multiple_of(4096)).step_by(4096) { + assert!(instance.is_memory_accessible(address, 4096, MemoryProtection::Read)); + if !instance.is_memory_accessible(address, 4096, MemoryProtection::ReadWrite) { + pages_made_writable.push(address); + instance.unprotect_memory(address, 4096).unwrap(); + } + } + instance.write_memory(address, &contents).unwrap(); + for address in pages_made_writable { + instance.protect_memory(address, 4096).unwrap(); + } + nth_step += 1; continue; } From 34f3580c24d6c068282b323e0a677897f4220bac Mon Sep 17 00:00:00 2001 From: Jan Bujak Date: Mon, 10 Nov 2025 11:03:53 +0000 Subject: [PATCH 20/21] Add more spectool tests --- ...gas_register_move_use_all_decode_slots.txt | 7 +++++++ ...start_execution_in_the_middle_of_block.txt | 8 ++++++++ tools/spectool/spec/src/gas_xor_and_shift.txt | 6 ++++++ ...multistep_ecalli_at_the_start_of_block.txt | 13 ++++++++++++ ...ultistep_ecalli_in_the_middle_of_block.txt | 13 ++++++++++++ ...multistep_paging_at_the_start_of_block.txt | 19 ++++++++++++++++++ ...ultistep_paging_in_the_middle_of_block.txt | 20 +++++++++++++++++++ 7 files changed, 86 insertions(+) create mode 100644 tools/spectool/spec/src/gas_register_move_use_all_decode_slots.txt create mode 100644 tools/spectool/spec/src/gas_start_execution_in_the_middle_of_block.txt create mode 100644 tools/spectool/spec/src/gas_xor_and_shift.txt create mode 100644 tools/spectool/spec/src/multistep_ecalli_at_the_start_of_block.txt create mode 100644 tools/spectool/spec/src/multistep_ecalli_in_the_middle_of_block.txt create mode 100644 tools/spectool/spec/src/multistep_paging_at_the_start_of_block.txt create mode 100644 tools/spectool/spec/src/multistep_paging_in_the_middle_of_block.txt diff --git a/tools/spectool/spec/src/gas_register_move_use_all_decode_slots.txt b/tools/spectool/spec/src/gas_register_move_use_all_decode_slots.txt new file mode 100644 index 00000000..d6d26838 --- /dev/null +++ b/tools/spectool/spec/src/gas_register_move_use_all_decode_slots.txt @@ -0,0 +1,7 @@ +pub @main: + s0 = a1 + a0 = a1 + a1 = t0 + a2 = s1 +pub @expected_exit: %no_fallthrough + trap diff --git a/tools/spectool/spec/src/gas_start_execution_in_the_middle_of_block.txt b/tools/spectool/spec/src/gas_start_execution_in_the_middle_of_block.txt new file mode 100644 index 00000000..2cd92c94 --- /dev/null +++ b/tools/spectool/spec/src/gas_start_execution_in_the_middle_of_block.txt @@ -0,0 +1,8 @@ + a1 = 1 + a2 = 2 +pub @main: %no_fallthrough + a3 = 3 + a4 = 4 + a0 = a1 + a2 +pub @expected_exit: %no_fallthrough + trap diff --git a/tools/spectool/spec/src/gas_xor_and_shift.txt b/tools/spectool/spec/src/gas_xor_and_shift.txt new file mode 100644 index 00000000..886fa7d9 --- /dev/null +++ b/tools/spectool/spec/src/gas_xor_and_shift.txt @@ -0,0 +1,6 @@ +pub @main: + a1 = a1 ^ 0xffffffffffffffff + a1 = a0 >> a1 + fallthrough +pub @expected_exit: %no_fallthrough + trap diff --git a/tools/spectool/spec/src/multistep_ecalli_at_the_start_of_block.txt b/tools/spectool/spec/src/multistep_ecalli_at_the_start_of_block.txt new file mode 100644 index 00000000..925ddd17 --- /dev/null +++ b/tools/spectool/spec/src/multistep_ecalli_at_the_start_of_block.txt @@ -0,0 +1,13 @@ +pub @main: + ecalli 3 + a0 = 0x30000 + a2 = a1 + a0 + ecalli 4 +pub @expected_exit: %no_fallthrough + ret + +step: run +step: a1 = 10 +step: run +step: ra = 0xffff0000 +step: run diff --git a/tools/spectool/spec/src/multistep_ecalli_in_the_middle_of_block.txt b/tools/spectool/spec/src/multistep_ecalli_in_the_middle_of_block.txt new file mode 100644 index 00000000..55254199 --- /dev/null +++ b/tools/spectool/spec/src/multistep_ecalli_in_the_middle_of_block.txt @@ -0,0 +1,13 @@ +pub @main: + a0 = 0x30000 + ecalli 3 + a2 = a1 + a0 + ecalli 4 +pub @expected_exit: %no_fallthrough + ret + +step: run +step: a1 = 10 +step: run +step: ra = 0xffff0000 +step: run diff --git a/tools/spectool/spec/src/multistep_paging_at_the_start_of_block.txt b/tools/spectool/spec/src/multistep_paging_at_the_start_of_block.txt new file mode 100644 index 00000000..e1429e59 --- /dev/null +++ b/tools/spectool/spec/src/multistep_paging_at_the_start_of_block.txt @@ -0,0 +1,19 @@ +pub @main: + a1 = u8 [0x30000] + a2 = u8 [0x30fff] + a3 = u8 [0x31fff] + a4 = a1 + a2 + a4 = a4 + a3 +pub @expected_exit: %no_fallthrough + ret + +step: run +step: run +step: map RW 0x30000..0x31000 +step: write 0x30000 01 +step: write 0x30fff 02 +step: run +step: map RW 0x31000..0x32000 +step: write 0x31fff 03 +step: ra = 0xffff0000 +step: run diff --git a/tools/spectool/spec/src/multistep_paging_in_the_middle_of_block.txt b/tools/spectool/spec/src/multistep_paging_in_the_middle_of_block.txt new file mode 100644 index 00000000..5118864e --- /dev/null +++ b/tools/spectool/spec/src/multistep_paging_in_the_middle_of_block.txt @@ -0,0 +1,20 @@ +pub @main: + a0 = 0x30000 + a1 = u8 [a0] + a2 = u8 [a0 + 0x0fff] + a3 = u8 [a0 + 0x1fff] + a4 = a1 + a2 + a4 = a4 + a3 +pub @expected_exit: %no_fallthrough + ret + +step: run +step: run +step: map RW 0x30000..0x31000 +step: write 0x30000 01 +step: write 0x30fff 02 +step: run +step: map RW 0x31000..0x32000 +step: write 0x31fff 03 +step: ra = 0xffff0000 +step: run From 1ce70ebb0ec83f656de2697b2a91705fd99eb365 Mon Sep 17 00:00:00 2001 From: Jan Bujak Date: Tue, 11 Nov 2025 07:11:38 +0000 Subject: [PATCH 21/21] Unconditionally format `page_set` with rustfmt too --- crates/polkavm/src/lib.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/polkavm/src/lib.rs b/crates/polkavm/src/lib.rs index 25db670d..d0d47017 100644 --- a/crates/polkavm/src/lib.rs +++ b/crates/polkavm/src/lib.rs @@ -110,6 +110,8 @@ mod compiler; #[cfg(rustfmt)] mod generic_allocator; #[cfg(rustfmt)] +mod page_set; +#[cfg(rustfmt)] mod sandbox; #[cfg(rustfmt)] mod shm_allocator;