polkavm-test-data

Signed-off-by: Jarkko Sakkinen <[email protected]>

Cargo.toml

@@ -10,6 +10,7 @@ members = [
     "crates/polkavm-common",
     "crates/polkavm-linux-raw",
     "crates/polkavm",
+    "crates/polkavm-test-data",
 
     "crates/simplealloc",
 
@@ -39,10 +40,13 @@ polkavm-derive-impl       = { version = "0.19.0", path = "crates/polkavm-derive-
 polkavm-derive-impl-macro = { version = "0.19.0", path = "crates/polkavm-derive-impl-macro" }
 polkavm-linker            = { version = "0.19.0", path = "crates/polkavm-linker" }
 polkavm-linux-raw         = { version = "0.19.0", path = "crates/polkavm-linux-raw" }
+polkavm-test-data         = { version = "0.19.0", path = "crates/polkavm-test-data" }
 
 blake3 = { version = "1.5.4", default-features = false }
 clap = "4.4.6"
 diff = "0.1.13"
+dirs = "6.0.0"
+duct = "0.13.7"
 env_logger = { version = "0.10.0", default-features = false }
 gimli = { version = "0.31.1", default-features = false }
 hashbrown = { version = "0.14.1", default-features = false }

crates/polkavm-test-data/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "polkavm-test-data"
+version.workspace = true
+authors.workspace = true
+license.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+repository.workspace = true
+
+[build-dependencies]
+dirs = { workspace = true }
+duct = { workspace = true }
+polkavm-linker = { workspace = true }
+
+[lints]
+workspace = true

crates/polkavm-test-data/build.cmd

@@ -0,0 +1,8 @@
+@echo off
+
+cd %1
+
+cargo build -q --profile no-lto --bin test-blob -p test-blob --target ..\crates\polkavm-linker\riscv32emac-unknown-none-polkavm.json -Zunstable-options -Zbuild-std="core,alloc"
+cargo build -q --profile no-lto --bin test-blob -p test-blob --target ..\crates\polkavm-linker\riscv64emac-unknown-none-polkavm.json -Zunstable-options -Zbuild-std="core,alloc"
+cargo build -q --profile release --bin bench-pinky -p bench-pinky --target ..\crates\polkavm-linker\riscv32emac-unknown-none-polkavm.json -Zunstable-options -Zbuild-std="core,alloc"
+cargo build -q --profile release --bin bench-pinky -p bench-pinky --target ..\crates\polkavm-linker\riscv64emac-unknown-none-polkavm.json -Zunstable-options -Zbuild-std="core,alloc"

crates/polkavm-test-data/build.rs

@@ -0,0 +1,91 @@
+use std::path::PathBuf;
+use std::process::Command;
+
+fn build_unix(project: &str, profile: &str, target: &str) -> Result<(), ()> {
+    let mut path = PathBuf::new();
+
+    path.push(std::env!("CARGO_MANIFEST_DIR"));
+    path.push("..");
+    path.push("..");
+    path.push("guest-programs");
+
+    let guest_programs = String::from(path.as_path().to_str().unwrap());
+    // path.push(project);
+    let cargo_pwd = path.as_path();
+
+    let home = dirs::home_dir().unwrap();
+    let home_str = home.to_str().unwrap();
+
+    // Remaps for RUSTFLAGS:
+    let remaps = std::format!("--remap-path-prefix={guest_programs}= --remap-path-prefix={home_str}=~");
+    std::env::set_current_dir(cargo_pwd).unwrap();
+
+    let mut cmd = Command::new("cargo");
+    cmd.env_clear()
+        .env("PATH", std::env!("PATH"))
+        .env("RUSTFLAGS", remaps)
+        .env("RUSTUP_HOME", std::env!("RUSTUP_HOME"))
+        .arg("build")
+        .arg("-q")
+        .arg("--profile")
+        .arg(profile)
+        .arg("--bin")
+        .arg(project)
+        .arg("-p")
+        .arg(project)
+        .arg("--target")
+        .arg(target)
+        .arg("-Zunstable-options")
+        .arg("-Zbuild-std=core,alloc");
+
+    let res = cmd.output().unwrap();
+
+    if res.status.success() {
+        return Ok(());
+    }
+
+    let err = String::from_utf8_lossy(&res.stderr).to_string();
+    println!("cargo::error={err}");
+
+    Err(())
+}
+
+fn main() -> Result<(), ()> {
+    println!("cargo:rerun-if-env-changed=OUT_DIR");
+
+    if cfg!(windows) {
+        let mut build_cmd = PathBuf::new();
+
+        build_cmd.push(std::env!("CARGO_MANIFEST_DIR"));
+        build_cmd.push("build.cmd");
+
+        let mut path = PathBuf::new();
+
+        path.push(std::env!("CARGO_MANIFEST_DIR"));
+        path.push("..");
+        path.push("..");
+        path.push("guest-programs");
+
+        let mut cmd = Command::new("powershell");
+        cmd.env_clear().env("PATH", std::env!("PATH")).arg(build_cmd).arg(path);
+    } else {
+        let target_32 = polkavm_linker::target_json_32_path()
+            .unwrap()
+            .into_os_string()
+            .into_string()
+            .unwrap();
+
+        let target_64 = polkavm_linker::target_json_64_path()
+            .unwrap()
+            .into_os_string()
+            .into_string()
+            .unwrap();
+
+        build_unix("test-blob", "no-lto", &target_32)?;
+        build_unix("test-blob", "no-lto", &target_64)?;
+        build_unix("bench-pinky", "release", &target_32)?;
+        build_unix("bench-pinky", "release", &target_64)?;
+    }
+
+    Ok(())
+}

crates/polkavm-test-data/src/lib.rs

@@ -0,0 +1,4 @@
+pub const BENCH_PINKY_32: &[u8] = include_bytes!("../../../guest-programs/target/riscv64emac-unknown-none-polkavm/release/bench-pinky");
+pub const BENCH_PINKY_64: &[u8] = include_bytes!("../../../guest-programs/target/riscv64emac-unknown-none-polkavm/release/bench-pinky");
+pub const TEST_BLOB_32: &[u8] = include_bytes!("../../../guest-programs/target/riscv32emac-unknown-none-polkavm/no-lto/test-blob");
+pub const TEST_BLOB_64: &[u8] = include_bytes!("../../../guest-programs/target/riscv64emac-unknown-none-polkavm/no-lto/test-blob");

crates/polkavm/Cargo.toml

@@ -26,6 +26,7 @@ libc = { workspace = true }
 env_logger = { workspace = true }
 polkavm-common = { workspace = true, features = ["export-internals-for-testing"] }
 polkavm-linker = { workspace = true }
+polkavm-test-data = { workspace = true }
 image = { workspace = true, features = ["tga"] }
 ruzstd = { workspace = true }
 paste = { workspace = true }

crates/polkavm/src/tests.rs

@@ -1319,18 +1319,18 @@ fn decompress_zstd(mut bytes: &[u8]) -> Vec<u8> {
 
 static BLOB_MAP: Mutex<Option<BTreeMap<(bool, bool, &'static [u8]), ProgramBlob>>> = Mutex::new(None);
 
-fn get_blob(elf: &'static [u8]) -> ProgramBlob {
-    get_blob_impl(true, false, elf)
+fn get_blob(elf: &'static [u8], decompress: bool) -> ProgramBlob {
+    get_blob_impl(true, false, decompress, elf)
 }
 
-fn get_blob_impl(optimize: bool, strip: bool, elf: &'static [u8]) -> ProgramBlob {
+fn get_blob_impl(optimize: bool, strip: bool, decompress: bool, elf: &'static [u8]) -> ProgramBlob {
     let mut blob_map = BLOB_MAP.lock();
     let blob_map = blob_map.get_or_insert_with(BTreeMap::new);
     blob_map
         .entry((optimize, strip, elf))
         .or_insert_with(|| {
             // This is slow, so cache it.
-            let elf = decompress_zstd(elf);
+            let elf = if decompress { decompress_zstd(elf) } else { elf.to_vec() };
             let mut config = polkavm_linker::Config::default();
             config.set_optimize(optimize);
             config.set_strip(strip);
@@ -1355,7 +1355,7 @@ fn doom(config: Config, elf: &'static [u8]) {
     const DOOM_WAD: &[u8] = include_bytes!("../../../examples/doom/roms/doom1.wad");
 
     let _ = env_logger::try_init();
-    let blob = get_blob(elf);
+    let blob = get_blob(elf, true /* decompress */);
     let engine = Engine::new(&config).unwrap();
     let mut module_config = ModuleConfig::default();
     module_config.set_page_size(16 * 1024); // TODO: Also test with other page sizes.
@@ -1507,9 +1507,9 @@ fn pinky_impl(config: Config, is_64_bit: bool) {
 
     let _ = env_logger::try_init();
     let blob = if !is_64_bit {
-        get_blob(include_bytes!("../../../test-data/bench-pinky_32.elf.zst"))
+        get_blob(polkavm_test_data::BENCH_PINKY_32, false /* no decompress */)
     } else {
-        get_blob(include_bytes!("../../../test-data/bench-pinky_64.elf.zst"))
+        get_blob(polkavm_test_data::BENCH_PINKY_64, false /* no decompress */)
     };
 
     let engine = Engine::new(&config).unwrap();
@@ -1998,14 +1998,15 @@ struct TestInstance {
     instance: crate::Instance,
 }
 
-const TEST_BLOB_32_ELF_ZST: &[u8] = include_bytes!("../../../test-data/test-blob_32.elf.zst");
-const TEST_BLOB_64_ELF_ZST: &[u8] = include_bytes!("../../../test-data/test-blob_64.elf.zst");
-
 impl TestInstance {
     fn new(config: &Config, optimize: bool, is_64_bit: bool) -> Self {
         let _ = env_logger::try_init();
-        let blob = if is_64_bit { TEST_BLOB_64_ELF_ZST } else { TEST_BLOB_32_ELF_ZST };
-        let blob = get_blob_impl(optimize, is_64_bit, blob);
+        let blob = if is_64_bit {
+            polkavm_test_data::TEST_BLOB_64
+        } else {
+            polkavm_test_data::TEST_BLOB_32
+        };
+        let blob = get_blob_impl(optimize, is_64_bit, false, blob);
 
         let engine = Engine::new(config).unwrap();
         let module = Module::from_blob(&engine, &Default::default(), blob).unwrap();
@@ -2662,7 +2663,7 @@ fn trapping_preserves_all_registers_segfault(config: Config) {
 
 fn test_basic_debug_info(raw_blob: &'static [u8]) {
     let _ = env_logger::try_init();
-    let program = get_blob(raw_blob);
+    let program = get_blob(raw_blob, false /* no decompress */);
     let entry_point = program.exports().find(|export| export == "read_u32").unwrap().program_counter();
     let mut line_program = program.get_debug_line_program_at(entry_point).unwrap().unwrap();
     let info = line_program.run().unwrap().unwrap();
@@ -2686,13 +2687,13 @@ fn test_basic_debug_info(raw_blob: &'static [u8]) {
 #[ignore]
 #[test]
 fn test_basic_debug_info_32() {
-    test_basic_debug_info(TEST_BLOB_32_ELF_ZST);
+    test_basic_debug_info(polkavm_test_data::TEST_BLOB_32);
 }
 
 #[ignore]
 #[test]
 fn test_basic_debug_info_64() {
-    test_basic_debug_info(TEST_BLOB_64_ELF_ZST);
+    test_basic_debug_info(polkavm_test_data::TEST_BLOB_64);
 }
 
 #[test]
@@ -2759,7 +2760,7 @@ fn module_cache(_config: Config) {}
 #[cfg(feature = "module-cache")]
 fn module_cache(mut config: Config) {
     let _ = env_logger::try_init();
-    let blob = get_blob(TEST_BLOB_32_ELF_ZST);
+    let blob = get_blob(polkavm_test_data::TEST_BLOB_32, false /* no decompress */);
 
     config.set_worker_count(0);