From a32334f70e0aa89ed1010bafae4718035f86d677 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Wed, 11 Sep 2024 17:55:39 -0600 Subject: [PATCH 1/7] move forwarder to prover build step --- Makefile | 31 +- arbitrator/prover/Cargo.toml | 3 + .../forward/src/main.rs => prover/build.rs} | 76 +- arbitrator/prover/src/binary.rs | 12 +- arbitrator/prover/src/lib.rs | 5 +- arbitrator/prover/src/machine.rs | 44 +- arbitrator/prover/src/main.rs | 3 + arbitrator/prover/src/test.rs | 2 +- arbitrator/prover/test-cases/forward-test.wat | 32 - .../prover/test-cases/forward/forward.wat | 8 - .../prover/test-cases/forward/target.wat | 27 - arbitrator/stylus/src/test/mod.rs | 1 + arbitrator/tools/module_roots/src/main.rs | 1 + arbitrator/wasm-libraries/Cargo.lock | 668 +++++++++++++++++- arbitrator/wasm-libraries/Cargo.toml | 1 - arbitrator/wasm-libraries/forward/.gitignore | 1 - arbitrator/wasm-libraries/forward/Cargo.toml | 8 - arbitrator/wasm-testsuite/src/main.rs | 2 + validator/server_arb/machine.go | 2 +- 19 files changed, 735 insertions(+), 192 deletions(-) rename arbitrator/{wasm-libraries/forward/src/main.rs => prover/build.rs} (78%) delete mode 100644 arbitrator/prover/test-cases/forward-test.wat delete mode 100644 arbitrator/prover/test-cases/forward/forward.wat delete mode 100644 arbitrator/prover/test-cases/forward/target.wat delete mode 100644 arbitrator/wasm-libraries/forward/.gitignore delete mode 100644 arbitrator/wasm-libraries/forward/Cargo.toml diff --git a/Makefile b/Makefile index 0a71d64f12..9ecba1ce2d 100644 --- a/Makefile +++ b/Makefile @@ -60,7 +60,7 @@ replay_wasm=$(output_latest)/replay.wasm arb_brotli_files = $(wildcard arbitrator/brotli/src/*.* arbitrator/brotli/src/*/*.* arbitrator/brotli/*.toml arbitrator/brotli/*.rs) .make/cbrotli-lib .make/cbrotli-wasm arbitrator_generated_header=$(output_root)/include/arbitrator.h -arbitrator_wasm_libs=$(patsubst %, $(output_root)/machines/latest/%.wasm, forward wasi_stub host_io soft-float arbcompress user_host program_exec) +arbitrator_wasm_libs=$(patsubst %, $(output_root)/machines/latest/%.wasm, wasi_stub host_io soft-float arbcompress user_host program_exec) arbitrator_stylus_lib=$(output_root)/lib/libstylus.a prover_bin=$(output_root)/bin/prover arbitrator_jit=$(output_root)/bin/jit @@ -75,16 +75,12 @@ arbitrator_test_wasms=$(patsubst %.wat,%.wasm, $(arbitrator_tests_wat)) $(patsub arbitrator_tests_link_info = $(shell cat $(arbitrator_cases)/link.txt | xargs) arbitrator_tests_link_deps = $(patsubst %,$(arbitrator_cases)/%.wasm, $(arbitrator_tests_link_info)) -arbitrator_tests_forward_wats = $(wildcard $(arbitrator_cases)/forward/*.wat) -arbitrator_tests_forward_deps = $(arbitrator_tests_forward_wats:wat=wasm) - WASI_SYSROOT?=/opt/wasi-sdk/wasi-sysroot arbitrator_wasm_lib_flags=$(patsubst %, -l %, $(arbitrator_wasm_libs)) rust_arbutil_files = $(wildcard arbitrator/arbutil/src/*.* arbitrator/arbutil/src/*/*.* arbitrator/arbutil/*.toml arbitrator/caller-env/src/*.* arbitrator/caller-env/src/*/*.* arbitrator/caller-env/*.toml) .make/cbrotli-lib -prover_direct_includes = $(patsubst %,$(output_latest)/%.wasm, forward forward_stub) prover_dir = arbitrator/prover/ rust_prover_files = $(wildcard $(prover_dir)/src/*.* $(prover_dir)/src/*/*.* $(prover_dir)/*.toml $(prover_dir)/*.rs) $(rust_arbutil_files) $(prover_direct_includes) $(arb_brotli_files) @@ -92,12 +88,9 @@ wasm_lib = arbitrator/wasm-libraries wasm_lib_cargo = $(wasm_lib)/.cargo/config.toml wasm_lib_deps = $(wildcard $(wasm_lib)/$(1)/*.toml $(wasm_lib)/$(1)/src/*.rs $(wasm_lib)/$(1)/*.rs) $(wasm_lib_cargo) $(rust_arbutil_files) $(arb_brotli_files) .make/machines wasm_lib_go_abi = $(call wasm_lib_deps,go-abi) -wasm_lib_forward = $(call wasm_lib_deps,forward) wasm_lib_user_host_trait = $(call wasm_lib_deps,user-host-trait) wasm_lib_user_host = $(call wasm_lib_deps,user-host) $(wasm_lib_user_host_trait) -forward_dir = $(wasm_lib)/forward - stylus_files = $(wildcard $(stylus_dir)/*.toml $(stylus_dir)/src/*.rs) $(wasm_lib_user_host_trait) $(rust_prover_files) jit_dir = arbitrator/jit @@ -281,7 +274,6 @@ clean: rm -f arbitrator/wasm-libraries/soft-float/*.o rm -f arbitrator/wasm-libraries/soft-float/SoftFloat/build/Wasm-Clang/*.o rm -f arbitrator/wasm-libraries/soft-float/SoftFloat/build/Wasm-Clang/*.a - rm -f arbitrator/wasm-libraries/forward/*.wat rm -rf arbitrator/stylus/tests/*/target/ arbitrator/stylus/tests/*/*.wasm @rm -rf contracts/build contracts/cache solgen/go/ @rm -f .make/* @@ -399,7 +391,7 @@ $(output_latest)/host_io.wasm: $(DEP_PREDICATE) $(call wasm_lib_deps,host-io) $( cargo build --manifest-path arbitrator/wasm-libraries/Cargo.toml --release --target wasm32-wasi --config $(wasm_lib_cargo) --package host-io install arbitrator/wasm-libraries/$(wasm32_wasi)/host_io.wasm $@ -$(output_latest)/user_host.wasm: $(DEP_PREDICATE) $(wasm_lib_user_host) $(rust_prover_files) $(output_latest)/forward_stub.wasm .make/machines +$(output_latest)/user_host.wasm: $(DEP_PREDICATE) $(wasm_lib_user_host) $(rust_prover_files) .make/machines cargo build --manifest-path arbitrator/wasm-libraries/Cargo.toml --release --target wasm32-wasi --config $(wasm_lib_cargo) --package user-host install arbitrator/wasm-libraries/$(wasm32_wasi)/user_host.wasm $@ @@ -415,17 +407,9 @@ $(output_latest)/arbcompress.wasm: $(DEP_PREDICATE) $(call wasm_lib_deps,brotli) cargo build --manifest-path arbitrator/wasm-libraries/Cargo.toml --release --target wasm32-wasi --config $(wasm_lib_cargo) --package arbcompress install arbitrator/wasm-libraries/$(wasm32_wasi)/arbcompress.wasm $@ -$(output_latest)/forward.wasm: $(DEP_PREDICATE) $(wasm_lib_forward) .make/machines - cargo run --manifest-path $(forward_dir)/Cargo.toml -- --path $(forward_dir)/forward.wat - wat2wasm $(wasm_lib)/forward/forward.wat -o $@ - -$(output_latest)/forward_stub.wasm: $(DEP_PREDICATE) $(wasm_lib_forward) .make/machines - cargo run --manifest-path $(forward_dir)/Cargo.toml -- --path $(forward_dir)/forward_stub.wat --stub - wat2wasm $(wasm_lib)/forward/forward_stub.wat -o $@ - $(output_latest)/machine.wavm.br: $(DEP_PREDICATE) $(prover_bin) $(arbitrator_wasm_libs) $(replay_wasm) - $(prover_bin) $(replay_wasm) --generate-binaries $(output_latest) \ - $(patsubst %,-l $(output_latest)/%.wasm, forward soft-float wasi_stub host_io user_host arbcompress program_exec) + $(prover_bin) $(replay_wasm) --generate-binaries $(output_latest) --with-forwarder \ + $(patsubst %,-l $(output_latest)/%.wasm, soft-float wasi_stub host_io user_host arbcompress program_exec) $(arbitrator_cases)/%.wasm: $(arbitrator_cases)/%.wat wat2wasm $< -o $@ @@ -492,10 +476,10 @@ target/testdata/preimages.bin: python3 scripts/create-test-preimages.py $@ contracts/test/prover/proofs/rust-%.json: $(arbitrator_cases)/rust/$(wasm32_wasi)/%.wasm $(prover_bin) $(arbitrator_wasm_libs) target/testdata/preimages.bin - $(prover_bin) $< $(arbitrator_wasm_lib_flags) -o $@ -b --allow-hostapi --require-success --inbox-add-stub-headers --inbox $(arbitrator_cases)/rust/data/msg0.bin --inbox $(arbitrator_cases)/rust/data/msg1.bin --delayed-inbox $(arbitrator_cases)/rust/data/msg0.bin --delayed-inbox $(arbitrator_cases)/rust/data/msg1.bin --preimages target/testdata/preimages.bin + $(prover_bin) $< $(arbitrator_wasm_lib_flags) -o $@ -b --allow-hostapi --require-success --inbox-add-stub-headers --inbox $(arbitrator_cases)/rust/data/msg0.bin --inbox $(arbitrator_cases)/rust/data/msg1.bin --delayed-inbox $(arbitrator_cases)/rust/data/msg0.bin --delayed-inbox $(arbitrator_cases)/rust/data/msg1.bin --preimages target/testdata/preimages.bin --with-forwarder contracts/test/prover/proofs/go.json: $(arbitrator_cases)/go/testcase.wasm $(prover_bin) $(arbitrator_wasm_libs) target/testdata/preimages.bin $(arbitrator_tests_link_deps) $(arbitrator_cases)/user.wasm - $(prover_bin) $< $(arbitrator_wasm_lib_flags) -o $@ -b --require-success --preimages target/testdata/preimages.bin --stylus-modules $(arbitrator_cases)/user.wasm + $(prover_bin) $< $(arbitrator_wasm_lib_flags) -o $@ -b --require-success --preimages target/testdata/preimages.bin --stylus-modules $(arbitrator_cases)/user.wasm --with-forwarder # avoid testing user.wasm in onestepproofs. It can only run as stylus program. contracts/test/prover/proofs/user.json: @@ -508,9 +492,6 @@ contracts/test/prover/proofs/read-inboxmsg-10.json: contracts/test/prover/proofs/global-state.json: echo "[]" > $@ -contracts/test/prover/proofs/forward-test.json: $(arbitrator_cases)/forward-test.wasm $(arbitrator_tests_forward_deps) $(prover_bin) - $(prover_bin) $< -o $@ --allow-hostapi $(patsubst %,-l %, $(arbitrator_tests_forward_deps)) - contracts/test/prover/proofs/link.json: $(arbitrator_cases)/link.wasm $(arbitrator_tests_link_deps) $(prover_bin) $(prover_bin) $< -o $@ --allow-hostapi --stylus-modules $(arbitrator_tests_link_deps) --require-success diff --git a/arbitrator/prover/Cargo.toml b/arbitrator/prover/Cargo.toml index 5475647765..a4a69fcb05 100644 --- a/arbitrator/prover/Cargo.toml +++ b/arbitrator/prover/Cargo.toml @@ -47,6 +47,9 @@ enum-iterator = "2.0.1" criterion = { version = "0.5.0", features = ["html_reports"] } rand = "0.8.4" +[build-dependencies] +wasmer = { path = "../tools/wasmer/lib/api" } + [[bench]] name = "merkle_bench" harness = false diff --git a/arbitrator/wasm-libraries/forward/src/main.rs b/arbitrator/prover/build.rs similarity index 78% rename from arbitrator/wasm-libraries/forward/src/main.rs rename to arbitrator/prover/build.rs index 05a949e8aa..8191771c79 100644 --- a/arbitrator/wasm-libraries/forward/src/main.rs +++ b/arbitrator/prover/build.rs @@ -1,12 +1,8 @@ -// Copyright 2022-2024, Offchain Labs, Inc. -// For license information, see https://github.com/OffchainLabs/nitro/blob/master/LICENSE - -use eyre::Result; -use std::{fs::File, io::Write, path::PathBuf}; -use structopt::StructOpt; +use core::str; +use std::{env, fmt::Write, fs, path::Path}; /// order matters! -const HOSTIOS: [[&str; 3]; 42] = [ +pub const HOSTIOS: [[&str; 3]; 42] = [ ["read_args", "i32", ""], ["write_result", "i32 i32", ""], ["exit_early", "i32", ""], @@ -51,44 +47,16 @@ const HOSTIOS: [[&str; 3]; 42] = [ ["pay_for_memory_grow", "i32", ""], ]; -#[derive(StructOpt)] -#[structopt(name = "arbitrator-prover")] -struct Opts { - #[structopt(long)] - path: PathBuf, - #[structopt(long)] - stub: bool, -} - -fn main() -> Result<()> { - let opts = Opts::from_args(); - let file = &mut File::options() - .create(true) - .write(true) - .truncate(true) - .open(opts.path)?; - - match opts.stub { - true => forward_stub(file), - false => forward(file), - } -} - -fn forward(file: &mut File) -> Result<()> { +pub fn gen_forwarder(out_path: &Path) { + let mut wat = String::new(); macro_rules! wln { ($($text:tt)*) => { - writeln!(file, $($text)*)?; + writeln!(wat, $($text)*).unwrap(); }; } let s = " "; - wln!( - ";; Copyright 2022-2023, Offchain Labs, Inc.\n\ - ;; For license information, see https://github.com/OffchainLabs/nitro/blob/master/LICENSE\n\ - ;; This file is auto-generated.\n\ - \n\ - (module" - ); + wln!("(module"); macro_rules! group { ($list:expr, $kind:expr) => { @@ -102,9 +70,7 @@ fn forward(file: &mut File) -> Result<()> { for [name, ins, outs] in HOSTIOS { let params = group!(ins, "param"); let result = group!(outs, "result"); - wln!( - r#"{s}(import "user_host" "arbitrator_forward__{name}" (func ${name}{params}{result}))"# - ); + wln!(r#"{s}(import "user_host" "{name}" (func $_{name}{params}{result}))"#); } wln!(); @@ -139,20 +105,25 @@ fn forward(file: &mut File) -> Result<()> { } wln!( - "{s}{s}call ${name}\n\ + "{s}{s}call $_{name}\n\ {s}{s}call $check\n\ {s})" ); } wln!(")"); - Ok(()) + + let wasm = wasmer::wat2wasm(wat.as_bytes()).unwrap(); + + fs::write(out_path, wasm.as_ref()).unwrap(); } -fn forward_stub(file: &mut File) -> Result<()> { +pub fn gen_forwarder_stub(out_path: &Path) { + let mut wat = String::new(); + macro_rules! wln { ($($text:tt)*) => { - writeln!(file, $($text)*)?; + writeln!(wat, $($text)*).unwrap(); }; } let s = " "; @@ -202,5 +173,16 @@ fn forward_stub(file: &mut File) -> Result<()> { } wln!(")"); - Ok(()) + + let wasm = wasmer::wat2wasm(wat.as_bytes()).unwrap(); + + fs::write(out_path, wasm.as_ref()).unwrap(); +} + +fn main() { + let out_dir = env::var("OUT_DIR").unwrap(); + let forwarder_path = Path::new(&out_dir).join("forwarder.wasm"); + let forwarder_stub_path = Path::new(&out_dir).join("forwarder_stub.wasm"); + gen_forwarder(&forwarder_path); + gen_forwarder_stub(&forwarder_stub_path); } diff --git a/arbitrator/prover/src/binary.rs b/arbitrator/prover/src/binary.rs index aa5537476c..b1d819deb2 100644 --- a/arbitrator/prover/src/binary.rs +++ b/arbitrator/prover/src/binary.rs @@ -471,10 +471,14 @@ pub fn parse<'a>(input: &'a [u8], path: &'_ Path) -> Result> { let export = export.rsplit("__").take(1); exports.extend(export); } - for import in &binary.imports { - let name = import.name; - if exports.contains(name) { - bail!("binary exports an import with the same name {}", name.red()); + // forwarder is allowed to re-export the same name + // TODO: is there a real problem if import_name == export_name but module names don't? + if path != Path::new("forwarder") { + for import in &binary.imports { + let name = import.name; + if exports.contains(name) { + bail!("binary exports an import with the same name {}", name.red()); + } } } diff --git a/arbitrator/prover/src/lib.rs b/arbitrator/prover/src/lib.rs index 0f537478eb..20472691c1 100644 --- a/arbitrator/prover/src/lib.rs +++ b/arbitrator/prover/src/lib.rs @@ -70,9 +70,10 @@ pub unsafe extern "C" fn arbitrator_load_machine( library_paths: *const *const c_char, library_paths_size: isize, debug_chain: usize, + with_forwarder: bool, ) -> *mut Machine { let debug_chain = debug_chain != 0; - match arbitrator_load_machine_impl(binary_path, library_paths, library_paths_size, debug_chain) + match arbitrator_load_machine_impl(binary_path, library_paths, library_paths_size, debug_chain, with_forwarder) { Ok(mach) => mach, Err(err) => { @@ -87,6 +88,7 @@ unsafe fn arbitrator_load_machine_impl( library_paths: *const *const c_char, library_paths_size: isize, debug_chain: bool, + with_forwarder: bool, ) -> Result<*mut Machine> { let binary_path = cstr_to_string(binary_path); let binary_path = Path::new(&binary_path); @@ -107,6 +109,7 @@ unsafe fn arbitrator_load_machine_impl( Default::default(), Default::default(), get_empty_preimage_resolver(), + with_forwarder, )?; Ok(Box::into_raw(Box::new(mach))) } diff --git a/arbitrator/prover/src/machine.rs b/arbitrator/prover/src/machine.rs index 358876bd25..da3a4e3a79 100644 --- a/arbitrator/prover/src/machine.rs +++ b/arbitrator/prover/src/machine.rs @@ -334,7 +334,7 @@ lazy_static! { static ref USER_IMPORTS: HashMap = { let mut imports = HashMap::default(); - let forward = include_bytes!("../../../target/machines/latest/forward_stub.wasm"); + let forward = include_bytes!(concat!(env!("OUT_DIR"), "/forwarder_stub.wasm")); let forward = binary::parse(forward, Path::new("forward")).unwrap(); for (name, &(export, kind)) in &forward.exports { @@ -352,13 +352,12 @@ lazy_static! { } impl Module { - const FORWARDING_PREFIX: &'static str = "arbitrator_forward__"; - fn from_binary( bin: &WasmBinary, available_imports: &HashMap, floating_point_impls: &FloatingPointImpls, allow_hostapi: bool, + is_forwarder: bool, debug_funcs: bool, stylus_data: Option, ) -> Result { @@ -371,16 +370,11 @@ impl Module { for import in &bin.imports { let module = import.module; let have_ty = &bin.types[import.offset as usize]; - let (forward, import_name) = match import.name.strip_prefix(Module::FORWARDING_PREFIX) { - Some(name) => (true, name), - None => (false, import.name), - }; - let mut qualified_name = format!("{module}__{import_name}"); - qualified_name = qualified_name.replace(&['/', '.', '-'] as &[char], "_"); + let qualified_name = format!("{module}__{}", import.name); let func = if let Some(import) = available_imports.get(&qualified_name) { - let call = match forward { + let call = match is_forwarder { true => Opcode::CrossModuleForward, false => Opcode::CrossModuleCall, }; @@ -393,18 +387,18 @@ impl Module { Instruction::simple(Opcode::Return), ]; Function::new_from_wavm(wavm, import.ty.clone(), vec![]) - } else if let Ok((hostio, debug)) = host::get_impl(import.module, import_name) { + } else if let Ok((hostio, debug)) = host::get_impl(import.module, import.name) { ensure!( (debug && debug_funcs) || (!debug && allow_hostapi), "Host func {} in {} not enabled debug_funcs={debug_funcs} hostapi={allow_hostapi} debug={debug}", - import_name.red(), + import.name.red(), import.module.red(), ); hostio } else { bail!( "No such import {} in {} for {}", - import_name.red(), + import.name.red(), import.module.red(), bin_name.red() ) @@ -412,12 +406,12 @@ impl Module { ensure!( &func.ty == have_ty, "Import {} for {} has different function signature than export.\nexpected {} in {}\nbut have {}", - import_name.red(), bin_name.red(), func.ty.red(), module.red(), have_ty.red(), + import.name.red(), bin_name.red(), func.ty.red(), module.red(), have_ty.red(), ); func_type_idxs.push(import.offset); code.push(func); - host_call_hooks.push(Some((import.module.into(), import_name.into()))); + host_call_hooks.push(Some((import.module.into(), import.name.into()))); } func_type_idxs.extend(bin.functions.iter()); @@ -616,6 +610,7 @@ impl Module { &USER_IMPORTS, &HashMap::default(), false, + false, debug_funcs, stylus_data, ) @@ -1227,6 +1222,7 @@ impl Machine { global_state: GlobalState, inbox_contents: HashMap<(InboxIdentifier, u64), Vec>, preimage_resolver: PreimageResolver, + with_forwarder: bool, ) -> Result { let bin_source = file_bytes(binary_path)?; let bin = parse(&bin_source, binary_path) @@ -1252,6 +1248,7 @@ impl Machine { inbox_contents, preimage_resolver, None, + with_forwarder, ) } @@ -1281,6 +1278,7 @@ impl Machine { HashMap::default(), Arc::new(|_, _, _| panic!("tried to read preimage")), Some(stylus_data), + false, )?; let footprint: u32 = stylus_data.footprint.into(); @@ -1328,6 +1326,7 @@ impl Machine { inbox_contents: HashMap<(InboxIdentifier, u64), Vec>, preimage_resolver: PreimageResolver, stylus_data: Option, + with_forwarder: bool, ) -> Result { use ArbValueType::*; @@ -1337,6 +1336,17 @@ impl Machine { let mut floating_point_impls = HashMap::default(); let main_module_index = u32::try_from(modules.len() + libraries.len())?; + let forwarder_wasm = include_bytes!(concat!(env!("OUT_DIR"), "/forwarder.wasm")); + let mut libs_vec = vec![]; + + let libraries = if with_forwarder { + libs_vec.push(parse(forwarder_wasm.as_ref(), Path::new("forwarder"))?); + libs_vec.extend_from_slice(libraries); + &libs_vec + } else { + libraries + }; + // make the main module's exports available to libraries for (name, &(export, kind)) in &bin.exports { if kind == ExportKind::Func { @@ -1367,12 +1377,13 @@ impl Machine { } } - for lib in libraries { + for (index, lib) in libraries.iter().enumerate() { let module = Module::from_binary( lib, &available_imports, &floating_point_impls, true, + with_forwarder && (index == 0), debug_funcs, None, )?; @@ -1408,6 +1419,7 @@ impl Machine { &available_imports, &floating_point_impls, allow_hostapi_from_main, + false, debug_funcs, stylus_data, )?); diff --git a/arbitrator/prover/src/main.rs b/arbitrator/prover/src/main.rs index dba32e0e72..442b35992d 100644 --- a/arbitrator/prover/src/main.rs +++ b/arbitrator/prover/src/main.rs @@ -86,6 +86,8 @@ struct Opts { skip_until_host_io: bool, #[structopt(long)] max_steps: Option, + #[structopt(short, long)] + with_forwarder: bool, } fn file_with_stub_header(path: &Path, headerlength: usize) -> Result> { @@ -211,6 +213,7 @@ fn main() -> Result<()> { global_state, inbox_contents, preimage_resolver, + opts.with_forwarder, )?; for path in &opts.stylus_modules { diff --git a/arbitrator/prover/src/test.rs b/arbitrator/prover/src/test.rs index 97170441ff..52bf394dd5 100644 --- a/arbitrator/prover/src/test.rs +++ b/arbitrator/prover/src/test.rs @@ -57,7 +57,7 @@ pub fn reject_ambiguous_imports() { #[test] pub fn test_compress() -> Result<()> { - let data = include_bytes!("../../../target/machines/latest/forward_stub.wasm"); + let data = include_bytes!(concat!(env!("OUT_DIR"), "/forwarder_stub.wasm")); let mut last = vec![]; for dict in [Dictionary::Empty, Dictionary::StylusProgram] { diff --git a/arbitrator/prover/test-cases/forward-test.wat b/arbitrator/prover/test-cases/forward-test.wat deleted file mode 100644 index b9beff0d82..0000000000 --- a/arbitrator/prover/test-cases/forward-test.wat +++ /dev/null @@ -1,32 +0,0 @@ - -(module - (import "forward" "add" (func $add (param i32 i32) (result i32))) - (import "forward" "sub" (func $sub (param i32 i32) (result i32))) - (import "forward" "mul" (func $mul (param i32 i32) (result i32))) - (func $start - ;; this address will update each time a forwarded call is made - i32.const 0xa4b - i32.const 805 - i32.store - - i32.const 11 - i32.const 5 - call $sub - - i32.const 3 - i32.const -2 - call $mul - - call $add - (if - (then (unreachable))) - - ;; check that the address has changed - i32.const 0xa4b - i32.load - i32.const 808 - i32.ne - (if - (then (unreachable)))) - (start $start) - (memory 1)) diff --git a/arbitrator/prover/test-cases/forward/forward.wat b/arbitrator/prover/test-cases/forward/forward.wat deleted file mode 100644 index ff55953e62..0000000000 --- a/arbitrator/prover/test-cases/forward/forward.wat +++ /dev/null @@ -1,8 +0,0 @@ - -(module - (import "target" "arbitrator_forward__add" (func $add (param i32 i32) (result i32))) - (import "target" "arbitrator_forward__sub" (func $sub (param i32 i32) (result i32))) - (import "target" "arbitrator_forward__mul" (func $mul (param i32 i32) (result i32))) - (export "forward__add" (func $add)) - (export "forward__sub" (func $sub)) - (export "forward__mul" (func $mul))) diff --git a/arbitrator/prover/test-cases/forward/target.wat b/arbitrator/prover/test-cases/forward/target.wat deleted file mode 100644 index 0779eb753d..0000000000 --- a/arbitrator/prover/test-cases/forward/target.wat +++ /dev/null @@ -1,27 +0,0 @@ - -(module - (import "env" "wavm_caller_load8" (func $load (param i32) (result i32))) - (import "env" "wavm_caller_store8" (func $store (param i32 i32))) - (func (export "target__add") (param i32 i32) (result i32) - call $write_caller - local.get 0 - local.get 1 - i32.add) - (func (export "target__sub") (param i32 i32) (result i32) - call $write_caller - local.get 0 - local.get 1 - i32.sub) - (func (export "target__mul") (param i32 i32) (result i32) - call $write_caller - local.get 0 - local.get 1 - i32.mul) - (func $write_caller (export "write_caller") - ;; increment the value at address 0xa4b in the caller - i32.const 0xa4b - i32.const 0xa4b - call $load - i32.const 1 - i32.add - call $store)) diff --git a/arbitrator/stylus/src/test/mod.rs b/arbitrator/stylus/src/test/mod.rs index 00c9c62ae4..42c9446564 100644 --- a/arbitrator/stylus/src/test/mod.rs +++ b/arbitrator/stylus/src/test/mod.rs @@ -164,6 +164,7 @@ fn new_test_machine(path: &str, compile: &CompileConfig) -> Result { HashMap::default(), Arc::new(|_, _, _| panic!("tried to read preimage")), Some(stylus_data), + false, )?; mach.set_ink(u64::MAX); mach.set_stack(u32::MAX); diff --git a/arbitrator/tools/module_roots/src/main.rs b/arbitrator/tools/module_roots/src/main.rs index 32e4764847..fc22bed5fa 100644 --- a/arbitrator/tools/module_roots/src/main.rs +++ b/arbitrator/tools/module_roots/src/main.rs @@ -39,6 +39,7 @@ fn main() -> Result<()> { GlobalState::default(), HashMap::default(), Arc::new(|_, _, _| panic!("tried to read preimage")), + false, )?; let mut stylus = vec![]; diff --git a/arbitrator/wasm-libraries/Cargo.lock b/arbitrator/wasm-libraries/Cargo.lock index 7620ff538b..b234424f69 100644 --- a/arbitrator/wasm-libraries/Cargo.lock +++ b/arbitrator/wasm-libraries/Cargo.lock @@ -2,6 +2,21 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" +dependencies = [ + "gimli 0.29.0", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + [[package]] name = "ahash" version = "0.7.8" @@ -91,6 +106,21 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +[[package]] +name = "backtrace" +version = "0.3.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" +dependencies = [ + "addr2line", + "cc", + "cfg-if 1.0.0", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + [[package]] name = "bincode" version = "1.3.3" @@ -187,6 +217,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.7.0" @@ -203,6 +239,15 @@ dependencies = [ "rand_pcg", ] +[[package]] +name = "cc" +version = "1.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57b6a275aa2903740dc87da01c62040406b8812552e97129a63ea8850a17c6e6" +dependencies = [ + "shlex", +] + [[package]] name = "cfg-if" version = "0.1.10" @@ -236,6 +281,19 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" +[[package]] +name = "corosensei" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80128832c58ea9cbd041d2a759ec449224487b2c1e400453d99d244eead87a8e" +dependencies = [ + "autocfg", + "cfg-if 1.0.0", + "libc", + "scopeguard", + "windows-sys 0.33.0", +] + [[package]] name = "cpufeatures" version = "0.2.12" @@ -245,6 +303,123 @@ dependencies = [ "libc", ] +[[package]] +name = "cranelift-bforest" +version = "0.91.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a2ab4512dfd3a6f4be184403a195f76e81a8a9f9e6c898e19d2dc3ce20e0115" +dependencies = [ + "cranelift-entity", +] + +[[package]] +name = "cranelift-codegen" +version = "0.91.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98b022ed2a5913a38839dfbafe6cf135342661293b08049843362df4301261dc" +dependencies = [ + "arrayvec", + "bumpalo", + "cranelift-bforest", + "cranelift-codegen-meta", + "cranelift-codegen-shared", + "cranelift-egraph", + "cranelift-entity", + "cranelift-isle", + "gimli 0.26.2", + "log", + "regalloc2", + "smallvec", + "target-lexicon", +] + +[[package]] +name = "cranelift-codegen-meta" +version = "0.91.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "639307b45434ad112a98f8300c0f0ab085cbefcd767efcdef9ef19d4c0756e74" +dependencies = [ + "cranelift-codegen-shared", +] + +[[package]] +name = "cranelift-codegen-shared" +version = "0.91.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "278e52e29c53fcf32431ef08406c295699a70306d05a0715c5b1bf50e33a9ab7" + +[[package]] +name = "cranelift-egraph" +version = "0.91.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624b54323b06e675293939311943ba82d323bb340468ce1889be5da7932c8d73" +dependencies = [ + "cranelift-entity", + "fxhash", + "hashbrown 0.12.3", + "indexmap 1.9.3", + "log", + "smallvec", +] + +[[package]] +name = "cranelift-entity" +version = "0.91.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a59bcbca89c3f1b70b93ab3cbba5e5e0cbf3e63dadb23c7525cb142e21a9d4c" + +[[package]] +name = "cranelift-frontend" +version = "0.91.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d70abacb8cfef3dc8ff7e8836e9c1d70f7967dfdac824a4cd5e30223415aca6" +dependencies = [ + "cranelift-codegen", + "log", + "smallvec", + "target-lexicon", +] + +[[package]] +name = "cranelift-isle" +version = "0.91.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "393bc73c451830ff8dbb3a07f61843d6cb41a084f9996319917c0b291ed785bb" + +[[package]] +name = "crossbeam-deque" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" + [[package]] name = "crunchy" version = "0.2.2" @@ -330,6 +505,19 @@ dependencies = [ "syn 2.0.72", ] +[[package]] +name = "dashmap" +version = "5.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" +dependencies = [ + "cfg-if 1.0.0", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + [[package]] name = "derivative" version = "2.2.0" @@ -456,26 +644,33 @@ dependencies = [ "once_cell", ] +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + [[package]] name = "fnv" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "forward" -version = "0.1.0" -dependencies = [ - "eyre", - "structopt", -] - [[package]] name = "funty" version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -497,6 +692,23 @@ dependencies = [ "wasi", ] +[[package]] +name = "gimli" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d" +dependencies = [ + "fallible-iterator", + "indexmap 1.9.3", + "stable_deref_trait", +] + +[[package]] +name = "gimli" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" + [[package]] name = "hashbrown" version = "0.12.3" @@ -595,6 +807,15 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +[[package]] +name = "js-sys" +version = "0.3.70" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "keccak" version = "0.1.5" @@ -632,6 +853,12 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "log" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" + [[package]] name = "lru" version = "0.12.4" @@ -641,12 +868,57 @@ dependencies = [ "hashbrown 0.14.5", ] +[[package]] +name = "mach" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa" +dependencies = [ + "libc", +] + +[[package]] +name = "mach2" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b955cdeb2a02b9117f121ce63aa52d08ade45de53e48fe6a38b39c10f6f709" +dependencies = [ + "libc", +] + [[package]] name = "memchr" version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "memmap2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327" +dependencies = [ + "libc", +] + +[[package]] +name = "memmap2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d28bba84adfe6646737845bc5ebbfa2c08424eb1c37e94a1fd2a82adb56a872" +dependencies = [ + "libc", +] + +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + [[package]] name = "memory_units" version = "0.4.0" @@ -659,6 +931,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" +dependencies = [ + "adler", +] + [[package]] name = "more-asserts" version = "0.2.2" @@ -791,6 +1072,15 @@ dependencies = [ "syn 2.0.72", ] +[[package]] +name = "object" +version = "0.36.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.19.0" @@ -832,6 +1122,12 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "pin-project-lite" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" + [[package]] name = "proc-macro-crate" version = "3.1.0" @@ -912,6 +1208,7 @@ dependencies = [ "smallvec", "static_assertions", "structopt", + "wasmer", "wasmer-types", "wasmparser", "wat", @@ -976,6 +1273,26 @@ dependencies = [ "rand_core", ] +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.5.3" @@ -985,6 +1302,30 @@ dependencies = [ "bitflags 2.6.0", ] +[[package]] +name = "regalloc2" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300d4fbfb40c1c66a78ba3ddd41c1110247cf52f97b87d0f2fc9209bd49b030c" +dependencies = [ + "fxhash", + "log", + "slice-group-by", + "smallvec", +] + +[[package]] +name = "region" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6b6ebd13bc009aef9cd476c1310d49ac354d36e240cf1bd753290f3dc7199a7" +dependencies = [ + "bitflags 1.3.2", + "libc", + "mach2", + "windows-sys 0.52.0", +] + [[package]] name = "rend" version = "0.4.2" @@ -1075,6 +1416,12 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" +[[package]] +name = "self_cell" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d369a96f978623eb3dc28807c4852d6cc617fed53da5d3c400feff1ef34a714a" + [[package]] name = "semver" version = "1.0.23" @@ -1090,6 +1437,17 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-wasm-bindgen" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3b4c031cd0d9014307d82b8abf653c0290fbdaeb4c02d00c63cf52f728628bf" +dependencies = [ + "js-sys", + "serde", + "wasm-bindgen", +] + [[package]] name = "serde_derive" version = "1.0.204" @@ -1181,6 +1539,22 @@ dependencies = [ "keccak", ] +[[package]] +name = "shared-buffer" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6c99835bad52957e7aa241d3975ed17c1e5f8c92026377d117a606f36b84b16" +dependencies = [ + "bytes", + "memmap2 0.6.2", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "simdutf8" version = "0.1.4" @@ -1193,6 +1567,12 @@ version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" +[[package]] +name = "slice-group-by" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "826167069c09b99d56f31e9ae5c99049e932a98c9dc2dac47645b08dbbf76ba7" + [[package]] name = "smallvec" version = "1.13.2" @@ -1202,6 +1582,12 @@ dependencies = [ "serde", ] +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "static_assertions" version = "1.1.0" @@ -1348,6 +1734,37 @@ dependencies = [ "winnow", ] +[[package]] +name = "tracing" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.72", +] + +[[package]] +name = "tracing-core" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", +] + [[package]] name = "typenum" version = "1.17.0" @@ -1445,13 +1862,147 @@ dependencies = [ "wee_alloc", ] +[[package]] +name = "wasm-bindgen" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" +dependencies = [ + "cfg-if 1.0.0", + "once_cell", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.72", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.72", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" + [[package]] name = "wasm-encoder" -version = "0.215.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb56df3e06b8e6b77e37d2969a50ba51281029a9aeb3855e76b7f49b6418847" +checksum = "1ba64e81215916eaeb48fee292f29401d69235d62d8b8fd92a7b2844ec5ae5f7" +dependencies = [ + "leb128", +] + +[[package]] +name = "wasmer" +version = "4.2.8" dependencies = [ + "bytes", + "cfg-if 1.0.0", + "derivative", + "indexmap 1.9.3", + "js-sys", + "more-asserts", + "rustc-demangle", + "serde", + "serde-wasm-bindgen", + "shared-buffer", + "target-lexicon", + "thiserror", + "tracing", + "wasm-bindgen", + "wasmer-compiler", + "wasmer-compiler-cranelift", + "wasmer-derive", + "wasmer-types", + "wasmer-vm", + "wat", + "winapi", +] + +[[package]] +name = "wasmer-compiler" +version = "4.2.8" +dependencies = [ + "backtrace", + "bytes", + "cfg-if 1.0.0", + "enum-iterator 0.7.0", + "enumset", + "lazy_static", "leb128", + "memmap2 0.5.10", + "more-asserts", + "region", + "rkyv", + "self_cell", + "shared-buffer", + "smallvec", + "thiserror", + "wasmer-types", + "wasmer-vm", + "wasmparser", + "winapi", +] + +[[package]] +name = "wasmer-compiler-cranelift" +version = "4.2.8" +dependencies = [ + "cranelift-codegen", + "cranelift-entity", + "cranelift-frontend", + "gimli 0.26.2", + "more-asserts", + "rayon", + "smallvec", + "target-lexicon", + "tracing", + "wasmer-compiler", + "wasmer-types", +] + +[[package]] +name = "wasmer-derive" +version = "4.2.8" +dependencies = [ + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.109", ] [[package]] @@ -1468,6 +2019,32 @@ dependencies = [ "thiserror", ] +[[package]] +name = "wasmer-vm" +version = "4.2.8" +dependencies = [ + "backtrace", + "cc", + "cfg-if 1.0.0", + "corosensei", + "crossbeam-queue", + "dashmap", + "derivative", + "enum-iterator 0.7.0", + "fnv", + "indexmap 1.9.3", + "lazy_static", + "libc", + "mach", + "memoffset", + "more-asserts", + "region", + "scopeguard", + "thiserror", + "wasmer-types", + "winapi", +] + [[package]] name = "wasmparser" version = "0.121.2" @@ -1481,11 +2058,10 @@ dependencies = [ [[package]] name = "wast" -version = "215.0.0" +version = "64.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ff1d00d893593249e60720be04a7c1f42f1c4dc3806a2869f4e66ab61eb54cb" +checksum = "a259b226fd6910225aa7baeba82f9d9933b6d00f2ce1b49b80fa4214328237cc" dependencies = [ - "bumpalo", "leb128", "memchr", "unicode-width", @@ -1494,9 +2070,9 @@ dependencies = [ [[package]] name = "wat" -version = "1.215.0" +version = "1.0.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "670bf4d9c8cf76ae242d70ded47c546525b6dafaa6871f9bcb065344bf2b4e3d" +checksum = "53253d920ab413fca1c7dc2161d601c79b4fdf631d0ba51dd4343bf9b556c3f6" dependencies = [ "wast", ] @@ -1535,6 +2111,28 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-sys" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43dbb096663629518eb1dfa72d80243ca5a6aca764cae62a2df70af760a9be75" +dependencies = [ + "windows_aarch64_msvc 0.33.0", + "windows_i686_gnu 0.33.0", + "windows_i686_msvc 0.33.0", + "windows_x86_64_gnu 0.33.0", + "windows_x86_64_msvc 0.33.0", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-targets" version = "0.52.6" @@ -1542,13 +2140,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_x86_64_msvc 0.52.6", ] [[package]] @@ -1557,12 +2155,24 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_msvc" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd761fd3eb9ab8cc1ed81e56e567f02dd82c4c837e48ac3b2181b9ffc5060807" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_i686_gnu" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab0cf703a96bab2dc0c02c0fa748491294bf9b7feb27e1f4f96340f208ada0e" + [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -1575,12 +2185,24 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_msvc" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cfdbe89cc9ad7ce618ba34abc34bbb6c36d99e96cae2245b7943cd75ee773d0" + [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_x86_64_gnu" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4dd9b0c0e9ece7bb22e84d70d01b71c6d6248b81a3c60d11869451b4cb24784" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" @@ -1593,6 +2215,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_msvc" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff1e4aa646495048ec7f3ffddc411e1d829c026a2ec62b39da15c1055e406eaa" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" diff --git a/arbitrator/wasm-libraries/Cargo.toml b/arbitrator/wasm-libraries/Cargo.toml index 837df8f4da..2fd585ee7d 100644 --- a/arbitrator/wasm-libraries/Cargo.toml +++ b/arbitrator/wasm-libraries/Cargo.toml @@ -7,6 +7,5 @@ members = [ "user-host-trait", "user-test", "program-exec", - "forward", ] resolver = "2" diff --git a/arbitrator/wasm-libraries/forward/.gitignore b/arbitrator/wasm-libraries/forward/.gitignore deleted file mode 100644 index 40da2042b7..0000000000 --- a/arbitrator/wasm-libraries/forward/.gitignore +++ /dev/null @@ -1 +0,0 @@ -**.wat diff --git a/arbitrator/wasm-libraries/forward/Cargo.toml b/arbitrator/wasm-libraries/forward/Cargo.toml deleted file mode 100644 index 73ed9d8827..0000000000 --- a/arbitrator/wasm-libraries/forward/Cargo.toml +++ /dev/null @@ -1,8 +0,0 @@ -[package] -name = "forward" -version = "0.1.0" -edition = "2021" - -[dependencies] -eyre = "0.6.5" -structopt = "0.3.26" diff --git a/arbitrator/wasm-testsuite/src/main.rs b/arbitrator/wasm-testsuite/src/main.rs index 2144dcf992..cb861e3a35 100644 --- a/arbitrator/wasm-testsuite/src/main.rs +++ b/arbitrator/wasm-testsuite/src/main.rs @@ -343,6 +343,7 @@ fn main() -> eyre::Result<()> { GlobalState::default(), HashMap::default(), machine::get_empty_preimage_resolver(), + false, ); if let Err(error) = &mech { @@ -429,6 +430,7 @@ fn main() -> eyre::Result<()> { GlobalState::default(), HashMap::default(), machine::get_empty_preimage_resolver(), + false, ) .expect_err(&format!("failed to reject invalid module {}", filename)); } diff --git a/validator/server_arb/machine.go b/validator/server_arb/machine.go index adca9695e2..e77e071d70 100644 --- a/validator/server_arb/machine.go +++ b/validator/server_arb/machine.go @@ -92,7 +92,7 @@ func LoadSimpleMachine(wasm string, libraries []string, debugChain bool) (*Arbit cWasm := C.CString(wasm) cLibraries := CreateCStringList(libraries) debug := usize(arbmath.BoolToUint32(debugChain)) - mach := C.arbitrator_load_machine(cWasm, cLibraries, C.long(len(libraries)), debug) + mach := C.arbitrator_load_machine(cWasm, cLibraries, C.long(len(libraries)), debug, false) C.free(unsafe.Pointer(cWasm)) FreeCStringList(cLibraries, len(libraries)) if mach == nil { From ed18d13032542989318a664c3c85d1c33756eca7 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Thu, 5 Sep 2024 19:35:58 -0600 Subject: [PATCH 2/7] remove forwarder_stub --- arbitrator/arbutil/src/hostios.rs | 64 +++++++++++++++ arbitrator/arbutil/src/lib.rs | 1 + arbitrator/prover/Cargo.toml | 1 + arbitrator/prover/build.rs | 131 ++++-------------------------- arbitrator/prover/src/machine.rs | 118 ++++++++++++--------------- arbitrator/prover/src/test.rs | 2 +- arbitrator/prover/src/value.rs | 12 ++- 7 files changed, 145 insertions(+), 184 deletions(-) create mode 100644 arbitrator/arbutil/src/hostios.rs diff --git a/arbitrator/arbutil/src/hostios.rs b/arbitrator/arbutil/src/hostios.rs new file mode 100644 index 0000000000..05bcb94f73 --- /dev/null +++ b/arbitrator/arbutil/src/hostios.rs @@ -0,0 +1,64 @@ +use std::fmt::Display; + +pub enum ParamType { + I32, + I64, +} + +impl Display for ParamType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + use ParamType::*; + match self { + I32 => write!(f, "i32"), + I64 => write!(f, "i64"), + } + } +} + +use ParamType::*; + +/// order matters! +pub const HOSTIOS: [(&str, &[ParamType], &[ParamType]); 42] = [ + ("read_args", &[I32], &[]), + ("write_result", &[I32, I32], &[]), + ("exit_early", &[I32], &[]), + ("storage_load_bytes32", &[I32, I32], &[]), + ("storage_cache_bytes32", &[I32, I32], &[]), + ("storage_flush_cache", &[I32], &[]), + ("transient_load_bytes32", &[I32, I32], &[]), + ("transient_store_bytes32", &[I32, I32], &[]), + ("call_contract", &[I32, I32, I32, I32, I64, I32], &[I32]), + ("delegate_call_contract", &[I32, I32, I32, I64, I32], &[I32]), + ("static_call_contract", &[I32, I32, I32, I64, I32], &[I32]), + ("create1", &[I32, I32, I32, I32, I32], &[]), + ("create2", &[I32, I32, I32, I32, I32, I32], &[]), + ("read_return_data", &[I32, I32, I32], &[I32]), + ("return_data_size", &[], &[I32]), + ("emit_log", &[I32, I32, I32], &[]), + ("account_balance", &[I32, I32], &[]), + ("account_code", &[I32, I32, I32, I32], &[I32]), + ("account_code_size", &[I32], &[I32]), + ("account_codehash", &[I32, I32], &[]), + ("evm_gas_left", &[], &[I64]), + ("evm_ink_left", &[], &[I64]), + ("block_basefee", &[I32], &[]), + ("chainid", &[], &[I64]), + ("block_coinbase", &[I32], &[]), + ("block_gas_limit", &[], &[I64]), + ("block_number", &[], &[I64]), + ("block_timestamp", &[], &[I64]), + ("contract_address", &[I32], &[]), + ("math_div", &[I32, I32], &[]), + ("math_mod", &[I32, I32], &[]), + ("math_pow", &[I32, I32], &[]), + ("math_add_mod", &[I32, I32, I32], &[]), + ("math_mul_mod", &[I32, I32, I32], &[]), + ("msg_reentrant", &[], &[I32]), + ("msg_sender", &[I32], &[]), + ("msg_value", &[I32], &[]), + ("native_keccak256", &[I32, I32, I32], &[]), + ("tx_gas_price", &[I32], &[]), + ("tx_ink_price", &[], &[I32]), + ("tx_origin", &[I32], &[]), + ("pay_for_memory_grow", &[I32], &[]), +]; diff --git a/arbitrator/arbutil/src/lib.rs b/arbitrator/arbutil/src/lib.rs index 9c48a9fefc..8d8c1d0fca 100644 --- a/arbitrator/arbutil/src/lib.rs +++ b/arbitrator/arbutil/src/lib.rs @@ -6,6 +6,7 @@ pub mod color; pub mod crypto; pub mod evm; pub mod format; +pub mod hostios; pub mod math; pub mod operator; pub mod pricing; diff --git a/arbitrator/prover/Cargo.toml b/arbitrator/prover/Cargo.toml index a4a69fcb05..a634dba19e 100644 --- a/arbitrator/prover/Cargo.toml +++ b/arbitrator/prover/Cargo.toml @@ -49,6 +49,7 @@ rand = "0.8.4" [build-dependencies] wasmer = { path = "../tools/wasmer/lib/api" } +arbutil = { path = "../arbutil/" } [[bench]] name = "merkle_bench" diff --git a/arbitrator/prover/build.rs b/arbitrator/prover/build.rs index 8191771c79..db0560b2c5 100644 --- a/arbitrator/prover/build.rs +++ b/arbitrator/prover/build.rs @@ -1,52 +1,6 @@ -use core::str; +use arbutil::hostios::HOSTIOS; use std::{env, fmt::Write, fs, path::Path}; -/// order matters! -pub const HOSTIOS: [[&str; 3]; 42] = [ - ["read_args", "i32", ""], - ["write_result", "i32 i32", ""], - ["exit_early", "i32", ""], - ["storage_load_bytes32", "i32 i32", ""], - ["storage_cache_bytes32", "i32 i32", ""], - ["storage_flush_cache", "i32", ""], - ["transient_load_bytes32", "i32 i32", ""], - ["transient_store_bytes32", "i32 i32", ""], - ["call_contract", "i32 i32 i32 i32 i64 i32", "i32"], - ["delegate_call_contract", "i32 i32 i32 i64 i32", "i32"], - ["static_call_contract", "i32 i32 i32 i64 i32", "i32"], - ["create1", "i32 i32 i32 i32 i32", ""], - ["create2", "i32 i32 i32 i32 i32 i32", ""], - ["read_return_data", "i32 i32 i32", "i32"], - ["return_data_size", "", "i32"], - ["emit_log", "i32 i32 i32", ""], - ["account_balance", "i32 i32", ""], - ["account_code", "i32 i32 i32 i32", "i32"], - ["account_code_size", "i32", "i32"], - ["account_codehash", "i32 i32", ""], - ["evm_gas_left", "", "i64"], - ["evm_ink_left", "", "i64"], - ["block_basefee", "i32", ""], - ["chainid", "", "i64"], - ["block_coinbase", "i32", ""], - ["block_gas_limit", "", "i64"], - ["block_number", "", "i64"], - ["block_timestamp", "", "i64"], - ["contract_address", "i32", ""], - ["math_div", "i32 i32", ""], - ["math_mod", "i32 i32", ""], - ["math_pow", "i32 i32", ""], - ["math_add_mod", "i32 i32 i32", ""], - ["math_mul_mod", "i32 i32 i32", ""], - ["msg_reentrant", "", "i32"], - ["msg_sender", "i32", ""], - ["msg_value", "i32", ""], - ["native_keccak256", "i32 i32 i32", ""], - ["tx_gas_price", "i32", ""], - ["tx_ink_price", "", "i32"], - ["tx_origin", "i32", ""], - ["pay_for_memory_grow", "i32", ""], -]; - pub fn gen_forwarder(out_path: &Path) { let mut wat = String::new(); macro_rules! wln { @@ -61,13 +15,23 @@ pub fn gen_forwarder(out_path: &Path) { macro_rules! group { ($list:expr, $kind:expr) => { (!$list.is_empty()) - .then(|| format!(" ({} {})", $kind, $list)) + .then(|| { + format!( + " ({} {})", + $kind, + $list + .iter() + .map(|x| x.to_string()) + .collect::>() + .join(" ") + ) + }) .unwrap_or_default() }; } wln!("{s};; symbols to re-export"); - for [name, ins, outs] in HOSTIOS { + for (name, ins, outs) in HOSTIOS { let params = group!(ins, "param"); let result = group!(outs, "result"); wln!(r#"{s}(import "user_host" "{name}" (func $_{name}{params}{result}))"#); @@ -94,13 +58,12 @@ pub fn gen_forwarder(out_path: &Path) { ); wln!("{s};; user linkage"); - for [name, ins, outs] in HOSTIOS { + for (name, ins, outs) in HOSTIOS { let params = group!(ins, "param"); let result = group!(outs, "result"); wln!("{s}(func (export \"vm_hooks__{name}\"){params}{result}"); - let gets = (1 + ins.len()) / 4; - for i in 0..gets { + for i in 0..ins.len() { wln!("{s}{s}local.get {i}"); } @@ -112,67 +75,7 @@ pub fn gen_forwarder(out_path: &Path) { } wln!(")"); - - let wasm = wasmer::wat2wasm(wat.as_bytes()).unwrap(); - - fs::write(out_path, wasm.as_ref()).unwrap(); -} - -pub fn gen_forwarder_stub(out_path: &Path) { - let mut wat = String::new(); - - macro_rules! wln { - ($($text:tt)*) => { - writeln!(wat, $($text)*).unwrap(); - }; - } - let s = " "; - - wln!( - ";; Copyright 2022-2023, Offchain Labs, Inc.\n\ - ;; For license information, see https://github.com/OffchainLabs/nitro/blob/master/LICENSE\n\ - ;; This file is auto-generated.\n\ - \n\ - (module" - ); - - macro_rules! group { - ($list:expr, $kind:expr) => { - (!$list.is_empty()) - .then(|| format!(" ({} {})", $kind, $list)) - .unwrap_or_default() - }; - } - - wln!("{s};; stubs for the symbols we re-export"); - for [name, ins, outs] in HOSTIOS { - let params = group!(ins, "param"); - let result = group!(outs, "result"); - wln!("{s}(func ${name}{params}{result} unreachable)"); - } - wln!(); - - wln!("{s};; reserved offsets for future user_host imports"); - for i in HOSTIOS.len()..512 { - wln!("{s}(func $reserved_{i} unreachable)"); - } - wln!(); - - wln!( - "{s};; allows user_host to request a trap\n\ - {s}(global $trap (mut i32) (i32.const 0))\n\ - {s}(func $check unreachable)\n\ - {s}(func (export \"forward__set_trap\") unreachable)" - ); - - wln!("{s};; user linkage"); - for [name, ins, outs] in HOSTIOS { - let params = group!(ins, "param"); - let result = group!(outs, "result"); - wln!("{s}(func (export \"vm_hooks__{name}\"){params}{result} unreachable)"); - } - - wln!(")"); + eprintln!("{}", &wat); let wasm = wasmer::wat2wasm(wat.as_bytes()).unwrap(); @@ -182,7 +85,5 @@ pub fn gen_forwarder_stub(out_path: &Path) { fn main() { let out_dir = env::var("OUT_DIR").unwrap(); let forwarder_path = Path::new(&out_dir).join("forwarder.wasm"); - let forwarder_stub_path = Path::new(&out_dir).join("forwarder_stub.wasm"); gen_forwarder(&forwarder_path); - gen_forwarder_stub(&forwarder_stub_path); } diff --git a/arbitrator/prover/src/machine.rs b/arbitrator/prover/src/machine.rs index da3a4e3a79..c045fc72c9 100644 --- a/arbitrator/prover/src/machine.rs +++ b/arbitrator/prover/src/machine.rs @@ -19,7 +19,7 @@ use crate::{ IBinOpType, IRelOpType, IUnOpType, Instruction, Opcode, }, }; -use arbutil::{crypto, math, Bytes32, Color, DebugColor, PreimageType}; +use arbutil::{crypto, hostios::HOSTIOS, math, Bytes32, Color, DebugColor, PreimageType}; use brotli::Dictionary; #[cfg(feature = "native")] use c_kzg::BYTES_PER_BLOB; @@ -330,22 +330,17 @@ pub struct Module { pub(crate) extra_hash: Arc, } +#[cfg(feature = "native")] +static FORWARDER_WASM: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/forwarder.wasm")); + lazy_static! { static ref USER_IMPORTS: HashMap = { let mut imports = HashMap::default(); - let forward = include_bytes!(concat!(env!("OUT_DIR"), "/forwarder_stub.wasm")); - let forward = binary::parse(forward, Path::new("forward")).unwrap(); - - for (name, &(export, kind)) in &forward.exports { - if kind == ExportKind::Func { - let ty = match forward.get_function(FunctionIndex::from_u32(export)) { - Ok(ty) => ty, - Err(error) => panic!("failed to read export {name}: {error:?}"), - }; - let import = AvailableImport::new(ty, 1, export); - imports.insert(name.to_owned(), import); - } + // 0-513 are internal + for (index, (name, ins, outs)) in HOSTIOS.iter().enumerate() { + let import = AvailableImport::new(FunctionType::new(ins.iter().map(|x|x.into()).collect::>(), outs.iter().map(|x|x.into()).collect::>()), 1, (index + 514).try_into().unwrap()); + imports.insert(format!("vm_hooks__{name}"), import); } imports }; @@ -1334,36 +1329,19 @@ impl Machine { let mut modules = vec![Module::default()]; let mut available_imports = HashMap::default(); let mut floating_point_impls = HashMap::default(); - let main_module_index = u32::try_from(modules.len() + libraries.len())?; - let forwarder_wasm = include_bytes!(concat!(env!("OUT_DIR"), "/forwarder.wasm")); - let mut libs_vec = vec![]; + let forwarder = if with_forwarder { + #[cfg(not(feature = "native"))] + bail!("forwarder not supported without native"); - let libraries = if with_forwarder { - libs_vec.push(parse(forwarder_wasm.as_ref(), Path::new("forwarder"))?); - libs_vec.extend_from_slice(libraries); - &libs_vec + #[cfg(feature = "native")] + Some(parse(FORWARDER_WASM, Path::new("forwarder"))?) } else { - libraries + None }; - // make the main module's exports available to libraries - for (name, &(export, kind)) in &bin.exports { - if kind == ExportKind::Func { - let index: usize = export.try_into()?; - if let Some(index) = index.checked_sub(bin.imports.len()) { - let ty: usize = bin.functions[index].try_into()?; - let ty = bin.types[ty].clone(); - available_imports.insert( - format!("env__wavm_guest_call__{name}"), - AvailableImport::new(ty, main_module_index, export), - ); - } - } - } - // collect all the library exports in advance so they can use each other's - for (index, lib) in libraries.iter().enumerate() { + for (index, lib) in forwarder.iter().chain(libraries.iter()).enumerate() { let module = 1 + index as u32; // off by one due to the entry point for (name, &(export, kind)) in &lib.exports { if kind == ExportKind::Func { @@ -1371,49 +1349,55 @@ impl Machine { Ok(ty) => ty, Err(error) => bail!("failed to read export {name}: {error}"), }; - let import = AvailableImport::new(ty, module, export); + let import = AvailableImport::new(ty.clone(), module, export); available_imports.insert(name.to_owned(), import); + if let Ok(op) = name.parse::() { + let mut sig = op.signature(); + // wavm codegen takes care of effecting this type change at callsites + for ty in sig.inputs.iter_mut().chain(sig.outputs.iter_mut()) { + if *ty == F32 { + *ty = I32; + } else if *ty == F64 { + *ty = I64; + } + } + ensure!( + ty == sig, + "Wrong type for floating point impl {} expecting {} but got {}", + name.red(), + sig.red(), + ty.red() + ); + floating_point_impls.insert(op, (module, export)); + } } } } - for (index, lib) in libraries.iter().enumerate() { - let module = Module::from_binary( + if let Some(lib) = forwarder { + modules.push(Module::from_binary( + &lib, + &available_imports, + &floating_point_impls, + true, + true, + debug_funcs, + None, + )?); + } + + for lib in libraries.iter() { + modules.push(Module::from_binary( lib, &available_imports, &floating_point_impls, true, - with_forwarder && (index == 0), + false, debug_funcs, None, - )?; - for (name, &func) in &*module.func_exports { - let ty = module.func_types[func as usize].clone(); - if let Ok(op) = name.parse::() { - let mut sig = op.signature(); - // wavm codegen takes care of effecting this type change at callsites - for ty in sig.inputs.iter_mut().chain(sig.outputs.iter_mut()) { - if *ty == F32 { - *ty = I32; - } else if *ty == F64 { - *ty = I64; - } - } - ensure!( - ty == sig, - "Wrong type for floating point impl {} expecting {} but got {}", - name.red(), - sig.red(), - ty.red() - ); - floating_point_impls.insert(op, (modules.len() as u32, func)); - } - } - modules.push(module); + )?); } - // Shouldn't be necessary, but to be safe, don't allow the main binary to import its own guest calls - available_imports.retain(|_, i| i.module as usize != modules.len()); modules.push(Module::from_binary( &bin, &available_imports, diff --git a/arbitrator/prover/src/test.rs b/arbitrator/prover/src/test.rs index 52bf394dd5..2f32aff012 100644 --- a/arbitrator/prover/src/test.rs +++ b/arbitrator/prover/src/test.rs @@ -57,7 +57,7 @@ pub fn reject_ambiguous_imports() { #[test] pub fn test_compress() -> Result<()> { - let data = include_bytes!(concat!(env!("OUT_DIR"), "/forwarder_stub.wasm")); + let data = include_bytes!(concat!(env!("OUT_DIR"), "/forwarder.wasm")); let mut last = vec![]; for dict in [Dictionary::Empty, Dictionary::StylusProgram] { diff --git a/arbitrator/prover/src/value.rs b/arbitrator/prover/src/value.rs index 6afffdf7a0..f2f54404f0 100644 --- a/arbitrator/prover/src/value.rs +++ b/arbitrator/prover/src/value.rs @@ -2,7 +2,7 @@ // For license information, see https://github.com/nitro/blob/master/LICENSE use crate::binary::FloatType; -use arbutil::{Bytes32, Color}; +use arbutil::{hostios::ParamType, Bytes32, Color}; use digest::Digest; use eyre::{bail, ErrReport, Result}; use serde::{Deserialize, Serialize}; @@ -77,6 +77,16 @@ impl From for ValType { } } +impl From<&ParamType> for ArbValueType { + fn from(ty: &ParamType) -> Self { + use ParamType as V; + match ty { + V::I32 => Self::I32, + V::I64 => Self::I64, + } + } +} + #[cfg(feature = "native")] pub fn parser_type(ty: &wasmer::Type) -> wasmer::wasmparser::ValType { match ty { From 8b719d0d0e7a948849f6735e597eb7d04b699594 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Wed, 11 Sep 2024 18:05:27 -0600 Subject: [PATCH 3/7] Merge remote-tracking branch 'origin/master' into forwarder-refactor --- arbnode/inbox_test.go | 4 +- arbos/programs/native.go | 70 +++++++++++------ arbos/programs/wasmstorehelper.go | 3 +- cmd/nitro/init.go | 6 +- cmd/nitro/init_test.go | 10 +++ cmd/nitro/nitro.go | 2 +- execution/gethexec/executionengine.go | 40 ++++++---- execution/gethexec/node.go | 46 ++++++++++-- go-ethereum | 2 +- staker/block_validator.go | 10 ++- staker/challenge_manager.go | 3 +- staker/stateless_block_validator.go | 5 +- system_tests/common_test.go | 52 +++++++------ system_tests/forwarder_test.go | 4 +- system_tests/program_test.go | 92 +++++++++++++++++++++-- system_tests/recreatestate_rpc_test.go | 18 ++--- system_tests/retryable_test.go | 2 +- system_tests/validation_mock_test.go | 6 +- validator/client/redis/producer.go | 9 ++- validator/client/validation_client.go | 15 ++-- validator/interface.go | 4 +- validator/server_api/json.go | 8 +- validator/server_arb/validator_spawner.go | 5 +- validator/server_jit/spawner.go | 5 +- validator/validation_entry.go | 4 +- validator/valnode/redis/consumer.go | 11 ++- validator/valnode/validation_api.go | 4 +- 27 files changed, 310 insertions(+), 130 deletions(-) diff --git a/arbnode/inbox_test.go b/arbnode/inbox_test.go index 1c46c593b9..d579b7c278 100644 --- a/arbnode/inbox_test.go +++ b/arbnode/inbox_test.go @@ -72,7 +72,9 @@ func NewTransactionStreamerForTest(t *testing.T, ownerAddress common.Address) (* if err != nil { Fail(t, err) } - if err := execEngine.Initialize(gethexec.DefaultCachingConfig.StylusLRUCache, &gethexec.DefaultStylusTargetConfig); err != nil { + stylusTargetConfig := &gethexec.DefaultStylusTargetConfig + Require(t, stylusTargetConfig.Validate()) // pre-processes config (i.a. parses wasmTargets) + if err := execEngine.Initialize(gethexec.DefaultCachingConfig.StylusLRUCache, stylusTargetConfig); err != nil { Fail(t, err) } execSeq := &execClientWrapper{execEngine, t} diff --git a/arbos/programs/native.go b/arbos/programs/native.go index 9316e8f230..377e25a31e 100644 --- a/arbos/programs/native.go +++ b/arbos/programs/native.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/offchainlabs/nitro/arbos/burn" "github.com/offchainlabs/nitro/arbos/util" @@ -71,7 +72,7 @@ func activateProgramInternal( version uint16, debug bool, gasLeft *uint64, -) (*activationInfo, map[rawdb.Target][]byte, error) { +) (*activationInfo, map[ethdb.WasmTarget][]byte, error) { output := &rustBytes{} moduleHash := &bytes32{} stylusData := &C.StylusData{} @@ -99,25 +100,50 @@ func activateProgramInternal( } return nil, nil, err } - target := rawdb.LocalTarget() - status_asm := C.stylus_compile( - goSlice(wasm), - u16(version), - cbool(debug), - goSlice([]byte(target)), - output, - ) - asm := output.intoBytes() - if status_asm != 0 { - return nil, nil, fmt.Errorf("%w: %s", ErrProgramActivation, string(asm)) + targets := db.Database().WasmTargets() + type result struct { + target ethdb.WasmTarget + asm []byte + err error } - asmMap := map[rawdb.Target][]byte{ - rawdb.TargetWavm: module, - target: asm, + results := make(chan result, len(targets)) + for _, target := range targets { + if target == rawdb.TargetWavm { + results <- result{target, module, nil} + } else { + target := target + go func() { + output := &rustBytes{} + status_asm := C.stylus_compile( + goSlice(wasm), + u16(version), + cbool(debug), + goSlice([]byte(target)), + output, + ) + asm := output.intoBytes() + if status_asm != 0 { + results <- result{target, nil, fmt.Errorf("%w: %s", ErrProgramActivation, string(asm))} + return + } + results <- result{target, asm, nil} + }() + } + } + asmMap := make(map[ethdb.WasmTarget][]byte, len(targets)) + for range targets { + res := <-results + if res.err != nil { + err = errors.Join(res.err, err) + } else { + asmMap[res.target] = res.asm + } + } + if err != nil { + return nil, nil, fmt.Errorf("compilation failed for one or more targets: %w", err) } hash := moduleHash.toHash() - info := &activationInfo{ moduleHash: hash, initGas: uint16(stylusData.init_cost), @@ -171,7 +197,7 @@ func getLocalAsm(statedb vm.StateDB, moduleHash common.Hash, addressForLogging c } asm, exists := asmMap[localTarget] if !exists { - var availableTargets []rawdb.Target + var availableTargets []ethdb.WasmTarget for target := range asmMap { availableTargets = append(availableTargets, target) } @@ -202,12 +228,8 @@ func callProgram( panic("missing asm") } - if db, ok := db.(*state.StateDB); ok { - targets := []rawdb.Target{ - rawdb.TargetWavm, - rawdb.LocalTarget(), - } - db.RecordProgram(targets, moduleHash) + if stateDb, ok := db.(*state.StateDB); ok { + stateDb.RecordProgram(db.Database().WasmTargets(), moduleHash) } evmApi := newApi(interpreter, tracingInfo, scope, memoryModel) @@ -291,7 +313,7 @@ func ResizeWasmLruCache(size uint32) { const DefaultTargetDescriptionArm = "arm64-linux-unknown+neon" const DefaultTargetDescriptionX86 = "x86_64-linux-unknown+sse4.2+lzcnt+bmi" -func SetTarget(name rawdb.Target, description string, native bool) error { +func SetTarget(name ethdb.WasmTarget, description string, native bool) error { output := &rustBytes{} status := userStatus(C.stylus_target_set( goSlice([]byte(name)), diff --git a/arbos/programs/wasmstorehelper.go b/arbos/programs/wasmstorehelper.go index 4f82d80282..434820dd9c 100644 --- a/arbos/programs/wasmstorehelper.go +++ b/arbos/programs/wasmstorehelper.go @@ -43,8 +43,9 @@ func (p Programs) SaveActiveProgramToWasmStore(statedb *state.StateDB, codeHash return err } + targets := statedb.Database().WasmTargets() // If already in wasm store then return early - _, err = statedb.TryGetActivatedAsmMap([]rawdb.Target{rawdb.TargetWavm, rawdb.LocalTarget()}, moduleHash) + _, err = statedb.TryGetActivatedAsmMap(targets, moduleHash) if err == nil { return nil } diff --git a/cmd/nitro/init.go b/cmd/nitro/init.go index 51d895c069..a8463a7d21 100644 --- a/cmd/nitro/init.go +++ b/cmd/nitro/init.go @@ -560,7 +560,7 @@ func rebuildLocalWasm(ctx context.Context, config *gethexec.Config, l2BlockChain return chainDb, l2BlockChain, nil } -func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeConfig, chainId *big.Int, cacheConfig *core.CacheConfig, persistentConfig *conf.PersistentConfig, l1Client arbutil.L1Interface, rollupAddrs chaininfo.RollupAddresses) (ethdb.Database, *core.BlockChain, error) { +func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeConfig, chainId *big.Int, cacheConfig *core.CacheConfig, targetConfig *gethexec.StylusTargetConfig, persistentConfig *conf.PersistentConfig, l1Client arbutil.L1Interface, rollupAddrs chaininfo.RollupAddresses) (ethdb.Database, *core.BlockChain, error) { if !config.Init.Force { if readOnlyDb, err := stack.OpenDatabaseWithFreezerWithExtraOptions("l2chaindata", 0, 0, config.Persistent.Ancient, "l2chaindata/", true, persistentConfig.Pebble.ExtraOptions("l2chaindata")); err == nil { if chainConfig := gethexec.TryReadStoredChainConfig(readOnlyDb); chainConfig != nil { @@ -585,7 +585,7 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo if err := dbutil.UnfinishedConversionCheck(wasmDb); err != nil { return nil, nil, fmt.Errorf("wasm unfinished database conversion check error: %w", err) } - chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmDb, 1) + chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmDb, 1, targetConfig.WasmTargets()) _, err = rawdb.ParseStateScheme(cacheConfig.StateScheme, chainDb) if err != nil { return nil, nil, err @@ -660,7 +660,7 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo if err := validateOrUpgradeWasmStoreSchemaVersion(wasmDb); err != nil { return nil, nil, err } - chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmDb, 1) + chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmDb, 1, targetConfig.WasmTargets()) _, err = rawdb.ParseStateScheme(cacheConfig.StateScheme, chainDb) if err != nil { return nil, nil, err diff --git a/cmd/nitro/init_test.go b/cmd/nitro/init_test.go index 7b9153a9cf..48d969f053 100644 --- a/cmd/nitro/init_test.go +++ b/cmd/nitro/init_test.go @@ -354,6 +354,12 @@ func TestEmptyDatabaseDir(t *testing.T) { } } +func defaultStylusTargetConfigForTest(t *testing.T) *gethexec.StylusTargetConfig { + targetConfig := gethexec.DefaultStylusTargetConfig + Require(t, targetConfig.Validate()) + return &targetConfig +} + func TestOpenInitializeChainDbIncompatibleStateScheme(t *testing.T) { t.Parallel() @@ -381,6 +387,7 @@ func TestOpenInitializeChainDbIncompatibleStateScheme(t *testing.T) { &nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), + defaultStylusTargetConfigForTest(t), &nodeConfig.Persistent, l1Client, chaininfo.RollupAddresses{}, @@ -397,6 +404,7 @@ func TestOpenInitializeChainDbIncompatibleStateScheme(t *testing.T) { &nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), + defaultStylusTargetConfigForTest(t), &nodeConfig.Persistent, l1Client, chaininfo.RollupAddresses{}, @@ -414,6 +422,7 @@ func TestOpenInitializeChainDbIncompatibleStateScheme(t *testing.T) { &nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), + defaultStylusTargetConfigForTest(t), &nodeConfig.Persistent, l1Client, chaininfo.RollupAddresses{}, @@ -549,6 +558,7 @@ func TestOpenInitializeChainDbEmptyInit(t *testing.T) { &nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), + defaultStylusTargetConfigForTest(t), &nodeConfig.Persistent, l1Client, chaininfo.RollupAddresses{}, diff --git a/cmd/nitro/nitro.go b/cmd/nitro/nitro.go index d4b2a87a30..146a0049e7 100644 --- a/cmd/nitro/nitro.go +++ b/cmd/nitro/nitro.go @@ -495,7 +495,7 @@ func mainImpl() int { } } - chainDb, l2BlockChain, err := openInitializeChainDb(ctx, stack, nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), &nodeConfig.Persistent, l1Client, rollupAddrs) + chainDb, l2BlockChain, err := openInitializeChainDb(ctx, stack, nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), &nodeConfig.Execution.StylusTarget, &nodeConfig.Persistent, l1Client, rollupAddrs) if l2BlockChain != nil { deferFuncs = append(deferFuncs, func() { l2BlockChain.Stop() }) } diff --git a/execution/gethexec/executionengine.go b/execution/gethexec/executionengine.go index 991c94540e..8594d5867d 100644 --- a/execution/gethexec/executionengine.go +++ b/execution/gethexec/executionengine.go @@ -151,19 +151,33 @@ func (s *ExecutionEngine) MarkFeedStart(to arbutil.MessageIndex) { } func populateStylusTargetCache(targetConfig *StylusTargetConfig) error { - var effectiveStylusTarget string - target := rawdb.LocalTarget() - switch target { - case rawdb.TargetArm64: - effectiveStylusTarget = targetConfig.Arm64 - case rawdb.TargetAmd64: - effectiveStylusTarget = targetConfig.Amd64 - case rawdb.TargetHost: - effectiveStylusTarget = targetConfig.Host - } - err := programs.SetTarget(target, effectiveStylusTarget, true) - if err != nil { - return fmt.Errorf("Failed to set stylus target: %w", err) + localTarget := rawdb.LocalTarget() + targets := targetConfig.WasmTargets() + var nativeSet bool + for _, target := range targets { + var effectiveStylusTarget string + switch target { + case rawdb.TargetWavm: + // skip wavm target + continue + case rawdb.TargetArm64: + effectiveStylusTarget = targetConfig.Arm64 + case rawdb.TargetAmd64: + effectiveStylusTarget = targetConfig.Amd64 + case rawdb.TargetHost: + effectiveStylusTarget = targetConfig.Host + default: + return fmt.Errorf("unsupported stylus target: %v", target) + } + isNative := target == localTarget + err := programs.SetTarget(target, effectiveStylusTarget, isNative) + if err != nil { + return fmt.Errorf("failed to set stylus target: %w", err) + } + nativeSet = nativeSet || isNative + } + if !nativeSet { + return fmt.Errorf("local target %v missing in list of archs %v", localTarget, targets) } return nil } diff --git a/execution/gethexec/node.go b/execution/gethexec/node.go index b864332e83..21c2b4bece 100644 --- a/execution/gethexec/node.go +++ b/execution/gethexec/node.go @@ -11,6 +11,7 @@ import ( "github.com/ethereum/go-ethereum/arbitrum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/filters" @@ -29,21 +30,51 @@ import ( ) type StylusTargetConfig struct { - Arm64 string `koanf:"arm64"` - Amd64 string `koanf:"amd64"` - Host string `koanf:"host"` + Arm64 string `koanf:"arm64"` + Amd64 string `koanf:"amd64"` + Host string `koanf:"host"` + ExtraArchs []string `koanf:"extra-archs"` + + wasmTargets []ethdb.WasmTarget +} + +func (c *StylusTargetConfig) WasmTargets() []ethdb.WasmTarget { + return c.wasmTargets +} + +func (c *StylusTargetConfig) Validate() error { + targetsSet := make(map[ethdb.WasmTarget]bool, len(c.ExtraArchs)) + for _, arch := range c.ExtraArchs { + target := ethdb.WasmTarget(arch) + if !rawdb.IsSupportedWasmTarget(target) { + return fmt.Errorf("unsupported architecture: %v, possible values: %s, %s, %s, %s", arch, rawdb.TargetWavm, rawdb.TargetArm64, rawdb.TargetAmd64, rawdb.TargetHost) + } + targetsSet[target] = true + } + if !targetsSet[rawdb.TargetWavm] { + return fmt.Errorf("%s target not found in archs list, archs: %v", rawdb.TargetWavm, c.ExtraArchs) + } + targetsSet[rawdb.LocalTarget()] = true + targets := make([]ethdb.WasmTarget, 0, len(c.ExtraArchs)+1) + for target := range targetsSet { + targets = append(targets, target) + } + c.wasmTargets = targets + return nil } var DefaultStylusTargetConfig = StylusTargetConfig{ - Arm64: programs.DefaultTargetDescriptionArm, - Amd64: programs.DefaultTargetDescriptionX86, - Host: "", + Arm64: programs.DefaultTargetDescriptionArm, + Amd64: programs.DefaultTargetDescriptionX86, + Host: "", + ExtraArchs: []string{string(rawdb.TargetWavm)}, } func StylusTargetConfigAddOptions(prefix string, f *flag.FlagSet) { f.String(prefix+".arm64", DefaultStylusTargetConfig.Arm64, "stylus programs compilation target for arm64 linux") f.String(prefix+".amd64", DefaultStylusTargetConfig.Amd64, "stylus programs compilation target for amd64 linux") f.String(prefix+".host", DefaultStylusTargetConfig.Host, "stylus programs compilation target for system other than 64-bit ARM or 64-bit x86") + f.StringSlice(prefix+".extra-archs", DefaultStylusTargetConfig.ExtraArchs, fmt.Sprintf("Comma separated list of extra architectures to cross-compile stylus program to and cache in wasm store (additionally to local target). Currently must include at least %s. (supported targets: %s, %s, %s, %s)", rawdb.TargetWavm, rawdb.TargetWavm, rawdb.TargetArm64, rawdb.TargetAmd64, rawdb.TargetHost)) } type Config struct { @@ -82,6 +113,9 @@ func (c *Config) Validate() error { if c.forwardingTarget != "" && c.Sequencer.Enable { return errors.New("ForwardingTarget set and sequencer enabled") } + if err := c.StylusTarget.Validate(); err != nil { + return err + } return nil } diff --git a/go-ethereum b/go-ethereum index 575062fad7..81114dde8a 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 575062fad7ff4db9d7c235f49472f658be29e2fe +Subproject commit 81114dde8a26bae90c188605c4a36d5919a4a265 diff --git a/staker/block_validator.go b/staker/block_validator.go index 2239952b37..7a7efca846 100644 --- a/staker/block_validator.go +++ b/staker/block_validator.go @@ -106,6 +106,7 @@ type BlockValidatorConfig struct { ValidationServerConfigs []rpcclient.ClientConfig `koanf:"validation-server-configs"` ValidationPoll time.Duration `koanf:"validation-poll" reload:"hot"` PrerecordedBlocks uint64 `koanf:"prerecorded-blocks" reload:"hot"` + RecordingIterLimit uint64 `koanf:"recording-iter-limit"` ForwardBlocks uint64 `koanf:"forward-blocks" reload:"hot"` CurrentModuleRoot string `koanf:"current-module-root"` // TODO(magic) requires reinitialization on hot reload PendingUpgradeModuleRoot string `koanf:"pending-upgrade-module-root"` // TODO(magic) requires StatelessBlockValidator recreation on hot reload @@ -174,6 +175,7 @@ func BlockValidatorConfigAddOptions(prefix string, f *pflag.FlagSet) { f.Uint64(prefix+".forward-blocks", DefaultBlockValidatorConfig.ForwardBlocks, "prepare entries for up to that many blocks ahead of validation (small footprint)") f.Uint64(prefix+".prerecorded-blocks", DefaultBlockValidatorConfig.PrerecordedBlocks, "record that many blocks ahead of validation (larger footprint)") f.String(prefix+".current-module-root", DefaultBlockValidatorConfig.CurrentModuleRoot, "current wasm module root ('current' read from chain, 'latest' from machines/latest dir, or provide hash)") + f.Uint64(prefix+".recording-iter-limit", DefaultBlockValidatorConfig.RecordingIterLimit, "limit on block recordings sent per iteration") f.String(prefix+".pending-upgrade-module-root", DefaultBlockValidatorConfig.PendingUpgradeModuleRoot, "pending upgrade wasm module root to additionally validate (hash, 'latest' or empty)") f.Bool(prefix+".failure-is-fatal", DefaultBlockValidatorConfig.FailureIsFatal, "failing a validation is treated as a fatal error") BlockValidatorDangerousConfigAddOptions(prefix+".dangerous", f) @@ -197,6 +199,7 @@ var DefaultBlockValidatorConfig = BlockValidatorConfig{ FailureIsFatal: true, Dangerous: DefaultBlockValidatorDangerousConfig, MemoryFreeLimit: "default", + RecordingIterLimit: 20, } var TestBlockValidatorConfig = BlockValidatorConfig{ @@ -207,6 +210,7 @@ var TestBlockValidatorConfig = BlockValidatorConfig{ ValidationPoll: 100 * time.Millisecond, ForwardBlocks: 128, PrerecordedBlocks: uint64(2 * runtime.NumCPU()), + RecordingIterLimit: 20, CurrentModuleRoot: "latest", PendingUpgradeModuleRoot: "latest", FailureIsFatal: true, @@ -500,7 +504,7 @@ func (v *BlockValidator) sendRecord(s *validationStatus) error { //nolint:gosec func (v *BlockValidator) writeToFile(validationEntry *validationEntry, moduleRoot common.Hash) error { - input, err := validationEntry.ToInput([]rawdb.Target{rawdb.TargetWavm}) + input, err := validationEntry.ToInput([]ethdb.WasmTarget{rawdb.TargetWavm}) if err != nil { return err } @@ -652,6 +656,10 @@ func (v *BlockValidator) sendNextRecordRequests(ctx context.Context) (bool, erro if recordUntil < pos { return false, nil } + recordUntilLimit := pos + arbutil.MessageIndex(v.config().RecordingIterLimit) + if recordUntil > recordUntilLimit { + recordUntil = recordUntilLimit + } log.Trace("preparing to record", "pos", pos, "until", recordUntil) // prepare could take a long time so we do it without a lock err := v.recorder.PrepareForRecord(ctx, pos, recordUntil) diff --git a/staker/challenge_manager.go b/staker/challenge_manager.go index ef431d3c79..c488c8a96e 100644 --- a/staker/challenge_manager.go +++ b/staker/challenge_manager.go @@ -16,6 +16,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rpc" "github.com/offchainlabs/nitro/arbutil" @@ -468,7 +469,7 @@ func (m *ChallengeManager) createExecutionBackend(ctx context.Context, step uint if err != nil { return fmt.Errorf("error creating validation entry for challenge %v msg %v for execution challenge: %w", m.challengeIndex, initialCount, err) } - input, err := entry.ToInput([]rawdb.Target{rawdb.TargetWavm}) + input, err := entry.ToInput([]ethdb.WasmTarget{rawdb.TargetWavm}) if err != nil { return fmt.Errorf("error getting validation entry input of challenge %v msg %v: %w", m.challengeIndex, initialCount, err) } diff --git a/staker/stateless_block_validator.go b/staker/stateless_block_validator.go index d5eeb8eb69..f54ec8b58c 100644 --- a/staker/stateless_block_validator.go +++ b/staker/stateless_block_validator.go @@ -12,7 +12,6 @@ import ( "github.com/offchainlabs/nitro/arbstate/daprovider" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" @@ -134,7 +133,7 @@ type validationEntry struct { DelayedMsg []byte } -func (e *validationEntry) ToInput(stylusArchs []rawdb.Target) (*validator.ValidationInput, error) { +func (e *validationEntry) ToInput(stylusArchs []ethdb.WasmTarget) (*validator.ValidationInput, error) { if e.Stage != Ready { return nil, errors.New("cannot create input from non-ready entry") } @@ -143,7 +142,7 @@ func (e *validationEntry) ToInput(stylusArchs []rawdb.Target) (*validator.Valida HasDelayedMsg: e.HasDelayedMsg, DelayedMsgNr: e.DelayedMsgNr, Preimages: e.Preimages, - UserWasms: make(map[rawdb.Target]map[common.Hash][]byte, len(e.UserWasms)), + UserWasms: make(map[ethdb.WasmTarget]map[common.Hash][]byte, len(e.UserWasms)), BatchInfo: e.BatchInfo, DelayedMsg: e.DelayedMsg, StartState: e.Start, diff --git a/system_tests/common_test.go b/system_tests/common_test.go index 6e7375a921..457dae0910 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -197,7 +197,7 @@ var TestSequencerConfig = gethexec.SequencerConfig{ EnableProfiling: false, } -func ExecConfigDefaultNonSequencerTest() *gethexec.Config { +func ExecConfigDefaultNonSequencerTest(t *testing.T) *gethexec.Config { config := gethexec.ConfigDefault config.Caching = TestCachingConfig config.ParentChainReader = headerreader.TestConfig @@ -206,12 +206,12 @@ func ExecConfigDefaultNonSequencerTest() *gethexec.Config { config.ForwardingTarget = "null" config.TxPreChecker.Strictness = gethexec.TxPreCheckerStrictnessNone - _ = config.Validate() + Require(t, config.Validate()) return &config } -func ExecConfigDefaultTest() *gethexec.Config { +func ExecConfigDefaultTest(t *testing.T) *gethexec.Config { config := gethexec.ConfigDefault config.Caching = TestCachingConfig config.Sequencer = TestSequencerConfig @@ -219,7 +219,7 @@ func ExecConfigDefaultTest() *gethexec.Config { config.ForwardingTarget = "null" config.TxPreChecker.Strictness = gethexec.TxPreCheckerStrictnessNone - _ = config.Validate() + Require(t, config.Validate()) return &config } @@ -272,7 +272,7 @@ func (b *NodeBuilder) DefaultConfig(t *testing.T, withL1 bool) *NodeBuilder { b.l2StackConfig = testhelpers.CreateStackConfigForTest(b.dataDir) cp := valnode.TestValidationConfig b.valnodeConfig = &cp - b.execConfig = ExecConfigDefaultTest() + b.execConfig = ExecConfigDefaultTest(t) return b } @@ -293,6 +293,11 @@ func (b *NodeBuilder) WithWasmRootDir(wasmRootDir string) *NodeBuilder { return b } +func (b *NodeBuilder) WithExtraArchs(targets []string) *NodeBuilder { + b.execConfig.StylusTarget.ExtraArchs = targets + return b +} + func (b *NodeBuilder) Build(t *testing.T) func() { b.CheckConfig(t) if b.withL1 { @@ -310,7 +315,7 @@ func (b *NodeBuilder) CheckConfig(t *testing.T) { b.nodeConfig = arbnode.ConfigDefaultL1Test() } if b.execConfig == nil { - b.execConfig = ExecConfigDefaultTest() + b.execConfig = ExecConfigDefaultTest(t) } if b.L1Info == nil { b.L1Info = NewL1TestInfo(t) @@ -346,7 +351,7 @@ func (b *NodeBuilder) BuildL2OnL1(t *testing.T) func() { var l2arbDb ethdb.Database var l2blockchain *core.BlockChain _, b.L2.Stack, l2chainDb, l2arbDb, l2blockchain = createL2BlockChainWithStackConfig( - t, b.L2Info, b.dataDir, b.chainConfig, b.initMessage, b.l2StackConfig, &b.execConfig.Caching) + t, b.L2Info, b.dataDir, b.chainConfig, b.initMessage, b.l2StackConfig, b.execConfig) var sequencerTxOptsPtr *bind.TransactOpts var dataSigner signature.DataSignerFunc @@ -409,7 +414,7 @@ func (b *NodeBuilder) BuildL2(t *testing.T) func() { var arbDb ethdb.Database var blockchain *core.BlockChain b.L2Info, b.L2.Stack, chainDb, arbDb, blockchain = createL2BlockChain( - t, b.L2Info, b.dataDir, b.chainConfig, &b.execConfig.Caching) + t, b.L2Info, b.dataDir, b.chainConfig, b.execConfig) Require(t, b.execConfig.Validate()) execConfig := b.execConfig @@ -460,7 +465,7 @@ func (b *NodeBuilder) RestartL2Node(t *testing.T) { } b.L2.cleanup() - l2info, stack, chainDb, arbDb, blockchain := createL2BlockChainWithStackConfig(t, b.L2Info, b.dataDir, b.chainConfig, b.initMessage, b.l2StackConfig, &b.execConfig.Caching) + l2info, stack, chainDb, arbDb, blockchain := createL2BlockChainWithStackConfig(t, b.L2Info, b.dataDir, b.chainConfig, b.initMessage, b.l2StackConfig, b.execConfig) execConfigFetcher := func() *gethexec.Config { return b.execConfig } execNode, err := gethexec.CreateExecutionNode(b.ctx, stack, chainDb, blockchain, nil, execConfigFetcher) @@ -1047,30 +1052,32 @@ func DeployOnTestL1( } func createL2BlockChain( - t *testing.T, l2info *BlockchainTestInfo, dataDir string, chainConfig *params.ChainConfig, cacheConfig *gethexec.CachingConfig, + t *testing.T, l2info *BlockchainTestInfo, dataDir string, chainConfig *params.ChainConfig, execConfig *gethexec.Config, ) (*BlockchainTestInfo, *node.Node, ethdb.Database, ethdb.Database, *core.BlockChain) { - return createL2BlockChainWithStackConfig(t, l2info, dataDir, chainConfig, nil, nil, cacheConfig) + return createL2BlockChainWithStackConfig(t, l2info, dataDir, chainConfig, nil, nil, execConfig) } func createL2BlockChainWithStackConfig( - t *testing.T, l2info *BlockchainTestInfo, dataDir string, chainConfig *params.ChainConfig, initMessage *arbostypes.ParsedInitMessage, stackConfig *node.Config, cacheConfig *gethexec.CachingConfig, + t *testing.T, l2info *BlockchainTestInfo, dataDir string, chainConfig *params.ChainConfig, initMessage *arbostypes.ParsedInitMessage, stackConfig *node.Config, execConfig *gethexec.Config, ) (*BlockchainTestInfo, *node.Node, ethdb.Database, ethdb.Database, *core.BlockChain) { if l2info == nil { l2info = NewArbTestInfo(t, chainConfig.ChainID) } - var stack *node.Node - var err error if stackConfig == nil { stackConfig = testhelpers.CreateStackConfigForTest(dataDir) } - stack, err = node.New(stackConfig) + if execConfig == nil { + execConfig = ExecConfigDefaultTest(t) + } + + stack, err := node.New(stackConfig) Require(t, err) chainData, err := stack.OpenDatabaseWithExtraOptions("l2chaindata", 0, 0, "l2chaindata/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("l2chaindata")) Require(t, err) wasmData, err := stack.OpenDatabaseWithExtraOptions("wasm", 0, 0, "wasm/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("wasm")) Require(t, err) - chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmData, 0) + chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmData, 0, execConfig.StylusTarget.WasmTargets()) arbDb, err := stack.OpenDatabaseWithExtraOptions("arbitrumdata", 0, 0, "arbitrumdata/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("arbitrumdata")) Require(t, err) @@ -1085,11 +1092,8 @@ func createL2BlockChainWithStackConfig( SerializedChainConfig: serializedChainConfig, } } - var coreCacheConfig *core.CacheConfig - if cacheConfig != nil { - coreCacheConfig = gethexec.DefaultCacheConfigFor(stack, cacheConfig) - } - blockchain, err := gethexec.WriteOrTestBlockChain(chainDb, coreCacheConfig, initReader, chainConfig, initMessage, ExecConfigDefaultTest().TxLookupLimit, 0) + coreCacheConfig := gethexec.DefaultCacheConfigFor(stack, &execConfig.Caching) + blockchain, err := gethexec.WriteOrTestBlockChain(chainDb, coreCacheConfig, initReader, chainConfig, initMessage, ExecConfigDefaultTest(t).TxLookupLimit, 0) Require(t, err) return l2info, stack, chainDb, arbDb, blockchain @@ -1149,7 +1153,7 @@ func Create2ndNodeWithConfig( nodeConfig = arbnode.ConfigDefaultL1NonSequencerTest() } if execConfig == nil { - execConfig = ExecConfigDefaultNonSequencerTest() + execConfig = ExecConfigDefaultNonSequencerTest(t) } feedErrChan := make(chan error, 10) l1rpcClient := l1stack.Attach() @@ -1165,7 +1169,7 @@ func Create2ndNodeWithConfig( Require(t, err) wasmData, err := l2stack.OpenDatabaseWithExtraOptions("wasm", 0, 0, "wasm/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("wasm")) Require(t, err) - l2chainDb := rawdb.WrapDatabaseWithWasm(l2chainData, wasmData, 0) + l2chainDb := rawdb.WrapDatabaseWithWasm(l2chainData, wasmData, 0, execConfig.StylusTarget.WasmTargets()) l2arbDb, err := l2stack.OpenDatabaseWithExtraOptions("arbitrumdata", 0, 0, "arbitrumdata/", false, conf.PersistentConfigDefault.Pebble.ExtraOptions("arbitrumdata")) Require(t, err) @@ -1179,7 +1183,7 @@ func Create2ndNodeWithConfig( chainConfig := firstExec.ArbInterface.BlockChain().Config() coreCacheConfig := gethexec.DefaultCacheConfigFor(l2stack, &execConfig.Caching) - l2blockchain, err := gethexec.WriteOrTestBlockChain(l2chainDb, coreCacheConfig, initReader, chainConfig, initMessage, ExecConfigDefaultTest().TxLookupLimit, 0) + l2blockchain, err := gethexec.WriteOrTestBlockChain(l2chainDb, coreCacheConfig, initReader, chainConfig, initMessage, ExecConfigDefaultTest(t).TxLookupLimit, 0) Require(t, err) AddValNodeIfNeeded(t, ctx, nodeConfig, true, "", valnodeConfig.Wasm.RootPath) diff --git a/system_tests/forwarder_test.go b/system_tests/forwarder_test.go index f87283432e..57381ca84e 100644 --- a/system_tests/forwarder_test.go +++ b/system_tests/forwarder_test.go @@ -38,7 +38,7 @@ func TestStaticForwarder(t *testing.T) { clientA := builder.L2.Client nodeConfigB := arbnode.ConfigDefaultL1Test() - execConfigB := ExecConfigDefaultTest() + execConfigB := ExecConfigDefaultTest(t) execConfigB.Sequencer.Enable = false nodeConfigB.Sequencer = false nodeConfigB.DelayedSequencer.Enable = false @@ -109,7 +109,7 @@ func createForwardingNode(t *testing.T, builder *NodeBuilder, ipcPath string, re nodeConfig.Sequencer = false nodeConfig.DelayedSequencer.Enable = false nodeConfig.BatchPoster.Enable = false - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.Sequencer.Enable = false execConfig.Forwarder.RedisUrl = redisUrl execConfig.ForwardingTarget = fallbackPath diff --git a/system_tests/program_test.go b/system_tests/program_test.go index c89a13e205..83c066fdb5 100644 --- a/system_tests/program_test.go +++ b/system_tests/program_test.go @@ -45,18 +45,31 @@ import ( var oneEth = arbmath.UintToBig(1e18) +var allWasmTargets = []string{string(rawdb.TargetWavm), string(rawdb.TargetArm64), string(rawdb.TargetAmd64), string(rawdb.TargetHost)} + func TestProgramKeccak(t *testing.T) { t.Parallel() - keccakTest(t, true) + t.Run("WithDefaultWasmTargets", func(t *testing.T) { + keccakTest(t, true) + }) + + t.Run("WithAllWasmTargets", func(t *testing.T) { + keccakTest(t, true, func(builder *NodeBuilder) { + builder.WithExtraArchs(allWasmTargets) + }) + }) } -func keccakTest(t *testing.T, jit bool) { - builder, auth, cleanup := setupProgramTest(t, jit) +func keccakTest(t *testing.T, jit bool, builderOpts ...func(*NodeBuilder)) { + builder, auth, cleanup := setupProgramTest(t, jit, builderOpts...) ctx := builder.ctx l2client := builder.L2.Client defer cleanup() programAddress := deployWasm(t, ctx, auth, l2client, rustFile("keccak")) + wasmDb := builder.L2.ExecNode.Backend.ArbInterface().BlockChain().StateCache().WasmStore() + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 1) + wasm, _ := readWasmFile(t, rustFile("keccak")) otherAddressSameCode := deployContract(t, ctx, auth, l2client, wasm) arbWasm, err := pgen.NewArbWasm(types.ArbWasmAddress, l2client) @@ -68,6 +81,7 @@ func keccakTest(t *testing.T, jit bool) { Fatal(t, "activate should have failed with ProgramUpToDate", err) } }) + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 1) if programAddress == otherAddressSameCode { Fatal(t, "expected to deploy at two separate program addresses") @@ -141,11 +155,18 @@ func keccakTest(t *testing.T, jit bool) { func TestProgramActivateTwice(t *testing.T) { t.Parallel() - testActivateTwice(t, true) + t.Run("WithDefaultWasmTargets", func(t *testing.T) { + testActivateTwice(t, true) + }) + t.Run("WithAllWasmTargets", func(t *testing.T) { + testActivateTwice(t, true, func(builder *NodeBuilder) { + builder.WithExtraArchs(allWasmTargets) + }) + }) } -func testActivateTwice(t *testing.T, jit bool) { - builder, auth, cleanup := setupProgramTest(t, jit) +func testActivateTwice(t *testing.T, jit bool, builderOpts ...func(*NodeBuilder)) { + builder, auth, cleanup := setupProgramTest(t, jit, builderOpts...) ctx := builder.ctx l2info := builder.L2Info l2client := builder.L2.Client @@ -171,6 +192,10 @@ func testActivateTwice(t *testing.T, jit bool) { colors.PrintBlue("keccak program B deployed to ", keccakB) multiAddr := deployWasm(t, ctx, auth, l2client, rustFile("multicall")) + + wasmDb := builder.L2.ExecNode.Backend.ArbInterface().BlockChain().StateCache().WasmStore() + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 1) + preimage := []byte("it's time to du-du-du-du d-d-d-d-d-d-d de-duplicate") keccakArgs := []byte{0x01} // keccak the preimage once @@ -194,6 +219,7 @@ func testActivateTwice(t *testing.T, jit bool) { // Calling the contract pre-activation should fail. checkReverts() + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 1) // mechanisms for creating calldata activateProgram, _ := util.NewCallParser(pgen.ArbWasmABI, "activateProgram") @@ -216,6 +242,7 @@ func testActivateTwice(t *testing.T, jit bool) { // Ensure the revert also reverted keccak's activation checkReverts() + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 1) // Activate keccak program A, then call into B, which should succeed due to being the same codehash args = argsForMulticall(vm.CALL, types.ArbWasmAddress, oneEth, pack(activateProgram(keccakA))) @@ -223,6 +250,7 @@ func testActivateTwice(t *testing.T, jit bool) { tx = l2info.PrepareTxTo("Owner", &multiAddr, 1e9, oneEth, args) ensure(tx, l2client.SendTransaction(ctx, tx)) + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 2) validateBlocks(t, 7, jit, builder) } @@ -1834,7 +1862,9 @@ func createMapFromDb(db ethdb.KeyValueStore) (map[string][]byte, error) { } func TestWasmStoreRebuilding(t *testing.T) { - builder, auth, cleanup := setupProgramTest(t, true) + builder, auth, cleanup := setupProgramTest(t, true, func(b *NodeBuilder) { + b.WithExtraArchs(allWasmTargets) + }) ctx := builder.ctx l2info := builder.L2Info l2client := builder.L2.Client @@ -1871,6 +1901,7 @@ func TestWasmStoreRebuilding(t *testing.T) { storeMap, err := createMapFromDb(wasmDb) Require(t, err) + checkWasmStoreContent(t, wasmDb, builder.execConfig.StylusTarget.ExtraArchs, 1) // close nodeB cleanupB() @@ -1927,5 +1958,52 @@ func TestWasmStoreRebuilding(t *testing.T) { } } + checkWasmStoreContent(t, wasmDbAfterRebuild, builder.execConfig.StylusTarget.ExtraArchs, 1) cleanupB() } + +func readModuleHashes(t *testing.T, wasmDb ethdb.KeyValueStore) []common.Hash { + modulesSet := make(map[common.Hash]struct{}) + asmPrefix := []byte{0x00, 'w'} + it := wasmDb.NewIterator(asmPrefix, nil) + defer it.Release() + for it.Next() { + key := it.Key() + if len(key) != rawdb.WasmKeyLen { + t.Fatalf("unexpected activated module key length, len: %d, key: %v", len(key), key) + } + moduleHash := key[rawdb.WasmPrefixLen:] + if len(moduleHash) != common.HashLength { + t.Fatalf("Invalid moduleHash length in key: %v, moduleHash: %v", key, moduleHash) + } + modulesSet[common.BytesToHash(moduleHash)] = struct{}{} + } + modules := make([]common.Hash, 0, len(modulesSet)) + for module := range modulesSet { + modules = append(modules, module) + } + return modules +} + +func checkWasmStoreContent(t *testing.T, wasmDb ethdb.KeyValueStore, targets []string, numModules int) { + modules := readModuleHashes(t, wasmDb) + if len(modules) != numModules { + t.Fatalf("Unexpected number of module hashes found in wasm store, want: %d, have: %d", numModules, len(modules)) + } + for _, module := range modules { + for _, target := range targets { + wasmTarget := ethdb.WasmTarget(target) + if !rawdb.IsSupportedWasmTarget(wasmTarget) { + t.Fatalf("internal test error - unsupported target passed to checkWasmStoreContent: %v", target) + } + func() { + defer func() { + if r := recover(); r != nil { + t.Fatalf("Failed to read activated asm for target: %v, module: %v", target, module) + } + }() + _ = rawdb.ReadActivatedAsm(wasmDb, wasmTarget, module) + }() + } + } +} diff --git a/system_tests/recreatestate_rpc_test.go b/system_tests/recreatestate_rpc_test.go index cd3904ca06..7b09552e28 100644 --- a/system_tests/recreatestate_rpc_test.go +++ b/system_tests/recreatestate_rpc_test.go @@ -95,7 +95,7 @@ func removeStatesFromDb(t *testing.T, bc *core.BlockChain, db ethdb.Database, fr func TestRecreateStateForRPCNoDepthLimit(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = arbitrum.InfiniteMaxRecreateStateDepth execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -134,7 +134,7 @@ func TestRecreateStateForRPCBigEnoughDepthLimit(t *testing.T) { defer cancel() // #nosec G115 depthGasLimit := int64(256 * util.NormalizeL2GasForL1GasInitial(800_000, params.GWei)) - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = depthGasLimit execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -171,7 +171,7 @@ func TestRecreateStateForRPCBigEnoughDepthLimit(t *testing.T) { func TestRecreateStateForRPCDepthLimitExceeded(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = int64(200) execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -208,7 +208,7 @@ func TestRecreateStateForRPCMissingBlockParent(t *testing.T) { var headerCacheLimit uint64 = 512 ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = arbitrum.InfiniteMaxRecreateStateDepth execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -256,7 +256,7 @@ func TestRecreateStateForRPCBeyondGenesis(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = arbitrum.InfiniteMaxRecreateStateDepth execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -294,7 +294,7 @@ func TestRecreateStateForRPCBlockNotFoundWhileRecreating(t *testing.T) { var blockCacheLimit uint64 = 256 ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = arbitrum.InfiniteMaxRecreateStateDepth execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -342,7 +342,7 @@ func testSkippingSavingStateAndRecreatingAfterRestart(t *testing.T, cacheConfig ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.RPC.MaxRecreateStateDepth = maxRecreateStateDepth execConfig.Sequencer.MaxBlockSpeed = 0 execConfig.Sequencer.MaxTxDataSize = 150 // 1 test tx ~= 110 @@ -483,7 +483,7 @@ func TestSkippingSavingStateAndRecreatingAfterRestart(t *testing.T) { func TestGettingStateForRPCFullNode(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.Caching.SnapshotCache = 0 // disable snapshots execConfig.Caching.BlockAge = 0 // use only Caching.BlockCount to keep only last N blocks in dirties cache, no matter how new they are execConfig.Sequencer.MaxBlockSpeed = 0 @@ -527,7 +527,7 @@ func TestGettingStateForRPCFullNode(t *testing.T) { func TestGettingStateForRPCHybridArchiveNode(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - execConfig := ExecConfigDefaultTest() + execConfig := ExecConfigDefaultTest(t) execConfig.Caching.Archive = true // For now Archive node should use HashScheme execConfig.Caching.StateScheme = rawdb.HashScheme diff --git a/system_tests/retryable_test.go b/system_tests/retryable_test.go index 106dfc6d46..aa9fbfd72e 100644 --- a/system_tests/retryable_test.go +++ b/system_tests/retryable_test.go @@ -1042,7 +1042,7 @@ func elevateL2Basefee(t *testing.T, ctx context.Context, builder *NodeBuilder) { _, err = precompilesgen.NewArbosTest(common.HexToAddress("0x69"), builder.L2.Client) Require(t, err, "failed to deploy ArbosTest") - burnAmount := ExecConfigDefaultTest().RPC.RPCGasCap + burnAmount := ExecConfigDefaultTest(t).RPC.RPCGasCap burnTarget := uint64(5 * l2pricing.InitialSpeedLimitPerSecondV6 * l2pricing.InitialBacklogTolerance) for i := uint64(0); i < (burnTarget+burnAmount)/burnAmount; i++ { burnArbGas := arbosTestAbi.Methods["burnArbGas"] diff --git a/system_tests/validation_mock_test.go b/system_tests/validation_mock_test.go index 88421e4c4b..2739c7545e 100644 --- a/system_tests/validation_mock_test.go +++ b/system_tests/validation_mock_test.go @@ -8,9 +8,9 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/rpc" "github.com/offchainlabs/nitro/arbnode" @@ -61,8 +61,8 @@ func (s *mockSpawner) WasmModuleRoots() ([]common.Hash, error) { return mockWasmModuleRoots, nil } -func (s *mockSpawner) StylusArchs() []rawdb.Target { - return []rawdb.Target{"mock"} +func (s *mockSpawner) StylusArchs() []ethdb.WasmTarget { + return []ethdb.WasmTarget{"mock"} } func (s *mockSpawner) Launch(entry *validator.ValidationInput, moduleRoot common.Hash) validator.ValidationRun { diff --git a/validator/client/redis/producer.go b/validator/client/redis/producer.go index f98c246d0e..adc2f34af5 100644 --- a/validator/client/redis/producer.go +++ b/validator/client/redis/producer.go @@ -7,6 +7,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/go-redis/redis/v8" "github.com/offchainlabs/nitro/pubsub" @@ -35,7 +36,7 @@ func (c ValidationClientConfig) Enabled() bool { func (c ValidationClientConfig) Validate() error { for _, arch := range c.StylusArchs { - if !rawdb.Target(arch).IsValid() { + if !rawdb.IsSupportedWasmTarget(ethdb.WasmTarget(arch)) { return fmt.Errorf("Invalid stylus arch: %v", arch) } } @@ -162,10 +163,10 @@ func (c *ValidationClient) Name() string { return c.config.Name } -func (c *ValidationClient) StylusArchs() []rawdb.Target { - stylusArchs := make([]rawdb.Target, 0, len(c.config.StylusArchs)) +func (c *ValidationClient) StylusArchs() []ethdb.WasmTarget { + stylusArchs := make([]ethdb.WasmTarget, 0, len(c.config.StylusArchs)) for _, arch := range c.config.StylusArchs { - stylusArchs = append(stylusArchs, rawdb.Target(arch)) + stylusArchs = append(stylusArchs, ethdb.WasmTarget(arch)) } return stylusArchs } diff --git a/validator/client/validation_client.go b/validator/client/validation_client.go index 00bd992f46..3b18ad1851 100644 --- a/validator/client/validation_client.go +++ b/validator/client/validation_client.go @@ -22,6 +22,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/rpc" @@ -31,7 +32,7 @@ type ValidationClient struct { stopwaiter.StopWaiter client *rpcclient.RpcClient name string - stylusArchs []rawdb.Target + stylusArchs []ethdb.WasmTarget room atomic.Int32 wasmModuleRoots []common.Hash } @@ -40,7 +41,7 @@ func NewValidationClient(config rpcclient.ClientConfigFetcher, stack *node.Node) return &ValidationClient{ client: rpcclient.NewRpcClient(config, stack), name: "not started", - stylusArchs: []rawdb.Target{"not started"}, + stylusArchs: []ethdb.WasmTarget{"not started"}, } } @@ -67,20 +68,20 @@ func (c *ValidationClient) Start(ctx context.Context) error { if len(name) == 0 { return errors.New("couldn't read name from server") } - var stylusArchs []rawdb.Target + var stylusArchs []ethdb.WasmTarget if err := c.client.CallContext(ctx, &stylusArchs, server_api.Namespace+"_stylusArchs"); err != nil { var rpcError rpc.Error ok := errors.As(err, &rpcError) if !ok || rpcError.ErrorCode() != -32601 { return fmt.Errorf("could not read stylus arch from server: %w", err) } - stylusArchs = []rawdb.Target{rawdb.Target("pre-stylus")} // invalid, will fail if trying to validate block with stylus + stylusArchs = []ethdb.WasmTarget{ethdb.WasmTarget("pre-stylus")} // invalid, will fail if trying to validate block with stylus } else { if len(stylusArchs) == 0 { return fmt.Errorf("could not read stylus archs from validation server") } for _, stylusArch := range stylusArchs { - if stylusArch != rawdb.TargetWavm && stylusArch != rawdb.LocalTarget() && stylusArch != "mock" { + if !rawdb.IsSupportedWasmTarget(ethdb.WasmTarget(stylusArch)) && stylusArch != "mock" { return fmt.Errorf("unsupported stylus architecture: %v", stylusArch) } } @@ -118,11 +119,11 @@ func (c *ValidationClient) WasmModuleRoots() ([]common.Hash, error) { return nil, errors.New("not started") } -func (c *ValidationClient) StylusArchs() []rawdb.Target { +func (c *ValidationClient) StylusArchs() []ethdb.WasmTarget { if c.Started() { return c.stylusArchs } - return []rawdb.Target{"not started"} + return []ethdb.WasmTarget{"not started"} } func (c *ValidationClient) Stop() { diff --git a/validator/interface.go b/validator/interface.go index 81b40ae5cf..af08629137 100644 --- a/validator/interface.go +++ b/validator/interface.go @@ -4,7 +4,7 @@ import ( "context" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/offchainlabs/nitro/util/containers" ) @@ -14,7 +14,7 @@ type ValidationSpawner interface { Start(context.Context) error Stop() Name() string - StylusArchs() []rawdb.Target + StylusArchs() []ethdb.WasmTarget Room() int } diff --git a/validator/server_api/json.go b/validator/server_api/json.go index dbe2bb1fee..6fe936e17d 100644 --- a/validator/server_api/json.go +++ b/validator/server_api/json.go @@ -11,7 +11,7 @@ import ( "os" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/offchainlabs/nitro/arbcompress" "github.com/offchainlabs/nitro/arbutil" @@ -64,7 +64,7 @@ type InputJSON struct { BatchInfo []BatchInfoJson DelayedMsgB64 string StartState validator.GoGlobalState - UserWasms map[rawdb.Target]map[common.Hash]string + UserWasms map[ethdb.WasmTarget]map[common.Hash]string DebugChain bool } @@ -96,7 +96,7 @@ func ValidationInputToJson(entry *validator.ValidationInput) *InputJSON { DelayedMsgB64: base64.StdEncoding.EncodeToString(entry.DelayedMsg), StartState: entry.StartState, PreimagesB64: jsonPreimagesMap, - UserWasms: make(map[rawdb.Target]map[common.Hash]string), + UserWasms: make(map[ethdb.WasmTarget]map[common.Hash]string), DebugChain: entry.DebugChain, } for _, binfo := range entry.BatchInfo { @@ -128,7 +128,7 @@ func ValidationInputFromJson(entry *InputJSON) (*validator.ValidationInput, erro DelayedMsgNr: entry.DelayedMsgNr, StartState: entry.StartState, Preimages: preimages, - UserWasms: make(map[rawdb.Target]map[common.Hash][]byte), + UserWasms: make(map[ethdb.WasmTarget]map[common.Hash][]byte), DebugChain: entry.DebugChain, } delayed, err := base64.StdEncoding.DecodeString(entry.DelayedMsgB64) diff --git a/validator/server_arb/validator_spawner.go b/validator/server_arb/validator_spawner.go index eb53070303..6f0d0cee1d 100644 --- a/validator/server_arb/validator_spawner.go +++ b/validator/server_arb/validator_spawner.go @@ -13,6 +13,7 @@ import ( "github.com/spf13/pflag" + "github.com/ethereum/go-ethereum/ethdb" "github.com/offchainlabs/nitro/arbutil" "github.com/offchainlabs/nitro/util/containers" "github.com/offchainlabs/nitro/util/stopwaiter" @@ -89,8 +90,8 @@ func (s *ArbitratorSpawner) WasmModuleRoots() ([]common.Hash, error) { return s.locator.ModuleRoots(), nil } -func (s *ArbitratorSpawner) StylusArchs() []rawdb.Target { - return []rawdb.Target{rawdb.TargetWavm} +func (s *ArbitratorSpawner) StylusArchs() []ethdb.WasmTarget { + return []ethdb.WasmTarget{rawdb.TargetWavm} } func (s *ArbitratorSpawner) Name() string { diff --git a/validator/server_jit/spawner.go b/validator/server_jit/spawner.go index 92b50b17cb..d77317d218 100644 --- a/validator/server_jit/spawner.go +++ b/validator/server_jit/spawner.go @@ -10,6 +10,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/offchainlabs/nitro/util/stopwaiter" "github.com/offchainlabs/nitro/validator" @@ -72,8 +73,8 @@ func (v *JitSpawner) WasmModuleRoots() ([]common.Hash, error) { return v.locator.ModuleRoots(), nil } -func (v *JitSpawner) StylusArchs() []rawdb.Target { - return []rawdb.Target{rawdb.LocalTarget()} +func (v *JitSpawner) StylusArchs() []ethdb.WasmTarget { + return []ethdb.WasmTarget{rawdb.LocalTarget()} } func (v *JitSpawner) execute( diff --git a/validator/validation_entry.go b/validator/validation_entry.go index 2c357659ad..d340993fa2 100644 --- a/validator/validation_entry.go +++ b/validator/validation_entry.go @@ -2,7 +2,7 @@ package validator import ( "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/offchainlabs/nitro/arbutil" ) @@ -17,7 +17,7 @@ type ValidationInput struct { HasDelayedMsg bool DelayedMsgNr uint64 Preimages map[arbutil.PreimageType]map[common.Hash][]byte - UserWasms map[rawdb.Target]map[common.Hash][]byte + UserWasms map[ethdb.WasmTarget]map[common.Hash][]byte BatchInfo []BatchInfo DelayedMsg []byte StartState GoGlobalState diff --git a/validator/valnode/redis/consumer.go b/validator/valnode/redis/consumer.go index 7456466533..2b025600cc 100644 --- a/validator/valnode/redis/consumer.go +++ b/validator/valnode/redis/consumer.go @@ -111,10 +111,12 @@ func (s *ValidationServer) Start(ctx_in context.Context) { req, err := c.Consume(ctx) if err != nil { log.Error("Consuming request", "error", err) + requestTokenQueue <- struct{}{} return 0 } if req == nil { - // There's nothing in the queue. + // There's nothing in the queue + requestTokenQueue <- struct{}{} return time.Second } select { @@ -152,9 +154,10 @@ func (s *ValidationServer) Start(ctx_in context.Context) { res, err := valRun.Await(ctx) if err != nil { log.Error("Error validating", "request value", work.req.Value, "error", err) - } - if err := s.consumers[work.moduleRoot].SetResult(ctx, work.req.ID, res); err != nil { - log.Error("Error setting result for request", "id", work.req.ID, "result", res, "error", err) + } else { + if err := s.consumers[work.moduleRoot].SetResult(ctx, work.req.ID, res); err != nil { + log.Error("Error setting result for request", "id", work.req.ID, "result", res, "error", err) + } } select { case <-ctx.Done(): diff --git a/validator/valnode/validation_api.go b/validator/valnode/validation_api.go index a79ac7fa55..a10d931dfc 100644 --- a/validator/valnode/validation_api.go +++ b/validator/valnode/validation_api.go @@ -12,7 +12,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/offchainlabs/nitro/util/stopwaiter" "github.com/offchainlabs/nitro/validator" @@ -45,7 +45,7 @@ func (a *ValidationServerAPI) WasmModuleRoots() ([]common.Hash, error) { return a.spawner.WasmModuleRoots() } -func (a *ValidationServerAPI) StylusArchs() ([]rawdb.Target, error) { +func (a *ValidationServerAPI) StylusArchs() ([]ethdb.WasmTarget, error) { return a.spawner.StylusArchs(), nil } From 925a623477d1ed3fa82cc42e6e90a947378d4d0b Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Wed, 11 Sep 2024 18:06:14 -0600 Subject: [PATCH 4/7] cargo_fmt --- arbitrator/prover/src/lib.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/arbitrator/prover/src/lib.rs b/arbitrator/prover/src/lib.rs index 20472691c1..e45d12c492 100644 --- a/arbitrator/prover/src/lib.rs +++ b/arbitrator/prover/src/lib.rs @@ -73,8 +73,13 @@ pub unsafe extern "C" fn arbitrator_load_machine( with_forwarder: bool, ) -> *mut Machine { let debug_chain = debug_chain != 0; - match arbitrator_load_machine_impl(binary_path, library_paths, library_paths_size, debug_chain, with_forwarder) - { + match arbitrator_load_machine_impl( + binary_path, + library_paths, + library_paths_size, + debug_chain, + with_forwarder, + ) { Ok(mach) => mach, Err(err) => { eprintln!("Error loading binary: {:?}", err); From c5f7181a9319f14dfd704c9e08e68508d3323788 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Wed, 11 Sep 2024 18:14:05 -0600 Subject: [PATCH 5/7] use import resolvers --- arbitrator/prover/src/host.rs | 9 +- arbitrator/prover/src/machine.rs | 148 ++++++++++++++++++++----------- 2 files changed, 102 insertions(+), 55 deletions(-) diff --git a/arbitrator/prover/src/host.rs b/arbitrator/prover/src/host.rs index 1d0fe658ec..dc0f0b74b2 100644 --- a/arbitrator/prover/src/host.rs +++ b/arbitrator/prover/src/host.rs @@ -405,7 +405,7 @@ impl Hostio { } } -pub fn get_impl(module: &str, name: &str) -> Result<(Function, bool)> { +pub fn get_impl(module: &str, name: &str) -> Result { let hostio: Hostio = format!("{module}__{name}").parse()?; let append = |code: &mut Vec| { @@ -414,8 +414,11 @@ pub fn get_impl(module: &str, name: &str) -> Result<(Function, bool)> { Ok(()) }; - let debug = module == "console" || module == "debug"; - Function::new(&[], append, hostio.ty(), &[]).map(|x| (x, debug)) + Function::new(&[], append, hostio.ty(), &[]) +} + +pub fn hostio_module_is_debug(module: &str) -> bool { + module == "console" || module == "debug" } /// Adds internal functions to a module. diff --git a/arbitrator/prover/src/machine.rs b/arbitrator/prover/src/machine.rs index c045fc72c9..9f2ba081dc 100644 --- a/arbitrator/prover/src/machine.rs +++ b/arbitrator/prover/src/machine.rs @@ -346,14 +346,95 @@ lazy_static! { }; } +trait ImportResolver: for<'a, 'b> Fn(&'a str, &'b str) -> Result + Sized { + fn compose(self, other: impl ImportResolver) -> impl ImportResolver { + move |module, name| { + self(module, name).or_else(|original_err| { + let mut result = other(module, name); + if result.is_err() { + for err_layer in original_err.chain().rev() { + result = result.wrap_err(format!("{err_layer}")); + } + } + result + }) + } + } + + fn condition(self, condition: bool) -> impl ImportResolver { + move |module, name| { + if condition { + self(module, name) + } else { + bail!("resolver disabled") + } + } + } +} + +impl Fn(&'a str, &'b str) -> Result> ImportResolver for F {} + impl Module { - fn from_binary( + fn make_imports_resolver(import_map: &HashMap) -> impl ImportResolver + '_ { + move |module, name| { + let qualified_name = format!("{module}__{name}"); + + let Some(import) = import_map.get(&qualified_name) else { + bail!("func not found {module} {name}") + }; + let wavm = vec![ + Instruction::simple(Opcode::InitFrame), + Instruction::with_data( + Opcode::CrossModuleCall, + pack_cross_module_call(import.module, import.func), + ), + Instruction::simple(Opcode::Return), + ]; + Ok(Function::new_from_wavm(wavm, import.ty.clone(), vec![])) + } + } + + fn make_forward_resolver(import_map: &HashMap) -> impl ImportResolver + '_ { + move |module, name| { + let qualified_name = format!("{module}__{name}"); + + let Some(import) = import_map.get(&qualified_name) else { + bail!("func not found {module} {name}") + }; + let wavm = vec![ + Instruction::simple(Opcode::InitFrame), + Instruction::with_data( + Opcode::CrossModuleForward, + pack_cross_module_call(import.module, import.func), + ), + Instruction::simple(Opcode::Return), + ]; + Ok(Function::new_from_wavm(wavm, import.ty.clone(), vec![])) + } + } + + fn hostio_resolver(module: &str, name: &str) -> Result { + if host::hostio_module_is_debug(module) { + return Self::notfound_resolver(module, name); + } + host::get_impl(module, name) + } + + fn debug_resolver(module: &str, name: &str) -> Result { + if !host::hostio_module_is_debug(module) { + return Self::notfound_resolver(module, name); + } + host::get_impl(module, name) + } + + fn notfound_resolver(module: &str, name: &str) -> Result { + bail!("import not found {module} {name}") + } + + fn from_binary( bin: &WasmBinary, - available_imports: &HashMap, + import_resolver: R, floating_point_impls: &FloatingPointImpls, - allow_hostapi: bool, - is_forwarder: bool, - debug_funcs: bool, stylus_data: Option, ) -> Result { let mut code = Vec::new(); @@ -366,38 +447,8 @@ impl Module { let module = import.module; let have_ty = &bin.types[import.offset as usize]; - let qualified_name = format!("{module}__{}", import.name); + let func = import_resolver(import.module, import.name)?; - let func = if let Some(import) = available_imports.get(&qualified_name) { - let call = match is_forwarder { - true => Opcode::CrossModuleForward, - false => Opcode::CrossModuleCall, - }; - let wavm = vec![ - Instruction::simple(Opcode::InitFrame), - Instruction::with_data( - call, - pack_cross_module_call(import.module, import.func), - ), - Instruction::simple(Opcode::Return), - ]; - Function::new_from_wavm(wavm, import.ty.clone(), vec![]) - } else if let Ok((hostio, debug)) = host::get_impl(import.module, import.name) { - ensure!( - (debug && debug_funcs) || (!debug && allow_hostapi), - "Host func {} in {} not enabled debug_funcs={debug_funcs} hostapi={allow_hostapi} debug={debug}", - import.name.red(), - import.module.red(), - ); - hostio - } else { - bail!( - "No such import {} in {} for {}", - import.name.red(), - import.module.red(), - bin_name.red() - ) - }; ensure!( &func.ty == have_ty, "Import {} for {} has different function signature than export.\nexpected {} in {}\nbut have {}", @@ -602,11 +653,9 @@ impl Module { ) -> Result { Self::from_binary( bin, - &USER_IMPORTS, + Module::make_imports_resolver(&USER_IMPORTS) + .compose(Module::debug_resolver.condition(debug_funcs)), &HashMap::default(), - false, - false, - debug_funcs, stylus_data, ) } @@ -1377,11 +1426,8 @@ impl Machine { if let Some(lib) = forwarder { modules.push(Module::from_binary( &lib, - &available_imports, + Module::make_forward_resolver(&available_imports), &floating_point_impls, - true, - true, - debug_funcs, None, )?); } @@ -1389,22 +1435,20 @@ impl Machine { for lib in libraries.iter() { modules.push(Module::from_binary( lib, - &available_imports, + Module::hostio_resolver + .compose(Module::debug_resolver.condition(debug_funcs)) + .compose(Module::make_imports_resolver(&available_imports)), &floating_point_impls, - true, - false, - debug_funcs, None, )?); } modules.push(Module::from_binary( &bin, - &available_imports, + Module::make_imports_resolver(&available_imports) + .compose(Module::hostio_resolver.condition(allow_hostapi_from_main)) + .compose(Module::debug_resolver.condition(debug_funcs)), &floating_point_impls, - allow_hostapi_from_main, - false, - debug_funcs, stylus_data, )?); From 06954ca91f186d8bcf58bd3a1bc4611f719ea6e5 Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Wed, 11 Sep 2024 18:54:08 -0600 Subject: [PATCH 6/7] cargo fmt --- arbitrator/prover/src/machine.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/arbitrator/prover/src/machine.rs b/arbitrator/prover/src/machine.rs index 9f2ba081dc..2418d3cc2e 100644 --- a/arbitrator/prover/src/machine.rs +++ b/arbitrator/prover/src/machine.rs @@ -375,7 +375,9 @@ trait ImportResolver: for<'a, 'b> Fn(&'a str, &'b str) -> Result + Siz impl Fn(&'a str, &'b str) -> Result> ImportResolver for F {} impl Module { - fn make_imports_resolver(import_map: &HashMap) -> impl ImportResolver + '_ { + fn make_imports_resolver( + import_map: &HashMap, + ) -> impl ImportResolver + '_ { move |module, name| { let qualified_name = format!("{module}__{name}"); @@ -394,7 +396,9 @@ impl Module { } } - fn make_forward_resolver(import_map: &HashMap) -> impl ImportResolver + '_ { + fn make_forward_resolver( + import_map: &HashMap, + ) -> impl ImportResolver + '_ { move |module, name| { let qualified_name = format!("{module}__{name}"); From a9b455ca5328cf31a4a243220588ed513ebc8dbf Mon Sep 17 00:00:00 2001 From: Tsahi Zidenberg Date: Thu, 12 Sep 2024 12:19:08 -0600 Subject: [PATCH 7/7] makefile lint --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9ecba1ce2d..b736b04906 100644 --- a/Makefile +++ b/Makefile @@ -479,7 +479,7 @@ contracts/test/prover/proofs/rust-%.json: $(arbitrator_cases)/rust/$(wasm32_wasi $(prover_bin) $< $(arbitrator_wasm_lib_flags) -o $@ -b --allow-hostapi --require-success --inbox-add-stub-headers --inbox $(arbitrator_cases)/rust/data/msg0.bin --inbox $(arbitrator_cases)/rust/data/msg1.bin --delayed-inbox $(arbitrator_cases)/rust/data/msg0.bin --delayed-inbox $(arbitrator_cases)/rust/data/msg1.bin --preimages target/testdata/preimages.bin --with-forwarder contracts/test/prover/proofs/go.json: $(arbitrator_cases)/go/testcase.wasm $(prover_bin) $(arbitrator_wasm_libs) target/testdata/preimages.bin $(arbitrator_tests_link_deps) $(arbitrator_cases)/user.wasm - $(prover_bin) $< $(arbitrator_wasm_lib_flags) -o $@ -b --require-success --preimages target/testdata/preimages.bin --stylus-modules $(arbitrator_cases)/user.wasm --with-forwarder + $(prover_bin) $< $(arbitrator_wasm_lib_flags) -o $@ -b --require-success --preimages target/testdata/preimages.bin --stylus-modules $(arbitrator_cases)/user.wasm --with-forwarder # avoid testing user.wasm in onestepproofs. It can only run as stylus program. contracts/test/prover/proofs/user.json: