feat: direct db onto zktrie (#38)

* upgrade zktrie

* noisy log

* fix typo

* add chunk mode

* add chunk mode to binary tool

* disable key cache

* fix unwrap

* build once

* clippy

* add tracker

* add tracker

* fix some order

* fix clippy

* add metrics exporter

* some adjustment

* some adjustment

* typo

* adjust range based on super2

* fix

* fix

* fix

* update Cargo.lock

* fix clippy

* add new traces

* remove aggregator unit test

* fix

* suppress measure log

* refactor

* remove StateDB

* remove unneeded

* remove get_account

* fix clippy

* fix

* refactor

Author: lightsing

Cargo.toml

@@ -6,6 +6,7 @@ edition = "2021"
 [dependencies]
 hex = "0.4"
 itertools = "0.13.0" # for compatibility msrv 1.75
+once_cell = "1.19"
 rkyv = { version = "0.7", features = ["validation"] }
 thiserror = "1.0"
 tiny-keccak = "2.0"
@@ -41,7 +42,6 @@ ctor = { version = "0.2", optional = true }
 hyper = { version = "1.4", features = ["server", "http1"], optional = true }
 hyper-util = { version = "0.1", features = ["tokio"], optional = true }
 http-body-util = { version = "0.1", optional = true }
-once_cell = { version = "1.19", optional = true }
 prometheus-client = { version = "0.22", optional = true }
 pprof = { version = "0.13", features = ["flamegraph"], optional = true }
 tracing = { version = "0.1", optional = true }
@@ -100,7 +100,7 @@ profiling = ["pprof"]
 debug-account = ["csv", "revm/serde"]
 debug-storage = ["csv", "revm/serde"]
 dev = ["ctor", "tracing", "tracing-subscriber"]
-metrics = ["hyper", "hyper-util", "http-body-util", "once_cell", "prometheus-client", "tokio", "tokio/macros", "tokio/signal"]
+metrics = ["hyper", "hyper-util", "http-body-util", "prometheus-client", "tokio", "tokio/macros", "tokio/signal"]
 
 # sp1 related
 sp1 = []

src/bin/trace-verifier/commands/run_file.rs

@@ -74,28 +74,27 @@ impl RunFileCommand {
         }
 
         let fork_config = fork_config(traces[0].chain_id);
-        let (chunk_info, zktrie_state) = ChunkInfo::from_block_traces(&traces);
+        let (chunk_info, mut zktrie_state) = ChunkInfo::from_block_traces(&traces);
 
         let tx_bytes_hasher = Rc::new(RefCell::new(Keccak::v256()));
 
-        let mut executor = EvmExecutorBuilder::new()
+        let mut executor = EvmExecutorBuilder::new(&zktrie_state)
             .hardfork_config(fork_config)
             .with_execute_hooks(|hooks| {
                 let hasher = tx_bytes_hasher.clone();
                 hooks.add_tx_rlp_handler(move |_, rlp| {
                     hasher.borrow_mut().update(rlp);
                 });
             })
-            .zktrie_state(&zktrie_state)
-            .build(&traces[0]);
+            .build(&traces[0])?;
         executor.handle_block(&traces[0])?;
 
         for trace in traces[1..].iter() {
-            executor.update_db(trace, &zktrie_state);
+            executor.update_db(trace)?;
             executor.handle_block(trace)?;
         }
 
-        let post_state_root = executor.commit_changes();
+        let post_state_root = executor.commit_changes(&mut zktrie_state);
         if post_state_root != chunk_info.post_state_root() {
             bail!("post state root mismatch");
         }

src/bin/trace-verifier/utils.rs

@@ -1,5 +1,8 @@
 use eth_types::l2_types::BlockTrace;
-use stateless_block_verifier::{post_check, EvmExecutorBuilder, HardforkConfig, VerificationError};
+use mpt_zktrie::ZktrieState;
+use stateless_block_verifier::{
+    post_check, utils::ext::BlockZktrieExt, EvmExecutorBuilder, HardforkConfig, VerificationError,
+};
 
 pub fn verify(
     l2_trace: &BlockTrace,
@@ -35,7 +38,19 @@ fn verify_inner(
         .build()
         .unwrap();
 
-    let mut executor = EvmExecutorBuilder::new()
+    let mut zktrie_state = cycle_track!(
+        {
+            let old_root = l2_trace.storage_trace.root_before;
+            let mut zktrie_state = ZktrieState::construct(old_root);
+            l2_trace.build_zktrie_state(&mut zktrie_state);
+            zktrie_state
+        },
+        "build ZktrieState"
+    );
+
+    cycle_tracker_end!("build ZktrieState");
+
+    let mut executor = EvmExecutorBuilder::new(&zktrie_state)
         .hardfork_config(*fork_config)
         .with_execute_hooks(|hooks| {
             let l2_trace = l2_trace.clone();
@@ -45,7 +60,7 @@ fn verify_inner(
                 })
             }
         })
-        .build(&l2_trace);
+        .build(&l2_trace)?;
 
     // TODO: change to Result::inspect_err when sp1 toolchain >= 1.76
     #[allow(clippy::map_identity)]
@@ -58,7 +73,7 @@ fn verify_inner(
         update_metrics_counter!(verification_error);
         e
     })?;
-    let revm_root_after = executor.commit_changes();
+    let revm_root_after = executor.commit_changes(&mut zktrie_state);
 
     #[cfg(feature = "profiling")]
     if let Ok(report) = guard.report().build() {

src/chunk.rs

@@ -135,11 +135,11 @@ mod tests {
         });
 
         let fork_config = HardforkConfig::default_from_chain_id(traces[0].chain_id);
-        let (chunk_info, zktrie_state) = ChunkInfo::from_block_traces(&traces);
+        let (chunk_info, mut zktrie_state) = ChunkInfo::from_block_traces(&traces);
 
         let tx_bytes_hasher = Rc::new(RefCell::new(Keccak::v256()));
 
-        let mut executor = EvmExecutorBuilder::new()
+        let mut executor = EvmExecutorBuilder::new(&zktrie_state)
             .hardfork_config(fork_config)
             .with_execute_hooks(|hooks| {
                 let hasher = tx_bytes_hasher.clone();
@@ -148,15 +148,16 @@ mod tests {
                 });
             })
             .zktrie_state(&zktrie_state)
-            .build(&traces[0]);
+            .build(&traces[0])
+            .unwrap();
         executor.handle_block(&traces[0]).unwrap();
 
         for trace in traces[1..].iter() {
-            executor.update_db(trace, &zktrie_state);
+            executor.update_db(trace).unwrap();
             executor.handle_block(trace).unwrap();
         }
 
-        let post_state_root = executor.commit_changes();
+        let post_state_root = executor.commit_changes(&mut zktrie_state);
         assert_eq!(post_state_root, chunk_info.post_state_root);
         drop(executor); // drop executor to release Rc<Keccek>