From 662fd87288bebee14762b4399b66ca0f55fb14a8 Mon Sep 17 00:00:00 2001 From: tomarST <46097541+tomarST@users.noreply.github.com> Date: Sun, 26 Feb 2023 05:10:52 -0330 Subject: [PATCH 001/191] chore: fix all the obvious clippy warnings (#1520) --- crates/stages/src/stages/index_account_history.rs | 8 +++----- crates/stages/src/stages/index_storage_history.rs | 11 ++++------- crates/storage/db/benches/criterion.rs | 6 +++--- crates/storage/db/benches/hash_keys.rs | 8 ++++---- 4 files changed, 14 insertions(+), 19 deletions(-) diff --git a/crates/stages/src/stages/index_account_history.rs b/crates/stages/src/stages/index_account_history.rs index 5ff888019a2..ccfda386f45 100644 --- a/crates/stages/src/stages/index_account_history.rs +++ b/crates/stages/src/stages/index_account_history.rs @@ -258,8 +258,8 @@ mod tests { } async fn run(tx: &TestTransaction, run_to: u64) { - let mut input = ExecInput::default(); - input.previous_stage = Some((PREV_STAGE_ID, run_to)); + let input = + ExecInput { previous_stage: Some((PREV_STAGE_ID, run_to)), ..Default::default() }; let mut stage = IndexAccountHistoryStage::default(); let mut tx = tx.inner(); let out = stage.execute(&mut tx, input).await.unwrap(); @@ -268,9 +268,7 @@ mod tests { } async fn unwind(tx: &TestTransaction, unwind_from: u64, unwind_to: u64) { - let mut input = UnwindInput::default(); - input.stage_progress = unwind_from; - input.unwind_to = unwind_to; + let input = UnwindInput { stage_progress: unwind_from, unwind_to, ..Default::default() }; let mut stage = IndexAccountHistoryStage::default(); let mut tx = tx.inner(); let out = stage.unwind(&mut tx, input).await.unwrap(); diff --git a/crates/stages/src/stages/index_storage_history.rs b/crates/stages/src/stages/index_storage_history.rs index 576ff0bd5c0..efef9e065fe 100644 --- a/crates/stages/src/stages/index_storage_history.rs +++ b/crates/stages/src/stages/index_storage_history.rs @@ -282,8 +282,8 @@ mod tests { } async fn run(tx: &TestTransaction, run_to: u64) { - let mut input = ExecInput::default(); - input.previous_stage = Some((PREV_STAGE_ID, run_to)); + let input = + ExecInput { previous_stage: Some((PREV_STAGE_ID, run_to)), ..Default::default() }; let mut stage = IndexStorageHistoryStage::default(); let mut tx = tx.inner(); let out = stage.execute(&mut tx, input).await.unwrap(); @@ -292,9 +292,7 @@ mod tests { } async fn unwind(tx: &TestTransaction, unwind_from: u64, unwind_to: u64) { - let mut input = UnwindInput::default(); - input.stage_progress = unwind_from; - input.unwind_to = unwind_to; + let input = UnwindInput { stage_progress: unwind_from, unwind_to, ..Default::default() }; let mut stage = IndexStorageHistoryStage::default(); let mut tx = tx.inner(); let out = stage.unwind(&mut tx, input).await.unwrap(); @@ -357,8 +355,7 @@ mod tests { async fn insert_index_to_full_shard() { // init let tx = TestTransaction::default(); - let mut input = ExecInput::default(); - input.previous_stage = Some((PREV_STAGE_ID, 5)); + let _input = ExecInput { previous_stage: Some((PREV_STAGE_ID, 5)), ..Default::default() }; // change does not matter only that account is present in changeset. let full_list = vec![3; NUM_OF_INDICES_IN_SHARD]; diff --git a/crates/storage/db/benches/criterion.rs b/crates/storage/db/benches/criterion.rs index c0f73d160b9..2034f7df428 100644 --- a/crates/storage/db/benches/criterion.rs +++ b/crates/storage/db/benches/criterion.rs @@ -147,7 +147,7 @@ where crsr.append(k, v).expect("submit"); } - tx.inner.commit().unwrap(); + tx.inner.commit().unwrap() }); }, ) @@ -171,7 +171,7 @@ where crsr.insert(k, v).expect("submit"); } - tx.inner.commit().unwrap(); + tx.inner.commit().unwrap() }); }, ) @@ -241,7 +241,7 @@ where crsr.append_dup(k, v).expect("submit"); } - tx.inner.commit().unwrap(); + tx.inner.commit().unwrap() }); }, ) diff --git a/crates/storage/db/benches/hash_keys.rs b/crates/storage/db/benches/hash_keys.rs index b7cb5929747..b372dca41fd 100644 --- a/crates/storage/db/benches/hash_keys.rs +++ b/crates/storage/db/benches/hash_keys.rs @@ -39,7 +39,7 @@ pub fn hash_keys(c: &mut Criterion) { group.sample_size(10); - for size in vec![10_000, 100_000, 1_000_000] { + for size in [10_000, 100_000, 1_000_000] { measure_table_insertion::(&mut group, size); } } @@ -176,7 +176,7 @@ where crsr.append(k, v).expect("submit"); } - tx.inner.commit().unwrap(); + tx.inner.commit().unwrap() }); } db @@ -197,7 +197,7 @@ where crsr.insert(k, v).expect("submit"); } - tx.inner.commit().unwrap(); + tx.inner.commit().unwrap() }); } db @@ -214,7 +214,7 @@ where tx.put::(k, v).expect("submit"); } - tx.inner.commit().unwrap(); + tx.inner.commit().unwrap() }); } db From 17474f309ff621c75863f6ffd6184e9aecd8be2d Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 27 Feb 2023 09:06:07 +0100 Subject: [PATCH 002/191] chore(deps): rm unused deps (#1557) --- Cargo.lock | 6 ------ bin/reth/Cargo.toml | 6 ------ bin/reth/src/lib.rs | 2 +- 3 files changed, 1 insertion(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 114e99007c0..23715789384 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4357,7 +4357,6 @@ dependencies = [ "futures", "human_bytes", "jsonrpsee", - "metrics", "metrics-exporter-prometheus", "metrics-util", "proptest", @@ -4365,8 +4364,6 @@ dependencies = [ "reth-db", "reth-discv4", "reth-downloaders", - "reth-eth-wire", - "reth-executor", "reth-interfaces", "reth-net-nat", "reth-network", @@ -4386,12 +4383,9 @@ dependencies = [ "serde_json", "shellexpand", "tempfile", - "thiserror", "tokio", - "tokio-stream", "tracing", "tui", - "walkdir", ] [[package]] diff --git a/bin/reth/Cargo.toml b/bin/reth/Cargo.toml index fea91f9e8b1..947ef36a6b3 100644 --- a/bin/reth/Cargo.toml +++ b/bin/reth/Cargo.toml @@ -17,8 +17,6 @@ reth-stages = { path = "../../crates/stages"} reth-interfaces = { path = "../../crates/interfaces", features = ["test-utils"] } reth-transaction-pool = { path = "../../crates/transaction-pool", features = ["test-utils"] } reth-consensus = { path = "../../crates/consensus" } -reth-executor = { path = "../../crates/executor" } -reth-eth-wire = { path = "../../crates/net/eth-wire" } reth-rpc-engine-api = { path = "../../crates/rpc/rpc-engine-api" } reth-rpc-builder = { path = "../../crates/rpc/rpc-builder" } reth-rpc = { path = "../../crates/rpc/rpc" } @@ -36,7 +34,6 @@ tracing = "0.1" # io fdlimit = "0.2.1" -walkdir = "2.3" serde = "1.0" serde_json = "1.0" shellexpand = "3.0.0" @@ -44,7 +41,6 @@ dirs-next = "2.0.0" confy = "0.5" # rpc/metrics -metrics = "0.20.1" metrics-exporter-prometheus = { version = "0.11.0", features = ["http-listener"] } metrics-util = "0.14.0" @@ -54,9 +50,7 @@ proptest = "1.0" # misc eyre = "0.6.8" clap = { version = "4.0", features = ["derive", "cargo"] } -thiserror = "1.0" tokio = { version = "1.21", features = ["sync", "macros", "rt-multi-thread"] } -tokio-stream = "0.1" futures = "0.3.25" tempfile = { version = "3.3.0" } backon = "0.4" diff --git a/bin/reth/src/lib.rs b/bin/reth/src/lib.rs index d22105b0506..36ebcf6324a 100644 --- a/bin/reth/src/lib.rs +++ b/bin/reth/src/lib.rs @@ -1,4 +1,4 @@ -#![warn(missing_docs, unreachable_pub)] +#![warn(missing_docs, unreachable_pub, unused_crate_dependencies)] #![deny(unused_must_use, rust_2018_idioms)] #![doc(test( no_crate_inject, From cea56f1e1d8d3913f3a7c7feb08922fac83beec1 Mon Sep 17 00:00:00 2001 From: Tom French <15848336+TomAFrench@users.noreply.github.com> Date: Mon, 27 Feb 2023 10:22:11 +0000 Subject: [PATCH 003/191] chore: silence unwanted clippy warning in test (#1566) --- crates/storage/db/src/implementation/mdbx/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/storage/db/src/implementation/mdbx/mod.rs b/crates/storage/db/src/implementation/mdbx/mod.rs index 8b9189c6956..92f4860ff89 100644 --- a/crates/storage/db/src/implementation/mdbx/mod.rs +++ b/crates/storage/db/src/implementation/mdbx/mod.rs @@ -279,6 +279,7 @@ mod tests { assert_eq!(walker.next(), None); } + #[allow(clippy::reversed_empty_ranges)] #[test] fn db_cursor_walk_range_invalid() { let db: Arc> = test_utils::create_test_db(EnvKind::RW); From 2d0699e98ab019245ca14ef1c0b1a27f91394ff6 Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Mon, 27 Feb 2023 14:56:27 +0200 Subject: [PATCH 004/191] fix: patch clap with fixed clippy warnings (#1573) --- Cargo.lock | 17 +++++++---------- bin/reth/Cargo.toml | 3 ++- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 23715789384..196a865ed33 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -757,13 +757,12 @@ dependencies = [ [[package]] name = "clap" -version = "4.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f13b9c79b5d1dd500d20ef541215a6423c75829ef43117e1b4d17fd8af0b5d76" +version = "4.1.6" +source = "git+https://github.com/rkrasiuk/clap?branch=rkrasiuk/fix-almost-swapped-lint#c1caf462ce40fc541066dfb2a9cccb586777a757" dependencies = [ "bitflags", "clap_derive", - "clap_lex 0.3.1", + "clap_lex 0.3.2", "is-terminal", "once_cell", "strsim 0.10.0", @@ -773,8 +772,7 @@ dependencies = [ [[package]] name = "clap_derive" version = "4.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "684a277d672e91966334af371f1a7b5833f9aa00b07c84e92fbce95e00208ce8" +source = "git+https://github.com/rkrasiuk/clap?branch=rkrasiuk/fix-almost-swapped-lint#c1caf462ce40fc541066dfb2a9cccb586777a757" dependencies = [ "heck", "proc-macro-error", @@ -794,9 +792,8 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "783fe232adfca04f90f56201b26d79682d4cd2625e0bc7290b95123afe558ade" +version = "0.3.2" +source = "git+https://github.com/rkrasiuk/clap?branch=rkrasiuk/fix-almost-swapped-lint#c1caf462ce40fc541066dfb2a9cccb586777a757" dependencies = [ "os_str_bytes", ] @@ -4347,7 +4344,7 @@ name = "reth" version = "0.1.0" dependencies = [ "backon", - "clap 4.1.4", + "clap 4.1.6", "comfy-table", "confy", "crossterm", diff --git a/bin/reth/Cargo.toml b/bin/reth/Cargo.toml index 947ef36a6b3..1c9a16d1c0e 100644 --- a/bin/reth/Cargo.toml +++ b/bin/reth/Cargo.toml @@ -49,7 +49,8 @@ proptest = "1.0" # misc eyre = "0.6.8" -clap = { version = "4.0", features = ["derive", "cargo"] } +# TODO: temp patch. tracked in https://github.com/paradigmxyz/reth/issues/1572 +clap = { git = "https://github.com/rkrasiuk/clap", branch = "rkrasiuk/fix-almost-swapped-lint", features = ["derive", "cargo"] } # { version = "4.0", features = ["derive", "cargo"] } tokio = { version = "1.21", features = ["sync", "macros", "rt-multi-thread"] } futures = "0.3.25" tempfile = { version = "3.3.0" } From 7275f8d9227287e9f863cb53deaaf023fce7ea01 Mon Sep 17 00:00:00 2001 From: Tom French <15848336+TomAFrench@users.noreply.github.com> Date: Mon, 27 Feb 2023 13:08:20 +0000 Subject: [PATCH 005/191] chore: check genesis mismatch in `init_genesis` (#1560) --- bin/reth/src/chain/init.rs | 11 +----- crates/staged-sync/Cargo.toml | 3 +- crates/staged-sync/src/utils/init.rs | 53 +++++++++++++++++++++++----- 3 files changed, 47 insertions(+), 20 deletions(-) diff --git a/bin/reth/src/chain/init.rs b/bin/reth/src/chain/init.rs index e8eb32afebe..82d7d1f434c 100644 --- a/bin/reth/src/chain/init.rs +++ b/bin/reth/src/chain/init.rs @@ -49,16 +49,7 @@ impl InitCommand { info!(target: "reth::cli", "Database opened"); info!(target: "reth::cli", "Writing genesis block"); - let genesis_hash = init_genesis(db, self.chain.clone())?; - - if genesis_hash != self.chain.genesis_hash() { - // TODO: better error text - return Err(eyre::eyre!( - "Genesis hash mismatch: expected {}, got {}", - self.chain.genesis_hash(), - genesis_hash - )) - } + init_genesis(db, self.chain.clone())?; Ok(()) } diff --git a/crates/staged-sync/Cargo.toml b/crates/staged-sync/Cargo.toml index dc440eaa634..5521c66634a 100644 --- a/crates/staged-sync/Cargo.toml +++ b/crates/staged-sync/Cargo.toml @@ -41,7 +41,7 @@ tracing = "0.1.37" rand = { version = "0.8", optional = true } # errors -thiserror = { version = "1", optional = true } +thiserror = "1" # enr enr = { version = "0.7.0", features = ["serde", "rust-secp256k1"], optional = true } @@ -89,7 +89,6 @@ test-utils = [ "dep:enr", "dep:ethers-core", "dep:tempfile", - "dep:thiserror", "dep:hex", "dep:rand", "dep:tokio", diff --git a/crates/staged-sync/src/utils/init.rs b/crates/staged-sync/src/utils/init.rs index 5551ee7d9dd..a43514881c4 100644 --- a/crates/staged-sync/src/utils/init.rs +++ b/crates/staged-sync/src/utils/init.rs @@ -21,14 +21,37 @@ pub fn init_db>(path: P) -> eyre::Result> { Ok(db) } +/// Database initialization error type. +#[derive(Debug, thiserror::Error, PartialEq, Eq, Clone)] +pub enum InitDatabaseError { + /// Attempted to reinitialize database with inconsistent genesis block + #[error("Genesis hash mismatch: expected {expected}, got {actual}")] + GenesisHashMismatch { expected: H256, actual: H256 }, + + /// Low-level database error. + #[error(transparent)] + DBError(#[from] reth_db::Error), +} + /// Write the genesis block if it has not already been written #[allow(clippy::field_reassign_with_default)] -pub fn init_genesis(db: Arc, chain: ChainSpec) -> Result { +pub fn init_genesis( + db: Arc, + chain: ChainSpec, +) -> Result { let genesis = chain.genesis(); + + let header = chain.genesis_header(); + let hash = header.hash_slow(); + let tx = db.tx()?; - if let Some((_, hash)) = tx.cursor_read::()?.first()? { - debug!("Genesis already written, skipping."); - return Ok(hash) + if let Some((_, db_hash)) = tx.cursor_read::()?.first()? { + if db_hash == hash { + debug!("Genesis already written, skipping."); + return Ok(hash) + } + + return Err(InitDatabaseError::GenesisHashMismatch { expected: hash, actual: db_hash }) } drop(tx); @@ -48,9 +71,6 @@ pub fn init_genesis(db: Arc, chain: ChainSpec) -> Result(0, hash)?; tx.put::(hash, 0)?; tx.put::(0, Default::default())?; @@ -65,7 +85,7 @@ pub fn init_genesis(db: Arc, chain: ChainSpec) -> Result Date: Mon, 27 Feb 2023 08:21:34 -0700 Subject: [PATCH 006/191] feat: Hook on Execution (#1567) --- Cargo.lock | 1 + crates/executor/Cargo.toml | 1 + crates/executor/src/executor.rs | 98 +++++---- crates/revm/revm-inspectors/src/lib.rs | 5 + crates/revm/revm-inspectors/src/stack.rs | 240 +++++++++++++++++++++++ 5 files changed, 312 insertions(+), 33 deletions(-) create mode 100644 crates/revm/revm-inspectors/src/stack.rs diff --git a/Cargo.lock b/Cargo.lock index 196a865ed33..6cabf34e4d4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4598,6 +4598,7 @@ dependencies = [ "reth-primitives", "reth-provider", "reth-revm", + "reth-revm-inspectors", "reth-rlp", "revm", "rlp", diff --git a/crates/executor/Cargo.toml b/crates/executor/Cargo.toml index 18ff91ecfca..41e42af20a2 100644 --- a/crates/executor/Cargo.toml +++ b/crates/executor/Cargo.toml @@ -11,6 +11,7 @@ readme = "README.md" reth-primitives = { path = "../primitives" } reth-interfaces = { path = "../interfaces" } reth-revm = { path = "../revm" } +reth-revm-inspectors = { path = "../revm/revm-inspectors" } reth-rlp = { path = "../rlp" } reth-db = { path = "../storage/db" } reth-provider = { path = "../storage/provider" } diff --git a/crates/executor/src/executor.rs b/crates/executor/src/executor.rs index 24125d5551e..5a7cd69776d 100644 --- a/crates/executor/src/executor.rs +++ b/crates/executor/src/executor.rs @@ -1,6 +1,7 @@ use crate::execution_result::{ AccountChangeSet, AccountInfoChangeSet, ExecutionResult, TransactionChangeSet, }; + use hashbrown::hash_map::Entry; use reth_interfaces::executor::{BlockExecutor, Error}; use reth_primitives::{ @@ -14,6 +15,7 @@ use reth_revm::{ env::{fill_cfg_and_block_env, fill_tx_env}, into_reth_log, to_reth_acc, }; +use reth_revm_inspectors::stack::{InspectorStack, InspectorStackConfig}; use revm::{ db::AccountState, primitives::{Account as RevmAccount, AccountInfo, Bytecode, ResultAndState}, @@ -28,19 +30,24 @@ where { chain_spec: &'a ChainSpec, evm: EVM<&'a mut SubState>, - /// Enable revm inspector printer. - /// In execution this will print opcode level traces directly to console. - pub use_printer_tracer: bool, + stack: InspectorStack, } impl<'a, DB> Executor<'a, DB> where DB: StateProvider, { - fn new(chain_spec: &'a ChainSpec, db: &'a mut SubState) -> Self { + /// Creates a new executor from the given chain spec and database. + pub fn new(chain_spec: &'a ChainSpec, db: &'a mut SubState) -> Self { let mut evm = EVM::new(); evm.database(db); - Executor { chain_spec, evm, use_printer_tracer: false } + Executor { chain_spec, evm, stack: InspectorStack::new(InspectorStackConfig::default()) } + } + + /// Configures the executor with the given inspectors. + pub fn with_stack(mut self, stack: InspectorStack) -> Self { + self.stack = stack; + self } fn db(&mut self) -> &mut SubState { @@ -327,18 +334,44 @@ where } } } -} -impl<'a, DB> BlockExecutor for Executor<'a, DB> -where - DB: StateProvider, -{ - fn execute( + /// Runs a single transaction in the configured environment and proceeds + /// to return the result and state diff (without applying it). + /// + /// Assumes the rest of the block environment has been filled via `init_block_env`. + pub fn transact( + &mut self, + transaction: &TransactionSigned, + sender: Address, + ) -> Result { + // Fill revm structure. + fill_tx_env(&mut self.evm.env.tx, transaction, sender); + + let out = if self.stack.should_inspect(&self.evm.env, transaction.hash()) { + // execution with inspector. + let output = self.evm.inspect(&mut self.stack); + tracing::trace!( + target: "evm", + hash = ?transaction.hash(), ?output, ?transaction, env = ?self.evm.env, + "Executed transaction" + ); + output + } else { + // main execution. + self.evm.transact() + }; + out.map_err(|e| Error::EVM(format!("{e:?}"))) + } + + /// Runs the provided transactions and commits their state. Will proceed + /// to return the total gas used by this batch of transaction as well as the + /// changesets generated by each tx. + pub fn execute_transactions( &mut self, block: &Block, total_difficulty: U256, senders: Option>, - ) -> Result { + ) -> Result<(Vec, u64), Error> { let senders = self.recover_senders(&block.body, senders)?; self.init_env(&block.header, total_difficulty); @@ -357,27 +390,8 @@ where block_available_gas, }) } - - // Fill revm structure. - fill_tx_env(&mut self.evm.env.tx, transaction, sender); - // Execute transaction. - let out = if self.use_printer_tracer { - // execution with inspector. - let output = self.evm.inspect(revm::inspectors::CustomPrintTracer::default()); - tracing::trace!( - target: "evm", - hash = ?transaction.hash(), ?output, ?transaction, env = ?self.evm.env, - "Executed transaction" - ); - output - } else { - // main execution. - self.evm.transact() - }; - - // cast the error and extract returnables. - let ResultAndState { result, state } = out.map_err(|e| Error::EVM(format!("{e:?}")))?; + let ResultAndState { result, state } = self.transact(transaction, sender)?; // commit changes let (changeset, new_bytecodes) = self.commit_changes(state); @@ -404,6 +418,23 @@ where }); } + Ok((tx_changesets, cumulative_gas_used)) + } +} + +impl<'a, DB> BlockExecutor for Executor<'a, DB> +where + DB: StateProvider, +{ + fn execute( + &mut self, + block: &Block, + total_difficulty: U256, + senders: Option>, + ) -> Result { + let (tx_changesets, cumulative_gas_used) = + self.execute_transactions(block, total_difficulty, senders)?; + // Check if gas used matches the value set in header. if block.gas_used != cumulative_gas_used { return Err(Error::BlockGasUsed { got: cumulative_gas_used, expected: block.gas_used }) @@ -485,7 +516,8 @@ pub fn execute( chain_spec: &ChainSpec, db: &mut SubState, ) -> Result { - let mut executor = Executor::new(chain_spec, db); + let mut executor = Executor::new(chain_spec, db) + .with_stack(InspectorStack::new(InspectorStackConfig::default())); executor.execute(block, total_difficulty, senders) } diff --git a/crates/revm/revm-inspectors/src/lib.rs b/crates/revm/revm-inspectors/src/lib.rs index cec3a9d0912..b4d79c40603 100644 --- a/crates/revm/revm-inspectors/src/lib.rs +++ b/crates/revm/revm-inspectors/src/lib.rs @@ -9,3 +9,8 @@ /// An inspector implementation for an EIP2930 Accesslist pub mod access_list; + +/// An inspector stack abstracting the implementation details of +/// each inspector and allowing to hook on block/transaciton execution, +/// used in the main RETH executor. +pub mod stack; diff --git a/crates/revm/revm-inspectors/src/stack.rs b/crates/revm/revm-inspectors/src/stack.rs new file mode 100644 index 00000000000..12e3df4050a --- /dev/null +++ b/crates/revm/revm-inspectors/src/stack.rs @@ -0,0 +1,240 @@ +use reth_primitives::{bytes::Bytes, Address, TxHash, H256}; +use revm::{ + inspectors::CustomPrintTracer, + interpreter::{CallInputs, CreateInputs, Gas, InstructionResult, Interpreter}, + primitives::Env, + Database, EVMData, Inspector, +}; + +/// One can hook on inspector execution in 3 ways: +/// - Block: Hook on block execution +/// - BlockWithIndex: Hook on block execution transaction index +/// - Transaction: Hook on a specific transaction hash +#[derive(Default)] +pub enum Hook { + #[default] + /// No hook. + None, + /// Hook on a specific block. + Block(u64), + /// Hook on a specific transaction hash. + Transaction(TxHash), + /// Hooks on every transaction in a block. + All, +} + +#[derive(Default)] +/// An inspector that calls multiple inspectors in sequence. +/// +/// If a call to an inspector returns a value other than [InstructionResult::Continue] (or +/// equivalent) the remaining inspectors are not called. +pub struct InspectorStack { + /// An inspector that prints the opcode traces to the console. + pub custom_print_tracer: Option, + /// The provided hook + pub hook: Hook, +} + +impl InspectorStack { + /// Create a new inspector stack. + pub fn new(config: InspectorStackConfig) -> Self { + let mut stack = InspectorStack { hook: config.hook, ..Default::default() }; + + if config.use_printer_tracer { + stack.custom_print_tracer = Some(CustomPrintTracer::default()); + } + + stack + } + + /// Check if the inspector should be used. + pub fn should_inspect(&self, env: &Env, tx_hash: TxHash) -> bool { + match self.hook { + Hook::None => false, + Hook::Block(block) => env.block.number.to::() == block, + Hook::Transaction(hash) => hash == tx_hash, + Hook::All => true, + } + } +} + +#[derive(Default)] +/// Configuration for the inspectors. +pub struct InspectorStackConfig { + /// Enable revm inspector printer. + /// In execution this will print opcode level traces directly to console. + pub use_printer_tracer: bool, + + /// Hook on a specific block or transaction. + pub hook: Hook, +} + +/// Helper macro to call the same method on multiple inspectors without resorting to dynamic +/// dispatch +#[macro_export] +macro_rules! call_inspectors { + ($id:ident, [ $($inspector:expr),+ ], $call:block) => { + $({ + if let Some($id) = $inspector { + $call; + } + })+ + } +} + +impl Inspector for InspectorStack +where + DB: Database, +{ + fn initialize_interp( + &mut self, + interpreter: &mut Interpreter, + data: &mut EVMData<'_, DB>, + is_static: bool, + ) -> InstructionResult { + call_inspectors!(inspector, [&mut self.custom_print_tracer], { + let status = inspector.initialize_interp(interpreter, data, is_static); + + // Allow inspectors to exit early + if status != InstructionResult::Continue { + return status + } + }); + + InstructionResult::Continue + } + + fn step( + &mut self, + interpreter: &mut Interpreter, + data: &mut EVMData<'_, DB>, + is_static: bool, + ) -> InstructionResult { + call_inspectors!(inspector, [&mut self.custom_print_tracer], { + let status = inspector.step(interpreter, data, is_static); + + // Allow inspectors to exit early + if status != InstructionResult::Continue { + return status + } + }); + + InstructionResult::Continue + } + + fn log( + &mut self, + evm_data: &mut EVMData<'_, DB>, + address: &Address, + topics: &[H256], + data: &Bytes, + ) { + call_inspectors!(inspector, [&mut self.custom_print_tracer], { + inspector.log(evm_data, address, topics, data); + }); + } + + fn step_end( + &mut self, + interpreter: &mut Interpreter, + data: &mut EVMData<'_, DB>, + is_static: bool, + eval: InstructionResult, + ) -> InstructionResult { + call_inspectors!(inspector, [&mut self.custom_print_tracer], { + let status = inspector.step_end(interpreter, data, is_static, eval); + + // Allow inspectors to exit early + if status != InstructionResult::Continue { + return status + } + }); + + InstructionResult::Continue + } + + fn call( + &mut self, + data: &mut EVMData<'_, DB>, + inputs: &mut CallInputs, + is_static: bool, + ) -> (InstructionResult, Gas, Bytes) { + call_inspectors!(inspector, [&mut self.custom_print_tracer], { + let (status, gas, retdata) = inspector.call(data, inputs, is_static); + + // Allow inspectors to exit early + if status != InstructionResult::Continue { + return (status, gas, retdata) + } + }); + + (InstructionResult::Continue, Gas::new(inputs.gas_limit), Bytes::new()) + } + + fn call_end( + &mut self, + data: &mut EVMData<'_, DB>, + inputs: &CallInputs, + remaining_gas: Gas, + ret: InstructionResult, + out: Bytes, + is_static: bool, + ) -> (InstructionResult, Gas, Bytes) { + call_inspectors!(inspector, [&mut self.custom_print_tracer], { + let (new_ret, new_gas, new_out) = + inspector.call_end(data, inputs, remaining_gas, ret, out.clone(), is_static); + + // If the inspector returns a different ret or a revert with a non-empty message, + // we assume it wants to tell us something + if new_ret != ret || (new_ret == InstructionResult::Revert && new_out != out) { + return (new_ret, new_gas, new_out) + } + }); + + (ret, remaining_gas, out) + } + + fn create( + &mut self, + data: &mut EVMData<'_, DB>, + inputs: &mut CreateInputs, + ) -> (InstructionResult, Option
, Gas, Bytes) { + call_inspectors!(inspector, [&mut self.custom_print_tracer], { + let (status, addr, gas, retdata) = inspector.create(data, inputs); + + // Allow inspectors to exit early + if status != InstructionResult::Continue { + return (status, addr, gas, retdata) + } + }); + + (InstructionResult::Continue, None, Gas::new(inputs.gas_limit), Bytes::new()) + } + + fn create_end( + &mut self, + data: &mut EVMData<'_, DB>, + inputs: &CreateInputs, + ret: InstructionResult, + address: Option
, + remaining_gas: Gas, + out: Bytes, + ) -> (InstructionResult, Option
, Gas, Bytes) { + call_inspectors!(inspector, [&mut self.custom_print_tracer], { + let (new_ret, new_address, new_gas, new_retdata) = + inspector.create_end(data, inputs, ret, address, remaining_gas, out.clone()); + + if new_ret != ret { + return (new_ret, new_address, new_gas, new_retdata) + } + }); + + (ret, address, remaining_gas, out) + } + + fn selfdestruct(&mut self) { + call_inspectors!(inspector, [&mut self.custom_print_tracer], { + Inspector::::selfdestruct(inspector); + }); + } +} From 494a9e88b2c9956ed3e0f7b04658d65bb2a6a4c1 Mon Sep 17 00:00:00 2001 From: Aditya Pandey Date: Mon, 27 Feb 2023 20:58:29 +0530 Subject: [PATCH 007/191] test: serde roundtrip test for rpc types (#1571) --- crates/rpc/rpc-types/src/eth/block.rs | 39 +++++++++++++++++++ crates/rpc/rpc-types/src/eth/log.rs | 30 ++++++++++++++ .../rpc/rpc-types/src/eth/transaction/mod.rs | 36 +++++++++++++++++ 3 files changed, 105 insertions(+) diff --git a/crates/rpc/rpc-types/src/eth/block.rs b/crates/rpc/rpc-types/src/eth/block.rs index a5dde53f097..5b6c563ffca 100644 --- a/crates/rpc/rpc-types/src/eth/block.rs +++ b/crates/rpc/rpc-types/src/eth/block.rs @@ -330,4 +330,43 @@ mod tests { let full = false; assert_eq!(BlockTransactionsKind::Hashes, full.into()); } + + #[test] + fn serde_block() { + let block = Block { + header: Header { + hash: Some(H256::from_low_u64_be(1)), + parent_hash: H256::from_low_u64_be(2), + uncles_hash: H256::from_low_u64_be(3), + author: Address::from_low_u64_be(4), + miner: Address::from_low_u64_be(4), + state_root: H256::from_low_u64_be(5), + transactions_root: H256::from_low_u64_be(6), + receipts_root: H256::from_low_u64_be(7), + withdrawals_root: Some(H256::from_low_u64_be(8)), + number: Some(U256::from(9)), + gas_used: U256::from(10), + gas_limit: U256::from(11), + extra_data: Bytes::from(vec![1, 2, 3]), + logs_bloom: Bloom::default(), + timestamp: U256::from(12), + difficulty: U256::from(13), + mix_hash: H256::from_low_u64_be(14), + nonce: Some(H64::from_low_u64_be(15)), + }, + total_difficulty: U256::from(100000), + uncles: vec![H256::from_low_u64_be(17)], + transactions: BlockTransactions::Hashes(vec![H256::from_low_u64_be(18)]), + size: Some(U256::from(19)), + base_fee_per_gas: Some(U256::from(20)), + withdrawals: None, + }; + let serialized = serde_json::to_string(&block).unwrap(); + assert_eq!( + serialized, + r#"{"hash":"0x0000000000000000000000000000000000000000000000000000000000000001","parentHash":"0x0000000000000000000000000000000000000000000000000000000000000002","sha3Uncles":"0x0000000000000000000000000000000000000000000000000000000000000003","author":"0x0000000000000000000000000000000000000004","miner":"0x0000000000000000000000000000000000000004","stateRoot":"0x0000000000000000000000000000000000000000000000000000000000000005","transactionsRoot":"0x0000000000000000000000000000000000000000000000000000000000000006","receiptsRoot":"0x0000000000000000000000000000000000000000000000000000000000000007","withdrawalsRoot":"0x0000000000000000000000000000000000000000000000000000000000000008","number":"0x9","gasUsed":"0xa","gasLimit":"0xb","extraData":"0x010203","logsBloom":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","timestamp":"0xc","difficulty":"0xd","mixHash":"0x000000000000000000000000000000000000000000000000000000000000000e","nonce":"0x000000000000000f","totalDifficulty":"0x186a0","uncles":["0x0000000000000000000000000000000000000000000000000000000000000011"],"transactions":["0x0000000000000000000000000000000000000000000000000000000000000012"],"size":"0x13","baseFeePerGas":"0x14","withdrawals":null}"# + ); + let deserialized: Block = serde_json::from_str(&serialized).unwrap(); + assert_eq!(block, deserialized); + } } diff --git a/crates/rpc/rpc-types/src/eth/log.rs b/crates/rpc/rpc-types/src/eth/log.rs index f17eda38d40..0eee01fbbd6 100644 --- a/crates/rpc/rpc-types/src/eth/log.rs +++ b/crates/rpc/rpc-types/src/eth/log.rs @@ -27,3 +27,33 @@ pub struct Log { #[serde(default)] pub removed: bool, } + +#[cfg(test)] +mod tests { + use super::*; + use serde_json; + + #[test] + fn serde_log() { + let log = Log { + address: Address::from_low_u64_be(0x1234), + topics: vec![H256::from_low_u64_be(0x1234)], + data: Bytes::from(vec![0x12, 0x34]), + block_hash: Some(H256::from_low_u64_be(0x1234)), + block_number: Some(U256::from(0x1234)), + transaction_hash: Some(H256::from_low_u64_be(0x1234)), + transaction_index: Some(U256::from(0x1234)), + log_index: Some(U256::from(0x1234)), + transaction_log_index: Some(U256::from(0x1234)), + removed: false, + }; + let serialized = serde_json::to_string(&log).unwrap(); + assert_eq!( + serialized, + r#"{"address":"0x0000000000000000000000000000000000001234","topics":["0x0000000000000000000000000000000000000000000000000000000000001234"],"data":"0x1234","blockHash":"0x0000000000000000000000000000000000000000000000000000000000001234","blockNumber":"0x1234","transactionHash":"0x0000000000000000000000000000000000000000000000000000000000001234","transactionIndex":"0x1234","logIndex":"0x1234","transactionLogIndex":"0x1234","removed":false}"# + ); + + let deserialized: Log = serde_json::from_str(&serialized).unwrap(); + assert_eq!(log, deserialized); + } +} diff --git a/crates/rpc/rpc-types/src/eth/transaction/mod.rs b/crates/rpc/rpc-types/src/eth/transaction/mod.rs index 940085948ab..bce73152339 100644 --- a/crates/rpc/rpc-types/src/eth/transaction/mod.rs +++ b/crates/rpc/rpc-types/src/eth/transaction/mod.rs @@ -153,3 +153,39 @@ impl Transaction { } } } + +#[cfg(test)] +mod tests { + use super::*; + use serde_json; + + #[test] + fn serde_transaction() { + let transaction = Transaction { + hash: H256::from_low_u64_be(1), + nonce: U256::from(2), + block_hash: Some(H256::from_low_u64_be(3)), + block_number: Some(U256::from(4)), + transaction_index: Some(U256::from(5)), + from: Address::from_low_u64_be(6), + to: Some(Address::from_low_u64_be(7)), + value: U256::from(8), + gas_price: Some(U128::from(9)), + gas: U256::from(10), + input: Bytes::from(vec![11, 12, 13]), + signature: Some(Signature { v: U256::from(14), r: U256::from(14), s: U256::from(14) }), + chain_id: Some(U64::from(17)), + access_list: None, + transaction_type: Some(U64::from(20)), + max_fee_per_gas: Some(U128::from(21)), + max_priority_fee_per_gas: Some(U128::from(22)), + }; + let serialized = serde_json::to_string(&transaction).unwrap(); + assert_eq!( + serialized, + r#"{"hash":"0x0000000000000000000000000000000000000000000000000000000000000001","nonce":"0x2","blockHash":"0x0000000000000000000000000000000000000000000000000000000000000003","blockNumber":"0x4","transactionIndex":"0x5","from":"0x0000000000000000000000000000000000000006","to":"0x0000000000000000000000000000000000000007","value":"0x8","gasPrice":"0x9","gas":"0xa","maxFeePerGas":"0x15","maxPriorityFeePerGas":"0x16","input":"0x0b0c0d","r":"0xe","s":"0xe","v":"0xe","chainId":"0x11","type":"0x14"}"# + ); + let deserialized: Transaction = serde_json::from_str(&serialized).unwrap(); + assert_eq!(transaction, deserialized); + } +} From 5e0fa44094a6073e8b5eea219221130369d3b896 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 27 Feb 2023 16:40:27 +0100 Subject: [PATCH 008/191] fix(rpc): return eth_call fee errors as invalid params (#1559) --- crates/rpc/rpc/src/eth/error.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/rpc/rpc/src/eth/error.rs b/crates/rpc/rpc/src/eth/error.rs index e211b29c468..82761d51117 100644 --- a/crates/rpc/rpc/src/eth/error.rs +++ b/crates/rpc/rpc/src/eth/error.rs @@ -79,7 +79,12 @@ impl From for RpcError { EthApiError::InvalidTransactionSignature | EthApiError::EmptyRawTransactionData | EthApiError::UnknownBlockNumber | - EthApiError::InvalidBlockRange => rpc_err(INVALID_PARAMS_CODE, value.to_string(), None), + EthApiError::InvalidBlockRange | + EthApiError::ConflictingRequestGasPrice { .. } | + EthApiError::ConflictingRequestGasPriceAndTipSet { .. } | + EthApiError::RequestLegacyGasPriceAndTipSet { .. } => { + rpc_err(INVALID_PARAMS_CODE, value.to_string(), None) + } EthApiError::InvalidTransaction(err) => err.into(), err => internal_rpc_err(err.to_string()), } From 530a4a059a1b32b04e95732b034069d418d1f18d Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 28 Feb 2023 08:55:40 +0100 Subject: [PATCH 009/191] chore(deps): bump clap (#1574) --- Cargo.lock | 15 +++++++++------ bin/reth/Cargo.toml | 3 +-- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6cabf34e4d4..df4acabceb4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -757,8 +757,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.1.6" -source = "git+https://github.com/rkrasiuk/clap?branch=rkrasiuk/fix-almost-swapped-lint#c1caf462ce40fc541066dfb2a9cccb586777a757" +version = "4.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f3061d6db6d8fcbbd4b05e057f2acace52e64e96b498c08c2d7a4e65addd340" dependencies = [ "bitflags", "clap_derive", @@ -771,8 +772,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.1.0" -source = "git+https://github.com/rkrasiuk/clap?branch=rkrasiuk/fix-almost-swapped-lint#c1caf462ce40fc541066dfb2a9cccb586777a757" +version = "4.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34d122164198950ba84a918270a3bb3f7ededd25e15f7451673d986f55bd2667" dependencies = [ "heck", "proc-macro-error", @@ -793,7 +795,8 @@ dependencies = [ [[package]] name = "clap_lex" version = "0.3.2" -source = "git+https://github.com/rkrasiuk/clap?branch=rkrasiuk/fix-almost-swapped-lint#c1caf462ce40fc541066dfb2a9cccb586777a757" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "350b9cf31731f9957399229e9b2adc51eeabdfbe9d71d9a0552275fd12710d09" dependencies = [ "os_str_bytes", ] @@ -4344,7 +4347,7 @@ name = "reth" version = "0.1.0" dependencies = [ "backon", - "clap 4.1.6", + "clap 4.1.7", "comfy-table", "confy", "crossterm", diff --git a/bin/reth/Cargo.toml b/bin/reth/Cargo.toml index 1c9a16d1c0e..abe8a6e4771 100644 --- a/bin/reth/Cargo.toml +++ b/bin/reth/Cargo.toml @@ -49,8 +49,7 @@ proptest = "1.0" # misc eyre = "0.6.8" -# TODO: temp patch. tracked in https://github.com/paradigmxyz/reth/issues/1572 -clap = { git = "https://github.com/rkrasiuk/clap", branch = "rkrasiuk/fix-almost-swapped-lint", features = ["derive", "cargo"] } # { version = "4.0", features = ["derive", "cargo"] } +clap = { version = "4", features = ["derive", "cargo"] } tokio = { version = "1.21", features = ["sync", "macros", "rt-multi-thread"] } futures = "0.3.25" tempfile = { version = "3.3.0" } From 6cdf0a3ea633e9ce7ef05d26f06077e2a0456081 Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Tue, 28 Feb 2023 23:45:04 +0800 Subject: [PATCH 010/191] fix: `seek_by_key_subkey` usage on `HistoricalStateProvider` (#1584) --- crates/storage/db/src/abstraction/cursor.rs | 2 +- crates/storage/provider/src/providers/state/historical.rs | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/crates/storage/db/src/abstraction/cursor.rs b/crates/storage/db/src/abstraction/cursor.rs index c5d1703b22a..3e28b78c14a 100644 --- a/crates/storage/db/src/abstraction/cursor.rs +++ b/crates/storage/db/src/abstraction/cursor.rs @@ -73,7 +73,7 @@ pub trait DbDupCursorRO<'tx, T: DupSort> { /// Returns the next `value` of a duplicate `key`. fn next_dup_val(&mut self) -> ValueOnlyResult; - /// Seek by key and subkey + /// Seek by key and subkey. Make sure that the returned value subkey matches the queried one. fn seek_by_key_subkey(&mut self, key: T::Key, subkey: T::SubKey) -> ValueOnlyResult; /// Returns an iterator starting at a key greater or equal than `start_key` of a DupSort diff --git a/crates/storage/provider/src/providers/state/historical.rs b/crates/storage/provider/src/providers/state/historical.rs index cd7ce2c8e77..b5339469128 100644 --- a/crates/storage/provider/src/providers/state/historical.rs +++ b/crates/storage/provider/src/providers/state/historical.rs @@ -56,6 +56,7 @@ impl<'a, 'b, TX: DbTx<'a>> AccountProvider for HistoricalStateProviderRef<'a, 'b .tx .cursor_dup_read::()? .seek_by_key_subkey(changeset_transition_id, address)? + .filter(|acc| acc.address == address) .ok_or(ProviderError::AccountChangeset { transition_id: changeset_transition_id, address, @@ -95,6 +96,7 @@ impl<'a, 'b, TX: DbTx<'a>> StateProvider for HistoricalStateProviderRef<'a, 'b, .tx .cursor_dup_read::()? .seek_by_key_subkey((changeset_transition_id, address).into(), storage_key)? + .filter(|entry| entry.key == storage_key) .ok_or(ProviderError::StorageChangeset { transition_id: changeset_transition_id, address, @@ -107,8 +109,9 @@ impl<'a, 'b, TX: DbTx<'a>> StateProvider for HistoricalStateProviderRef<'a, 'b, Ok(self .tx .cursor_dup_read::()? - .seek_by_key_subkey(address, storage_key) - .map(|r| r.map(|entry| entry.value))?) + .seek_by_key_subkey(address, storage_key)? + .filter(|entry| entry.key == storage_key) + .map(|entry| entry.value)) } } From dc2f6047db71ef77eb4b599199f3a0bf7595f22e Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Tue, 28 Feb 2023 17:50:43 +0200 Subject: [PATCH 011/191] chore(deps): bump clap (#1585) --- Cargo.lock | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index df4acabceb4..3df3be67b16 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -757,9 +757,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.1.7" +version = "4.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f3061d6db6d8fcbbd4b05e057f2acace52e64e96b498c08c2d7a4e65addd340" +checksum = "c3d7ae14b20b94cb02149ed21a86c423859cbe18dc7ed69845cace50e52b40a5" dependencies = [ "bitflags", "clap_derive", @@ -772,9 +772,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.1.7" +version = "4.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34d122164198950ba84a918270a3bb3f7ededd25e15f7451673d986f55bd2667" +checksum = "44bec8e5c9d09e439c4335b1af0abaab56dcf3b94999a936e1bb47b9134288f0" dependencies = [ "heck", "proc-macro-error", @@ -4347,7 +4347,7 @@ name = "reth" version = "0.1.0" dependencies = [ "backon", - "clap 4.1.7", + "clap 4.1.8", "comfy-table", "confy", "crossterm", From 7a01e1e231194823836278a0044715f318bb6914 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 28 Feb 2023 20:46:25 +0100 Subject: [PATCH 012/191] feat: add eth state cache (#1561) --- Cargo.lock | 13 + crates/executor/src/executor.rs | 8 +- crates/interfaces/Cargo.toml | 2 +- crates/interfaces/src/provider.rs | 3 + crates/revm/revm-primitives/src/env.rs | 11 +- crates/rpc/rpc/Cargo.toml | 2 + crates/rpc/rpc/src/eth/cache.rs | 285 ++++++++++++++++++ crates/rpc/rpc/src/eth/mod.rs | 1 + crates/storage/provider/src/providers/mod.rs | 15 +- .../storage/provider/src/test_utils/mock.rs | 16 +- .../storage/provider/src/test_utils/noop.rs | 16 +- crates/storage/provider/src/traits/evm_env.rs | 14 +- 12 files changed, 364 insertions(+), 22 deletions(-) create mode 100644 crates/rpc/rpc/src/eth/cache.rs diff --git a/Cargo.lock b/Cargo.lock index 3df3be67b16..bfca56c1f7c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4913,6 +4913,7 @@ name = "reth-rpc" version = "0.1.0" dependencies = [ "async-trait", + "futures", "hex", "http", "http-body", @@ -4933,6 +4934,7 @@ dependencies = [ "reth-tasks", "reth-transaction-pool", "revm", + "schnellru", "secp256k1 0.26.0", "serde", "serde_json", @@ -5436,6 +5438,17 @@ dependencies = [ "windows-sys 0.42.0", ] +[[package]] +name = "schnellru" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "772575a524feeb803e5b0fcbc6dd9f367e579488197c94c6e4023aad2305774d" +dependencies = [ + "ahash 0.8.3", + "cfg-if", + "hashbrown 0.13.2", +] + [[package]] name = "scopeguard" version = "1.1.0" diff --git a/crates/executor/src/executor.rs b/crates/executor/src/executor.rs index 5a7cd69776d..f383d84e0bc 100644 --- a/crates/executor/src/executor.rs +++ b/crates/executor/src/executor.rs @@ -72,7 +72,13 @@ where /// Initializes the config and block env. fn init_env(&mut self, header: &Header, total_difficulty: U256) { - fill_cfg_and_block_env(&mut self.evm.env, self.chain_spec, header, total_difficulty); + fill_cfg_and_block_env( + &mut self.evm.env.cfg, + &mut self.evm.env.block, + self.chain_spec, + header, + total_difficulty, + ); } /// Commit change to database and return change diff that is used to update state and create diff --git a/crates/interfaces/Cargo.toml b/crates/interfaces/Cargo.toml index aeb1b909c5f..9448840bf3a 100644 --- a/crates/interfaces/Cargo.toml +++ b/crates/interfaces/Cargo.toml @@ -22,7 +22,7 @@ reth-eth-wire = { path = "../net/eth-wire" } # codecs parity-scale-codec = { version = "3.2.1", features = ["bytes"] } -futures = "0.3.25" +futures = "0.3" tokio-stream = "0.1.11" rand = "0.8.5" arbitrary = { version = "1.1.7", features = ["derive"], optional = true } diff --git a/crates/interfaces/src/provider.rs b/crates/interfaces/src/provider.rs index be00f651271..ac24be67a6c 100644 --- a/crates/interfaces/src/provider.rs +++ b/crates/interfaces/src/provider.rs @@ -71,4 +71,7 @@ pub enum ProviderError { /// Thrown when required header related data was not found but was required. #[error("requested data not found")] HeaderNotFound, + /// Thrown when the cache service task dropped + #[error("cache service task stopped")] + CacheServiceUnavailable, } diff --git a/crates/revm/revm-primitives/src/env.rs b/crates/revm/revm-primitives/src/env.rs index c6bbf1802f4..470684a7603 100644 --- a/crates/revm/revm-primitives/src/env.rs +++ b/crates/revm/revm-primitives/src/env.rs @@ -3,18 +3,19 @@ use reth_primitives::{ Address, ChainSpec, Head, Header, Transaction, TransactionKind, TransactionSigned, TxEip1559, TxEip2930, TxLegacy, U256, }; -use revm::primitives::{AnalysisKind, BlockEnv, CfgEnv, Env, SpecId, TransactTo, TxEnv}; +use revm::primitives::{AnalysisKind, BlockEnv, CfgEnv, SpecId, TransactTo, TxEnv}; /// Convenience function to call both [fill_cfg_env] and [fill_block_env] pub fn fill_cfg_and_block_env( - env: &mut Env, + cfg: &mut CfgEnv, + block_env: &mut BlockEnv, chain_spec: &ChainSpec, header: &Header, total_difficulty: U256, ) { - fill_cfg_env(&mut env.cfg, chain_spec, header, total_difficulty); - let after_merge = env.cfg.spec_id >= SpecId::MERGE; - fill_block_env(&mut env.block, header, after_merge); + fill_cfg_env(cfg, chain_spec, header, total_difficulty); + let after_merge = cfg.spec_id >= SpecId::MERGE; + fill_block_env(block_env, header, after_merge); } /// Fill [CfgEnv] fields according to the chain spec and given header diff --git a/crates/rpc/rpc/Cargo.toml b/crates/rpc/rpc/Cargo.toml index f86cb8a25ee..9e1a9a6126a 100644 --- a/crates/rpc/rpc/Cargo.toml +++ b/crates/rpc/rpc/Cargo.toml @@ -50,6 +50,8 @@ thiserror = "1.0" hex = "0.4" rand = "0.8.5" tracing = "0.1" +schnellru = "0.2" +futures = "0.3.26" [dev-dependencies] jsonrpsee = { version = "0.16", features = ["client"]} diff --git a/crates/rpc/rpc/src/eth/cache.rs b/crates/rpc/rpc/src/eth/cache.rs new file mode 100644 index 00000000000..edc7ed06091 --- /dev/null +++ b/crates/rpc/rpc/src/eth/cache.rs @@ -0,0 +1,285 @@ +//! Async caching support for eth RPC + +use futures::StreamExt; +use reth_interfaces::{provider::ProviderError, Result}; +use reth_primitives::{Block, H256}; +use reth_provider::{BlockProvider, EvmEnvProvider, StateProviderFactory}; +use reth_tasks::TaskSpawner; +use revm::primitives::{BlockEnv, CfgEnv}; +use schnellru::{ByMemoryUsage, Limiter, LruMap}; +use std::{ + collections::{hash_map::Entry, HashMap}, + future::Future, + hash::Hash, + pin::Pin, + task::{ready, Context, Poll}, +}; +use tokio::sync::{ + mpsc::{unbounded_channel, UnboundedSender}, + oneshot, +}; +use tokio_stream::wrappers::UnboundedReceiverStream; + +/// The type that can send the response to a requested [Block] +type BlockResponseSender = oneshot::Sender>>; + +/// The type that can send the response to a requested env +type EnvResponseSender = oneshot::Sender>; + +type BlockLruCache = MultiConsumerLruCache; + +type EnvLruCache = MultiConsumerLruCache; + +/// Provides async access to cached eth data +/// +/// This is the frontend to the [EthStateCacheService] which manages cached data on a different +/// task. +#[derive(Debug, Clone)] +pub(crate) struct EthStateCache { + to_service: UnboundedSender, +} + +impl EthStateCache { + /// Creates and returns both [EthStateCache] frontend and the memory bound service. + fn create( + client: Client, + action_task_spawner: Box, + max_block_bytes: usize, + max_env_bytes: usize, + ) -> (Self, EthStateCacheService) { + let (to_service, rx) = unbounded_channel(); + let service = EthStateCacheService { + client, + full_block_cache: BlockLruCache::with_memory_budget(max_block_bytes), + evm_env_cache: EnvLruCache::with_memory_budget(max_env_bytes), + action_tx: to_service.clone(), + action_rx: UnboundedReceiverStream::new(rx), + action_task_spawner, + }; + let cache = EthStateCache { to_service }; + (cache, service) + } + + /// Creates a new async LRU backed cache service task and spawns it to a new task via the given + /// spawner. + /// + /// The cache is memory limited by the given max bytes values. + pub(crate) fn spawn( + client: Client, + spawner: Box, + max_block_bytes: usize, + max_env_bytes: usize, + ) -> Self + where + Client: StateProviderFactory + BlockProvider + EvmEnvProvider + Clone + Unpin + 'static, + { + let (this, service) = Self::create(client, spawner.clone(), max_block_bytes, max_env_bytes); + spawner.spawn(Box::pin(service)); + this + } + + /// Requests the [Block] for the block hash + /// + /// Returns `None` if the block does not exist. + pub(crate) async fn get_block(&self, block_hash: H256) -> Result> { + let (response_tx, rx) = oneshot::channel(); + let _ = self.to_service.send(CacheAction::GetBlock { block_hash, response_tx }); + rx.await.map_err(|_| ProviderError::CacheServiceUnavailable)? + } + + /// Requests the evm env config for the block hash. + /// + /// Returns an error if the corresponding header (required for populating the envs) was not + /// found. + pub(crate) async fn get_evm_evn(&self, block_hash: H256) -> Result<(CfgEnv, BlockEnv)> { + let (response_tx, rx) = oneshot::channel(); + let _ = self.to_service.send(CacheAction::GetEnv { block_hash, response_tx }); + rx.await.map_err(|_| ProviderError::CacheServiceUnavailable)? + } +} + +/// A task than manages caches for data required by the `eth` rpc implementation. +/// +/// It provides a caching layer on top of the given [StateProvider](reth_provider::StateProvider) +/// and keeps data fetched via the provider in memory in an LRU cache. If the requested data is +/// missing in the cache it is fetched and inserted into the cache afterwards. While fetching data +/// from disk is sync, this service is async since requests and data is shared via channels. +/// +/// This type is an endless future that listens for incoming messages from the user facing +/// [EthStateCache] via a channel. If the requested data is not cached then it spawns a new task +/// that does the IO and sends the result back to it. This way the [EthStateCacheService] only +/// handles messages and does LRU lookups and never blocking IO. +/// +/// Caution: The channel for the data is _unbounded_ it is assumed that this is mainly used by the +/// [EthApi](crate::EthApi) which is typically invoked by the RPC server, which already uses permits +/// to limit concurrent requests. +#[must_use = "Type does nothing unless spawned"] +pub(crate) struct EthStateCacheService< + Client, + LimitBlocks = ByMemoryUsage, + LimitEnvs = ByMemoryUsage, +> where + LimitBlocks: Limiter, + LimitEnvs: Limiter, +{ + /// The type used to lookup data from disk + client: Client, + /// The LRU cache for full blocks grouped by their hash. + full_block_cache: BlockLruCache, + /// The LRU cache for revm environments + evm_env_cache: EnvLruCache, + /// Sender half of the action channel. + action_tx: UnboundedSender, + /// Receiver half of the action channel. + action_rx: UnboundedReceiverStream, + /// The type that's used to spawn tasks that do the actual work + action_task_spawner: Box, +} + +impl Future for EthStateCacheService +where + Client: StateProviderFactory + BlockProvider + EvmEnvProvider + Clone + Unpin + 'static, +{ + type Output = (); + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.get_mut(); + + loop { + match ready!(this.action_rx.poll_next_unpin(cx)) { + None => { + unreachable!("can't close") + } + Some(action) => { + match action { + CacheAction::GetBlock { block_hash, response_tx } => { + // check if block is cached + if let Some(block) = + this.full_block_cache.cache.get(&block_hash).cloned() + { + let _ = response_tx.send(Ok(Some(block))); + continue + } + + // block is not in the cache, request it if this is the first consumer + if this.full_block_cache.queue(block_hash, response_tx) { + let client = this.client.clone(); + let action_tx = this.action_tx.clone(); + this.action_task_spawner.spawn(Box::pin(async move { + let res = client.block_by_hash(block_hash); + let _ = action_tx + .send(CacheAction::BlockResult { block_hash, res }); + })); + } + } + CacheAction::GetEnv { block_hash, response_tx } => { + // check if env data is cached + if let Some(env) = this.evm_env_cache.cache.get(&block_hash).cloned() { + let _ = response_tx.send(Ok(env)); + continue + } + + // env data is not in the cache, request it if this is the first + // consumer + if this.evm_env_cache.queue(block_hash, response_tx) { + let client = this.client.clone(); + let action_tx = this.action_tx.clone(); + this.action_task_spawner.spawn(Box::pin(async move { + let mut cfg = CfgEnv::default(); + let mut block_env = BlockEnv::default(); + let res = client + .fill_env_at(&mut cfg, &mut block_env, block_hash.into()) + .map(|_| (cfg, block_env)); + let _ = action_tx.send(CacheAction::EnvResult { + block_hash, + res: Box::new(res), + }); + })); + } + } + CacheAction::BlockResult { block_hash, res } => { + if let Some(queued) = this.full_block_cache.queued.remove(&block_hash) { + // send the response to queued senders + for tx in queued { + let _ = tx.send(res.clone()); + } + } + + // cache good block + if let Ok(Some(block)) = res { + this.full_block_cache.cache.insert(block_hash, block); + } + } + CacheAction::EnvResult { block_hash, res } => { + let res = *res; + if let Some(queued) = this.evm_env_cache.queued.remove(&block_hash) { + // send the response to queued senders + for tx in queued { + let _ = tx.send(res.clone()); + } + } + + // cache good env data + if let Ok(data) = res { + this.evm_env_cache.cache.insert(block_hash, data); + } + } + } + } + } + } + } +} + +struct MultiConsumerLruCache +where + K: Hash + Eq, + L: Limiter, +{ + /// The LRU cache for the + cache: LruMap, + /// All queued consumers + queued: HashMap>, +} + +impl MultiConsumerLruCache +where + K: Hash + Eq, + L: Limiter, +{ + /// Adds the sender to the queue for the given key. + /// + /// Returns true if this is the first queued sender for the key + fn queue(&mut self, key: K, sender: S) -> bool { + match self.queued.entry(key) { + Entry::Occupied(mut entry) => { + entry.get_mut().push(sender); + false + } + Entry::Vacant(entry) => { + entry.insert(vec![sender]); + true + } + } + } +} + +impl MultiConsumerLruCache +where + K: Hash + Eq, +{ + /// Creates a new empty map with a given `memory_budget`. + /// + /// See also [LruMap::with_memory_budget] + fn with_memory_budget(memory_budget: usize) -> Self { + Self { cache: LruMap::with_memory_budget(memory_budget), queued: Default::default() } + } +} + +/// All message variants sent through the channel +enum CacheAction { + GetBlock { block_hash: H256, response_tx: BlockResponseSender }, + GetEnv { block_hash: H256, response_tx: EnvResponseSender }, + BlockResult { block_hash: H256, res: Result> }, + EnvResult { block_hash: H256, res: Box> }, +} diff --git a/crates/rpc/rpc/src/eth/mod.rs b/crates/rpc/rpc/src/eth/mod.rs index 598e3721a36..0a79cc7630a 100644 --- a/crates/rpc/rpc/src/eth/mod.rs +++ b/crates/rpc/rpc/src/eth/mod.rs @@ -1,6 +1,7 @@ //! `eth` namespace handler implementation. mod api; +mod cache; pub(crate) mod error; mod filter; mod pubsub; diff --git a/crates/storage/provider/src/providers/mod.rs b/crates/storage/provider/src/providers/mod.rs index c67de018fb5..8ea781b06f3 100644 --- a/crates/storage/provider/src/providers/mod.rs +++ b/crates/storage/provider/src/providers/mod.rs @@ -17,7 +17,7 @@ use reth_revm_primitives::{ config::revm_spec, env::{fill_block_env, fill_cfg_and_block_env, fill_cfg_env}, }; -use revm_primitives::{BlockEnv, CfgEnv, Env, SpecId}; +use revm_primitives::{BlockEnv, CfgEnv, SpecId}; use std::{ops::RangeBounds, sync::Arc}; mod state; @@ -241,16 +241,21 @@ impl WithdrawalsProvider for ShareableDatabase { } impl EvmEnvProvider for ShareableDatabase { - fn fill_env_at(&self, env: &mut Env, at: BlockId) -> Result<()> { + fn fill_env_at(&self, cfg: &mut CfgEnv, block_env: &mut BlockEnv, at: BlockId) -> Result<()> { let hash = self.block_hash_for_id(at)?.ok_or(ProviderError::HeaderNotFound)?; let header = self.header(&hash)?.ok_or(ProviderError::HeaderNotFound)?; - self.fill_env_with_header(env, &header) + self.fill_env_with_header(cfg, block_env, &header) } - fn fill_env_with_header(&self, env: &mut Env, header: &Header) -> Result<()> { + fn fill_env_with_header( + &self, + cfg: &mut CfgEnv, + block_env: &mut BlockEnv, + header: &Header, + ) -> Result<()> { let total_difficulty = self.header_td_by_number(header.number)?.ok_or(ProviderError::HeaderNotFound)?; - fill_cfg_and_block_env(env, &self.chain_spec, header, total_difficulty); + fill_cfg_and_block_env(cfg, block_env, &self.chain_spec, header, total_difficulty); Ok(()) } diff --git a/crates/storage/provider/src/test_utils/mock.rs b/crates/storage/provider/src/test_utils/mock.rs index a95bc0b7a15..f6946f95ef9 100644 --- a/crates/storage/provider/src/test_utils/mock.rs +++ b/crates/storage/provider/src/test_utils/mock.rs @@ -8,7 +8,7 @@ use reth_primitives::{ keccak256, Account, Address, Block, BlockHash, BlockId, BlockNumber, BlockNumberOrTag, Bytes, ChainInfo, Header, StorageKey, StorageValue, TransactionSigned, TxHash, H256, U256, }; -use revm_primitives::{BlockEnv, CfgEnv, Env}; +use revm_primitives::{BlockEnv, CfgEnv}; use std::{collections::HashMap, ops::RangeBounds, sync::Arc}; /// A mock implementation for Provider interfaces. @@ -240,11 +240,21 @@ impl StateProvider for MockEthProvider { } impl EvmEnvProvider for MockEthProvider { - fn fill_env_at(&self, _env: &mut Env, _at: BlockId) -> Result<()> { + fn fill_env_at( + &self, + _cfg: &mut CfgEnv, + _block_env: &mut BlockEnv, + _at: BlockId, + ) -> Result<()> { unimplemented!() } - fn fill_env_with_header(&self, _env: &mut Env, _header: &Header) -> Result<()> { + fn fill_env_with_header( + &self, + _cfg: &mut CfgEnv, + _block_env: &mut BlockEnv, + _header: &Header, + ) -> Result<()> { unimplemented!() } diff --git a/crates/storage/provider/src/test_utils/noop.rs b/crates/storage/provider/src/test_utils/noop.rs index 8f24ea6fb47..28eba66138d 100644 --- a/crates/storage/provider/src/test_utils/noop.rs +++ b/crates/storage/provider/src/test_utils/noop.rs @@ -7,7 +7,7 @@ use reth_primitives::{ Account, Address, Block, BlockHash, BlockId, BlockNumber, Bytes, ChainInfo, Header, StorageKey, StorageValue, TransactionSigned, TxHash, TxNumber, H256, U256, }; -use revm_primitives::{BlockEnv, CfgEnv, Env}; +use revm_primitives::{BlockEnv, CfgEnv}; use std::ops::RangeBounds; /// Supports various api interfaces for testing purposes. @@ -102,11 +102,21 @@ impl StateProvider for NoopProvider { } impl EvmEnvProvider for NoopProvider { - fn fill_env_at(&self, _env: &mut Env, _at: BlockId) -> Result<()> { + fn fill_env_at( + &self, + _cfg: &mut CfgEnv, + _block_env: &mut BlockEnv, + _at: BlockId, + ) -> Result<()> { Ok(()) } - fn fill_env_with_header(&self, _env: &mut Env, _header: &Header) -> Result<()> { + fn fill_env_with_header( + &self, + _cfg: &mut CfgEnv, + _block_env: &mut BlockEnv, + _header: &Header, + ) -> Result<()> { Ok(()) } diff --git a/crates/storage/provider/src/traits/evm_env.rs b/crates/storage/provider/src/traits/evm_env.rs index 6f422e298c6..9afe1ce4715 100644 --- a/crates/storage/provider/src/traits/evm_env.rs +++ b/crates/storage/provider/src/traits/evm_env.rs @@ -1,17 +1,23 @@ use reth_interfaces::Result; use reth_primitives::{BlockId, Header}; -use revm_primitives::{BlockEnv, CfgEnv, Env}; +use revm_primitives::{BlockEnv, CfgEnv}; -/// A provider type that knows chain specific information required to configure an [Env] +/// A provider type that knows chain specific information required to configure an +/// [Env](revm_primitives::Env) /// /// This type is mainly used to provide required data to configure the EVM environment. #[auto_impl::auto_impl(&, Arc)] pub trait EvmEnvProvider: Send + Sync { /// Fills the [CfgEnv] and [BlockEnv] fields with values specific to the given [BlockId]. - fn fill_env_at(&self, env: &mut Env, at: BlockId) -> Result<()>; + fn fill_env_at(&self, cfg: &mut CfgEnv, block_env: &mut BlockEnv, at: BlockId) -> Result<()>; /// Fills the [CfgEnv] and [BlockEnv] fields with values specific to the given [Header]. - fn fill_env_with_header(&self, env: &mut Env, header: &Header) -> Result<()>; + fn fill_env_with_header( + &self, + cfg: &mut CfgEnv, + block_env: &mut BlockEnv, + header: &Header, + ) -> Result<()>; /// Fills the [BlockEnv] fields with values specific to the given [BlockId]. fn fill_block_env_at(&self, block_env: &mut BlockEnv, at: BlockId) -> Result<()>; From 3790a142685581a2f8692e41a4d91e829cd10144 Mon Sep 17 00:00:00 2001 From: Paul Lange Date: Tue, 28 Feb 2023 21:03:14 +0100 Subject: [PATCH 013/191] fix(rpc): accept jwt's starting with `0x` (#1589) Co-authored-by: Bjerg --- crates/rpc/rpc/src/layers/jwt_secret.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/crates/rpc/rpc/src/layers/jwt_secret.rs b/crates/rpc/rpc/src/layers/jwt_secret.rs index 9a349174831..1f2d170c5e6 100644 --- a/crates/rpc/rpc/src/layers/jwt_secret.rs +++ b/crates/rpc/rpc/src/layers/jwt_secret.rs @@ -58,8 +58,10 @@ impl JwtSecret { /// Returns an error if one of the following applies: /// - `hex` is not a valid hexadecimal string /// - `hex` argument length is less than `JWT_SECRET_LEN` + /// + /// This strips the leading `0x`, if any. pub fn from_hex>(hex: S) -> Result { - let hex: &str = hex.as_ref().trim(); + let hex: &str = hex.as_ref().trim().trim_start_matches("0x"); if hex.len() != JWT_SECRET_LEN { Err(JwtError::InvalidLength(JWT_SECRET_LEN, hex.len())) } else { @@ -212,6 +214,14 @@ mod tests { assert_eq!(hex.len(), expected_len); } + #[test] + fn creation_ok_hex_string_with_0x() { + let hex: String = + "0x7365637265747365637265747365637265747365637265747365637265747365".into(); + let result = JwtSecret::from_hex(hex); + assert!(matches!(result, Ok(_))); + } + #[test] fn creation_error_wrong_len() { let hex = "f79ae8046"; From 71bc1451af3d43a931c5b11bdf7ee31d17a2e4a2 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Tue, 28 Feb 2023 15:20:02 -0500 Subject: [PATCH 014/191] fix: use correct type for NewPooledTransactionHashes68 (#1548) --- Cargo.lock | 1 + crates/net/eth-wire/Cargo.toml | 1 + crates/net/eth-wire/src/types/broadcast.rs | 220 ++++++++++++++++++++- 3 files changed, 220 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bfca56c1f7c..152e38dc628 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4564,6 +4564,7 @@ dependencies = [ "bytes", "ethers-core", "futures", + "hex", "hex-literal", "metrics", "pin-project", diff --git a/crates/net/eth-wire/Cargo.toml b/crates/net/eth-wire/Cargo.toml index fb76a352b66..b4cc8118d4f 100644 --- a/crates/net/eth-wire/Cargo.toml +++ b/crates/net/eth-wire/Cargo.toml @@ -45,6 +45,7 @@ ethers-core = { git = "https://github.com/gakonst/ethers-rs", default-features = test-fuzz = "3.0.4" tokio-util = { version = "0.7.4", features = ["io", "codec"] } hex-literal = "0.3" +hex = "0.4" rand = "0.8" secp256k1 = { version = "0.24.2", features = ["global-context", "rand-std", "recovery"] } diff --git a/crates/net/eth-wire/src/types/broadcast.rs b/crates/net/eth-wire/src/types/broadcast.rs index c6fef5457fc..163f16bf8b7 100644 --- a/crates/net/eth-wire/src/types/broadcast.rs +++ b/crates/net/eth-wire/src/types/broadcast.rs @@ -1,8 +1,11 @@ //! Types for broadcasting new data. use crate::{EthMessage, EthVersion}; +use bytes::Bytes; use reth_codecs::derive_arbitrary; use reth_primitives::{Block, TransactionSigned, H256, U128}; -use reth_rlp::{RlpDecodable, RlpDecodableWrapper, RlpEncodable, RlpEncodableWrapper}; +use reth_rlp::{ + Decodable, Encodable, RlpDecodable, RlpDecodableWrapper, RlpEncodable, RlpEncodableWrapper, +}; use std::sync::Arc; #[cfg(feature = "serde")] @@ -193,10 +196,32 @@ impl From> for NewPooledTransactionHashes66 { /// Same as [`NewPooledTransactionHashes66`] but extends that that beside the transaction hashes, /// the node sends the transaction types and their sizes (as defined in EIP-2718) as well. #[derive_arbitrary(rlp)] -#[derive(Clone, Debug, PartialEq, Eq, RlpEncodable, RlpDecodable, Default)] +#[derive(Clone, Debug, PartialEq, Eq, Default)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub struct NewPooledTransactionHashes68 { /// Transaction types for new transactions that have appeared on the network. + /// + /// ## Note on RLP encoding and decoding + /// + /// In the [eth/68 spec](https://eips.ethereum.org/EIPS/eip-5793#specification) this is defined + /// the following way: + /// * `[type_0: B_1, type_1: B_1, ...]` + /// + /// This would make it seem like the [`Encodable`](reth_rlp::Encodable) and + /// [`Decodable`](reth_rlp::Decodable) implementations should directly use a `Vec` for + /// encoding and decoding, because it looks like this field should be encoded as a _list_ of + /// bytes. + /// + /// However, [this is implemented in geth as a `[]byte` + /// type](https://github.com/ethereum/go-ethereum/blob/82d934b1dd80cdd8190803ea9f73ed2c345e2576/eth/protocols/eth/protocol.go#L308-L313), + /// which [ends up being encoded as a RLP + /// string](https://github.com/ethereum/go-ethereum/blob/82d934b1dd80cdd8190803ea9f73ed2c345e2576/rlp/encode_test.go#L171-L176), + /// **not** a RLP list. + /// + /// Because of this, we do not directly use the `Vec` when encoding and decoding, and + /// instead use the [`Encodable`](reth_rlp::Encodable) and [`Decodable`](reth_rlp::Decodable) + /// implementations for `&[u8]` instead, which encodes into a RLP string, and expects an RLP + /// string when decoding. pub types: Vec, /// Transaction sizes for new transactions that have appeared on the network. pub sizes: Vec, @@ -204,10 +229,80 @@ pub struct NewPooledTransactionHashes68 { pub hashes: Vec, } +impl Encodable for NewPooledTransactionHashes68 { + fn length(&self) -> usize { + #[derive(RlpEncodable)] + struct EncodableNewPooledTransactionHashes68<'a> { + types: &'a [u8], + sizes: &'a Vec, + hashes: &'a Vec, + } + + let encodable = EncodableNewPooledTransactionHashes68 { + types: &self.types[..], + sizes: &self.sizes, + hashes: &self.hashes, + }; + + encodable.length() + } + fn encode(&self, out: &mut dyn bytes::BufMut) { + #[derive(RlpEncodable)] + struct EncodableNewPooledTransactionHashes68<'a> { + types: &'a [u8], + sizes: &'a Vec, + hashes: &'a Vec, + } + + let encodable = EncodableNewPooledTransactionHashes68 { + types: &self.types[..], + sizes: &self.sizes, + hashes: &self.hashes, + }; + + encodable.encode(out); + } +} + +impl Decodable for NewPooledTransactionHashes68 { + fn decode(buf: &mut &[u8]) -> Result { + #[derive(RlpDecodable)] + struct EncodableNewPooledTransactionHashes68 { + types: Bytes, + sizes: Vec, + hashes: Vec, + } + + let encodable = EncodableNewPooledTransactionHashes68::decode(buf)?; + Ok(Self { types: encodable.types.into(), sizes: encodable.sizes, hashes: encodable.hashes }) + } +} + #[cfg(test)] mod tests { + use std::str::FromStr; + + use bytes::BytesMut; + use hex_literal::hex; + use reth_rlp::{Decodable, Encodable}; + use super::*; + /// Takes as input a struct / encoded hex message pair, ensuring that we encode to the exact hex + /// message, and decode to the exact struct. + fn test_encoding_vector( + input: (T, &[u8]), + ) { + let (expected_decoded, expected_encoded) = input; + let mut encoded = BytesMut::new(); + expected_decoded.encode(&mut encoded); + + assert_eq!(hex::encode(&encoded), hex::encode(expected_encoded)); + + let decoded = T::decode(&mut encoded.as_ref()).unwrap(); + assert_eq!(expected_decoded, decoded); + } + #[test] fn can_return_latest_block() { let mut blocks = NewBlockHashes(vec![BlockHashNumber { hash: H256::random(), number: 0 }]); @@ -219,4 +314,125 @@ mod tests { let latest = blocks.latest().unwrap(); assert_eq!(latest.number, 100); } + + #[test] + fn eth_68_tx_hash_roundtrip() { + let vectors = vec![ + ( + NewPooledTransactionHashes68 { types: vec![], sizes: vec![], hashes: vec![] }, + &hex!("c380c0c0")[..], + ), + ( + NewPooledTransactionHashes68 { + types: vec![0x00], + sizes: vec![0x00], + hashes: vec![H256::from_str( + "0x0000000000000000000000000000000000000000000000000000000000000000", + ) + .unwrap()], + }, + &hex!("e500c180e1a00000000000000000000000000000000000000000000000000000000000000000")[..], + ), + ( + NewPooledTransactionHashes68 { + types: vec![0x00, 0x00], + sizes: vec![0x00, 0x00], + hashes: vec![ + H256::from_str( + "0x0000000000000000000000000000000000000000000000000000000000000000", + ) + .unwrap(), + H256::from_str( + "0x0000000000000000000000000000000000000000000000000000000000000000", + ) + .unwrap(), + ], + }, + &hex!("f84a820000c28080f842a00000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000")[..], + ), + ( + NewPooledTransactionHashes68 { + types: vec![0x02], + sizes: vec![0xb6], + hashes: vec![H256::from_str( + "0xfecbed04c7b88d8e7221a0a3f5dc33f220212347fc167459ea5cc9c3eb4c1124", + ) + .unwrap()], + }, + &hex!("e602c281b6e1a0fecbed04c7b88d8e7221a0a3f5dc33f220212347fc167459ea5cc9c3eb4c1124")[..], + ), + ( + NewPooledTransactionHashes68 { + types: vec![0xff, 0xff], + sizes: vec![0xffffffff, 0xffffffff], + hashes: vec![ + H256::from_str( + "0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + ) + .unwrap(), + H256::from_str( + "0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + ) + .unwrap(), + ], + }, + &hex!("f85282ffffca84ffffffff84fffffffff842a0ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffa0ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")[..], + ), + ( + NewPooledTransactionHashes68 { + types: vec![0xff, 0xff], + sizes: vec![0xffffffff, 0xffffffff], + hashes: vec![ + H256::from_str( + "0xbeefcafebeefcafebeefcafebeefcafebeefcafebeefcafebeefcafebeefcafe", + ) + .unwrap(), + H256::from_str( + "0xbeefcafebeefcafebeefcafebeefcafebeefcafebeefcafebeefcafebeefcafe", + ) + .unwrap(), + ], + }, + &hex!("f85282ffffca84ffffffff84fffffffff842a0beefcafebeefcafebeefcafebeefcafebeefcafebeefcafebeefcafebeefcafea0beefcafebeefcafebeefcafebeefcafebeefcafebeefcafebeefcafebeefcafe")[..], + ), + ( + NewPooledTransactionHashes68 { + types: vec![0x10, 0x10], + sizes: vec![0xdeadc0de, 0xdeadc0de], + hashes: vec![ + H256::from_str( + "0x3b9aca00f0671c9a2a1b817a0a78d3fe0c0f776cccb2a8c3c1b412a4f4e4d4e2", + ) + .unwrap(), + H256::from_str( + "0x3b9aca00f0671c9a2a1b817a0a78d3fe0c0f776cccb2a8c3c1b412a4f4e4d4e2", + ) + .unwrap(), + ], + }, + &hex!("f852821010ca84deadc0de84deadc0def842a03b9aca00f0671c9a2a1b817a0a78d3fe0c0f776cccb2a8c3c1b412a4f4e4d4e2a03b9aca00f0671c9a2a1b817a0a78d3fe0c0f776cccb2a8c3c1b412a4f4e4d4e2")[..], + ), + ( + NewPooledTransactionHashes68 { + types: vec![0x6f, 0x6f], + sizes: vec![0x7fffffff, 0x7fffffff], + hashes: vec![ + H256::from_str( + "0x0000000000000000000000000000000000000000000000000000000000000002", + ) + .unwrap(), + H256::from_str( + "0x0000000000000000000000000000000000000000000000000000000000000002", + ) + .unwrap(), + ], + }, + &hex!("f852826f6fca847fffffff847ffffffff842a00000000000000000000000000000000000000000000000000000000000000002a00000000000000000000000000000000000000000000000000000000000000002")[..], + ), + ]; + + for vector in vectors { + test_encoding_vector(vector); + } + } } From 4285186dbdceafbe8867fd869ba5f2aaceed0b9e Mon Sep 17 00:00:00 2001 From: Georgios Konstantopoulos Date: Tue, 28 Feb 2023 16:55:51 -0700 Subject: [PATCH 015/191] refactor: Execution Stage owns Executor (#1568) Co-authored-by: Roman Krasiuk Co-authored-by: Oliver Nordbjerg --- bin/reth/src/chain/import.rs | 9 +- bin/reth/src/dump_stage/execution.rs | 7 +- bin/reth/src/node/mod.rs | 9 +- bin/reth/src/stage/mod.rs | 6 +- bin/reth/src/test_eth_chain/runner.rs | 4 +- crates/executor/src/executor.rs | 155 +++++++++++--------- crates/revm/revm-inspectors/src/stack.rs | 4 +- crates/revm/src/lib.rs | 3 + crates/rpc/rpc-engine-api/src/engine_api.rs | 21 ++- crates/stages/src/lib.rs | 5 + crates/stages/src/sets.rs | 2 +- crates/stages/src/stages/execution.rs | 103 ++++++------- 12 files changed, 181 insertions(+), 147 deletions(-) diff --git a/bin/reth/src/chain/import.rs b/bin/reth/src/chain/import.rs index f93228a3f6a..967ea2fa183 100644 --- a/bin/reth/src/chain/import.rs +++ b/bin/reth/src/chain/import.rs @@ -27,6 +27,7 @@ use reth_staged_sync::{ use reth_stages::{ prelude::*, stages::{ExecutionStage, SenderRecoveryStage, TotalDifficultyStage}, + DefaultDB, }; use std::sync::Arc; use tracing::{debug, info}; @@ -156,9 +157,11 @@ impl ImportCommand { .set(SenderRecoveryStage { commit_threshold: config.stages.sender_recovery.commit_threshold, }) - .set(ExecutionStage { - chain_spec: self.chain.clone(), - commit_threshold: config.stages.execution.commit_threshold, + .set({ + let mut stage: ExecutionStage<'_, DefaultDB<'_>> = + ExecutionStage::from(self.chain.clone()); + stage.commit_threshold = config.stages.execution.commit_threshold; + stage }), ) .with_max_block(0) diff --git a/bin/reth/src/dump_stage/execution.rs b/bin/reth/src/dump_stage/execution.rs index 49d6de7de35..5a5ae04decb 100644 --- a/bin/reth/src/dump_stage/execution.rs +++ b/bin/reth/src/dump_stage/execution.rs @@ -7,8 +7,9 @@ use eyre::Result; use reth_db::{ cursor::DbCursorRO, database::Database, table::TableImporter, tables, transaction::DbTx, }; +use reth_primitives::MAINNET; use reth_provider::Transaction; -use reth_stages::{stages::ExecutionStage, Stage, StageId, UnwindInput}; +use reth_stages::{stages::ExecutionStage, DefaultDB, Stage, StageId, UnwindInput}; use std::ops::DerefMut; use tracing::info; @@ -96,7 +97,7 @@ async fn unwind_and_copy( output_db: &reth_db::mdbx::Env, ) -> eyre::Result<()> { let mut unwind_tx = Transaction::new(db_tool.db)?; - let mut exec_stage = ExecutionStage::default(); + let mut exec_stage: ExecutionStage<'_, DefaultDB<'_>> = ExecutionStage::from(MAINNET.clone()); exec_stage .unwind( @@ -125,7 +126,7 @@ async fn dry_run( info!(target: "reth::cli", "Executing stage. [dry-run]"); let mut tx = Transaction::new(&output_db)?; - let mut exec_stage = ExecutionStage::default(); + let mut exec_stage: ExecutionStage<'_, DefaultDB<'_>> = ExecutionStage::from(MAINNET.clone()); exec_stage .execute( diff --git a/bin/reth/src/node/mod.rs b/bin/reth/src/node/mod.rs index 55a4a9ef368..082d2c5cbba 100644 --- a/bin/reth/src/node/mod.rs +++ b/bin/reth/src/node/mod.rs @@ -50,6 +50,7 @@ use reth_staged_sync::{ use reth_stages::{ prelude::*, stages::{ExecutionStage, SenderRecoveryStage, TotalDifficultyStage, FINISH}, + DefaultDB, }; use reth_tasks::TaskExecutor; use std::{net::SocketAddr, path::PathBuf, sync::Arc}; @@ -448,9 +449,11 @@ impl Command { .set(SenderRecoveryStage { commit_threshold: stage_conf.sender_recovery.commit_threshold, }) - .set(ExecutionStage { - chain_spec: self.chain.clone(), - commit_threshold: stage_conf.execution.commit_threshold, + .set({ + let mut stage: ExecutionStage<'_, DefaultDB<'_>> = + ExecutionStage::from(self.chain.clone()); + stage.commit_threshold = stage_conf.execution.commit_threshold; + stage }), ) .build(); diff --git a/bin/reth/src/stage/mod.rs b/bin/reth/src/stage/mod.rs index 8153567a4fd..27f82e25d95 100644 --- a/bin/reth/src/stage/mod.rs +++ b/bin/reth/src/stage/mod.rs @@ -17,7 +17,7 @@ use reth_staged_sync::{ }; use reth_stages::{ stages::{BodyStage, ExecutionStage, SenderRecoveryStage}, - ExecInput, Stage, StageId, UnwindInput, + DefaultDB, ExecInput, Stage, StageId, UnwindInput, }; use std::{net::SocketAddr, sync::Arc}; use tracing::*; @@ -171,8 +171,8 @@ impl Command { stage.execute(&mut tx, input).await?; } StageEnum::Execution => { - let mut stage = - ExecutionStage { chain_spec: self.chain.clone(), commit_threshold: num_blocks }; + let mut stage = ExecutionStage::>::from(self.chain.clone()); + stage.commit_threshold = num_blocks; if !self.skip_unwind { stage.unwind(&mut tx, unwind).await?; } diff --git a/bin/reth/src/test_eth_chain/runner.rs b/bin/reth/src/test_eth_chain/runner.rs index 45c1225bf26..2e1b2f99238 100644 --- a/bin/reth/src/test_eth_chain/runner.rs +++ b/bin/reth/src/test_eth_chain/runner.rs @@ -15,7 +15,7 @@ use reth_primitives::{ }; use reth_provider::Transaction; use reth_rlp::Decodable; -use reth_stages::{stages::ExecutionStage, ExecInput, Stage, StageId}; +use reth_stages::{stages::ExecutionStage, DefaultDB, ExecInput, Stage, StageId}; use std::{ collections::HashMap, ffi::OsStr, @@ -193,7 +193,7 @@ pub async fn run_test(path: PathBuf) -> eyre::Result { // Initialize the execution stage // Hardcode the chain_id to Ethereum 1. - let mut stage = ExecutionStage::new(chain_spec, 1000); + let mut stage = ExecutionStage::>::from(chain_spec); // Call execution stage let input = ExecInput { diff --git a/crates/executor/src/executor.rs b/crates/executor/src/executor.rs index f383d84e0bc..ab27c6c639e 100644 --- a/crates/executor/src/executor.rs +++ b/crates/executor/src/executor.rs @@ -21,26 +21,47 @@ use revm::{ primitives::{Account as RevmAccount, AccountInfo, Bytecode, ResultAndState}, EVM, }; -use std::collections::{BTreeMap, HashMap}; +use std::{ + collections::{BTreeMap, HashMap}, + sync::Arc, +}; /// Main block executor pub struct Executor<'a, DB> where DB: StateProvider, { - chain_spec: &'a ChainSpec, + /// The configured chain-spec + pub chain_spec: Arc, evm: EVM<&'a mut SubState>, stack: InspectorStack, } +impl<'a, DB> From for Executor<'a, DB> +where + DB: StateProvider, +{ + /// Instantiates a new executor from the chainspec. Must call + /// `with_db` to set the database before executing. + fn from(chain_spec: ChainSpec) -> Self { + let evm = EVM::new(); + Executor { + chain_spec: Arc::new(chain_spec), + evm, + stack: InspectorStack::new(InspectorStackConfig::default()), + } + } +} + impl<'a, DB> Executor<'a, DB> where DB: StateProvider, { /// Creates a new executor from the given chain spec and database. - pub fn new(chain_spec: &'a ChainSpec, db: &'a mut SubState) -> Self { + pub fn new(chain_spec: Arc, db: &'a mut SubState) -> Self { let mut evm = EVM::new(); evm.database(db); + Executor { chain_spec, evm, stack: InspectorStack::new(InspectorStackConfig::default()) } } @@ -50,10 +71,21 @@ where self } - fn db(&mut self) -> &mut SubState { + /// Gives a reference to the database + pub fn db(&mut self) -> &mut SubState { self.evm.db().expect("db to not be moved") } + /// Overrides the database + pub fn with_db( + &self, + db: &'a mut SubState, + ) -> Executor<'a, OtherDB> { + let mut evm = EVM::new(); + evm.database(db); + Executor { chain_spec: self.chain_spec.clone(), evm, stack: self.stack.clone() } + } + fn recover_senders( &self, body: &[TransactionSigned], @@ -75,7 +107,7 @@ where fill_cfg_and_block_env( &mut self.evm.env.cfg, &mut self.evm.env.block, - self.chain_spec, + &self.chain_spec, header, total_difficulty, ); @@ -341,6 +373,30 @@ where } } + /// Execute and verify block + pub fn execute_and_verify_receipt( + &mut self, + block: &Block, + total_difficulty: U256, + senders: Option>, + ) -> Result { + let execution_result = self.execute(block, total_difficulty, senders)?; + + let receipts_iter = + execution_result.tx_changesets.iter().map(|changeset| &changeset.receipt); + + if self.chain_spec.fork(Hardfork::Byzantium).active_at_block(block.header.number) { + verify_receipt(block.header.receipts_root, block.header.logs_bloom, receipts_iter)?; + } + + // TODO Before Byzantium, receipts contained state root that would mean that expensive + // operation as hashing that is needed for state root got calculated in every + // transaction This was replaced with is_success flag. + // See more about EIP here: https://eips.ethereum.org/EIPS/eip-658 + + Ok(execution_result) + } + /// Runs a single transaction in the configured environment and proceeds /// to return the result and state diff (without applying it). /// @@ -464,30 +520,6 @@ where } } -/// Execute and verify block -pub fn execute_and_verify_receipt( - block: &Block, - total_difficulty: U256, - senders: Option>, - chain_spec: &ChainSpec, - db: &mut SubState, -) -> Result { - let execution_result = execute(block, total_difficulty, senders, chain_spec, db)?; - - let receipts_iter = execution_result.tx_changesets.iter().map(|changeset| &changeset.receipt); - - if chain_spec.fork(Hardfork::Byzantium).active_at_block(block.header.number) { - verify_receipt(block.header.receipts_root, block.header.logs_bloom, receipts_iter)?; - } - - // TODO Before Byzantium, receipts contained state root that would mean that expensive operation - // as hashing that is needed for state root got calculated in every transaction - // This was replaced with is_success flag. - // See more about EIP here: https://eips.ethereum.org/EIPS/eip-658 - - Ok(execution_result) -} - /// Verify receipts pub fn verify_receipt<'a>( expected_receipts_root: H256, @@ -511,22 +543,6 @@ pub fn verify_receipt<'a>( Ok(()) } -/// Verify block. Execute all transaction and compare results. -/// Returns ChangeSet on transaction granularity. -/// NOTE: If block reward is still active (Before Paris/Merge) we would return -/// additional TransactionStatechangeset for account that receives the reward. -pub fn execute( - block: &Block, - total_difficulty: U256, - senders: Option>, - chain_spec: &ChainSpec, - db: &mut SubState, -) -> Result { - let mut executor = Executor::new(chain_spec, db) - .with_stack(InspectorStack::new(InspectorStackConfig::default())); - executor.execute(block, total_difficulty, senders) -} - #[cfg(test)] mod tests { use super::*; @@ -640,13 +656,13 @@ mod tests { ); // spec at berlin fork - let chain_spec = ChainSpecBuilder::mainnet().berlin_activated().build(); + let chain_spec = Arc::new(ChainSpecBuilder::mainnet().berlin_activated().build()); let mut db = SubState::new(State::new(db)); // execute chain and verify receipts - let out = - execute_and_verify_receipt(&block, U256::ZERO, None, &chain_spec, &mut db).unwrap(); + let mut executor = Executor::new(chain_spec, &mut db); + let out = executor.execute_and_verify_receipt(&block, U256::ZERO, None).unwrap(); assert_eq!(out.tx_changesets.len(), 1, "Should executed one transaction"); @@ -765,21 +781,23 @@ mod tests { beneficiary_balance += i; } - let chain_spec = ChainSpecBuilder::from(&*MAINNET) - .homestead_activated() - .with_fork(Hardfork::Dao, ForkCondition::Block(1)) - .build(); + let chain_spec = Arc::new( + ChainSpecBuilder::from(&*MAINNET) + .homestead_activated() + .with_fork(Hardfork::Dao, ForkCondition::Block(1)) + .build(), + ); let mut db = SubState::new(State::new(db)); // execute chain and verify receipts - let out = execute_and_verify_receipt( - &Block { header, body: vec![], ommers: vec![], withdrawals: None }, - U256::ZERO, - None, - &chain_spec, - &mut db, - ) - .unwrap(); + let mut executor = Executor::new(chain_spec, &mut db); + let out = executor + .execute_and_verify_receipt( + &Block { header, body: vec![], ommers: vec![], withdrawals: None }, + U256::ZERO, + None, + ) + .unwrap(); assert_eq!(out.tx_changesets.len(), 0, "No tx"); // Check if cache is set @@ -858,13 +876,13 @@ mod tests { ); // spec at berlin fork - let chain_spec = ChainSpecBuilder::mainnet().berlin_activated().build(); + let chain_spec = Arc::new(ChainSpecBuilder::mainnet().berlin_activated().build()); let mut db = SubState::new(State::new(db)); // execute chain and verify receipts - let out = - execute_and_verify_receipt(&block, U256::ZERO, None, &chain_spec, &mut db).unwrap(); + let mut executor = Executor::new(chain_spec, &mut db); + let out = executor.execute_and_verify_receipt(&block, U256::ZERO, None).unwrap(); assert_eq!(out.tx_changesets.len(), 1, "Should executed one transaction"); @@ -907,17 +925,17 @@ mod tests { Address::from_str("c94f5374fce5edbc8e2a8697c15331677e6ebf0b").unwrap(); // spec at shanghai fork - let chain_spec = ChainSpecBuilder::mainnet().shanghai_activated().build(); + let chain_spec = Arc::new(ChainSpecBuilder::mainnet().shanghai_activated().build()); let mut db = SubState::new(State::new(StateProviderTest::default())); // execute chain and verify receipts - let out = - execute_and_verify_receipt(&block, U256::ZERO, None, &chain_spec, &mut db).unwrap(); + let mut executor = Executor::new(chain_spec, &mut db); + let out = executor.execute_and_verify_receipt(&block, U256::ZERO, None).unwrap(); assert_eq!(out.tx_changesets.len(), 0, "No tx"); let withdrawal_sum = withdrawals.iter().fold(U256::ZERO, |sum, w| sum + w.amount_wei()); - let beneficiary_account = db.accounts.get(&withdrawal_beneficiary).unwrap(); + let beneficiary_account = executor.db().accounts.get(&withdrawal_beneficiary).unwrap(); assert_eq!(beneficiary_account.info.balance, withdrawal_sum); assert_eq!(beneficiary_account.info.nonce, 0); assert_eq!(beneficiary_account.account_state, AccountState::StorageCleared); @@ -931,8 +949,7 @@ mod tests { ); // Execute same block again - let out = - execute_and_verify_receipt(&block, U256::ZERO, None, &chain_spec, &mut db).unwrap(); + let out = executor.execute_and_verify_receipt(&block, U256::ZERO, None).unwrap(); assert_eq!(out.tx_changesets.len(), 0, "No tx"); assert_eq!(out.block_changesets.len(), 1); diff --git a/crates/revm/revm-inspectors/src/stack.rs b/crates/revm/revm-inspectors/src/stack.rs index 12e3df4050a..042dd339f9b 100644 --- a/crates/revm/revm-inspectors/src/stack.rs +++ b/crates/revm/revm-inspectors/src/stack.rs @@ -10,7 +10,7 @@ use revm::{ /// - Block: Hook on block execution /// - BlockWithIndex: Hook on block execution transaction index /// - Transaction: Hook on a specific transaction hash -#[derive(Default)] +#[derive(Default, Clone)] pub enum Hook { #[default] /// No hook. @@ -23,7 +23,7 @@ pub enum Hook { All, } -#[derive(Default)] +#[derive(Default, Clone)] /// An inspector that calls multiple inspectors in sequence. /// /// If a call to an inspector returns a value other than [InstructionResult::Continue] (or diff --git a/crates/revm/src/lib.rs b/crates/revm/src/lib.rs index 21cc3790af8..781ae36c332 100644 --- a/crates/revm/src/lib.rs +++ b/crates/revm/src/lib.rs @@ -12,3 +12,6 @@ pub mod database; /// reexport for convenience pub use reth_revm_primitives::*; + +/// Re-export everything +pub use revm; diff --git a/crates/rpc/rpc-engine-api/src/engine_api.rs b/crates/rpc/rpc-engine-api/src/engine_api.rs index c5e8df1d3f2..d52daa3033e 100644 --- a/crates/rpc/rpc-engine-api/src/engine_api.rs +++ b/crates/rpc/rpc-engine-api/src/engine_api.rs @@ -1,6 +1,5 @@ use crate::{message::EngineApiMessageVersion, EngineApiError, EngineApiMessage, EngineApiResult}; use futures::StreamExt; -use reth_executor::executor; use reth_interfaces::consensus::ForkchoiceState; use reth_primitives::{ proofs::{self, EMPTY_LIST_HASH}, @@ -17,6 +16,7 @@ use reth_rpc_types::engine::{ use std::{ future::Future, pin::Pin, + sync::Arc, task::{ready, Context, Poll}, }; use tokio::sync::{mpsc, oneshot, watch}; @@ -37,7 +37,7 @@ const MAX_PAYLOAD_BODIES_LIMIT: u64 = 1024; pub struct EngineApi { client: Client, /// Consensus configuration - chain_spec: ChainSpec, + chain_spec: Arc, message_rx: UnboundedReceiverStream, forkchoice_state_tx: watch::Sender, // TODO: Placeholder for storing future blocks. Make cache bounded. Use lru @@ -57,7 +57,7 @@ impl Self { Self { client, - chain_spec, + chain_spec: Arc::new(chain_spec), message_rx: UnboundedReceiverStream::new(message_rx), forkchoice_state_tx, } @@ -304,13 +304,10 @@ impl Ok(PayloadStatus::new(PayloadStatusEnum::Valid, block_hash)), Err(err) => Ok(PayloadStatus::new( PayloadStatusEnum::Invalid { validation_error: err.to_string() }, @@ -443,7 +440,7 @@ mod tests { }; fn setup_engine_api() -> (EngineApiTestHandle, EngineApi>) { - let chain_spec = MAINNET.clone(); + let chain_spec = Arc::new(MAINNET.clone()); let client = Arc::new(MockEthProvider::default()); let (msg_tx, msg_rx) = unbounded_channel(); let (forkchoice_state_tx, forkchoice_state_rx) = watch::channel(ForkchoiceState::default()); @@ -458,7 +455,7 @@ mod tests { } struct EngineApiTestHandle { - chain_spec: ChainSpec, + chain_spec: Arc, client: Arc, msg_tx: UnboundedSender, forkchoice_state_rx: WatchReceiver, diff --git a/crates/stages/src/lib.rs b/crates/stages/src/lib.rs index 139bc314d1c..59e6c075e86 100644 --- a/crates/stages/src/lib.rs +++ b/crates/stages/src/lib.rs @@ -56,6 +56,11 @@ mod stage; mod trie; mod util; +/// The real database type we use in Reth using MDBX. +pub type DefaultDB<'a> = LatestStateProviderRef<'a, 'a, Tx<'a, RW, WriteMap>>; +use reth_db::mdbx::{tx::Tx, WriteMap, RW}; +use reth_provider::LatestStateProviderRef; + #[allow(missing_docs)] #[cfg(any(test, feature = "test-utils"))] pub mod test_utils; diff --git a/crates/stages/src/sets.rs b/crates/stages/src/sets.rs index 6b2d70031fe..dd86ef121ca 100644 --- a/crates/stages/src/sets.rs +++ b/crates/stages/src/sets.rs @@ -170,7 +170,7 @@ impl StageSet for ExecutionStages { fn builder(self) -> StageSetBuilder { StageSetBuilder::default() .add_stage(SenderRecoveryStage::default()) - .add_stage(ExecutionStage { chain_spec: self.chain_spec, ..Default::default() }) + .add_stage(ExecutionStage::>::from(self.chain_spec)) } } diff --git a/crates/stages/src/stages/execution.rs b/crates/stages/src/stages/execution.rs index 62490016b15..f7b58eb9b81 100644 --- a/crates/stages/src/stages/execution.rs +++ b/crates/stages/src/stages/execution.rs @@ -1,6 +1,6 @@ use crate::{ - exec_or_return, ExecAction, ExecInput, ExecOutput, Stage, StageError, StageId, UnwindInput, - UnwindOutput, + exec_or_return, DefaultDB, ExecAction, ExecInput, ExecOutput, Stage, StageError, StageId, + UnwindInput, UnwindOutput, }; use reth_db::{ cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO}, @@ -9,12 +9,11 @@ use reth_db::{ tables, transaction::{DbTx, DbTxMut}, }; -use reth_executor::execution_result::AccountChangeSet; +use reth_executor::{execution_result::AccountChangeSet, executor::Executor}; use reth_interfaces::provider::ProviderError; -use reth_primitives::{Address, Block, ChainSpec, Hardfork, StorageEntry, H256, MAINNET, U256}; -use reth_provider::{LatestStateProviderRef, Transaction}; +use reth_primitives::{Address, Block, ChainSpec, Hardfork, StorageEntry, H256, U256}; +use reth_provider::{LatestStateProviderRef, StateProvider, Transaction}; use reth_revm::database::{State, SubState}; -use std::fmt::Debug; use tracing::*; /// The [`StageId`] of the execution stage. @@ -48,24 +47,35 @@ pub const EXECUTION: StageId = StageId("Execution"); /// - [tables::AccountHistory] to remove change set and apply old values to /// - [tables::PlainAccountState] [tables::StorageHistory] to remove change set and apply old values /// to [tables::PlainStorageState] -#[derive(Debug)] -pub struct ExecutionStage { - /// Executor configuration. - pub chain_spec: ChainSpec, +// false positive, we cannot derive it if !DB: Debug. +#[allow(missing_debug_implementations)] +pub struct ExecutionStage<'a, DB = DefaultDB<'a>> +where + DB: StateProvider, +{ + /// The stage's internal executor + pub executor: Executor<'a, DB>, /// Commit threshold pub commit_threshold: u64, } -impl Default for ExecutionStage { - fn default() -> Self { - Self { chain_spec: MAINNET.clone(), commit_threshold: 1_000 } +impl<'a, DB: StateProvider> From> for ExecutionStage<'a, DB> { + fn from(executor: Executor<'a, DB>) -> Self { + Self { executor, commit_threshold: 1_000 } } } -impl ExecutionStage { +impl<'a, DB: StateProvider> From for ExecutionStage<'a, DB> { + fn from(chain_spec: ChainSpec) -> Self { + let executor = Executor::from(chain_spec); + Self::from(executor) + } +} + +impl<'a, S: StateProvider> ExecutionStage<'a, S> { /// Execute the stage. pub fn execute_inner( - &self, + &mut self, tx: &mut Transaction<'_, DB>, input: ExecInput, ) -> Result { @@ -87,7 +97,6 @@ impl ExecutionStage { let mut tx_cursor = tx.cursor_read::()?; // Skip sender recovery and load signer from database. let mut tx_sender = tx.cursor_read::()?; - // Get block headers and bodies let block_batch = headers_cursor .walk_range(start_block..=end_block)? @@ -106,6 +115,7 @@ impl ExecutionStage { .collect::, _>>()?; // Create state provider with cached state + let mut state_provider = SubState::new(State::new(LatestStateProviderRef::new(&**tx))); // Fetch transactions, execute them and generate results @@ -143,14 +153,15 @@ impl ExecutionStage { trace!(target: "sync::stages::execution", number = block_number, txs = transactions.len(), "Executing block"); - let changeset = reth_executor::executor::execute_and_verify_receipt( - &Block { header, body: transactions, ommers, withdrawals }, - td, - Some(signers), - &self.chain_spec, - &mut state_provider, - ) - .map_err(|error| StageError::ExecutionError { block: block_number, error })?; + // Configure the executor to use the current state. + let mut executor = self.executor.with_db(&mut state_provider); + let changeset = executor + .execute_and_verify_receipt( + &Block { header, body: transactions, ommers, withdrawals }, + td, + Some(signers), + ) + .map_err(|error| StageError::ExecutionError { block: block_number, error })?; block_change_patches.push((changeset, block_number)); } @@ -160,8 +171,11 @@ impl ExecutionStage { // apply changes to plain database. for (results, block_number) in block_change_patches.into_iter() { - let spurious_dragon_active = - self.chain_spec.fork(Hardfork::SpuriousDragon).active_at_block(block_number); + let spurious_dragon_active = self + .executor + .chain_spec + .fork(Hardfork::SpuriousDragon) + .active_at_block(block_number); // insert state change set for result in results.tx_changesets.into_iter() { for (address, account_change_set) in result.changeset.into_iter() { @@ -266,15 +280,15 @@ impl ExecutionStage { } } -impl ExecutionStage { +impl<'a, DB: StateProvider> ExecutionStage<'a, DB> { /// Create new execution stage with specified config. - pub fn new(chain_spec: ChainSpec, commit_threshold: u64) -> Self { - Self { chain_spec, commit_threshold } + pub fn new(executor: Executor<'a, DB>, commit_threshold: u64) -> Self { + Self { executor, commit_threshold } } } #[async_trait::async_trait] -impl Stage for ExecutionStage { +impl Stage for ExecutionStage<'_, State> { /// Return the id of the stage fn id(&self) -> StageId { EXECUTION @@ -392,20 +406,18 @@ impl Stage for ExecutionStage { #[cfg(test)] mod tests { - use std::ops::{Deref, DerefMut}; - - use crate::test_utils::{TestTransaction, PREV_STAGE_ID}; - use super::*; + use crate::test_utils::{TestTransaction, PREV_STAGE_ID}; use reth_db::{ mdbx::{test_utils::create_test_db, EnvKind, WriteMap}, models::AccountBeforeTx, }; use reth_primitives::{ - hex_literal::hex, keccak256, Account, ChainSpecBuilder, SealedBlock, H160, U256, + hex_literal::hex, keccak256, Account, ChainSpecBuilder, SealedBlock, H160, MAINNET, U256, }; use reth_provider::insert_canonical_block; use reth_rlp::Decodable; + use std::ops::{Deref, DerefMut}; #[tokio::test] async fn sanity_execution_of_block() { @@ -448,11 +460,8 @@ mod tests { db_tx.put::(code_hash, code.to_vec()).unwrap(); tx.commit().unwrap(); - // execute - let mut execution_stage = ExecutionStage { - chain_spec: ChainSpecBuilder::mainnet().berlin_activated().build(), - ..Default::default() - }; + let chain_spec = ChainSpecBuilder::mainnet().berlin_activated().build(); + let mut execution_stage = ExecutionStage::>::from(chain_spec); let output = execution_stage.execute(&mut tx, input).await.unwrap(); tx.commit().unwrap(); assert_eq!(output, ExecOutput { stage_progress: 1, done: true }); @@ -536,14 +545,12 @@ mod tests { tx.commit().unwrap(); // execute - let mut execution_stage = ExecutionStage { - chain_spec: ChainSpecBuilder::mainnet().berlin_activated().build(), - ..Default::default() - }; + let chain_spec = ChainSpecBuilder::mainnet().berlin_activated().build(); + let mut execution_stage = ExecutionStage::>::from(chain_spec); let _ = execution_stage.execute(&mut tx, input).await.unwrap(); tx.commit().unwrap(); - let o = ExecutionStage::default() + let o = ExecutionStage::>::from(MAINNET.clone()) .unwind(&mut tx, UnwindInput { stage_progress: 1, unwind_to: 0, bad_block: None }) .await .unwrap(); @@ -624,10 +631,8 @@ mod tests { tx.commit().unwrap(); // execute - let mut execution_stage = ExecutionStage { - chain_spec: ChainSpecBuilder::mainnet().berlin_activated().build(), - ..Default::default() - }; + let chain_spec = ChainSpecBuilder::mainnet().berlin_activated().build(); + let mut execution_stage = ExecutionStage::>::from(chain_spec); let _ = execution_stage.execute(&mut tx, input).await.unwrap(); tx.commit().unwrap(); From 4839adeaef13e6f3095ee501e80dbba98aa57333 Mon Sep 17 00:00:00 2001 From: Francisco Krause Arnim <56402156+fkrause98@users.noreply.github.com> Date: Tue, 28 Feb 2023 22:35:37 -0300 Subject: [PATCH 016/191] test: fix hive tests (#1530) Co-authored-by: lambdaclass-user Co-authored-by: Dan Cline <6798349+Rjected@users.noreply.github.com> --- Cargo.lock | 179 ++++++++++++---------------- crates/primitives/src/chain/spec.rs | 94 ++++++++++++++- crates/primitives/src/genesis.rs | 15 +-- 3 files changed, 175 insertions(+), 113 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 152e38dc628..d2046ac0615 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -142,7 +142,7 @@ dependencies = [ "proc-macro-error", "proc-macro2 1.0.51", "quote 1.0.23", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -208,7 +208,7 @@ checksum = "10f203db73a71dfa2fb6dd22763990fa26f3d2625a6da2da900d23b87d26be27" dependencies = [ "proc-macro2 1.0.51", "quote 1.0.23", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -219,7 +219,7 @@ checksum = "1cd7fce9ba8c3c042128ce72d8b2ddbf3a05747efb67ea0313c635e10bda47a2" dependencies = [ "proc-macro2 1.0.51", "quote 1.0.23", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -265,18 +265,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "auto_impl" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7862e21c893d65a1650125d157eaeec691439379a1cee17ee49031b79236ada4" -dependencies = [ - "proc-macro-error", - "proc-macro2 1.0.51", - "quote 1.0.23", - "syn 1.0.107", -] - [[package]] name = "auto_impl" version = "1.0.1" @@ -286,7 +274,7 @@ dependencies = [ "proc-macro-error", "proc-macro2 1.0.51", "quote 1.0.23", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -418,7 +406,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -780,7 +768,7 @@ dependencies = [ "proc-macro-error", "proc-macro2 1.0.51", "quote 1.0.23", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -816,7 +804,7 @@ dependencies = [ "proc-macro2 1.0.51", "quote 1.0.23", "serde", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -1183,7 +1171,7 @@ dependencies = [ "proc-macro2 1.0.51", "quote 1.0.23", "scratch", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -1200,7 +1188,7 @@ checksum = "357f40d1f06a24b60ae1fe122542c1fb05d28d32acb2aed064e84bc2ad1e252e" dependencies = [ "proc-macro2 1.0.51", "quote 1.0.23", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -1234,7 +1222,7 @@ dependencies = [ "proc-macro2 1.0.51", "quote 1.0.23", "strsim 0.9.3", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -1248,7 +1236,7 @@ dependencies = [ "proc-macro2 1.0.51", "quote 1.0.23", "strsim 0.10.0", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -1259,7 +1247,7 @@ checksum = "d9b5a2f4ac4969822c62224815d069952656cadc7084fdca9751e6d959189b72" dependencies = [ "darling_core 0.10.2", "quote 1.0.23", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -1270,7 +1258,7 @@ checksum = "7618812407e9402654622dd402b0a89dff9ba93badd6540781526117b92aab7e" dependencies = [ "darling_core 0.14.2", "quote 1.0.23", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -1329,7 +1317,7 @@ checksum = "8beee4701e2e229e8098bbdecdca12449bc3e322f137d269182fa1291e20bd00" dependencies = [ "proc-macro2 1.0.51", "quote 1.0.23", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -1342,7 +1330,7 @@ dependencies = [ "derive_builder_core", "proc-macro2 1.0.51", "quote 1.0.23", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -1354,7 +1342,7 @@ dependencies = [ "darling 0.10.2", "proc-macro2 1.0.51", "quote 1.0.23", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -1367,7 +1355,7 @@ dependencies = [ "proc-macro2 1.0.51", "quote 1.0.23", "rustc_version", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -1543,7 +1531,7 @@ dependencies = [ "enum-ordinalize", "proc-macro2 1.0.51", "quote 1.0.23", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -1636,7 +1624,7 @@ dependencies = [ "heck", "proc-macro2 1.0.51", "quote 1.0.23", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -1648,7 +1636,7 @@ dependencies = [ "heck", "proc-macro2 1.0.51", "quote 1.0.23", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -1662,7 +1650,7 @@ dependencies = [ "proc-macro2 1.0.51", "quote 1.0.23", "rustc_version", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -1673,7 +1661,7 @@ checksum = "e88bcb3a067a6555d577aba299e75eff9942da276e6506fc6274327daa026132" dependencies = [ "proc-macro2 1.0.51", "quote 1.0.23", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -1770,7 +1758,7 @@ dependencies = [ [[package]] name = "ethers-contract" version = "1.0.2" -source = "git+https://github.com/gakonst/ethers-rs#228f9447269e61ca7d7f868edd58c08772e38650" +source = "git+https://github.com/gakonst/ethers-rs#537d0a9deb8b1fc549bf21b48684e9155f12707b" dependencies = [ "ethers-core", "ethers-providers", @@ -1786,7 +1774,7 @@ dependencies = [ [[package]] name = "ethers-core" version = "1.0.2" -source = "git+https://github.com/gakonst/ethers-rs#228f9447269e61ca7d7f868edd58c08772e38650" +source = "git+https://github.com/gakonst/ethers-rs#537d0a9deb8b1fc549bf21b48684e9155f12707b" dependencies = [ "arrayvec", "bytes", @@ -1807,7 +1795,7 @@ dependencies = [ "serde", "serde_json", "strum", - "syn 1.0.107", + "syn 1.0.109", "tempfile", "thiserror", "tiny-keccak", @@ -1817,7 +1805,7 @@ dependencies = [ [[package]] name = "ethers-etherscan" version = "1.0.2" -source = "git+https://github.com/gakonst/ethers-rs#228f9447269e61ca7d7f868edd58c08772e38650" +source = "git+https://github.com/gakonst/ethers-rs#537d0a9deb8b1fc549bf21b48684e9155f12707b" dependencies = [ "ethers-core", "getrandom 0.2.8", @@ -1833,10 +1821,10 @@ dependencies = [ [[package]] name = "ethers-middleware" version = "1.0.2" -source = "git+https://github.com/gakonst/ethers-rs#228f9447269e61ca7d7f868edd58c08772e38650" +source = "git+https://github.com/gakonst/ethers-rs#537d0a9deb8b1fc549bf21b48684e9155f12707b" dependencies = [ "async-trait", - "auto_impl 0.5.0", + "auto_impl", "ethers-contract", "ethers-core", "ethers-etherscan", @@ -1858,10 +1846,10 @@ dependencies = [ [[package]] name = "ethers-providers" version = "1.0.2" -source = "git+https://github.com/gakonst/ethers-rs#228f9447269e61ca7d7f868edd58c08772e38650" +source = "git+https://github.com/gakonst/ethers-rs#537d0a9deb8b1fc549bf21b48684e9155f12707b" dependencies = [ "async-trait", - "auto_impl 1.0.1", + "auto_impl", "base64 0.21.0", "enr 0.7.0", "ethers-core", @@ -1895,7 +1883,7 @@ dependencies = [ [[package]] name = "ethers-signers" version = "1.0.2" -source = "git+https://github.com/gakonst/ethers-rs#228f9447269e61ca7d7f868edd58c08772e38650" +source = "git+https://github.com/gakonst/ethers-rs#537d0a9deb8b1fc549bf21b48684e9155f12707b" dependencies = [ "async-trait", "coins-bip32", @@ -1907,6 +1895,7 @@ dependencies = [ "rand 0.8.5", "sha2 0.10.6", "thiserror", + "tracing", ] [[package]] @@ -2101,7 +2090,7 @@ checksum = "95a73af87da33b5acf53acfebdc339fe592ecf5357ac7c0a7734ab9d8c876a70" dependencies = [ "proc-macro2 1.0.51", "quote 1.0.23", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -2720,7 +2709,7 @@ checksum = "11d7a9f6330b71fea57921c9b61c47ee6e84f72d394754eff6163ae67e7395eb" dependencies = [ "proc-macro2 1.0.51", "quote 1.0.23", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -2962,7 +2951,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2 1.0.51", "quote 1.0.23", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -3274,7 +3263,7 @@ checksum = "731f8ecebd9f3a4aa847dfe75455e4757a45da40a7793d2f0b1f9b6ed18b23f3" dependencies = [ "proc-macro2 1.0.51", "quote 1.0.23", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -3359,7 +3348,7 @@ checksum = "5a7d5f7076603ebc68de2dc6a650ec331a062a13abaa346975be747bbfa4b789" dependencies = [ "proc-macro2 1.0.51", "quote 1.0.23", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -3516,23 +3505,23 @@ dependencies = [ [[package]] name = "num_enum" -version = "0.5.9" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d829733185c1ca374f17e52b762f24f535ec625d2cc1f070e34c8a9068f341b" +checksum = "1f646caf906c20226733ed5b1374287eb97e3c2a5c227ce668c1f2ce20ae57c9" dependencies = [ "num_enum_derive", ] [[package]] name = "num_enum_derive" -version = "0.5.9" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2be1598bf1c313dcdd12092e3f1920f463462525a21b7b4e11b4168353d0123e" +checksum = "dcbff9bc912032c62bf65ef1d5aea88983b420f4f839db1e9b0c281a25c9c799" dependencies = [ "proc-macro-crate", "proc-macro2 1.0.51", "quote 1.0.23", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -3575,7 +3564,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "786393f80485445794f6043fd3138854dd109cc6c4bd1a6383db304c9ce9b9ce" dependencies = [ "arrayvec", - "auto_impl 1.0.1", + "auto_impl", "bytes", "ethereum-types", "open-fastrlp-derive", @@ -3590,7 +3579,7 @@ dependencies = [ "bytes", "proc-macro2 1.0.51", "quote 1.0.23", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -3654,7 +3643,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2 1.0.51", "quote 1.0.23", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -3812,7 +3801,7 @@ checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" dependencies = [ "proc-macro2 1.0.51", "quote 1.0.23", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -3964,7 +3953,7 @@ dependencies = [ "proc-macro-error-attr", "proc-macro2 1.0.51", "quote 1.0.23", - "syn 1.0.107", + "syn 1.0.109", "version_check", ] @@ -4284,15 +4273,6 @@ version = "0.6.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" -[[package]] -name = "remove_dir_all" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" -dependencies = [ - "winapi", -] - [[package]] name = "reqwest" version = "0.11.14" @@ -4593,7 +4573,7 @@ name = "reth-executor" version = "0.1.0" dependencies = [ "async-trait", - "auto_impl 1.0.1", + "auto_impl", "hash-db", "hashbrown 0.13.2", "plain_hasher", @@ -4619,7 +4599,7 @@ version = "0.1.0" dependencies = [ "arbitrary", "async-trait", - "auto_impl 1.0.1", + "auto_impl", "futures", "hex-literal", "modular-bitfield", @@ -4705,7 +4685,7 @@ dependencies = [ "quote 1.0.23", "regex", "serial_test", - "syn 1.0.107", + "syn 1.0.109", "trybuild", ] @@ -4738,7 +4718,7 @@ version = "0.1.0" dependencies = [ "aquamarine", "async-trait", - "auto_impl 1.0.1", + "auto_impl", "enr 0.7.0", "ethers-core", "ethers-middleware", @@ -4840,7 +4820,7 @@ dependencies = [ name = "reth-provider" version = "0.1.0" dependencies = [ - "auto_impl 1.0.1", + "auto_impl", "parking_lot 0.12.1", "reth-db", "reth-interfaces", @@ -4883,7 +4863,7 @@ name = "reth-rlp" version = "0.1.2" dependencies = [ "arrayvec", - "auto_impl 1.0.1", + "auto_impl", "bytes", "criterion", "enr 0.7.0", @@ -4906,7 +4886,7 @@ version = "0.1.1" dependencies = [ "proc-macro2 1.0.51", "quote 1.0.23", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -5126,7 +5106,7 @@ version = "0.1.0" dependencies = [ "aquamarine", "async-trait", - "auto_impl 1.0.1", + "auto_impl", "bitflags", "fnv", "futures-util", @@ -5149,7 +5129,7 @@ name = "revm" version = "3.0.0" source = "git+https://github.com/bluealloy/revm#4d2f0741c5f9daec0ceb7cc7733d65ea4c496170" dependencies = [ - "auto_impl 1.0.1", + "auto_impl", "revm-interpreter", "revm-precompile", ] @@ -5187,7 +5167,7 @@ version = "1.0.0" source = "git+https://github.com/bluealloy/revm#4d2f0741c5f9daec0ceb7cc7733d65ea4c496170" dependencies = [ "arbitrary", - "auto_impl 1.0.1", + "auto_impl", "bytes", "derive_more", "enumn", @@ -5266,7 +5246,7 @@ checksum = "e33d7b2abe0c340d8797fe2907d3f20d3b5ea5908683618bfe80df7f621f672a" dependencies = [ "proc-macro2 1.0.51", "quote 1.0.23", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -5427,7 +5407,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2 1.0.51", "quote 1.0.23", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -5626,7 +5606,7 @@ checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" dependencies = [ "proc-macro2 1.0.51", "quote 1.0.23", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -5677,7 +5657,7 @@ dependencies = [ "darling 0.14.2", "proc-macro2 1.0.51", "quote 1.0.23", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -5702,7 +5682,7 @@ checksum = "b64f9e531ce97c88b4778aad0ceee079216071cffec6ac9b904277f8f92e7fe3" dependencies = [ "proc-macro2 1.0.51", "quote 1.0.23", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -5998,7 +5978,7 @@ dependencies = [ "proc-macro2 1.0.51", "quote 1.0.23", "rustversion", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -6075,9 +6055,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.107" +version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ "proc-macro2 1.0.51", "quote 1.0.23", @@ -6092,7 +6072,7 @@ checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" dependencies = [ "proc-macro2 1.0.51", "quote 1.0.23", - "syn 1.0.107", + "syn 1.0.109", "unicode-xid 0.2.4", ] @@ -6104,16 +6084,15 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tempfile" -version = "3.3.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" +checksum = "af18f7ae1acd354b992402e9ec5864359d693cd8a79dcbef59f76891701c1e95" dependencies = [ "cfg-if", "fastrand", - "libc", "redox_syscall", - "remove_dir_all", - "winapi", + "rustix", + "windows-sys 0.42.0", ] [[package]] @@ -6162,7 +6141,7 @@ dependencies = [ "proc-macro2 1.0.51", "quote 1.0.23", "subprocess", - "syn 1.0.107", + "syn 1.0.109", "test-fuzz-internal", "toolchain_find", "unzip-n", @@ -6205,7 +6184,7 @@ checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f" dependencies = [ "proc-macro2 1.0.51", "quote 1.0.23", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -6306,7 +6285,7 @@ checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" dependencies = [ "proc-macro2 1.0.51", "quote 1.0.23", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -6521,7 +6500,7 @@ checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" dependencies = [ "proc-macro2 1.0.51", "quote 1.0.23", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -6606,7 +6585,7 @@ checksum = "744324b12d69a9fc1edea4b38b7b1311295b662d161ad5deac17bb1358224a08" dependencies = [ "lazy_static", "quote 1.0.23", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -6870,7 +6849,7 @@ checksum = "c2e7e85a0596447f0f2ac090e16bc4c516c6fe91771fb0c0ccf7fa3dae896b9c" dependencies = [ "proc-macro2 1.0.51", "quote 1.0.23", - "syn 1.0.107", + "syn 1.0.109", ] [[package]] @@ -6996,7 +6975,7 @@ dependencies = [ "once_cell", "proc-macro2 1.0.51", "quote 1.0.23", - "syn 1.0.107", + "syn 1.0.109", "wasm-bindgen-shared", ] @@ -7030,7 +7009,7 @@ checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" dependencies = [ "proc-macro2 1.0.51", "quote 1.0.23", - "syn 1.0.107", + "syn 1.0.109", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -7278,6 +7257,6 @@ checksum = "44bf07cb3e50ea2003396695d58bf46bc9887a1f362260446fad6bc4e79bd36c" dependencies = [ "proc-macro2 1.0.51", "quote 1.0.23", - "syn 1.0.107", + "syn 1.0.109", "synstructure", ] diff --git a/crates/primitives/src/chain/spec.rs b/crates/primitives/src/chain/spec.rs index 0fdaa93ec14..06a91fc4e0d 100644 --- a/crates/primitives/src/chain/spec.rs +++ b/crates/primitives/src/chain/spec.rs @@ -298,7 +298,7 @@ impl From for ChainSpec { } /// A helper type for compatibility with geth's config -#[derive(Debug, Clone, Deserialize)] +#[derive(Debug, Clone, Deserialize, Serialize)] #[serde(untagged)] pub enum AllGenesisFormats { /// The geth genesis format @@ -583,13 +583,12 @@ impl ForkCondition { #[cfg(test)] mod tests { - use revm_primitives::U256; - use crate::{ - Chain, ChainSpec, ChainSpecBuilder, ForkCondition, ForkHash, ForkId, Genesis, Hardfork, - Head, GOERLI, MAINNET, SEPOLIA, + AllGenesisFormats, Chain, ChainSpec, ChainSpecBuilder, ForkCondition, ForkHash, ForkId, + Genesis, Hardfork, Head, GOERLI, H256, MAINNET, SEPOLIA, }; - + use ethers_core::types as EtherType; + use revm_primitives::U256; fn test_fork_ids(spec: &ChainSpec, cases: &[(Head, ForkId)]) { for (block, expected_id) in cases { let computed_id = spec.fork_id(block); @@ -1028,4 +1027,87 @@ mod tests { &ForkCondition::Timestamp(0) ); } + + #[test] + fn hive_geth_json() { + let hive_json = r#" + { + "nonce": "0x0000000000000042", + "difficulty": "0x2123456", + "mixHash": "0x123456789abcdef123456789abcdef123456789abcdef123456789abcdef1234", + "coinbase": "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "timestamp": "0x123456", + "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "extraData": "0xfafbfcfd", + "gasLimit": "0x2fefd8", + "alloc": { + "dbdbdb2cbd23b783741e8d7fcf51e459b497e4a6": { + "balance": "0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + }, + "e6716f9544a56c530d868e4bfbacb172315bdead": { + "balance": "0x11", + "code": "0x12" + }, + "b9c015918bdaba24b4ff057a92a3873d6eb201be": { + "balance": "0x21", + "storage": { + "0x0000000000000000000000000000000000000000000000000000000000000001": "0x22" + } + }, + "1a26338f0d905e295fccb71fa9ea849ffa12aaf4": { + "balance": "0x31", + "nonce": "0x32" + }, + "0000000000000000000000000000000000000001": { + "balance": "0x41" + }, + "0000000000000000000000000000000000000002": { + "balance": "0x51" + }, + "0000000000000000000000000000000000000003": { + "balance": "0x61" + }, + "0000000000000000000000000000000000000004": { + "balance": "0x71" + } + }, + "config": { + "ethash": {}, + "chainId": 10, + "homesteadBlock": 0, + "eip150Block": 0, + "eip155Block": 0, + "eip158Block": 0, + "byzantiumBlock": 0, + "constantinopleBlock": 0, + "petersburgBlock": 0, + "istanbulBlock": 0 + } + } + "#; + let genesis = serde_json::from_str::(hive_json).unwrap(); + let chainspec: ChainSpec = genesis.into(); + assert_eq!(chainspec.genesis_hash, None); + assert_eq!(Chain::Named(EtherType::Chain::Optimism), chainspec.chain); + let expected_state_root: H256 = + hex_literal::hex!("9a6049ac535e3dc7436c189eaa81c73f35abd7f282ab67c32944ff0301d63360") + .into(); + assert_eq!(chainspec.genesis_header().state_root, expected_state_root); + let hard_forks = vec![ + Hardfork::Byzantium, + Hardfork::Homestead, + Hardfork::Istanbul, + Hardfork::Petersburg, + Hardfork::Constantinople, + ]; + for ref fork in hard_forks { + assert_eq!(chainspec.hardforks.get(fork).unwrap(), &ForkCondition::Block(0)); + } + + let expected_hash: H256 = + hex_literal::hex!("5ae31c6522bd5856129f66be3d582b842e4e9faaa87f21cce547128339a9db3c") + .into(); + let hash = chainspec.genesis_header().hash_slow(); + assert_eq!(hash, expected_hash); + } } diff --git a/crates/primitives/src/genesis.rs b/crates/primitives/src/genesis.rs index 56d6e1dddc4..4fd8752f58e 100644 --- a/crates/primitives/src/genesis.rs +++ b/crates/primitives/src/genesis.rs @@ -6,9 +6,8 @@ use crate::{ utils::serde_helpers::deserialize_stringified_u64, Address, Bytes, H256, KECCAK_EMPTY, U256, }; -use bytes::BytesMut; use ethers_core::utils::GenesisAccount as EthersGenesisAccount; -use reth_rlp::{length_of_length, Encodable, Header as RlpHeader}; +use reth_rlp::{encode_fixed_size, length_of_length, Encodable, Header as RlpHeader}; use serde::{Deserialize, Serialize}; use triehash::sec_trie_root; @@ -62,7 +61,8 @@ impl GenesisAccount { // rather than rlp-encoding the storage, we just return the length of a single hash // hashes are a fixed size, so it is safe to use the empty root for this len += EMPTY_ROOT.length(); - len += self.code.as_ref().map_or(KECCAK_EMPTY, keccak256).length(); + // we are encoding a hash, so let's just use the length of the empty hash for the code hash + len += KECCAK_EMPTY.length(); len } } @@ -83,13 +83,14 @@ impl Encodable for GenesisAccount { self.storage .as_ref() .map_or(EMPTY_ROOT, |storage| { + if storage.is_empty() { + return EMPTY_ROOT + } let storage_values = storage.iter().filter(|(_k, &v)| v != KECCAK_EMPTY).map(|(&k, v)| { - let mut value_rlp = BytesMut::new(); - v.encode(&mut value_rlp); - (k, value_rlp.freeze()) + let value = U256::from_be_bytes(**v); + (k, encode_fixed_size(&value)) }); - sec_trie_root::(storage_values) }) .encode(out); From 2884eae0755ea2809c4d70dd39863997ca55dfa4 Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Wed, 1 Mar 2023 14:20:00 +0800 Subject: [PATCH 017/191] perf: bench merkle stage (#1497) --- bin/reth/src/dump_stage/merkle.rs | 137 +++++++++++++++ bin/reth/src/dump_stage/mod.rs | 8 + crates/interfaces/Cargo.toml | 16 +- .../interfaces/src/test_utils/generators.rs | 116 ++++++++++++- crates/primitives/src/storage.rs | 6 + crates/stages/benches/criterion.rs | 134 +++++++++------ .../stages/benches/setup/account_hashing.rs | 10 +- crates/stages/benches/setup/mod.rs | 151 ++++++++++++++-- crates/stages/src/lib.rs | 1 + crates/stages/src/stages/hashing_storage.rs | 41 +++-- crates/stages/src/stages/merkle.rs | 161 +++++------------- crates/stages/src/test_utils/test_db.rs | 151 +++++++++++----- crates/stages/src/trie/mod.rs | 106 ++++++++---- crates/storage/db/benches/utils.rs | 4 + 14 files changed, 764 insertions(+), 278 deletions(-) create mode 100644 bin/reth/src/dump_stage/merkle.rs diff --git a/bin/reth/src/dump_stage/merkle.rs b/bin/reth/src/dump_stage/merkle.rs new file mode 100644 index 00000000000..8e4b2b56115 --- /dev/null +++ b/bin/reth/src/dump_stage/merkle.rs @@ -0,0 +1,137 @@ +use crate::{ + db::DbTool, + dirs::{DbPath, PlatformPath}, + dump_stage::setup, +}; +use eyre::Result; +use reth_db::{database::Database, table::TableImporter, tables, transaction::DbTx}; +use reth_primitives::MAINNET; +use reth_provider::Transaction; +use reth_stages::{ + stages::{AccountHashingStage, ExecutionStage, MerkleStage, StorageHashingStage}, + DefaultDB, Stage, StageId, UnwindInput, +}; +use std::ops::DerefMut; +use tracing::info; + +pub(crate) async fn dump_merkle_stage( + db_tool: &mut DbTool<'_, DB>, + from: u64, + to: u64, + output_db: &PlatformPath, + should_run: bool, +) -> Result<()> { + let (output_db, tip_block_number) = setup::(from, to, output_db, db_tool)?; + + output_db.update(|tx| { + tx.import_table_with_range::(&db_tool.db.tx()?, Some(from), to) + })??; + + let tx = db_tool.db.tx()?; + let from_transition_rev = + tx.get::(from)?.expect("there should be at least one."); + let to_transition_rev = + tx.get::(to)?.expect("there should be at least one."); + + output_db.update(|tx| { + tx.import_table_with_range::( + &db_tool.db.tx()?, + Some(from_transition_rev), + to_transition_rev, + ) + })??; + + unwind_and_copy::(db_tool, (from, to), tip_block_number, &output_db).await?; + + if should_run { + println!( + "\n# Merkle stage does not support dry run, so it will actually be committing changes." + ); + run(output_db, to, from).await?; + } + + Ok(()) +} + +/// Dry-run an unwind to FROM block and copy the necessary table data to the new database. +async fn unwind_and_copy( + db_tool: &mut DbTool<'_, DB>, + range: (u64, u64), + tip_block_number: u64, + output_db: &reth_db::mdbx::Env, +) -> eyre::Result<()> { + let (from, to) = range; + let mut unwind_tx = Transaction::new(db_tool.db)?; + let unwind = UnwindInput { unwind_to: from, stage_progress: tip_block_number, bad_block: None }; + let execute_input = reth_stages::ExecInput { + previous_stage: Some((StageId("Another"), to)), + stage_progress: Some(from), + }; + + // Unwind hashes all the way to FROM + StorageHashingStage::default().unwind(&mut unwind_tx, unwind).await.unwrap(); + AccountHashingStage::default().unwind(&mut unwind_tx, unwind).await.unwrap(); + + MerkleStage::default_unwind().unwind(&mut unwind_tx, unwind).await?; + + // Bring Plainstate to TO (hashing stage execution requires it) + let mut exec_stage: ExecutionStage<'_, DefaultDB<'_>> = ExecutionStage::from(MAINNET.clone()); + exec_stage.commit_threshold = u64::MAX; + exec_stage + .unwind( + &mut unwind_tx, + UnwindInput { unwind_to: to, stage_progress: tip_block_number, bad_block: None }, + ) + .await?; + + // Bring hashes to TO + AccountHashingStage { clean_threshold: u64::MAX, commit_threshold: u64::MAX } + .execute(&mut unwind_tx, execute_input) + .await + .unwrap(); + StorageHashingStage { clean_threshold: u64::MAX, commit_threshold: u64::MAX } + .execute(&mut unwind_tx, execute_input) + .await + .unwrap(); + + let unwind_inner_tx = unwind_tx.deref_mut(); + + // TODO optimize we can actually just get the entries we need + output_db.update(|tx| tx.import_dupsort::(unwind_inner_tx))??; + + output_db.update(|tx| tx.import_table::(unwind_inner_tx))??; + output_db.update(|tx| tx.import_dupsort::(unwind_inner_tx))??; + output_db.update(|tx| tx.import_table::(unwind_inner_tx))??; + output_db.update(|tx| tx.import_dupsort::(unwind_inner_tx))??; + + unwind_tx.drop()?; + + Ok(()) +} + +/// Try to re-execute the stage straightaway +async fn run( + output_db: reth_db::mdbx::Env, + to: u64, + from: u64, +) -> eyre::Result<()> { + info!(target: "reth::cli", "Executing stage."); + + let mut tx = Transaction::new(&output_db)?; + + MerkleStage::Execution { + clean_threshold: u64::MAX, // Forces updating the root instead of calculating from scratch + } + .execute( + &mut tx, + reth_stages::ExecInput { + previous_stage: Some((StageId("Another"), to)), + stage_progress: Some(from), + }, + ) + .await?; + + info!(target: "reth::cli", "Success."); + + Ok(()) +} diff --git a/bin/reth/src/dump_stage/mod.rs b/bin/reth/src/dump_stage/mod.rs index 0bc3863a342..4eddb3817da 100644 --- a/bin/reth/src/dump_stage/mod.rs +++ b/bin/reth/src/dump_stage/mod.rs @@ -8,6 +8,9 @@ use hashing_account::dump_hashing_account_stage; mod execution; use execution::dump_execution_stage; +mod merkle; +use merkle::dump_merkle_stage; + use crate::{ db::DbTool, dirs::{DbPath, PlatformPath}, @@ -45,6 +48,8 @@ pub enum Stages { StorageHashing(StageCommand), /// AccountHashing stage. AccountHashing(StageCommand), + /// Merkle stage. + Merkle(StageCommand), } /// Stage command that takes a range @@ -94,6 +99,9 @@ impl Command { Stages::AccountHashing(StageCommand { output_db, from, to, dry_run, .. }) => { dump_hashing_account_stage(&mut tool, *from, *to, output_db, *dry_run).await? } + Stages::Merkle(StageCommand { output_db, from, to, dry_run, .. }) => { + dump_merkle_stage(&mut tool, *from, *to, output_db, *dry_run).await? + } } Ok(()) diff --git a/crates/interfaces/Cargo.toml b/crates/interfaces/Cargo.toml index 9448840bf3a..a1b2dfcf286 100644 --- a/crates/interfaces/Cargo.toml +++ b/crates/interfaces/Cargo.toml @@ -10,7 +10,7 @@ readme = "README.md" reth-codecs = { path = "../storage/codecs" } reth-primitives = { path = "../primitives" } reth-rpc-types = { path = "../rpc/rpc-types" } -reth-network-api = { path = "../net/network-api"} +reth-network-api = { path = "../net/network-api" } revm-primitives = "1.0" async-trait = "0.1.57" thiserror = "1.0.37" @@ -26,16 +26,24 @@ futures = "0.3" tokio-stream = "0.1.11" rand = "0.8.5" arbitrary = { version = "1.1.7", features = ["derive"], optional = true } -secp256k1 = { version = "0.24.2", default-features = false, features = ["alloc", "recovery", "rand"], optional = true } +secp256k1 = { version = "0.24.2", default-features = false, features = [ + "alloc", + "recovery", + "rand", +], optional = true } modular-bitfield = "0.11.2" [dev-dependencies] reth-db = { path = "../storage/db", features = ["test-utils"] } tokio = { version = "1.21.2", features = ["full"] } tokio-stream = { version = "0.1.11", features = ["sync"] } -arbitrary = { version = "1.1.7", features = ["derive"]} +arbitrary = { version = "1.1.7", features = ["derive"] } hex-literal = "0.3" -secp256k1 = { version = "0.24.2", default-features = false, features = ["alloc", "recovery", "rand"] } +secp256k1 = { version = "0.24.2", default-features = false, features = [ + "alloc", + "recovery", + "rand", +] } [features] bench = [] diff --git a/crates/interfaces/src/test_utils/generators.rs b/crates/interfaces/src/test_utils/generators.rs index 4d0b7fa07f4..d7e17aee632 100644 --- a/crates/interfaces/src/test_utils/generators.rs +++ b/crates/interfaces/src/test_utils/generators.rs @@ -1,9 +1,10 @@ -use rand::{distributions::uniform::SampleRange, thread_rng, Rng}; +use rand::{distributions::uniform::SampleRange, seq::SliceRandom, thread_rng, Rng}; use reth_primitives::{ - proofs, Account, Address, Bytes, Header, SealedBlock, SealedHeader, Signature, Transaction, - TransactionKind, TransactionSigned, TxLegacy, H160, H256, U256, + proofs, Account, Address, Bytes, Header, SealedBlock, SealedHeader, Signature, StorageEntry, + Transaction, TransactionKind, TransactionSigned, TxLegacy, H160, H256, U256, }; use secp256k1::{KeyPair, Message as SecpMessage, Secp256k1, SecretKey}; +use std::{collections::BTreeMap, ops::Sub}; // TODO(onbjerg): Maybe we should split this off to its own crate, or move the helpers to the // relevant crates? @@ -165,6 +166,115 @@ pub fn random_block_range( blocks } +type Transition = Vec<(Address, Account, Vec)>; +type AccountState = (Account, Vec); + +/// Generate a range of transitions for given blocks and accounts. +/// Assumes all accounts start with an empty storage. +/// +/// Returns a Vec of account and storage changes for each transition, +/// along with the final state of all accounts and storages. +pub fn random_transition_range<'a, IBlk, IAcc>( + blocks: IBlk, + accounts: IAcc, + n_changes: std::ops::Range, + key_range: std::ops::Range, +) -> (Vec, BTreeMap) +where + IBlk: IntoIterator, + IAcc: IntoIterator))>, +{ + let mut rng = rand::thread_rng(); + let mut state: BTreeMap<_, _> = accounts + .into_iter() + .map(|(addr, (acc, st))| (addr, (acc, st.into_iter().map(|e| (e.key, e.value)).collect()))) + .collect(); + + let valid_addresses = state.keys().copied().collect(); + + let num_transitions: usize = blocks.into_iter().map(|block| block.body.len()).sum(); + let mut transitions = Vec::with_capacity(num_transitions); + + (0..num_transitions).for_each(|i| { + let mut transition = Vec::new(); + let (from, to, mut transfer, new_entries) = + random_account_change(&valid_addresses, n_changes.clone(), key_range.clone()); + + // extract from sending account + let (prev_from, _) = state.get_mut(&from).unwrap(); + transition.push((from, *prev_from, Vec::new())); + + transfer = transfer.min(prev_from.balance).max(U256::from(1)); + prev_from.balance = prev_from.balance.wrapping_sub(transfer); + + // deposit in receiving account and update storage + let (prev_to, storage): &mut (Account, BTreeMap) = state.get_mut(&to).unwrap(); + + let old_entries = new_entries + .into_iter() + .filter_map(|entry| { + let old = if entry.value != U256::ZERO { + storage.insert(entry.key, entry.value) + } else { + let old = storage.remove(&entry.key); + if matches!(old, Some(U256::ZERO)) { + return None + } + old + }; + Some(StorageEntry { value: old.unwrap_or(U256::from(0)), ..entry }) + }) + .collect(); + + transition.push((to, *prev_to, old_entries)); + + prev_to.balance = prev_to.balance.wrapping_add(transfer); + + transitions.push(transition); + }); + + let final_state = state + .into_iter() + .map(|(addr, (acc, storage))| { + (addr, (acc, storage.into_iter().map(|v| v.into()).collect())) + }) + .collect(); + (transitions, final_state) +} + +/// Generate a random account change. +/// +/// Returns two addresses, a balance_change, and a Vec of new storage entries. +pub fn random_account_change( + valid_addresses: &Vec
, + n_changes: std::ops::Range, + key_range: std::ops::Range, +) -> (Address, Address, U256, Vec) { + let mut rng = rand::thread_rng(); + let mut addresses = valid_addresses.choose_multiple(&mut rng, 2).cloned(); + + let addr_from = addresses.next().unwrap_or_else(Address::random); + let addr_to = addresses.next().unwrap_or_else(Address::random); + + let balance_change = U256::from(rng.gen::()); + + let storage_changes = (0..n_changes.sample_single(&mut rng)) + .map(|_| random_storage_entry(key_range.clone())) + .collect(); + + (addr_from, addr_to, balance_change, storage_changes) +} + +/// Generate a random storage change. +pub fn random_storage_entry(key_range: std::ops::Range) -> StorageEntry { + let mut rng = rand::thread_rng(); + + let key = H256::from_low_u64_be(key_range.sample_single(&mut rng)); + let value = U256::from(rng.gen::()); + + StorageEntry { key, value } +} + /// Generate random Externaly Owned Account (EOA account without contract). pub fn random_eoa_account() -> (Address, Account) { let nonce: u64 = rand::random(); diff --git a/crates/primitives/src/storage.rs b/crates/primitives/src/storage.rs index f64554c88c4..67e027c34dc 100644 --- a/crates/primitives/src/storage.rs +++ b/crates/primitives/src/storage.rs @@ -12,6 +12,12 @@ pub struct StorageEntry { pub value: U256, } +impl From<(H256, U256)> for StorageEntry { + fn from((key, value): (H256, U256)) -> Self { + StorageEntry { key, value } + } +} + // NOTE: Removing main_codec and manually encode subkey // and compress second part of the value. If we have compression // over whole value (Even SubKey) that would mess up fetching of values with seek_by_key_subkey diff --git a/crates/stages/benches/criterion.rs b/crates/stages/benches/criterion.rs index 3fa21ee6f1e..0a34cbb4360 100644 --- a/crates/stages/benches/criterion.rs +++ b/crates/stages/benches/criterion.rs @@ -5,21 +5,24 @@ use criterion::{ use pprof::criterion::{Output, PProfProfiler}; use reth_db::mdbx::{Env, WriteMap}; use reth_stages::{ - stages::{SenderRecoveryStage, TotalDifficultyStage, TransactionLookupStage}, + stages::{MerkleStage, SenderRecoveryStage, TotalDifficultyStage, TransactionLookupStage}, test_utils::TestTransaction, ExecInput, Stage, StageId, UnwindInput, }; use std::path::PathBuf; mod setup; +use setup::StageRange; criterion_group! { name = benches; - config = Criterion::default().with_profiler(PProfProfiler::new(100, Output::Flamegraph(None))); - targets = transaction_lookup, account_hashing, senders, total_difficulty + config = Criterion::default().with_profiler(PProfProfiler::new(1000, Output::Flamegraph(None))); + targets = transaction_lookup, account_hashing, senders, total_difficulty, merkle } criterion_main!(benches); +const DEFAULT_NUM_BLOCKS: u64 = 10_000; + fn account_hashing(c: &mut Criterion) { let mut group = c.benchmark_group("Stages"); @@ -29,32 +32,42 @@ fn account_hashing(c: &mut Criterion) { let num_blocks = 10_000; let (path, stage, execution_range) = setup::prepare_account_hashing(num_blocks); - measure_stage_with_path(&mut group, stage, path, "AccountHashing".to_string(), execution_range); + measure_stage_with_path( + path, + &mut group, + setup::stage_unwind, + stage, + execution_range, + "AccountHashing".to_string(), + ); } fn senders(c: &mut Criterion) { let mut group = c.benchmark_group("Stages"); - // don't need to run each stage for that many times group.sample_size(10); for batch in [1000usize, 10_000, 100_000, 250_000] { - let num_blocks = 10_000; - let stage = SenderRecoveryStage { commit_threshold: num_blocks, ..Default::default() }; + let stage = SenderRecoveryStage { commit_threshold: DEFAULT_NUM_BLOCKS }; let label = format!("SendersRecovery-batch-{batch}"); - measure_stage(&mut group, stage, num_blocks, label); + + measure_stage(&mut group, setup::stage_unwind, stage, 0..DEFAULT_NUM_BLOCKS, label); } } fn transaction_lookup(c: &mut Criterion) { let mut group = c.benchmark_group("Stages"); - // don't need to run each stage for that many times group.sample_size(10); + let stage = TransactionLookupStage::new(DEFAULT_NUM_BLOCKS); - let num_blocks = 10_000; - let stage = TransactionLookupStage::new(num_blocks); - measure_stage(&mut group, stage, num_blocks, "TransactionLookup".to_string()); + measure_stage( + &mut group, + setup::stage_unwind, + stage, + 0..DEFAULT_NUM_BLOCKS, + "TransactionLookup".to_string(), + ); } fn total_difficulty(c: &mut Criterion) { @@ -63,44 +76,60 @@ fn total_difficulty(c: &mut Criterion) { group.warm_up_time(std::time::Duration::from_millis(2000)); // don't need to run each stage for that many times group.sample_size(10); - - let num_blocks = 10_000; let stage = TotalDifficultyStage::default(); - measure_stage(&mut group, stage, num_blocks, "TotalDifficulty".to_string()); + + measure_stage( + &mut group, + setup::stage_unwind, + stage, + 0..DEFAULT_NUM_BLOCKS, + "TotalDifficulty".to_string(), + ); +} + +fn merkle(c: &mut Criterion) { + let mut group = c.benchmark_group("Stages"); + // don't need to run each stage for that many times + group.sample_size(10); + + let stage = MerkleStage::Both { clean_threshold: u64::MAX }; + measure_stage( + &mut group, + setup::unwind_hashes, + stage, + 1..DEFAULT_NUM_BLOCKS + 1, + "Merkle-incremental".to_string(), + ); + + let stage = MerkleStage::Both { clean_threshold: 0 }; + measure_stage( + &mut group, + setup::unwind_hashes, + stage, + 1..DEFAULT_NUM_BLOCKS + 1, + "Merkle-fullhash".to_string(), + ); } -fn measure_stage_with_path>>( +fn measure_stage_with_path( + path: PathBuf, group: &mut BenchmarkGroup, + setup: F, stage: S, - path: PathBuf, + stage_range: StageRange, label: String, - stage_range: (ExecInput, UnwindInput), -) { +) where + S: Clone + Stage>, + F: Fn(S, &TestTransaction, StageRange), +{ let tx = TestTransaction::new(&path); - let (input, unwind) = stage_range; + let (input, _) = stage_range; group.bench_function(label, move |b| { b.to_async(FuturesExecutor).iter_with_setup( || { // criterion setup does not support async, so we have to use our own runtime - tokio::runtime::Runtime::new().unwrap().block_on(async { - let mut stage = stage.clone(); - let mut db_tx = tx.inner(); - - // Clear previous run - stage - .unwind(&mut db_tx, unwind) - .await - .map_err(|e| { - eyre::eyre!(format!( - "{e}\nMake sure your test database at `{}` isn't too old and incompatible with newer stage changes.", - path.display() - )) - }) - .unwrap(); - - db_tx.commit().unwrap(); - }); + setup(stage.clone(), &tx, stage_range) }, |_| async { let mut stage = stage.clone(); @@ -112,25 +141,34 @@ fn measure_stage_with_path>>( }); } -fn measure_stage>>( +fn measure_stage( group: &mut BenchmarkGroup, + setup: F, stage: S, - num_blocks: u64, + block_interval: std::ops::Range, label: String, -) { - let path = setup::txs_testdata(num_blocks as usize); +) where + S: Clone + Stage>, + F: Fn(S, &TestTransaction, StageRange), +{ + let path = setup::txs_testdata(block_interval.end); measure_stage_with_path( + path, group, + setup, stage, - path, - label, ( ExecInput { - previous_stage: Some((StageId("Another"), num_blocks)), - ..Default::default() + previous_stage: Some((StageId("Another"), block_interval.end)), + stage_progress: Some(block_interval.start), + }, + UnwindInput { + stage_progress: block_interval.end, + unwind_to: block_interval.start, + bad_block: None, }, - UnwindInput::default(), ), - ) + label, + ); } diff --git a/crates/stages/benches/setup/account_hashing.rs b/crates/stages/benches/setup/account_hashing.rs index ec408c358ff..033b71da9fa 100644 --- a/crates/stages/benches/setup/account_hashing.rs +++ b/crates/stages/benches/setup/account_hashing.rs @@ -1,4 +1,4 @@ -use super::constants; +use super::{constants, StageRange}; use reth_db::{ cursor::DbCursorRO, database::Database, tables, transaction::DbTx, Error as DbError, }; @@ -15,9 +15,7 @@ use std::path::{Path, PathBuf}; /// generate its own random data. /// /// Returns the path to the database file, stage and range of stage execution if it exists. -pub fn prepare_account_hashing( - num_blocks: u64, -) -> (PathBuf, AccountHashingStage, (ExecInput, UnwindInput)) { +pub fn prepare_account_hashing(num_blocks: u64) -> (PathBuf, AccountHashingStage, StageRange) { let (path, stage_range) = match std::env::var(constants::ACCOUNT_HASHING_DB) { Ok(db) => { let path = Path::new(&db).to_path_buf(); @@ -30,7 +28,7 @@ pub fn prepare_account_hashing( (path, AccountHashingStage::default(), stage_range) } -fn find_stage_range(db: &Path) -> (ExecInput, UnwindInput) { +fn find_stage_range(db: &Path) -> StageRange { let mut stage_range = None; TestTransaction::new(db) .tx @@ -54,7 +52,7 @@ fn find_stage_range(db: &Path) -> (ExecInput, UnwindInput) { stage_range.expect("Could not find the stage range from the external DB.") } -fn generate_testdata_db(num_blocks: u64) -> (PathBuf, (ExecInput, UnwindInput)) { +fn generate_testdata_db(num_blocks: u64) -> (PathBuf, StageRange) { let opts = SeedOpts { blocks: 0..num_blocks + 1, accounts: 0..10_000, diff --git a/crates/stages/benches/setup/mod.rs b/crates/stages/benches/setup/mod.rs index 7f7932bfede..34876d031ad 100644 --- a/crates/stages/benches/setup/mod.rs +++ b/crates/stages/benches/setup/mod.rs @@ -1,34 +1,165 @@ +use itertools::concat; use reth_db::{ cursor::DbCursorRO, + mdbx::{Env, WriteMap}, tables, transaction::{DbTx, DbTxMut}, }; -use reth_interfaces::test_utils::generators::random_block_range; -use reth_primitives::H256; -use reth_stages::test_utils::TestTransaction; -use std::path::{Path, PathBuf}; +use reth_interfaces::test_utils::generators::{ + random_block_range, random_contract_account_range, random_eoa_account_range, + random_transition_range, +}; +use reth_primitives::{Account, Address, SealedBlock, H256}; +use reth_stages::{ + stages::{AccountHashingStage, StorageHashingStage}, + test_utils::TestTransaction, + DBTrieLoader, ExecInput, Stage, UnwindInput, +}; +use std::{ + collections::BTreeMap, + path::{Path, PathBuf}, +}; mod constants; mod account_hashing; pub use account_hashing::*; -// Helper for generating testdata for the sender recovery stage and tx lookup stages (512MB). -// Returns the path to the database file and the number of blocks written. -pub fn txs_testdata(num_blocks: usize) -> PathBuf { +pub(crate) type StageRange = (ExecInput, UnwindInput); + +pub(crate) fn stage_unwind>>( + stage: S, + tx: &TestTransaction, + range: StageRange, +) { + let (_, unwind) = range; + + tokio::runtime::Runtime::new().unwrap().block_on(async { + let mut stage = stage.clone(); + let mut db_tx = tx.inner(); + + // Clear previous run + stage + .unwind(&mut db_tx, unwind) + .await + .map_err(|e| { + eyre::eyre!(format!( + "{e}\nMake sure your test database at `{}` isn't too old and incompatible with newer stage changes.", + tx.path.as_ref().unwrap().display() + )) + }) + .unwrap(); + + db_tx.commit().unwrap(); + }); +} + +pub(crate) fn unwind_hashes>>( + stage: S, + tx: &TestTransaction, + range: StageRange, +) { + let (input, unwind) = range; + + tokio::runtime::Runtime::new().unwrap().block_on(async { + let mut stage = stage.clone(); + let mut db_tx = tx.inner(); + + StorageHashingStage::default().unwind(&mut db_tx, unwind).await.unwrap(); + AccountHashingStage::default().unwind(&mut db_tx, unwind).await.unwrap(); + + let target_root = db_tx.get_header(unwind.unwind_to).unwrap().state_root; + let _ = db_tx.delete::(target_root, None); + + // Clear previous run + stage.unwind(&mut db_tx, unwind).await.unwrap(); + + AccountHashingStage::default().execute(&mut db_tx, input).await.unwrap(); + StorageHashingStage::default().execute(&mut db_tx, input).await.unwrap(); + + db_tx.commit().unwrap(); + }); +} + +// Helper for generating testdata for the benchmarks. +// Returns the path to the database file. +pub(crate) fn txs_testdata(num_blocks: u64) -> PathBuf { let path = Path::new(env!("CARGO_MANIFEST_DIR")).join("testdata").join("txs-bench"); let txs_range = 100..150; + // number of storage changes per transition + let n_changes = 0..3; + + // range of possible values for a storage key + let key_range = 0..300; + + // number of accounts + let n_eoa = 131; + let n_contract = 31; + if !path.exists() { // create the dirs std::fs::create_dir_all(&path).unwrap(); println!("Transactions testdata not found, generating to {:?}", path.display()); let tx = TestTransaction::new(&path); - // This takes a while because it does sig recovery internally - let blocks = random_block_range(0..num_blocks as u64 + 1, H256::zero(), txs_range); + let accounts: BTreeMap = concat([ + random_eoa_account_range(0..n_eoa), + random_contract_account_range(&mut (0..n_contract)), + ]) + .into_iter() + .collect(); + + let mut blocks = random_block_range(0..num_blocks + 1, H256::zero(), txs_range); + + let (transitions, start_state) = random_transition_range( + blocks.iter().take(2), + accounts.into_iter().map(|(addr, acc)| (addr, (acc, Vec::new()))), + n_changes.clone(), + key_range.clone(), + ); + + tx.insert_accounts_and_storages(start_state.clone()).unwrap(); + + // make first block after genesis have valid state root + let root = DBTrieLoader::default().calculate_root(&tx.inner()).unwrap(); + let second_block = blocks.get_mut(1).unwrap(); + let cloned_second = second_block.clone(); + let mut updated_header = cloned_second.header.unseal(); + updated_header.state_root = root; + *second_block = SealedBlock { header: updated_header.seal_slow(), ..cloned_second }; + + let offset = transitions.len() as u64; + + tx.insert_transitions(transitions, None).unwrap(); + + let (transitions, final_state) = + random_transition_range(blocks.iter().skip(2), start_state, n_changes, key_range); + + tx.insert_transitions(transitions, Some(offset)).unwrap(); + + tx.insert_accounts_and_storages(final_state).unwrap(); + + // make last block have valid state root + let root = { + let mut tx_mut = tx.inner(); + let root = DBTrieLoader::default().calculate_root(&tx_mut).unwrap(); + tx_mut.commit().unwrap(); + root + }; + + tx.query(|tx| { + assert!(tx.get::(root)?.is_some()); + Ok(()) + }) + .unwrap(); + + let last_block = blocks.last_mut().unwrap(); + let cloned_last = last_block.clone(); + let mut updated_header = cloned_last.header.unseal(); + updated_header.state_root = root; + *last_block = SealedBlock { header: updated_header.seal_slow(), ..cloned_last }; - // insert all blocks tx.insert_blocks(blocks.iter(), None).unwrap(); // initialize TD diff --git a/crates/stages/src/lib.rs b/crates/stages/src/lib.rs index 59e6c075e86..21dd5b57c7b 100644 --- a/crates/stages/src/lib.rs +++ b/crates/stages/src/lib.rs @@ -77,6 +77,7 @@ pub use error::*; pub use id::*; pub use pipeline::*; pub use stage::*; +pub use trie::DBTrieLoader; // NOTE: Needed so the link in the module-level rustdoc works. #[allow(unused_extern_crates)] diff --git a/crates/stages/src/stages/hashing_storage.rs b/crates/stages/src/stages/hashing_storage.rs index 2427d90a50f..e7b5d3c49a3 100644 --- a/crates/stages/src/stages/hashing_storage.rs +++ b/crates/stages/src/stages/hashing_storage.rs @@ -160,33 +160,40 @@ impl Stage for StorageHashingStage { // Assumption we are okay with is that plain state represent // `previous_stage_progress` state. .map(|(address, storage)| { - storage - .into_iter() - .map(|key| { - plain_storage - .seek_by_key_subkey(address, key) - .map(|ret| (keccak256(key), ret.map(|e| e.value))) - }) - .collect::, _>>() - .map(|storage| (keccak256(address), storage)) + let res = ( + keccak256(address), + storage + .into_iter() + .map(|key| { + Ok::, reth_db::Error>( + plain_storage + .seek_by_key_subkey(address, key)? + .filter(|v| v.key == key) + .map(|ret| (keccak256(key), ret.value)), + ) + }) + .collect::>, _>>()? + .into_iter() + .flatten() + .collect::>(), + ); + Ok::<_, reth_db::Error>(res) }) .collect::, _>>()? .into_iter() // Hash the address and key and apply them to HashedStorage (if Storage is None // just remove it); - .try_for_each(|(address, storage)| { + .try_for_each(|(hashed_address, storage)| { storage.into_iter().try_for_each(|(key, val)| -> Result<(), StageError> { if hashed_storage - .seek_by_key_subkey(address, key)? + .seek_by_key_subkey(hashed_address, key)? .filter(|entry| entry.key == key) .is_some() { hashed_storage.delete_current()?; } - if let Some(value) = val { - hashed_storage.upsert(address, StorageEntry { key, value })?; - } + hashed_storage.upsert(hashed_address, StorageEntry { key, value: val })?; Ok(()) }) })?; @@ -232,9 +239,9 @@ impl Stage for StorageHashingStage { .collect::>() .into_iter() // Apply values to HashedStorage (if Value is zero just remove it); - .try_for_each(|((address, key), value)| -> Result<(), StageError> { + .try_for_each(|((hashed_address, key), value)| -> Result<(), StageError> { if hashed_storage - .seek_by_key_subkey(address, key)? + .seek_by_key_subkey(hashed_address, key)? .filter(|entry| entry.key == key) .is_some() { @@ -242,7 +249,7 @@ impl Stage for StorageHashingStage { } if value != U256::ZERO { - hashed_storage.upsert(address, StorageEntry { key, value })?; + hashed_storage.upsert(hashed_address, StorageEntry { key, value })?; } Ok(()) })?; diff --git a/crates/stages/src/stages/merkle.rs b/crates/stages/src/stages/merkle.rs index d86470bff89..884c137a86d 100644 --- a/crates/stages/src/stages/merkle.rs +++ b/crates/stages/src/stages/merkle.rs @@ -14,6 +14,9 @@ pub const MERKLE_EXECUTION: StageId = StageId("MerkleExecute"); /// The [`StageId`] of the merkle hashing unwind stage. pub const MERKLE_UNWIND: StageId = StageId("MerkleUnwind"); +/// The [`StageId`] of the merkle hashing unwind and execution stage. +pub const MERKLE_BOTH: StageId = StageId("MerkleBoth"); + /// The merkle hashing stage uses input from /// [`AccountHashingStage`][crate::stages::AccountHashingStage] and /// [`StorageHashingStage`][crate::stages::AccountHashingStage] to calculate intermediate hashes @@ -35,7 +38,7 @@ pub const MERKLE_UNWIND: StageId = StageId("MerkleUnwind"); /// - [`AccountHashingStage`][crate::stages::AccountHashingStage] /// - [`StorageHashingStage`][crate::stages::StorageHashingStage] /// - [`MerkleStage::Execution`] -#[derive(Debug)] +#[derive(Debug, Clone)] pub enum MerkleStage { /// The execution portion of the merkle stage. Execution { @@ -46,7 +49,9 @@ pub enum MerkleStage { /// The unwind portion of the merkle stage. Unwind, - #[cfg(test)] + /// Able to execute and unwind. Used for tests + #[cfg(any(test, feature = "test-utils"))] + #[allow(missing_docs)] Both { clean_threshold: u64 }, } @@ -69,8 +74,8 @@ impl Stage for MerkleStage { match self { MerkleStage::Execution { .. } => MERKLE_EXECUTION, MerkleStage::Unwind => MERKLE_UNWIND, - #[cfg(test)] - MerkleStage::Both { .. } => unreachable!(), + #[cfg(any(test, feature = "test-utils"))] + MerkleStage::Both { .. } => MERKLE_BOTH, } } @@ -89,7 +94,7 @@ impl Stage for MerkleStage { }) } MerkleStage::Execution { clean_threshold } => *clean_threshold, - #[cfg(test)] + #[cfg(any(test, feature = "test-utils"))] MerkleStage::Both { clean_threshold } => *clean_threshold, }; @@ -156,10 +161,22 @@ impl Stage for MerkleStage { let from_transition = tx.get_block_transition(input.unwind_to)?; let to_transition = tx.get_block_transition(input.stage_progress)?; - loader + let block_root = loader .update_root(tx, current_root, from_transition..to_transition) .map_err(|e| StageError::Fatal(Box::new(e)))?; + if block_root != target_root { + let unwind_to = input.unwind_to; + warn!(target: "sync::stages::merkle::unwind", ?unwind_to, got = ?block_root, expected = ?target_root, "Block's root state failed verification"); + return Err(StageError::Validation { + block: unwind_to, + error: consensus::Error::BodyStateRootDiff { + got: block_root, + expected: target_root, + }, + }) + } + info!(target: "sync::stages::merkle::unwind", "Stage finished"); Ok(UnwindOutput { stage_progress: input.unwind_to }) } @@ -175,12 +192,11 @@ mod tests { use assert_matches::assert_matches; use reth_db::{ cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO, DbDupCursorRW}, - models::{AccountBeforeTx, StoredBlockBody}, tables, transaction::{DbTx, DbTxMut}, }; use reth_interfaces::test_utils::generators::{ - random_block, random_block_range, random_contract_account_range, + random_block, random_block_range, random_contract_account_range, random_transition_range, }; use reth_primitives::{keccak256, Account, Address, SealedBlock, StorageEntry, H256, U256}; use std::collections::BTreeMap; @@ -276,12 +292,16 @@ mod tests { let end = input.previous_stage_progress() + 1; let n_accounts = 31; - let mut accounts = random_contract_account_range(&mut (0..n_accounts)); + let accounts = random_contract_account_range(&mut (0..n_accounts)) + .into_iter() + .collect::>(); let SealedBlock { header, body, ommers, withdrawals } = random_block(stage_progress, None, Some(0), None); let mut header = header.unseal(); - header.state_root = self.generate_initial_trie(&accounts)?; + + header.state_root = + self.generate_initial_trie(accounts.iter().map(|(k, v)| (*k, *v)))?; let sealed_head = SealedBlock { header: header.seal_slow(), body, ommers, withdrawals }; let head_hash = sealed_head.hash(); @@ -289,64 +309,18 @@ mod tests { blocks.extend(random_block_range((stage_progress + 1)..end, head_hash, 0..3)); - self.tx.insert_headers(blocks.iter().map(|block| &block.header))?; - - let (mut transition_id, mut tx_id) = (0, 0); - - let mut storages: BTreeMap> = BTreeMap::new(); - - for progress in blocks.iter() { - // Insert last progress data - self.tx.commit(|tx| { - let body = StoredBlockBody { - start_tx_id: tx_id, - tx_count: progress.body.len() as u64, - }; - - progress.body.iter().try_for_each(|transaction| { - tx.put::(transaction.hash(), tx_id)?; - tx.put::(tx_id, transaction.clone())?; - tx.put::(tx_id, transition_id)?; - - // seed account changeset - let (addr, prev_acc) = accounts - .get_mut(rand::random::() % n_accounts as usize) - .unwrap(); - let acc_before_tx = - AccountBeforeTx { address: *addr, info: Some(*prev_acc) }; - - tx.put::(transition_id, acc_before_tx)?; - - prev_acc.nonce += 1; - prev_acc.balance = prev_acc.balance.wrapping_add(U256::from(1)); - - let new_entry = StorageEntry { - key: keccak256([rand::random::()]), - value: U256::from(rand::random::() % 30 + 1), - }; - let storage = storages.entry(*addr).or_default(); - let old_value = storage.entry(new_entry.key).or_default(); - - tx.put::( - (transition_id, *addr).into(), - StorageEntry { key: new_entry.key, value: *old_value }, - )?; - - *old_value = new_entry.value; - - tx_id += 1; - transition_id += 1; + self.tx.insert_blocks(blocks.iter(), None)?; - Ok(()) - })?; + let (transitions, final_state) = random_transition_range( + blocks.iter(), + accounts.into_iter().map(|(addr, acc)| (addr, (acc, Vec::new()))), + 0..3, + 0..256, + ); - tx.put::(progress.number, transition_id)?; - tx.put::(progress.number, body) - })?; - } + self.tx.insert_transitions(transitions, None)?; - self.insert_accounts(&accounts)?; - self.insert_storages(&storages)?; + self.tx.insert_accounts_and_storages(final_state)?; let last_block_number = end - 1; let root = self.state_root()?; @@ -471,9 +445,11 @@ mod tests { pub(crate) fn generate_initial_trie( &self, - accounts: &[(Address, Account)], + accounts: impl IntoIterator, ) -> Result { - self.insert_accounts(accounts)?; + self.tx.insert_accounts_and_storages( + accounts.into_iter().map(|(addr, acc)| (addr, (acc, std::iter::empty()))), + )?; let loader = DBTrieLoader::default(); @@ -485,57 +461,6 @@ mod tests { Ok(root) } - pub(crate) fn insert_accounts( - &self, - accounts: &[(Address, Account)], - ) -> Result<(), TestRunnerError> { - for (addr, acc) in accounts.iter() { - self.tx.commit(|tx| { - tx.put::(*addr, *acc)?; - tx.put::(keccak256(addr), *acc)?; - Ok(()) - })?; - } - - Ok(()) - } - - fn insert_storages( - &self, - storages: &BTreeMap>, - ) -> Result<(), TestRunnerError> { - self.tx - .commit(|tx| { - storages.iter().try_for_each(|(&addr, storage)| { - storage.iter().try_for_each(|(&key, &value)| { - let entry = StorageEntry { key, value }; - tx.put::(addr, entry) - }) - })?; - storages - .iter() - .map(|(addr, storage)| { - ( - keccak256(addr), - storage - .iter() - .filter(|(_, &value)| value != U256::ZERO) - .map(|(key, value)| (keccak256(key), value)), - ) - }) - .collect::>() - .into_iter() - .try_for_each(|(addr, storage)| { - storage.into_iter().try_for_each(|(key, &value)| { - let entry = StorageEntry { key, value }; - tx.put::(addr, entry) - }) - })?; - Ok(()) - }) - .map_err(|e| e.into()) - } - fn check_root(&self, previous_stage_progress: u64) -> Result<(), TestRunnerError> { if previous_stage_progress != 0 { let block_root = diff --git a/crates/stages/src/test_utils/test_db.rs b/crates/stages/src/test_utils/test_db.rs index a040c651386..bf1a0043f28 100644 --- a/crates/stages/src/test_utils/test_db.rs +++ b/crates/stages/src/test_utils/test_db.rs @@ -1,19 +1,26 @@ use reth_db::{ - cursor::{DbCursorRO, DbCursorRW}, + cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO}, mdbx::{ test_utils::{create_test_db, create_test_db_with_path}, tx::Tx, Env, EnvKind, WriteMap, RW, }, - models::StoredBlockBody, + models::{AccountBeforeTx, BlockNumHash, StoredBlockBody}, table::Table, tables, transaction::{DbTx, DbTxMut}, Error as DbError, }; -use reth_primitives::{BlockNumber, SealedBlock, SealedHeader, U256}; +use reth_primitives::{ + keccak256, Account, Address, BlockNumber, SealedBlock, SealedHeader, StorageEntry, H256, U256, +}; use reth_provider::Transaction; -use std::{borrow::Borrow, path::Path, sync::Arc}; +use std::{ + borrow::Borrow, + collections::BTreeMap, + path::{Path, PathBuf}, + sync::Arc, +}; /// The [TestTransaction] is used as an internal /// database for testing stage implementation. @@ -26,18 +33,22 @@ use std::{borrow::Borrow, path::Path, sync::Arc}; pub struct TestTransaction { /// WriteMap DB pub tx: Arc>, + pub path: Option, } impl Default for TestTransaction { /// Create a new instance of [TestTransaction] fn default() -> Self { - Self { tx: create_test_db::(EnvKind::RW) } + Self { tx: create_test_db::(EnvKind::RW), path: None } } } impl TestTransaction { pub fn new(path: &Path) -> Self { - Self { tx: Arc::new(create_test_db_with_path::(EnvKind::RW, path)) } + Self { + tx: Arc::new(create_test_db_with_path::(EnvKind::RW, path)), + path: Some(path.to_path_buf()), + } } /// Return a database wrapped in [Transaction]. @@ -177,23 +188,20 @@ impl TestTransaction { }) } + /// Inserts a single [SealedHeader] into the corresponding tables of the headers stage. + fn insert_header(tx: &mut Tx<'_, RW, WriteMap>, header: &SealedHeader) -> Result<(), DbError> { + tx.put::(header.number, header.hash())?; + tx.put::(header.hash(), header.number)?; + tx.put::(header.number, header.clone().unseal()) + } + /// Insert ordered collection of [SealedHeader] into the corresponding tables /// that are supposed to be populated by the headers stage. pub fn insert_headers<'a, I>(&self, headers: I) -> Result<(), DbError> where I: Iterator, { - self.commit(|tx| { - let headers = headers.collect::>(); - - for header in headers { - tx.put::(header.number, header.hash())?; - tx.put::(header.hash(), header.number)?; - tx.put::(header.number, header.clone().unseal())?; - } - - Ok(()) - }) + self.commit(|tx| headers.into_iter().try_for_each(|header| Self::insert_header(tx, header))) } /// Inserts total difficulty of headers into the corresponding tables. @@ -204,23 +212,19 @@ impl TestTransaction { I: Iterator, { self.commit(|tx| { - let headers = headers.collect::>(); - let mut td = U256::ZERO; - for header in headers { + headers.into_iter().try_for_each(|header| { + Self::insert_header(tx, header)?; td += header.difficulty; - tx.put::(header.number, td.into())?; - tx.put::(header.number, header.hash())?; - tx.put::(header.hash(), header.number)?; - tx.put::(header.number, header.clone().unseal())?; - } - - Ok(()) + tx.put::(header.number, td.into()) + }) }) } /// Insert ordered collection of [SealedBlock] into corresponding tables. /// Superset functionality of [TestTransaction::insert_headers]. + /// + /// Assumes that there's a single transition for each transaction (i.e. no block rewards). pub fn insert_blocks<'a, I>(&self, blocks: I, tx_offset: Option) -> Result<(), DbError> where I: Iterator, @@ -228,12 +232,8 @@ impl TestTransaction { self.commit(|tx| { let mut current_tx_id = tx_offset.unwrap_or_default(); - for block in blocks { - // Insert into header tables. - tx.put::(block.number, block.hash())?; - tx.put::(block.hash(), block.number)?; - tx.put::(block.number, block.header.clone().unseal())?; - + blocks.into_iter().try_for_each(|block| { + Self::insert_header(tx, &block.header)?; // Insert into body tables. tx.put::( block.number, @@ -242,13 +242,88 @@ impl TestTransaction { tx_count: block.body.len() as u64, }, )?; - for body_tx in block.body.clone() { - tx.put::(current_tx_id, body_tx)?; + block.body.iter().try_for_each(|body_tx| { + tx.put::(current_tx_id, current_tx_id)?; + tx.put::(current_tx_id, body_tx.clone())?; current_tx_id += 1; - } - } + Ok(()) + })?; + tx.put::(block.number, current_tx_id) + }) + }) + } - Ok(()) + /// Insert collection of ([Address], [Account]) into corresponding tables. + pub fn insert_accounts_and_storages(&self, accounts: I) -> Result<(), DbError> + where + I: IntoIterator, + S: IntoIterator, + { + self.commit(|tx| { + accounts.into_iter().try_for_each(|(address, (account, storage))| { + let hashed_address = keccak256(address); + + // Insert into account tables. + tx.put::(address, account)?; + tx.put::(hashed_address, account)?; + + // Insert into storage tables. + storage.into_iter().filter(|e| e.value != U256::ZERO).try_for_each(|entry| { + let hashed_entry = StorageEntry { key: keccak256(entry.key), ..entry }; + + let mut cursor = tx.cursor_dup_write::()?; + if let Some(e) = cursor + .seek_by_key_subkey(address, entry.key)? + .filter(|e| e.key == entry.key) + { + cursor.delete_current()?; + } + cursor.upsert(address, entry)?; + + let mut cursor = tx.cursor_dup_write::()?; + if let Some(e) = cursor + .seek_by_key_subkey(hashed_address, hashed_entry.key)? + .filter(|e| e.key == hashed_entry.key) + { + cursor.delete_current()?; + } + cursor.upsert(hashed_address, hashed_entry)?; + + Ok(()) + }) + }) + }) + } + + /// Insert collection of Vec<([Address], [Account], Vec<[StorageEntry]>)> into + /// corresponding tables. + pub fn insert_transitions( + &self, + transitions: I, + transition_offset: Option, + ) -> Result<(), DbError> + where + I: IntoIterator)>>, + { + let offset = transition_offset.unwrap_or_default(); + self.commit(|tx| { + transitions.into_iter().enumerate().try_for_each(|(transition_id, changes)| { + changes.into_iter().try_for_each(|(address, old_account, old_storage)| { + let tid = offset + transition_id as u64; + // Insert into account changeset. + tx.put::( + tid, + AccountBeforeTx { address, info: Some(old_account) }, + )?; + + let tid_address = (tid, address).into(); + + // Insert into storage changeset. + old_storage.into_iter().try_for_each(|entry| { + tx.put::(tid_address, entry) + }) + }) + }) }) } } diff --git a/crates/stages/src/trie/mod.rs b/crates/stages/src/trie/mod.rs index d5f4cb94b80..199df9f8417 100644 --- a/crates/stages/src/trie/mod.rs +++ b/crates/stages/src/trie/mod.rs @@ -24,7 +24,7 @@ use std::{ use tracing::*; #[derive(Debug, thiserror::Error)] -pub(crate) enum TrieError { +pub enum TrieError { #[error("Some error occurred: {0}")] InternalError(#[from] cita_trie::TrieError), #[error("The root node wasn't found in the DB")] @@ -54,17 +54,33 @@ where Ok(::get(self, key)?.is_some()) } - fn insert(&self, key: Vec, value: Vec) -> Result<(), Self::Error> { - // Caching and bulk inserting shouldn't be needed, as the data is ordered - self.tx.put::(H256::from_slice(key.as_slice()), value)?; + fn insert(&self, _key: Vec, _value: Vec) -> Result<(), Self::Error> { + unreachable!("Use batch instead."); + } + + // Insert a batch of data into the cache. + fn insert_batch(&self, keys: Vec>, values: Vec>) -> Result<(), Self::Error> { + let mut cursor = self.tx.cursor_write::()?; + for (key, value) in keys.into_iter().zip(values.into_iter()) { + cursor.upsert(H256::from_slice(key.as_slice()), value)?; + } Ok(()) } - fn remove(&self, key: &[u8]) -> Result<(), Self::Error> { - self.tx.delete::(H256::from_slice(key), None)?; + fn remove_batch(&self, keys: &[Vec]) -> Result<(), Self::Error> { + let mut cursor = self.tx.cursor_write::()?; + for key in keys { + if cursor.seek_exact(H256::from_slice(key.as_slice()))?.is_some() { + cursor.delete_current()?; + } + } Ok(()) } + fn remove(&self, _key: &[u8]) -> Result<(), Self::Error> { + unreachable!("Use batch instead."); + } + fn flush(&self) -> Result<(), Self::Error> { Ok(()) } @@ -104,31 +120,49 @@ where fn get(&self, key: &[u8]) -> Result>, Self::Error> { let mut cursor = self.tx.cursor_dup_read::()?; - Ok(cursor.seek_by_key_subkey(self.key, H256::from_slice(key))?.map(|entry| entry.node)) + let subkey = H256::from_slice(key); + Ok(cursor + .seek_by_key_subkey(self.key, subkey)? + .filter(|entry| entry.hash == subkey) + .map(|entry| entry.node)) } fn contains(&self, key: &[u8]) -> Result { Ok(::get(self, key)?.is_some()) } - fn insert(&self, key: Vec, value: Vec) -> Result<(), Self::Error> { - // Caching and bulk inserting shouldn't be needed, as the data is ordered - self.tx.put::( - self.key, - StorageTrieEntry { hash: H256::from_slice(key.as_slice()), node: value }, - )?; + fn insert(&self, _key: Vec, _value: Vec) -> Result<(), Self::Error> { + unreachable!("Use batch instead."); + } + + /// Insert a batch of data into the cache. + fn insert_batch(&self, keys: Vec>, values: Vec>) -> Result<(), Self::Error> { + let mut cursor = self.tx.cursor_dup_write::()?; + for (key, node) in keys.into_iter().zip(values.into_iter()) { + let hash = H256::from_slice(key.as_slice()); + if cursor.seek_by_key_subkey(self.key, hash)?.filter(|e| e.hash == hash).is_some() { + cursor.delete_current()?; + } + cursor.upsert(self.key, StorageTrieEntry { hash, node })?; + } Ok(()) } - fn remove(&self, key: &[u8]) -> Result<(), Self::Error> { + fn remove_batch(&self, keys: &[Vec]) -> Result<(), Self::Error> { let mut cursor = self.tx.cursor_dup_write::()?; - cursor - .seek_by_key_subkey(self.key, H256::from_slice(key))? - .map(|_| cursor.delete_current()) - .transpose()?; + for key in keys { + let hash = H256::from_slice(key.as_slice()); + if cursor.seek_by_key_subkey(self.key, hash)?.filter(|e| e.hash == hash).is_some() { + cursor.delete_current()?; + } + } Ok(()) } + fn remove(&self, _key: &[u8]) -> Result<(), Self::Error> { + unreachable!("Use batch instead."); + } + fn flush(&self) -> Result<(), Self::Error> { Ok(()) } @@ -139,7 +173,7 @@ impl<'tx, 'itx, DB: Database> DupHashDatabase<'tx, 'itx, DB> { fn new(tx: &'tx Transaction<'itx, DB>, key: H256) -> Result { let root = EMPTY_ROOT; let mut cursor = tx.cursor_dup_write::()?; - if cursor.seek_by_key_subkey(key, root)?.is_none() { + if cursor.seek_by_key_subkey(key, root)?.filter(|entry| entry.hash == root).is_none() { tx.put::( key, StorageTrieEntry { hash: root, node: [EMPTY_STRING_CODE].to_vec() }, @@ -155,6 +189,7 @@ impl<'tx, 'itx, DB: Database> DupHashDatabase<'tx, 'itx, DB> { } tx.cursor_dup_read::()? .seek_by_key_subkey(key, root)? + .filter(|entry| entry.hash == root) .ok_or(TrieError::MissingRoot(root))?; Ok(Self { tx, key }) } @@ -190,12 +225,14 @@ impl EthAccount { } } +/// Struct for calculating the root of a merkle patricia tree, +/// while populating the database with intermediate hashes. #[derive(Debug, Default)] -pub(crate) struct DBTrieLoader; +pub struct DBTrieLoader; impl DBTrieLoader { /// Calculates the root of the state trie, saving intermediate hashes in the database. - pub(crate) fn calculate_root( + pub fn calculate_root( &self, tx: &Transaction<'_, DB>, ) -> Result { @@ -255,7 +292,7 @@ impl DBTrieLoader { } /// Calculates the root of the state trie by updating an existing trie. - pub(crate) fn update_root( + pub fn update_root( &self, tx: &Transaction<'_, DB>, root: H256, @@ -272,20 +309,21 @@ impl DBTrieLoader { let mut trie = PatriciaTrie::from(Arc::clone(&db), Arc::clone(&hasher), root.as_bytes())?; for (address, changed_storages) in changed_accounts { - if let Some(account) = trie.get(address.as_slice())? { - let storage_root = EthAccount::decode(&mut account.as_slice())?.storage_root; + let storage_root = if let Some(account) = trie.get(address.as_slice())? { trie.remove(address.as_bytes())?; - if let Some((_, account)) = accounts_cursor.seek_exact(address)? { - let value = EthAccount::from_with_root( - account, - self.update_storage_root(tx, storage_root, address, changed_storages)?, - ); + let storage_root = EthAccount::decode(&mut account.as_slice())?.storage_root; + self.update_storage_root(tx, storage_root, address, changed_storages)? + } else { + self.calculate_storage_root(tx, address)? + }; - let mut out = Vec::new(); - Encodable::encode(&value, &mut out); - trie.insert(address.as_bytes().to_vec(), out)?; - } + if let Some((_, account)) = accounts_cursor.seek_exact(address)? { + let value = EthAccount::from_with_root(account, storage_root); + + let mut out = Vec::new(); + Encodable::encode(&value, &mut out); + trie.insert(address.as_bytes().to_vec(), out)?; } } @@ -310,7 +348,7 @@ impl DBTrieLoader { for key in changed_storages { if let Some(StorageEntry { value, .. }) = - storage_cursor.seek_by_key_subkey(address, key)? + storage_cursor.seek_by_key_subkey(address, key)?.filter(|e| e.key == key) { let out = encode_fixed_size(&value).to_vec(); trie.insert(key.as_bytes().to_vec(), out)?; diff --git a/crates/storage/db/benches/utils.rs b/crates/storage/db/benches/utils.rs index 08d88a9ba09..29561dfe629 100644 --- a/crates/storage/db/benches/utils.rs +++ b/crates/storage/db/benches/utils.rs @@ -1,3 +1,4 @@ +#[allow(unused_imports)] use reth_db::{ database::Database, mdbx::{test_utils::create_test_db_with_path, EnvKind, WriteMap}, @@ -7,12 +8,15 @@ use reth_db::{ use std::path::Path; /// Path where the DB is initialized for benchmarks. +#[allow(unused)] const BENCH_DB_PATH: &str = "/tmp/reth-benches"; /// Used for RandomRead and RandomWrite benchmarks. +#[allow(unused)] const RANDOM_INDEXES: [usize; 10] = [23, 2, 42, 5, 3, 99, 54, 0, 33, 64]; /// Returns bench vectors in the format: `Vec<(Key, EncodedKey, Value, CompressedValue)>`. +#[allow(unused)] fn load_vectors() -> Vec<(T::Key, bytes::Bytes, T::Value, bytes::Bytes)> where T: Default, From 14811dae4063f32264acd67c6e21fe5fc486955d Mon Sep 17 00:00:00 2001 From: grantkee <49913008+grantkee@users.noreply.github.com> Date: Wed, 1 Mar 2023 13:47:24 -0600 Subject: [PATCH 018/191] Implement Decodable for TransactionSignedEcRecovered (#1591) --- crates/primitives/src/transaction/mod.rs | 27 +++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/crates/primitives/src/transaction/mod.rs b/crates/primitives/src/transaction/mod.rs index 4561bf2d695..70ad0ef3496 100644 --- a/crates/primitives/src/transaction/mod.rs +++ b/crates/primitives/src/transaction/mod.rs @@ -899,6 +899,16 @@ impl Encodable for TransactionSignedEcRecovered { } } +impl Decodable for TransactionSignedEcRecovered { + fn decode(buf: &mut &[u8]) -> Result { + let signed_transaction = TransactionSigned::decode(buf)?; + let signer = signed_transaction + .recover_signer() + .ok_or(DecodeError::Custom("Unable to recover decoded transaction signer."))?; + Ok(TransactionSignedEcRecovered { signer, signed_transaction }) + } +} + /// The inverse of [`FromRecoveredTransaction`] that ensure the transaction can be sent over the /// network pub trait IntoRecoveredTransaction { @@ -919,7 +929,8 @@ impl IntoRecoveredTransaction for TransactionSignedEcRecovered { mod tests { use crate::{ transaction::{signature::Signature, TransactionKind, TxEip1559, TxEip2930, TxLegacy}, - AccessList, Address, Bytes, Transaction, TransactionSigned, H256, U256, + AccessList, Address, Bytes, Transaction, TransactionSigned, TransactionSignedEcRecovered, + H256, U256, }; use bytes::BytesMut; use ethers_core::utils::hex; @@ -1244,4 +1255,18 @@ mod tests { let encoded = decoded.envelope_encoded(); assert_eq!(encoded, input); } + + #[test] + fn test_decode_signed_ec_recovered_transaction() { + // random tx: + let input = hex::decode("02f871018302a90f808504890aef60826b6c94ddf4c5025d1a5742cf12f74eec246d4432c295e487e09c3bbcc12b2b80c080a0f21a4eacd0bf8fea9c5105c543be5a1d8c796516875710fafafdf16d16d8ee23a001280915021bb446d1973501a67f93d2b38894a514b976e7b46dc2fe54598d76").unwrap(); + let tx = TransactionSigned::decode(&mut &input[..]).unwrap(); + let recovered = tx.into_ecrecovered().unwrap(); + + let mut encoded = BytesMut::new(); + recovered.encode(&mut encoded); + + let decoded = TransactionSignedEcRecovered::decode(&mut &encoded[..]).unwrap(); + assert_eq!(recovered, decoded) + } } From 6136e0deb446b7e4030d332a985c36ec9b586b72 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Wed, 1 Mar 2023 15:13:37 -0500 Subject: [PATCH 019/191] chore: bump ethers (#1598) Co-authored-by: Matthias Seitz --- Cargo.lock | 88 +++++++++++++++++++++++++++++++++++++++++++++++++----- deny.toml | 2 ++ 2 files changed, 82 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d2046ac0615..8668d9c0ad1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,16 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "Inflector" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" +dependencies = [ + "lazy_static", + "regex", +] + [[package]] name = "addr2line" version = "0.19.0" @@ -605,9 +615,9 @@ dependencies = [ [[package]] name = "cargo_metadata" -version = "0.15.2" +version = "0.15.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "982a0cf6a99c350d7246035613882e376d58cebe571785abc5da4f648d53ac0a" +checksum = "08a1ec454bc3eead8719cb56e15dbbfecdbc14e4b3a3ae4936cc6e31f5fc0d07" dependencies = [ "camino", "cargo-platform", @@ -1481,6 +1491,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "dunce" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bd4b30a6560bbd9b4620f4de34c3f14f60848e58a9b7216801afcb4c7b31c3c" + [[package]] name = "dyn-clone" version = "1.0.10" @@ -1758,8 +1774,10 @@ dependencies = [ [[package]] name = "ethers-contract" version = "1.0.2" -source = "git+https://github.com/gakonst/ethers-rs#537d0a9deb8b1fc549bf21b48684e9155f12707b" +source = "git+https://github.com/gakonst/ethers-rs#73636a906e40810beb3b55b305f5e81640478a01" dependencies = [ + "ethers-contract-abigen", + "ethers-contract-derive", "ethers-core", "ethers-providers", "futures-util", @@ -1771,13 +1789,56 @@ dependencies = [ "thiserror", ] +[[package]] +name = "ethers-contract-abigen" +version = "1.0.2" +source = "git+https://github.com/gakonst/ethers-rs#73636a906e40810beb3b55b305f5e81640478a01" +dependencies = [ + "Inflector", + "cfg-if", + "dunce", + "ethers-core", + "ethers-etherscan", + "eyre", + "getrandom 0.2.8", + "hex", + "prettyplease", + "proc-macro2 1.0.51", + "quote 1.0.23", + "regex", + "reqwest", + "serde", + "serde_json", + "syn 1.0.109", + "tokio", + "toml", + "url", + "walkdir", +] + +[[package]] +name = "ethers-contract-derive" +version = "1.0.2" +source = "git+https://github.com/gakonst/ethers-rs#73636a906e40810beb3b55b305f5e81640478a01" +dependencies = [ + "ethers-contract-abigen", + "ethers-core", + "eyre", + "hex", + "proc-macro2 1.0.51", + "quote 1.0.23", + "serde_json", + "syn 1.0.109", +] + [[package]] name = "ethers-core" version = "1.0.2" -source = "git+https://github.com/gakonst/ethers-rs#537d0a9deb8b1fc549bf21b48684e9155f12707b" +source = "git+https://github.com/gakonst/ethers-rs#73636a906e40810beb3b55b305f5e81640478a01" dependencies = [ "arrayvec", "bytes", + "cargo_metadata", "chrono", "convert_case 0.6.0", "elliptic-curve", @@ -1787,6 +1848,7 @@ dependencies = [ "hex", "k256", "num_enum", + "once_cell", "open-fastrlp", "proc-macro2 1.0.51", "rand 0.8.5", @@ -1805,7 +1867,7 @@ dependencies = [ [[package]] name = "ethers-etherscan" version = "1.0.2" -source = "git+https://github.com/gakonst/ethers-rs#537d0a9deb8b1fc549bf21b48684e9155f12707b" +source = "git+https://github.com/gakonst/ethers-rs#73636a906e40810beb3b55b305f5e81640478a01" dependencies = [ "ethers-core", "getrandom 0.2.8", @@ -1821,7 +1883,7 @@ dependencies = [ [[package]] name = "ethers-middleware" version = "1.0.2" -source = "git+https://github.com/gakonst/ethers-rs#537d0a9deb8b1fc549bf21b48684e9155f12707b" +source = "git+https://github.com/gakonst/ethers-rs#73636a906e40810beb3b55b305f5e81640478a01" dependencies = [ "async-trait", "auto_impl", @@ -1846,7 +1908,7 @@ dependencies = [ [[package]] name = "ethers-providers" version = "1.0.2" -source = "git+https://github.com/gakonst/ethers-rs#537d0a9deb8b1fc549bf21b48684e9155f12707b" +source = "git+https://github.com/gakonst/ethers-rs#73636a906e40810beb3b55b305f5e81640478a01" dependencies = [ "async-trait", "auto_impl", @@ -1883,7 +1945,7 @@ dependencies = [ [[package]] name = "ethers-signers" version = "1.0.2" -source = "git+https://github.com/gakonst/ethers-rs#537d0a9deb8b1fc549bf21b48684e9155f12707b" +source = "git+https://github.com/gakonst/ethers-rs#73636a906e40810beb3b55b305f5e81640478a01" dependencies = [ "async-trait", "coins-bip32", @@ -3920,6 +3982,16 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +[[package]] +name = "prettyplease" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e97e3215779627f01ee256d2fad52f3d95e8e1c11e9fc6fd08f7cd455d5d5c78" +dependencies = [ + "proc-macro2 1.0.51", + "syn 1.0.109", +] + [[package]] name = "primitive-types" version = "0.12.1" diff --git a/deny.toml b/deny.toml index 7c1d703f3f8..6fe43eb8b35 100644 --- a/deny.toml +++ b/deny.toml @@ -57,6 +57,8 @@ exceptions = [ { allow = ["CC0-1.0"], name = "secp256k1-sys" }, { allow = ["CC0-1.0"], name = "tiny-keccak" }, { allow = ["CC0-1.0"], name = "more-asserts" }, + # MIT with no attribution https://choosealicense.com/licenses/mit-0/ + { allow = ["MIT-0"], name = "dunce" }, # TODO: ethers transitive deps { allow = ["GPL-3.0"], name = "fastrlp" }, From 42e3f561089e1269ae560bef9dee0e731c19d05c Mon Sep 17 00:00:00 2001 From: rakita Date: Wed, 1 Mar 2023 21:42:45 +0100 Subject: [PATCH 020/191] feat: refactor few stages to providers, introduce `insert_block` (#1474) Co-authored-by: Georgios Konstantopoulos --- Cargo.lock | 12 +- crates/executor/src/lib.rs | 4 +- crates/primitives/Cargo.toml | 18 +- crates/primitives/src/chain/spec.rs | 3 +- crates/stages/Cargo.toml | 4 - crates/stages/benches/setup/mod.rs | 3 +- crates/stages/src/lib.rs | 2 - crates/stages/src/stages/execution.rs | 121 +--- crates/stages/src/stages/hashing_account.rs | 43 +- crates/stages/src/stages/hashing_storage.rs | 74 +-- .../src/stages/index_account_history.rs | 72 +-- .../src/stages/index_storage_history.rs | 80 +-- crates/stages/src/stages/merkle.rs | 7 +- crates/storage/provider/Cargo.toml | 15 +- .../provider}/src/execution_result.rs | 4 +- crates/storage/provider/src/lib.rs | 6 + crates/storage/provider/src/transaction.rs | 544 +++++++++++++++++- .../provider}/src/trie/mod.rs | 11 +- crates/storage/provider/src/utils.rs | 42 +- 19 files changed, 659 insertions(+), 406 deletions(-) rename crates/{executor => storage/provider}/src/execution_result.rs (99%) rename crates/{stages => storage/provider}/src/trie/mod.rs (99%) diff --git a/Cargo.lock b/Cargo.lock index 8668d9c0ad1..f42fba15ca7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4858,6 +4858,7 @@ dependencies = [ "crunchy", "derive_more", "ethers-core", + "eyre", "fixed-hash", "hash-db", "hex", @@ -4879,6 +4880,7 @@ dependencies = [ "serde", "serde_json", "serde_with", + "shellexpand", "strum", "sucds", "test-fuzz", @@ -4892,14 +4894,22 @@ dependencies = [ name = "reth-provider" version = "0.1.0" dependencies = [ + "assert_matches", "auto_impl", + "cita_trie", + "hasher", + "itertools 0.10.5", "parking_lot 0.12.1", + "proptest", "reth-db", "reth-interfaces", "reth-primitives", "reth-revm-primitives", + "reth-rlp", + "reth-tracing", "revm-primitives", "thiserror", + "triehash", ] [[package]] @@ -5116,11 +5126,9 @@ dependencies = [ "arbitrary", "assert_matches", "async-trait", - "cita_trie", "criterion", "eyre", "futures-util", - "hasher", "itertools 0.10.5", "metrics", "num-traits", diff --git a/crates/executor/src/lib.rs b/crates/executor/src/lib.rs index 523fc7a0f0d..6d28d96482c 100644 --- a/crates/executor/src/lib.rs +++ b/crates/executor/src/lib.rs @@ -9,7 +9,7 @@ pub mod eth_dao_fork; -/// Execution result types -pub mod execution_result; +/// Execution result types. +pub use reth_provider::execution_result; /// Executor pub mod executor; diff --git a/crates/primitives/Cargo.toml b/crates/primitives/Cargo.toml index df03f8b5e3f..91bd5718fef 100644 --- a/crates/primitives/Cargo.toml +++ b/crates/primitives/Cargo.toml @@ -9,15 +9,11 @@ description = "Commonly used types in reth." [dependencies] # reth -reth-rlp = { path = "../rlp", features = [ - "std", - "derive", - "ethereum-types", -] } +reth-rlp = { path = "../rlp", features = ["std", "derive", "ethereum-types"] } reth-rlp-derive = { path = "../rlp/rlp-derive" } reth-codecs = { version = "0.1.0", path = "../storage/codecs" } -revm-primitives = { version="1.0.0", features = ["serde"] } +revm-primitives = { version = "1.0.0", features = ["serde"] } # ethereum ethers-core = { git = "https://github.com/gakonst/ethers-rs", default-features = false } @@ -61,6 +57,10 @@ triehash = "0.8" plain_hasher = "0.2" hash-db = "0.15" +# used for clap value parser +eyre = "0.6" +shellexpand = "3.0" + # arbitrary utils arbitrary = { version = "1.1.7", features = ["derive"], optional = true } proptest = { version = "1.0", optional = true } @@ -81,7 +81,11 @@ proptest-derive = "0.3" # https://github.com/paradigmxyz/reth/pull/177#discussion_r1021172198 secp256k1 = "0.24.2" criterion = "0.4.0" -pprof = { version = "0.11", features = ["flamegraph", "frame-pointer", "criterion"] } +pprof = { version = "0.11", features = [ + "flamegraph", + "frame-pointer", + "criterion", +] } [features] default = [] diff --git a/crates/primitives/src/chain/spec.rs b/crates/primitives/src/chain/spec.rs index 06a91fc4e0d..98ca08338eb 100644 --- a/crates/primitives/src/chain/spec.rs +++ b/crates/primitives/src/chain/spec.rs @@ -585,10 +585,9 @@ impl ForkCondition { mod tests { use crate::{ AllGenesisFormats, Chain, ChainSpec, ChainSpecBuilder, ForkCondition, ForkHash, ForkId, - Genesis, Hardfork, Head, GOERLI, H256, MAINNET, SEPOLIA, + Genesis, Hardfork, Head, GOERLI, H256, MAINNET, SEPOLIA, U256, }; use ethers_core::types as EtherType; - use revm_primitives::U256; fn test_fork_ids(spec: &ChainSpec, cases: &[(Head, ForkId)]) { for (block, expected_id) in cases { let computed_id = spec.fork_id(block); diff --git a/crates/stages/Cargo.toml b/crates/stages/Cargo.toml index eb14a8e7c10..3a50de8a2ab 100644 --- a/crates/stages/Cargo.toml +++ b/crates/stages/Cargo.toml @@ -43,10 +43,6 @@ itertools = "0.10.5" rayon = "1.6.0" num-traits = "0.2.15" -# trie -cita_trie = "4.0.0" -hasher = "0.1.4" - # arbitrary utils arbitrary = { version = "1.1.7", features = ["derive"], optional = true } proptest = { version = "1.0", optional = true } diff --git a/crates/stages/benches/setup/mod.rs b/crates/stages/benches/setup/mod.rs index 34876d031ad..aaed854bd82 100644 --- a/crates/stages/benches/setup/mod.rs +++ b/crates/stages/benches/setup/mod.rs @@ -10,10 +10,11 @@ use reth_interfaces::test_utils::generators::{ random_transition_range, }; use reth_primitives::{Account, Address, SealedBlock, H256}; +use reth_provider::trie::DBTrieLoader; use reth_stages::{ stages::{AccountHashingStage, StorageHashingStage}, test_utils::TestTransaction, - DBTrieLoader, ExecInput, Stage, UnwindInput, + ExecInput, Stage, UnwindInput, }; use std::{ collections::BTreeMap, diff --git a/crates/stages/src/lib.rs b/crates/stages/src/lib.rs index 21dd5b57c7b..8e5e75ec4bf 100644 --- a/crates/stages/src/lib.rs +++ b/crates/stages/src/lib.rs @@ -53,7 +53,6 @@ mod error; mod id; mod pipeline; mod stage; -mod trie; mod util; /// The real database type we use in Reth using MDBX. @@ -77,7 +76,6 @@ pub use error::*; pub use id::*; pub use pipeline::*; pub use stage::*; -pub use trie::DBTrieLoader; // NOTE: Needed so the link in the module-level rustdoc works. #[allow(unused_extern_crates)] diff --git a/crates/stages/src/stages/execution.rs b/crates/stages/src/stages/execution.rs index f7b58eb9b81..633bfec4adf 100644 --- a/crates/stages/src/stages/execution.rs +++ b/crates/stages/src/stages/execution.rs @@ -9,9 +9,9 @@ use reth_db::{ tables, transaction::{DbTx, DbTxMut}, }; -use reth_executor::{execution_result::AccountChangeSet, executor::Executor}; +use reth_executor::executor::Executor; use reth_interfaces::provider::ProviderError; -use reth_primitives::{Address, Block, ChainSpec, Hardfork, StorageEntry, H256, U256}; +use reth_primitives::{Address, Block, ChainSpec, U256}; use reth_provider::{LatestStateProviderRef, StateProvider, Transaction}; use reth_revm::database::{State, SubState}; use tracing::*; @@ -119,7 +119,7 @@ impl<'a, S: StateProvider> ExecutionStage<'a, S> { let mut state_provider = SubState::new(State::new(LatestStateProviderRef::new(&**tx))); // Fetch transactions, execute them and generate results - let mut block_change_patches = Vec::with_capacity(block_batch.len()); + let mut changesets = Vec::with_capacity(block_batch.len()); for (header, td, body, ommers, withdrawals) in block_batch.into_iter() { let block_number = header.number; tracing::trace!(target: "sync::stages::execution", ?block_number, "Execute block."); @@ -162,117 +162,11 @@ impl<'a, S: StateProvider> ExecutionStage<'a, S> { Some(signers), ) .map_err(|error| StageError::ExecutionError { block: block_number, error })?; - block_change_patches.push((changeset, block_number)); + changesets.push(changeset); } - // Get last tx count so that we can know amount of transaction in the block. - let mut current_transition_id = tx.get_block_transition(last_block)?; - info!(target: "sync::stages::execution", current_transition_id, blocks = block_change_patches.len(), "Inserting execution results"); - - // apply changes to plain database. - for (results, block_number) in block_change_patches.into_iter() { - let spurious_dragon_active = self - .executor - .chain_spec - .fork(Hardfork::SpuriousDragon) - .active_at_block(block_number); - // insert state change set - for result in results.tx_changesets.into_iter() { - for (address, account_change_set) in result.changeset.into_iter() { - let AccountChangeSet { account, wipe_storage, storage } = account_change_set; - // apply account change to db. Updates AccountChangeSet and PlainAccountState - // tables. - trace!(target: "sync::stages::execution", ?address, current_transition_id, ?account, wipe_storage, "Applying account changeset"); - account.apply_to_db( - &**tx, - address, - current_transition_id, - spurious_dragon_active, - )?; - - let storage_id = TransitionIdAddress((current_transition_id, address)); - - // cast key to H256 and trace the change - let storage = storage - .into_iter() - .map(|(key, (old_value,new_value))| { - let hkey = H256(key.to_be_bytes()); - trace!(target: "sync::stages::execution", ?address, current_transition_id, ?hkey, ?old_value, ?new_value, "Applying storage changeset"); - (hkey, old_value,new_value) - }) - .collect::>(); - - let mut cursor_storage_changeset = - tx.cursor_write::()?; - cursor_storage_changeset.seek_exact(storage_id)?; - - if wipe_storage { - // iterate over storage and save them before entry is deleted. - tx.cursor_read::()? - .walk(Some(address))? - .take_while(|res| { - res.as_ref().map(|(k, _)| *k == address).unwrap_or_default() - }) - .try_for_each(|entry| { - let (_, old_value) = entry?; - cursor_storage_changeset.append(storage_id, old_value) - })?; - - // delete all entries - tx.delete::(address, None)?; - - // insert storage changeset - for (key, _, new_value) in storage { - // old values are already cleared. - if new_value != U256::ZERO { - tx.put::( - address, - StorageEntry { key, value: new_value }, - )?; - } - } - } else { - // insert storage changeset - for (key, old_value, new_value) in storage { - let old_entry = StorageEntry { key, value: old_value }; - let new_entry = StorageEntry { key, value: new_value }; - // insert into StorageChangeSet - cursor_storage_changeset.append(storage_id, old_entry)?; - - // Always delete old value as duplicate table, put will not override it - tx.delete::(address, Some(old_entry))?; - if new_value != U256::ZERO { - tx.put::(address, new_entry)?; - } - } - } - } - // insert bytecode - for (hash, bytecode) in result.new_bytecodes.into_iter() { - // make different types of bytecode. Checked and maybe even analyzed (needs to - // be packed). Currently save only raw bytes. - let bytecode = bytecode.bytes(); - trace!(target: "sync::stages::execution", ?hash, ?bytecode, len = bytecode.len(), "Inserting bytecode"); - tx.put::(hash, bytecode[..bytecode.len()].to_vec())?; - - // NOTE: bytecode bytes are not inserted in change set and can be found in - // separate table - } - current_transition_id += 1; - } - - // If there are any post block changes, we will add account changesets to db. - for (address, changeset) in results.block_changesets.into_iter() { - trace!(target: "sync::stages::execution", ?address, current_transition_id, "Applying block reward"); - changeset.apply_to_db( - &**tx, - address, - current_transition_id, - spurious_dragon_active, - )?; - } - current_transition_id += 1; - } + // put execution results to database + tx.insert_execution_result(changesets, &self.executor.chain_spec, last_block)?; let done = !capped; info!(target: "sync::stages::execution", stage_progress = end_block, done, "Sync iteration finished"); @@ -413,7 +307,8 @@ mod tests { models::AccountBeforeTx, }; use reth_primitives::{ - hex_literal::hex, keccak256, Account, ChainSpecBuilder, SealedBlock, H160, MAINNET, U256, + hex_literal::hex, keccak256, Account, ChainSpecBuilder, SealedBlock, StorageEntry, H160, + H256, MAINNET, U256, }; use reth_provider::insert_canonical_block; use reth_rlp::Decodable; diff --git a/crates/stages/src/stages/hashing_account.rs b/crates/stages/src/stages/hashing_account.rs index ffba350c873..ceaedf30c05 100644 --- a/crates/stages/src/stages/hashing_account.rs +++ b/crates/stages/src/stages/hashing_account.rs @@ -7,11 +7,7 @@ use reth_db::{ }; use reth_primitives::{keccak256, Account, Address}; use reth_provider::Transaction; -use std::{ - collections::{BTreeMap, BTreeSet}, - fmt::Debug, - ops::Range, -}; +use std::{collections::BTreeMap, fmt::Debug, ops::Range}; use tracing::*; /// The [`StageId`] of the account hashing stage. @@ -180,38 +176,15 @@ impl Stage for AccountHashingStage { break } } else { - let mut plain_accounts = tx.cursor_read::()?; - let mut hashed_accounts = tx.cursor_write::()?; - // Aggregate all transition changesets and and make list of account that have been // changed. - tx.cursor_read::()? - .walk_range(from_transition..to_transition)? - .collect::, _>>()? - .into_iter() - // fold all account to one set of changed accounts - .fold(BTreeSet::new(), |mut accounts: BTreeSet
, (_, account_before)| { - accounts.insert(account_before.address); - accounts - }) - .into_iter() - // iterate over plain state and get newest value. - // Assumption we are okay to make is that plainstate represent - // `previous_stage_progress` state. - .map(|address| { - plain_accounts.seek_exact(address).map(|a| (address, a.map(|(_, v)| v))) - }) - .collect::, _>>()? - .into_iter() - .try_for_each(|(address, account)| -> Result<(), StageError> { - let hashed_address = keccak256(address); - if let Some(account) = account { - hashed_accounts.upsert(hashed_address, account)? - } else if hashed_accounts.seek_exact(hashed_address)?.is_some() { - hashed_accounts.delete_current()?; - } - Ok(()) - })?; + let lists = tx.get_addresses_of_changed_accounts(from_transition, to_transition)?; + // iterate over plain state and get newest value. + // Assumption we are okay to make is that plainstate represent + // `previous_stage_progress` state. + let accounts = tx.get_plainstate_accounts(lists.into_iter())?; + // insert and hash accounts to hashing table + tx.insert_account_for_hashing(accounts.into_iter())?; } info!(target: "sync::stages::hashing_account", "Stage finished"); diff --git a/crates/stages/src/stages/hashing_storage.rs b/crates/stages/src/stages/hashing_storage.rs index e7b5d3c49a3..7e54b37fcd7 100644 --- a/crates/stages/src/stages/hashing_storage.rs +++ b/crates/stages/src/stages/hashing_storage.rs @@ -9,10 +9,7 @@ use reth_db::{ }; use reth_primitives::{keccak256, Address, StorageEntry, H256, U256}; use reth_provider::Transaction; -use std::{ - collections::{BTreeMap, BTreeSet}, - fmt::Debug, -}; +use std::{collections::BTreeMap, fmt::Debug}; use tracing::*; /// The [`StageId`] of the storage hashing stage. @@ -133,70 +130,15 @@ impl Stage for StorageHashingStage { } } } else { - let mut plain_storage = tx.cursor_dup_read::()?; - let mut hashed_storage = tx.cursor_dup_write::()?; - // Aggregate all transition changesets and and make list of storages that have been // changed. - tx.cursor_read::()? - .walk_range( - TransitionIdAddress((from_transition, Address::zero())).. - TransitionIdAddress((to_transition, Address::zero())), - )? - .collect::, _>>()? - .into_iter() - // fold all storages and save its old state so we can remove it from HashedStorage - // it is needed as it is dup table. - .fold( - BTreeMap::new(), - |mut accounts: BTreeMap>, - (TransitionIdAddress((_, address)), storage_entry)| { - accounts.entry(address).or_default().insert(storage_entry.key); - accounts - }, - ) - .into_iter() - // iterate over plain state and get newest storage value. - // Assumption we are okay with is that plain state represent - // `previous_stage_progress` state. - .map(|(address, storage)| { - let res = ( - keccak256(address), - storage - .into_iter() - .map(|key| { - Ok::, reth_db::Error>( - plain_storage - .seek_by_key_subkey(address, key)? - .filter(|v| v.key == key) - .map(|ret| (keccak256(key), ret.value)), - ) - }) - .collect::>, _>>()? - .into_iter() - .flatten() - .collect::>(), - ); - Ok::<_, reth_db::Error>(res) - }) - .collect::, _>>()? - .into_iter() - // Hash the address and key and apply them to HashedStorage (if Storage is None - // just remove it); - .try_for_each(|(hashed_address, storage)| { - storage.into_iter().try_for_each(|(key, val)| -> Result<(), StageError> { - if hashed_storage - .seek_by_key_subkey(hashed_address, key)? - .filter(|entry| entry.key == key) - .is_some() - { - hashed_storage.delete_current()?; - } - - hashed_storage.upsert(hashed_address, StorageEntry { key, value: val })?; - Ok(()) - }) - })?; + let lists = + tx.get_addresses_and_keys_of_changed_storages(from_transition, to_transition)?; + // iterate over plain state and get newest storage value. + // Assumption we are okay with is that plain state represent + // `previous_stage_progress` state. + let storages = tx.get_plainstate_storages(lists.into_iter())?; + tx.insert_storage_for_hashing(storages.into_iter())?; } info!(target: "sync::stages::hashing_storage", "Stage finished"); diff --git a/crates/stages/src/stages/index_account_history.rs b/crates/stages/src/stages/index_account_history.rs index ccfda386f45..7cfa4fdcd9d 100644 --- a/crates/stages/src/stages/index_account_history.rs +++ b/crates/stages/src/stages/index_account_history.rs @@ -1,9 +1,8 @@ use crate::{ExecInput, ExecOutput, Stage, StageError, StageId, UnwindInput, UnwindOutput}; -use itertools::Itertools; use reth_db::{ cursor::{DbCursorRO, DbCursorRW}, database::{Database, DatabaseGAT}, - models::{sharded_key::NUM_OF_INDICES_IN_SHARD, ShardedKey}, + models::ShardedKey, tables, transaction::{DbTx, DbTxMut, DbTxMutGAT}, TransitionList, @@ -57,52 +56,10 @@ impl Stage for IndexAccountHistoryStage { std::cmp::min(stage_progress + self.commit_threshold, previous_stage_progress); let to_transition = tx.get_block_transition(to_block)?; - let account_changesets = tx - .cursor_read::()? - .walk(Some(from_transition))? - .take_while(|res| res.as_ref().map(|(k, _)| *k < to_transition).unwrap_or_default()) - .collect::, _>>()?; - - let account_changeset_lists = account_changesets - .into_iter() - // fold all account to one set of changed accounts - .fold( - BTreeMap::new(), - |mut accounts: BTreeMap>, (index, account)| { - accounts.entry(account.address).or_default().push(index); - accounts - }, - ); - // insert indexes to AccontHistory. - for (address, mut indices) in account_changeset_lists { - let mut last_shard = take_last_account_shard(tx, address)?; - last_shard.append(&mut indices); - // chunk indices and insert them in shards of N size. - let mut chunks = last_shard - .iter() - .chunks(NUM_OF_INDICES_IN_SHARD) - .into_iter() - .map(|chunks| chunks.map(|i| *i as usize).collect::>()) - .collect::>(); - let last_chunk = chunks.pop(); - - chunks.into_iter().try_for_each(|list| { - tx.put::( - ShardedKey::new( - address, - *list.last().expect("Chuck does not return empty list") as TransitionId, - ), - TransitionList::new(list).expect("Indices are presorted and not empty"), - ) - })?; - // Insert last list with u64::MAX - if let Some(last_list) = last_chunk { - tx.put::( - ShardedKey::new(address, u64::MAX), - TransitionList::new(last_list).expect("Indices are presorted and not empty"), - )? - } - } + let indices = + tx.get_account_transition_ids_from_changeset(from_transition, to_transition)?; + // Insert changeset to history index + tx.insert_account_history_index(indices)?; info!(target: "sync::stages::index_account_history", "Stage finished"); Ok(ExecOutput { stage_progress: to_block, done: true }) @@ -155,23 +112,6 @@ impl Stage for IndexAccountHistoryStage { } } -/// Load last shard and check if it is full and remove if it is not. If list is empty, last shard -/// was full or there is no shards at all. -pub fn take_last_account_shard( - tx: &Transaction<'_, DB>, - address: Address, -) -> Result, StageError> { - let mut cursor = tx.cursor_read::()?; - let last = cursor.seek_exact(ShardedKey::new(address, u64::MAX))?; - if let Some((shard_key, list)) = last { - // delete old shard so new one can be inserted. - tx.delete::(shard_key, None)?; - let list = list.iter(0).map(|i| i as u64).collect::>(); - return Ok(list) - } - Ok(Vec::new()) -} - /// Unwind all history shards. For boundary shard, remove it from database and /// return last part of shard with still valid items. If all full shard were removed, return list /// would be empty. @@ -212,7 +152,7 @@ pub fn unwind_account_history_shards( mod tests { use super::*; use crate::test_utils::{TestTransaction, PREV_STAGE_ID}; - use reth_db::models::AccountBeforeTx; + use reth_db::models::{sharded_key::NUM_OF_INDICES_IN_SHARD, AccountBeforeTx}; use reth_primitives::{hex_literal::hex, H160}; const ADDRESS: H160 = H160(hex!("0000000000000000000000000000000000000001")); diff --git a/crates/stages/src/stages/index_storage_history.rs b/crates/stages/src/stages/index_storage_history.rs index efef9e065fe..e29091030ec 100644 --- a/crates/stages/src/stages/index_storage_history.rs +++ b/crates/stages/src/stages/index_storage_history.rs @@ -1,9 +1,8 @@ use crate::{ExecInput, ExecOutput, Stage, StageError, StageId, UnwindInput, UnwindOutput}; -use itertools::Itertools; use reth_db::{ cursor::{DbCursorRO, DbCursorRW}, database::{Database, DatabaseGAT}, - models::{sharded_key::NUM_OF_INDICES_IN_SHARD, storage_sharded_key::StorageShardedKey}, + models::storage_sharded_key::StorageShardedKey, tables, transaction::{DbTx, DbTxMut, DbTxMutGAT}, TransitionList, @@ -57,58 +56,9 @@ impl Stage for IndexStorageHistoryStage { std::cmp::min(stage_progress + self.commit_threshold, previous_stage_progress); let to_transition = tx.get_block_transition(to_block)?; - let storage_chageset = tx - .cursor_read::()? - .walk(Some((from_transition, Address::zero()).into()))? - .take_while(|res| { - res.as_ref().map(|(k, _)| k.transition_id() < to_transition).unwrap_or_default() - }) - .collect::, _>>()?; - - // fold all storages to one set of changes - let storage_changeset_lists = storage_chageset.into_iter().fold( - BTreeMap::new(), - |mut storages: BTreeMap<(Address, H256), Vec>, (index, storage)| { - storages - .entry((index.address(), storage.key)) - .or_default() - .push(index.transition_id()); - storages - }, - ); - - for ((address, storage_key), mut indices) in storage_changeset_lists { - let mut last_shard = take_last_storage_shard(tx, address, storage_key)?; - last_shard.append(&mut indices); - - // chunk indices and insert them in shards of N size. - let mut chunks = last_shard - .iter() - .chunks(NUM_OF_INDICES_IN_SHARD) - .into_iter() - .map(|chunks| chunks.map(|i| *i as usize).collect::>()) - .collect::>(); - let last_chunk = chunks.pop(); - - // chunk indices and insert them in shards of N size. - chunks.into_iter().try_for_each(|list| { - tx.put::( - StorageShardedKey::new( - address, - storage_key, - *list.last().expect("Chuck does not return empty list") as TransitionId, - ), - TransitionList::new(list).expect("Indices are presorted and not empty"), - ) - })?; - // Insert last list with u64::MAX - if let Some(last_list) = last_chunk { - tx.put::( - StorageShardedKey::new(address, storage_key, u64::MAX), - TransitionList::new(last_list).expect("Indices are presorted and not empty"), - )?; - } - } + let indices = + tx.get_storage_transition_ids_from_changeset(from_transition, to_transition)?; + tx.insert_storage_history_index(indices)?; info!(target: "sync::stages::index_storage_history", "Stage finished"); Ok(ExecOutput { stage_progress: to_block, done: true }) @@ -164,24 +114,6 @@ impl Stage for IndexStorageHistoryStage { } } -/// Load last shard and check if it is full and remove if it is not. If list is empty, last shard -/// was full or there is no shards at all. -pub fn take_last_storage_shard( - tx: &Transaction<'_, DB>, - address: Address, - storage_key: H256, -) -> Result, StageError> { - let mut cursor = tx.cursor_read::()?; - let last = cursor.seek_exact(StorageShardedKey::new(address, storage_key, u64::MAX))?; - if let Some((storage_shard_key, list)) = last { - // delete old shard so new one can be inserted. - tx.delete::(storage_shard_key, None)?; - let list = list.iter(0).map(|i| i as u64).collect::>(); - return Ok(list) - } - Ok(Vec::new()) -} - /// Unwind all history shards. For boundary shard, remove it from database and /// return last part of shard with still valid items. If all full shard were removed, return list /// would be empty but this does not mean that there is none shard left but that there is no @@ -226,7 +158,9 @@ mod tests { use super::*; use crate::test_utils::{TestTransaction, PREV_STAGE_ID}; - use reth_db::models::{ShardedKey, TransitionIdAddress}; + use reth_db::models::{ + storage_sharded_key::NUM_OF_INDICES_IN_SHARD, ShardedKey, TransitionIdAddress, + }; use reth_primitives::{hex_literal::hex, StorageEntry, H160, U256}; const ADDRESS: H160 = H160(hex!("0000000000000000000000000000000000000001")); diff --git a/crates/stages/src/stages/merkle.rs b/crates/stages/src/stages/merkle.rs index 884c137a86d..ee29dd74d91 100644 --- a/crates/stages/src/stages/merkle.rs +++ b/crates/stages/src/stages/merkle.rs @@ -1,10 +1,7 @@ -use crate::{ - trie::DBTrieLoader, ExecInput, ExecOutput, Stage, StageError, StageId, UnwindInput, - UnwindOutput, -}; +use crate::{ExecInput, ExecOutput, Stage, StageError, StageId, UnwindInput, UnwindOutput}; use reth_db::{database::Database, tables, transaction::DbTx}; use reth_interfaces::consensus; -use reth_provider::Transaction; +use reth_provider::{trie::DBTrieLoader, Transaction}; use std::fmt::Debug; use tracing::*; diff --git a/crates/storage/provider/Cargo.toml b/crates/storage/provider/Cargo.toml index 509009cbaf2..0a1802d14e8 100644 --- a/crates/storage/provider/Cargo.toml +++ b/crates/storage/provider/Cargo.toml @@ -13,18 +13,31 @@ reth-primitives = { path = "../../primitives" } reth-interfaces = { path = "../../interfaces" } reth-revm-primitives = { path = "../../revm/revm-primitives" } reth-db = { path = "../db" } +reth-tracing = {path = "../../tracing"} +reth-rlp = {path = "../../rlp"} + +revm-primitives = "1.0.0" + +# trie +cita_trie = "4.0.0" +hasher = "0.1.4" # misc thiserror = "1.0.37" auto_impl = "1.0" +itertools = "0.10" # feature test-utils parking_lot = { version = "0.12", optional = true } -revm-primitives = "1.0" [dev-dependencies] reth-db = { path = "../db", features = ["test-utils"] } parking_lot = "0.12" +proptest = { version = "1.0" } +assert_matches = "1.5" + +# trie +triehash = "0.8" [features] bench = [] diff --git a/crates/executor/src/execution_result.rs b/crates/storage/provider/src/execution_result.rs similarity index 99% rename from crates/executor/src/execution_result.rs rename to crates/storage/provider/src/execution_result.rs index 3f1b433facc..747f08eabc2 100644 --- a/crates/executor/src/execution_result.rs +++ b/crates/storage/provider/src/execution_result.rs @@ -1,6 +1,8 @@ +//! Output of execution. + use reth_db::{models::AccountBeforeTx, tables, transaction::DbTxMut, Error as DbError}; use reth_primitives::{Account, Address, Receipt, H256, U256}; -use revm::primitives::Bytecode; +use revm_primitives::Bytecode; use std::collections::BTreeMap; /// Execution Result containing vector of transaction changesets diff --git a/crates/storage/provider/src/lib.rs b/crates/storage/provider/src/lib.rs index d7941257ae6..c9f2496dc3d 100644 --- a/crates/storage/provider/src/lib.rs +++ b/crates/storage/provider/src/lib.rs @@ -22,6 +22,12 @@ pub use providers::{ LatestStateProviderRef, ShareableDatabase, }; +/// Merkle trie +pub mod trie; + +/// Execution result +pub mod execution_result; + /// Helper types for interacting with the database mod transaction; pub use transaction::{Transaction, TransactionError}; diff --git a/crates/storage/provider/src/transaction.rs b/crates/storage/provider/src/transaction.rs index b1c1898fdcf..5da17221cf6 100644 --- a/crates/storage/provider/src/transaction.rs +++ b/crates/storage/provider/src/transaction.rs @@ -1,19 +1,36 @@ -#![allow(dead_code)] +use itertools::Itertools; use reth_db::{ - cursor::DbCursorRO, + cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO}, database::{Database, DatabaseGAT}, - models::StoredBlockBody, + models::{ + sharded_key, + storage_sharded_key::{self, StorageShardedKey}, + ShardedKey, StoredBlockBody, TransitionIdAddress, + }, table::Table, tables, transaction::{DbTx, DbTxMut}, + TransitionList, }; use reth_interfaces::{db::Error as DbError, provider::ProviderError}; -use reth_primitives::{BlockHash, BlockNumber, Header, TransitionId, TxNumber, U256}; +use reth_primitives::{ + keccak256, Account, Address, BlockHash, BlockNumber, ChainSpec, Hardfork, Header, SealedBlock, + StorageEntry, TransitionId, TxNumber, H256, U256, +}; +use reth_tracing::tracing::{info, trace}; use std::{ + collections::{BTreeMap, BTreeSet}, fmt::Debug, ops::{Deref, DerefMut}, }; +use crate::{ + insert_canonical_block, + trie::{DBTrieLoader, TrieError}, +}; + +use crate::execution_result::{AccountChangeSet, ExecutionResult}; + /// A container for any DB transaction that will open a new inner transaction when the current /// one is committed. // NOTE: This container is needed since `Transaction::commit` takes `mut self`, so methods in @@ -71,6 +88,11 @@ where Ok(Self { db, tx: Some(db.tx_mut()?) }) } + /// Creates a new container with given database and transaction handles. + pub fn new_raw(db: &'this DB, tx: >::TXMut) -> Self { + Self { db, tx: Some(tx) } + } + /// Accessor to the internal Database pub fn inner(&self) -> &'this DB { self.db @@ -213,6 +235,505 @@ where } Ok(()) } + + /// Load last shard and check if it is full and remove if it is not. If list is empty, last + /// shard was full or there is no shards at all. + fn take_last_account_shard(&self, address: Address) -> Result, TransactionError> { + let mut cursor = self.cursor_read::()?; + let last = cursor.seek_exact(ShardedKey::new(address, u64::MAX))?; + if let Some((shard_key, list)) = last { + // delete old shard so new one can be inserted. + self.delete::(shard_key, None)?; + let list = list.iter(0).map(|i| i as u64).collect::>(); + return Ok(list) + } + Ok(Vec::new()) + } + + /// Load last shard and check if it is full and remove if it is not. If list is empty, last + /// shard was full or there is no shards at all. + pub fn take_last_storage_shard( + &self, + address: Address, + storage_key: H256, + ) -> Result, TransactionError> { + let mut cursor = self.cursor_read::()?; + let last = cursor.seek_exact(StorageShardedKey::new(address, storage_key, u64::MAX))?; + if let Some((storage_shard_key, list)) = last { + // delete old shard so new one can be inserted. + self.delete::(storage_shard_key, None)?; + let list = list.iter(0).map(|i| i as u64).collect::>(); + return Ok(list) + } + Ok(Vec::new()) + } +} + +/// Stages impl +impl<'this, DB> Transaction<'this, DB> +where + DB: Database, +{ + /// Insert full block and make it canonical + /// + /// This is atomic operation and transaction will do one commit at the end of the function. + pub fn insert_block( + &mut self, + block: &SealedBlock, + chain_spec: &ChainSpec, + changeset: ExecutionResult, + ) -> Result<(), TransactionError> { + // Header, Body, SenderRecovery, TD, TxLookup stages + let (from, to) = insert_canonical_block(self.deref_mut(), block, false).unwrap(); + + let parent_block_number = block.number - 1; + + // execution stage + self.insert_execution_result(vec![changeset], chain_spec, parent_block_number)?; + + // storage hashing stage + { + let lists = self.get_addresses_and_keys_of_changed_storages(from, to)?; + let storages = self.get_plainstate_storages(lists.into_iter())?; + self.insert_storage_for_hashing(storages.into_iter())?; + } + + // account hashing stage + { + let lists = self.get_addresses_of_changed_accounts(from, to)?; + let accounts = self.get_plainstate_accounts(lists.into_iter())?; + self.insert_account_for_hashing(accounts.into_iter())?; + } + + // merkle tree + { + let current_root = self.get_header(parent_block_number)?.state_root; + let loader = DBTrieLoader::default(); + let root = loader.update_root(self, current_root, from..to)?; + if root != block.state_root { + return Err(TransactionError::StateTrieRootMismatch { + got: root, + expected: block.state_root, + block_number: block.number, + block_hash: block.hash(), + }) + } + } + + // account history stage + { + let indices = self.get_account_transition_ids_from_changeset(from, to)?; + self.insert_account_history_index(indices)?; + } + + // storage history stage + { + let indices = self.get_storage_transition_ids_from_changeset(from, to)?; + self.insert_storage_history_index(indices)?; + } + + // commit block to database + self.commit()?; + Ok(()) + } + + /// Iterate over account changesets and return all account address that were changed. + pub fn get_addresses_and_keys_of_changed_storages( + &self, + from: TransitionId, + to: TransitionId, + ) -> Result>, TransactionError> { + Ok(self + .cursor_read::()? + .walk_range( + TransitionIdAddress((from, Address::zero())).. + TransitionIdAddress((to, Address::zero())), + )? + .collect::, _>>()? + .into_iter() + // fold all storages and save its old state so we can remove it from HashedStorage + // it is needed as it is dup table. + .fold( + BTreeMap::new(), + |mut accounts: BTreeMap>, + (TransitionIdAddress((_, address)), storage_entry)| { + accounts.entry(address).or_default().insert(storage_entry.key); + accounts + }, + )) + } + + /// Get plainstate storages + #[allow(clippy::type_complexity)] + pub fn get_plainstate_storages( + &self, + iter: impl IntoIterator)>, + ) -> Result)>, TransactionError> { + let mut plain_storage = self.cursor_dup_read::()?; + + iter.into_iter() + .map(|(address, storage)| { + storage + .into_iter() + .map(|key| -> Result<_, TransactionError> { + let ret = plain_storage + .seek_by_key_subkey(address, key)? + .filter(|v| v.key == key) + .unwrap_or_default(); + Ok((key, ret.value)) + }) + .collect::, _>>() + .map(|storage| (address, storage)) + }) + .collect::, _>>() + } + + /// iterate over storages and insert them to hashing table + pub fn insert_storage_for_hashing( + &self, + storages: impl IntoIterator)>, + ) -> Result<(), TransactionError> { + // hash values + let hashed = storages.into_iter().fold(BTreeMap::new(), |mut map, (address, storage)| { + let storage = storage.into_iter().fold(BTreeMap::new(), |mut map, (key, value)| { + map.insert(keccak256(key), value); + map + }); + map.insert(keccak256(address), storage); + map + }); + + let mut hashed_storage = self.cursor_dup_write::()?; + // Hash the address and key and apply them to HashedStorage (if Storage is None + // just remove it); + hashed.into_iter().try_for_each(|(hashed_address, storage)| { + storage.into_iter().try_for_each(|(key, value)| -> Result<(), TransactionError> { + if hashed_storage + .seek_by_key_subkey(hashed_address, key)? + .filter(|entry| entry.key == key) + .is_some() + { + hashed_storage.delete_current()?; + } + + if value != U256::ZERO { + hashed_storage.upsert(hashed_address, StorageEntry { key, value })?; + } + Ok(()) + }) + })?; + Ok(()) + } + + /// Iterate over account changesets and return all account address that were changed. + pub fn get_addresses_of_changed_accounts( + &self, + from: TransitionId, + to: TransitionId, + ) -> Result, TransactionError> { + Ok(self + .cursor_read::()? + .walk_range(from..to)? + .collect::, _>>()? + .into_iter() + // fold all account to one set of changed accounts + .fold(BTreeSet::new(), |mut accounts: BTreeSet
, (_, account_before)| { + accounts.insert(account_before.address); + accounts + })) + } + + /// Get plainstate account from iterator + pub fn get_plainstate_accounts( + &self, + iter: impl IntoIterator, + ) -> Result)>, TransactionError> { + let mut plain_accounts = self.cursor_read::()?; + Ok(iter + .into_iter() + .map(|address| plain_accounts.seek_exact(address).map(|a| (address, a.map(|(_, v)| v)))) + .collect::, _>>()?) + } + + /// iterate over accounts and insert them to hashing table + pub fn insert_account_for_hashing( + &self, + accounts: impl IntoIterator)>, + ) -> Result<(), TransactionError> { + let mut hashed_accounts = self.cursor_write::()?; + + let hashes_accounts = accounts.into_iter().fold( + BTreeMap::new(), + |mut map: BTreeMap>, (address, account)| { + map.insert(keccak256(address), account); + map + }, + ); + + hashes_accounts.into_iter().try_for_each( + |(hashed_address, account)| -> Result<(), TransactionError> { + if let Some(account) = account { + hashed_accounts.upsert(hashed_address, account)? + } else if hashed_accounts.seek_exact(hashed_address)?.is_some() { + hashed_accounts.delete_current()?; + } + Ok(()) + }, + )?; + Ok(()) + } + + /// Get all transaction ids where account got changed. + pub fn get_storage_transition_ids_from_changeset( + &self, + from: TransitionId, + to: TransitionId, + ) -> Result>, TransactionError> { + let storage_changeset = self + .cursor_read::()? + .walk(Some((from, Address::zero()).into()))? + .take_while(|res| res.as_ref().map(|(k, _)| k.transition_id() < to).unwrap_or_default()) + .collect::, _>>()?; + + // fold all storages to one set of changes + let storage_changeset_lists = storage_changeset.into_iter().fold( + BTreeMap::new(), + |mut storages: BTreeMap<(Address, H256), Vec>, (index, storage)| { + storages + .entry((index.address(), storage.key)) + .or_default() + .push(index.transition_id()); + storages + }, + ); + + Ok(storage_changeset_lists) + } + + /// Get all transaction ids where account got changed. + pub fn get_account_transition_ids_from_changeset( + &self, + from: TransitionId, + to: TransitionId, + ) -> Result>, TransactionError> { + let account_changesets = self + .cursor_read::()? + .walk(Some(from))? + .take_while(|res| res.as_ref().map(|(k, _)| *k < to).unwrap_or_default()) + .collect::, _>>()?; + + let account_transtions = account_changesets + .into_iter() + // fold all account to one set of changed accounts + .fold( + BTreeMap::new(), + |mut accounts: BTreeMap>, (index, account)| { + accounts.entry(account.address).or_default().push(index); + accounts + }, + ); + + Ok(account_transtions) + } + + /// Insert storage change index to database. Used inside StorageHistoryIndex stage + pub fn insert_storage_history_index( + &self, + storage_transitions: BTreeMap<(Address, H256), Vec>, + ) -> Result<(), TransactionError> { + for ((address, storage_key), mut indices) in storage_transitions { + let mut last_shard = self.take_last_storage_shard(address, storage_key)?; + last_shard.append(&mut indices); + + // chunk indices and insert them in shards of N size. + let mut chunks = last_shard + .iter() + .chunks(storage_sharded_key::NUM_OF_INDICES_IN_SHARD) + .into_iter() + .map(|chunks| chunks.map(|i| *i as usize).collect::>()) + .collect::>(); + let last_chunk = chunks.pop(); + + // chunk indices and insert them in shards of N size. + chunks.into_iter().try_for_each(|list| { + self.put::( + StorageShardedKey::new( + address, + storage_key, + *list.last().expect("Chuck does not return empty list") as TransitionId, + ), + TransitionList::new(list).expect("Indices are presorted and not empty"), + ) + })?; + // Insert last list with u64::MAX + if let Some(last_list) = last_chunk { + self.put::( + StorageShardedKey::new(address, storage_key, u64::MAX), + TransitionList::new(last_list).expect("Indices are presorted and not empty"), + )?; + } + } + Ok(()) + } + + /// Insert account change index to database. Used inside AccountHistoryIndex stage + pub fn insert_account_history_index( + &self, + account_transitions: BTreeMap>, + ) -> Result<(), TransactionError> { + // insert indexes to AccountHistory. + for (address, mut indices) in account_transitions { + let mut last_shard = self.take_last_account_shard(address)?; + last_shard.append(&mut indices); + // chunk indices and insert them in shards of N size. + let mut chunks = last_shard + .iter() + .chunks(sharded_key::NUM_OF_INDICES_IN_SHARD) + .into_iter() + .map(|chunks| chunks.map(|i| *i as usize).collect::>()) + .collect::>(); + let last_chunk = chunks.pop(); + + chunks.into_iter().try_for_each(|list| { + self.put::( + ShardedKey::new( + address, + *list.last().expect("Chuck does not return empty list") as TransitionId, + ), + TransitionList::new(list).expect("Indices are presorted and not empty"), + ) + })?; + // Insert last list with u64::MAX + if let Some(last_list) = last_chunk { + self.put::( + ShardedKey::new(address, u64::MAX), + TransitionList::new(last_list).expect("Indices are presorted and not empty"), + )? + } + } + Ok(()) + } + + /// Used inside execution stage to commit created account storage changesets for transaction or + /// block state change. + pub fn insert_execution_result( + &self, + changesets: Vec, + chain_spec: &ChainSpec, + parent_block_number: u64, + ) -> Result<(), TransactionError> { + // Get last tx count so that we can know amount of transaction in the block. + let mut current_transition_id = self + .get::(parent_block_number)? + .ok_or(ProviderError::BlockTransition { block_number: parent_block_number })?; + + info!(target: "sync::stages::execution", current_transition_id, blocks = changesets.len(), "Inserting execution results"); + + // apply changes to plain database. + let mut block_number = parent_block_number; + for results in changesets.into_iter() { + block_number += 1; + let spurious_dragon_active = + chain_spec.fork(Hardfork::SpuriousDragon).active_at_block(block_number); + // insert state change set + for result in results.tx_changesets.into_iter() { + for (address, account_change_set) in result.changeset.into_iter() { + let AccountChangeSet { account, wipe_storage, storage } = account_change_set; + // apply account change to db. Updates AccountChangeSet and PlainAccountState + // tables. + trace!(target: "sync::stages::execution", ?address, current_transition_id, ?account, wipe_storage, "Applying account changeset"); + account.apply_to_db( + &**self, + address, + current_transition_id, + spurious_dragon_active, + )?; + + let storage_id = TransitionIdAddress((current_transition_id, address)); + + // cast key to H256 and trace the change + let storage = storage + .into_iter() + .map(|(key, (old_value,new_value))| { + let hkey = H256(key.to_be_bytes()); + trace!(target: "sync::stages::execution", ?address, current_transition_id, ?hkey, ?old_value, ?new_value, "Applying storage changeset"); + (hkey, old_value,new_value) + }) + .collect::>(); + + let mut cursor_storage_changeset = + self.cursor_write::()?; + cursor_storage_changeset.seek_exact(storage_id)?; + + if wipe_storage { + // iterate over storage and save them before entry is deleted. + self.cursor_read::()? + .walk(Some(address))? + .take_while(|res| { + res.as_ref().map(|(k, _)| *k == address).unwrap_or_default() + }) + .try_for_each(|entry| { + let (_, old_value) = entry?; + cursor_storage_changeset.append(storage_id, old_value) + })?; + + // delete all entries + self.delete::(address, None)?; + + // insert storage changeset + for (key, _, new_value) in storage { + // old values are already cleared. + if new_value != U256::ZERO { + self.put::( + address, + StorageEntry { key, value: new_value }, + )?; + } + } + } else { + // insert storage changeset + for (key, old_value, new_value) in storage { + let old_entry = StorageEntry { key, value: old_value }; + let new_entry = StorageEntry { key, value: new_value }; + // insert into StorageChangeSet + cursor_storage_changeset.append(storage_id, old_entry)?; + + // Always delete old value as duplicate table, put will not override it + self.delete::(address, Some(old_entry))?; + if new_value != U256::ZERO { + self.put::(address, new_entry)?; + } + } + } + } + // insert bytecode + for (hash, bytecode) in result.new_bytecodes.into_iter() { + // make different types of bytecode. Checked and maybe even analyzed (needs to + // be packed). Currently save only raw bytes. + let bytecode = bytecode.bytes(); + trace!(target: "sync::stages::execution", ?hash, ?bytecode, len = bytecode.len(), "Inserting bytecode"); + self.put::(hash, bytecode[..bytecode.len()].to_vec())?; + + // NOTE: bytecode bytes are not inserted in change set and can be found in + // separate table + } + current_transition_id += 1; + } + + // If there are any post block changes, we will add account changesets to db. + for (address, changeset) in results.block_changesets.into_iter() { + trace!(target: "sync::stages::execution", ?address, current_transition_id, "Applying block reward"); + changeset.apply_to_db( + &**self, + address, + current_transition_id, + spurious_dragon_active, + )?; + } + current_transition_id += 1; + } + Ok(()) + } } /// An error that can occur when using the transaction container @@ -224,4 +745,19 @@ pub enum TransactionError { /// The transaction encountered a database integrity error. #[error("A database integrity error occurred: {0}")] DatabaseIntegrity(#[from] ProviderError), + /// The transaction encountered merkle trie error. + #[error("Merkle trie calculation error: {0}")] + MerkleTrie(#[from] TrieError), + /// Root mismatch + #[error("Merkle trie root mismatch on block: #{block_number:?} {block_hash:?}. got: {got:?} expected:{got:?}")] + StateTrieRootMismatch { + /// Expected root + expected: H256, + /// Calculated root + got: H256, + /// Block number + block_number: BlockNumber, + /// Block hash + block_hash: BlockHash, + }, } diff --git a/crates/stages/src/trie/mod.rs b/crates/storage/provider/src/trie/mod.rs similarity index 99% rename from crates/stages/src/trie/mod.rs rename to crates/storage/provider/src/trie/mod.rs index 199df9f8417..c99689b8461 100644 --- a/crates/stages/src/trie/mod.rs +++ b/crates/storage/provider/src/trie/mod.rs @@ -1,3 +1,4 @@ +use crate::Transaction; use cita_trie::{PatriciaTrie, Trie}; use hasher::HasherKeccak; use reth_db::{ @@ -11,18 +12,19 @@ use reth_primitives::{ keccak256, proofs::EMPTY_ROOT, Account, Address, StorageEntry, StorageTrieEntry, TransitionId, H256, KECCAK_EMPTY, U256, }; -use reth_provider::Transaction; use reth_rlp::{ encode_fixed_size, Decodable, DecodeError, Encodable, RlpDecodable, RlpEncodable, EMPTY_STRING_CODE, }; +use reth_tracing::tracing::*; use std::{ collections::{BTreeMap, BTreeSet}, ops::Range, sync::Arc, }; -use tracing::*; +/// Merkle Trie error types +#[allow(missing_docs)] #[derive(Debug, thiserror::Error)] pub enum TrieError { #[error("Some error occurred: {0}")] @@ -410,9 +412,8 @@ mod tests { hex_literal::hex, keccak256, proofs::{genesis_state_root, KeccakHasher, EMPTY_ROOT}, - Address, ChainSpec, + Address, ChainSpec, MAINNET, }; - use reth_staged_sync::utils::chainspec::chain_spec_value_parser; use std::{collections::HashMap, str::FromStr}; use triehash::sec_trie_root; @@ -553,7 +554,7 @@ mod tests { let trie = DBTrieLoader::default(); let db = create_test_rw_db(); let mut tx = Transaction::new(db.as_ref()).unwrap(); - let ChainSpec { genesis, .. } = chain_spec_value_parser("mainnet").unwrap(); + let ChainSpec { genesis, .. } = MAINNET.clone(); // Insert account state for (address, account) in &genesis.alloc { diff --git a/crates/storage/provider/src/utils.rs b/crates/storage/provider/src/utils.rs index 086cc7fc8c6..13888706205 100644 --- a/crates/storage/provider/src/utils.rs +++ b/crates/storage/provider/src/utils.rs @@ -4,34 +4,39 @@ use reth_db::{ transaction::{DbTx, DbTxMut}, }; use reth_interfaces::{provider::ProviderError, Result}; -use reth_primitives::{SealedBlock, U256}; +use reth_primitives::{SealedBlock, TransitionId, U256}; /// Insert block data into corresponding tables. Used mainly for testing & internal tooling. /// /// /// Check parent dependency in [tables::HeaderNumbers] and in [tables::BlockBodies] tables. -/// Inserts blocks data to [tables::CanonicalHeaders], [tables::Headers], [tables::HeaderNumbers], -/// and transactions data to [tables::TxSenders], [tables::Transactions], -/// [tables::BlockBodies] and [tables::BlockBodies] +/// Inserts header data to [tables::CanonicalHeaders], [tables::Headers], [tables::HeaderNumbers]. +/// and transactions data to [tables::TxSenders], [tables::Transactions], [tables::TxHashNumber]. +/// and transition indexes to [tables::BlockTransitionIndex] and [tables::TxTransitionIndex]. +/// And block data [tables::BlockBodies], [tables::BlockBodies] and [tables::BlockWithdrawals]. +/// +/// Return [TransitionId] `(from,to)` pub fn insert_block<'a, TX: DbTxMut<'a> + DbTx<'a>>( tx: &TX, block: &SealedBlock, has_block_reward: bool, parent_tx_num_transition_id: Option<(u64, u64)>, -) -> Result<()> { +) -> Result<(TransitionId, TransitionId)> { tx.put::(block.number, block.hash())?; // Put header with canonical hashes. tx.put::(block.number, block.header.as_ref().clone())?; tx.put::(block.hash(), block.number)?; - tx.put::( - block.number, - if has_block_reward { - U256::ZERO - } else { - U256::from(58_750_000_000_000_000_000_000_u128) + block.difficulty - } - .into(), - )?; + + // total difficulty + let ttd = if block.number == 0 { + U256::ZERO + } else { + let parent_block_number = block.number - 1; + let parent_ttd = tx.get::(parent_block_number)?.unwrap_or_default(); + parent_ttd.0 + block.difficulty + }; + + tx.put::(block.number, ttd.into())?; // insert body ommers data if !block.ommers.is_empty() { @@ -56,7 +61,7 @@ pub fn insert_block<'a, TX: DbTxMut<'a> + DbTx<'a>>( .ok_or(ProviderError::BlockTransition { block_number: prev_block_num })?; (prev_body.start_tx_id + prev_body.tx_count, last_transition_id) }; - + let from_transition = transition_id; // insert body data tx.put::( block.number, @@ -65,9 +70,11 @@ pub fn insert_block<'a, TX: DbTxMut<'a> + DbTx<'a>>( for transaction in block.body.iter() { let rec_tx = transaction.clone().into_ecrecovered().unwrap(); + let hash = rec_tx.hash(); tx.put::(current_tx_id, rec_tx.signer())?; tx.put::(current_tx_id, rec_tx.into())?; tx.put::(current_tx_id, transition_id)?; + tx.put::(hash, current_tx_id)?; transition_id += 1; current_tx_id += 1; } @@ -88,7 +95,8 @@ pub fn insert_block<'a, TX: DbTxMut<'a> + DbTx<'a>>( } tx.put::(block.number, transition_id)?; - Ok(()) + let to_transition = transition_id; + Ok((from_transition, to_transition)) } /// Inserts canonical block in blockchain. Parent tx num and transition id is taken from @@ -97,6 +105,6 @@ pub fn insert_canonical_block<'a, TX: DbTxMut<'a> + DbTx<'a>>( tx: &TX, block: &SealedBlock, has_block_reward: bool, -) -> Result<()> { +) -> Result<(TransitionId, TransitionId)> { insert_block(tx, block, has_block_reward, None) } From ad0ce8cf1a02796cf78015ad9b3bc8d1f990d8ce Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Wed, 1 Mar 2023 17:35:13 -0500 Subject: [PATCH 021/191] fix(rpc): make exp claim optional (#1601) --- crates/rpc/rpc/src/layers/auth_layer.rs | 6 +++--- crates/rpc/rpc/src/layers/jwt_secret.rs | 28 ++++++++++++++++++------- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/crates/rpc/rpc/src/layers/auth_layer.rs b/crates/rpc/rpc/src/layers/auth_layer.rs index c71c8f99ddb..537bdd33f5a 100644 --- a/crates/rpc/rpc/src/layers/auth_layer.rs +++ b/crates/rpc/rpc/src/layers/auth_layer.rs @@ -191,7 +191,7 @@ mod tests { } async fn valid_jwt() { - let claims = Claims { iat: to_u64(SystemTime::now()), exp: 10000000000 }; + let claims = Claims { iat: to_u64(SystemTime::now()), exp: Some(10000000000) }; let secret = JwtSecret::from_hex(SECRET).unwrap(); // Same secret as the server let jwt = secret.encode(&claims).unwrap(); let (status, _) = send_request(Some(jwt)).await; @@ -209,7 +209,7 @@ mod tests { // This secret is different from the server. This will generate a // different signature let secret = JwtSecret::random(); - let claims = Claims { iat: to_u64(SystemTime::now()), exp: 10000000000 }; + let claims = Claims { iat: to_u64(SystemTime::now()), exp: Some(10000000000) }; let jwt = secret.encode(&claims).unwrap(); let (status, body) = send_request(Some(jwt)).await; @@ -222,7 +222,7 @@ mod tests { let secret = JwtSecret::from_hex(SECRET).unwrap(); // Same secret as the server let iat = to_u64(SystemTime::now()) + 1000; - let claims = Claims { iat, exp: 10000000000 }; + let claims = Claims { iat, exp: Some(10000000000) }; let jwt = secret.encode(&claims).unwrap(); let (status, body) = send_request(Some(jwt)).await; diff --git a/crates/rpc/rpc/src/layers/jwt_secret.rs b/crates/rpc/rpc/src/layers/jwt_secret.rs index 1f2d170c5e6..bf757292c9b 100644 --- a/crates/rpc/rpc/src/layers/jwt_secret.rs +++ b/crates/rpc/rpc/src/layers/jwt_secret.rs @@ -111,7 +111,9 @@ impl JwtSecret { /// /// See also: [JWT Claims - Engine API specs](https://github.com/ethereum/execution-apis/blob/main/src/engine/authentication.md#jwt-claims) pub fn validate(&self, jwt: String) -> Result<(), JwtError> { - let validation = Validation::new(JWT_SIGNATURE_ALGO); + let mut validation = Validation::new(JWT_SIGNATURE_ALGO); + // ensure that the JWT has an `iat` claim + validation.set_required_spec_claims(&["iat"]); let bytes = &self.0; match decode::(&jwt, &DecodingKey::from_secret(bytes), &validation) { @@ -165,7 +167,7 @@ pub(crate) struct Claims { /// - [`RFC-7519 - Spec`](https://www.rfc-editor.org/rfc/rfc7519#section-4.1.6) /// - [`RFC-7519 - Notations`](https://www.rfc-editor.org/rfc/rfc7519#section-2) pub(crate) iat: u64, - pub(crate) exp: u64, + pub(crate) exp: Option, } impl Claims { @@ -239,7 +241,7 @@ mod tests { #[test] fn validation_ok() { let secret = JwtSecret::random(); - let claims = Claims { iat: to_u64(SystemTime::now()), exp: 10000000000 }; + let claims = Claims { iat: to_u64(SystemTime::now()), exp: Some(10000000000) }; let jwt: String = secret.encode(&claims).unwrap(); let result = secret.validate(jwt); @@ -254,7 +256,7 @@ mod tests { // Check past 'iat' claim more than 60 secs let offset = Duration::from_secs(JWT_MAX_IAT_DIFF.as_secs() + 1); let out_of_window_time = SystemTime::now().checked_sub(offset).unwrap(); - let claims = Claims { iat: to_u64(out_of_window_time), exp: 10000000000 }; + let claims = Claims { iat: to_u64(out_of_window_time), exp: Some(10000000000) }; let jwt: String = secret.encode(&claims).unwrap(); let result = secret.validate(jwt); @@ -264,7 +266,7 @@ mod tests { // Check future 'iat' claim more than 60 secs let offset = Duration::from_secs(JWT_MAX_IAT_DIFF.as_secs() + 1); let out_of_window_time = SystemTime::now().checked_add(offset).unwrap(); - let claims = Claims { iat: to_u64(out_of_window_time), exp: 10000000000 }; + let claims = Claims { iat: to_u64(out_of_window_time), exp: Some(10000000000) }; let jwt: String = secret.encode(&claims).unwrap(); let result = secret.validate(jwt); @@ -275,7 +277,7 @@ mod tests { #[test] fn validation_error_wrong_signature() { let secret_1 = JwtSecret::random(); - let claims = Claims { iat: to_u64(SystemTime::now()), exp: 10000000000 }; + let claims = Claims { iat: to_u64(SystemTime::now()), exp: Some(10000000000) }; let jwt: String = secret_1.encode(&claims).unwrap(); // A different secret will generate a different signature. @@ -292,13 +294,25 @@ mod tests { let key = EncodingKey::from_secret(bytes); let unsupported_algo = Header::new(Algorithm::HS384); - let claims = Claims { iat: to_u64(SystemTime::now()), exp: 10000000000 }; + let claims = Claims { iat: to_u64(SystemTime::now()), exp: Some(10000000000) }; let jwt: String = encode(&unsupported_algo, &claims, &key).unwrap(); let result = secret.validate(jwt); assert!(matches!(result, Err(JwtError::UnsupportedSignatureAlgorithm))); } + #[test] + fn valid_without_exp_claim() { + let secret = JwtSecret::random(); + + let claims = Claims { iat: to_u64(SystemTime::now()), exp: None }; + let jwt: String = secret.encode(&claims).unwrap(); + + let result = secret.validate(jwt); + + assert!(matches!(result, Ok(()))); + } + #[test] fn ephemeral_secret_created() { let fpath: &Path = Path::new("secret0.hex"); From 626c076c85610421d51d29d276670c523f66430e Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 1 Mar 2023 23:46:57 +0100 Subject: [PATCH 022/191] feat(rpc): integrate eth cache and task executor in pipeline (#1596) --- Cargo.lock | 1 + bin/reth/src/args/rpc_server_args.rs | 23 +++- bin/reth/src/node/mod.rs | 8 +- crates/rpc/rpc-builder/Cargo.toml | 1 + crates/rpc/rpc-builder/src/auth.rs | 36 ++++-- crates/rpc/rpc-builder/src/lib.rs | 138 +++++++++++++++------ crates/rpc/rpc-builder/tests/it/utils.rs | 5 +- crates/rpc/rpc/src/eth/api/mod.rs | 8 +- crates/rpc/rpc/src/eth/api/server.rs | 15 ++- crates/rpc/rpc/src/eth/api/transactions.rs | 8 +- crates/rpc/rpc/src/eth/cache.rs | 66 +++++++--- crates/rpc/rpc/src/eth/mod.rs | 2 +- crates/rpc/rpc/src/lib.rs | 2 +- crates/tasks/src/lib.rs | 2 +- 14 files changed, 239 insertions(+), 76 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f42fba15ca7..cfef28ee277 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5032,6 +5032,7 @@ dependencies = [ "reth-rpc-api", "reth-rpc-engine-api", "reth-rpc-types", + "reth-tasks", "reth-tracing", "reth-transaction-pool", "serde", diff --git a/bin/reth/src/args/rpc_server_args.rs b/bin/reth/src/args/rpc_server_args.rs index 0afb2a016ea..54c16749f0f 100644 --- a/bin/reth/src/args/rpc_server_args.rs +++ b/bin/reth/src/args/rpc_server_args.rs @@ -11,6 +11,7 @@ use reth_rpc_builder::{ RpcServerHandle, ServerBuilder, TransportRpcModuleConfig, }; use reth_rpc_engine_api::EngineApiHandle; +use reth_tasks::TaskSpawner; use reth_transaction_pool::TransactionPool; use std::{ net::{IpAddr, Ipv4Addr, SocketAddr}, @@ -103,11 +104,12 @@ impl RpcServerArgs { } /// Convenience function for starting a rpc server with configs which extracted from cli args. - pub(crate) async fn start_rpc_server( + pub(crate) async fn start_rpc_server( &self, client: Client, pool: Pool, network: Network, + executor: Tasks, ) -> Result where Client: BlockProvider @@ -115,9 +117,11 @@ impl RpcServerArgs { + StateProviderFactory + EvmEnvProvider + Clone + + Unpin + 'static, Pool: TransactionPool + Clone + 'static, Network: NetworkInfo + Peers + Clone + 'static, + Tasks: TaskSpawner + Clone + 'static, { reth_rpc_builder::launch( client, @@ -125,16 +129,18 @@ impl RpcServerArgs { network, self.transport_rpc_module_config(), self.rpc_server_config(), + executor, ) .await } /// Create Engine API server. - pub(crate) async fn start_auth_server( + pub(crate) async fn start_auth_server( &self, client: Client, pool: Pool, network: Network, + executor: Tasks, handle: EngineApiHandle, ) -> Result where @@ -143,16 +149,27 @@ impl RpcServerArgs { + StateProviderFactory + EvmEnvProvider + Clone + + Unpin + 'static, Pool: TransactionPool + Clone + 'static, Network: NetworkInfo + Peers + Clone + 'static, + Tasks: TaskSpawner + Clone + 'static, { let socket_address = SocketAddr::new( self.auth_addr.unwrap_or(IpAddr::V4(Ipv4Addr::UNSPECIFIED)), self.auth_port.unwrap_or(constants::DEFAULT_AUTH_PORT), ); let secret = self.jwt_secret().map_err(|err| RpcError::Custom(err.to_string()))?; - reth_rpc_builder::auth::launch(client, pool, network, handle, socket_address, secret).await + reth_rpc_builder::auth::launch( + client, + pool, + network, + executor, + handle, + socket_address, + secret, + ) + .await } /// Creates the [TransportRpcModuleConfig] from cli args. diff --git a/bin/reth/src/node/mod.rs b/bin/reth/src/node/mod.rs index 082d2c5cbba..1c8753b6082 100644 --- a/bin/reth/src/node/mod.rs +++ b/bin/reth/src/node/mod.rs @@ -160,7 +160,12 @@ impl Command { let _rpc_server = self .rpc - .start_rpc_server(shareable_db.clone(), test_transaction_pool.clone(), network.clone()) + .start_rpc_server( + shareable_db.clone(), + test_transaction_pool.clone(), + network.clone(), + ctx.task_executor.clone(), + ) .await?; info!(target: "reth::cli", "Started RPC server"); @@ -174,6 +179,7 @@ impl Command { shareable_db, test_transaction_pool, network.clone(), + ctx.task_executor.clone(), engine_api_handle, ) .await?; diff --git a/crates/rpc/rpc-builder/Cargo.toml b/crates/rpc/rpc-builder/Cargo.toml index 67e32a50adc..6f9e41a588a 100644 --- a/crates/rpc/rpc-builder/Cargo.toml +++ b/crates/rpc/rpc-builder/Cargo.toml @@ -15,6 +15,7 @@ reth-rpc = { path = "../rpc" } reth-rpc-api = { path = "../rpc-api" } reth-rpc-engine-api = { path = "../rpc-engine-api" } reth-rpc-types = { path = "../rpc-types" } +reth-tasks = { path = "../../tasks" } reth-transaction-pool = { path = "../../transaction-pool" } jsonrpsee = { version = "0.16", features = ["server"] } diff --git a/crates/rpc/rpc-builder/src/auth.rs b/crates/rpc/rpc-builder/src/auth.rs index 4bccd977a1f..48ca32d922a 100644 --- a/crates/rpc/rpc-builder/src/auth.rs +++ b/crates/rpc/rpc-builder/src/auth.rs @@ -14,11 +14,12 @@ pub use reth_ipc::server::{Builder as IpcServerBuilder, Endpoint}; use reth_network_api::{NetworkInfo, Peers}; use reth_provider::{BlockProvider, EvmEnvProvider, HeaderProvider, StateProviderFactory}; use reth_rpc::{ - AdminApi, AuthLayer, DebugApi, EngineApi, EthApi, JwtAuthValidator, JwtSecret, NetApi, - TraceApi, Web3Api, + eth::cache::EthStateCache, AdminApi, AuthLayer, DebugApi, EngineApi, EthApi, JwtAuthValidator, + JwtSecret, NetApi, TraceApi, Web3Api, }; use reth_rpc_api::servers::*; use reth_rpc_engine_api::EngineApiHandle; +use reth_tasks::TaskSpawner; use reth_transaction_pool::TransactionPool; use serde::{Deserialize, Serialize, Serializer}; use std::{ @@ -31,22 +32,32 @@ use strum::{AsRefStr, EnumString, EnumVariantNames, ParseError, VariantNames}; use tower::layer::util::{Identity, Stack}; use tower_http::cors::{AllowOrigin, Any, CorsLayer}; -/// Configure and launch an auth server with `engine` and `eth` namespaces. -pub async fn launch( +/// Configure and launch an auth server with `engine` and a _new_ `eth` namespace. +pub async fn launch( client: Client, pool: Pool, network: Network, + executor: Tasks, handle: EngineApiHandle, socket_addr: SocketAddr, secret: JwtSecret, ) -> Result where - Client: - BlockProvider + HeaderProvider + StateProviderFactory + EvmEnvProvider + Clone + 'static, + Client: BlockProvider + + HeaderProvider + + StateProviderFactory + + EvmEnvProvider + + Clone + + Unpin + + 'static, Pool: TransactionPool + Clone + 'static, Network: NetworkInfo + Peers + Clone + 'static, + Tasks: TaskSpawner + Clone + 'static, { - launch_with_eth_api(EthApi::new(client, pool, network), handle, socket_addr, secret).await + // spawn a new cache task + let eth_cache = EthStateCache::spawn_with(client.clone(), Default::default(), executor); + launch_with_eth_api(EthApi::new(client, pool, network, eth_cache), handle, socket_addr, secret) + .await } /// Configure and launch an auth server with existing EthApi implementation. @@ -57,8 +68,13 @@ pub async fn launch_with_eth_api( secret: JwtSecret, ) -> Result where - Client: - BlockProvider + HeaderProvider + StateProviderFactory + EvmEnvProvider + Clone + 'static, + Client: BlockProvider + + HeaderProvider + + StateProviderFactory + + EvmEnvProvider + + Clone + + Unpin + + 'static, Pool: TransactionPool + Clone + 'static, Network: NetworkInfo + Peers + Clone + 'static, { @@ -71,7 +87,7 @@ where let middleware = tower::ServiceBuilder::new().layer(AuthLayer::new(JwtAuthValidator::new(secret))); - // By default both http and ws are enabled. + // By default, both http and ws are enabled. let server = ServerBuilder::new().set_middleware(middleware).build(socket_addr).await?; server.start(module) diff --git a/crates/rpc/rpc-builder/src/lib.rs b/crates/rpc/rpc-builder/src/lib.rs index 1a87511228a..2f9b16dbc41 100644 --- a/crates/rpc/rpc-builder/src/lib.rs +++ b/crates/rpc/rpc-builder/src/lib.rs @@ -28,10 +28,11 @@ //! use reth_network_api::{NetworkInfo, Peers}; //! use reth_provider::{BlockProvider, HeaderProvider, StateProviderFactory, EvmEnvProvider}; //! use reth_rpc_builder::{RethRpcModule, RpcModuleBuilder, RpcServerConfig, ServerBuilder, TransportRpcModuleConfig}; +//! use reth_tasks::TokioTaskExecutor; //! use reth_transaction_pool::TransactionPool; //! pub async fn launch(client: Client, pool: Pool, network: Network) //! where -//! Client: BlockProvider + HeaderProvider + StateProviderFactory + EvmEnvProvider + Clone + 'static, +//! Client: BlockProvider + HeaderProvider + StateProviderFactory + EvmEnvProvider + Clone + Unpin + 'static, //! Pool: TransactionPool + Clone + 'static, //! Network: NetworkInfo + Peers + Clone + 'static, //! { @@ -42,7 +43,7 @@ //! RethRpcModule::Eth, //! RethRpcModule::Web3, //! ]); -//! let transport_modules = RpcModuleBuilder::new(client, pool, network).build(transports); +//! let transport_modules = RpcModuleBuilder::new(client, pool, network, TokioTaskExecutor::default()).build(transports); //! let handle = RpcServerConfig::default() //! .with_http(ServerBuilder::default()) //! .start(transport_modules) @@ -86,27 +87,36 @@ pub mod auth; /// Common RPC constants. pub mod constants; use constants::*; +use reth_rpc::eth::cache::EthStateCache; +use reth_tasks::TaskSpawner; /// Cors utilities. mod cors; /// Convenience function for starting a server in one step. -pub async fn launch( +pub async fn launch( client: Client, pool: Pool, network: Network, module_config: impl Into, server_config: impl Into, + executor: Tasks, ) -> Result where - Client: - BlockProvider + HeaderProvider + StateProviderFactory + EvmEnvProvider + Clone + 'static, + Client: BlockProvider + + HeaderProvider + + StateProviderFactory + + EvmEnvProvider + + Clone + + Unpin + + 'static, Pool: TransactionPool + Clone + 'static, Network: NetworkInfo + Peers + Clone + 'static, + Tasks: TaskSpawner + Clone + 'static, { let module_config = module_config.into(); let server_config = server_config.into(); - RpcModuleBuilder::new(client, pool, network) + RpcModuleBuilder::new(client, pool, network, executor) .build(module_config) .start_server(server_config) .await @@ -116,57 +126,74 @@ where /// /// This is the main entrypoint for up RPC servers. #[derive(Debug)] -pub struct RpcModuleBuilder { +pub struct RpcModuleBuilder { /// The Client type to when creating all rpc handlers client: Client, /// The Pool type to when creating all rpc handlers pool: Pool, /// The Network type to when creating all rpc handlers network: Network, + /// How additional tasks are spawned, for example in the eth pubsub namespace + executor: Tasks, } // === impl RpcBuilder === -impl RpcModuleBuilder { +impl RpcModuleBuilder { /// Create a new instance of the builder - pub fn new(client: Client, pool: Pool, network: Network) -> Self { - Self { client, pool, network } + pub fn new(client: Client, pool: Pool, network: Network, executor: Tasks) -> Self { + Self { client, pool, network, executor } } /// Configure the client instance. - pub fn with_client(self, client: C) -> RpcModuleBuilder + pub fn with_client(self, client: C) -> RpcModuleBuilder where C: BlockProvider + StateProviderFactory + EvmEnvProvider + 'static, { - let Self { pool, network, .. } = self; - RpcModuleBuilder { client, network, pool } + let Self { pool, network, executor, .. } = self; + RpcModuleBuilder { client, network, pool, executor } } /// Configure the transaction pool instance. - pub fn with_pool

(self, pool: P) -> RpcModuleBuilder + pub fn with_pool

(self, pool: P) -> RpcModuleBuilder where P: TransactionPool + 'static, { - let Self { client, network, .. } = self; - RpcModuleBuilder { client, network, pool } + let Self { client, network, executor, .. } = self; + RpcModuleBuilder { client, network, pool, executor } } /// Configure the network instance. - pub fn with_network(self, network: N) -> RpcModuleBuilder + pub fn with_network(self, network: N) -> RpcModuleBuilder where N: NetworkInfo + Peers + 'static, { - let Self { client, pool, .. } = self; - RpcModuleBuilder { client, network, pool } + let Self { client, pool, executor, .. } = self; + RpcModuleBuilder { client, network, pool, executor } + } + + /// Configure the task executor to use for additional tasks. + pub fn with_executor(self, executor: T) -> RpcModuleBuilder + where + T: TaskSpawner + 'static, + { + let Self { pool, network, client, .. } = self; + RpcModuleBuilder { client, network, pool, executor } } } -impl RpcModuleBuilder +impl RpcModuleBuilder where - Client: - BlockProvider + HeaderProvider + StateProviderFactory + EvmEnvProvider + Clone + 'static, + Client: BlockProvider + + HeaderProvider + + StateProviderFactory + + EvmEnvProvider + + Clone + + Unpin + + 'static, Pool: TransactionPool + Clone + 'static, Network: NetworkInfo + Peers + Clone + 'static, + Tasks: TaskSpawner + Clone + 'static, { /// Configures all [RpcModule]s specific to the given [TransportRpcModuleConfig] which can be /// used to start the transport server(s). @@ -175,9 +202,9 @@ where pub fn build(self, module_config: TransportRpcModuleConfig) -> TransportRpcModules<()> { let mut modules = TransportRpcModules::default(); - let Self { client, pool, network } = self; + let Self { client, pool, network, executor } = self; - let mut registry = RethModuleRegistry::new(client, pool, network); + let mut registry = RethModuleRegistry::new(client, pool, network, executor); if !module_config.is_empty() { let TransportRpcModuleConfig { http, ws, ipc } = module_config; @@ -190,9 +217,9 @@ where } } -impl Default for RpcModuleBuilder<(), (), ()> { +impl Default for RpcModuleBuilder<(), (), (), ()> { fn default() -> Self { - RpcModuleBuilder::new((), (), ()) + RpcModuleBuilder::new((), (), (), ()) } } @@ -258,11 +285,12 @@ impl RpcModuleSelection { /// Note: This will always create new instance of the module handlers and is therefor only /// recommended for launching standalone transports. If multiple transports need to be /// configured it's recommended to use the [RpcModuleBuilder]. - pub fn standalone_module( + pub fn standalone_module( &self, client: Client, pool: Pool, network: Network, + executor: Tasks, ) -> RpcModule<()> where Client: BlockProvider @@ -270,11 +298,13 @@ impl RpcModuleSelection { + StateProviderFactory + EvmEnvProvider + Clone + + Unpin + 'static, Pool: TransactionPool + Clone + 'static, Network: NetworkInfo + Peers + Clone + 'static, + Tasks: TaskSpawner + Clone + 'static, { - let mut registry = RethModuleRegistry::new(client, pool, network); + let mut registry = RethModuleRegistry::new(client, pool, network, executor); registry.module_for(self) } @@ -354,10 +384,13 @@ impl Serialize for RethRpcModule { } /// A Helper type the holds instances of the configured modules. -pub struct RethModuleRegistry { +pub struct RethModuleRegistry { client: Client, pool: Pool, network: Network, + executor: Tasks, + /// Holds a clone of the async [EthStateCache] channel. + eth_cache: Option, /// Holds a clone of the actual [EthApi] namespace impl since this can be required by other /// namespaces eth_api: Option>, @@ -367,10 +400,18 @@ pub struct RethModuleRegistry { // === impl RethModuleRegistry === -impl RethModuleRegistry { +impl RethModuleRegistry { /// Creates a new, empty instance. - pub fn new(client: Client, pool: Pool, network: Network) -> Self { - Self { client, pool, network, eth_api: None, modules: Default::default() } + pub fn new(client: Client, pool: Pool, network: Network, executor: Tasks) -> Self { + Self { + client, + pool, + network, + eth_api: None, + executor, + modules: Default::default(), + eth_cache: None, + } } /// Returns all installed methods @@ -388,7 +429,7 @@ impl RethModuleRegistry { } } -impl RethModuleRegistry +impl RethModuleRegistry where Network: NetworkInfo + Peers + Clone + 'static, { @@ -407,12 +448,18 @@ where } } -impl RethModuleRegistry +impl RethModuleRegistry where - Client: - BlockProvider + HeaderProvider + StateProviderFactory + EvmEnvProvider + Clone + 'static, + Client: BlockProvider + + HeaderProvider + + StateProviderFactory + + EvmEnvProvider + + Clone + + Unpin + + 'static, Pool: TransactionPool + Clone + 'static, Network: NetworkInfo + Peers + Clone + 'static, + Tasks: TaskSpawner + Clone + 'static, { /// Register Eth Namespace pub fn register_eth(&mut self) -> &mut Self { @@ -486,11 +533,28 @@ where .collect::>() } + /// Returns the [EthStateCache] frontend + /// + /// This will spawn exactly one [EthStateCache] service if this is the first time the cache is + /// requested. + pub fn eth_cache(&mut self) -> EthStateCache { + self.eth_cache + .get_or_insert_with(|| { + EthStateCache::spawn_with( + self.client.clone(), + Default::default(), + self.executor.clone(), + ) + }) + .clone() + } + /// Returns the configured [EthApi] or creates it if it does not exist yet fn eth_api(&mut self) -> EthApi { + let cache = self.eth_cache(); self.eth_api .get_or_insert_with(|| { - EthApi::new(self.client.clone(), self.pool.clone(), self.network.clone()) + EthApi::new(self.client.clone(), self.pool.clone(), self.network.clone(), cache) }) .clone() } diff --git a/crates/rpc/rpc-builder/tests/it/utils.rs b/crates/rpc/rpc-builder/tests/it/utils.rs index bc9f696591d..5e8e8527b0e 100644 --- a/crates/rpc/rpc-builder/tests/it/utils.rs +++ b/crates/rpc/rpc-builder/tests/it/utils.rs @@ -4,6 +4,7 @@ use reth_rpc_builder::{ RpcModuleBuilder, RpcModuleSelection, RpcServerConfig, RpcServerHandle, TransportRpcModuleConfig, }; +use reth_tasks::TokioTaskExecutor; use reth_transaction_pool::test_utils::{testing_pool, TestPool}; use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4}; @@ -50,9 +51,11 @@ pub async fn launch_http_ws(modules: impl Into) -> RpcServer } /// Returns an [RpcModuleBuilder] with testing components. -pub fn test_rpc_builder() -> RpcModuleBuilder { +pub fn test_rpc_builder() -> RpcModuleBuilder +{ RpcModuleBuilder::default() .with_client(NoopProvider::default()) .with_pool(testing_pool()) .with_network(NoopNetwork::default()) + .with_executor(TokioTaskExecutor::default()) } diff --git a/crates/rpc/rpc/src/eth/api/mod.rs b/crates/rpc/rpc/src/eth/api/mod.rs index 960c19d4b2f..fa3dfed1d83 100644 --- a/crates/rpc/rpc/src/eth/api/mod.rs +++ b/crates/rpc/rpc/src/eth/api/mod.rs @@ -13,7 +13,7 @@ use reth_primitives::{ use reth_provider::{BlockProvider, EvmEnvProvider, StateProviderFactory}; use std::num::NonZeroUsize; -use crate::eth::error::EthResult; +use crate::eth::{cache::EthStateCache, error::EthResult}; use reth_provider::providers::ChainState; use reth_rpc_types::FeeHistoryCache; use reth_transaction_pool::TransactionPool; @@ -67,8 +67,8 @@ pub struct EthApi { impl EthApi { /// Creates a new, shareable instance. - pub fn new(client: Client, pool: Pool, network: Network) -> Self { - let inner = EthApiInner { client, pool, network, signers: Default::default() }; + pub fn new(client: Client, pool: Pool, network: Network, eth_cache: EthStateCache) -> Self { + let inner = EthApiInner { client, pool, network, signers: Default::default(), eth_cache }; Self { inner: Arc::new(inner), fee_history_cache: FeeHistoryCache::new( @@ -216,4 +216,6 @@ struct EthApiInner { network: Network, /// All configured Signers signers: Vec>, + /// The async cache frontend for eth related data + eth_cache: EthStateCache, } diff --git a/crates/rpc/rpc/src/eth/api/server.rs b/crates/rpc/rpc/src/eth/api/server.rs index 1e382017837..ea8afb35564 100644 --- a/crates/rpc/rpc/src/eth/api/server.rs +++ b/crates/rpc/rpc/src/eth/api/server.rs @@ -378,6 +378,7 @@ where #[cfg(test)] mod tests { + use crate::eth::cache::EthStateCache; use jsonrpsee::{ core::{error::Error as RpcError, RpcResult}, types::error::{CallError, INVALID_PARAMS_CODE}, @@ -394,7 +395,12 @@ mod tests { #[tokio::test] /// Handler for: `eth_test_fee_history` async fn test_fee_history() { - let eth_api = EthApi::new(NoopProvider::default(), testing_pool(), NoopNetwork::default()); + let eth_api = EthApi::new( + NoopProvider::default(), + testing_pool(), + NoopNetwork::default(), + EthStateCache::spawn(NoopProvider::default(), Default::default()), + ); let response = eth_api.fee_history(1.into(), BlockNumberOrTag::Latest.into(), None).await; assert!(matches!(response, RpcResult::Err(RpcError::Call(CallError::Custom(_))))); @@ -434,7 +440,12 @@ mod tests { .push(base_fee_per_gas.map(|fee| U256::try_from(fee).unwrap()).unwrap_or_default()); } - let eth_api = EthApi::new(mock_provider, testing_pool(), NoopNetwork::default()); + let eth_api = EthApi::new( + mock_provider, + testing_pool(), + NoopNetwork::default(), + EthStateCache::spawn(NoopProvider::default(), Default::default()), + ); let response = eth_api.fee_history((newest_block + 1).into(), newest_block.into(), None).await; diff --git a/crates/rpc/rpc/src/eth/api/transactions.rs b/crates/rpc/rpc/src/eth/api/transactions.rs index e33e6459745..f861c0585b4 100644 --- a/crates/rpc/rpc/src/eth/api/transactions.rs +++ b/crates/rpc/rpc/src/eth/api/transactions.rs @@ -46,6 +46,7 @@ where #[cfg(test)] mod tests { + use crate::eth::cache::EthStateCache; use reth_primitives::{hex_literal::hex, Bytes}; use reth_provider::test_utils::NoopProvider; use reth_transaction_pool::{test_utils::testing_pool, TransactionPool}; @@ -58,7 +59,12 @@ mod tests { let pool = testing_pool(); - let eth_api = EthApi::new(noop_provider, pool.clone(), ()); + let eth_api = EthApi::new( + noop_provider, + pool.clone(), + (), + EthStateCache::spawn(NoopProvider::default(), Default::default()), + ); // https://etherscan.io/tx/0xa694b71e6c128a2ed8e2e0f6770bddbe52e3bb8f10e8472f9a79ab81497a8b5d let tx_1 = Bytes::from(hex!("02f871018303579880850555633d1b82520894eee27662c2b8eba3cd936a23f039f3189633e4c887ad591c62bdaeb180c080a07ea72c68abfb8fca1bd964f0f99132ed9280261bdca3e549546c0205e800f7d0a05b4ef3039e9c9b9babc179a1878fb825b5aaf5aed2fa8744854150157b08d6f3")); diff --git a/crates/rpc/rpc/src/eth/cache.rs b/crates/rpc/rpc/src/eth/cache.rs index edc7ed06091..adab25f93b2 100644 --- a/crates/rpc/rpc/src/eth/cache.rs +++ b/crates/rpc/rpc/src/eth/cache.rs @@ -4,9 +4,10 @@ use futures::StreamExt; use reth_interfaces::{provider::ProviderError, Result}; use reth_primitives::{Block, H256}; use reth_provider::{BlockProvider, EvmEnvProvider, StateProviderFactory}; -use reth_tasks::TaskSpawner; +use reth_tasks::{TaskSpawner, TokioTaskExecutor}; use revm::primitives::{BlockEnv, CfgEnv}; use schnellru::{ByMemoryUsage, Limiter, LruMap}; +use serde::{Deserialize, Serialize}; use std::{ collections::{hash_map::Entry, HashMap}, future::Future, @@ -30,23 +31,43 @@ type BlockLruCache = MultiConsumerLruCache = MultiConsumerLruCache; +/// Settings for the [EthStateCache] +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct EthStateCacheConfig { + /// Max number of bytes for cached block data. + /// + /// Default is 50MB + pub max_block_bytes: usize, + /// Max number of bytes for cached env data. + /// + /// Default is 500kb (env configs are very small) + pub max_env_bytes: usize, +} + +impl Default for EthStateCacheConfig { + fn default() -> Self { + Self { max_block_bytes: 50 * 1024 * 1024, max_env_bytes: 500 * 1024 } + } +} + /// Provides async access to cached eth data /// -/// This is the frontend to the [EthStateCacheService] which manages cached data on a different +/// This is the frontend for the async caching service which manages cached data on a different /// task. #[derive(Debug, Clone)] -pub(crate) struct EthStateCache { +pub struct EthStateCache { to_service: UnboundedSender, } impl EthStateCache { /// Creates and returns both [EthStateCache] frontend and the memory bound service. - fn create( + fn create( client: Client, - action_task_spawner: Box, + action_task_spawner: Tasks, max_block_bytes: usize, max_env_bytes: usize, - ) -> (Self, EthStateCacheService) { + ) -> (Self, EthStateCacheService) { let (to_service, rx) = unbounded_channel(); let service = EthStateCacheService { client, @@ -60,21 +81,34 @@ impl EthStateCache { (cache, service) } + /// Creates a new async LRU backed cache service task and spawns it to a new task via + /// [tokio::spawn]. + /// + /// See also [Self::spawn_with] + pub fn spawn(client: Client, config: EthStateCacheConfig) -> Self + where + Client: StateProviderFactory + BlockProvider + EvmEnvProvider + Clone + Unpin + 'static, + { + Self::spawn_with(client, config, TokioTaskExecutor::default()) + } + /// Creates a new async LRU backed cache service task and spawns it to a new task via the given /// spawner. /// /// The cache is memory limited by the given max bytes values. - pub(crate) fn spawn( + pub fn spawn_with( client: Client, - spawner: Box, - max_block_bytes: usize, - max_env_bytes: usize, + config: EthStateCacheConfig, + executor: Tasks, ) -> Self where Client: StateProviderFactory + BlockProvider + EvmEnvProvider + Clone + Unpin + 'static, + Tasks: TaskSpawner + Clone + 'static, { - let (this, service) = Self::create(client, spawner.clone(), max_block_bytes, max_env_bytes); - spawner.spawn(Box::pin(service)); + let EthStateCacheConfig { max_block_bytes, max_env_bytes } = config; + let (this, service) = + Self::create(client, executor.clone(), max_block_bytes, max_env_bytes); + executor.spawn(Box::pin(service)); this } @@ -107,7 +141,7 @@ impl EthStateCache { /// /// This type is an endless future that listens for incoming messages from the user facing /// [EthStateCache] via a channel. If the requested data is not cached then it spawns a new task -/// that does the IO and sends the result back to it. This way the [EthStateCacheService] only +/// that does the IO and sends the result back to it. This way the caching service only /// handles messages and does LRU lookups and never blocking IO. /// /// Caution: The channel for the data is _unbounded_ it is assumed that this is mainly used by the @@ -116,6 +150,7 @@ impl EthStateCache { #[must_use = "Type does nothing unless spawned"] pub(crate) struct EthStateCacheService< Client, + Tasks, LimitBlocks = ByMemoryUsage, LimitEnvs = ByMemoryUsage, > where @@ -133,12 +168,13 @@ pub(crate) struct EthStateCacheService< /// Receiver half of the action channel. action_rx: UnboundedReceiverStream, /// The type that's used to spawn tasks that do the actual work - action_task_spawner: Box, + action_task_spawner: Tasks, } -impl Future for EthStateCacheService +impl Future for EthStateCacheService where Client: StateProviderFactory + BlockProvider + EvmEnvProvider + Clone + Unpin + 'static, + Tasks: TaskSpawner + Clone + 'static, { type Output = (); diff --git a/crates/rpc/rpc/src/eth/mod.rs b/crates/rpc/rpc/src/eth/mod.rs index 0a79cc7630a..42500801a36 100644 --- a/crates/rpc/rpc/src/eth/mod.rs +++ b/crates/rpc/rpc/src/eth/mod.rs @@ -1,7 +1,7 @@ //! `eth` namespace handler implementation. mod api; -mod cache; +pub mod cache; pub(crate) mod error; mod filter; mod pubsub; diff --git a/crates/rpc/rpc/src/lib.rs b/crates/rpc/rpc/src/lib.rs index 2281d828e69..f5c60506da7 100644 --- a/crates/rpc/rpc/src/lib.rs +++ b/crates/rpc/rpc/src/lib.rs @@ -14,7 +14,7 @@ mod admin; mod debug; mod engine; -mod eth; +pub mod eth; mod layers; mod net; mod trace; diff --git a/crates/tasks/src/lib.rs b/crates/tasks/src/lib.rs index e9e93ac7092..3e408a1cc1f 100644 --- a/crates/tasks/src/lib.rs +++ b/crates/tasks/src/lib.rs @@ -66,7 +66,7 @@ pub mod shutdown; /// ``` /// /// The [TaskSpawner] trait is [DynClone] so `Box` are also `Clone`. -pub trait TaskSpawner: Send + Sync + std::fmt::Debug + DynClone { +pub trait TaskSpawner: Send + Sync + Unpin + std::fmt::Debug + DynClone { /// Spawns the task onto the runtime. /// See also [`Handle::spawn`]. fn spawn(&self, fut: BoxFuture<'static, ()>) -> JoinHandle<()>; From 9326b45821ef191d992e0eaf2f7ff17aa175aa07 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 2 Mar 2023 13:49:15 +0100 Subject: [PATCH 023/191] chore(deps): rm unused deps in primitives (#1604) --- Cargo.lock | 3 --- crates/primitives/Cargo.toml | 11 +++-------- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cfef28ee277..954fcb32cbd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4858,7 +4858,6 @@ dependencies = [ "crunchy", "derive_more", "ethers-core", - "eyre", "fixed-hash", "hash-db", "hex", @@ -4866,7 +4865,6 @@ dependencies = [ "impl-serde", "modular-bitfield", "once_cell", - "parity-scale-codec", "plain_hasher", "pprof", "proptest", @@ -4880,7 +4878,6 @@ dependencies = [ "serde", "serde_json", "serde_with", - "shellexpand", "strum", "sucds", "test-fuzz", diff --git a/crates/primitives/Cargo.toml b/crates/primitives/Cargo.toml index 91bd5718fef..5a872aeab4d 100644 --- a/crates/primitives/Cargo.toml +++ b/crates/primitives/Cargo.toml @@ -13,11 +13,10 @@ reth-rlp = { path = "../rlp", features = ["std", "derive", "ethereum-types"] } reth-rlp-derive = { path = "../rlp/rlp-derive" } reth-codecs = { version = "0.1.0", path = "../storage/codecs" } -revm-primitives = { version = "1.0.0", features = ["serde"] } +revm-primitives = { version = "1.0", features = ["serde"] } # ethereum ethers-core = { git = "https://github.com/gakonst/ethers-rs", default-features = false } -parity-scale-codec = { version = "3.2.1", features = ["derive", "bytes"] } tiny-keccak = { version = "2.0", features = ["keccak"] } crunchy = { version = "0.2.2", default-features = false, features = ["limit_256"] } @@ -57,10 +56,6 @@ triehash = "0.8" plain_hasher = "0.2" hash-db = "0.15" -# used for clap value parser -eyre = "0.6" -shellexpand = "3.0" - # arbitrary utils arbitrary = { version = "1.1.7", features = ["derive"], optional = true } proptest = { version = "1.0", optional = true } @@ -72,7 +67,7 @@ serde_json = "1.0" hex-literal = "0.3" test-fuzz = "3.0.4" rand = "0.8" -revm-primitives = "1.0.0" +revm-primitives = { version = "1.0", features = ["arbitrary"] } arbitrary = { version = "1.1.7", features = ["derive"] } proptest = { version = "1.0" } proptest-derive = "0.3" @@ -98,4 +93,4 @@ arbitrary = [ [[bench]] name = "recover_ecdsa_crit" -harness = false +harness = false \ No newline at end of file From 6501ce29560b6277dc596c7f77f618fbfeda6819 Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Thu, 2 Mar 2023 16:29:00 +0200 Subject: [PATCH 024/191] chore: expose methods and structs (#1607) --- bin/reth/src/cli.rs | 10 ++++++---- bin/reth/src/dirs.rs | 3 ++- crates/stages/src/stages/headers.rs | 11 +++++++---- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/bin/reth/src/cli.rs b/bin/reth/src/cli.rs index e984a0b9371..9f6a5e46335 100644 --- a/bin/reth/src/cli.rs +++ b/bin/reth/src/cli.rs @@ -88,9 +88,10 @@ struct Cli { verbosity: Verbosity, } +/// The log configuration. #[derive(Args)] #[command(next_help_heading = "Logging")] -struct Logs { +pub struct Logs { /// The path to put log files in. #[arg( long = "log.directory", @@ -112,7 +113,7 @@ struct Logs { impl Logs { /// Builds a tracing layer from the current log options. - fn layer(&self) -> (BoxedLayer, Option) + pub fn layer(&self) -> (BoxedLayer, Option) where S: Subscriber, for<'a> S: LookupSpan<'a>, @@ -129,9 +130,10 @@ impl Logs { } } +/// The verbosity settings for the cli. #[derive(Args)] #[command(next_help_heading = "Display")] -struct Verbosity { +pub struct Verbosity { /// Set the minimum log level. /// /// -v Errors @@ -150,7 +152,7 @@ struct Verbosity { impl Verbosity { /// Get the corresponding [Directive] for the given verbosity, or none if the verbosity /// corresponds to silent. - fn directive(&self) -> Directive { + pub fn directive(&self) -> Directive { if self.quiet { LevelFilter::OFF.into() } else { diff --git a/bin/reth/src/dirs.rs b/bin/reth/src/dirs.rs index b1a8aa575c5..c5ac09bbc0d 100644 --- a/bin/reth/src/dirs.rs +++ b/bin/reth/src/dirs.rs @@ -116,7 +116,8 @@ impl XdgPath for LogsDir { /// A small helper trait for unit structs that represent a standard path following the XDG /// path specification. -trait XdgPath { +pub trait XdgPath { + /// Resolve the standard path. fn resolve() -> Option; } diff --git a/crates/stages/src/stages/headers.rs b/crates/stages/src/stages/headers.rs index ef258ce2cf1..b656652131e 100644 --- a/crates/stages/src/stages/headers.rs +++ b/crates/stages/src/stages/headers.rs @@ -236,9 +236,12 @@ where /// Represents a gap to sync: from `local_head` to `target` #[derive(Debug)] -struct SyncGap { - local_head: SealedHeader, - target: SyncTarget, +pub struct SyncGap { + /// The local head block. Represents lower bound of sync range. + pub local_head: SealedHeader, + + /// The sync target. Represents upper bound of sync range. + pub target: SyncTarget, } // === impl SyncGap === @@ -246,7 +249,7 @@ struct SyncGap { impl SyncGap { /// Returns `true` if the gap from the head to the target was closed #[inline] - fn is_closed(&self) -> bool { + pub fn is_closed(&self) -> bool { self.local_head.hash() == self.target.tip() } } From 5fbd3f0c9bcd79add3a58c4a51926cb3e520be11 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 2 Mar 2023 15:39:00 +0100 Subject: [PATCH 025/191] feat(rpc): Add RpcModuleConfig and integrate in builder (#1605) --- crates/rpc/rpc-builder/src/eth.rs | 25 ++++++ crates/rpc/rpc-builder/src/lib.rs | 144 ++++++++++++++++++++++-------- crates/rpc/rpc/src/eth/api/mod.rs | 7 +- crates/rpc/rpc/src/eth/cache.rs | 2 +- crates/rpc/rpc/src/eth/pubsub.rs | 3 +- 5 files changed, 141 insertions(+), 40 deletions(-) create mode 100644 crates/rpc/rpc-builder/src/eth.rs diff --git a/crates/rpc/rpc-builder/src/eth.rs b/crates/rpc/rpc-builder/src/eth.rs new file mode 100644 index 00000000000..45c136e91b8 --- /dev/null +++ b/crates/rpc/rpc-builder/src/eth.rs @@ -0,0 +1,25 @@ +use reth_rpc::{ + eth::cache::{EthStateCache, EthStateCacheConfig}, + EthApi, EthFilter, EthPubSub, +}; +use serde::{Deserialize, Serialize}; + +/// All handlers for the `eth` namespace +#[derive(Debug, Clone)] +pub struct EthHandlers { + /// Main `eth_` request handler + pub api: EthApi, + /// The async caching layer used by the eth handlers + pub eth_cache: EthStateCache, + /// Polling based filter handler available on all transports + pub filter: EthFilter, + /// Handler for subscriptions only available for transports that support it (ws, ipc) + pub pubsub: Option>, +} + +/// Additional config values for the eth namespace +#[derive(Debug, Clone, Eq, PartialEq, Default, Serialize, Deserialize)] +pub struct EthConfig { + /// Settings for the caching layer + pub cache: EthStateCacheConfig, +} diff --git a/crates/rpc/rpc-builder/src/lib.rs b/crates/rpc/rpc-builder/src/lib.rs index 2f9b16dbc41..c09caf4ca64 100644 --- a/crates/rpc/rpc-builder/src/lib.rs +++ b/crates/rpc/rpc-builder/src/lib.rs @@ -64,7 +64,7 @@ use jsonrpsee::{ use reth_ipc::server::IpcServer; use reth_network_api::{NetworkInfo, Peers}; use reth_provider::{BlockProvider, EvmEnvProvider, HeaderProvider, StateProviderFactory}; -use reth_rpc::{AdminApi, DebugApi, EthApi, NetApi, TraceApi, Web3Api}; +use reth_rpc::{AdminApi, DebugApi, EthApi, EthFilter, NetApi, TraceApi, Web3Api}; use reth_rpc_api::servers::*; use reth_transaction_pool::TransactionPool; use serde::{Deserialize, Serialize, Serializer}; @@ -84,8 +84,12 @@ pub use reth_ipc::server::{Builder as IpcServerBuilder, Endpoint}; /// Auth server utilities. pub mod auth; +/// Eth utils +mod eth; + /// Common RPC constants. pub mod constants; +pub use crate::eth::{EthConfig, EthHandlers}; use constants::*; use reth_rpc::eth::cache::EthStateCache; use reth_tasks::TaskSpawner; @@ -204,10 +208,17 @@ where let Self { client, pool, network, executor } = self; - let mut registry = RethModuleRegistry::new(client, pool, network, executor); - if !module_config.is_empty() { - let TransportRpcModuleConfig { http, ws, ipc } = module_config; + let TransportRpcModuleConfig { http, ws, ipc, config } = module_config; + + let mut registry = RethModuleRegistry::new( + client, + pool, + network, + executor, + config.unwrap_or_default(), + ); + modules.http = registry.maybe_module(http.as_ref()); modules.ws = registry.maybe_module(ws.as_ref()); modules.ipc = registry.maybe_module(ipc.as_ref()); @@ -223,6 +234,44 @@ impl Default for RpcModuleBuilder<(), (), (), ()> { } } +/// Bundles settings for modules +#[derive(Debug, Default, Clone, Eq, PartialEq, Serialize, Deserialize)] +pub struct RpcModuleConfig { + /// `eth` namespace settings + eth: EthConfig, +} + +// === impl RpcModuleConfig === + +impl RpcModuleConfig { + /// Convenience method to create a new [RpcModuleConfigBuilder] + pub fn builder() -> RpcModuleConfigBuilder { + RpcModuleConfigBuilder::default() + } +} + +/// Configures [RpcModuleConfig] +#[derive(Default)] +pub struct RpcModuleConfigBuilder { + eth: Option, +} + +// === impl RpcModuleConfigBuilder === + +impl RpcModuleConfigBuilder { + /// Configures a custom eth namespace config + pub fn eth(mut self, eth: EthConfig) -> Self { + self.eth = Some(eth); + self + } + + /// Consumes the type and creates the [RpcModuleConfig] + pub fn build(self) -> RpcModuleConfig { + let RpcModuleConfigBuilder { eth } = self; + RpcModuleConfig { eth: eth.unwrap_or_default() } + } +} + /// Describes the modules that should be installed. /// /// # Example @@ -244,7 +293,7 @@ pub enum RpcModuleSelection { Selection(Vec), } -// === impl RpcModuleConfig === +// === impl RpcModuleSelection === impl RpcModuleSelection { /// The standard modules to instantiate by default `eth`, `net`, `web3` @@ -291,6 +340,7 @@ impl RpcModuleSelection { pool: Pool, network: Network, executor: Tasks, + config: RpcModuleConfig, ) -> RpcModule<()> where Client: BlockProvider @@ -304,7 +354,7 @@ impl RpcModuleSelection { Network: NetworkInfo + Peers + Clone + 'static, Tasks: TaskSpawner + Clone + 'static, { - let mut registry = RethModuleRegistry::new(client, pool, network, executor); + let mut registry = RethModuleRegistry::new(client, pool, network, executor, config); registry.module_for(self) } @@ -389,11 +439,10 @@ pub struct RethModuleRegistry { pool: Pool, network: Network, executor: Tasks, - /// Holds a clone of the async [EthStateCache] channel. - eth_cache: Option, - /// Holds a clone of the actual [EthApi] namespace impl since this can be required by other - /// namespaces - eth_api: Option>, + /// Additional settings for handlers. + config: RpcModuleConfig, + /// Holds a clone of all the eth namespace handlers + eth: Option>, /// Contains the [Methods] of a module modules: HashMap, } @@ -402,16 +451,14 @@ pub struct RethModuleRegistry { impl RethModuleRegistry { /// Creates a new, empty instance. - pub fn new(client: Client, pool: Pool, network: Network, executor: Tasks) -> Self { - Self { - client, - pool, - network, - eth_api: None, - executor, - modules: Default::default(), - eth_cache: None, - } + pub fn new( + client: Client, + pool: Pool, + network: Network, + executor: Tasks, + config: RpcModuleConfig, + ) -> Self { + Self { client, pool, network, eth: None, executor, modules: Default::default(), config } } /// Returns all installed methods @@ -538,25 +585,39 @@ where /// This will spawn exactly one [EthStateCache] service if this is the first time the cache is /// requested. pub fn eth_cache(&mut self) -> EthStateCache { - self.eth_cache - .get_or_insert_with(|| { - EthStateCache::spawn_with( - self.client.clone(), - Default::default(), - self.executor.clone(), - ) - }) - .clone() + self.with_eth(|handlers| handlers.eth_cache.clone()) + } + + /// Creates the [EthHandlers] type the first time this is called. + fn with_eth(&mut self, f: F) -> R + where + F: FnOnce(&EthHandlers) -> R, + { + if self.eth.is_none() { + let eth_cache = EthStateCache::spawn_with( + self.client.clone(), + self.config.eth.cache.clone(), + self.executor.clone(), + ); + let api = EthApi::new( + self.client.clone(), + self.pool.clone(), + self.network.clone(), + eth_cache.clone(), + ); + let filter = EthFilter::new(self.client.clone(), self.pool.clone()); + + // TODO: install pubsub + + let eth = EthHandlers { api, eth_cache, filter, pubsub: None }; + self.eth = Some(eth); + } + f(self.eth.as_ref().expect("exists; qed")) } /// Returns the configured [EthApi] or creates it if it does not exist yet fn eth_api(&mut self) -> EthApi { - let cache = self.eth_cache(); - self.eth_api - .get_or_insert_with(|| { - EthApi::new(self.client.clone(), self.pool.clone(), self.network.clone(), cache) - }) - .clone() + self.with_eth(|handlers| handlers.api.clone()) } } @@ -731,7 +792,7 @@ impl RpcServerConfig { /// /// # Example /// -/// Configure an http transport only +/// Configure a http transport only /// /// ``` /// use reth_rpc_builder::{RethRpcModule, TransportRpcModuleConfig}; @@ -746,6 +807,8 @@ pub struct TransportRpcModuleConfig { ws: Option, /// ipc module configuration ipc: Option, + /// Config for the modules + config: Option, } // === impl TransportRpcModuleConfig === @@ -784,6 +847,12 @@ impl TransportRpcModuleConfig { self } + /// Sets a custom [RpcModuleConfig] for the configured modules. + pub fn with_config(mut self, config: RpcModuleConfig) -> Self { + self.config = Some(config); + self + } + /// Returns true if no transports are configured pub fn is_empty(&self) -> bool { self.http.is_none() && self.ws.is_none() && self.ipc.is_none() @@ -1058,6 +1127,7 @@ mod tests { ])), ws: None, ipc: None, + config: None, } ) } diff --git a/crates/rpc/rpc/src/eth/api/mod.rs b/crates/rpc/rpc/src/eth/api/mod.rs index fa3dfed1d83..3e4d14063de 100644 --- a/crates/rpc/rpc/src/eth/api/mod.rs +++ b/crates/rpc/rpc/src/eth/api/mod.rs @@ -58,7 +58,6 @@ pub trait EthApiSpec: Send + Sync { /// the main impls. This way [`EthApi`] is not limited to [`jsonrpsee`] and can be used standalone /// or in other network handlers (for example ipc). #[derive(Clone)] -#[allow(missing_debug_implementations)] pub struct EthApi { /// All nested fields bundled together. inner: Arc>, @@ -172,6 +171,12 @@ where } } +impl std::fmt::Debug for EthApi { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("EthApi").finish_non_exhaustive() + } +} + #[async_trait] impl EthApiSpec for EthApi where diff --git a/crates/rpc/rpc/src/eth/cache.rs b/crates/rpc/rpc/src/eth/cache.rs index adab25f93b2..f0f543d02df 100644 --- a/crates/rpc/rpc/src/eth/cache.rs +++ b/crates/rpc/rpc/src/eth/cache.rs @@ -32,7 +32,7 @@ type BlockLruCache = MultiConsumerLruCache = MultiConsumerLruCache; /// Settings for the [EthStateCache] -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct EthStateCacheConfig { /// Max number of bytes for cached block data. diff --git a/crates/rpc/rpc/src/eth/pubsub.rs b/crates/rpc/rpc/src/eth/pubsub.rs index e4a9a5f145c..57f66f974f9 100644 --- a/crates/rpc/rpc/src/eth/pubsub.rs +++ b/crates/rpc/rpc/src/eth/pubsub.rs @@ -18,7 +18,8 @@ use tokio_stream::{ /// `Eth` pubsub RPC implementation. /// -/// This handles +/// This handles `eth_subscribe` RPC calls. +#[derive(Clone)] pub struct EthPubSub { /// All nested fields bundled together. inner: EthPubSubInner, From 5720ede5500d1c0d2813581f19e973872d1a2e9d Mon Sep 17 00:00:00 2001 From: Bjerg Date: Thu, 2 Mar 2023 15:46:00 +0100 Subject: [PATCH 026/191] feat: db list paging (#1542) --- bin/reth/src/db/mod.rs | 6 +- bin/reth/src/db/tui.rs | 291 +++++++++++++++++++++++++++++++---------- 2 files changed, 224 insertions(+), 73 deletions(-) diff --git a/bin/reth/src/db/mod.rs b/bin/reth/src/db/mod.rs index 6a9ccd48f41..fb17e8cc829 100644 --- a/bin/reth/src/db/mod.rs +++ b/bin/reth/src/db/mod.rs @@ -151,8 +151,10 @@ impl Command { ); return Ok(()); } - let map = tool.list::($start, $len)?; - tui::DbListTUI::::show_tui(map, $start, total_entries) + + tui::DbListTUI::<_, tables::$table>::new(|start, count| { + tool.list::(start, count).unwrap() + }, $start, $len, total_entries).run() })?? },)* _ => { diff --git a/bin/reth/src/db/tui.rs b/bin/reth/src/db/tui.rs index 1121d1af683..c4ceaf9e868 100644 --- a/bin/reth/src/db/tui.rs +++ b/bin/reth/src/db/tui.rs @@ -19,7 +19,14 @@ use tui::{ }; /// Available keybindings for the [DbListTUI] -static CMDS: [(&str, &str); 3] = [("q", "Quit"), ("up", "Entry Above"), ("down", "Entry Below")]; +static CMDS: [(&str, &str); 6] = [ + ("q", "Quit"), + ("↑", "Entry above"), + ("↓", "Entry below"), + ("←", "Previous page"), + ("→", "Next page"), + ("G", "Go to a specific page"), +]; /// Modified version of the [ListState] struct that exposes the `offset` field. /// Used to make the [DbListTUI] keys clickable. @@ -27,26 +34,62 @@ struct ExpListState { pub(crate) offset: usize, } +#[derive(Default, Eq, PartialEq)] +pub(crate) enum ViewMode { + /// Normal list view mode + #[default] + Normal, + /// Currently wanting to go to a page + GoToPage, +} + #[derive(Default)] -pub(crate) struct DbListTUI { - /// The state of the key list. - pub(crate) state: ListState, +pub(crate) struct DbListTUI +where + F: FnMut(usize, usize) -> BTreeMap, +{ + /// Fetcher for the next page of items. + /// + /// The fetcher is passed the index of the first item to fetch, and the number of items to + /// fetch from that item. + fetch: F, /// The starting index of the key list in the DB. - pub(crate) start: usize, + start: usize, + /// The amount of entries to show per page + count: usize, /// The total number of entries in the database - pub(crate) total_entries: usize, + total_entries: usize, + /// The current view mode + mode: ViewMode, + /// The current state of the input buffer + input: String, + /// The state of the key list. + list_state: ListState, /// Entries to show in the TUI. - pub(crate) entries: BTreeMap, + entries: BTreeMap, } -impl DbListTUI { - fn new(entries: BTreeMap, start: usize, total_entries: usize) -> Self { - Self { state: ListState::default(), start, total_entries, entries } +impl DbListTUI +where + F: FnMut(usize, usize) -> BTreeMap, +{ + /// Create a new database list TUI + pub(crate) fn new(fetch: F, start: usize, count: usize, total_entries: usize) -> Self { + Self { + fetch, + start, + count, + total_entries, + mode: ViewMode::Normal, + input: String::new(), + list_state: ListState::default(), + entries: BTreeMap::new(), + } } /// Move to the next list selection fn next(&mut self) { - let i = match self.state.selected() { + let i = match self.list_state.selected() { Some(i) => { if i >= self.entries.len() - 1 { 0 @@ -56,12 +99,12 @@ impl DbListTUI { } None => 0, }; - self.state.select(Some(i)); + self.list_state.select(Some(i)); } /// Move to the previous list selection fn previous(&mut self) { - let i = match self.state.selected() { + let i = match self.list_state.selected() { Some(i) => { if i == 0 { self.entries.len() - 1 @@ -71,89 +114,177 @@ impl DbListTUI { } None => 0, }; - self.state.select(Some(i)); + self.list_state.select(Some(i)); + } + + fn reset(&mut self) { + self.list_state.select(Some(0)); + } + + /// Fetch the next page of items + fn next_page(&mut self) { + if self.start + self.count >= self.total_entries { + return + } + + self.start += self.count; + self.fetch_page(); + } + + /// Fetch the previous page of items + fn previous_page(&mut self) { + if self.start == 0 { + return + } + + self.start -= self.count; + self.fetch_page(); + } + + /// Go to a specific page. + fn go_to_page(&mut self, page: usize) { + self.start = (self.count * page).min(self.total_entries - self.count); + self.fetch_page(); + } + + /// Fetch the current page + fn fetch_page(&mut self) { + self.entries = (self.fetch)(self.start, self.count); + self.reset(); } /// Show the [DbListTUI] in the terminal. - pub(crate) fn show_tui( - entries: BTreeMap, - start: usize, - total_entries: usize, - ) -> eyre::Result<()> { - // setup terminal + pub(crate) fn run(mut self) -> eyre::Result<()> { + // Setup backend enable_raw_mode()?; let mut stdout = io::stdout(); execute!(stdout, EnterAlternateScreen, EnableMouseCapture)?; let backend = CrosstermBackend::new(stdout); let mut terminal = Terminal::new(backend)?; - // create app and run it + // Load initial page + self.fetch_page(); + + // Run event loop let tick_rate = Duration::from_millis(250); - let mut app = DbListTUI::::new(entries, start, total_entries); - app.state.select(Some(0)); - let res = run(&mut terminal, app, tick_rate); + let res = event_loop(&mut terminal, &mut self, tick_rate); - // restore terminal + // Restore terminal disable_raw_mode()?; execute!(terminal.backend_mut(), LeaveAlternateScreen, DisableMouseCapture)?; terminal.show_cursor()?; + // Handle errors if let Err(err) = res { error!("{:?}", err) } - Ok(()) } } -fn run( +/// Run the event loop +fn event_loop( terminal: &mut Terminal, - mut app: DbListTUI, + app: &mut DbListTUI, tick_rate: Duration, -) -> io::Result<()> { +) -> io::Result<()> +where + F: FnMut(usize, usize) -> BTreeMap, +{ let mut last_tick = Instant::now(); - loop { - terminal.draw(|f| ui(f, &mut app))?; + let mut running = true; + while running { + // Render + terminal.draw(|f| ui(f, app))?; + // Calculate timeout let timeout = tick_rate.checked_sub(last_tick.elapsed()).unwrap_or_else(|| Duration::from_secs(0)); + + // Poll events if crossterm::event::poll(timeout)? { - match event::read()? { - Event::Key(key) => match key.code { - KeyCode::Char('q') | KeyCode::Char('Q') => return Ok(()), - KeyCode::Down => app.next(), - KeyCode::Up => app.previous(), - _ => {} - }, - Event::Mouse(e) => match e.kind { - MouseEventKind::ScrollDown => app.next(), - MouseEventKind::ScrollUp => app.previous(), - // TODO: This click event can be triggered outside of the list widget. - MouseEventKind::Down(_) => { - // SAFETY: The pointer to the app's state will always be valid for - // reads here, and the source is larger than the destination. - // - // This is technically unsafe, but because the alignment requirements - // in both the source and destination are the same and we can ensure - // that the pointer to `app.state` is valid for reads, this is safe. - let state: ExpListState = unsafe { std::mem::transmute_copy(&app.state) }; - let new_idx = (e.row as usize + state.offset).saturating_sub(1); - if new_idx < app.entries.len() { - app.state.select(Some(new_idx)); - } - } - _ => {} - }, - _ => {} - } + running = !handle_event(app, event::read()?)?; } + if last_tick.elapsed() >= tick_rate { last_tick = Instant::now(); } } + + Ok(()) +} + +/// Handle incoming events +fn handle_event(app: &mut DbListTUI, event: Event) -> io::Result +where + F: FnMut(usize, usize) -> BTreeMap, +{ + if app.mode == ViewMode::GoToPage { + if let Event::Key(key) = event { + match key.code { + KeyCode::Enter => { + let input = std::mem::take(&mut app.input); + if let Ok(page) = input.parse() { + app.go_to_page(page); + } + app.mode = ViewMode::Normal; + } + KeyCode::Char(c) => { + app.input.push(c); + } + KeyCode::Backspace => { + app.input.pop(); + } + KeyCode::Esc => app.mode = ViewMode::Normal, + _ => {} + } + } + + return Ok(false) + } + + match event { + Event::Key(key) => match key.code { + KeyCode::Char('q') | KeyCode::Char('Q') => return Ok(true), + KeyCode::Down => app.next(), + KeyCode::Up => app.previous(), + KeyCode::Right => app.next_page(), + KeyCode::Left => app.previous_page(), + KeyCode::Char('G') => { + app.mode = ViewMode::GoToPage; + } + _ => {} + }, + Event::Mouse(e) => match e.kind { + MouseEventKind::ScrollDown => app.next(), + MouseEventKind::ScrollUp => app.previous(), + // TODO: This click event can be triggered outside of the list widget. + MouseEventKind::Down(_) => { + // SAFETY: The pointer to the app's state will always be valid for + // reads here, and the source is larger than the destination. + // + // This is technically unsafe, but because the alignment requirements + // in both the source and destination are the same and we can ensure + // that the pointer to `app.state` is valid for reads, this is safe. + let state: ExpListState = unsafe { std::mem::transmute_copy(&app.list_state) }; + let new_idx = (e.row as usize + state.offset).saturating_sub(1); + if new_idx < app.entries.len() { + app.list_state.select(Some(new_idx)); + } + } + _ => {} + }, + _ => {} + } + + Ok(false) } -fn ui(f: &mut Frame<'_, B>, app: &mut DbListTUI) { +/// Render the UI +fn ui(f: &mut Frame<'_, B>, app: &mut DbListTUI) +where + F: FnMut(usize, usize) -> BTreeMap, +{ let outer_chunks = Layout::default() .direction(Direction::Vertical) .constraints([Constraint::Percentage(95), Constraint::Percentage(5)].as_ref()) @@ -166,16 +297,19 @@ fn ui(f: &mut Frame<'_, B>, app: &mut DbListTUI) { .constraints([Constraint::Percentage(50), Constraint::Percentage(50)]) .split(outer_chunks[0]); + let key_length = format!("{}", app.start + app.count - 1).len(); let formatted_keys = app .entries .keys() .enumerate() - .map(|(i, k)| ListItem::new(format!("[{}] - {k:?}", i + app.start))) + .map(|(i, k)| { + ListItem::new(format!("[{:0>width$}]: {k:?}", i + app.start, width = key_length)) + }) .collect::>>(); let key_list = List::new(formatted_keys) .block(Block::default().borders(Borders::ALL).title(format!( - "Keys (Showing range [{}, {}] out of {} entries)", + "Keys (Showing entries {}-{} out of {} entries)", app.start, app.start + app.entries.len() - 1, app.total_entries @@ -184,13 +318,18 @@ fn ui(f: &mut Frame<'_, B>, app: &mut DbListTUI) { .highlight_style(Style::default().fg(Color::Cyan).add_modifier(Modifier::ITALIC)) .highlight_symbol("➜ ") .start_corner(Corner::TopLeft); - f.render_stateful_widget(key_list, inner_chunks[0], &mut app.state); + f.render_stateful_widget(key_list, inner_chunks[0], &mut app.list_state); + let values = app.entries.values().collect::>(); let value_display = Paragraph::new( - serde_json::to_string_pretty( - &app.entries.values().collect::>()[app.state.selected().unwrap_or(0)], - ) - .unwrap_or_else(|_| String::from("Error serializing value!")), + app.list_state + .selected() + .and_then(|selected| values.get(selected)) + .map(|entry| { + serde_json::to_string_pretty(entry) + .unwrap_or(String::from("Error serializing value")) + }) + .unwrap_or("No value selected".to_string()), ) .block(Block::default().borders(Borders::ALL).title("Value (JSON)")) .wrap(Wrap { trim: false }) @@ -199,11 +338,21 @@ fn ui(f: &mut Frame<'_, B>, app: &mut DbListTUI) { } // Footer - let footer = Paragraph::new( - CMDS.iter().map(|(k, v)| format!("[{k}] {v}")).collect::>().join(" | "), - ) + let footer = match app.mode { + ViewMode::Normal => Paragraph::new( + CMDS.iter().map(|(k, v)| format!("[{k}] {v}")).collect::>().join(" | "), + ), + ViewMode::GoToPage => Paragraph::new(format!( + "Go to page (max {}): {}", + app.total_entries / app.count, + app.input + )), + } .block(Block::default().borders(Borders::ALL)) - .alignment(Alignment::Center) + .alignment(match app.mode { + ViewMode::Normal => Alignment::Center, + ViewMode::GoToPage => Alignment::Left, + }) .style(Style::default().fg(Color::Cyan).add_modifier(Modifier::BOLD)); f.render_widget(footer, outer_chunks[1]); } From e9c2e884a48770fad5e0a9ff959cda41c21ecca2 Mon Sep 17 00:00:00 2001 From: Bjerg Date: Thu, 2 Mar 2023 16:08:40 +0100 Subject: [PATCH 027/191] chore: `get_evm_evn` -> `get_evm_env` (#1610) --- crates/rpc/rpc/src/eth/cache.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/rpc/rpc/src/eth/cache.rs b/crates/rpc/rpc/src/eth/cache.rs index f0f543d02df..3990e9dc35e 100644 --- a/crates/rpc/rpc/src/eth/cache.rs +++ b/crates/rpc/rpc/src/eth/cache.rs @@ -125,7 +125,7 @@ impl EthStateCache { /// /// Returns an error if the corresponding header (required for populating the envs) was not /// found. - pub(crate) async fn get_evm_evn(&self, block_hash: H256) -> Result<(CfgEnv, BlockEnv)> { + pub(crate) async fn get_evm_env(&self, block_hash: H256) -> Result<(CfgEnv, BlockEnv)> { let (response_tx, rx) = oneshot::channel(); let _ = self.to_service.send(CacheAction::GetEnv { block_hash, response_tx }); rx.await.map_err(|_| ProviderError::CacheServiceUnavailable)? From f89b504245af3c6daaa7f3c8726ea229392c64e9 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 2 Mar 2023 17:07:36 +0100 Subject: [PATCH 028/191] feat(rpc): impl eth_estimate (#1599) --- Cargo.lock | 2 + crates/primitives/src/constants.rs | 3 + crates/rpc/rpc/Cargo.toml | 2 + crates/rpc/rpc/src/eth/api/call.rs | 186 +++++++++++++++++++++++++++-- crates/rpc/rpc/src/eth/api/mod.rs | 5 + crates/rpc/rpc/src/eth/error.rs | 80 ++++++++++++- 6 files changed, 262 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 954fcb32cbd..060f8738579 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4973,6 +4973,8 @@ name = "reth-rpc" version = "0.1.0" dependencies = [ "async-trait", + "bytes", + "ethers-core", "futures", "hex", "http", diff --git a/crates/primitives/src/constants.rs b/crates/primitives/src/constants.rs index 4739074f827..c38819ae35a 100644 --- a/crates/primitives/src/constants.rs +++ b/crates/primitives/src/constants.rs @@ -3,6 +3,9 @@ use crate::H256; use hex_literal::hex; +/// The first four bytes of the call data for a function call specifies the function to be called. +pub const SELECTOR_LEN: usize = 4; + /// Initial base fee as defined in [EIP-1559](https://eips.ethereum.org/EIPS/eip-1559) pub const EIP1559_INITIAL_BASE_FEE: u64 = 1_000_000_000; diff --git a/crates/rpc/rpc/Cargo.toml b/crates/rpc/rpc/Cargo.toml index 9e1a9a6126a..34e20387a12 100644 --- a/crates/rpc/rpc/Cargo.toml +++ b/crates/rpc/rpc/Cargo.toml @@ -24,6 +24,7 @@ reth-tasks = { path = "../../tasks" } # eth revm = { version = "3.0.0", features = ["optional_block_gas_limit"] } +ethers-core = { git = "https://github.com/gakonst/ethers-rs" } # rpc jsonrpsee = { version = "0.16" } @@ -39,6 +40,7 @@ tower = "0.4" tokio-stream = "0.1" pin-project = "1.0" +bytes = "1.4" secp256k1 = { version = "0.26.0", features = [ "global-context", "rand-std", diff --git a/crates/rpc/rpc/src/eth/api/call.rs b/crates/rpc/rpc/src/eth/api/call.rs index 60cb62bd66a..46d4cca807f 100644 --- a/crates/rpc/rpc/src/eth/api/call.rs +++ b/crates/rpc/rpc/src/eth/api/call.rs @@ -3,20 +3,25 @@ #![allow(unused)] // TODO rm later use crate::{ - eth::error::{EthApiError, EthResult, InvalidTransactionError}, + eth::error::{EthApiError, EthResult, InvalidTransactionError, RevertError}, EthApi, }; -use reth_primitives::{AccessList, Address, BlockId, Bytes, TransactionKind, U128, U256}; +use reth_primitives::{ + AccessList, Address, BlockId, BlockNumberOrTag, Bytes, TransactionKind, U128, U256, +}; use reth_provider::{BlockProvider, EvmEnvProvider, StateProvider, StateProviderFactory}; use reth_revm::database::{State, SubState}; use reth_rpc_types::CallRequest; use revm::{ - primitives::{ruint::Uint, BlockEnv, CfgEnv, Env, ResultAndState, TransactTo, TxEnv}, + primitives::{ + ruint::Uint, BlockEnv, CfgEnv, Env, ExecutionResult, Halt, ResultAndState, TransactTo, + TxEnv, + }, Database, }; // Gas per transaction not creating a contract. -pub(crate) const MIN_TRANSACTION_GAS: U256 = Uint::from_limbs([21_000, 0, 0, 0]); +const MIN_TRANSACTION_GAS: u64 = 21_000u64; impl EthApi where @@ -53,13 +58,34 @@ where transact(&mut db, env) } - /// Estimate gas needed for execution of the `request` at the [BlockId] . - pub(crate) fn estimate_gas_at(&self, mut request: CallRequest, at: BlockId) -> EthResult { - // TODO get a StateProvider for the given blockId and BlockEnv - todo!() + /// Estimate gas needed for execution of the `request` at the [BlockId]. + pub(crate) async fn estimate_gas_at( + &self, + request: CallRequest, + at: BlockId, + ) -> EthResult { + // TODO handle Pending state's env + let (cfg, block_env) = match at { + BlockId::Number(BlockNumberOrTag::Pending) => { + // This should perhaps use the latest env settings and update block specific + // settings like basefee/number + unimplemented!("support pending state env") + } + hash_or_num => { + let block_hash = self + .client() + .block_hash_for_id(hash_or_num)? + .ok_or_else(|| EthApiError::UnknownBlockNumber)?; + self.cache().get_evm_env(block_hash).await? + } + }; + let state = self.state_at_block_id(at)?.ok_or_else(|| EthApiError::UnknownBlockNumber)?; + self.estimate_gas_with(cfg, block_env, request, state) } /// Estimates the gas usage of the `request` with the state. + /// + /// This will execute the [CallRequest] and find the best gas limit via binary search fn estimate_gas_with( &self, cfg: CfgEnv, @@ -70,26 +96,160 @@ where where S: StateProvider, { + // keep a copy of gas related request values + let request_gas = request.gas; + let request_gas_price = request.gas_price; + let env_gas_limit = block.gas_limit; + // get the highest possible gas limit, either the request's set value or the currently // configured gas limit let mut highest_gas_limit = request.gas.unwrap_or(block.gas_limit); // Configure the evm env let mut env = build_call_evm_env(cfg, block, request)?; + let mut db = SubState::new(State::new(state)); // if the request is a simple transfer we can optimize if env.tx.data.is_empty() { if let TransactTo::Call(to) = env.tx.transact_to { - let no_code = state.account_code(to)?.map(|code| code.is_empty()).unwrap_or(true); - if no_code { - return Ok(MIN_TRANSACTION_GAS) + if let Ok(code) = db.db.state().account_code(to) { + let no_code_callee = code.map(|code| code.is_empty()).unwrap_or(true); + if no_code_callee { + // simple transfer, check if caller has sufficient funds + let mut available_funds = + db.basic(env.tx.caller)?.map(|acc| acc.balance).unwrap_or_default(); + if env.tx.value > available_funds { + return Err(InvalidTransactionError::InsufficientFundsForTransfer.into()) + } + return Ok(U256::from(MIN_TRANSACTION_GAS)) + } } } } - let mut db = SubState::new(State::new(state)); + // check funds of the sender + let gas_price = env.tx.gas_price; + if gas_price > U256::ZERO { + let mut available_funds = + db.basic(env.tx.caller)?.map(|acc| acc.balance).unwrap_or_default(); + if env.tx.value > available_funds { + return Err(InvalidTransactionError::InsufficientFunds.into()) + } + // subtract transferred value from available funds + // SAFETY: value < available_funds, checked above + available_funds -= env.tx.value; + // amount of gas the sender can afford with the `gas_price` + // SAFETY: gas_price not zero + let allowance = available_funds.checked_div(gas_price).unwrap_or_default(); - todo!() + if highest_gas_limit > allowance { + // cap the highest gas limit by max gas caller can afford with given gas price + highest_gas_limit = allowance; + } + } + + // if the provided gas limit is less than computed cap, use that + let gas_limit = std::cmp::min(U256::from(env.tx.gas_limit), highest_gas_limit); + env.block.gas_limit = gas_limit; + + // execute the call without writing to db + let (res, mut env) = transact(&mut db, env)?; + match res.result { + ExecutionResult::Success { .. } => { + // succeeded + } + ExecutionResult::Halt { reason, .. } => { + return match reason { + Halt::OutOfGas(_) => Err(InvalidTransactionError::OutOfGas(gas_limit).into()), + Halt::NonceOverflow => Err(InvalidTransactionError::NonceMaxValue.into()), + err => Err(InvalidTransactionError::EvmHalt(err).into()), + } + } + ExecutionResult::Revert { output, .. } => { + // if price or limit was included in the request then we can execute the request + // again with the block's gas limit to check if revert is gas related or not + return if request_gas.is_some() || request_gas_price.is_some() { + let req_gas_limit = env.tx.gas_limit; + env.tx.gas_limit = env_gas_limit.try_into().unwrap_or(u64::MAX); + let (res, _) = transact(&mut db, env)?; + match res.result { + ExecutionResult::Success { .. } => { + // transaction succeeded by manually increasing the gas limit to + // highest, which means the caller lacks funds to pay for the tx + Err(InvalidTransactionError::OutOfGas(U256::from(req_gas_limit)).into()) + } + ExecutionResult::Revert { .. } => { + // reverted again after bumping the limit + Err(InvalidTransactionError::Revert(RevertError::new(output)).into()) + } + ExecutionResult::Halt { reason, .. } => { + Err(InvalidTransactionError::EvmHalt(reason).into()) + } + } + } else { + // the transaction did revert + Err(InvalidTransactionError::Revert(RevertError::new(output)).into()) + } + } + } + + // at this point we know the call succeeded but want to find the _best_ (lowest) gas the + // transaction succeeds with. we find this by doing a binary search over the + // possible range NOTE: this is the gas the transaction used, which is less than the + // transaction requires to succeed + let gas_used = res.result.gas_used(); + // the lowest value is capped by the gas it takes for a transfer + let mut lowest_gas_limit = MIN_TRANSACTION_GAS; + let mut highest_gas_limit: u64 = highest_gas_limit.try_into().unwrap_or(u64::MAX); + // pick a point that's close to the estimated gas + let mut mid_gas_limit = + std::cmp::min(gas_used * 3, (highest_gas_limit + lowest_gas_limit) / 2); + + let mut last_highest_gas_limit = highest_gas_limit; + + // binary search + while (highest_gas_limit - lowest_gas_limit) > 1 { + let mut env = env.clone(); + env.tx.gas_limit = mid_gas_limit; + let (res, _) = transact(&mut db, env)?; + match res.result { + ExecutionResult::Success { .. } => { + // cap the highest gas limit with succeeding gas limit + highest_gas_limit = mid_gas_limit; + // if last two successful estimations only vary by 10%, we consider this to be + // sufficiently accurate + const ACCURACY: u64 = 10; + if (last_highest_gas_limit - highest_gas_limit) * ACCURACY / + last_highest_gas_limit < + 1u64 + { + return Ok(U256::from(highest_gas_limit)) + } + last_highest_gas_limit = highest_gas_limit; + } + ExecutionResult::Revert { .. } => { + // increase the lowest gas limit + lowest_gas_limit = mid_gas_limit; + } + ExecutionResult::Halt { reason, .. } => { + match reason { + Halt::OutOfGas(_) => { + // increase the lowest gas limit + lowest_gas_limit = mid_gas_limit; + } + err => { + // these should be unreachable because we know the transaction succeeds, + // but we consider these cases an error + return Err(InvalidTransactionError::EvmHalt(err).into()) + } + } + } + } + // new midpoint + mid_gas_limit = (highest_gas_limit + lowest_gas_limit) / 2; + } + + Ok(U256::from(highest_gas_limit)) } } diff --git a/crates/rpc/rpc/src/eth/api/mod.rs b/crates/rpc/rpc/src/eth/api/mod.rs index 3e4d14063de..29f321b482c 100644 --- a/crates/rpc/rpc/src/eth/api/mod.rs +++ b/crates/rpc/rpc/src/eth/api/mod.rs @@ -76,6 +76,11 @@ impl EthApi { } } + /// Returns the state cache frontend + pub(crate) fn cache(&self) -> &EthStateCache { + &self.inner.eth_cache + } + /// Returns the inner `Client` pub(crate) fn client(&self) -> &Client { &self.inner.client diff --git a/crates/rpc/rpc/src/eth/error.rs b/crates/rpc/rpc/src/eth/error.rs index 82761d51117..ee1e42ab0eb 100644 --- a/crates/rpc/rpc/src/eth/error.rs +++ b/crates/rpc/rpc/src/eth/error.rs @@ -2,10 +2,10 @@ use crate::result::{internal_rpc_err, rpc_err}; use jsonrpsee::{core::Error as RpcError, types::error::INVALID_PARAMS_CODE}; -use reth_primitives::U128; +use reth_primitives::{constants::SELECTOR_LEN, U128, U256}; use reth_rpc_types::BlockError; use reth_transaction_pool::error::PoolError; -use revm::primitives::EVMError; +use revm::primitives::{EVMError, Halt}; /// Result alias pub(crate) type EthResult = Result; @@ -166,6 +166,15 @@ pub enum InvalidTransactionError { /// Thrown if the sender of a transaction is a contract. #[error("sender not an eoa")] SenderNoEOA, + /// Thrown during estimate if caller has insufficient funds to cover the tx. + #[error("Out of gas: gas required exceeds allowance: {0:?}")] + OutOfGas(U256), + /// Thrown if executing a transaction failed during estimate/call + #[error("{0}")] + Revert(RevertError), + /// Unspecific evm halt error + #[error("EVM error {0:?}")] + EvmHalt(Halt), } impl InvalidTransactionError { @@ -175,6 +184,7 @@ impl InvalidTransactionError { InvalidTransactionError::GasTooLow | InvalidTransactionError::GasTooHigh => { EthRpcErrorCode::InvalidInput.code() } + InvalidTransactionError::Revert(_) => EthRpcErrorCode::ExecutionError.code(), _ => EthRpcErrorCode::TransactionRejected.code(), } } @@ -182,7 +192,17 @@ impl InvalidTransactionError { impl From for RpcError { fn from(err: InvalidTransactionError) -> Self { - rpc_err(err.error_code(), err.to_string(), None) + match err { + InvalidTransactionError::Revert(revert) => { + // include out data if some + rpc_err( + revert.error_code(), + revert.to_string(), + revert.output.as_ref().map(|out| out.as_ref()), + ) + } + err => rpc_err(err.error_code(), err.to_string(), None), + } } } @@ -215,6 +235,48 @@ impl From for InvalidTransactionError { } } +/// Represents a reverted transaction and its output data. +/// +/// Displays "execution reverted(: reason)?" if the reason is a string. +#[derive(Debug, Clone)] +pub struct RevertError { + /// The transaction output data + /// + /// Note: this is `None` if output was empty + output: Option, +} + +// === impl RevertError == + +impl RevertError { + /// Wraps the output bytes + /// + /// Note: this is intended to wrap an revm output + pub fn new(output: bytes::Bytes) -> Self { + if output.is_empty() { + Self { output: None } + } else { + Self { output: Some(output) } + } + } + + fn error_code(&self) -> i32 { + EthRpcErrorCode::ExecutionError.code() + } +} + +impl std::fmt::Display for RevertError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("execution reverted")?; + if let Some(reason) = self.output.as_ref().and_then(decode_revert_reason) { + write!(f, ": {reason}")?; + } + Ok(()) + } +} + +impl std::error::Error for RevertError {} + /// A helper error type that mirrors `geth` Txpool's error messages #[derive(Debug, thiserror::Error)] pub(crate) enum GethTxPoolError { @@ -254,3 +316,15 @@ impl From for EthApiError { EthApiError::PoolError(GethTxPoolError::from(err)) } } + +/// Returns the revert reason from the `revm::TransactOut` data, if it's an abi encoded String. +/// +/// **Note:** it's assumed the `out` buffer starts with the call's signature +pub(crate) fn decode_revert_reason(out: impl AsRef<[u8]>) -> Option { + use ethers_core::abi::AbiDecode; + let out = out.as_ref(); + if out.len() < SELECTOR_LEN { + return None + } + String::decode(&out[SELECTOR_LEN..]).ok() +} From 1ae74feeb511c37f94c9c3c8cce775e8e8b17849 Mon Sep 17 00:00:00 2001 From: Bjerg Date: Thu, 2 Mar 2023 17:25:06 +0100 Subject: [PATCH 029/191] chore: remove unused config table (#1611) Co-authored-by: Roman Krasiuk --- crates/storage/db/src/tables/mod.rs | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/crates/storage/db/src/tables/mod.rs b/crates/storage/db/src/tables/mod.rs index a88e48a2125..43904cf483f 100644 --- a/crates/storage/db/src/tables/mod.rs +++ b/crates/storage/db/src/tables/mod.rs @@ -32,7 +32,7 @@ pub enum TableType { } /// Default tables that should be present inside database. -pub const TABLES: [(TableType, &str); 27] = [ +pub const TABLES: [(TableType, &str); 26] = [ (TableType::Table, CanonicalHeaders::const_name()), (TableType::Table, HeaderTD::const_name()), (TableType::Table, HeaderNumbers::const_name()), @@ -58,7 +58,6 @@ pub const TABLES: [(TableType, &str); 27] = [ (TableType::Table, AccountsTrie::const_name()), (TableType::DupSort, StoragesTrie::const_name()), (TableType::Table, TxSenders::const_name()), - (TableType::Table, Config::const_name()), (TableType::Table, SyncStage::const_name()), ]; @@ -289,11 +288,6 @@ table!( ( TxSenders ) TxNumber | Address ); -table!( - /// Configuration values. - ( Config ) ConfigKey | ConfigValue -); - table!( /// Stores the highest synced block number of each stage. ( SyncStage ) StageId | BlockNumber @@ -311,9 +305,5 @@ pub type StageId = Vec; // TODO: Temporary types, until they're properly defined alongside with the Encode and Decode Trait // -/// Temporary placeholder type for DB. -pub type ConfigKey = Vec; -/// Temporary placeholder type for DB. -pub type ConfigValue = Vec; /// Temporary placeholder type for DB. pub type Bytecode = Vec; From 161795945b21969faa06cd1234a0b5e3bf62f530 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 2 Mar 2023 18:42:41 +0100 Subject: [PATCH 030/191] feat(rpc): impl eth_call (#1612) --- crates/rpc/rpc/src/eth/api/call.rs | 45 +++++++++++++++++++----------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/crates/rpc/rpc/src/eth/api/call.rs b/crates/rpc/rpc/src/eth/api/call.rs index 46d4cca807f..81414db12a3 100644 --- a/crates/rpc/rpc/src/eth/api/call.rs +++ b/crates/rpc/rpc/src/eth/api/call.rs @@ -27,13 +27,38 @@ impl EthApi where Client: BlockProvider + StateProviderFactory + EvmEnvProvider + 'static, { + /// Returns the revm evm env for the requested [BlockId] + /// + /// If the [BlockId] this will return the [BlockId::Hash] of the block the env was configured + /// for. + async fn evm_env_at(&self, at: BlockId) -> EthResult<(CfgEnv, BlockEnv, BlockId)> { + // TODO handle Pending state's env + match at { + BlockId::Number(BlockNumberOrTag::Pending) => { + // This should perhaps use the latest env settings and update block specific + // settings like basefee/number + unimplemented!("support pending state env") + } + hash_or_num => { + let block_hash = self + .client() + .block_hash_for_id(hash_or_num)? + .ok_or_else(|| EthApiError::UnknownBlockNumber)?; + let (cfg, env) = self.cache().get_evm_env(block_hash).await?; + Ok((cfg, env, block_hash.into())) + } + } + } + /// Executes the call request at the given [BlockId] - pub(crate) fn call_at( + pub(crate) async fn call_at( &self, request: CallRequest, at: BlockId, ) -> EthResult<(ResultAndState, Env)> { - todo!() + let (cfg, block_env, at) = self.evm_env_at(at).await?; + let state = self.state_at_block_id(at)?.ok_or_else(|| EthApiError::UnknownBlockNumber)?; + self.call_with(cfg, block_env, request, state) } /// Executes the call request using the given environment against the state provider @@ -64,21 +89,7 @@ where request: CallRequest, at: BlockId, ) -> EthResult { - // TODO handle Pending state's env - let (cfg, block_env) = match at { - BlockId::Number(BlockNumberOrTag::Pending) => { - // This should perhaps use the latest env settings and update block specific - // settings like basefee/number - unimplemented!("support pending state env") - } - hash_or_num => { - let block_hash = self - .client() - .block_hash_for_id(hash_or_num)? - .ok_or_else(|| EthApiError::UnknownBlockNumber)?; - self.cache().get_evm_env(block_hash).await? - } - }; + let (cfg, block_env, at) = self.evm_env_at(at).await?; let state = self.state_at_block_id(at)?.ok_or_else(|| EthApiError::UnknownBlockNumber)?; self.estimate_gas_with(cfg, block_env, request, state) } From fbfec26a36ea2f253b3b0b9d14de72e24a77d259 Mon Sep 17 00:00:00 2001 From: Tirth Patel Date: Fri, 3 Mar 2023 05:06:01 -0330 Subject: [PATCH 031/191] Support soft limit in NewPooledTransactions (#1577) --- crates/net/network/src/transactions.rs | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/crates/net/network/src/transactions.rs b/crates/net/network/src/transactions.rs index 3b470781365..e59d3c927c1 100644 --- a/crates/net/network/src/transactions.rs +++ b/crates/net/network/src/transactions.rs @@ -36,6 +36,9 @@ use tracing::trace; /// Cache limit of transactions to keep track of for a single peer. const PEER_TRANSACTION_CACHE_LIMIT: usize = 1024 * 10; +/// Soft limit for NewPooledTransactions +const NEW_POOLED_TRANSACTION_HASHES_SOFT_LIMT: usize = 4096; + /// The future for inserting a function into the pool pub type PoolImportFuture = Pin> + Send + 'static>>; @@ -352,7 +355,23 @@ where // Send a `NewPooledTransactionHashes` to the peer with _all_ transactions in the // pool if !self.network.is_syncing() { - todo!("get access to full tx"); + let mut hashes = PooledTransactionsHashesBuilder::new(version); + let to_propogate = self.pool.pooled_transactions().into_iter().map(|tx| { + let tx = Arc::new(tx.transaction.to_recovered_transaction().into_signed()); + PropagateTransaction::new(tx) + }); + + for tx in to_propogate.take(NEW_POOLED_TRANSACTION_HASHES_SOFT_LIMT) { + let peer = self.peers.get_mut(&peer_id).unwrap(); + + if peer.transactions.insert(tx.hash()) { + hashes.push(&tx); + } + } + + let hashes = hashes.build(); + + self.network.send_transactions_hashes(peer_id, hashes); // let msg = NewPooledTransactionHashes66(self.pool.pooled_transactions()); // self.network.send_message(NetworkHandleMessage::SendPooledTransactionHashes { // peer_id, From a553c1d7000b1d531a87ad620b0417177a0700cf Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 3 Mar 2023 14:12:00 +0100 Subject: [PATCH 032/191] chore: add spawn_critical function to trait (#1622) --- crates/tasks/src/lib.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/crates/tasks/src/lib.rs b/crates/tasks/src/lib.rs index 3e408a1cc1f..0802c0ba9ee 100644 --- a/crates/tasks/src/lib.rs +++ b/crates/tasks/src/lib.rs @@ -70,6 +70,9 @@ pub trait TaskSpawner: Send + Sync + Unpin + std::fmt::Debug + DynClone { /// Spawns the task onto the runtime. /// See also [`Handle::spawn`]. fn spawn(&self, fut: BoxFuture<'static, ()>) -> JoinHandle<()>; + + /// This spawns a critical task onto the runtime. + fn spawn_critical(&self, name: &'static str, fut: BoxFuture<'static, ()>) -> JoinHandle<()>; } dyn_clone::clone_trait_object!(TaskSpawner); @@ -83,6 +86,10 @@ impl TaskSpawner for TokioTaskExecutor { fn spawn(&self, fut: BoxFuture<'static, ()>) -> JoinHandle<()> { tokio::task::spawn(fut) } + + fn spawn_critical(&self, _name: &'static str, fut: BoxFuture<'static, ()>) -> JoinHandle<()> { + tokio::task::spawn(fut) + } } /// Many reth components require to spawn tasks for long-running jobs. For example `discovery` @@ -331,6 +338,10 @@ impl TaskSpawner for TaskExecutor { fn spawn(&self, fut: BoxFuture<'static, ()>) -> JoinHandle<()> { self.spawn(fut) } + + fn spawn_critical(&self, name: &'static str, fut: BoxFuture<'static, ()>) -> JoinHandle<()> { + TaskExecutor::spawn_critical(self, name, fut) + } } /// Determines how a task is spawned From 789ae514af4a9aae73bfb40e714b24928415bfbc Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 3 Mar 2023 17:59:18 +0100 Subject: [PATCH 033/191] feat: add derive contract address functions (#1620) --- crates/primitives/src/contract.rs | 147 ++++++++++++++++++++++++++++++ crates/primitives/src/lib.rs | 1 + 2 files changed, 148 insertions(+) create mode 100644 crates/primitives/src/contract.rs diff --git a/crates/primitives/src/contract.rs b/crates/primitives/src/contract.rs new file mode 100644 index 00000000000..fe8ddc35751 --- /dev/null +++ b/crates/primitives/src/contract.rs @@ -0,0 +1,147 @@ +//! Helpers for deriving contract addresses + +use crate::{keccak256, Address, H256, U256}; +use reth_rlp::Encodable; +use reth_rlp_derive::RlpEncodable; + +/// The address for an Ethereum contract is deterministically computed from the +/// address of its creator (sender) and how many transactions the creator has +/// sent (nonce). The sender and nonce are RLP encoded and then hashed with Keccak-256. +pub fn get_contract_address(sender: impl Into

, nonce: impl Into) -> Address { + #[derive(RlpEncodable)] + struct S { + sender: Address, + nonce: U256, + } + let sender = S { sender: sender.into(), nonce: nonce.into() }; + let mut buf = Vec::new(); + sender.encode(&mut buf); + let hash = keccak256(buf); + let addr: [u8; 20] = hash[12..].try_into().expect("correct len"); + Address::from(addr) +} + +/// Returns the CREATE2 address of a smart contract as specified in +/// [EIP1014](https://github.com/ethereum/EIPs/blob/master/EIPS/eip-1014.md) +/// +/// keccak256( 0xff ++ senderAddress ++ salt ++ keccak256(init_code))[12..] +/// +/// where `salt` is always 32 bytes (a stack item). +pub fn get_create2_address( + from: impl Into
, + salt: [u8; 32], + init_code: impl AsRef<[u8]>, +) -> Address { + let init_code_hash = keccak256(init_code); + get_create2_address_from_hash(from, salt, init_code_hash) +} + +/// Returns the CREATE2 address of a smart contract as specified in +/// [EIP1014](https://github.com/ethereum/EIPs/blob/master/EIPS/eip-1014.md), +/// taking the pre-computed hash of the init code as input. +/// +/// keccak256( 0xff ++ senderAddress ++ salt ++ keccak256(init_code))[12..] +pub fn get_create2_address_from_hash( + from: impl Into
, + salt: [u8; 32], + init_code_hash: impl Into, +) -> Address { + let from = from.into(); + let init_code_hash = init_code_hash.into(); + // always 85 bytes: 0xff+20+salt+code_hash + let mut preimage = [0xff; 85]; + + // 20bytes address + preimage[1..21].copy_from_slice(from.as_bytes()); + // 32bytes salt + preimage[21..53].copy_from_slice(&salt[..]); + // 32bytes code hash + preimage[53..].copy_from_slice(init_code_hash.as_ref()); + + let hash = keccak256(&preimage[..]); + let addr: [u8; 20] = hash[12..].try_into().expect("correct len"); + Address::from(addr) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn contract_address() { + // http://ethereum.stackexchange.com/questions/760/how-is-the-address-of-an-ethereum-contract-computed + let from = "6ac7ea33f8831ea9dcc53393aaa88b25a785dbf0".parse::
().unwrap(); + for (nonce, expected) in [ + "cd234a471b72ba2f1ccf0a70fcaba648a5eecd8d", + "343c43a37d37dff08ae8c4a11544c718abb4fcf8", + "f778b86fa74e846c4f0a1fbd1335fe81c00a0c91", + "fffd933a0bc612844eaf0c6fe3e5b8e9b6c1d19c", + ] + .iter() + .enumerate() + { + let address = get_contract_address(from, U256::from(nonce)); + assert_eq!(address, expected.parse::
().unwrap()); + } + } + + #[test] + // Test vectors from https://github.com/ethereum/EIPs/blob/master/EIPS/eip-1014.md#examples + fn create2_address() { + for (from, salt, init_code, expected) in &[ + ( + "0000000000000000000000000000000000000000", + "0000000000000000000000000000000000000000000000000000000000000000", + "00", + "4D1A2e2bB4F88F0250f26Ffff098B0b30B26BF38", + ), + ( + "deadbeef00000000000000000000000000000000", + "0000000000000000000000000000000000000000000000000000000000000000", + "00", + "B928f69Bb1D91Cd65274e3c79d8986362984fDA3", + ), + ( + "deadbeef00000000000000000000000000000000", + "000000000000000000000000feed000000000000000000000000000000000000", + "00", + "D04116cDd17beBE565EB2422F2497E06cC1C9833", + ), + ( + "0000000000000000000000000000000000000000", + "0000000000000000000000000000000000000000000000000000000000000000", + "deadbeef", + "70f2b2914A2a4b783FaEFb75f459A580616Fcb5e", + ), + ( + "00000000000000000000000000000000deadbeef", + "00000000000000000000000000000000000000000000000000000000cafebabe", + "deadbeef", + "60f3f640a8508fC6a86d45DF051962668E1e8AC7", + ), + ( + "00000000000000000000000000000000deadbeef", + "00000000000000000000000000000000000000000000000000000000cafebabe", + "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef", + "1d8bfDC5D46DC4f61D6b6115972536eBE6A8854C", + ), + ( + "0000000000000000000000000000000000000000", + "0000000000000000000000000000000000000000000000000000000000000000", + "", + "E33C0C7F7df4809055C3ebA6c09CFe4BaF1BD9e0", + ), + ] { + // get_create2_address() + let from = from.parse::
().unwrap(); + let salt = hex::decode(salt).unwrap(); + let init_code = hex::decode(init_code).unwrap(); + let expected = expected.parse::
().unwrap(); + assert_eq!(expected, get_create2_address(from, salt.clone().try_into().unwrap(), init_code.clone())); + + // get_create2_address_from_hash() + let init_code_hash = keccak256(init_code); + assert_eq!(expected, get_create2_address_from_hash(from, salt.try_into().unwrap(), init_code_hash)) + } + } +} diff --git a/crates/primitives/src/lib.rs b/crates/primitives/src/lib.rs index 5a530528120..4685244ccbf 100644 --- a/crates/primitives/src/lib.rs +++ b/crates/primitives/src/lib.rs @@ -15,6 +15,7 @@ mod block; pub mod bloom; mod chain; pub mod constants; +pub mod contract; mod error; pub mod filter; mod forkid; From bc538770b1cf7c216e8b6e90f2efd391e7b29618 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 3 Mar 2023 20:17:28 +0100 Subject: [PATCH 034/191] fix: use correct serde impl for Withdrawal (#1616) --- crates/primitives/src/lib.rs | 7 +-- .../src/{ => serde_helper}/jsonu256.rs | 0 crates/primitives/src/serde_helper/mod.rs | 45 +++++++++++++++++++ crates/primitives/src/withdrawal.rs | 20 ++++++++- 4 files changed, 66 insertions(+), 6 deletions(-) rename crates/primitives/src/{ => serde_helper}/jsonu256.rs (100%) create mode 100644 crates/primitives/src/serde_helper/mod.rs diff --git a/crates/primitives/src/lib.rs b/crates/primitives/src/lib.rs index 4685244ccbf..722ce553c24 100644 --- a/crates/primitives/src/lib.rs +++ b/crates/primitives/src/lib.rs @@ -24,7 +24,6 @@ mod hardfork; mod header; mod hex_bytes; mod integer_list; -mod jsonu256; mod log; mod net; mod peer; @@ -53,11 +52,11 @@ pub use hardfork::Hardfork; pub use header::{Head, Header, HeadersDirection, SealedHeader}; pub use hex_bytes::Bytes; pub use integer_list::IntegerList; -pub use jsonu256::JsonU256; pub use log::Log; pub use net::NodeRecord; pub use peer::{PeerId, WithPeerId}; pub use receipt::Receipt; +pub use serde_helper::JsonU256; pub use storage::{StorageEntry, StorageTrieEntry}; pub use transaction::{ AccessList, AccessListItem, FromRecoveredTransaction, IntoRecoveredTransaction, Signature, @@ -109,9 +108,7 @@ pub mod utils { } /// Helpers for working with serde -pub mod serde_helper { - pub use crate::jsonu256::deserialize_json_u256; -} +pub mod serde_helper; /// Returns the keccak256 hash for the given data. #[inline] diff --git a/crates/primitives/src/jsonu256.rs b/crates/primitives/src/serde_helper/jsonu256.rs similarity index 100% rename from crates/primitives/src/jsonu256.rs rename to crates/primitives/src/serde_helper/jsonu256.rs diff --git a/crates/primitives/src/serde_helper/mod.rs b/crates/primitives/src/serde_helper/mod.rs new file mode 100644 index 00000000000..7efbed1daca --- /dev/null +++ b/crates/primitives/src/serde_helper/mod.rs @@ -0,0 +1,45 @@ +//! Various serde utilities + +mod jsonu256; +pub use jsonu256::*; + +/// serde functions for handling primitive `u64` as [U64](crate::U64) +pub mod u64_hex { + use crate::U64; + use serde::{Deserialize, Deserializer, Serialize, Serializer}; + + /// Deserializes an `u64` from [U64] accepting a hex quantity string with optional 0x prefix + pub fn deserialize<'de, D>(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + U64::deserialize(deserializer).map(|val| val.as_u64()) + } + + /// Serializes u64 as hex string + pub fn serialize(value: &u64, s: S) -> Result { + U64::from(*value).serialize(s) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde::{Deserialize, Serialize}; + + #[test] + fn test_hex_u64() { + #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] + struct Value { + #[serde(with = "u64_hex")] + inner: u64, + } + + let val = Value { inner: 1000 }; + let s = serde_json::to_string(&val).unwrap(); + assert_eq!(s, "{\"inner\":\"0x3e8\"}"); + + let deserialized: Value = serde_json::from_str(&s).unwrap(); + assert_eq!(val, deserialized); + } +} diff --git a/crates/primitives/src/withdrawal.rs b/crates/primitives/src/withdrawal.rs index ae49d3a38d8..f94175f8f0d 100644 --- a/crates/primitives/src/withdrawal.rs +++ b/crates/primitives/src/withdrawal.rs @@ -1,4 +1,4 @@ -use crate::{constants::GWEI_TO_WEI, Address, U256}; +use crate::{constants::GWEI_TO_WEI, serde_helper::u64_hex, Address, U256}; use reth_codecs::{main_codec, Compact}; use reth_rlp::{RlpDecodable, RlpEncodable}; @@ -7,12 +7,15 @@ use reth_rlp::{RlpDecodable, RlpEncodable}; #[derive(Debug, Clone, PartialEq, Eq, Default, RlpEncodable, RlpDecodable)] pub struct Withdrawal { /// Monotonically increasing identifier issued by consensus layer. + #[serde(with = "u64_hex")] pub index: u64, /// Index of validator associated with withdrawal. + #[serde(with = "u64_hex", rename = "validatorIndex")] pub validator_index: u64, /// Target address for withdrawn ether. pub address: Address, /// Value of the withdrawal in gwei. + #[serde(with = "u64_hex")] pub amount: u64, } @@ -22,3 +25,18 @@ impl Withdrawal { U256::from(self.amount) * U256::from(GWEI_TO_WEI) } } + +#[cfg(test)] +mod tests { + use super::*; + + // + #[test] + fn test_withdrawal_serde_roundtrip() { + let input = r#"[{"index":"0x0","validatorIndex":"0x0","address":"0x0000000000000000000000000000000000001000","amount":"0x1"},{"index":"0x1","validatorIndex":"0x1","address":"0x0000000000000000000000000000000000001001","amount":"0x1"},{"index":"0x2","validatorIndex":"0x2","address":"0x0000000000000000000000000000000000001002","amount":"0x1"},{"index":"0x3","validatorIndex":"0x3","address":"0x0000000000000000000000000000000000001003","amount":"0x1"},{"index":"0x4","validatorIndex":"0x4","address":"0x0000000000000000000000000000000000001004","amount":"0x1"},{"index":"0x5","validatorIndex":"0x5","address":"0x0000000000000000000000000000000000001005","amount":"0x1"},{"index":"0x6","validatorIndex":"0x6","address":"0x0000000000000000000000000000000000001006","amount":"0x1"},{"index":"0x7","validatorIndex":"0x7","address":"0x0000000000000000000000000000000000001007","amount":"0x1"},{"index":"0x8","validatorIndex":"0x8","address":"0x0000000000000000000000000000000000001008","amount":"0x1"},{"index":"0x9","validatorIndex":"0x9","address":"0x0000000000000000000000000000000000001009","amount":"0x1"},{"index":"0xa","validatorIndex":"0xa","address":"0x000000000000000000000000000000000000100a","amount":"0x1"},{"index":"0xb","validatorIndex":"0xb","address":"0x000000000000000000000000000000000000100b","amount":"0x1"},{"index":"0xc","validatorIndex":"0xc","address":"0x000000000000000000000000000000000000100c","amount":"0x1"},{"index":"0xd","validatorIndex":"0xd","address":"0x000000000000000000000000000000000000100d","amount":"0x1"},{"index":"0xe","validatorIndex":"0xe","address":"0x000000000000000000000000000000000000100e","amount":"0x1"},{"index":"0xf","validatorIndex":"0xf","address":"0x000000000000000000000000000000000000100f","amount":"0x1"}]"#; + + let withdrawals: Vec = serde_json::from_str(input).unwrap(); + let s = serde_json::to_string(&withdrawals).unwrap(); + assert_eq!(input, s); + } +} From 2d1bcd17d8020aff3b4bebd16ec25183c0a58601 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 3 Mar 2023 20:50:49 +0100 Subject: [PATCH 035/191] perf: improve initial pooled tx exchange (#1615) --- crates/net/network/src/transactions.rs | 58 ++++++++++++++----------- crates/transaction-pool/src/lib.rs | 3 +- crates/transaction-pool/src/validate.rs | 5 +++ 3 files changed, 39 insertions(+), 27 deletions(-) diff --git a/crates/net/network/src/transactions.rs b/crates/net/network/src/transactions.rs index e59d3c927c1..39bf239df6b 100644 --- a/crates/net/network/src/transactions.rs +++ b/crates/net/network/src/transactions.rs @@ -19,7 +19,8 @@ use reth_primitives::{ }; use reth_rlp::Encodable; use reth_transaction_pool::{ - error::PoolResult, PropagateKind, PropagatedTransactions, TransactionPool, + error::PoolResult, PoolTransaction, PropagateKind, PropagatedTransactions, TransactionPool, + ValidPoolTransaction, }; use std::{ collections::{hash_map::Entry, HashMap}, @@ -37,7 +38,7 @@ use tracing::trace; const PEER_TRANSACTION_CACHE_LIMIT: usize = 1024 * 10; /// Soft limit for NewPooledTransactions -const NEW_POOLED_TRANSACTION_HASHES_SOFT_LIMT: usize = 4096; +const NEW_POOLED_TRANSACTION_HASHES_SOFT_LIMIT: usize = 4096; /// The future for inserting a function into the pool pub type PoolImportFuture = Pin> + Send + 'static>>; @@ -340,7 +341,7 @@ where self.peers.remove(&peer_id); } NetworkEvent::SessionEstablished { peer_id, messages, version, .. } => { - // insert a new peer + // insert a new peer into the peerset self.peers.insert( peer_id, Peer { @@ -352,31 +353,26 @@ where }, ); - // Send a `NewPooledTransactionHashes` to the peer with _all_ transactions in the + // Send a `NewPooledTransactionHashes` to the peer with up to + // `NEW_POOLED_TRANSACTION_HASHES_SOFT_LIMIT` transactions in the // pool if !self.network.is_syncing() { - let mut hashes = PooledTransactionsHashesBuilder::new(version); - let to_propogate = self.pool.pooled_transactions().into_iter().map(|tx| { - let tx = Arc::new(tx.transaction.to_recovered_transaction().into_signed()); - PropagateTransaction::new(tx) - }); - - for tx in to_propogate.take(NEW_POOLED_TRANSACTION_HASHES_SOFT_LIMT) { - let peer = self.peers.get_mut(&peer_id).unwrap(); - - if peer.transactions.insert(tx.hash()) { - hashes.push(&tx); - } + let peer = self.peers.get_mut(&peer_id).expect("is present; qed"); + + let mut msg_builder = PooledTransactionsHashesBuilder::new(version); + + for pooled_tx in self + .pool + .pooled_transactions() + .into_iter() + .take(NEW_POOLED_TRANSACTION_HASHES_SOFT_LIMIT) + { + peer.transactions.insert(*pooled_tx.hash()); + msg_builder.push_pooled(pooled_tx); } - let hashes = hashes.build(); - - self.network.send_transactions_hashes(peer_id, hashes); - // let msg = NewPooledTransactionHashes66(self.pool.pooled_transactions()); - // self.network.send_message(NetworkHandleMessage::SendPooledTransactionHashes { - // peer_id, - // msg, - // }) + let msg = msg_builder.build(); + self.network.send_transactions_hashes(peer_id, msg); } } // TODO Add remaining events @@ -536,7 +532,7 @@ where } } -/// A transaction that's about to be propagated +/// A transaction that's about to be propagated to multiple peers. struct PropagateTransaction { tx_type: u8, length: usize, @@ -565,6 +561,18 @@ enum PooledTransactionsHashesBuilder { // === impl PooledTransactionsHashesBuilder === impl PooledTransactionsHashesBuilder { + /// Push a transaction from the pool to the list. + fn push_pooled(&mut self, pooled_tx: Arc>) { + match self { + PooledTransactionsHashesBuilder::Eth66(msg) => msg.0.push(*pooled_tx.hash()), + PooledTransactionsHashesBuilder::Eth68(msg) => { + msg.hashes.push(*pooled_tx.hash()); + msg.sizes.push(pooled_tx.encoded_length); + msg.types.push(pooled_tx.transaction.tx_type()); + } + } + } + fn push(&mut self, tx: &PropagateTransaction) { match self { PooledTransactionsHashesBuilder::Eth66(msg) => msg.0.push(tx.hash()), diff --git a/crates/transaction-pool/src/lib.rs b/crates/transaction-pool/src/lib.rs index 5b356d58309..f9d099a32d5 100644 --- a/crates/transaction-pool/src/lib.rs +++ b/crates/transaction-pool/src/lib.rs @@ -86,13 +86,12 @@ pub use crate::{ BestTransactions, OnNewBlockEvent, PoolTransaction, PropagateKind, PropagatedTransactions, TransactionOrigin, TransactionPool, }, - validate::{TransactionValidationOutcome, TransactionValidator}, + validate::{TransactionValidationOutcome, TransactionValidator, ValidPoolTransaction}, }; use crate::{ error::PoolResult, pool::PoolInner, traits::{NewTransactionEvent, PoolSize}, - validate::ValidPoolTransaction, }; use reth_primitives::{TxHash, U256}; use std::{collections::HashMap, sync::Arc}; diff --git a/crates/transaction-pool/src/validate.rs b/crates/transaction-pool/src/validate.rs index a5d855b3a09..7200d4f8501 100644 --- a/crates/transaction-pool/src/validate.rs +++ b/crates/transaction-pool/src/validate.rs @@ -106,6 +106,11 @@ impl ValidPoolTransaction { self.transaction.hash() } + /// Returns the type identifier of the transaction + pub fn tx_type(&self) -> u8 { + self.transaction.tx_type() + } + /// Returns the address of the sender pub fn sender(&self) -> Address { self.transaction.sender() From 9bdbbb2b9ec30d88a9cf39eaf35729d1ec3e53a7 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 3 Mar 2023 21:07:19 +0100 Subject: [PATCH 036/191] fix(disc): trigger bootstrap if table empty (#1621) --- crates/net/discv4/src/lib.rs | 44 ++++++++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/crates/net/discv4/src/lib.rs b/crates/net/discv4/src/lib.rs index eb1424ba47e..d0fadddac83 100644 --- a/crates/net/discv4/src/lib.rs +++ b/crates/net/discv4/src/lib.rs @@ -24,8 +24,8 @@ use crate::{ use discv5::{ kbucket, kbucket::{ - BucketInsertResult, Distance, Entry as BucketEntry, KBucketsTable, NodeStatus, - MAX_NODES_PER_BUCKET, + BucketInsertResult, Distance, Entry as BucketEntry, InsertResult, KBucketsTable, + NodeStatus, MAX_NODES_PER_BUCKET, }, ConnectionDirection, ConnectionState, }; @@ -553,20 +553,24 @@ impl Discv4Service { /// **Note:** This is a noop if there are no bootnodes. pub fn bootstrap(&mut self) { for record in self.config.bootstrap_nodes.clone() { - debug!(target : "discv4", ?record, "Adding bootstrap node"); + debug!(target : "discv4", ?record, "pinging boot node"); let key = kad_key(record.id); let entry = NodeEntry::new(record); // insert the boot node in the table - let _ = self.kbuckets.insert_or_update( + match self.kbuckets.insert_or_update( &key, entry, NodeStatus { state: ConnectionState::Connected, direction: ConnectionDirection::Outgoing, }, - ); - self.try_ping(record, PingReason::Initial); + ) { + InsertResult::Failed(_) => {} + _ => { + self.try_ping(record, PingReason::Initial); + } + } } } @@ -625,13 +629,23 @@ impl Discv4Service { target_key.clone(), self.kbuckets .closest_values(&target_key) + .filter(|node| !self.pending_find_nodes.contains_key(&node.key.preimage().0)) .take(MAX_NODES_PER_BUCKET) .map(|n| (target_key.distance(&n.key), n.value.record)), tx, ); // From those 16, pick the 3 closest to start the concurrent lookup. - let closest = ctx.closest(ALPHA, |node| !self.pending_find_nodes.contains_key(&node.id)); + let closest = ctx.closest(ALPHA); + + if closest.is_empty() && self.pending_find_nodes.is_empty() { + // no closest nodes, and no lookup in progress: table is empty. + // This could happen if all records were deleted from the table due to missed pongs + // (e.g. connectivity problems over a long period of time, or issues during initial + // bootstrapping) so we attempt to bootstrap again + self.bootstrap(); + return + } trace!(target : "net::discv4", ?target, num = closest.len(), "Start lookup closest nodes"); @@ -1172,7 +1186,8 @@ impl Discv4Service { } // get the next closest nodes, not yet queried nodes and start over. - let closest = ctx.closest(ALPHA, |node| !self.pending_find_nodes.contains_key(&node.id)); + let closest = + ctx.filter_closest(ALPHA, |node| !self.pending_find_nodes.contains_key(&node.id)); for closest in closest { let key = kad_key(closest.id); @@ -1704,8 +1719,19 @@ impl LookupContext { self.inner.target.preimage().0 } + fn closest(&self, num: usize) -> Vec { + self.inner + .closest_nodes + .borrow() + .iter() + .filter(|(_, node)| !node.queried) + .map(|(_, n)| n.record) + .take(num) + .collect() + } + /// Returns the closest nodes that have not been queried yet. - fn closest

(&self, num: usize, filter: P) -> Vec + fn filter_closest

(&self, num: usize, filter: P) -> Vec where P: FnMut(&NodeRecord) -> bool, { From 1bb25ae8b485acc5b7b7ed1e76538d28abf553e8 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 3 Mar 2023 22:21:38 +0100 Subject: [PATCH 037/191] style: take ownership of header arg (#1626) --- crates/net/downloaders/src/bodies/test_utils.rs | 2 +- crates/net/eth-wire/src/types/blocks.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/net/downloaders/src/bodies/test_utils.rs b/crates/net/downloaders/src/bodies/test_utils.rs index 13103f9cbda..c69d26d47cd 100644 --- a/crates/net/downloaders/src/bodies/test_utils.rs +++ b/crates/net/downloaders/src/bodies/test_utils.rs @@ -41,7 +41,7 @@ pub(crate) fn create_raw_bodies<'a>( .into_iter() .map(|header| { let body = bodies.remove(&header.hash()).expect("body exists"); - body.create_block(header) + body.create_block(header.as_ref().clone()) }) .collect() } diff --git a/crates/net/eth-wire/src/types/blocks.rs b/crates/net/eth-wire/src/types/blocks.rs index 2f93b2dc419..1890b355d66 100644 --- a/crates/net/eth-wire/src/types/blocks.rs +++ b/crates/net/eth-wire/src/types/blocks.rs @@ -87,9 +87,9 @@ pub struct BlockBody { impl BlockBody { /// Create a [`Block`](reth_primitives::Block) from the body and its header. - pub fn create_block(&self, header: &Header) -> Block { + pub fn create_block(&self, header: Header) -> Block { Block { - header: header.clone(), + header, body: self.transactions.clone(), ommers: self.ommers.clone(), withdrawals: self.withdrawals.clone(), From f605e7f3a06dae07495a119a4df1088811bee3f2 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 3 Mar 2023 22:35:11 +0100 Subject: [PATCH 038/191] chore: reexport revm create address (#1624) --- crates/primitives/src/contract.rs | 69 ++++++------------------------- 1 file changed, 13 insertions(+), 56 deletions(-) diff --git a/crates/primitives/src/contract.rs b/crates/primitives/src/contract.rs index fe8ddc35751..43ce75b53f2 100644 --- a/crates/primitives/src/contract.rs +++ b/crates/primitives/src/contract.rs @@ -1,25 +1,8 @@ //! Helpers for deriving contract addresses -use crate::{keccak256, Address, H256, U256}; -use reth_rlp::Encodable; -use reth_rlp_derive::RlpEncodable; - -/// The address for an Ethereum contract is deterministically computed from the -/// address of its creator (sender) and how many transactions the creator has -/// sent (nonce). The sender and nonce are RLP encoded and then hashed with Keccak-256. -pub fn get_contract_address(sender: impl Into

, nonce: impl Into) -> Address { - #[derive(RlpEncodable)] - struct S { - sender: Address, - nonce: U256, - } - let sender = S { sender: sender.into(), nonce: nonce.into() }; - let mut buf = Vec::new(); - sender.encode(&mut buf); - let hash = keccak256(buf); - let addr: [u8; 20] = hash[12..].try_into().expect("correct len"); - Address::from(addr) -} +// re-export from revm +use crate::{keccak256, Address, U256}; +pub use revm_primitives::utilities::{create2_address, create_address}; /// Returns the CREATE2 address of a smart contract as specified in /// [EIP1014](https://github.com/ethereum/EIPs/blob/master/EIPS/eip-1014.md) @@ -27,40 +10,13 @@ pub fn get_contract_address(sender: impl Into
, nonce: impl Into) /// keccak256( 0xff ++ senderAddress ++ salt ++ keccak256(init_code))[12..] /// /// where `salt` is always 32 bytes (a stack item). -pub fn get_create2_address( - from: impl Into
, - salt: [u8; 32], +pub fn create2_address_from_code( + from: Address, init_code: impl AsRef<[u8]>, + salt: U256, ) -> Address { let init_code_hash = keccak256(init_code); - get_create2_address_from_hash(from, salt, init_code_hash) -} - -/// Returns the CREATE2 address of a smart contract as specified in -/// [EIP1014](https://github.com/ethereum/EIPs/blob/master/EIPS/eip-1014.md), -/// taking the pre-computed hash of the init code as input. -/// -/// keccak256( 0xff ++ senderAddress ++ salt ++ keccak256(init_code))[12..] -pub fn get_create2_address_from_hash( - from: impl Into
, - salt: [u8; 32], - init_code_hash: impl Into, -) -> Address { - let from = from.into(); - let init_code_hash = init_code_hash.into(); - // always 85 bytes: 0xff+20+salt+code_hash - let mut preimage = [0xff; 85]; - - // 20bytes address - preimage[1..21].copy_from_slice(from.as_bytes()); - // 32bytes salt - preimage[21..53].copy_from_slice(&salt[..]); - // 32bytes code hash - preimage[53..].copy_from_slice(init_code_hash.as_ref()); - - let hash = keccak256(&preimage[..]); - let addr: [u8; 20] = hash[12..].try_into().expect("correct len"); - Address::from(addr) + create2_address(from, init_code_hash, salt) } #[cfg(test)] @@ -80,14 +36,14 @@ mod tests { .iter() .enumerate() { - let address = get_contract_address(from, U256::from(nonce)); + let address = create_address(from, nonce as u64); assert_eq!(address, expected.parse::
().unwrap()); } } #[test] // Test vectors from https://github.com/ethereum/EIPs/blob/master/EIPS/eip-1014.md#examples - fn create2_address() { + fn test_create2_address() { for (from, salt, init_code, expected) in &[ ( "0000000000000000000000000000000000000000", @@ -132,16 +88,17 @@ mod tests { "E33C0C7F7df4809055C3ebA6c09CFe4BaF1BD9e0", ), ] { - // get_create2_address() + // create2_address() let from = from.parse::
().unwrap(); let salt = hex::decode(salt).unwrap(); + let salt = U256::try_from_be_slice(&salt).unwrap(); let init_code = hex::decode(init_code).unwrap(); let expected = expected.parse::
().unwrap(); - assert_eq!(expected, get_create2_address(from, salt.clone().try_into().unwrap(), init_code.clone())); + assert_eq!(expected, create2_address_from_code(from, init_code.clone(),salt )); // get_create2_address_from_hash() let init_code_hash = keccak256(init_code); - assert_eq!(expected, get_create2_address_from_hash(from, salt.try_into().unwrap(), init_code_hash)) + assert_eq!(expected, create2_address(from, init_code_hash, salt)) } } } From 5dc56cc76f6f0d9e67e7006ba2c17b70b5ad5767 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 3 Mar 2023 22:56:35 +0100 Subject: [PATCH 039/191] chore: drop tokio rt on separate thread (#1627) --- bin/reth/src/runner.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bin/reth/src/runner.rs b/bin/reth/src/runner.rs index 84afc3847f0..d0a171f33df 100644 --- a/bin/reth/src/runner.rs +++ b/bin/reth/src/runner.rs @@ -37,12 +37,17 @@ impl CliRunner { // fires the shutdown signal to all tasks spawned via the task executor drop(task_manager); + // drop the tokio runtime on a separate thread because drop blocks until its pools + // (including blocking pool) are shutdown. In other words `drop(tokio_runtime)` would block + // the current thread but we want to exit right away. + std::thread::spawn(move || drop(tokio_runtime)); + // give all tasks that are now being shut down some time to finish before tokio leaks them // see [Runtime::shutdown_timeout](tokio::runtime::Runtime::shutdown_timeout) // TODO: enable this again, when pipeline/stages are not longer blocking tasks - std::process::exit(0); // warn!(target: "reth::cli", "Received shutdown signal, waiting up to 30 seconds for // tasks."); tokio_runtime.shutdown_timeout(Duration::from_secs(30)); + Ok(()) } /// Executes a regular future until completion or until external signal received. From cef4c425fd0eb8887f7563e5643952c51110358d Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sat, 4 Mar 2023 01:26:45 +0100 Subject: [PATCH 040/191] perf: allocate ping buffer and move to standalone function (#1629) --- crates/net/eth-wire/src/p2pstream.rs | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/crates/net/eth-wire/src/p2pstream.rs b/crates/net/eth-wire/src/p2pstream.rs index 25be3ae0d2b..a5990950183 100644 --- a/crates/net/eth-wire/src/p2pstream.rs +++ b/crates/net/eth-wire/src/p2pstream.rs @@ -250,7 +250,15 @@ impl P2PStream { let pong = P2PMessage::Pong; let mut pong_bytes = BytesMut::with_capacity(pong.length()); pong.encode(&mut pong_bytes); - self.outgoing_messages.push_back(pong_bytes.into()); + self.outgoing_messages.push_back(pong_bytes.freeze()); + } + + /// Queues in a _snappy_ encoded [`P2PMessage::Ping`] message. + fn send_ping(&mut self) { + let ping = P2PMessage::Ping; + let mut ping_bytes = BytesMut::with_capacity(ping.length()); + ping.encode(&mut ping_bytes); + self.outgoing_messages.push_back(ping_bytes.freeze()); } /// Starts to gracefully disconnect the connection by sending a Disconnect message and stop @@ -439,17 +447,7 @@ where match this.pinger.poll_ping(cx) { Poll::Pending => {} Poll::Ready(Ok(PingerEvent::Ping)) => { - // encode the ping message - let mut ping_bytes = BytesMut::new(); - P2PMessage::Ping.encode(&mut ping_bytes); - - // check if the buffer is full - if this.outgoing_messages.len() >= MAX_P2P_CAPACITY { - return Poll::Ready(Err(P2PStreamError::SendBufferFull)) - } - - // if the sink is not ready, buffer the message - this.outgoing_messages.push_back(ping_bytes.into()); + this.send_ping(); } _ => { // encode the disconnect message From 91652a0b0c5a9a869362506c086bf55e5a697bc6 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sat, 4 Mar 2023 14:14:16 +0100 Subject: [PATCH 041/191] chore: add From bytes::Bytes impl (#1635) --- crates/primitives/src/hex_bytes.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/crates/primitives/src/hex_bytes.rs b/crates/primitives/src/hex_bytes.rs index b7447afa72f..33b58285312 100644 --- a/crates/primitives/src/hex_bytes.rs +++ b/crates/primitives/src/hex_bytes.rs @@ -98,6 +98,12 @@ impl From for Bytes { } } +impl From for bytes::Bytes { + fn from(src: Bytes) -> Self { + src.0 + } +} + impl From> for Bytes { fn from(src: Vec) -> Self { Self(src.into()) @@ -147,12 +153,12 @@ impl PartialEq for Bytes { } impl Encodable for Bytes { - fn length(&self) -> usize { - self.0.length() - } fn encode(&self, out: &mut dyn bytes::BufMut) { self.0.encode(out) } + fn length(&self) -> usize { + self.0.length() + } } impl Decodable for Bytes { From 0e9d18d4e923dc9f972c8c6eee46da489a17ce48 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sat, 4 Mar 2023 21:44:33 +0100 Subject: [PATCH 042/191] feat(rpc): support eth_call state overrides (#1634) --- crates/rpc/rpc-api/src/eth.rs | 11 ++- crates/rpc/rpc-builder/tests/it/http.rs | 2 +- crates/rpc/rpc-types/src/eth/state.rs | 2 +- crates/rpc/rpc/src/eth/api/call.rs | 89 +++++++++++++++++++++++-- crates/rpc/rpc/src/eth/api/server.rs | 12 +++- crates/rpc/rpc/src/eth/error.rs | 6 +- 6 files changed, 109 insertions(+), 13 deletions(-) diff --git a/crates/rpc/rpc-api/src/eth.rs b/crates/rpc/rpc-api/src/eth.rs index 5b6553e7318..b16cf6bf0fc 100644 --- a/crates/rpc/rpc-api/src/eth.rs +++ b/crates/rpc/rpc-api/src/eth.rs @@ -4,8 +4,8 @@ use reth_primitives::{ H256, H64, U256, U64, }; use reth_rpc_types::{ - CallRequest, EIP1186AccountProofResponse, FeeHistory, Index, RichBlock, SyncStatus, - Transaction, TransactionReceipt, TransactionRequest, Work, + state::StateOverride, CallRequest, EIP1186AccountProofResponse, FeeHistory, Index, RichBlock, + SyncStatus, Transaction, TransactionReceipt, TransactionRequest, Work, }; /// Eth rpc interface: @@ -135,7 +135,12 @@ pub trait EthApi { /// Executes a new message call immediately without creating a transaction on the block chain. #[method(name = "eth_call")] - async fn call(&self, request: CallRequest, block_number: Option) -> Result; + async fn call( + &self, + request: CallRequest, + block_number: Option, + state_overrides: Option, + ) -> Result; /// Generates an access list for a transaction. /// diff --git a/crates/rpc/rpc-builder/tests/it/http.rs b/crates/rpc/rpc-builder/tests/it/http.rs index f6eec08ba3c..c709f9d0794 100644 --- a/crates/rpc/rpc-builder/tests/it/http.rs +++ b/crates/rpc/rpc-builder/tests/it/http.rs @@ -98,7 +98,7 @@ where )); assert!(is_unimplemented(EthApiClient::transaction_receipt(client, hash).await.err().unwrap())); assert!(is_unimplemented( - EthApiClient::call(client, call_request.clone(), None).await.err().unwrap() + EthApiClient::call(client, call_request.clone(), None, None).await.err().unwrap() )); assert!(is_unimplemented( EthApiClient::create_access_list(client, call_request.clone(), None).await.err().unwrap() diff --git a/crates/rpc/rpc-types/src/eth/state.rs b/crates/rpc/rpc-types/src/eth/state.rs index c2d1edadce7..9fbc0d8a138 100644 --- a/crates/rpc/rpc-types/src/eth/state.rs +++ b/crates/rpc/rpc-types/src/eth/state.rs @@ -9,7 +9,7 @@ pub type StateOverride = HashMap; /// Custom account override used in call #[derive(Clone, Debug, PartialEq, Eq, Default, Serialize, Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] +#[serde(default, rename_all = "camelCase", deny_unknown_fields)] #[allow(missing_docs)] pub struct AccountOverride { pub nonce: Option, diff --git a/crates/rpc/rpc/src/eth/api/call.rs b/crates/rpc/rpc/src/eth/api/call.rs index 81414db12a3..7c8ffc70699 100644 --- a/crates/rpc/rpc/src/eth/api/call.rs +++ b/crates/rpc/rpc/src/eth/api/call.rs @@ -11,11 +11,15 @@ use reth_primitives::{ }; use reth_provider::{BlockProvider, EvmEnvProvider, StateProvider, StateProviderFactory}; use reth_revm::database::{State, SubState}; -use reth_rpc_types::CallRequest; +use reth_rpc_types::{ + state::{AccountOverride, StateOverride}, + CallRequest, +}; use revm::{ + db::{CacheDB, DatabaseRef}, primitives::{ - ruint::Uint, BlockEnv, CfgEnv, Env, ExecutionResult, Halt, ResultAndState, TransactTo, - TxEnv, + ruint::Uint, BlockEnv, Bytecode, CfgEnv, Env, ExecutionResult, Halt, ResultAndState, + TransactTo, TxEnv, }, Database, }; @@ -55,10 +59,11 @@ where &self, request: CallRequest, at: BlockId, + state_overrides: Option, ) -> EthResult<(ResultAndState, Env)> { let (cfg, block_env, at) = self.evm_env_at(at).await?; let state = self.state_at_block_id(at)?.ok_or_else(|| EthApiError::UnknownBlockNumber)?; - self.call_with(cfg, block_env, request, state) + self.call_with(cfg, block_env, request, state, state_overrides) } /// Executes the call request using the given environment against the state provider @@ -70,6 +75,7 @@ where block: BlockEnv, mut request: CallRequest, state: S, + state_overrides: Option, ) -> EthResult<(ResultAndState, Env)> where S: StateProvider, @@ -80,6 +86,12 @@ where let mut env = build_call_evm_env(cfg, block, request)?; let mut db = SubState::new(State::new(state)); + + // apply state overrides + if let Some(state_overrides) = state_overrides { + apply_state_overrides(state_overrides, &mut db)?; + } + transact(&mut db, env) } @@ -391,3 +403,72 @@ impl CallFees { } } } + +/// Applies the given state overrides (a set of [AccountOverride]) to the [CacheDB]. +fn apply_state_overrides(overrides: StateOverride, db: &mut CacheDB) -> EthResult<()> +where + DB: DatabaseRef, + EthApiError: From<::Error>, +{ + for (account, account_overrides) in overrides { + apply_account_override(account, account_overrides, db)?; + } + Ok(()) +} + +/// Applies a single [AccountOverride] to the [CacheDB]. +fn apply_account_override( + account: Address, + account_override: AccountOverride, + db: &mut CacheDB, +) -> EthResult<()> +where + DB: DatabaseRef, + EthApiError: From<::Error>, +{ + let mut account_info = db.basic(account)?.unwrap_or_default(); + + if let Some(nonce) = account_override.nonce { + account_info.nonce = nonce; + } + if let Some(code) = account_override.code { + account_info.code = Some(Bytecode::new_raw(code.0)); + } + if let Some(balance) = account_override.balance { + account_info.balance = balance; + } + + db.insert_account_info(account, account_info); + + // We ensure that not both state and state_diff are set. + // If state is set, we must mark the account as "NewlyCreated", so that the old storage + // isn't read from + match (account_override.state, account_override.state_diff) { + (Some(_), Some(_)) => return Err(EthApiError::BothStateAndStateDiffInOverride(account)), + (None, None) => { + // nothing to do + } + (Some(new_account_state), None) => { + db.replace_account_storage( + account, + new_account_state + .into_iter() + .map(|(slot, value)| { + (U256::from_be_bytes(slot.0), U256::from_be_bytes(value.0)) + }) + .collect(), + )?; + } + (None, Some(account_state_diff)) => { + for (slot, value) in account_state_diff { + db.insert_account_storage( + account, + U256::from_be_bytes(slot.0), + U256::from_be_bytes(value.0), + )?; + } + } + }; + + Ok(()) +} diff --git a/crates/rpc/rpc/src/eth/api/server.rs b/crates/rpc/rpc/src/eth/api/server.rs index ea8afb35564..471ff2eb166 100644 --- a/crates/rpc/rpc/src/eth/api/server.rs +++ b/crates/rpc/rpc/src/eth/api/server.rs @@ -14,8 +14,9 @@ use reth_primitives::{ use reth_provider::{BlockProvider, EvmEnvProvider, HeaderProvider, StateProviderFactory}; use reth_rpc_api::EthApiServer; use reth_rpc_types::{ - CallRequest, EIP1186AccountProofResponse, FeeHistory, FeeHistoryCacheItem, Index, RichBlock, - SyncStatus, TransactionReceipt, TransactionRequest, Work, + state::StateOverride, CallRequest, EIP1186AccountProofResponse, FeeHistory, + FeeHistoryCacheItem, Index, RichBlock, SyncStatus, TransactionReceipt, TransactionRequest, + Work, }; use reth_transaction_pool::TransactionPool; use serde_json::Value; @@ -177,7 +178,12 @@ where } /// Handler for: `eth_call` - async fn call(&self, _request: CallRequest, _block_number: Option) -> Result { + async fn call( + &self, + _request: CallRequest, + _block_number: Option, + _state_overrides: Option, + ) -> Result { Err(internal_rpc_err("unimplemented")) } diff --git a/crates/rpc/rpc/src/eth/error.rs b/crates/rpc/rpc/src/eth/error.rs index ee1e42ab0eb..8a080c7b52a 100644 --- a/crates/rpc/rpc/src/eth/error.rs +++ b/crates/rpc/rpc/src/eth/error.rs @@ -2,7 +2,7 @@ use crate::result::{internal_rpc_err, rpc_err}; use jsonrpsee::{core::Error as RpcError, types::error::INVALID_PARAMS_CODE}; -use reth_primitives::{constants::SELECTOR_LEN, U128, U256}; +use reth_primitives::{constants::SELECTOR_LEN, Address, U128, U256}; use reth_rpc_types::BlockError; use reth_transaction_pool::error::PoolError; use revm::primitives::{EVMError, Halt}; @@ -67,6 +67,10 @@ pub(crate) enum EthApiError { /// Thrown when constructing an RPC block from a primitive block data failed. #[error(transparent)] InvalidBlockData(#[from] BlockError), + /// Thrown when a [AccountOverride](reth_rpc_types::state::AccountOverride) contains + /// conflicting `state` and `stateDiff` fields + #[error("account {0:?} has both 'state' and 'stateDiff'")] + BothStateAndStateDiffInOverride(Address), /// Other internal error #[error(transparent)] Internal(#[from] reth_interfaces::Error), From 10d24b5cd634e849799a563240b5ebfb017ca976 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sat, 4 Mar 2023 21:44:59 +0100 Subject: [PATCH 043/191] chore: spawn cache task as critical (#1633) --- crates/rpc/rpc/src/eth/cache.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/rpc/rpc/src/eth/cache.rs b/crates/rpc/rpc/src/eth/cache.rs index 3990e9dc35e..71706dc7cc4 100644 --- a/crates/rpc/rpc/src/eth/cache.rs +++ b/crates/rpc/rpc/src/eth/cache.rs @@ -108,7 +108,7 @@ impl EthStateCache { let EthStateCacheConfig { max_block_bytes, max_env_bytes } = config; let (this, service) = Self::create(client, executor.clone(), max_block_bytes, max_env_bytes); - executor.spawn(Box::pin(service)); + executor.spawn_critical("eth state cache", Box::pin(service)); this } From 446d76bc295d0ce1e355eb4536bf87013ec878cb Mon Sep 17 00:00:00 2001 From: grantkee <49913008+grantkee@users.noreply.github.com> Date: Sat, 4 Mar 2023 14:45:32 -0600 Subject: [PATCH 044/191] feat: Genesis Setters (#1632) --- crates/primitives/src/genesis.rs | 181 ++++++++++++++++++++++++++++++- 1 file changed, 180 insertions(+), 1 deletion(-) diff --git a/crates/primitives/src/genesis.rs b/crates/primitives/src/genesis.rs index 4fd8752f58e..8ae6bfce357 100644 --- a/crates/primitives/src/genesis.rs +++ b/crates/primitives/src/genesis.rs @@ -12,7 +12,7 @@ use serde::{Deserialize, Serialize}; use triehash::sec_trie_root; /// The genesis block specification. -#[derive(Clone, Debug, Default, Serialize, Deserialize)] +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "camelCase", default)] pub struct Genesis { /// The genesis header nonce. @@ -36,6 +36,60 @@ pub struct Genesis { pub alloc: HashMap, } +impl Genesis { + /// Set the nonce. + pub fn with_nonce(mut self, nonce: u64) -> Self { + self.nonce = nonce; + self + } + + /// Set the timestamp. + pub fn with_timestamp(mut self, timestamp: u64) -> Self { + self.timestamp = timestamp; + self + } + + /// Set the extra data. + pub fn with_extra_data(mut self, extra_data: Bytes) -> Self { + self.extra_data = extra_data; + self + } + + /// Set the gas limit. + pub fn with_gas_limit(mut self, gas_limit: u64) -> Self { + self.gas_limit = gas_limit; + self + } + + /// Set the difficulty. + pub fn with_difficulty(mut self, difficulty: U256) -> Self { + self.difficulty = difficulty; + self + } + + /// Set the mix hash of the header. + pub fn with_mix_hash(mut self, mix_hash: H256) -> Self { + self.mix_hash = mix_hash; + self + } + + /// Set the coinbase address. + pub fn with_coinbase(mut self, address: Address) -> Self { + self.coinbase = address; + self + } + + /// Add accounts to the genesis block. If the address is already present, + /// the account is updated. + pub fn extend_accounts( + mut self, + accounts: impl IntoIterator, + ) -> Self { + self.alloc.extend(accounts); + self + } +} + /// An account in the state of the genesis block. #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct GenesisAccount { @@ -65,6 +119,30 @@ impl GenesisAccount { len += KECCAK_EMPTY.length(); len } + + /// Set the nonce. + pub fn with_nonce(mut self, nonce: Option) -> Self { + self.nonce = nonce; + self + } + + /// Set the balance. + pub fn with_balance(mut self, balance: U256) -> Self { + self.balance = balance; + self + } + + /// Set the code. + pub fn with_code(mut self, code: Option) -> Self { + self.code = code; + self + } + + /// Set the storage. + pub fn with_storage(mut self, storage: Option>) -> Self { + self.storage = storage; + self + } } impl Encodable for GenesisAccount { @@ -110,3 +188,104 @@ impl From for GenesisAccount { } } } + +#[cfg(test)] +mod tests { + use super::*; + use hex_literal::hex; + + #[test] + fn test_genesis() { + let default_genesis = Genesis::default(); + + let nonce = 999; + let timestamp = 12345; + let extra_data = Bytes::from(b"extra-data"); + let gas_limit = 333333; + let difficulty = U256::from(9000); + let mix_hash = + hex!("74385b512f1e0e47100907efe2b00ac78df26acba6dd16b0772923068a5801a8").into(); + let coinbase = hex!("265873b6faf3258b3ab0827805386a2a20ed040e").into(); + // create dummy account + let first_address: Address = hex!("7618a8c597b89e01c66a1f662078992c52a30c9a").into(); + let mut account = HashMap::default(); + account.insert(first_address, GenesisAccount::default()); + + // check values updated + let custom_genesis = Genesis::default() + .with_nonce(nonce) + .with_timestamp(timestamp) + .with_extra_data(extra_data.clone()) + .with_gas_limit(gas_limit) + .with_difficulty(difficulty) + .with_mix_hash(mix_hash) + .with_coinbase(coinbase) + .extend_accounts(account.clone()); + + assert_ne!(custom_genesis, default_genesis); + // check every field + assert_eq!(custom_genesis.nonce, nonce); + assert_eq!(custom_genesis.timestamp, timestamp); + assert_eq!(custom_genesis.extra_data, extra_data); + assert_eq!(custom_genesis.gas_limit, gas_limit); + assert_eq!(custom_genesis.difficulty, difficulty); + assert_eq!(custom_genesis.mix_hash, mix_hash); + assert_eq!(custom_genesis.coinbase, coinbase); + assert_eq!(custom_genesis.alloc, account.clone()); + + // update existing account + assert_eq!(custom_genesis.alloc.len(), 1); + let same_address = first_address; + let new_alloc_account = GenesisAccount { + nonce: Some(1), + balance: U256::from(1), + code: Some(Bytes::from(b"code")), + storage: Some(HashMap::default()), + }; + let mut updated_account = HashMap::default(); + updated_account.insert(same_address, new_alloc_account); + let custom_genesis = custom_genesis.extend_accounts(updated_account.clone()); + assert_ne!(account, updated_account); + assert_eq!(custom_genesis.alloc.len(), 1); + + // add second account + let different_address = hex!("94e0681e3073dd71cec54b53afe988f39078fd1a").into(); + let more_accounts = HashMap::from([(different_address, GenesisAccount::default())]); + let custom_genesis = custom_genesis.extend_accounts(more_accounts); + assert_eq!(custom_genesis.alloc.len(), 2); + + // ensure accounts are different + let first_account = custom_genesis.alloc.get(&first_address); + let second_account = custom_genesis.alloc.get(&different_address); + assert!(first_account.is_some()); + assert!(second_account.is_some()); + assert_ne!(first_account, second_account); + } + + #[test] + fn test_genesis_account() { + let default_account = GenesisAccount::default(); + + let nonce = Some(1); + let balance = U256::from(33); + let code = Some(Bytes::from(b"code")); + let root = hex!("9474ddfcea39c5a690d2744103e39d1ff1b03d18db10fc147d970ad24699395a").into(); + let value = hex!("58eb8294d9bb16832a9dabfcb270fff99ab8ee1d8764e4f3d9fdf59ec1dee469").into(); + let mut map = HashMap::default(); + map.insert(root, value); + let storage = Some(map); + + let genesis_account = GenesisAccount::default() + .with_nonce(nonce) + .with_balance(balance) + .with_code(code.clone()) + .with_storage(storage.clone()); + + assert_ne!(default_account, genesis_account); + // check every field + assert_eq!(genesis_account.nonce, nonce); + assert_eq!(genesis_account.balance, balance); + assert_eq!(genesis_account.code, code); + assert_eq!(genesis_account.storage, storage); + } +} From 7b71af24097f16b0337fc97f8a7c83bfb05cb2d9 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sat, 4 Mar 2023 21:51:35 +0100 Subject: [PATCH 045/191] test: add empty decode buffer check (#1628) --- crates/net/eth-wire/src/p2pstream.rs | 18 ++++++++++++++++-- crates/storage/codecs/derive/src/arbitrary.rs | 10 ++++++---- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/crates/net/eth-wire/src/p2pstream.rs b/crates/net/eth-wire/src/p2pstream.rs index a5990950183..9a487cc3783 100644 --- a/crates/net/eth-wire/src/p2pstream.rs +++ b/crates/net/eth-wire/src/p2pstream.rs @@ -698,8 +698,22 @@ impl Encodable for P2PMessage { /// The [`Decodable`](reth_rlp::Decodable) implementation for [`P2PMessage`] assumes that each of /// the message variants are snappy compressed, except for the [`P2PMessage::Hello`] variant since /// the hello message is never compressed in the `p2p` subprotocol. +/// The [`Decodable`] implementation for [`P2PMessage::Ping`] and +/// [`P2PMessage::Pong`] expects a snappy encoded payload, see [`Encodable`] implementation. impl Decodable for P2PMessage { fn decode(buf: &mut &[u8]) -> Result { + /// Removes the snappy prefix from the Ping/Pong buffer + fn advance_snappy_ping_pong_payload(buf: &mut &[u8]) -> Result<(), DecodeError> { + if buf.len() < 3 { + return Err(DecodeError::InputTooShort) + } + if buf[..3] != [0x01, 0x00, EMPTY_LIST_CODE] { + return Err(DecodeError::Custom("expected snappy payload")) + } + buf.advance(3); + Ok(()) + } + let message_id = u8::decode(&mut &buf[..])?; let id = P2PMessageID::try_from(message_id) .or(Err(DecodeError::Custom("unknown p2p message id")))?; @@ -708,11 +722,11 @@ impl Decodable for P2PMessage { P2PMessageID::Hello => Ok(P2PMessage::Hello(HelloMessage::decode(buf)?)), P2PMessageID::Disconnect => Ok(P2PMessage::Disconnect(DisconnectReason::decode(buf)?)), P2PMessageID::Ping => { - buf.advance(1); + advance_snappy_ping_pong_payload(buf)?; Ok(P2PMessage::Ping) } P2PMessageID::Pong => { - buf.advance(1); + advance_snappy_ping_pong_payload(buf)?; Ok(P2PMessage::Pong) } } diff --git a/crates/storage/codecs/derive/src/arbitrary.rs b/crates/storage/codecs/derive/src/arbitrary.rs index 96c259b49ff..61f82057b59 100644 --- a/crates/storage/codecs/derive/src/arbitrary.rs +++ b/crates/storage/codecs/derive/src/arbitrary.rs @@ -36,10 +36,12 @@ pub fn maybe_generate_tests(args: TokenStream, ast: &DeriveInput) -> TokenStream { let mut buf = vec![]; - let len = field.clone().encode(&mut buf); - let decoded = super::#type_ident::decode(&mut buf.as_slice()).unwrap(); - - assert!(field == decoded); + let len = field.encode(&mut buf); + let mut b = &mut buf.as_slice(); + let decoded = super::#type_ident::decode(b).unwrap(); + assert_eq!(field, decoded); + // ensure buffer is fully consumed by decode + assert!(b.is_empty()); } }); } else if let Ok(num) = arg.to_string().parse() { From 31f62c11d5544ac8aaa78a10f6f5a4201c64cd61 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sat, 4 Mar 2023 21:52:12 +0100 Subject: [PATCH 046/191] test: add test for display help (#1623) --- bin/reth/src/cli.rs | 28 ++++++++++++++++++++++++---- bin/reth/src/node/mod.rs | 6 ++++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/bin/reth/src/cli.rs b/bin/reth/src/cli.rs index 9f6a5e46335..4afa9093921 100644 --- a/bin/reth/src/cli.rs +++ b/bin/reth/src/cli.rs @@ -38,7 +38,7 @@ pub fn run() -> eyre::Result<()> { } /// Commands to be executed -#[derive(Subcommand)] +#[derive(Debug, Subcommand)] pub enum Commands { /// Start the node #[command(name = "node")] @@ -74,7 +74,7 @@ pub enum Commands { TestVectors(test_vectors::Command), } -#[derive(Parser)] +#[derive(Debug, Parser)] #[command(author, version = "0.1", about = "Reth", long_about = None)] struct Cli { /// The command to run @@ -89,7 +89,7 @@ struct Cli { } /// The log configuration. -#[derive(Args)] +#[derive(Debug, Args)] #[command(next_help_heading = "Logging")] pub struct Logs { /// The path to put log files in. @@ -131,7 +131,7 @@ impl Logs { } /// The verbosity settings for the cli. -#[derive(Args)] +#[derive(Debug, Copy, Clone, Args)] #[command(next_help_heading = "Display")] pub struct Verbosity { /// Set the minimum log level. @@ -168,3 +168,23 @@ impl Verbosity { } } } + +#[cfg(test)] +mod tests { + use super::*; + use clap::CommandFactory; + + /// Tests that the help message is parsed correctly. This ensures that clap args are configured + /// correctly and no conflicts are introduced via attributes that would result in a panic at + /// runtime + #[test] + fn test_parse_help_all_subcommands() { + let reth = Cli::command(); + for sub_command in reth.get_subcommands() { + let err = Cli::try_parse_from(["reth", sub_command.get_name(), "--help"]).unwrap_err(); + // --help is treated as error, but + // > Not a true "error" as it means --help or similar was used. The help message will be sent to stdout. + assert_eq!(err.kind(), clap::error::ErrorKind::DisplayHelp); + } + } +} diff --git a/bin/reth/src/node/mod.rs b/bin/reth/src/node/mod.rs index 1c8753b6082..5852caec680 100644 --- a/bin/reth/src/node/mod.rs +++ b/bin/reth/src/node/mod.rs @@ -504,6 +504,12 @@ async fn run_network_until_shutdown( mod tests { use super::*; + #[test] + fn parse_help_node_command() { + let err = Command::try_parse_from(["reth", "--help"]).unwrap_err(); + assert_eq!(err.kind(), clap::error::ErrorKind::DisplayHelp); + } + #[test] fn parse_common_node_command_chain_args() { for chain in ["mainnet", "sepolia", "goerli"] { From 9dfee497e9dc7267a905aeb9b7e514cabd1b100c Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sun, 5 Mar 2023 13:33:19 +0100 Subject: [PATCH 047/191] chore: add tx type constants (#1639) Co-authored-by: Roman Krasiuk --- crates/primitives/src/lib.rs | 2 +- crates/primitives/src/transaction/mod.rs | 2 +- crates/primitives/src/transaction/tx_type.rs | 16 +++++++++++++--- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/crates/primitives/src/lib.rs b/crates/primitives/src/lib.rs index 722ce553c24..3b0e600ea77 100644 --- a/crates/primitives/src/lib.rs +++ b/crates/primitives/src/lib.rs @@ -61,7 +61,7 @@ pub use storage::{StorageEntry, StorageTrieEntry}; pub use transaction::{ AccessList, AccessListItem, FromRecoveredTransaction, IntoRecoveredTransaction, Signature, Transaction, TransactionKind, TransactionSigned, TransactionSignedEcRecovered, TxEip1559, - TxEip2930, TxLegacy, TxType, + TxEip2930, TxLegacy, TxType, EIP1559_TX_TYPE_ID, EIP2930_TX_TYPE_ID, LEGACY_TX_TYPE_ID, }; pub use withdrawal::Withdrawal; diff --git a/crates/primitives/src/transaction/mod.rs b/crates/primitives/src/transaction/mod.rs index 70ad0ef3496..12b495a7636 100644 --- a/crates/primitives/src/transaction/mod.rs +++ b/crates/primitives/src/transaction/mod.rs @@ -7,7 +7,7 @@ use reth_rlp::{ length_of_length, Decodable, DecodeError, Encodable, Header, EMPTY_LIST_CODE, EMPTY_STRING_CODE, }; pub use signature::Signature; -pub use tx_type::TxType; +pub use tx_type::{TxType, EIP1559_TX_TYPE_ID, EIP2930_TX_TYPE_ID, LEGACY_TX_TYPE_ID}; mod access_list; mod signature; diff --git a/crates/primitives/src/transaction/tx_type.rs b/crates/primitives/src/transaction/tx_type.rs index c2d86fe6879..e73e230a46c 100644 --- a/crates/primitives/src/transaction/tx_type.rs +++ b/crates/primitives/src/transaction/tx_type.rs @@ -1,6 +1,16 @@ use reth_codecs::{derive_arbitrary, Compact}; use serde::{Deserialize, Serialize}; +/// Identifier for legacy transaction, however [TxLegacy](crate::TxLegacy) this is technically not +/// typed. +pub const LEGACY_TX_TYPE_ID: u8 = 0; + +/// Identifier for [TxEip2930](crate::TxEip2930) transaction. +pub const EIP2930_TX_TYPE_ID: u8 = 1; + +/// Identifier for [TxEip1559](crate::TxEip1559) transaction. +pub const EIP1559_TX_TYPE_ID: u8 = 2; + /// Transaction Type #[derive_arbitrary(compact)] #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Default, Serialize, Deserialize)] @@ -17,9 +27,9 @@ pub enum TxType { impl From for u8 { fn from(value: TxType) -> Self { match value { - TxType::Legacy => 0, - TxType::EIP2930 => 1, - TxType::EIP1559 => 2, + TxType::Legacy => LEGACY_TX_TYPE_ID, + TxType::EIP2930 => EIP2930_TX_TYPE_ID, + TxType::EIP1559 => EIP1559_TX_TYPE_ID, } } } From e58fb6714f2de939ffa90b70ffbef96cc2f7a20f Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sun, 5 Mar 2023 16:01:45 +0100 Subject: [PATCH 048/191] chore(deps): cargo update (#1637) --- Cargo.lock | 477 ++++++++++++++++++++++++----------------------------- 1 file changed, 218 insertions(+), 259 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 060f8738579..776cd62b6d2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -138,15 +138,15 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anyhow" -version = "1.0.68" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cb2f989d18dd141ab8ae82f64d1a8cdd37e0840f73a406896cf5e99502fab61" +checksum = "224afbd727c3d6e4b90103ece64b8d1b67fbb1973b1046c2281eed3f3803f800" [[package]] name = "aquamarine" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d7cba9b073f771a6f76683be98624dd68e867ff9e6adcad3afbb3d2044c3afa" +checksum = "5bf310f0dd77f453bc43ec61506ead8283ab49423686244e474d0cc16226400c" dependencies = [ "itertools 0.9.0", "proc-macro-error", @@ -192,29 +192,29 @@ dependencies = [ [[package]] name = "async-lock" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8101efe8695a6c17e02911402145357e718ac92d3ff88ae8419e84b1707b685" +checksum = "fa24f727524730b077666307f2734b4a1a1c57acb79193127dcc8914d5242dd7" dependencies = [ "event-listener", - "futures-lite", ] [[package]] name = "async-stream" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dad5c83079eae9969be7fadefe640a1c566901f05ff91ab221de4b6f68d9507e" +checksum = "ad445822218ce64be7a341abfb0b1ea43b5c23aa83902542a4542e78309d8e5e" dependencies = [ "async-stream-impl", "futures-core", + "pin-project-lite", ] [[package]] name = "async-stream-impl" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10f203db73a71dfa2fb6dd22763990fa26f3d2625a6da2da900d23b87d26be27" +checksum = "e4655ae1a7b0cdf149156f780c5bf3f1352bc53cbd9e0a361a7ef7b22947e965" dependencies = [ "proc-macro2 1.0.51", "quote 1.0.23", @@ -223,9 +223,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.64" +version = "0.1.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd7fce9ba8c3c042128ce72d8b2ddbf3a05747efb67ea0313c635e10bda47a2" +checksum = "095183a3539c7c7649b2beb87c2d3f0591f3a7fed07761cc546d244e27e0238c" dependencies = [ "proc-macro2 1.0.51", "quote 1.0.23", @@ -362,9 +362,9 @@ checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" [[package]] name = "base64ct" -version = "1.5.3" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b645a089122eccb6111b4f81cbc1a49f5900ac4666bb93ac027feaecf15607bf" +checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" [[package]] name = "basic-toml" @@ -458,6 +458,7 @@ checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" dependencies = [ "funty", "radium 0.7.0", + "serde", "tap", "wyz", ] @@ -548,9 +549,9 @@ checksum = "771fe0050b883fcc3ea2359b1a96bcfbc090b7116eae7c3c512c7a083fdf23d3" [[package]] name = "bstr" -version = "1.1.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b45ea9b00a7b3f2988e9a65ad3917e62123c38dba709b666506207be96d1790b" +checksum = "5ffdb39cb703212f3c11973452c2861b972f757b021158f3516ba10f2fa8b2c1" dependencies = [ "memchr", "serde", @@ -576,9 +577,9 @@ checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" [[package]] name = "bytemuck" -version = "1.13.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c041d3eab048880cb0b86b256447da3f18859a163c3b8d8893f4e6368abe6393" +checksum = "17febce684fd15d89027105661fec94afb475cb995fbc59d2865198446ba2eea" [[package]] name = "byteorder" @@ -597,9 +598,9 @@ dependencies = [ [[package]] name = "camino" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c77df041dc383319cc661b428b6961a005db4d6808d5e12536931b1ca9556055" +checksum = "6031a462f977dd38968b6f23378356512feeace69cef817e1a4475108093cec3" dependencies = [ "serde", ] @@ -732,9 +733,9 @@ dependencies = [ [[package]] name = "clang-sys" -version = "1.4.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa2e27ae6ab525c3d369ded447057bca5438d86dc3a68f6faafb8269ba82ebf3" +checksum = "77ed9a53e5d4d9c573ae844bfac6872b159cb1d1585a83b29e7a64b7eef7332a" dependencies = [ "glob", "libc", @@ -910,9 +911,9 @@ dependencies = [ [[package]] name = "const-oid" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cec318a675afcb6a1ea1d4340e2d377e56e47c266f28043ceccbf4412ddfdd3b" +checksum = "520fbf3c07483f94e3e3ca9d0cfd913d7718ef2483d2cfd91c0d9e91474ab913" [[package]] name = "convert_case" @@ -1032,9 +1033,9 @@ checksum = "6548a0ad5d2549e111e1f6a11a6c2e2d00ce6a3dafe22948d67c2b443f775e52" [[package]] name = "crossbeam-channel" -version = "0.5.6" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" +checksum = "cf2b3e8478797446514c91ef04bafcb59faba183e621ad488df88983cc14128c" dependencies = [ "cfg-if", "crossbeam-utils", @@ -1042,9 +1043,9 @@ dependencies = [ [[package]] name = "crossbeam-deque" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" dependencies = [ "cfg-if", "crossbeam-epoch", @@ -1053,9 +1054,9 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.13" +version = "0.9.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a" +checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695" dependencies = [ "autocfg", "cfg-if", @@ -1066,9 +1067,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.14" +version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f" +checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" dependencies = [ "cfg-if", ] @@ -1159,9 +1160,9 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.88" +version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "322296e2f2e5af4270b54df9e85a02ff037e271af20ba3e7fe1575515dc840b8" +checksum = "86d3488e7665a7a483b57e25bdd90d0aeb2bc7608c8d0346acf2ad3f1caf1d62" dependencies = [ "cc", "cxxbridge-flags", @@ -1171,9 +1172,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.88" +version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "017a1385b05d631e7875b1f151c9f012d37b53491e2a87f65bff5c262b2111d8" +checksum = "48fcaf066a053a41a81dfb14d57d99738b767febb8b735c3016e469fac5da690" dependencies = [ "cc", "codespan-reporting", @@ -1186,15 +1187,15 @@ dependencies = [ [[package]] name = "cxxbridge-flags" -version = "1.0.88" +version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c26bbb078acf09bc1ecda02d4223f03bdd28bd4874edcb0379138efc499ce971" +checksum = "a2ef98b8b717a829ca5603af80e1f9e2e48013ab227b68ef37872ef84ee479bf" [[package]] name = "cxxbridge-macro" -version = "1.0.88" +version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "357f40d1f06a24b60ae1fe122542c1fb05d28d32acb2aed064e84bc2ad1e252e" +checksum = "086c685979a698443656e5cf7856c95c642295a38599f12fb1ff76fb28d19892" dependencies = [ "proc-macro2 1.0.51", "quote 1.0.23", @@ -1213,12 +1214,12 @@ dependencies = [ [[package]] name = "darling" -version = "0.14.2" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0dd3cd20dc6b5a876612a6e5accfe7f3dd883db6d07acfbf14c128f61550dfa" +checksum = "c0808e1bd8671fb44a113a14e13497557533369847788fa2ae912b6ebfce9fa8" dependencies = [ - "darling_core 0.14.2", - "darling_macro 0.14.2", + "darling_core 0.14.3", + "darling_macro 0.14.3", ] [[package]] @@ -1237,9 +1238,9 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.14.2" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a784d2ccaf7c98501746bf0be29b2022ba41fd62a2e622af997a03e9f972859f" +checksum = "001d80444f28e193f30c2f293455da62dcf9a6b29918a4253152ae2b1de592cb" dependencies = [ "fnv", "ident_case", @@ -1262,11 +1263,11 @@ dependencies = [ [[package]] name = "darling_macro" -version = "0.14.2" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7618812407e9402654622dd402b0a89dff9ba93badd6540781526117b92aab7e" +checksum = "b36230598a2d5de7ec1c6f51f72d8a99a9208daff41de2084d06e3fd3ea56685" dependencies = [ - "darling_core 0.14.2", + "darling_core 0.14.3", "quote 1.0.23", "syn 1.0.109", ] @@ -1281,7 +1282,7 @@ dependencies = [ "hashbrown 0.12.3", "lock_api", "once_cell", - "parking_lot_core 0.9.6", + "parking_lot_core 0.9.7", ] [[package]] @@ -1449,14 +1450,14 @@ dependencies = [ [[package]] name = "discv5" -version = "0.1.0" -source = "git+https://github.com/sigp/discv5#97a806ccf7817a420b5f43efa23e6127b475d839" +version = "0.2.1" +source = "git+https://github.com/sigp/discv5#e3a6fe7c6efcdfb52b0782c232ef7a3659d46e80" dependencies = [ "aes 0.7.5", "aes-gcm", "arrayvec", "delay_map", - "enr 0.6.2", + "enr", "fnv", "futures", "hashlink", @@ -1499,9 +1500,9 @@ checksum = "0bd4b30a6560bbd9b4620f4de34c3f14f60848e58a9b7216801afcb4c7b31c3c" [[package]] name = "dyn-clone" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9b0705efd4599c15a38151f4721f7bc388306f61084d3bfd50bd07fbca5cb60" +checksum = "68b0cf012f1230e43cd00ebb729c6bb58707ecfa8ad08b52ef3a4ccd2697fc30" [[package]] name = "ecdsa" @@ -1578,9 +1579,9 @@ dependencies = [ [[package]] name = "encoding_rs" -version = "0.8.31" +version = "0.8.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b" +checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" dependencies = [ "cfg-if", ] @@ -1591,26 +1592,6 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" -[[package]] -name = "enr" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26fa0a0be8915790626d5759eb51fe47435a8eac92c2f212bd2da9aa7f30ea56" -dependencies = [ - "base64 0.13.1", - "bs58", - "bytes", - "ed25519-dalek", - "hex", - "k256", - "log", - "rand 0.8.5", - "rlp", - "serde", - "sha3", - "zeroize", -] - [[package]] name = "enr" version = "0.7.0" @@ -1620,6 +1601,7 @@ dependencies = [ "base64 0.13.1", "bs58", "bytes", + "ed25519-dalek", "hex", "k256", "log", @@ -1774,7 +1756,7 @@ dependencies = [ [[package]] name = "ethers-contract" version = "1.0.2" -source = "git+https://github.com/gakonst/ethers-rs#73636a906e40810beb3b55b305f5e81640478a01" +source = "git+https://github.com/gakonst/ethers-rs#e54666b467c9cc5848873935622930449075828a" dependencies = [ "ethers-contract-abigen", "ethers-contract-derive", @@ -1792,7 +1774,7 @@ dependencies = [ [[package]] name = "ethers-contract-abigen" version = "1.0.2" -source = "git+https://github.com/gakonst/ethers-rs#73636a906e40810beb3b55b305f5e81640478a01" +source = "git+https://github.com/gakonst/ethers-rs#e54666b467c9cc5848873935622930449075828a" dependencies = [ "Inflector", "cfg-if", @@ -1819,7 +1801,7 @@ dependencies = [ [[package]] name = "ethers-contract-derive" version = "1.0.2" -source = "git+https://github.com/gakonst/ethers-rs#73636a906e40810beb3b55b305f5e81640478a01" +source = "git+https://github.com/gakonst/ethers-rs#e54666b467c9cc5848873935622930449075828a" dependencies = [ "ethers-contract-abigen", "ethers-core", @@ -1834,7 +1816,7 @@ dependencies = [ [[package]] name = "ethers-core" version = "1.0.2" -source = "git+https://github.com/gakonst/ethers-rs#73636a906e40810beb3b55b305f5e81640478a01" +source = "git+https://github.com/gakonst/ethers-rs#e54666b467c9cc5848873935622930449075828a" dependencies = [ "arrayvec", "bytes", @@ -1867,7 +1849,7 @@ dependencies = [ [[package]] name = "ethers-etherscan" version = "1.0.2" -source = "git+https://github.com/gakonst/ethers-rs#73636a906e40810beb3b55b305f5e81640478a01" +source = "git+https://github.com/gakonst/ethers-rs#e54666b467c9cc5848873935622930449075828a" dependencies = [ "ethers-core", "getrandom 0.2.8", @@ -1883,7 +1865,7 @@ dependencies = [ [[package]] name = "ethers-middleware" version = "1.0.2" -source = "git+https://github.com/gakonst/ethers-rs#73636a906e40810beb3b55b305f5e81640478a01" +source = "git+https://github.com/gakonst/ethers-rs#e54666b467c9cc5848873935622930449075828a" dependencies = [ "async-trait", "auto_impl", @@ -1908,12 +1890,12 @@ dependencies = [ [[package]] name = "ethers-providers" version = "1.0.2" -source = "git+https://github.com/gakonst/ethers-rs#73636a906e40810beb3b55b305f5e81640478a01" +source = "git+https://github.com/gakonst/ethers-rs#e54666b467c9cc5848873935622930449075828a" dependencies = [ "async-trait", "auto_impl", "base64 0.21.0", - "enr 0.7.0", + "enr", "ethers-core", "futures-channel", "futures-core", @@ -1945,7 +1927,7 @@ dependencies = [ [[package]] name = "ethers-signers" version = "1.0.2" -source = "git+https://github.com/gakonst/ethers-rs#73636a906e40810beb3b55b305f5e81640478a01" +source = "git+https://github.com/gakonst/ethers-rs#e54666b467c9cc5848873935622930449075828a" dependencies = [ "async-trait", "coins-bip32", @@ -1990,9 +1972,9 @@ checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" [[package]] name = "fastrand" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a407cfaa3385c4ae6b23e84623d48c2798d06e3e6a1878f7f59f17b3f86499" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" dependencies = [ "instant", ] @@ -2119,21 +2101,6 @@ version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfb8371b6fb2aeb2d280374607aeabfc99d95c72edfe51692e42d3d7f0d08531" -[[package]] -name = "futures-lite" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7694489acd39452c77daa48516b894c153f192c3578d5a839b62c58099fcbf48" -dependencies = [ - "fastrand", - "futures-core", - "futures-io", - "memchr", - "parking", - "pin-project-lite", - "waker-fn", -] - [[package]] name = "futures-locks" version = "0.7.1" @@ -2340,9 +2307,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.15" +version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f9f29bc9dda355256b2916cf526ab02ce0aeaaaf2bad60d65ef3f12f11dd0f4" +checksum = "5be7b54589b581f624f566bf5d8eb2bab1db736c51528720b6bd36b96b55924d" dependencies = [ "bytes", "fnv", @@ -2353,7 +2320,7 @@ dependencies = [ "indexmap", "slab", "tokio", - "tokio-util 0.7.4", + "tokio-util 0.7.7", "tracing", ] @@ -2453,15 +2420,15 @@ dependencies = [ "hash32", "rustc_version", "serde", - "spin 0.9.4", + "spin 0.9.5", "stable_deref_trait", ] [[package]] name = "heck" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" @@ -2542,9 +2509,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" +checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" dependencies = [ "bytes", "fnv", @@ -2833,12 +2800,12 @@ dependencies = [ [[package]] name = "io-lifetimes" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7d6c6f8c91b4b9ed43484ad1a938e393caf35960fce7f82a040497207bd8e9e" +checksum = "1abeb7a0dd0f8181267ff8adc397075586500b81b28a73e8a0208b00fc170fb3" dependencies = [ "libc", - "windows-sys 0.42.0", + "windows-sys 0.45.0", ] [[package]] @@ -2870,9 +2837,9 @@ dependencies = [ [[package]] name = "is-terminal" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22e18b0a45d56fe973d6db23972bf5bc46f988a4a2385deac9cc29572f09daef" +checksum = "21b6b32576413a8e69b90e952e4a026476040d81017b80445deda5f2d3921857" dependencies = [ "hermit-abi 0.3.1", "io-lifetimes", @@ -2900,15 +2867,15 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" +checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" [[package]] name = "js-sys" -version = "0.3.60" +version = "0.3.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47" +checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" dependencies = [ "wasm-bindgen", ] @@ -2950,7 +2917,7 @@ dependencies = [ "thiserror", "tokio", "tokio-rustls", - "tokio-util 0.7.4", + "tokio-util 0.7.7", "tracing", "webpki-roots", ] @@ -3033,7 +3000,7 @@ dependencies = [ "soketto", "tokio", "tokio-stream", - "tokio-util 0.7.4", + "tokio-util 0.7.7", "tower", "tracing", ] @@ -3271,18 +3238,18 @@ checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "memmap2" -version = "0.5.8" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b182332558b18d807c4ce1ca8ca983b34c3ee32765e47b3f0f69b90355cc1dc" +checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327" dependencies = [ "libc", ] [[package]] name = "memoffset" -version = "0.7.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" +checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" dependencies = [ "autocfg", ] @@ -3382,14 +3349,14 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de" +checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" dependencies = [ "libc", "log", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.42.0", + "windows-sys 0.45.0", ] [[package]] @@ -3430,13 +3397,14 @@ dependencies = [ [[package]] name = "nix" -version = "0.24.3" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa52e972a9a719cecb6864fb88568781eb706bac2cd1d4f04a648542dbf78069" +checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a" dependencies = [ "bitflags", "cfg-if", "libc", + "static_assertions", ] [[package]] @@ -3449,15 +3417,6 @@ dependencies = [ "minimal-lexical", ] -[[package]] -name = "nom8" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae01545c9c7fc4486ab7debaf2aad7003ac19431791868fb2e8066df97fad2f8" -dependencies = [ - "memchr", -] - [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -3683,9 +3642,9 @@ dependencies = [ [[package]] name = "parity-scale-codec" -version = "3.3.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3840933452adf7b3b9145e27086a5a3376c619dca1a21b1e5a5af0d54979bed" +checksum = "637935964ff85a605d114591d4d2c13c5d1ba2806dae97cea6bf180238a749ac" dependencies = [ "arrayvec", "bitvec 1.0.1", @@ -3722,12 +3681,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "parking" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" - [[package]] name = "parking_lot" version = "0.11.2" @@ -3746,7 +3699,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ "lock_api", - "parking_lot_core 0.9.6", + "parking_lot_core 0.9.7", ] [[package]] @@ -3765,15 +3718,15 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.6" +version = "0.9.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba1ef8814b5c993410bb3adfad7a5ed269563e4a2f90c41f5d85be7fb47133bf" +checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" dependencies = [ "cfg-if", "libc", "redox_syscall", "smallvec", - "windows-sys 0.42.0", + "windows-sys 0.45.0", ] [[package]] @@ -3828,9 +3781,9 @@ checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" [[package]] name = "pest" -version = "2.5.4" +version = "2.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ab62d2fa33726dbe6321cc97ef96d8cde531e3eeaf858a058de53a8a6d40d8f" +checksum = "8cbd939b234e95d72bc393d51788aec68aeeb5d51e748ca08ff3aad58cb722f7" dependencies = [ "thiserror", "ucd-trie", @@ -3945,9 +3898,9 @@ checksum = "26f6a7b87c2e435a3241addceeeff740ff8b7e76b74c13bf9acb17fa454ea00b" [[package]] name = "postcard" -version = "1.0.2" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c2b180dc0bade59f03fd005cb967d3f1e5f69b13922dad0cd6e047cb8af2363" +checksum = "cfa512cd0d087cc9f99ad30a1bf64795b67871edbead083ffc3a4dfafa59aa00" dependencies = [ "cobs", "heapless", @@ -3956,9 +3909,9 @@ dependencies = [ [[package]] name = "pprof" -version = "0.11.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e20150f965e0e4c925982b9356da71c84bcd56cb66ef4e894825837cbcf6613e" +checksum = "196ded5d4be535690899a4631cc9f18cdc41b7ebf24a79400f46f48e49a11059" dependencies = [ "backtrace", "cfg-if", @@ -4008,9 +3961,9 @@ dependencies = [ [[package]] name = "proc-macro-crate" -version = "1.3.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66618389e4ec1c7afe67d51a9bf34ff9236480f8d51e7489b7d5ab0303c13f34" +checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919" dependencies = [ "once_cell", "toml_edit", @@ -4270,18 +4223,18 @@ dependencies = [ [[package]] name = "raw-cpuid" -version = "10.6.0" +version = "10.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6823ea29436221176fe662da99998ad3b4db2c7f31e7b6f5fe43adccd6320bb" +checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332" dependencies = [ "bitflags", ] [[package]] name = "rayon" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7" +checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" dependencies = [ "either", "rayon-core", @@ -4289,9 +4242,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.10.2" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "356a0625f1954f730c0201cdab48611198dc6ce21f4acff55089b5a78e6e835b" +checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" dependencies = [ "crossbeam-channel", "crossbeam-deque", @@ -4507,7 +4460,7 @@ name = "reth-discv4" version = "0.1.0" dependencies = [ "discv5", - "enr 0.7.0", + "enr", "generic-array 0.14.6", "hex", "rand 0.8.5", @@ -4531,7 +4484,7 @@ version = "0.1.0" dependencies = [ "async-trait", "data-encoding", - "enr 0.7.0", + "enr", "linked_hash_set", "lru 0.9.0", "parking_lot 0.12.1", @@ -4572,7 +4525,7 @@ dependencies = [ "thiserror", "tokio", "tokio-stream", - "tokio-util 0.7.4", + "tokio-util 0.7.7", "tracing", ] @@ -4602,7 +4555,7 @@ dependencies = [ "thiserror", "tokio", "tokio-stream", - "tokio-util 0.7.4", + "tokio-util 0.7.7", "tracing", "typenum", ] @@ -4636,7 +4589,7 @@ dependencies = [ "thiserror", "tokio", "tokio-stream", - "tokio-util 0.7.4", + "tokio-util 0.7.7", "tracing", ] @@ -4703,7 +4656,7 @@ dependencies = [ "serde_json", "thiserror", "tokio", - "tokio-util 0.7.4", + "tokio-util 0.7.7", "tower", "tracing", "tracing-test", @@ -4791,7 +4744,7 @@ dependencies = [ "aquamarine", "async-trait", "auto_impl", - "enr 0.7.0", + "enr", "ethers-core", "ethers-middleware", "ethers-providers", @@ -4831,7 +4784,7 @@ dependencies = [ "thiserror", "tokio", "tokio-stream", - "tokio-util 0.7.4", + "tokio-util 0.7.7", "tracing", ] @@ -4945,7 +4898,7 @@ dependencies = [ "auto_impl", "bytes", "criterion", - "enr 0.7.0", + "enr", "ethereum-types", "ethnum", "hex-literal", @@ -5083,7 +5036,7 @@ version = "0.1.0" dependencies = [ "async-trait", "confy", - "enr 0.7.0", + "enr", "ethers-core", "ethers-middleware", "ethers-providers", @@ -5207,7 +5160,7 @@ dependencies = [ [[package]] name = "revm" version = "3.0.0" -source = "git+https://github.com/bluealloy/revm#4d2f0741c5f9daec0ceb7cc7733d65ea4c496170" +source = "git+https://github.com/bluealloy/revm#3a17ca8556ee2933100d1e1ed4e200712715f76f" dependencies = [ "auto_impl", "revm-interpreter", @@ -5217,8 +5170,9 @@ dependencies = [ [[package]] name = "revm-interpreter" version = "1.0.0" -source = "git+https://github.com/bluealloy/revm#4d2f0741c5f9daec0ceb7cc7733d65ea4c496170" +source = "git+https://github.com/bluealloy/revm#3a17ca8556ee2933100d1e1ed4e200712715f76f" dependencies = [ + "bitvec 1.0.1", "derive_more", "enumn", "revm-primitives", @@ -5228,7 +5182,7 @@ dependencies = [ [[package]] name = "revm-precompile" version = "2.0.0" -source = "git+https://github.com/bluealloy/revm#4d2f0741c5f9daec0ceb7cc7733d65ea4c496170" +source = "git+https://github.com/bluealloy/revm#3a17ca8556ee2933100d1e1ed4e200712715f76f" dependencies = [ "k256", "num", @@ -5244,10 +5198,11 @@ dependencies = [ [[package]] name = "revm-primitives" version = "1.0.0" -source = "git+https://github.com/bluealloy/revm#4d2f0741c5f9daec0ceb7cc7733d65ea4c496170" +source = "git+https://github.com/bluealloy/revm#3a17ca8556ee2933100d1e1ed4e200712715f76f" dependencies = [ "arbitrary", "auto_impl", + "bitvec 1.0.1", "bytes", "derive_more", "enumn", @@ -5277,9 +5232,9 @@ dependencies = [ [[package]] name = "rgb" -version = "0.8.35" +version = "0.8.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7495acf66551cdb696b7711408144bcd3194fc78e32f3a09e809bfe7dd4a7ce3" +checksum = "20ec2d3e3fc7a92ced357df9cebd5a10b6fb2aa1ee797bf7e9ce2f17dffc8f59" dependencies = [ "bytemuck", ] @@ -5379,16 +5334,16 @@ dependencies = [ [[package]] name = "rustix" -version = "0.36.7" +version = "0.36.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4fdebc4b395b7fbb9ab11e462e20ed9051e7b16e42d24042c776eca0ac81b03" +checksum = "fd5c6ff11fecd55b40746d1995a02f2eb375bf8c00d192d521ee09f42bef37bc" dependencies = [ "bitflags", "errno", "io-lifetimes", "libc", "linux-raw-sys", - "windows-sys 0.42.0", + "windows-sys 0.45.0", ] [[package]] @@ -5444,9 +5399,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" +checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" [[package]] name = "salsa20" @@ -5518,9 +5473,9 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "scratch" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddccb15bcce173023b3fedd9436f882a0739b8dfb45e4f6b6002bee5929f61b2" +checksum = "5d5e082f6ea090deaf0e6dd04b68360fd5cddb152af6ce8927c9d25db299f98c" [[package]] name = "scrypt" @@ -5691,9 +5646,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.91" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877c235533714907a8c2464236f5c4b2a17262ef1bd71f38f35ea592c8da6883" +checksum = "cad406b69c91885b5107daf2c29572f6c8cdb3c66826821e286c533490c0bc76" dependencies = [ "itoa", "ryu", @@ -5734,7 +5689,7 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1966009f3c05f095697c537312f5415d1e3ed31ce0a56942bac4c771c5c335e" dependencies = [ - "darling 0.14.2", + "darling 0.14.3", "proc-macro2 1.0.51", "quote 1.0.23", "syn 1.0.109", @@ -5872,9 +5827,9 @@ checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" [[package]] name = "signal-hook" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a253b5e89e2698464fc26b545c9edceb338e18a89effeeecfea192c3025be29d" +checksum = "732768f1176d21d09e076c23a93123d40bba92d50c4058da34d45c8de8e682b9" dependencies = [ "libc", "signal-hook-registry", @@ -5893,9 +5848,9 @@ dependencies = [ [[package]] name = "signal-hook-registry" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" dependencies = [ "libc", ] @@ -5930,9 +5885,9 @@ checksum = "ceb945e54128e09c43d8e4f1277851bd5044c6fc540bbaa2ad888f60b3da9ae7" [[package]] name = "slab" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" +checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" dependencies = [ "autocfg", ] @@ -5945,9 +5900,9 @@ checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" [[package]] name = "smol_str" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7475118a28b7e3a2e157ce0131ba8c5526ea96e90ee601d9f6bb2e286a35ab44" +checksum = "fad6c857cbab2627dcf01ec85a623ca4e7dcb5691cbaa3d7fb7653671f0d09c9" dependencies = [ "serde", ] @@ -5960,9 +5915,9 @@ checksum = "5e9f0ab6ef7eb7353d9119c170a436d1bf248eea575ac42d19d12f4e34130831" [[package]] name = "socket2" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" dependencies = [ "libc", "winapi", @@ -5992,9 +5947,9 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] name = "spin" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f6002a767bff9e83f8eeecf883ecb8011875a21ae8da43bffb817a57e78cc09" +checksum = "7dccf47db1b41fa1573ed27ccf5e08e3ca771cb994f776668c5ebda893b248fc" dependencies = [ "lock_api", ] @@ -6215,7 +6170,7 @@ version = "3.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57d187b450bfb5b7939f82f9747dc1ebb15a7a9c4a93cd304a41aece7149608b" dependencies = [ - "darling 0.14.2", + "darling 0.14.3", "if_chain", "lazy_static", "proc-macro2 1.0.51", @@ -6269,18 +6224,19 @@ dependencies = [ [[package]] name = "thread_local" -version = "1.1.4" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180" +checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" dependencies = [ + "cfg-if", "once_cell", ] [[package]] name = "time" -version = "0.3.17" +version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376" +checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" dependencies = [ "itoa", "serde", @@ -6296,9 +6252,9 @@ checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" [[package]] name = "time-macros" -version = "0.2.6" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d967f99f534ca7e495c575c62638eebc2898a8c84c119b89e250477bc4ba16b2" +checksum = "fd80a657e71da814b8e5d60d3374fc6d35045062245d80224748ae522dd76f36" dependencies = [ "time-core", ] @@ -6333,15 +6289,15 @@ dependencies = [ [[package]] name = "tinyvec_macros" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.25.0" +version = "1.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e00990ebabbe4c14c08aca901caed183ecd5c09562a12c824bb53d3c3fd3af" +checksum = "03201d01c3c27a29c8a5cee5b55a93ddae1ccf6f08f65365c2c918f8c1b76f64" dependencies = [ "autocfg", "bytes", @@ -6354,7 +6310,7 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys 0.42.0", + "windows-sys 0.45.0", ] [[package]] @@ -6381,14 +6337,14 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d660770404473ccd7bc9f8b28494a811bc18542b915c0855c51e8f419d5223ce" +checksum = "8fb52b74f05dbf495a8fba459fdc331812b96aa086d9eb78101fa0d4569c3313" dependencies = [ "futures-core", "pin-project-lite", "tokio", - "tokio-util 0.7.4", + "tokio-util 0.7.7", ] [[package]] @@ -6433,9 +6389,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.4" +version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bb2e075f03b3d66d8d8785356224ba688d2906a371015e225beeb65ca92c740" +checksum = "5427d89453009325de0d8f342c9490009f76e999cb7672d77e46267448f7e6b2" dependencies = [ "bytes", "futures-core", @@ -6457,19 +6413,19 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.5.1" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4553f467ac8e3d374bc9a177a26801e5d0f9b211aa1673fb137a403afd1c9cf5" +checksum = "3ab8ed2edee10b50132aed5f331333428b011c99402b5a534154ed15746f9622" [[package]] name = "toml_edit" -version = "0.18.1" +version = "0.19.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56c59d8dd7d0dcbc6428bf7aa2f0e823e26e43b3c9aca15bbc9475d23e5fa12b" +checksum = "9a1eb0622d28f4b9c90adc4ea4b2b46b47663fde9ac5fafcb14a1369d5508825" dependencies = [ "indexmap", - "nom8", "toml_datetime", + "winnow", ] [[package]] @@ -6500,7 +6456,7 @@ dependencies = [ "rand 0.8.5", "slab", "tokio", - "tokio-util 0.7.4", + "tokio-util 0.7.7", "tower-layer", "tower-service", "tracing", @@ -6528,7 +6484,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "tokio", - "tokio-util 0.7.4", + "tokio-util 0.7.7", "tower", "tower-layer", "tower-service", @@ -6647,9 +6603,9 @@ dependencies = [ [[package]] name = "tracing-test" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e3d272c44878d2bbc9f4a20ad463724f03e19dbc667c6e84ac433ab7ffcc70b" +checksum = "3a2c0ff408fe918a94c428a3f2ad04e4afd5c95bbc08fcf868eff750c15728a4" dependencies = [ "lazy_static", "tracing-core", @@ -6659,9 +6615,9 @@ dependencies = [ [[package]] name = "tracing-test-macro" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "744324b12d69a9fc1edea4b38b7b1311295b662d161ad5deac17bb1358224a08" +checksum = "258bc1c4f8e2e73a977812ab339d503e6feeb92700f6d07a6de4d321522d5c08" dependencies = [ "lazy_static", "quote 1.0.23", @@ -6776,9 +6732,9 @@ checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" [[package]] name = "trybuild" -version = "1.0.77" +version = "1.0.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44da5a6f2164c8e14d3bbc0657d69c5966af9f5f6930d4f600b1f5c4a673413" +checksum = "223fc354447478d08231355617eb8c37affad0e83d33aeac30a8c275786b905a" dependencies = [ "basic-toml", "glob", @@ -6868,9 +6824,9 @@ checksum = "d54675592c1dbefd78cbd98db9bacd89886e1ca50692a0692baefffdeb92dd58" [[package]] name = "unicode-ident" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" +checksum = "775c11906edafc97bc378816b94585fbd9a054eabaf86fdd0ced94af449efab7" [[package]] name = "unicode-normalization" @@ -6883,9 +6839,9 @@ dependencies = [ [[package]] name = "unicode-segmentation" -version = "1.10.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fdbf052a0783de01e944a6ce7a8cb939e295b1e7be835a1112c3b9a7f047a5a" +checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" [[package]] name = "unicode-width" @@ -6989,12 +6945,6 @@ dependencies = [ "libc", ] -[[package]] -name = "waker-fn" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" - [[package]] name = "walkdir" version = "2.3.2" @@ -7036,9 +6986,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" +checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -7046,9 +6996,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" +checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" dependencies = [ "bumpalo", "log", @@ -7061,9 +7011,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.33" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23639446165ca5a5de86ae1d8896b737ae80319560fbaa4c2887b7da6e7ebd7d" +checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454" dependencies = [ "cfg-if", "js-sys", @@ -7073,9 +7023,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" +checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" dependencies = [ "quote 1.0.23", "wasm-bindgen-macro-support", @@ -7083,9 +7033,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" +checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" dependencies = [ "proc-macro2 1.0.51", "quote 1.0.23", @@ -7096,9 +7046,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" +checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" [[package]] name = "wasm-timer" @@ -7117,9 +7067,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.60" +version = "0.3.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f" +checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" dependencies = [ "js-sys", "wasm-bindgen", @@ -7268,6 +7218,15 @@ version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" +[[package]] +name = "winnow" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c95fb4ff192527911dd18eb138ac30908e7165b8944e528b6af93aa4c842d345" +dependencies = [ + "memchr", +] + [[package]] name = "winreg" version = "0.10.1" From 92a0ebbf04cb9a82d77061a4e875def4ee13a941 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sun, 5 Mar 2023 20:20:59 +0100 Subject: [PATCH 049/191] ci: no longer pin nightly version (#1638) --- .github/workflows/sanity.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/sanity.yml b/.github/workflows/sanity.yml index 59de0b5b9bd..1b33133fde8 100644 --- a/.github/workflows/sanity.yml +++ b/.github/workflows/sanity.yml @@ -25,7 +25,6 @@ jobs: - name: Install toolchain uses: dtolnay/rust-toolchain@nightly with: - toolchain: nightly-2022-12-27 components: llvm-tools-preview - uses: Swatinem/rust-cache@v2 with: From fab0e6a933fe4fcadaa184b04de6519309d12dc4 Mon Sep 17 00:00:00 2001 From: clabby Date: Sun, 5 Mar 2023 13:08:35 -0700 Subject: [PATCH 050/191] WIP: OP Goerli genesis --- crates/primitives/res/genesis/goerli_op.json | 92 ++++++++++++++++++++ crates/primitives/src/chain/mod.rs | 15 ++++ crates/primitives/src/chain/spec.rs | 46 ++++++++++ crates/primitives/src/constants.rs | 5 ++ crates/primitives/src/lib.rs | 2 + crates/rpc/rpc-engine-api/Cargo.toml | 3 + crates/rpc/rpc-engine-api/src/engine_api.rs | 7 +- crates/rpc/rpc-engine-api/src/error.rs | 5 ++ 8 files changed, 174 insertions(+), 1 deletion(-) create mode 100644 crates/primitives/res/genesis/goerli_op.json diff --git a/crates/primitives/res/genesis/goerli_op.json b/crates/primitives/res/genesis/goerli_op.json new file mode 100644 index 00000000000..0ea4638d942 --- /dev/null +++ b/crates/primitives/res/genesis/goerli_op.json @@ -0,0 +1,92 @@ +{ + "nonce": "0x00", + "timestamp": "0x00", + "extraData": "0x000000000000000000000000000000000000000000000000000000000000000027770a9694e4B4b1E130Ab91Bc327C36855f612E0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "difficulty": "0x01", + "gasLimit": "15000000", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "coinbase": "0x0000000000000000000000000000000000000000", + "gasUsed": "0x00", + "number": "0x00", + "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "stateRoot": "0x9e6b478a1cd331a979c39e4bddf42c676bcf5a63382f898dc441fe3fe5eb0837", + "alloc": { + "0x4200000000000000000000000000000000000000": { + "balance": "00", + "storage": {}, + "code": "0x608060405234801561001057600080fd5b50600436106100365760003560e01c806382e3702d1461003b578063cafa81dc14610072575b600080fd5b61005e610049366004610112565b60006020819052908152604090205460ff1681565b604051901515815260200160405180910390f35b61008561008036600461015a565b610087565b005b6001600080833360405160200161009f929190610229565b604080518083037fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe001815291815281516020928301208352908201929092520160002080547fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff001691151591909117905550565b60006020828403121561012457600080fd5b5035919050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052604160045260246000fd5b60006020828403121561016c57600080fd5b813567ffffffffffffffff8082111561018457600080fd5b818401915084601f83011261019857600080fd5b8135818111156101aa576101aa61012b565b604051601f82017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0908116603f011681019083821181831017156101f0576101f061012b565b8160405282815287602084870101111561020957600080fd5b826020860160208301376000928101602001929092525095945050505050565b6000835160005b8181101561024a5760208187018101518583015201610230565b81811115610259576000828501525b5060609390931b7fffffffffffffffffffffffffffffffffffffffff00000000000000000000000016919092019081526014019291505056fea26469706673582212209ffc0b44ce8a27c46cae74a3b3b620a72f10aaea97ed37c15b5d36792abd2aa464736f6c63430008090033" + }, + "0x4200000000000000000000000000000000000002": { + "balance": "00", + "storage": { + "0x0000000000000000000000000000000000000000000000000000000000000000": "0x0000000000000000000000000000000000000000000000000000000000000000" + }, + "code": "0x608060405234801561001057600080fd5b50600436106100725760003560e01c80639b19251a116100505780639b19251a146100e9578063b1540a011461011c578063bdc7b54f1461012f57600080fd5b806308fd63221461007757806313af40351461008c5780638da5cb5b1461009f575b600080fd5b61008a610085366004610614565b610137565b005b61008a61009a366004610650565b610271565b6000546100bf9073ffffffffffffffffffffffffffffffffffffffff1681565b60405173ffffffffffffffffffffffffffffffffffffffff90911681526020015b60405180910390f35b61010c6100f7366004610650565b60016020526000908152604090205460ff1681565b60405190151581526020016100e0565b61010c61012a366004610650565b61047c565b61008a6104cd565b60005473ffffffffffffffffffffffffffffffffffffffff1633146101e3576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152603a60248201527f46756e6374696f6e2063616e206f6e6c792062652063616c6c6564206279207460448201527f6865206f776e6572206f66207468697320636f6e74726163742e00000000000060648201526084015b60405180910390fd5b73ffffffffffffffffffffffffffffffffffffffff821660008181526001602090815260409182902080547fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff00168515159081179091558251938452908301527f8daaf060c3306c38e068a75c054bf96ecd85a3db1252712c4d93632744c42e0d910160405180910390a15050565b60005473ffffffffffffffffffffffffffffffffffffffff163314610318576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152603a60248201527f46756e6374696f6e2063616e206f6e6c792062652063616c6c6564206279207460448201527f6865206f776e6572206f66207468697320636f6e74726163742e00000000000060648201526084016101da565b73ffffffffffffffffffffffffffffffffffffffff81166103e1576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152605160248201527f4f564d5f4465706c6f79657257686974656c6973743a2063616e206f6e6c792060448201527f62652064697361626c65642076696120656e61626c654172626974726172794360648201527f6f6e74726163744465706c6f796d656e74000000000000000000000000000000608482015260a4016101da565b6000546040805173ffffffffffffffffffffffffffffffffffffffff928316815291831660208301527fb532073b38c83145e3e5135377a08bf9aab55bc0fd7c1179cd4fb995d2a5159c910160405180910390a1600080547fffffffffffffffffffffffff00000000000000000000000000000000000000001673ffffffffffffffffffffffffffffffffffffffff92909216919091179055565b6000805473ffffffffffffffffffffffffffffffffffffffff1615806104c7575073ffffffffffffffffffffffffffffffffffffffff821660009081526001602052604090205460ff165b92915050565b60005473ffffffffffffffffffffffffffffffffffffffff163314610574576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152603a60248201527f46756e6374696f6e2063616e206f6e6c792062652063616c6c6564206279207460448201527f6865206f776e6572206f66207468697320636f6e74726163742e00000000000060648201526084016101da565b60005460405173ffffffffffffffffffffffffffffffffffffffff90911681527fc0e106cf568e50698fdbde1eff56f5a5c966cc7958e37e276918e9e4ccdf8cd49060200160405180910390a1600080547fffffffffffffffffffffffff0000000000000000000000000000000000000000169055565b803573ffffffffffffffffffffffffffffffffffffffff8116811461060f57600080fd5b919050565b6000806040838503121561062757600080fd5b610630836105eb565b91506020830135801515811461064557600080fd5b809150509250929050565b60006020828403121561066257600080fd5b61066b826105eb565b939250505056fea264697066735822122045a02b3906eca00a51b37c2965ab13be381f71f60af681951849865fb2daa75f64736f6c63430008090033" + }, + "0x4200000000000000000000000000000000000007": { + "balance": "00", + "storage": { + "0x0000000000000000000000000000000000000000000000000000000000000004": "0x000000000000000000000000000000000000000000000000000000000000dead", + "0x0000000000000000000000000000000000000000000000000000000000000005": "0x0000000000000000000000005086d1eef304eb5284a0f6720f79403b4e9be294", + "0x0000000000000000000000000000000000000000000000000000000000000003": "0x00000000000000000000000000000000000000000000000000000000000186a0" + }, + "code": "0x608060405234801561001057600080fd5b50600436106100885760003560e01c8063a71198691161005b578063a71198691461012a578063b1b1b2091461014a578063cbd4ece91461016d578063ecc704281461018057600080fd5b806321d800ec1461008d5780633dbb202b146100c55780636e296e45146100da57806382e3702d14610107575b600080fd5b6100b061009b366004610826565b60006020819052908152604090205460ff1681565b60405190151581526020015b60405180910390f35b6100d86100d3366004610942565b610197565b005b6100e26102e2565b60405173ffffffffffffffffffffffffffffffffffffffff90911681526020016100bc565b6100b0610115366004610826565b60026020526000908152604090205460ff1681565b6005546100e29073ffffffffffffffffffffffffffffffffffffffff1681565b6100b0610158366004610826565b60016020526000908152604090205460ff1681565b6100d861017b3660046109ad565b61038b565b61018960035481565b6040519081526020016100bc565b60006101a784338560035461078d565b80516020808301919091206000908152600290915260409081902080547fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff00166001179055517fcafa81dc0000000000000000000000000000000000000000000000000000000081529091507342000000000000000000000000000000000000009063cafa81dc9061023c908490600401610a89565b600060405180830381600087803b15801561025657600080fd5b505af115801561026a573d6000803e3d6000fd5b505050508373ffffffffffffffffffffffffffffffffffffffff167fcb0f7ffd78f9aee47a248fae8db181db6eee833039123e026dcbff529522e52a3385600354866040516102bc9493929190610aa3565b60405180910390a26001600360008282546102d79190610aef565b909155505050505050565b60045460009073ffffffffffffffffffffffffffffffffffffffff1661dead141561036e576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601f60248201527f78446f6d61696e4d65737361676553656e646572206973206e6f74207365740060448201526064015b60405180910390fd5b5060045473ffffffffffffffffffffffffffffffffffffffff1690565b60055473ffffffffffffffffffffffffffffffffffffffff167fffffffffffffffffffffffffeeeeffffffffffffffffffffffffffffffffeeef330173ffffffffffffffffffffffffffffffffffffffff161461046a576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152602760248201527f50726f7669646564206d65737361676520636f756c64206e6f7420626520766560448201527f7269666965642e000000000000000000000000000000000000000000000000006064820152608401610365565b60006104788585858561078d565b8051602080830191909120600081815260019092526040909120549192509060ff1615610527576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152602b60248201527f50726f7669646564206d6573736167652068617320616c72656164792062656560448201527f6e2072656365697665642e0000000000000000000000000000000000000000006064820152608401610365565b73ffffffffffffffffffffffffffffffffffffffff8616734200000000000000000000000000000000000000141561059957600090815260016020819052604090912080547fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0016909117905550610787565b600480547fffffffffffffffffffffffff00000000000000000000000000000000000000001673ffffffffffffffffffffffffffffffffffffffff878116919091179091556040516000918816906105f2908790610b2e565b6000604051808303816000865af19150503d806000811461062f576040519150601f19603f3d011682016040523d82523d6000602084013e610634565b606091505b5050600480547fffffffffffffffffffffffff00000000000000000000000000000000000000001661dead1790559050801515600114156106d557600082815260016020819052604080832080547fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff00169092179091555183917f4641df4a962071e12719d8c8c8e5ac7fc4d97b927346a3d7a335b1f7517e133c91a2610701565b60405182907f99d0e048484baa1b1540b1367cb128acd7ab2946d1ed91ec10e3c85e4bf51b8f90600090a25b600083334360405160200161071893929190610b4a565b604080517fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe08184030181529181528151602092830120600090815291829052902080547fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff00166001179055505050505b50505050565b6060848484846040516024016107a69493929190610b9c565b604080517fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe08184030181529190526020810180517bffffffffffffffffffffffffffffffffffffffffffffffffffffffff167fcbd4ece9000000000000000000000000000000000000000000000000000000001790529050949350505050565b60006020828403121561083857600080fd5b5035919050565b803573ffffffffffffffffffffffffffffffffffffffff8116811461086357600080fd5b919050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052604160045260246000fd5b600082601f8301126108a857600080fd5b813567ffffffffffffffff808211156108c3576108c3610868565b604051601f83017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0908116603f0116810190828211818310171561090957610909610868565b8160405283815286602085880101111561092257600080fd5b836020870160208301376000602085830101528094505050505092915050565b60008060006060848603121561095757600080fd5b6109608461083f565b9250602084013567ffffffffffffffff81111561097c57600080fd5b61098886828701610897565b925050604084013563ffffffff811681146109a257600080fd5b809150509250925092565b600080600080608085870312156109c357600080fd5b6109cc8561083f565b93506109da6020860161083f565b9250604085013567ffffffffffffffff8111156109f657600080fd5b610a0287828801610897565b949793965093946060013593505050565b60005b83811015610a2e578181015183820152602001610a16565b838111156107875750506000910152565b60008151808452610a57816020860160208601610a13565b601f017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0169290920160200192915050565b602081526000610a9c6020830184610a3f565b9392505050565b73ffffffffffffffffffffffffffffffffffffffff85168152608060208201526000610ad26080830186610a3f565b905083604083015263ffffffff8316606083015295945050505050565b60008219821115610b29577f4e487b7100000000000000000000000000000000000000000000000000000000600052601160045260246000fd5b500190565b60008251610b40818460208701610a13565b9190910192915050565b60008451610b5c818460208901610a13565b60609490941b7fffffffffffffffffffffffffffffffffffffffff0000000000000000000000001691909301908152601481019190915260340192915050565b600073ffffffffffffffffffffffffffffffffffffffff808716835280861660208401525060806040830152610bd56080830185610a3f565b90508260608301529594505050505056fea26469706673582212200afb9537c52292543adecf37773b8e4d795c984adf4e80970731e434679b4ec264736f6c63430008090033" + }, + "0x420000000000000000000000000000000000000F": { + "balance": "00", + "storage": { + "0x0000000000000000000000000000000000000000000000000000000000000000": "0x000000000000000000000000a693b8f8207ff043f6bbc2e2120bbe4c2251efe9", + "0x0000000000000000000000000000000000000000000000000000000000000001": "0x0000000000000000000000000000000000000000000000000000000000000001", + "0x0000000000000000000000000000000000000000000000000000000000000002": "0x0000000000000000000000000000000000000000000000000000000000000001", + "0x0000000000000000000000000000000000000000000000000000000000000003": "0x0000000000000000000000000000000000000000000000000000000000000abe", + "0x0000000000000000000000000000000000000000000000000000000000000004": "0x000000000000000000000000000000000000000000000000000000000016e360", + "0x0000000000000000000000000000000000000000000000000000000000000005": "0x0000000000000000000000000000000000000000000000000000000000000006" + }, + "code": "0x608060405234801561001057600080fd5b50600436106100f55760003560e01c80638c8885c811610097578063de26c4a111610066578063de26c4a1146101cc578063f2fde38b146101df578063f45e65d8146101f2578063fe173b97146101fb57600080fd5b80638c8885c81461016b5780638da5cb5b1461017e578063bede39b5146101a6578063bf1fe420146101b957600080fd5b806349948e0e116100d357806349948e0e14610134578063519b4bd3146101475780637046559714610150578063715018a61461016357600080fd5b80630c18c162146100fa578063313ce567146101165780633577afc51461011f575b600080fd5b61010360035481565b6040519081526020015b60405180910390f35b61010360055481565b61013261012d3660046108d0565b610204565b005b610103610142366004610918565b6102c6565b61010360025481565b61013261015e3660046108d0565b610322565b6101326103d8565b6101326101793660046108d0565b610465565b60005460405173ffffffffffffffffffffffffffffffffffffffff909116815260200161010d565b6101326101b43660046108d0565b61051b565b6101326101c73660046108d0565b6105d1565b6101036101da366004610918565b610687565b6101326101ed3660046109e7565b61072b565b61010360045481565b61010360015481565b60005473ffffffffffffffffffffffffffffffffffffffff16331461028a576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820181905260248201527f4f776e61626c653a2063616c6c6572206973206e6f7420746865206f776e657260448201526064015b60405180910390fd5b60038190556040518181527f32740b35c0ea213650f60d44366b4fb211c9033b50714e4a1d34e65d5beb9bb4906020015b60405180910390a150565b6000806102d283610687565b90506000600254826102e49190610a53565b90506000600554600a6102f79190610bb2565b90506000600454836103099190610a53565b905060006103178383610bbe565b979650505050505050565b60005473ffffffffffffffffffffffffffffffffffffffff1633146103a3576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820181905260248201527f4f776e61626c653a2063616c6c6572206973206e6f7420746865206f776e65726044820152606401610281565b60048190556040518181527f3336cd9708eaf2769a0f0dc0679f30e80f15dcd88d1921b5a16858e8b85c591a906020016102bb565b60005473ffffffffffffffffffffffffffffffffffffffff163314610459576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820181905260248201527f4f776e61626c653a2063616c6c6572206973206e6f7420746865206f776e65726044820152606401610281565b610463600061085b565b565b60005473ffffffffffffffffffffffffffffffffffffffff1633146104e6576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820181905260248201527f4f776e61626c653a2063616c6c6572206973206e6f7420746865206f776e65726044820152606401610281565b60058190556040518181527fd68112a8707e326d08be3656b528c1bcc5bbbfc47f4177e2179b14d8640838c1906020016102bb565b60005473ffffffffffffffffffffffffffffffffffffffff16331461059c576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820181905260248201527f4f776e61626c653a2063616c6c6572206973206e6f7420746865206f776e65726044820152606401610281565b60028190556040518181527f351fb23757bb5ea0546c85b7996ddd7155f96b939ebaa5ff7bc49c75f27f2c44906020016102bb565b60005473ffffffffffffffffffffffffffffffffffffffff163314610652576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820181905260248201527f4f776e61626c653a2063616c6c6572206973206e6f7420746865206f776e65726044820152606401610281565b60018190556040518181527ffcdccc6074c6c42e4bd578aa9870c697dc976a270968452d2b8c8dc369fae396906020016102bb565b600080805b8351811015610704578381815181106106a7576106a7610bf9565b01602001517fff00000000000000000000000000000000000000000000000000000000000000166106e4576106dd600483610c28565b91506106f2565b6106ef601083610c28565b91505b806106fc81610c40565b91505061068c565b506000600354826107159190610c28565b905061072381610440610c28565b949350505050565b60005473ffffffffffffffffffffffffffffffffffffffff1633146107ac576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820181905260248201527f4f776e61626c653a2063616c6c6572206973206e6f7420746865206f776e65726044820152606401610281565b73ffffffffffffffffffffffffffffffffffffffff811661084f576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152602660248201527f4f776e61626c653a206e6577206f776e657220697320746865207a65726f206160448201527f64647265737300000000000000000000000000000000000000000000000000006064820152608401610281565b6108588161085b565b50565b6000805473ffffffffffffffffffffffffffffffffffffffff8381167fffffffffffffffffffffffff0000000000000000000000000000000000000000831681178455604051919092169283917f8be0079c531659141344cd1fd0a4f28419497f9722a3daafe3b4186f6b6457e09190a35050565b6000602082840312156108e257600080fd5b5035919050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052604160045260246000fd5b60006020828403121561092a57600080fd5b813567ffffffffffffffff8082111561094257600080fd5b818401915084601f83011261095657600080fd5b813581811115610968576109686108e9565b604051601f82017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0908116603f011681019083821181831017156109ae576109ae6108e9565b816040528281528760208487010111156109c757600080fd5b826020860160208301376000928101602001929092525095945050505050565b6000602082840312156109f957600080fd5b813573ffffffffffffffffffffffffffffffffffffffff81168114610a1d57600080fd5b9392505050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052601160045260246000fd5b6000817fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0483118215151615610a8b57610a8b610a24565b500290565b600181815b80851115610ae957817fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff04821115610acf57610acf610a24565b80851615610adc57918102915b93841c9390800290610a95565b509250929050565b600082610b0057506001610bac565b81610b0d57506000610bac565b8160018114610b235760028114610b2d57610b49565b6001915050610bac565b60ff841115610b3e57610b3e610a24565b50506001821b610bac565b5060208310610133831016604e8410600b8410161715610b6c575081810a610bac565b610b768383610a90565b807fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff04821115610ba857610ba8610a24565b0290505b92915050565b6000610a1d8383610af1565b600082610bf4577f4e487b7100000000000000000000000000000000000000000000000000000000600052601260045260246000fd5b500490565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052603260045260246000fd5b60008219821115610c3b57610c3b610a24565b500190565b60007fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff821415610c7257610c72610a24565b506001019056fea2646970667358221220b949ef5f9defd6c0aab6259672d00d239cb8854c9972ba1866af1c6ec6433d4c64736f6c63430008090033" + }, + "0x4200000000000000000000000000000000000010": { + "balance": "00", + "storage": { + "0x0000000000000000000000000000000000000000000000000000000000000001": "0x000000000000000000000000636af16bf2f682dd3109e60102b8e1a089fedaa8", + "0x0000000000000000000000000000000000000000000000000000000000000000": "0x0000000000000000000000004200000000000000000000000000000000000007" + }, + "code": "0x608060405234801561001057600080fd5b50600436106100675760003560e01c80633cb747bf116100505780633cb747bf146100ca578063662a633a146100ea578063a3a79548146100fd57600080fd5b806332b7006d1461006c57806336c717c114610081575b600080fd5b61007f61007a366004610d0f565b610110565b005b6001546100a19073ffffffffffffffffffffffffffffffffffffffff1681565b60405173ffffffffffffffffffffffffffffffffffffffff909116815260200160405180910390f35b6000546100a19073ffffffffffffffffffffffffffffffffffffffff1681565b61007f6100f8366004610d80565b610126565b61007f61010b366004610e18565b6106c1565b61011f853333878787876106d8565b5050505050565b60015473ffffffffffffffffffffffffffffffffffffffff1661015e60005473ffffffffffffffffffffffffffffffffffffffff1690565b73ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff161461021d576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152602e60248201527f4f564d5f58434841494e3a206d657373656e67657220636f6e7472616374207560448201527f6e61757468656e7469636174656400000000000000000000000000000000000060648201526084015b60405180910390fd5b8073ffffffffffffffffffffffffffffffffffffffff1661025360005473ffffffffffffffffffffffffffffffffffffffff1690565b73ffffffffffffffffffffffffffffffffffffffff16636e296e456040518163ffffffff1660e01b815260040160206040518083038186803b15801561029857600080fd5b505afa1580156102ac573d6000803e3d6000fd5b505050506040513d601f19601f820116820180604052508101906102d09190610e9b565b73ffffffffffffffffffffffffffffffffffffffff1614610373576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152603060248201527f4f564d5f58434841494e3a2077726f6e672073656e646572206f662063726f7360448201527f732d646f6d61696e206d657373616765000000000000000000000000000000006064820152608401610214565b61039d877f1d1d8b6300000000000000000000000000000000000000000000000000000000610a32565b801561045357508673ffffffffffffffffffffffffffffffffffffffff1663c01e1bd66040518163ffffffff1660e01b8152600401602060405180830381600087803b1580156103ec57600080fd5b505af1158015610400573d6000803e3d6000fd5b505050506040513d601f19601f820116820180604052508101906104249190610e9b565b73ffffffffffffffffffffffffffffffffffffffff168873ffffffffffffffffffffffffffffffffffffffff16145b15610567576040517f40c10f1900000000000000000000000000000000000000000000000000000000815273ffffffffffffffffffffffffffffffffffffffff8681166004830152602482018690528816906340c10f1990604401600060405180830381600087803b1580156104c857600080fd5b505af11580156104dc573d6000803e3d6000fd5b505050508573ffffffffffffffffffffffffffffffffffffffff168773ffffffffffffffffffffffffffffffffffffffff168973ffffffffffffffffffffffffffffffffffffffff167fb0444523268717a02698be47d0803aa7468c00acbed2f8bd93a0459cde61dd898888888860405161055a9493929190610f08565b60405180910390a46106b7565b600063a9f9e67560e01b8989888a89898960405160240161058e9796959493929190610f3e565b604080517fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe08184030181529190526020810180517bffffffffffffffffffffffffffffffffffffffffffffffffffffffff167fffffffff00000000000000000000000000000000000000000000000000000000909316929092179091526001549091506106339073ffffffffffffffffffffffffffffffffffffffff16600083610a57565b8673ffffffffffffffffffffffffffffffffffffffff168873ffffffffffffffffffffffffffffffffffffffff168a73ffffffffffffffffffffffffffffffffffffffff167f7ea89a4591614515571c2b51f5ea06494056f261c10ab1ed8c03c7590d87bce0898989896040516106ad9493929190610f08565b60405180910390a4505b5050505050505050565b6106d0863387878787876106d8565b505050505050565b6040517f9dc29fac0000000000000000000000000000000000000000000000000000000081523360048201526024810185905273ffffffffffffffffffffffffffffffffffffffff881690639dc29fac90604401600060405180830381600087803b15801561074657600080fd5b505af115801561075a573d6000803e3d6000fd5b5050505060008773ffffffffffffffffffffffffffffffffffffffff1663c01e1bd66040518163ffffffff1660e01b8152600401602060405180830381600087803b1580156107a857600080fd5b505af11580156107bc573d6000803e3d6000fd5b505050506040513d601f19601f820116820180604052508101906107e09190610e9b565b9050606073ffffffffffffffffffffffffffffffffffffffff891673deaddeaddeaddeaddeaddeaddeaddeaddead000014156108d5576040517f1532ec340000000000000000000000000000000000000000000000000000000090610851908a908a908a9089908990602401610f9b565b604080517fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe08184030181529190526020810180517bffffffffffffffffffffffffffffffffffffffffffffffffffffffff167fffffffff00000000000000000000000000000000000000000000000000000000909316929092179091529050610994565b6040517fa9f9e67500000000000000000000000000000000000000000000000000000000906109149084908c908c908c908c908b908b90602401610f3e565b604080517fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe08184030181529190526020810180517bffffffffffffffffffffffffffffffffffffffffffffffffffffffff167fffffffff000000000000000000000000000000000000000000000000000000009093169290921790915290505b6001546109b89073ffffffffffffffffffffffffffffffffffffffff168683610a57565b3373ffffffffffffffffffffffffffffffffffffffff168973ffffffffffffffffffffffffffffffffffffffff168373ffffffffffffffffffffffffffffffffffffffff167f73d170910aba9e6d50b102db522b1dbcd796216f5128b445aa2135272886497e8a8a89896040516106ad9493929190610f08565b6000610a3d83610ae8565b8015610a4e5750610a4e8383610b4c565b90505b92915050565b6000546040517f3dbb202b00000000000000000000000000000000000000000000000000000000815273ffffffffffffffffffffffffffffffffffffffff90911690633dbb202b90610ab190869085908790600401611016565b600060405180830381600087803b158015610acb57600080fd5b505af1158015610adf573d6000803e3d6000fd5b50505050505050565b6000610b14827f01ffc9a700000000000000000000000000000000000000000000000000000000610b4c565b8015610a515750610b45827fffffffff00000000000000000000000000000000000000000000000000000000610b4c565b1592915050565b604080517fffffffff00000000000000000000000000000000000000000000000000000000831660248083019190915282518083039091018152604490910182526020810180517bffffffffffffffffffffffffffffffffffffffffffffffffffffffff167f01ffc9a7000000000000000000000000000000000000000000000000000000001790529051600091908290819073ffffffffffffffffffffffffffffffffffffffff87169061753090610c06908690611092565b6000604051808303818686fa925050503d8060008114610c42576040519150601f19603f3d011682016040523d82523d6000602084013e610c47565b606091505b5091509150602081511015610c625760009350505050610a51565b818015610c7e575080806020019051810190610c7e91906110ae565b9695505050505050565b73ffffffffffffffffffffffffffffffffffffffff81168114610caa57600080fd5b50565b803563ffffffff81168114610cc157600080fd5b919050565b60008083601f840112610cd857600080fd5b50813567ffffffffffffffff811115610cf057600080fd5b602083019150836020828501011115610d0857600080fd5b9250929050565b600080600080600060808688031215610d2757600080fd5b8535610d3281610c88565b945060208601359350610d4760408701610cad565b9250606086013567ffffffffffffffff811115610d6357600080fd5b610d6f88828901610cc6565b969995985093965092949392505050565b600080600080600080600060c0888a031215610d9b57600080fd5b8735610da681610c88565b96506020880135610db681610c88565b95506040880135610dc681610c88565b94506060880135610dd681610c88565b93506080880135925060a088013567ffffffffffffffff811115610df957600080fd5b610e058a828b01610cc6565b989b979a50959850939692959293505050565b60008060008060008060a08789031215610e3157600080fd5b8635610e3c81610c88565b95506020870135610e4c81610c88565b945060408701359350610e6160608801610cad565b9250608087013567ffffffffffffffff811115610e7d57600080fd5b610e8989828a01610cc6565b979a9699509497509295939492505050565b600060208284031215610ead57600080fd5b8151610eb881610c88565b9392505050565b8183528181602085013750600060208284010152600060207fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0601f840116840101905092915050565b73ffffffffffffffffffffffffffffffffffffffff85168152836020820152606060408201526000610c7e606083018486610ebf565b600073ffffffffffffffffffffffffffffffffffffffff808a1683528089166020840152808816604084015280871660608401525084608083015260c060a0830152610f8e60c083018486610ebf565b9998505050505050505050565b600073ffffffffffffffffffffffffffffffffffffffff808816835280871660208401525084604083015260806060830152610fdb608083018486610ebf565b979650505050505050565b60005b83811015611001578181015183820152602001610fe9565b83811115611010576000848401525b50505050565b73ffffffffffffffffffffffffffffffffffffffff841681526060602082015260008351806060840152611051816080850160208801610fe6565b63ffffffff93909316604083015250601f919091017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0160160800192915050565b600082516110a4818460208701610fe6565b9190910192915050565b6000602082840312156110c057600080fd5b81518015158114610eb857600080fdfea2646970667358221220df892c82e9ba0fc965240aa38614ff1bd6af4d227ce2c0e9933d7f50711a886264736f6c63430008090033" + }, + "0x4200000000000000000000000000000000000011": { + "balance": "00", + "storage": { + "0x0000000000000000000000000000000000000000000000000000000000000000": "0x000000000000000000000000fd1d2e729ae8eee2e146c033bf4400fe75284301" + }, + "code": "0x6080604052600436106100385760003560e01c80633ccfd60b14610044578063d3e5792b1461005b578063d4ff92181461008a57600080fd5b3661003f57005b600080fd5b34801561005057600080fd5b506100596100dc565b005b34801561006757600080fd5b5061007767d02ab486cedc000081565b6040519081526020015b60405180910390f35b34801561009657600080fd5b506000546100b79073ffffffffffffffffffffffffffffffffffffffff1681565b60405173ffffffffffffffffffffffffffffffffffffffff9091168152602001610081565b67d02ab486cedc000047101561019e576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152605760248201527f4f564d5f53657175656e6365724665655661756c743a2077697468647261776160448201527f6c20616d6f756e74206d7573742062652067726561746572207468616e206d6960648201527f6e696d756d207769746864726177616c20616d6f756e74000000000000000000608482015260a40160405180910390fd5b600080546040805160208101825283815290517fa3a795480000000000000000000000000000000000000000000000000000000081527342000000000000000000000000000000000000109363a3a79548936102309373deaddeaddeaddeaddeaddeaddeaddeaddead00009373ffffffffffffffffffffffffffffffffffffffff909216924792909190600401610264565b600060405180830381600087803b15801561024a57600080fd5b505af115801561025e573d6000803e3d6000fd5b50505050565b600073ffffffffffffffffffffffffffffffffffffffff808816835260208188168185015286604085015263ffffffff8616606085015260a06080850152845191508160a085015260005b828110156102cb5785810182015185820160c0015281016102af565b828111156102dd57600060c084870101525b5050601f017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0169190910160c001969550505050505056fea2646970667358221220387a6116dde263ea48767352a397053c8cffa776aecb43cded2f25a4a9cfbdc264736f6c63430008090033" + }, + "0x4200000000000000000000000000000000000012": { + "balance": "00", + "storage": {}, + "code": "0x608060405234801561001057600080fd5b506004361061002b5760003560e01c8063896f93d114610030575b600080fd5b61004361003e36600461025f565b610045565b005b73ffffffffffffffffffffffffffffffffffffffff83166100c6576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601d60248201527f4d7573742070726f76696465204c3120746f6b656e2061646472657373000000604482015260640160405180910390fd5b60007342000000000000000000000000000000000000108484846040516100ec90610178565b6100f99493929190610359565b604051809103906000f080158015610115573d6000803e3d6000fd5b5090508073ffffffffffffffffffffffffffffffffffffffff168473ffffffffffffffffffffffffffffffffffffffff167fceeb8e7d520d7f3b65fc11a262b91066940193b05d4f93df07cfdced0eb551cf60405160405180910390a350505050565b6113d7806103b083390190565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052604160045260246000fd5b600082601f8301126101c557600080fd5b813567ffffffffffffffff808211156101e0576101e0610185565b604051601f83017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0908116603f0116810190828211818310171561022657610226610185565b8160405283815286602085880101111561023f57600080fd5b836020870160208301376000602085830101528094505050505092915050565b60008060006060848603121561027457600080fd5b833573ffffffffffffffffffffffffffffffffffffffff8116811461029857600080fd5b9250602084013567ffffffffffffffff808211156102b557600080fd5b6102c1878388016101b4565b935060408601359150808211156102d757600080fd5b506102e4868287016101b4565b9150509250925092565b6000815180845260005b81811015610314576020818501810151868301820152016102f8565b81811115610326576000602083870101525b50601f017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0169290920160200192915050565b600073ffffffffffffffffffffffffffffffffffffffff80871683528086166020840152506080604083015261039260808301856102ee565b82810360608401526103a481856102ee565b97965050505050505056fe60806040523480156200001157600080fd5b50604051620013d7380380620013d783398101604081905262000034916200022f565b8151829082906200004d9060039060208501906200009f565b508051620000639060049060208401906200009f565b5050600580546001600160a01b039586166001600160a01b031991821617909155600680549690951695169490941790925550620002fc915050565b828054620000ad90620002bf565b90600052602060002090601f016020900481019282620000d157600085556200011c565b82601f10620000ec57805160ff19168380011785556200011c565b828001600101855582156200011c579182015b828111156200011c578251825591602001919060010190620000ff565b506200012a9291506200012e565b5090565b5b808211156200012a57600081556001016200012f565b80516001600160a01b03811681146200015d57600080fd5b919050565b634e487b7160e01b600052604160045260246000fd5b600082601f8301126200018a57600080fd5b81516001600160401b0380821115620001a757620001a762000162565b604051601f8301601f19908116603f01168101908282118183101715620001d257620001d262000162565b81604052838152602092508683858801011115620001ef57600080fd5b600091505b83821015620002135785820183015181830184015290820190620001f4565b83821115620002255760008385830101525b9695505050505050565b600080600080608085870312156200024657600080fd5b620002518562000145565b9350620002616020860162000145565b60408601519093506001600160401b03808211156200027f57600080fd5b6200028d8883890162000178565b93506060870151915080821115620002a457600080fd5b50620002b38782880162000178565b91505092959194509250565b600181811c90821680620002d457607f821691505b60208210811415620002f657634e487b7160e01b600052602260045260246000fd5b50919050565b6110cb806200030c6000396000f3fe608060405234801561001057600080fd5b50600436106101005760003560e01c806370a0823111610097578063a9059cbb11610066578063a9059cbb14610215578063ae1f6aaf14610228578063c01e1bd61461026d578063dd62ed3e1461028d57600080fd5b806370a08231146101b157806395d89b41146101e75780639dc29fac146101ef578063a457c2d71461020257600080fd5b806323b872dd116100d357806323b872dd14610167578063313ce5671461017a578063395093511461018957806340c10f191461019c57600080fd5b806301ffc9a71461010557806306fdde031461012d578063095ea7b31461014257806318160ddd14610155575b600080fd5b610118610113366004610e4a565b6102d3565b60405190151581526020015b60405180910390f35b610135610393565b6040516101249190610e93565b610118610150366004610f2f565b610425565b6002545b604051908152602001610124565b610118610175366004610f59565b61043b565b60405160128152602001610124565b610118610197366004610f2f565b61050c565b6101af6101aa366004610f2f565b610555565b005b6101596101bf366004610f95565b73ffffffffffffffffffffffffffffffffffffffff1660009081526020819052604090205490565b61013561061a565b6101af6101fd366004610f2f565b610629565b610118610210366004610f2f565b6106e2565b610118610223366004610f2f565b6107a0565b6006546102489073ffffffffffffffffffffffffffffffffffffffff1681565b60405173ffffffffffffffffffffffffffffffffffffffff9091168152602001610124565b6005546102489073ffffffffffffffffffffffffffffffffffffffff1681565b61015961029b366004610fb0565b73ffffffffffffffffffffffffffffffffffffffff918216600090815260016020908152604080832093909416825291909152205490565b60007f01ffc9a7a5cef8baa21ed3c5c0d7e23accb804b619e9333b597f47a0d84076e27f1d1d8b63000000000000000000000000000000000000000000000000000000007fffffffff0000000000000000000000000000000000000000000000000000000084167f01ffc9a700000000000000000000000000000000000000000000000000000000148061038b57507fffffffff00000000000000000000000000000000000000000000000000000000848116908216145b949350505050565b6060600380546103a290610fe3565b80601f01602080910402602001604051908101604052809291908181526020018280546103ce90610fe3565b801561041b5780601f106103f05761010080835404028352916020019161041b565b820191906000526020600020905b8154815290600101906020018083116103fe57829003601f168201915b5050505050905090565b60006104323384846107ad565b50600192915050565b600061044884848461092d565b73ffffffffffffffffffffffffffffffffffffffff84166000908152600160209081526040808320338452909152902054828110156104f45760405162461bcd60e51b815260206004820152602860248201527f45524332303a207472616e7366657220616d6f756e742065786365656473206160448201527f6c6c6f77616e636500000000000000000000000000000000000000000000000060648201526084015b60405180910390fd5b61050185338584036107ad565b506001949350505050565b33600081815260016020908152604080832073ffffffffffffffffffffffffffffffffffffffff871684529091528120549091610432918590610550908690611066565b6107ad565b60065473ffffffffffffffffffffffffffffffffffffffff1633146105bc5760405162461bcd60e51b815260206004820181905260248201527f4f6e6c79204c32204272696467652063616e206d696e7420616e64206275726e60448201526064016104eb565b6105c68282610b93565b8173ffffffffffffffffffffffffffffffffffffffff167f0f6798a560793a54c3bcfe86a93cde1e73087d944c0ea20544137d41213968858260405161060e91815260200190565b60405180910390a25050565b6060600480546103a290610fe3565b60065473ffffffffffffffffffffffffffffffffffffffff1633146106905760405162461bcd60e51b815260206004820181905260248201527f4f6e6c79204c32204272696467652063616e206d696e7420616e64206275726e60448201526064016104eb565b61069a8282610c99565b8173ffffffffffffffffffffffffffffffffffffffff167fcc16f5dbb4873280815c1ee09dbd06736cffcc184412cf7a71a0fdb75d397ca58260405161060e91815260200190565b33600090815260016020908152604080832073ffffffffffffffffffffffffffffffffffffffff86168452909152812054828110156107895760405162461bcd60e51b815260206004820152602560248201527f45524332303a2064656372656173656420616c6c6f77616e63652062656c6f7760448201527f207a65726f00000000000000000000000000000000000000000000000000000060648201526084016104eb565b61079633858584036107ad565b5060019392505050565b600061043233848461092d565b73ffffffffffffffffffffffffffffffffffffffff83166108355760405162461bcd60e51b8152602060048201526024808201527f45524332303a20617070726f76652066726f6d20746865207a65726f2061646460448201527f726573730000000000000000000000000000000000000000000000000000000060648201526084016104eb565b73ffffffffffffffffffffffffffffffffffffffff82166108be5760405162461bcd60e51b815260206004820152602260248201527f45524332303a20617070726f766520746f20746865207a65726f20616464726560448201527f737300000000000000000000000000000000000000000000000000000000000060648201526084016104eb565b73ffffffffffffffffffffffffffffffffffffffff83811660008181526001602090815260408083209487168084529482529182902085905590518481527f8c5be1e5ebec7d5bd14f71427d1e84f3dd0314c0f7b2291e5b200ac8c7c3b92591015b60405180910390a3505050565b73ffffffffffffffffffffffffffffffffffffffff83166109b65760405162461bcd60e51b815260206004820152602560248201527f45524332303a207472616e736665722066726f6d20746865207a65726f20616460448201527f647265737300000000000000000000000000000000000000000000000000000060648201526084016104eb565b73ffffffffffffffffffffffffffffffffffffffff8216610a3f5760405162461bcd60e51b815260206004820152602360248201527f45524332303a207472616e7366657220746f20746865207a65726f206164647260448201527f657373000000000000000000000000000000000000000000000000000000000060648201526084016104eb565b73ffffffffffffffffffffffffffffffffffffffff831660009081526020819052604090205481811015610adb5760405162461bcd60e51b815260206004820152602660248201527f45524332303a207472616e7366657220616d6f756e742065786365656473206260448201527f616c616e6365000000000000000000000000000000000000000000000000000060648201526084016104eb565b73ffffffffffffffffffffffffffffffffffffffff808516600090815260208190526040808220858503905591851681529081208054849290610b1f908490611066565b925050819055508273ffffffffffffffffffffffffffffffffffffffff168473ffffffffffffffffffffffffffffffffffffffff167fddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef84604051610b8591815260200190565b60405180910390a350505050565b73ffffffffffffffffffffffffffffffffffffffff8216610bf65760405162461bcd60e51b815260206004820152601f60248201527f45524332303a206d696e7420746f20746865207a65726f20616464726573730060448201526064016104eb565b8060026000828254610c089190611066565b909155505073ffffffffffffffffffffffffffffffffffffffff821660009081526020819052604081208054839290610c42908490611066565b909155505060405181815273ffffffffffffffffffffffffffffffffffffffff8316906000907fddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef9060200160405180910390a35050565b73ffffffffffffffffffffffffffffffffffffffff8216610d225760405162461bcd60e51b815260206004820152602160248201527f45524332303a206275726e2066726f6d20746865207a65726f2061646472657360448201527f730000000000000000000000000000000000000000000000000000000000000060648201526084016104eb565b73ffffffffffffffffffffffffffffffffffffffff821660009081526020819052604090205481811015610dbe5760405162461bcd60e51b815260206004820152602260248201527f45524332303a206275726e20616d6f756e7420657863656564732062616c616e60448201527f636500000000000000000000000000000000000000000000000000000000000060648201526084016104eb565b73ffffffffffffffffffffffffffffffffffffffff83166000908152602081905260408120838303905560028054849290610dfa90849061107e565b909155505060405182815260009073ffffffffffffffffffffffffffffffffffffffff8516907fddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef90602001610920565b600060208284031215610e5c57600080fd5b81357fffffffff0000000000000000000000000000000000000000000000000000000081168114610e8c57600080fd5b9392505050565b600060208083528351808285015260005b81811015610ec057858101830151858201604001528201610ea4565b81811115610ed2576000604083870101525b50601f017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe016929092016040019392505050565b803573ffffffffffffffffffffffffffffffffffffffff81168114610f2a57600080fd5b919050565b60008060408385031215610f4257600080fd5b610f4b83610f06565b946020939093013593505050565b600080600060608486031215610f6e57600080fd5b610f7784610f06565b9250610f8560208501610f06565b9150604084013590509250925092565b600060208284031215610fa757600080fd5b610e8c82610f06565b60008060408385031215610fc357600080fd5b610fcc83610f06565b9150610fda60208401610f06565b90509250929050565b600181811c90821680610ff757607f821691505b60208210811415611031577f4e487b7100000000000000000000000000000000000000000000000000000000600052602260045260246000fd5b50919050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052601160045260246000fd5b6000821982111561107957611079611037565b500190565b60008282101561109057611090611037565b50039056fea264697066735822122069a2d33039157f3f2f180571262ca2a5d0a3a24d33bf9448f3b7c2ce9ff757f964736f6c63430008090033a2646970667358221220d2e13f28319115807ec7308d1cd88642a8542d0b838e00b8769f8a85d696f26764736f6c63430008090033" + }, + "0x4200000000000000000000000000000000000013": { + "balance": "00", + "storage": {}, + "code": "0x4B60005260206000F3" + }, + "0xDeadDeAddeAddEAddeadDEaDDEAdDeaDDeAD0000": { + "balance": "00", + "storage": { + "0x0000000000000000000000000000000000000000000000000000000000000006": "0x0000000000000000000000004200000000000000000000000000000000000010", + "0x0000000000000000000000000000000000000000000000000000000000000005": "0x0000000000000000000000000000000000000000000000000000000000000000", + "0x0000000000000000000000000000000000000000000000000000000000000003": "0x457468657200000000000000000000000000000000000000000000000000000a", + "0x0000000000000000000000000000000000000000000000000000000000000004": "0x4554480000000000000000000000000000000000000000000000000000000006" + }, + "code": "0x608060405234801561001057600080fd5b50600436106101005760003560e01c806370a0823111610097578063a9059cbb11610066578063a9059cbb14610215578063ae1f6aaf14610228578063c01e1bd61461026d578063dd62ed3e1461028d57600080fd5b806370a08231146101b157806395d89b41146101e75780639dc29fac146101ef578063a457c2d71461020257600080fd5b806323b872dd116100d357806323b872dd14610167578063313ce5671461017a578063395093511461018957806340c10f191461019c57600080fd5b806301ffc9a71461010557806306fdde031461012d578063095ea7b31461014257806318160ddd14610155575b600080fd5b610118610113366004610c6d565b6102d3565b60405190151581526020015b60405180910390f35b610135610393565b6040516101249190610cb6565b610118610150366004610d52565b610425565b6002545b604051908152602001610124565b610118610175366004610d7c565b6104db565b60405160128152602001610124565b610118610197366004610d52565b61058c565b6101af6101aa366004610d52565b61063d565b005b6101596101bf366004610db8565b73ffffffffffffffffffffffffffffffffffffffff1660009081526020819052604090205490565b61013561071c565b6101af6101fd366004610d52565b61072b565b610118610210366004610d52565b6107fe565b610118610223366004610d52565b6108af565b6006546102489073ffffffffffffffffffffffffffffffffffffffff1681565b60405173ffffffffffffffffffffffffffffffffffffffff9091168152602001610124565b6005546102489073ffffffffffffffffffffffffffffffffffffffff1681565b61015961029b366004610dd3565b73ffffffffffffffffffffffffffffffffffffffff918216600090815260016020908152604080832093909416825291909152205490565b60007f01ffc9a7a5cef8baa21ed3c5c0d7e23accb804b619e9333b597f47a0d84076e27f1d1d8b63000000000000000000000000000000000000000000000000000000007fffffffff0000000000000000000000000000000000000000000000000000000084167f01ffc9a700000000000000000000000000000000000000000000000000000000148061038b57507fffffffff00000000000000000000000000000000000000000000000000000000848116908216145b949350505050565b6060600380546103a290610e06565b80601f01602080910402602001604051908101604052809291908181526020018280546103ce90610e06565b801561041b5780601f106103f05761010080835404028352916020019161041b565b820191906000526020600020905b8154815290600101906020018083116103fe57829003601f168201915b5050505050905090565b6040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152604260248201527f4f564d5f4554483a20617070726f76652069732064697361626c65642070656e60448201527f64696e67206675727468657220636f6d6d756e6974792064697363757373696f60648201527f6e2e000000000000000000000000000000000000000000000000000000000000608482015260009060a4015b60405180910390fd5b6040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152604760248201527f4f564d5f4554483a207472616e7366657246726f6d2069732064697361626c6560448201527f642070656e64696e67206675727468657220636f6d6d756e697479206469736360648201527f757373696f6e2e00000000000000000000000000000000000000000000000000608482015260009060a4016104d2565b6040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152604c60248201527f4f564d5f4554483a20696e637265617365416c6c6f77616e636520697320646960448201527f7361626c65642070656e64696e67206675727468657220636f6d6d756e69747960648201527f2064697363757373696f6e2e0000000000000000000000000000000000000000608482015260009060a4016104d2565b60065473ffffffffffffffffffffffffffffffffffffffff1633146106be576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820181905260248201527f4f6e6c79204c32204272696467652063616e206d696e7420616e64206275726e60448201526064016104d2565b6106c88282610960565b8173ffffffffffffffffffffffffffffffffffffffff167f0f6798a560793a54c3bcfe86a93cde1e73087d944c0ea20544137d41213968858260405161071091815260200190565b60405180910390a25050565b6060600480546103a290610e06565b60065473ffffffffffffffffffffffffffffffffffffffff1633146107ac576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820181905260248201527f4f6e6c79204c32204272696467652063616e206d696e7420616e64206275726e60448201526064016104d2565b6107b68282610a80565b8173ffffffffffffffffffffffffffffffffffffffff167fcc16f5dbb4873280815c1ee09dbd06736cffcc184412cf7a71a0fdb75d397ca58260405161071091815260200190565b6040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152604c60248201527f4f564d5f4554483a206465637265617365416c6c6f77616e636520697320646960448201527f7361626c65642070656e64696e67206675727468657220636f6d6d756e69747960648201527f2064697363757373696f6e2e0000000000000000000000000000000000000000608482015260009060a4016104d2565b6040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152604360248201527f4f564d5f4554483a207472616e736665722069732064697361626c656420706560448201527f6e64696e67206675727468657220636f6d6d756e69747920646973637573736960648201527f6f6e2e0000000000000000000000000000000000000000000000000000000000608482015260009060a4016104d2565b73ffffffffffffffffffffffffffffffffffffffff82166109dd576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152601f60248201527f45524332303a206d696e7420746f20746865207a65726f20616464726573730060448201526064016104d2565b80600260008282546109ef9190610e89565b909155505073ffffffffffffffffffffffffffffffffffffffff821660009081526020819052604081208054839290610a29908490610e89565b909155505060405181815273ffffffffffffffffffffffffffffffffffffffff8316906000907fddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef9060200160405180910390a35050565b73ffffffffffffffffffffffffffffffffffffffff8216610b23576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152602160248201527f45524332303a206275726e2066726f6d20746865207a65726f2061646472657360448201527f730000000000000000000000000000000000000000000000000000000000000060648201526084016104d2565b73ffffffffffffffffffffffffffffffffffffffff821660009081526020819052604090205481811015610bd9576040517f08c379a000000000000000000000000000000000000000000000000000000000815260206004820152602260248201527f45524332303a206275726e20616d6f756e7420657863656564732062616c616e60448201527f636500000000000000000000000000000000000000000000000000000000000060648201526084016104d2565b73ffffffffffffffffffffffffffffffffffffffff83166000908152602081905260408120838303905560028054849290610c15908490610ea1565b909155505060405182815260009073ffffffffffffffffffffffffffffffffffffffff8516907fddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef9060200160405180910390a3505050565b600060208284031215610c7f57600080fd5b81357fffffffff0000000000000000000000000000000000000000000000000000000081168114610caf57600080fd5b9392505050565b600060208083528351808285015260005b81811015610ce357858101830151858201604001528201610cc7565b81811115610cf5576000604083870101525b50601f017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe016929092016040019392505050565b803573ffffffffffffffffffffffffffffffffffffffff81168114610d4d57600080fd5b919050565b60008060408385031215610d6557600080fd5b610d6e83610d29565b946020939093013593505050565b600080600060608486031215610d9157600080fd5b610d9a84610d29565b9250610da860208501610d29565b9150604084013590509250925092565b600060208284031215610dca57600080fd5b610caf82610d29565b60008060408385031215610de657600080fd5b610def83610d29565b9150610dfd60208401610d29565b90509250929050565b600181811c90821680610e1a57607f821691505b60208210811415610e54577f4e487b7100000000000000000000000000000000000000000000000000000000600052602260045260246000fd5b50919050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052601160045260246000fd5b60008219821115610e9c57610e9c610e5a565b500190565b600082821015610eb357610eb3610e5a565b50039056fea2646970667358221220b71535a5111461b42945e5d842957b3a5926f7ed07d271872f6da21952b5f8b464736f6c63430008090033" + }, + "0x4200000000000000000000000000000000000006": { + "balance": "00", + "storage": { + "0x0000000000000000000000000000000000000000000000000000000000000000": "0x577261707065642045746865720000000000000000000000000000000000001a", + "0x0000000000000000000000000000000000000000000000000000000000000001": "0x5745544800000000000000000000000000000000000000000000000000000008", + "0x0000000000000000000000000000000000000000000000000000000000000002": "0x0000000000000000000000000000000000000000000000000000000000000012" + }, + "code": "0x6080604052600436106100bc5760003560e01c8063313ce56711610074578063a9059cbb1161004e578063a9059cbb146102cb578063d0e30db0146100bc578063dd62ed3e14610311576100bc565b8063313ce5671461024b57806370a082311461027657806395d89b41146102b6576100bc565b806318160ddd116100a557806318160ddd146101aa57806323b872dd146101d15780632e1a7d4d14610221576100bc565b806306fdde03146100c6578063095ea7b314610150575b6100c4610359565b005b3480156100d257600080fd5b506100db6103a8565b6040805160208082528351818301528351919283929083019185019080838360005b838110156101155781810151838201526020016100fd565b50505050905090810190601f1680156101425780820380516001836020036101000a031916815260200191505b509250505060405180910390f35b34801561015c57600080fd5b506101966004803603604081101561017357600080fd5b5073ffffffffffffffffffffffffffffffffffffffff8135169060200135610454565b604080519115158252519081900360200190f35b3480156101b657600080fd5b506101bf6104c7565b60408051918252519081900360200190f35b3480156101dd57600080fd5b50610196600480360360608110156101f457600080fd5b5073ffffffffffffffffffffffffffffffffffffffff8135811691602081013590911690604001356104cb565b34801561022d57600080fd5b506100c46004803603602081101561024457600080fd5b503561066b565b34801561025757600080fd5b50610260610700565b6040805160ff9092168252519081900360200190f35b34801561028257600080fd5b506101bf6004803603602081101561029957600080fd5b503573ffffffffffffffffffffffffffffffffffffffff16610709565b3480156102c257600080fd5b506100db61071b565b3480156102d757600080fd5b50610196600480360360408110156102ee57600080fd5b5073ffffffffffffffffffffffffffffffffffffffff8135169060200135610793565b34801561031d57600080fd5b506101bf6004803603604081101561033457600080fd5b5073ffffffffffffffffffffffffffffffffffffffff813581169160200135166107a7565b33600081815260036020908152604091829020805434908101909155825190815291517fe1fffcc4923d04b559f4d29a8bfc6cda04eb5b0d3c460751c2402c5c5cc9109c9281900390910190a2565b6000805460408051602060026001851615610100027fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0190941693909304601f8101849004840282018401909252818152929183018282801561044c5780601f106104215761010080835404028352916020019161044c565b820191906000526020600020905b81548152906001019060200180831161042f57829003601f168201915b505050505081565b33600081815260046020908152604080832073ffffffffffffffffffffffffffffffffffffffff8716808552908352818420869055815186815291519394909390927f8c5be1e5ebec7d5bd14f71427d1e84f3dd0314c0f7b2291e5b200ac8c7c3b925928290030190a350600192915050565b4790565b73ffffffffffffffffffffffffffffffffffffffff83166000908152600360205260408120548211156104fd57600080fd5b73ffffffffffffffffffffffffffffffffffffffff84163314801590610573575073ffffffffffffffffffffffffffffffffffffffff841660009081526004602090815260408083203384529091529020547fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff14155b156105ed5773ffffffffffffffffffffffffffffffffffffffff841660009081526004602090815260408083203384529091529020548211156105b557600080fd5b73ffffffffffffffffffffffffffffffffffffffff841660009081526004602090815260408083203384529091529020805483900390555b73ffffffffffffffffffffffffffffffffffffffff808516600081815260036020908152604080832080548890039055938716808352918490208054870190558351868152935191937fddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef929081900390910190a35060019392505050565b3360009081526003602052604090205481111561068757600080fd5b33600081815260036020526040808220805485900390555183156108fc0291849190818181858888f193505050501580156106c6573d6000803e3d6000fd5b5060408051828152905133917f7fcf532c15f0a6db0bd6d0e038bea71d30d808c7d98cb3bf7268a95bf5081b65919081900360200190a250565b60025460ff1681565b60036020526000908152604090205481565b60018054604080516020600284861615610100027fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0190941693909304601f8101849004840282018401909252818152929183018282801561044c5780601f106104215761010080835404028352916020019161044c565b60006107a03384846104cb565b9392505050565b60046020908152600092835260408084209091529082529020548156fea265627a7a7231582091c18790e0cca5011d2518024840ee00fecc67e11f56fd746f2cf84d5b583e0064736f6c63430005110032" + } + } +} diff --git a/crates/primitives/src/chain/mod.rs b/crates/primitives/src/chain/mod.rs index d03aff38aa1..d051c63b852 100644 --- a/crates/primitives/src/chain/mod.rs +++ b/crates/primitives/src/chain/mod.rs @@ -11,6 +11,9 @@ pub use spec::{ AllGenesisFormats, ChainSpec, ChainSpecBuilder, ForkCondition, GOERLI, MAINNET, SEPOLIA, }; +#[cfg(feature = "optimism")] +pub use spec::OP_GOERLI; + // The chain info module. mod info; pub use info::ChainInfo; @@ -41,6 +44,18 @@ impl Chain { Chain::Named(ethers_core::types::Chain::Sepolia) } + /// Returns the optimism goerli chain. + #[cfg(feature = "optimism")] + pub const fn optimism_goerli() -> Self { + Chain::Named(ethers_core::types::Chain::OptimismGoerli) + } + + /// Returns the optimism mainnet chain. + #[cfg(feature = "optimism")] + pub const fn optimism_mainnet() -> Self { + Chain::Named(ethers_core::types::Chain::Optimism) + } + /// The id of the chain pub fn id(&self) -> u64 { match self { diff --git a/crates/primitives/src/chain/spec.rs b/crates/primitives/src/chain/spec.rs index e9798f78f06..1b3e4dd831b 100644 --- a/crates/primitives/src/chain/spec.rs +++ b/crates/primitives/src/chain/spec.rs @@ -100,6 +100,30 @@ pub static SEPOLIA: Lazy = Lazy::new(|| ChainSpec { optimism: None, }); +/// The Optimism Goerli spec +#[cfg(feature = "optimism")] +pub static OP_GOERLI: Lazy = Lazy::new(|| ChainSpec { + chain: Chain::optimism_goerli(), + genesis: serde_json::from_str(include_str!("../../res/genesis/goerli_op.json")) + .expect("Can't deserialize Optimism Goerli genesis json"), + genesis_hash: Some(H256(hex!( + "c1fc15cd51159b1f1e5cbc4b82e85c1447ddfa33c52cf1d98d14fba0d6354be1" + ))), + hardforks: BTreeMap::from([ + (Hardfork::Byzantium, ForkCondition::Block(0)), + (Hardfork::Constantinople, ForkCondition::Block(0)), + (Hardfork::Petersburg, ForkCondition::Block(0)), + (Hardfork::Istanbul, ForkCondition::Block(0)), + (Hardfork::MuirGlacier, ForkCondition::Block(0)), + (Hardfork::Berlin, ForkCondition::Block(0)), + (Hardfork::London, ForkCondition::Block(4061224)), + (Hardfork::ArrowGlacier, ForkCondition::Block(4061224)), + (Hardfork::GrayGlacier, ForkCondition::Block(4061224)), + (Hardfork::Bedrock, ForkCondition::Block(4061224)), + ]), + optimism: Some(OptimismConfig { eip_1559_elasticity: 10, eip_1559_denominator: 50 }), +}); + /// An Ethereum chain specification. /// /// A chain specification describes: @@ -634,6 +658,9 @@ mod tests { Head, GOERLI, MAINNET, SEPOLIA, }; + #[cfg(feature = "optimism")] + use crate::OP_GOERLI; + fn test_fork_ids(spec: &ChainSpec, cases: &[(Head, ForkId)]) { for (block, expected_id) in cases { let computed_id = spec.fork_id(block); @@ -809,6 +836,25 @@ mod tests { ); } + #[cfg(feature = "optimism")] + #[test] + fn optimism_goerli_forkids() { + // TODO + test_fork_ids( + &OP_GOERLI, + &[ + ( + Head { number: 0, ..Default::default() }, + ForkId { hash: ForkHash([0x00, 0x00, 0x00, 0x00]), next: 4061224 }, + ), + ( + Head { number: 4061224, ..Default::default() }, + ForkId { hash: ForkHash([0x00, 0x00, 0x00, 0x00]), next: 0 }, + ), + ], + ); + } + /// Checks that time-based forks work /// /// This is based off of the test vectors here: https://github.com/ethereum/go-ethereum/blob/5c8cc10d1e05c23ff1108022f4150749e73c0ca1/core/forkid/forkid_test.go#L155-L188 diff --git a/crates/primitives/src/constants.rs b/crates/primitives/src/constants.rs index 4739074f827..913459f9921 100644 --- a/crates/primitives/src/constants.rs +++ b/crates/primitives/src/constants.rs @@ -27,6 +27,11 @@ pub const GOERLI_GENESIS: H256 = pub const SEPOLIA_GENESIS: H256 = H256(hex!("25a5cc106eea7138acab33231d7160d69cb777ee0c2c553fcddf5138993e6dd9")); +/// Optimism goerli genesis hash. +#[cfg(feature = "optimism")] +pub const GOERLI_OP_GENESIS: H256 = + H256(hex!("c1fc15cd51159b1f1e5cbc4b82e85c1447ddfa33c52cf1d98d14fba0d6354be1")); + /// Keccak256 over empty array. pub const KECCAK_EMPTY: H256 = H256(hex!("c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")); diff --git a/crates/primitives/src/lib.rs b/crates/primitives/src/lib.rs index 3007a20847d..a02714e39fa 100644 --- a/crates/primitives/src/lib.rs +++ b/crates/primitives/src/lib.rs @@ -39,6 +39,8 @@ pub use account::Account; pub use bits::H512; pub use block::{Block, BlockHashOrNumber, BlockId, BlockNumberOrTag, SealedBlock}; pub use bloom::Bloom; +#[cfg(feature = "optimism")] +pub use chain::OP_GOERLI; pub use chain::{ AllGenesisFormats, Chain, ChainInfo, ChainSpec, ChainSpecBuilder, ForkCondition, GOERLI, MAINNET, SEPOLIA, diff --git a/crates/rpc/rpc-engine-api/Cargo.toml b/crates/rpc/rpc-engine-api/Cargo.toml index 3be1201f493..b2d68f31d8d 100644 --- a/crates/rpc/rpc-engine-api/Cargo.toml +++ b/crates/rpc/rpc-engine-api/Cargo.toml @@ -31,3 +31,6 @@ thiserror = "1.0.37" reth-interfaces = { path = "../../interfaces", features = ["test-utils"] } reth-provider = { path = "../../storage/provider", features = ["test-utils"] } assert_matches = "1.5.0" + +[features] +optimism = ["reth-primitives/optimism", "reth-rpc-types/optimism"] diff --git a/crates/rpc/rpc-engine-api/src/engine_api.rs b/crates/rpc/rpc-engine-api/src/engine_api.rs index c5e8df1d3f2..6526bf67534 100644 --- a/crates/rpc/rpc-engine-api/src/engine_api.rs +++ b/crates/rpc/rpc-engine-api/src/engine_api.rs @@ -348,7 +348,12 @@ impl Date: Sun, 5 Mar 2023 15:02:33 -0700 Subject: [PATCH 051/191] Signature fix --- crates/primitives/src/transaction/signature.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crates/primitives/src/transaction/signature.rs b/crates/primitives/src/transaction/signature.rs index b20c4f79b2e..0489cfe14f1 100644 --- a/crates/primitives/src/transaction/signature.rs +++ b/crates/primitives/src/transaction/signature.rs @@ -38,6 +38,11 @@ impl Signature { /// Output the `v` of the signature depends on chain_id #[inline] pub fn v(&self, chain_id: Option) -> u64 { + #[cfg(feature = "optimism")] + if self.r == U256::ZERO && self.s == U256::ZERO { + return 0; + } + if let Some(chain_id) = chain_id { // EIP-155: v = {0, 1} + CHAIN_ID * 2 + 35 self.odd_y_parity as u64 + chain_id * 2 + 35 From ce2b83e7740c2739bc917a81306db570262f3d70 Mon Sep 17 00:00:00 2001 From: Francisco Krause Arnim <56402156+fkrause98@users.noreply.github.com> Date: Mon, 6 Mar 2023 11:43:44 -0300 Subject: [PATCH 052/191] feat(rpc): rpc handler for eth_getUncleByBlock{Hash/Number}AndIndex (#1595) Co-authored-by: lambdaclass-user --- crates/rpc/rpc-builder/tests/it/http.rs | 12 ++----- crates/rpc/rpc-types/src/eth/block.rs | 46 +++++++++++++++++++++---- crates/rpc/rpc/src/eth/api/block.rs | 16 ++++++++- crates/rpc/rpc/src/eth/api/server.rs | 12 +++---- 4 files changed, 62 insertions(+), 24 deletions(-) diff --git a/crates/rpc/rpc-builder/tests/it/http.rs b/crates/rpc/rpc-builder/tests/it/http.rs index c709f9d0794..10b2d44ca78 100644 --- a/crates/rpc/rpc-builder/tests/it/http.rs +++ b/crates/rpc/rpc-builder/tests/it/http.rs @@ -73,19 +73,11 @@ where EthApiClient::block_transaction_count_by_hash(client, hash).await.unwrap(); EthApiClient::block_uncles_count_by_hash(client, hash).await.unwrap(); EthApiClient::block_uncles_count_by_number(client, block_number).await.unwrap(); - + EthApiClient::uncle_by_block_hash_and_index(client, hash, index).await.unwrap(); + EthApiClient::uncle_by_block_number_and_index(client, block_number, index).await.unwrap(); // Unimplemented assert!(is_unimplemented(EthApiClient::syncing(client).await.err().unwrap())); assert!(is_unimplemented(EthApiClient::author(client).await.err().unwrap())); - assert!(is_unimplemented( - EthApiClient::uncle_by_block_hash_and_index(client, hash, index).await.err().unwrap() - )); - assert!(is_unimplemented( - EthApiClient::uncle_by_block_number_and_index(client, block_number, index) - .await - .err() - .unwrap() - )); assert!(is_unimplemented(EthApiClient::transaction_by_hash(client, hash).await.err().unwrap())); assert!(is_unimplemented( EthApiClient::transaction_by_block_hash_and_index(client, hash, index).await.err().unwrap() diff --git a/crates/rpc/rpc-types/src/eth/block.rs b/crates/rpc/rpc-types/src/eth/block.rs index 5b6c563ffca..aa74d1a63ef 100644 --- a/crates/rpc/rpc-types/src/eth/block.rs +++ b/crates/rpc/rpc-types/src/eth/block.rs @@ -8,7 +8,8 @@ use reth_rlp::Encodable; use serde::{ser::Error, Deserialize, Serialize, Serializer}; use std::{collections::BTreeMap, ops::Deref}; -/// Block Transactions depending on the boolean attribute of `eth_getBlockBy*` +/// Block Transactions depending on the boolean attribute of `eth_getBlockBy*`, +/// or if used by `eth_getUncle*` #[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] #[serde(untagged)] pub enum BlockTransactions { @@ -16,8 +17,16 @@ pub enum BlockTransactions { Hashes(Vec), /// Full transactions Full(Vec), + /// Special case for uncle response. + Uncle, +} +impl BlockTransactions { + /// Check if the enum variant is + /// used for an uncle response. + pub fn is_uncle(&self) -> bool { + matches!(self, Self::Uncle) + } } - /// Determines how the `transactions` field of [Block] should be filled. /// /// This essentially represents the `full:bool` argument in RPC calls that determine whether the @@ -55,11 +64,14 @@ pub struct Block { /// Header of the block #[serde(flatten)] pub header: Header, - /// Total difficulty - pub total_difficulty: U256, + /// Total difficulty, this field is None only if representing + /// an Uncle block. + #[serde(skip_serializing_if = "Option::is_none")] + pub total_difficulty: Option, /// Uncles' hashes pub uncles: Vec, /// Transactions + #[serde(skip_serializing_if = "BlockTransactions::is_uncle")] pub transactions: BlockTransactions, /// Integer the size of this block in bytes. pub size: Option, @@ -67,6 +79,7 @@ pub struct Block { #[serde(skip_serializing_if = "Option::is_none")] pub base_fee_per_gas: Option, /// Withdrawals + #[serde(skip_serializing_if = "Option::is_none")] pub withdrawals: Option>, } @@ -160,11 +173,29 @@ impl Block { uncles, transactions, base_fee_per_gas: base_fee_per_gas.map(U256::from), - total_difficulty, + total_difficulty: Some(total_difficulty), size: Some(U256::from(block_length)), withdrawals: block.withdrawals, } } + + /// Build an RPC block response representing + /// an Uncle from its header. + pub fn uncle_block_from_header(header: PrimitiveHeader) -> Self { + let hash = header.hash_slow(); + let rpc_header = Header::from_primitive_with_hash(header.clone(), hash); + let uncle_block = PrimitiveBlock { header, ..Default::default() }; + let size = Some(U256::from(uncle_block.length())); + Self { + uncles: vec![], + header: rpc_header, + transactions: BlockTransactions::Uncle, + base_fee_per_gas: None, + withdrawals: None, + size, + total_difficulty: None, + } + } } /// Block header representation. @@ -189,6 +220,7 @@ pub struct Header { /// Transactions receipts root hash pub receipts_root: H256, /// Withdrawals root hash + #[serde(skip_serializing_if = "Option::is_none")] pub withdrawals_root: Option, /// Block number pub number: Option, @@ -354,7 +386,7 @@ mod tests { mix_hash: H256::from_low_u64_be(14), nonce: Some(H64::from_low_u64_be(15)), }, - total_difficulty: U256::from(100000), + total_difficulty: Some(U256::from(100000)), uncles: vec![H256::from_low_u64_be(17)], transactions: BlockTransactions::Hashes(vec![H256::from_low_u64_be(18)]), size: Some(U256::from(19)), @@ -364,7 +396,7 @@ mod tests { let serialized = serde_json::to_string(&block).unwrap(); assert_eq!( serialized, - r#"{"hash":"0x0000000000000000000000000000000000000000000000000000000000000001","parentHash":"0x0000000000000000000000000000000000000000000000000000000000000002","sha3Uncles":"0x0000000000000000000000000000000000000000000000000000000000000003","author":"0x0000000000000000000000000000000000000004","miner":"0x0000000000000000000000000000000000000004","stateRoot":"0x0000000000000000000000000000000000000000000000000000000000000005","transactionsRoot":"0x0000000000000000000000000000000000000000000000000000000000000006","receiptsRoot":"0x0000000000000000000000000000000000000000000000000000000000000007","withdrawalsRoot":"0x0000000000000000000000000000000000000000000000000000000000000008","number":"0x9","gasUsed":"0xa","gasLimit":"0xb","extraData":"0x010203","logsBloom":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","timestamp":"0xc","difficulty":"0xd","mixHash":"0x000000000000000000000000000000000000000000000000000000000000000e","nonce":"0x000000000000000f","totalDifficulty":"0x186a0","uncles":["0x0000000000000000000000000000000000000000000000000000000000000011"],"transactions":["0x0000000000000000000000000000000000000000000000000000000000000012"],"size":"0x13","baseFeePerGas":"0x14","withdrawals":null}"# + r#"{"hash":"0x0000000000000000000000000000000000000000000000000000000000000001","parentHash":"0x0000000000000000000000000000000000000000000000000000000000000002","sha3Uncles":"0x0000000000000000000000000000000000000000000000000000000000000003","author":"0x0000000000000000000000000000000000000004","miner":"0x0000000000000000000000000000000000000004","stateRoot":"0x0000000000000000000000000000000000000000000000000000000000000005","transactionsRoot":"0x0000000000000000000000000000000000000000000000000000000000000006","receiptsRoot":"0x0000000000000000000000000000000000000000000000000000000000000007","withdrawalsRoot":"0x0000000000000000000000000000000000000000000000000000000000000008","number":"0x9","gasUsed":"0xa","gasLimit":"0xb","extraData":"0x010203","logsBloom":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","timestamp":"0xc","difficulty":"0xd","mixHash":"0x000000000000000000000000000000000000000000000000000000000000000e","nonce":"0x000000000000000f","totalDifficulty":"0x186a0","uncles":["0x0000000000000000000000000000000000000000000000000000000000000011"],"transactions":["0x0000000000000000000000000000000000000000000000000000000000000012"],"size":"0x13","baseFeePerGas":"0x14"}"# ); let deserialized: Block = serde_json::from_str(&serialized).unwrap(); assert_eq!(block, deserialized); diff --git a/crates/rpc/rpc/src/eth/api/block.rs b/crates/rpc/rpc/src/eth/api/block.rs index 158ba31d568..30dd290f46a 100644 --- a/crates/rpc/rpc/src/eth/api/block.rs +++ b/crates/rpc/rpc/src/eth/api/block.rs @@ -6,7 +6,7 @@ use crate::{ }; use reth_primitives::BlockId; use reth_provider::{BlockProvider, EvmEnvProvider, StateProviderFactory}; -use reth_rpc_types::{Block, RichBlock}; +use reth_rpc_types::{Block, Index, RichBlock}; impl EthApi where @@ -23,6 +23,20 @@ where Ok(self.client().ommers(block_id)?) } + pub(crate) async fn ommer_by_block_and_index( + &self, + block_id: impl Into, + index: Index, + ) -> EthResult> { + let block_id = block_id.into(); + let index = usize::from(index); + let uncles = self.client().ommers(block_id)?.unwrap_or_default(); + let uncle = uncles + .into_iter() + .nth(index) + .map(|header| Block::uncle_block_from_header(header).into()); + Ok(uncle) + } pub(crate) async fn block_transaction_count( &self, block_id: impl Into, diff --git a/crates/rpc/rpc/src/eth/api/server.rs b/crates/rpc/rpc/src/eth/api/server.rs index 471ff2eb166..b5ce8bdb973 100644 --- a/crates/rpc/rpc/src/eth/api/server.rs +++ b/crates/rpc/rpc/src/eth/api/server.rs @@ -102,19 +102,19 @@ where /// Handler for: `eth_getUncleByBlockHashAndIndex` async fn uncle_by_block_hash_and_index( &self, - _hash: H256, - _index: Index, + hash: H256, + index: Index, ) -> Result> { - Err(internal_rpc_err("unimplemented")) + Ok(EthApi::ommer_by_block_and_index(self, hash, index).await?) } /// Handler for: `eth_getUncleByBlockNumberAndIndex` async fn uncle_by_block_number_and_index( &self, - _number: BlockNumberOrTag, - _index: Index, + number: BlockNumberOrTag, + index: Index, ) -> Result> { - Err(internal_rpc_err("unimplemented")) + Ok(EthApi::ommer_by_block_and_index(self, number, index).await?) } /// Handler for: `eth_getTransactionByHash` From 809d0cab03beaabd261269eadf0eac6d73bc318e Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Mon, 6 Mar 2023 16:49:48 +0200 Subject: [PATCH 053/191] fix(txpool): queued tx ordering (#1643) --- crates/transaction-pool/src/pool/mod.rs | 1 - crates/transaction-pool/src/pool/parked.rs | 9 ++++++--- crates/transaction-pool/src/pool/transaction.rs | 1 - 3 files changed, 6 insertions(+), 5 deletions(-) delete mode 100644 crates/transaction-pool/src/pool/transaction.rs diff --git a/crates/transaction-pool/src/pool/mod.rs b/crates/transaction-pool/src/pool/mod.rs index 3772b41861b..334dec5eb29 100644 --- a/crates/transaction-pool/src/pool/mod.rs +++ b/crates/transaction-pool/src/pool/mod.rs @@ -92,7 +92,6 @@ mod parked; pub(crate) mod pending; pub(crate) mod size; pub(crate) mod state; -mod transaction; pub mod txpool; mod update; diff --git a/crates/transaction-pool/src/pool/parked.rs b/crates/transaction-pool/src/pool/parked.rs index 94105a16098..ed16256b6dd 100644 --- a/crates/transaction-pool/src/pool/parked.rs +++ b/crates/transaction-pool/src/pool/parked.rs @@ -231,8 +231,8 @@ impl Ord for BasefeeOrd { /// `Queued` transactions are transactions that are currently blocked by other parked (basefee, /// queued) or missing transactions. /// -/// The primary order function for is always compares via the timestamp when the transaction was -/// created +/// The primary order function always compares the transaction costs first. In case these +/// are equal, it compares the timestamps when the transactions were created. #[derive(Debug)] pub(crate) struct QueuedOrd(Arc>); @@ -240,6 +240,9 @@ impl_ord_wrapper!(QueuedOrd); impl Ord for QueuedOrd { fn cmp(&self, other: &Self) -> Ordering { - other.timestamp.cmp(&self.timestamp) + // Higher cost is better + self.cost.cmp(&other.cost).then_with(|| + // Lower timestamp is better + other.timestamp.cmp(&self.timestamp)) } } diff --git a/crates/transaction-pool/src/pool/transaction.rs b/crates/transaction-pool/src/pool/transaction.rs deleted file mode 100644 index 8b137891791..00000000000 --- a/crates/transaction-pool/src/pool/transaction.rs +++ /dev/null @@ -1 +0,0 @@ - From 1e007d69113a02550a35e7c98f227f0712fb0334 Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Mon, 6 Mar 2023 17:59:01 +0200 Subject: [PATCH 054/191] docs(txpool): minor fixes (#1645) --- crates/transaction-pool/src/pool/txpool.rs | 37 ++++++++++++---------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/crates/transaction-pool/src/pool/txpool.rs b/crates/transaction-pool/src/pool/txpool.rs index 983a9d48ac2..eeeeb087231 100644 --- a/crates/transaction-pool/src/pool/txpool.rs +++ b/crates/transaction-pool/src/pool/txpool.rs @@ -26,9 +26,11 @@ use std::{ sync::Arc, }; -/// The minimal value the basefee can decrease to +/// The minimal value the basefee can decrease to. /// -/// The `BASE_FEE_MAX_CHANGE_DENOMINATOR` is `8`, or 12.5%, once the base fee has dropped to `7` WEI it cannot decrease further because 12.5% of 7 is less than 1. +/// The `BASE_FEE_MAX_CHANGE_DENOMINATOR` is `8`, or 12.5%. +/// Once the base fee has dropped to `7` WEI it cannot decrease further because 12.5% of 7 is less +/// than 1. pub(crate) const MIN_PROTOCOL_BASE_FEE: u128 = 7; /// A pool that manages transactions. @@ -66,9 +68,11 @@ pub(crate) const MIN_PROTOCOL_BASE_FEE: u128 = 7; /// new --> |apply state changes| pool /// ``` pub struct TxPool { - /// Contains the currently known info + /// Contains the currently known information about the senders. sender_info: FnvHashMap, /// pending subpool + /// + /// Holds transactions that are ready to be executed on the current state. pending_pool: PendingPool, /// Pool settings to enforce limits etc. config: PoolConfig, @@ -136,7 +140,7 @@ impl TxPool { self.pending_pool.best() } - /// Returns if the transaction for the given hash is already included in this pool + /// Returns `true` if the transaction with the given hash is already included in this pool. pub(crate) fn contains(&self, tx_hash: &TxHash) -> bool { self.all_transactions.contains(tx_hash) } @@ -149,7 +153,7 @@ impl TxPool { self.all_transactions.by_hash.get(tx_hash).cloned() } - /// Returns all transaction for the hashes, if it exists. + /// Returns transactions for the multiple given hashes, if they exist. pub(crate) fn get_all<'a>( &'a self, txs: impl IntoIterator + 'a, @@ -189,21 +193,21 @@ impl TxPool { /// This pool consists of two three-pools: `Queued`, `Pending` and `BaseFee`. /// /// The `Queued` pool contains transactions with gaps in its dependency tree: It requires - /// additional transaction that are note yet present in the pool. And transactions that the + /// additional transactions that are note yet present in the pool. And transactions that the /// sender can not afford with the current balance. /// /// The `Pending` pool contains all transactions that have no nonce gaps, and can be afforded by /// the sender. It only contains transactions that are ready to be included in the pending - /// block. In the pending pool are all transactions that could be listed currently, but not + /// block. The pending pool contains all transactions that could be listed currently, but not /// necessarily independently. However, this pool never contains transactions with nonce gaps. A /// transaction is considered `ready` when it has the lowest nonce of all transactions from the /// same sender. Which is equals to the chain nonce of the sender in the pending pool. /// - /// The `BaseFee` pool contains transaction that currently can't satisfy the dynamic fee + /// The `BaseFee` pool contains transactions that currently can't satisfy the dynamic fee /// requirement. With EIP-1559, transactions can become executable or not without any changes to - /// the sender's balance or nonce and instead their feeCap determines whether the + /// the sender's balance or nonce and instead their `feeCap` determines whether the /// transaction is _currently_ (on the current state) ready or needs to be parked until the - /// feeCap satisfies the block's baseFee. + /// `feeCap` satisfies the block's `baseFee`. pub(crate) fn add_transaction( &mut self, tx: ValidPoolTransaction, @@ -294,11 +298,11 @@ impl TxPool { /// Moves a transaction from one sub pool to another. /// - /// This will remove the given transaction from one sub-pool and insert it in the other + /// This will remove the given transaction from one sub-pool and insert it into the other /// sub-pool. fn move_transaction(&mut self, from: SubPool, to: SubPool, id: &TransactionId) { if let Some(tx) = self.remove_from_subpool(from, id) { - self.add_transaction_to_pool(to, tx); + self.add_transaction_to_subpool(to, tx); } } @@ -372,8 +376,8 @@ impl TxPool { } } - /// Removes the transaction from the given pool - fn add_transaction_to_pool( + /// Inserts the transaction into the given sub-pool. + fn add_transaction_to_subpool( &mut self, pool: SubPool, tx: Arc>, @@ -391,7 +395,8 @@ impl TxPool { } } - /// Inserts the transaction into the given sub-pool + /// Inserts the transaction into the given sub-pool. + /// Optionally, removes the replacement transaction. fn add_new_transaction( &mut self, transaction: Arc>, @@ -403,7 +408,7 @@ impl TxPool { self.remove_from_subpool(replaced_pool, replaced.id()); } - self.add_transaction_to_pool(pool, transaction) + self.add_transaction_to_subpool(pool, transaction) } /// Ensures that the transactions in the sub-pools are within the given bounds. From 75e677cfde92e5ef35254330d72985192726a89f Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Mon, 6 Mar 2023 19:22:12 +0200 Subject: [PATCH 055/191] chore(txpool): expose `PooledTransaction` (#1649) --- crates/transaction-pool/src/lib.rs | 4 ++-- crates/transaction-pool/src/traits.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/transaction-pool/src/lib.rs b/crates/transaction-pool/src/lib.rs index f9d099a32d5..3dd3ee29287 100644 --- a/crates/transaction-pool/src/lib.rs +++ b/crates/transaction-pool/src/lib.rs @@ -83,8 +83,8 @@ pub use crate::{ config::PoolConfig, ordering::TransactionOrdering, traits::{ - BestTransactions, OnNewBlockEvent, PoolTransaction, PropagateKind, PropagatedTransactions, - TransactionOrigin, TransactionPool, + BestTransactions, OnNewBlockEvent, PoolTransaction, PooledTransaction, PropagateKind, + PropagatedTransactions, TransactionOrigin, TransactionPool, }, validate::{TransactionValidationOutcome, TransactionValidator, ValidPoolTransaction}, }; diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index 94a4d358094..1d33567c661 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -302,7 +302,7 @@ pub trait PoolTransaction: fmt::Debug + Send + Sync + FromRecoveredTransaction { /// This type is essentially a wrapper around [TransactionSignedEcRecovered] with additional fields /// derived from the transaction that are frequently used by the pools for ordering. #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub(crate) struct PooledTransaction { +pub struct PooledTransaction { /// EcRecovered transaction info pub(crate) transaction: TransactionSignedEcRecovered, From 3503444342f1ba41cde5605f4d1e8224facbdceb Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Mon, 6 Mar 2023 21:55:41 +0200 Subject: [PATCH 056/191] chore(txpool): expose underlying tx (#1651) --- crates/transaction-pool/src/traits.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index 1d33567c661..300699e8616 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -313,6 +313,13 @@ pub struct PooledTransaction { pub(crate) effective_gas_price: u128, } +impl PooledTransaction { + /// Return the reference to the underlying transaction. + pub fn transaction(&self) -> &TransactionSignedEcRecovered { + &self.transaction + } +} + impl PoolTransaction for PooledTransaction { /// Returns hash of the transaction. fn hash(&self) -> &TxHash { From 32e31b240dbfe9c98f6f7bfba9f76b878a3dc728 Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Mon, 6 Mar 2023 23:18:51 +0200 Subject: [PATCH 057/191] perf(provider): remove redundant lookup (#1650) --- crates/storage/provider/src/providers/mod.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/crates/storage/provider/src/providers/mod.rs b/crates/storage/provider/src/providers/mod.rs index 8ea781b06f3..19801249eff 100644 --- a/crates/storage/provider/src/providers/mod.rs +++ b/crates/storage/provider/src/providers/mod.rs @@ -62,11 +62,7 @@ impl HeaderProvider for ShareableDatabase { } fn header_by_number(&self, num: BlockNumber) -> Result> { - if let Some(hash) = self.db.view(|tx| tx.get::(num))?? { - self.header(&hash) - } else { - Ok(None) - } + Ok(self.db.view(|tx| tx.get::(num))??) } fn header_td(&self, hash: &BlockHash) -> Result> { From ce39c9810db8062ce34e53d2e268d6514cf9b7c6 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 7 Mar 2023 11:33:05 +0100 Subject: [PATCH 058/191] chore: add InvalidChainId error variant (#1657) --- Cargo.lock | 8 ++++---- crates/rpc/rpc/src/eth/error.rs | 9 ++++++--- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 776cd62b6d2..a243ad0c806 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5160,7 +5160,7 @@ dependencies = [ [[package]] name = "revm" version = "3.0.0" -source = "git+https://github.com/bluealloy/revm#3a17ca8556ee2933100d1e1ed4e200712715f76f" +source = "git+https://github.com/bluealloy/revm#7bb73da23d0278829f9669f175a6eede247c0538" dependencies = [ "auto_impl", "revm-interpreter", @@ -5170,7 +5170,7 @@ dependencies = [ [[package]] name = "revm-interpreter" version = "1.0.0" -source = "git+https://github.com/bluealloy/revm#3a17ca8556ee2933100d1e1ed4e200712715f76f" +source = "git+https://github.com/bluealloy/revm#7bb73da23d0278829f9669f175a6eede247c0538" dependencies = [ "bitvec 1.0.1", "derive_more", @@ -5182,7 +5182,7 @@ dependencies = [ [[package]] name = "revm-precompile" version = "2.0.0" -source = "git+https://github.com/bluealloy/revm#3a17ca8556ee2933100d1e1ed4e200712715f76f" +source = "git+https://github.com/bluealloy/revm#7bb73da23d0278829f9669f175a6eede247c0538" dependencies = [ "k256", "num", @@ -5198,7 +5198,7 @@ dependencies = [ [[package]] name = "revm-primitives" version = "1.0.0" -source = "git+https://github.com/bluealloy/revm#3a17ca8556ee2933100d1e1ed4e200712715f76f" +source = "git+https://github.com/bluealloy/revm#7bb73da23d0278829f9669f175a6eede247c0538" dependencies = [ "arbitrary", "auto_impl", diff --git a/crates/rpc/rpc/src/eth/error.rs b/crates/rpc/rpc/src/eth/error.rs index 8a080c7b52a..ce38462f0df 100644 --- a/crates/rpc/rpc/src/eth/error.rs +++ b/crates/rpc/rpc/src/eth/error.rs @@ -179,15 +179,17 @@ pub enum InvalidTransactionError { /// Unspecific evm halt error #[error("EVM error {0:?}")] EvmHalt(Halt), + #[error("Invalid chain id")] + InvalidChainId, } impl InvalidTransactionError { /// Returns the rpc error code for this error. fn error_code(&self) -> i32 { match self { - InvalidTransactionError::GasTooLow | InvalidTransactionError::GasTooHigh => { - EthRpcErrorCode::InvalidInput.code() - } + InvalidTransactionError::InvalidChainId | + InvalidTransactionError::GasTooLow | + InvalidTransactionError::GasTooHigh => EthRpcErrorCode::InvalidInput.code(), InvalidTransactionError::Revert(_) => EthRpcErrorCode::ExecutionError.code(), _ => EthRpcErrorCode::TransactionRejected.code(), } @@ -214,6 +216,7 @@ impl From for InvalidTransactionError { fn from(err: revm::primitives::InvalidTransaction) -> Self { use revm::primitives::InvalidTransaction; match err { + InvalidTransaction::InvalidChainId => InvalidTransactionError::InvalidChainId, InvalidTransaction::GasMaxFeeGreaterThanPriorityFee => { InvalidTransactionError::TipAboveFeeCap } From e06d0a4a0bffcd1faddb600da1a99ae8e0e492ca Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 7 Mar 2023 11:38:10 +0100 Subject: [PATCH 059/191] chore(rpc): match all error variants (#1658) --- crates/rpc/rpc/src/eth/error.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/crates/rpc/rpc/src/eth/error.rs b/crates/rpc/rpc/src/eth/error.rs index ce38462f0df..49cecc7cd58 100644 --- a/crates/rpc/rpc/src/eth/error.rs +++ b/crates/rpc/rpc/src/eth/error.rs @@ -77,8 +77,8 @@ pub(crate) enum EthApiError { } impl From for RpcError { - fn from(value: EthApiError) -> Self { - match value { + fn from(error: EthApiError) -> Self { + match error { EthApiError::FailedToDecodeSignedTransaction | EthApiError::InvalidTransactionSignature | EthApiError::EmptyRawTransactionData | @@ -86,11 +86,15 @@ impl From for RpcError { EthApiError::InvalidBlockRange | EthApiError::ConflictingRequestGasPrice { .. } | EthApiError::ConflictingRequestGasPriceAndTipSet { .. } | - EthApiError::RequestLegacyGasPriceAndTipSet { .. } => { - rpc_err(INVALID_PARAMS_CODE, value.to_string(), None) + EthApiError::RequestLegacyGasPriceAndTipSet { .. } | + EthApiError::BothStateAndStateDiffInOverride(_) => { + rpc_err(INVALID_PARAMS_CODE, error.to_string(), None) } EthApiError::InvalidTransaction(err) => err.into(), - err => internal_rpc_err(err.to_string()), + EthApiError::PoolError(_) | + EthApiError::PrevrandaoNotSet | + EthApiError::InvalidBlockData(_) | + EthApiError::Internal(_) => internal_rpc_err(error.to_string()), } } } From a1d0b537a7549717956cb750d10045f353a9f8e6 Mon Sep 17 00:00:00 2001 From: rakita Date: Tue, 7 Mar 2023 13:49:43 +0100 Subject: [PATCH 060/191] bug: Remove old account/storage root in merkle trie (#1646) --- crates/storage/provider/src/trie/mod.rs | 34 ++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/crates/storage/provider/src/trie/mod.rs b/crates/storage/provider/src/trie/mod.rs index c99689b8461..4b8f5d744ab 100644 --- a/crates/storage/provider/src/trie/mod.rs +++ b/crates/storage/provider/src/trie/mod.rs @@ -290,6 +290,11 @@ impl DBTrieLoader { let root = H256::from_slice(trie.root()?.as_slice()); + // if root is empty remove it from db + if root == EMPTY_ROOT { + tx.delete::(address, None)?; + } + Ok(root) } @@ -329,9 +334,15 @@ impl DBTrieLoader { } } - let root = H256::from_slice(trie.root()?.as_slice()); + let new_root = H256::from_slice(trie.root()?.as_slice()); + if new_root != root { + let mut cursor = tx.cursor_write::()?; + if cursor.seek_exact(root)?.is_some() { + cursor.delete_current()?; + } + } - Ok(root) + Ok(new_root) } fn update_storage_root( @@ -359,9 +370,24 @@ impl DBTrieLoader { } } - let root = H256::from_slice(trie.root()?.as_slice()); + let new_root = H256::from_slice(trie.root()?.as_slice()); + if new_root != root { + let mut cursor = tx.cursor_dup_write::()?; + if cursor + .seek_by_key_subkey(address, root)? + .filter(|entry| entry.hash == root) + .is_some() + { + cursor.delete_current()?; + } + } - Ok(root) + // if root is empty remove it from db + if new_root == EMPTY_ROOT { + tx.delete::(address, None)?; + } + + Ok(new_root) } fn gather_changes( From edeedd519e005e030b229ffdc89c4515209b998e Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 7 Mar 2023 17:56:05 +0100 Subject: [PATCH 061/191] fix(net): enforce soft limit when propagating tx hashes (#1660) --- crates/net/eth-wire/src/types/broadcast.rs | 29 ++++++++++++++++++++++ crates/net/network/src/transactions.rs | 19 +++++++++----- 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/crates/net/eth-wire/src/types/broadcast.rs b/crates/net/eth-wire/src/types/broadcast.rs index 163f16bf8b7..b7de075db25 100644 --- a/crates/net/eth-wire/src/types/broadcast.rs +++ b/crates/net/eth-wire/src/types/broadcast.rs @@ -152,6 +152,35 @@ impl NewPooledTransactionHashes { NewPooledTransactionHashes::Eth68(msg) => msg.hashes.into_iter(), } } + + /// Shortens the number of hashes in the message, keeping the first `len` hashes and dropping + /// the rest. If `len` is greater than the number of hashes, this has no effect. + pub fn truncate(&mut self, len: usize) { + match self { + NewPooledTransactionHashes::Eth66(msg) => msg.0.truncate(len), + NewPooledTransactionHashes::Eth68(msg) => { + msg.types.truncate(len); + msg.sizes.truncate(len); + msg.hashes.truncate(len); + } + } + } + + /// Returns true if the message is empty + pub fn is_empty(&self) -> bool { + match self { + NewPooledTransactionHashes::Eth66(msg) => msg.0.is_empty(), + NewPooledTransactionHashes::Eth68(msg) => msg.hashes.is_empty(), + } + } + + /// Returns the number of hashes in the message + pub fn len(&self) -> usize { + match self { + NewPooledTransactionHashes::Eth66(msg) => msg.0.len(), + NewPooledTransactionHashes::Eth68(msg) => msg.hashes.len(), + } + } } impl From for EthMessage { diff --git a/crates/net/network/src/transactions.rs b/crates/net/network/src/transactions.rs index 39bf239df6b..1cf1f0afe9a 100644 --- a/crates/net/network/src/transactions.rs +++ b/crates/net/network/src/transactions.rs @@ -227,7 +227,7 @@ where let max_num_full = (self.peers.len() as f64).sqrt() as usize + 1; // Note: Assuming ~random~ order due to random state of the peers map hasher - for (idx, (peer_id, peer)) in self.peers.iter_mut().enumerate() { + for (peer_idx, (peer_id, peer)) in self.peers.iter_mut().enumerate() { // filter all transactions unknown to the peer let mut hashes = PooledTransactionsHashesBuilder::new(peer.version); let mut full_transactions = Vec::new(); @@ -237,20 +237,27 @@ where full_transactions.push(Arc::clone(&tx.transaction)); } } - let hashes = hashes.build(); + let mut new_pooled_hashes = hashes.build(); if !full_transactions.is_empty() { - if idx > max_num_full { - for hash in hashes.iter_hashes().copied() { + // determine whether to send full tx objects or hashes. + if peer_idx > max_num_full { + // enforce tx soft limit per message for the (unlikely) event the number of + // hashes exceeds it + new_pooled_hashes.truncate(NEW_POOLED_TRANSACTION_HASHES_SOFT_LIMIT); + + for hash in new_pooled_hashes.iter_hashes().copied() { propagated.0.entry(hash).or_default().push(PropagateKind::Hash(*peer_id)); } // send hashes of transactions - self.network.send_transactions_hashes(*peer_id, hashes); + self.network.send_transactions_hashes(*peer_id, new_pooled_hashes); } else { + // TODO ensure max message size + // send full transactions self.network.send_transactions(*peer_id, full_transactions); - for hash in hashes.into_iter_hashes() { + for hash in new_pooled_hashes.into_iter_hashes() { propagated.0.entry(hash).or_default().push(PropagateKind::Full(*peer_id)); } } From a34226c8e5f5fb4bfea431036e0c1fa19283ce47 Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Tue, 7 Mar 2023 19:56:34 +0200 Subject: [PATCH 062/191] chore(executor): reuse revm hashmap (#1663) --- Cargo.lock | 1 - crates/executor/Cargo.toml | 3 +-- crates/executor/src/executor.rs | 19 +++++++++++-------- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a243ad0c806..c91026d0fbb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4600,7 +4600,6 @@ dependencies = [ "async-trait", "auto_impl", "hash-db", - "hashbrown 0.13.2", "plain_hasher", "reth-db", "reth-interfaces", diff --git a/crates/executor/Cargo.toml b/crates/executor/Cargo.toml index 41e42af20a2..095aa06084c 100644 --- a/crates/executor/Cargo.toml +++ b/crates/executor/Cargo.toml @@ -16,9 +16,8 @@ reth-rlp = { path = "../rlp" } reth-db = { path = "../storage/db" } reth-provider = { path = "../storage/provider" } +# revm revm = { version = "3.0.0"} -# remove from reth and reexport from revm -hashbrown = "0.13" # common async-trait = "0.1.57" diff --git a/crates/executor/src/executor.rs b/crates/executor/src/executor.rs index ab27c6c639e..9ce50bc0a19 100644 --- a/crates/executor/src/executor.rs +++ b/crates/executor/src/executor.rs @@ -1,14 +1,14 @@ -use crate::execution_result::{ - AccountChangeSet, AccountInfoChangeSet, ExecutionResult, TransactionChangeSet, -}; - -use hashbrown::hash_map::Entry; use reth_interfaces::executor::{BlockExecutor, Error}; use reth_primitives::{ bloom::logs_bloom, Account, Address, Block, Bloom, ChainSpec, Hardfork, Header, Log, Receipt, TransactionSigned, H256, U256, }; -use reth_provider::StateProvider; +use reth_provider::{ + execution_result::{ + AccountChangeSet, AccountInfoChangeSet, ExecutionResult, TransactionChangeSet, + }, + StateProvider, +}; use reth_revm::{ config::{WEI_2ETH, WEI_3ETH, WEI_5ETH}, database::SubState, @@ -18,7 +18,10 @@ use reth_revm::{ use reth_revm_inspectors::stack::{InspectorStack, InspectorStackConfig}; use revm::{ db::AccountState, - primitives::{Account as RevmAccount, AccountInfo, Bytecode, ResultAndState}, + primitives::{ + hash_map::{self, Entry}, + Account as RevmAccount, AccountInfo, Bytecode, ResultAndState, + }, EVM, }; use std::{ @@ -123,7 +126,7 @@ where /// BTreeMap is used to have sorted values fn commit_changes( &mut self, - changes: hashbrown::HashMap, + changes: hash_map::HashMap, ) -> (BTreeMap, BTreeMap) { let db = self.db(); From 77d108660e78660d32aa3a35fc664fd5fbfdf985 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s?= <47506558+MegaRedHand@users.noreply.github.com> Date: Wed, 8 Mar 2023 06:07:09 -0300 Subject: [PATCH 063/191] feat(rpc): add `eth_createAccessList` implementation (#1652) Co-authored-by: lambdaclass-user --- Cargo.lock | 1 + crates/primitives/src/lib.rs | 7 +- .../primitives/src/transaction/access_list.rs | 11 ++ crates/primitives/src/transaction/mod.rs | 2 +- crates/revm/Cargo.toml | 3 +- crates/revm/src/lib.rs | 2 + crates/rpc/rpc-api/src/eth.rs | 3 +- crates/rpc/rpc-builder/tests/it/http.rs | 5 +- crates/rpc/rpc-types/src/eth/log.rs | 1 - .../rpc/rpc-types/src/eth/transaction/mod.rs | 1 - crates/rpc/rpc/Cargo.toml | 7 +- crates/rpc/rpc/src/eth/api/call.rs | 115 ++++++++++++++++-- crates/rpc/rpc/src/eth/api/server.rs | 13 +- crates/storage/provider/src/providers/mod.rs | 1 + 14 files changed, 139 insertions(+), 33 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c91026d0fbb..595da4031f5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4868,6 +4868,7 @@ dependencies = [ "reth-interfaces", "reth-primitives", "reth-provider", + "reth-revm-inspectors", "reth-revm-primitives", "revm", ] diff --git a/crates/primitives/src/lib.rs b/crates/primitives/src/lib.rs index 3b0e600ea77..52f349c3af6 100644 --- a/crates/primitives/src/lib.rs +++ b/crates/primitives/src/lib.rs @@ -59,9 +59,10 @@ pub use receipt::Receipt; pub use serde_helper::JsonU256; pub use storage::{StorageEntry, StorageTrieEntry}; pub use transaction::{ - AccessList, AccessListItem, FromRecoveredTransaction, IntoRecoveredTransaction, Signature, - Transaction, TransactionKind, TransactionSigned, TransactionSignedEcRecovered, TxEip1559, - TxEip2930, TxLegacy, TxType, EIP1559_TX_TYPE_ID, EIP2930_TX_TYPE_ID, LEGACY_TX_TYPE_ID, + AccessList, AccessListItem, AccessListWithGasUsed, FromRecoveredTransaction, + IntoRecoveredTransaction, Signature, Transaction, TransactionKind, TransactionSigned, + TransactionSignedEcRecovered, TxEip1559, TxEip2930, TxLegacy, TxType, EIP1559_TX_TYPE_ID, + EIP2930_TX_TYPE_ID, LEGACY_TX_TYPE_ID, }; pub use withdrawal::Withdrawal; diff --git a/crates/primitives/src/transaction/access_list.rs b/crates/primitives/src/transaction/access_list.rs index 823a98bbc45..d7295f5dccb 100644 --- a/crates/primitives/src/transaction/access_list.rs +++ b/crates/primitives/src/transaction/access_list.rs @@ -3,6 +3,7 @@ use revm_primitives::U256; use reth_codecs::{main_codec, Compact}; use reth_rlp::{RlpDecodable, RlpDecodableWrapper, RlpEncodable, RlpEncodableWrapper}; +use serde::{Deserialize, Serialize}; /// A list of addresses and storage keys that the transaction plans to access. /// Accesses outside the list are possible, but become more expensive. @@ -34,3 +35,13 @@ impl AccessList { .collect() } } + +/// Access list with gas used appended. +#[derive(Serialize, Deserialize, Clone, Debug)] +#[serde(rename_all = "camelCase")] +pub struct AccessListWithGasUsed { + /// List with accounts accessed during transaction. + pub access_list: AccessList, + /// Estimated gas used with access list. + pub gas_used: U256, +} diff --git a/crates/primitives/src/transaction/mod.rs b/crates/primitives/src/transaction/mod.rs index 12b495a7636..45708a048c5 100644 --- a/crates/primitives/src/transaction/mod.rs +++ b/crates/primitives/src/transaction/mod.rs @@ -1,5 +1,5 @@ use crate::{keccak256, Address, Bytes, ChainId, TxHash, H256}; -pub use access_list::{AccessList, AccessListItem}; +pub use access_list::{AccessList, AccessListItem, AccessListWithGasUsed}; use bytes::{Buf, BytesMut}; use derive_more::{AsRef, Deref}; use reth_codecs::{add_arbitrary_tests, main_codec, Compact}; diff --git a/crates/revm/Cargo.toml b/crates/revm/Cargo.toml index de419c346d1..ae1a639554e 100644 --- a/crates/revm/Cargo.toml +++ b/crates/revm/Cargo.toml @@ -12,5 +12,6 @@ reth-primitives = { path = "../primitives" } reth-interfaces = { path = "../interfaces" } reth-provider = { path = "../storage/provider" } reth-revm-primitives = { path = "./revm-primitives" } +reth-revm-inspectors = { path = "./revm-inspectors" } -revm = { version = "3.0.0"} \ No newline at end of file +revm = { version = "3.0.0" } diff --git a/crates/revm/src/lib.rs b/crates/revm/src/lib.rs index 781ae36c332..c4541a2ee88 100644 --- a/crates/revm/src/lib.rs +++ b/crates/revm/src/lib.rs @@ -10,6 +10,8 @@ /// Contains glue code for integrating reth database into revm's [Database](revm::Database). pub mod database; +/// reexport for convenience +pub use reth_revm_inspectors::*; /// reexport for convenience pub use reth_revm_primitives::*; diff --git a/crates/rpc/rpc-api/src/eth.rs b/crates/rpc/rpc-api/src/eth.rs index b16cf6bf0fc..dc715088f62 100644 --- a/crates/rpc/rpc-api/src/eth.rs +++ b/crates/rpc/rpc-api/src/eth.rs @@ -1,7 +1,6 @@ use jsonrpsee::{core::RpcResult as Result, proc_macros::rpc}; use reth_primitives::{ - rpc::transaction::eip2930::AccessListWithGasUsed, Address, BlockId, BlockNumberOrTag, Bytes, - H256, H64, U256, U64, + AccessListWithGasUsed, Address, BlockId, BlockNumberOrTag, Bytes, H256, H64, U256, U64, }; use reth_rpc_types::{ state::StateOverride, CallRequest, EIP1186AccountProofResponse, FeeHistory, Index, RichBlock, diff --git a/crates/rpc/rpc-builder/tests/it/http.rs b/crates/rpc/rpc-builder/tests/it/http.rs index 10b2d44ca78..79783781483 100644 --- a/crates/rpc/rpc-builder/tests/it/http.rs +++ b/crates/rpc/rpc-builder/tests/it/http.rs @@ -75,6 +75,8 @@ where EthApiClient::block_uncles_count_by_number(client, block_number).await.unwrap(); EthApiClient::uncle_by_block_hash_and_index(client, hash, index).await.unwrap(); EthApiClient::uncle_by_block_number_and_index(client, block_number, index).await.unwrap(); + EthApiClient::create_access_list(client, call_request.clone(), None).await.unwrap(); + // Unimplemented assert!(is_unimplemented(EthApiClient::syncing(client).await.err().unwrap())); assert!(is_unimplemented(EthApiClient::author(client).await.err().unwrap())); @@ -92,9 +94,6 @@ where assert!(is_unimplemented( EthApiClient::call(client, call_request.clone(), None, None).await.err().unwrap() )); - assert!(is_unimplemented( - EthApiClient::create_access_list(client, call_request.clone(), None).await.err().unwrap() - )); assert!(is_unimplemented( EthApiClient::estimate_gas(client, call_request.clone(), None).await.err().unwrap() )); diff --git a/crates/rpc/rpc-types/src/eth/log.rs b/crates/rpc/rpc-types/src/eth/log.rs index 0eee01fbbd6..06becfe555c 100644 --- a/crates/rpc/rpc-types/src/eth/log.rs +++ b/crates/rpc/rpc-types/src/eth/log.rs @@ -31,7 +31,6 @@ pub struct Log { #[cfg(test)] mod tests { use super::*; - use serde_json; #[test] fn serde_log() { diff --git a/crates/rpc/rpc-types/src/eth/transaction/mod.rs b/crates/rpc/rpc-types/src/eth/transaction/mod.rs index bce73152339..a04a3cbd785 100644 --- a/crates/rpc/rpc-types/src/eth/transaction/mod.rs +++ b/crates/rpc/rpc-types/src/eth/transaction/mod.rs @@ -157,7 +157,6 @@ impl Transaction { #[cfg(test)] mod tests { use super::*; - use serde_json; #[test] fn serde_transaction() { diff --git a/crates/rpc/rpc/Cargo.toml b/crates/rpc/rpc/Cargo.toml index 34e20387a12..949fb3313ef 100644 --- a/crates/rpc/rpc/Cargo.toml +++ b/crates/rpc/rpc/Cargo.toml @@ -16,7 +16,9 @@ reth-rpc-api = { path = "../rpc-api" } reth-rlp = { path = "../../rlp" } reth-rpc-types = { path = "../rpc-types" } reth-provider = { path = "../../storage/provider", features = ["test-utils"] } -reth-transaction-pool = { path = "../../transaction-pool", features = ["test-utils"]} +reth-transaction-pool = { path = "../../transaction-pool", features = [ + "test-utils", +] } reth-network-api = { path = "../../net/network-api", features = ["test-utils"] } reth-rpc-engine-api = { path = "../rpc-engine-api" } reth-revm = { path = "../../revm" } @@ -56,5 +58,4 @@ schnellru = "0.2" futures = "0.3.26" [dev-dependencies] -jsonrpsee = { version = "0.16", features = ["client"]} - +jsonrpsee = { version = "0.16", features = ["client"] } diff --git a/crates/rpc/rpc/src/eth/api/call.rs b/crates/rpc/rpc/src/eth/api/call.rs index 7c8ffc70699..dc6e4146ab6 100644 --- a/crates/rpc/rpc/src/eth/api/call.rs +++ b/crates/rpc/rpc/src/eth/api/call.rs @@ -6,26 +6,33 @@ use crate::{ eth::error::{EthApiError, EthResult, InvalidTransactionError, RevertError}, EthApi, }; +use ethers_core::utils::get_contract_address; use reth_primitives::{ - AccessList, Address, BlockId, BlockNumberOrTag, Bytes, TransactionKind, U128, U256, + AccessList, AccessListItem, AccessListWithGasUsed, Address, BlockId, BlockNumberOrTag, Bytes, + TransactionKind, H256, U128, U256, }; use reth_provider::{BlockProvider, EvmEnvProvider, StateProvider, StateProviderFactory}; -use reth_revm::database::{State, SubState}; +use reth_revm::{ + access_list::AccessListInspector, + database::{State, SubState}, +}; use reth_rpc_types::{ state::{AccountOverride, StateOverride}, CallRequest, }; use revm::{ db::{CacheDB, DatabaseRef}, + precompile::{Precompiles, SpecId as PrecompilesSpecId}, primitives::{ ruint::Uint, BlockEnv, Bytecode, CfgEnv, Env, ExecutionResult, Halt, ResultAndState, - TransactTo, TxEnv, + SpecId, TransactTo, TxEnv, }, - Database, + Database, Inspector, }; // Gas per transaction not creating a contract. const MIN_TRANSACTION_GAS: u64 = 21_000u64; +const MIN_CREATE_GAS: u64 = 53_000u64; impl EthApi where @@ -222,11 +229,14 @@ where // transaction requires to succeed let gas_used = res.result.gas_used(); // the lowest value is capped by the gas it takes for a transfer - let mut lowest_gas_limit = MIN_TRANSACTION_GAS; + let mut lowest_gas_limit = + if env.tx.transact_to.is_create() { MIN_CREATE_GAS } else { MIN_TRANSACTION_GAS }; let mut highest_gas_limit: u64 = highest_gas_limit.try_into().unwrap_or(u64::MAX); // pick a point that's close to the estimated gas - let mut mid_gas_limit = - std::cmp::min(gas_used * 3, (highest_gas_limit + lowest_gas_limit) / 2); + let mut mid_gas_limit = std::cmp::min( + gas_used * 3, + ((highest_gas_limit as u128 + lowest_gas_limit as u128) / 2) as u64, + ); let mut last_highest_gas_limit = highest_gas_limit; @@ -241,10 +251,10 @@ where highest_gas_limit = mid_gas_limit; // if last two successful estimations only vary by 10%, we consider this to be // sufficiently accurate - const ACCURACY: u64 = 10; - if (last_highest_gas_limit - highest_gas_limit) * ACCURACY / - last_highest_gas_limit < - 1u64 + const ACCURACY: u128 = 10; + if (last_highest_gas_limit - highest_gas_limit) as u128 * ACCURACY / + (last_highest_gas_limit as u128) < + 1u128 { return Ok(U256::from(highest_gas_limit)) } @@ -269,11 +279,77 @@ where } } // new midpoint - mid_gas_limit = (highest_gas_limit + lowest_gas_limit) / 2; + mid_gas_limit = ((highest_gas_limit as u128 + lowest_gas_limit as u128) / 2) as u64; } Ok(U256::from(highest_gas_limit)) } + + pub(crate) async fn create_access_list_at( + &self, + request: CallRequest, + at: Option, + ) -> EthResult { + let block_id = at.unwrap_or(BlockId::Number(BlockNumberOrTag::Latest)); + let (mut cfg, block, at) = self.evm_env_at(block_id).await?; + let state = self.state_at_block_id(at)?.ok_or_else(|| EthApiError::UnknownBlockNumber)?; + + // we want to disable this in eth_call, since this is common practice used by other node + // impls and providers + cfg.disable_block_gas_limit = true; + + let mut env = build_call_evm_env(cfg, block, request.clone())?; + let mut db = SubState::new(State::new(state)); + + let from = request.from.unwrap_or_default(); + let to = if let Some(to) = request.to { + to + } else { + let nonce = db.basic(from)?.unwrap_or_default().nonce; + get_contract_address(from, nonce).into() + }; + + let initial = request.access_list.clone().unwrap_or_default(); + + let precompiles = get_precompiles(&env.cfg.spec_id); + let mut inspector = AccessListInspector::new(initial, from, to, precompiles); + let (result, _env) = inspect(&mut db, env, &mut inspector)?; + + match result.result { + ExecutionResult::Halt { reason, .. } => Err(match reason { + Halt::NonceOverflow => InvalidTransactionError::NonceMaxValue, + halt => InvalidTransactionError::EvmHalt(halt), + }), + ExecutionResult::Revert { output, .. } => { + Err(InvalidTransactionError::Revert(RevertError::new(output))) + } + ExecutionResult::Success { .. } => Ok(()), + }?; + Ok(inspector.into_access_list()) + } +} + +/// Returns the addresses of the precompiles corresponding to the SpecId. +fn get_precompiles(spec_id: &SpecId) -> Vec { + let spec = match spec_id { + SpecId::FRONTIER | SpecId::FRONTIER_THAWING => return vec![], + SpecId::HOMESTEAD | SpecId::DAO_FORK | SpecId::TANGERINE | SpecId::SPURIOUS_DRAGON => { + PrecompilesSpecId::HOMESTEAD + } + SpecId::BYZANTIUM | SpecId::CONSTANTINOPLE | SpecId::PETERSBURG => { + PrecompilesSpecId::BYZANTIUM + } + SpecId::ISTANBUL | SpecId::MUIR_GLACIER => PrecompilesSpecId::ISTANBUL, + SpecId::BERLIN | + SpecId::LONDON | + SpecId::ARROW_GLACIER | + SpecId::GRAY_GLACIER | + SpecId::MERGE | + SpecId::SHANGHAI | + SpecId::CANCUN => PrecompilesSpecId::BERLIN, + SpecId::LATEST => PrecompilesSpecId::LATEST, + }; + Precompiles::new(spec).addresses().into_iter().map(Address::from).collect() } /// Executes the [Env] against the given [Database] without committing state changes. @@ -288,6 +364,19 @@ where Ok((res, evm.env)) } +/// Executes the [Env] against the given [Database] without committing state changes. +pub(crate) fn inspect(db: S, env: Env, inspector: I) -> EthResult<(ResultAndState, Env)> +where + S: Database, + ::Error: Into, + I: Inspector, +{ + let mut evm = revm::EVM::with_env(env); + evm.database(db); + let res = evm.inspect(inspector)?; + Ok((res, evm.env)) +} + /// Creates a new [Env] to be used for executing the [CallRequest] in `eth_call` pub(crate) fn build_call_evm_env( mut cfg: CfgEnv, @@ -317,7 +406,7 @@ fn create_txn_env(block_env: &BlockEnv, request: CallRequest) -> EthResult, + mut request: CallRequest, + block_number: Option, ) -> Result { - Err(internal_rpc_err("unimplemented")) + let block_id = block_number.unwrap_or(BlockId::Number(BlockNumberOrTag::Latest)); + let access_list = self.create_access_list_at(request.clone(), block_number).await?; + request.access_list = Some(access_list.clone()); + let gas_used = self.estimate_gas_at(request, block_id).await?; + Ok(AccessListWithGasUsed { access_list, gas_used }) } /// Handler for: `eth_estimateGas` diff --git a/crates/storage/provider/src/providers/mod.rs b/crates/storage/provider/src/providers/mod.rs index 19801249eff..cc843246f49 100644 --- a/crates/storage/provider/src/providers/mod.rs +++ b/crates/storage/provider/src/providers/mod.rs @@ -298,6 +298,7 @@ impl EvmEnvProvider for ShareableDatabase { impl StateProviderFactory for ShareableDatabase { type HistorySP<'a> = HistoricalStateProvider<'a,>::TX> where Self: 'a; type LatestSP<'a> = LatestStateProvider<'a,>::TX> where Self: 'a; + /// Storage provider for latest block fn latest(&self) -> Result> { Ok(LatestStateProvider::new(self.db.tx()?)) From 2c6a35929e1ce169f1a4eff214f3bee6426aced9 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 8 Mar 2023 13:57:37 +0100 Subject: [PATCH 064/191] docs(txpool): clarify ParkedPool transaction type (#1671) --- crates/transaction-pool/src/pool/parked.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/crates/transaction-pool/src/pool/parked.rs b/crates/transaction-pool/src/pool/parked.rs index ed16256b6dd..b931d83d473 100644 --- a/crates/transaction-pool/src/pool/parked.rs +++ b/crates/transaction-pool/src/pool/parked.rs @@ -4,10 +4,15 @@ use crate::{ use fnv::FnvHashMap; use std::{cmp::Ordering, collections::BTreeSet, ops::Deref, sync::Arc}; -/// A pool of transaction that are currently parked and wait for external changes that eventually -/// move the transaction into the pending pool. +/// A pool of transactions that are currently parked and are waiting for external changes (e.g. +/// basefee, ancestor transactions, balance) that eventually move the transaction into the pending +/// pool. /// -/// This pool is a bijection: at all times each set contains the same transactions. +/// This pool is a bijection: at all times each set (`best`, `by_id`) contains the same +/// transactions. +/// +/// Note: This type is generic over [ParkedPool] which enforces that the underlying transaction type +/// is [ValidPoolTransaction] wrapped in an [Arc]. pub(crate) struct ParkedPool { /// Keeps track of transactions inserted in the pool. /// From 57e36223f7b35b2b3ff023b7ba304bac976cac4d Mon Sep 17 00:00:00 2001 From: Bjerg Date: Wed, 8 Mar 2023 14:12:09 +0100 Subject: [PATCH 065/191] feat: add `debug-fast` profile (#1670) --- Cargo.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index c94e7294bff..ad7253f8624 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,6 +41,11 @@ members = [ exclude = ["crate-template"] default-members = ["bin/reth"] +# Like release, but with full debug symbols. Useful for e.g. `perf`. +[profile.debug-fast] +inherits = "release" +debug = true + [patch.crates-io] revm = { git = "https://github.com/bluealloy/revm" } revm-primitives = { git = "https://github.com/bluealloy/revm" } From 21c66621ddcc9d7af1e1d6b5695b512afeeafc75 Mon Sep 17 00:00:00 2001 From: rakita Date: Wed, 8 Mar 2023 16:01:56 +0100 Subject: [PATCH 066/191] feat: ExecutorFactory (#1672) --- Cargo.lock | 1 + bin/reth/Cargo.toml | 1 + bin/reth/src/chain/import.rs | 11 +- bin/reth/src/dump_stage/execution.rs | 13 ++- bin/reth/src/dump_stage/merkle.rs | 9 +- bin/reth/src/node/mod.rs | 31 ++--- bin/reth/src/stage/mod.rs | 5 +- bin/reth/src/test_eth_chain/runner.rs | 6 +- crates/executor/src/executor.rs | 107 ++++++++---------- crates/executor/src/factory.rs | 34 ++++++ crates/executor/src/lib.rs | 4 + crates/interfaces/src/executor.rs | 22 +--- crates/rpc/rpc-engine-api/src/engine_api.rs | 10 +- crates/stages/Cargo.toml | 2 +- crates/stages/src/lib.rs | 11 +- crates/stages/src/sets.rs | 74 ++++++++---- crates/stages/src/stages/execution.rs | 77 ++++++------- crates/storage/db/benches/utils.rs | 1 + crates/storage/provider/src/lib.rs | 5 +- .../storage/provider/src/traits/executor.rs | 45 ++++++++ crates/storage/provider/src/traits/mod.rs | 3 + 21 files changed, 276 insertions(+), 196 deletions(-) create mode 100644 crates/executor/src/factory.rs create mode 100644 crates/storage/provider/src/traits/executor.rs diff --git a/Cargo.lock b/Cargo.lock index 595da4031f5..2389ae8b758 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4369,6 +4369,7 @@ dependencies = [ "reth-db", "reth-discv4", "reth-downloaders", + "reth-executor", "reth-interfaces", "reth-net-nat", "reth-network", diff --git a/bin/reth/Cargo.toml b/bin/reth/Cargo.toml index abe8a6e4771..11328d60c0d 100644 --- a/bin/reth/Cargo.toml +++ b/bin/reth/Cargo.toml @@ -17,6 +17,7 @@ reth-stages = { path = "../../crates/stages"} reth-interfaces = { path = "../../crates/interfaces", features = ["test-utils"] } reth-transaction-pool = { path = "../../crates/transaction-pool", features = ["test-utils"] } reth-consensus = { path = "../../crates/consensus" } +reth-executor = { path = "../../crates/executor" } reth-rpc-engine-api = { path = "../../crates/rpc/rpc-engine-api" } reth-rpc-builder = { path = "../../crates/rpc/rpc-builder" } reth-rpc = { path = "../../crates/rpc/rpc" } diff --git a/bin/reth/src/chain/import.rs b/bin/reth/src/chain/import.rs index 967ea2fa183..a694ba2001a 100644 --- a/bin/reth/src/chain/import.rs +++ b/bin/reth/src/chain/import.rs @@ -27,7 +27,6 @@ use reth_staged_sync::{ use reth_stages::{ prelude::*, stages::{ExecutionStage, SenderRecoveryStage, TotalDifficultyStage}, - DefaultDB, }; use std::sync::Arc; use tracing::{debug, info}; @@ -141,6 +140,8 @@ impl ImportCommand { .build(file_client.clone(), consensus.clone(), db) .into_task(); + let factory = reth_executor::Factory::new(Arc::new(self.chain.clone())); + let mut pipeline = Pipeline::builder() .with_sync_state_updater(file_client) .add_stages( @@ -149,6 +150,7 @@ impl ImportCommand { header_downloader, body_downloader, NoopStatusUpdater::default(), + factory.clone(), ) .set(TotalDifficultyStage { chain_spec: self.chain.clone(), @@ -157,12 +159,7 @@ impl ImportCommand { .set(SenderRecoveryStage { commit_threshold: config.stages.sender_recovery.commit_threshold, }) - .set({ - let mut stage: ExecutionStage<'_, DefaultDB<'_>> = - ExecutionStage::from(self.chain.clone()); - stage.commit_threshold = config.stages.execution.commit_threshold; - stage - }), + .set(ExecutionStage::new(factory, config.stages.execution.commit_threshold)), ) .with_max_block(0) .build(); diff --git a/bin/reth/src/dump_stage/execution.rs b/bin/reth/src/dump_stage/execution.rs index 5a5ae04decb..7e164d9eb45 100644 --- a/bin/reth/src/dump_stage/execution.rs +++ b/bin/reth/src/dump_stage/execution.rs @@ -9,8 +9,8 @@ use reth_db::{ }; use reth_primitives::MAINNET; use reth_provider::Transaction; -use reth_stages::{stages::ExecutionStage, DefaultDB, Stage, StageId, UnwindInput}; -use std::ops::DerefMut; +use reth_stages::{stages::ExecutionStage, Stage, StageId, UnwindInput}; +use std::{ops::DerefMut, sync::Arc}; use tracing::info; pub(crate) async fn dump_execution_stage( @@ -97,7 +97,10 @@ async fn unwind_and_copy( output_db: &reth_db::mdbx::Env, ) -> eyre::Result<()> { let mut unwind_tx = Transaction::new(db_tool.db)?; - let mut exec_stage: ExecutionStage<'_, DefaultDB<'_>> = ExecutionStage::from(MAINNET.clone()); + + let mut exec_stage = ExecutionStage::new_default_threshold(reth_executor::Factory::new( + Arc::new(MAINNET.clone()), + )); exec_stage .unwind( @@ -126,7 +129,9 @@ async fn dry_run( info!(target: "reth::cli", "Executing stage. [dry-run]"); let mut tx = Transaction::new(&output_db)?; - let mut exec_stage: ExecutionStage<'_, DefaultDB<'_>> = ExecutionStage::from(MAINNET.clone()); + let mut exec_stage = ExecutionStage::new_default_threshold(reth_executor::Factory::new( + Arc::new(MAINNET.clone()), + )); exec_stage .execute( diff --git a/bin/reth/src/dump_stage/merkle.rs b/bin/reth/src/dump_stage/merkle.rs index 8e4b2b56115..a9e8a8f6204 100644 --- a/bin/reth/src/dump_stage/merkle.rs +++ b/bin/reth/src/dump_stage/merkle.rs @@ -9,9 +9,9 @@ use reth_primitives::MAINNET; use reth_provider::Transaction; use reth_stages::{ stages::{AccountHashingStage, ExecutionStage, MerkleStage, StorageHashingStage}, - DefaultDB, Stage, StageId, UnwindInput, + Stage, StageId, UnwindInput, }; -use std::ops::DerefMut; +use std::{ops::DerefMut, sync::Arc}; use tracing::info; pub(crate) async fn dump_merkle_stage( @@ -75,7 +75,10 @@ async fn unwind_and_copy( MerkleStage::default_unwind().unwind(&mut unwind_tx, unwind).await?; // Bring Plainstate to TO (hashing stage execution requires it) - let mut exec_stage: ExecutionStage<'_, DefaultDB<'_>> = ExecutionStage::from(MAINNET.clone()); + let mut exec_stage = ExecutionStage::new_default_threshold(reth_executor::Factory::new( + Arc::new(MAINNET.clone()), + )); + exec_stage.commit_threshold = u64::MAX; exec_stage .unwind( diff --git a/bin/reth/src/node/mod.rs b/bin/reth/src/node/mod.rs index 5852caec680..1cd2a6b2a6c 100644 --- a/bin/reth/src/node/mod.rs +++ b/bin/reth/src/node/mod.rs @@ -50,7 +50,6 @@ use reth_staged_sync::{ use reth_stages::{ prelude::*, stages::{ExecutionStage, SenderRecoveryStage, TotalDifficultyStage, FINISH}, - DefaultDB, }; use reth_tasks::TaskExecutor; use std::{net::SocketAddr, path::PathBuf, sync::Arc}; @@ -444,23 +443,25 @@ impl Command { builder = builder.with_max_block(max_block) } + let factory = reth_executor::Factory::new(Arc::new(self.chain.clone())); let pipeline = builder .with_sync_state_updater(updater.clone()) .add_stages( - DefaultStages::new(consensus.clone(), header_downloader, body_downloader, updater) - .set(TotalDifficultyStage { - chain_spec: self.chain.clone(), - commit_threshold: stage_conf.total_difficulty.commit_threshold, - }) - .set(SenderRecoveryStage { - commit_threshold: stage_conf.sender_recovery.commit_threshold, - }) - .set({ - let mut stage: ExecutionStage<'_, DefaultDB<'_>> = - ExecutionStage::from(self.chain.clone()); - stage.commit_threshold = stage_conf.execution.commit_threshold; - stage - }), + DefaultStages::new( + consensus.clone(), + header_downloader, + body_downloader, + updater, + factory.clone(), + ) + .set(TotalDifficultyStage { + chain_spec: self.chain.clone(), + commit_threshold: stage_conf.total_difficulty.commit_threshold, + }) + .set(SenderRecoveryStage { + commit_threshold: stage_conf.sender_recovery.commit_threshold, + }) + .set(ExecutionStage::new(factory, stage_conf.execution.commit_threshold)), ) .build(); diff --git a/bin/reth/src/stage/mod.rs b/bin/reth/src/stage/mod.rs index 27f82e25d95..5c9ab23fd83 100644 --- a/bin/reth/src/stage/mod.rs +++ b/bin/reth/src/stage/mod.rs @@ -17,7 +17,7 @@ use reth_staged_sync::{ }; use reth_stages::{ stages::{BodyStage, ExecutionStage, SenderRecoveryStage}, - DefaultDB, ExecInput, Stage, StageId, UnwindInput, + ExecInput, Stage, StageId, UnwindInput, }; use std::{net::SocketAddr, sync::Arc}; use tracing::*; @@ -171,7 +171,8 @@ impl Command { stage.execute(&mut tx, input).await?; } StageEnum::Execution => { - let mut stage = ExecutionStage::>::from(self.chain.clone()); + let factory = reth_executor::Factory::new(Arc::new(self.chain.clone())); + let mut stage = ExecutionStage::new(factory, 10_000); stage.commit_threshold = num_blocks; if !self.skip_unwind { stage.unwind(&mut tx, unwind).await?; diff --git a/bin/reth/src/test_eth_chain/runner.rs b/bin/reth/src/test_eth_chain/runner.rs index 2e1b2f99238..cae43ba4ef3 100644 --- a/bin/reth/src/test_eth_chain/runner.rs +++ b/bin/reth/src/test_eth_chain/runner.rs @@ -15,11 +15,12 @@ use reth_primitives::{ }; use reth_provider::Transaction; use reth_rlp::Decodable; -use reth_stages::{stages::ExecutionStage, DefaultDB, ExecInput, Stage, StageId}; +use reth_stages::{stages::ExecutionStage, ExecInput, Stage, StageId}; use std::{ collections::HashMap, ffi::OsStr, path::{Path, PathBuf}, + sync::Arc, }; use tracing::{debug, trace}; @@ -193,7 +194,8 @@ pub async fn run_test(path: PathBuf) -> eyre::Result { // Initialize the execution stage // Hardcode the chain_id to Ethereum 1. - let mut stage = ExecutionStage::>::from(chain_spec); + let factory = reth_executor::Factory::new(Arc::new(chain_spec)); + let mut stage = ExecutionStage::new(factory, 1_000); // Call execution stage let input = ExecInput { diff --git a/crates/executor/src/executor.rs b/crates/executor/src/executor.rs index 9ce50bc0a19..03d28ee9c91 100644 --- a/crates/executor/src/executor.rs +++ b/crates/executor/src/executor.rs @@ -1,14 +1,12 @@ -use reth_interfaces::executor::{BlockExecutor, Error}; +use crate::execution_result::{ + AccountChangeSet, AccountInfoChangeSet, ExecutionResult, TransactionChangeSet, +}; +use reth_interfaces::executor::Error; use reth_primitives::{ bloom::logs_bloom, Account, Address, Block, Bloom, ChainSpec, Hardfork, Header, Log, Receipt, TransactionSigned, H256, U256, }; -use reth_provider::{ - execution_result::{ - AccountChangeSet, AccountInfoChangeSet, ExecutionResult, TransactionChangeSet, - }, - StateProvider, -}; +use reth_provider::{BlockExecutor, StateProvider}; use reth_revm::{ config::{WEI_2ETH, WEI_3ETH, WEI_5ETH}, database::SubState, @@ -30,38 +28,34 @@ use std::{ }; /// Main block executor -pub struct Executor<'a, DB> +pub struct Executor where DB: StateProvider, { /// The configured chain-spec pub chain_spec: Arc, - evm: EVM<&'a mut SubState>, + evm: EVM>, stack: InspectorStack, } -impl<'a, DB> From for Executor<'a, DB> +impl From> for Executor where DB: StateProvider, { /// Instantiates a new executor from the chainspec. Must call /// `with_db` to set the database before executing. - fn from(chain_spec: ChainSpec) -> Self { + fn from(chain_spec: Arc) -> Self { let evm = EVM::new(); - Executor { - chain_spec: Arc::new(chain_spec), - evm, - stack: InspectorStack::new(InspectorStackConfig::default()), - } + Executor { chain_spec, evm, stack: InspectorStack::new(InspectorStackConfig::default()) } } } -impl<'a, DB> Executor<'a, DB> +impl Executor where DB: StateProvider, { /// Creates a new executor from the given chain spec and database. - pub fn new(chain_spec: Arc, db: &'a mut SubState) -> Self { + pub fn new(chain_spec: Arc, db: SubState) -> Self { let mut evm = EVM::new(); evm.database(db); @@ -79,16 +73,6 @@ where self.evm.db().expect("db to not be moved") } - /// Overrides the database - pub fn with_db( - &self, - db: &'a mut SubState, - ) -> Executor<'a, OtherDB> { - let mut evm = EVM::new(); - evm.database(db); - Executor { chain_spec: self.chain_spec.clone(), evm, stack: self.stack.clone() } - } - fn recover_senders( &self, body: &[TransactionSigned], @@ -376,30 +360,6 @@ where } } - /// Execute and verify block - pub fn execute_and_verify_receipt( - &mut self, - block: &Block, - total_difficulty: U256, - senders: Option>, - ) -> Result { - let execution_result = self.execute(block, total_difficulty, senders)?; - - let receipts_iter = - execution_result.tx_changesets.iter().map(|changeset| &changeset.receipt); - - if self.chain_spec.fork(Hardfork::Byzantium).active_at_block(block.header.number) { - verify_receipt(block.header.receipts_root, block.header.logs_bloom, receipts_iter)?; - } - - // TODO Before Byzantium, receipts contained state root that would mean that expensive - // operation as hashing that is needed for state root got calculated in every - // transaction This was replaced with is_success flag. - // See more about EIP here: https://eips.ethereum.org/EIPS/eip-658 - - Ok(execution_result) - } - /// Runs a single transaction in the configured environment and proceeds /// to return the result and state diff (without applying it). /// @@ -487,7 +447,7 @@ where } } -impl<'a, DB> BlockExecutor for Executor<'a, DB> +impl BlockExecutor for Executor where DB: StateProvider, { @@ -521,6 +481,29 @@ where Ok(ExecutionResult { tx_changesets, block_changesets }) } + + fn execute_and_verify_receipt( + &mut self, + block: &Block, + total_difficulty: U256, + senders: Option>, + ) -> Result { + let execution_result = self.execute(block, total_difficulty, senders)?; + + let receipts_iter = + execution_result.tx_changesets.iter().map(|changeset| &changeset.receipt); + + if self.chain_spec.fork(Hardfork::Byzantium).active_at_block(block.header.number) { + verify_receipt(block.header.receipts_root, block.header.logs_bloom, receipts_iter)?; + } + + // TODO Before Byzantium, receipts contained state root that would mean that expensive + // operation as hashing that is needed for state root got calculated in every + // transaction This was replaced with is_success flag. + // See more about EIP here: https://eips.ethereum.org/EIPS/eip-658 + + Ok(execution_result) + } } /// Verify receipts @@ -661,10 +644,10 @@ mod tests { // spec at berlin fork let chain_spec = Arc::new(ChainSpecBuilder::mainnet().berlin_activated().build()); - let mut db = SubState::new(State::new(db)); + let db = SubState::new(State::new(db)); // execute chain and verify receipts - let mut executor = Executor::new(chain_spec, &mut db); + let mut executor = Executor::new(chain_spec, db); let out = executor.execute_and_verify_receipt(&block, U256::ZERO, None).unwrap(); assert_eq!(out.tx_changesets.len(), 1, "Should executed one transaction"); @@ -689,6 +672,7 @@ mod tests { // Check if cache is set // account1 + let db = executor.db(); let cached_acc1 = db.accounts.get(&account1).unwrap(); assert_eq!(cached_acc1.info.balance, account1_info.balance); assert_eq!(cached_acc1.info.nonce, account1_info.nonce); @@ -791,9 +775,9 @@ mod tests { .build(), ); - let mut db = SubState::new(State::new(db)); + let db = SubState::new(State::new(db)); // execute chain and verify receipts - let mut executor = Executor::new(chain_spec, &mut db); + let mut executor = Executor::new(chain_spec, db); let out = executor .execute_and_verify_receipt( &Block { header, body: vec![], ommers: vec![], withdrawals: None }, @@ -805,6 +789,7 @@ mod tests { // Check if cache is set // beneficiary + let db = executor.db(); let dao_beneficiary = db.accounts.get(&crate::eth_dao_fork::DAO_HARDFORK_BENEFICIARY).unwrap(); @@ -881,10 +866,10 @@ mod tests { // spec at berlin fork let chain_spec = Arc::new(ChainSpecBuilder::mainnet().berlin_activated().build()); - let mut db = SubState::new(State::new(db)); + let db = SubState::new(State::new(db)); // execute chain and verify receipts - let mut executor = Executor::new(chain_spec, &mut db); + let mut executor = Executor::new(chain_spec, db); let out = executor.execute_and_verify_receipt(&block, U256::ZERO, None).unwrap(); assert_eq!(out.tx_changesets.len(), 1, "Should executed one transaction"); @@ -930,10 +915,10 @@ mod tests { // spec at shanghai fork let chain_spec = Arc::new(ChainSpecBuilder::mainnet().shanghai_activated().build()); - let mut db = SubState::new(State::new(StateProviderTest::default())); + let db = SubState::new(State::new(StateProviderTest::default())); // execute chain and verify receipts - let mut executor = Executor::new(chain_spec, &mut db); + let mut executor = Executor::new(chain_spec, db); let out = executor.execute_and_verify_receipt(&block, U256::ZERO, None).unwrap(); assert_eq!(out.tx_changesets.len(), 0, "No tx"); diff --git a/crates/executor/src/factory.rs b/crates/executor/src/factory.rs new file mode 100644 index 00000000000..5bd59d0297d --- /dev/null +++ b/crates/executor/src/factory.rs @@ -0,0 +1,34 @@ +use reth_primitives::ChainSpec; +use reth_provider::{ExecutorFactory, StateProvider}; +use reth_revm::database::{State, SubState}; + +use crate::executor::Executor; +use std::sync::Arc; + +/// Factory that spawn Executor. +#[derive(Clone, Debug)] +pub struct Factory { + chain_spec: Arc, +} + +impl Factory { + /// Create new factory + pub fn new(chain_spec: Arc) -> Self { + Self { chain_spec } + } +} + +impl ExecutorFactory for Factory { + type Executor = Executor; + + /// Executor with [`StateProvider`] + fn with_sp(&self, sp: SP) -> Self::Executor { + let substate = SubState::new(State::new(sp)); + Executor::new(self.chain_spec.clone(), substate) + } + + /// Return internal chainspec + fn chain_spec(&self) -> &ChainSpec { + self.chain_spec.as_ref() + } +} diff --git a/crates/executor/src/lib.rs b/crates/executor/src/lib.rs index 6d28d96482c..4bf17e8dc52 100644 --- a/crates/executor/src/lib.rs +++ b/crates/executor/src/lib.rs @@ -13,3 +13,7 @@ pub mod eth_dao_fork; pub use reth_provider::execution_result; /// Executor pub mod executor; + +/// ExecutorFactory impl +pub mod factory; +pub use factory::Factory; diff --git a/crates/interfaces/src/executor.rs b/crates/interfaces/src/executor.rs index 489c00810e8..0165708da30 100644 --- a/crates/interfaces/src/executor.rs +++ b/crates/interfaces/src/executor.rs @@ -1,26 +1,6 @@ -use async_trait::async_trait; -use reth_primitives::{Address, Block, Bloom, H256, U256}; +use reth_primitives::{Bloom, H256}; use thiserror::Error; -/// An executor capable of executing a block. -#[async_trait] -pub trait BlockExecutor { - /// Execute a block. - /// - /// The number of `senders` should be equal to the number of transactions in the block. - /// - /// If no senders are specified, the `execute` function MUST recover the senders for the - /// provided block's transactions internally. We use this to allow for calculating senders in - /// parallel in e.g. staged sync, so that execution can happen without paying for sender - /// recovery costs. - fn execute( - &mut self, - block: &Block, - total_difficulty: U256, - senders: Option>, - ) -> Result; -} - /// BlockExecutor Errors #[allow(missing_docs)] #[derive(Error, Debug, Clone, PartialEq, Eq)] diff --git a/crates/rpc/rpc-engine-api/src/engine_api.rs b/crates/rpc/rpc-engine-api/src/engine_api.rs index d52daa3033e..cc91f293d78 100644 --- a/crates/rpc/rpc-engine-api/src/engine_api.rs +++ b/crates/rpc/rpc-engine-api/src/engine_api.rs @@ -6,8 +6,10 @@ use reth_primitives::{ BlockHash, BlockId, BlockNumber, ChainSpec, Hardfork, Header, SealedBlock, TransactionSigned, H64, U256, }; -use reth_provider::{BlockProvider, EvmEnvProvider, HeaderProvider, StateProviderFactory}; -use reth_revm::database::{State, SubState}; +use reth_provider::{ + BlockExecutor, BlockProvider, EvmEnvProvider, ExecutorFactory, HeaderProvider, + StateProviderFactory, +}; use reth_rlp::Decodable; use reth_rpc_types::engine::{ ExecutionPayload, ExecutionPayloadBodies, ForkchoiceUpdated, PayloadAttributes, PayloadStatus, @@ -305,8 +307,8 @@ impl Ok(PayloadStatus::new(PayloadStatusEnum::Valid, block_hash)), Err(err) => Ok(PayloadStatus::new( diff --git a/crates/stages/Cargo.toml b/crates/stages/Cargo.toml index 3a50de8a2ab..f08754b58af 100644 --- a/crates/stages/Cargo.toml +++ b/crates/stages/Cargo.toml @@ -17,7 +17,6 @@ normal = [ # reth libs reth-primitives = { path = "../primitives" } reth-interfaces = { path = "../interfaces" } -reth-executor = { path = "../executor" } reth-revm = { path = "../revm" } reth-rlp = { path = "../rlp" } reth-db = { path = "../storage/db" } @@ -54,6 +53,7 @@ reth-db = { path = "../storage/db", features = ["test-utils", "mdbx"] } reth-interfaces = { path = "../interfaces", features = ["test-utils"] } reth-downloaders = { path = "../net/downloaders" } reth-eth-wire = { path = "../net/eth-wire" } # TODO(onbjerg): We only need this for [BlockBody] +reth-executor = { path = "../executor" } tokio = { version = "*", features = ["rt", "sync", "macros"] } tempfile = "3.3.0" assert_matches = "1.5.0" diff --git a/crates/stages/src/lib.rs b/crates/stages/src/lib.rs index 8e5e75ec4bf..562d61665ab 100644 --- a/crates/stages/src/lib.rs +++ b/crates/stages/src/lib.rs @@ -27,7 +27,8 @@ //! # use reth_interfaces::consensus::Consensus; //! # use reth_interfaces::sync::NoopSyncStateUpdate; //! # use reth_interfaces::test_utils::{TestBodiesClient, TestConsensus, TestHeadersClient, TestStatusUpdater}; -//! # use reth_primitives::PeerId; +//! # use reth_executor::Factory; +//! # use reth_primitives::{PeerId,MAINNET}; //! # use reth_stages::Pipeline; //! # use reth_stages::sets::DefaultStages; //! # let consensus: Arc = Arc::new(TestConsensus::default()); @@ -40,12 +41,13 @@ //! # consensus.clone(), //! # create_test_rw_db() //! # ); +//! # let factory = Factory::new(Arc::new(MAINNET.clone())); //! # let (status_updater, _) = TestStatusUpdater::new(); //! // Create a pipeline that can fully sync //! # let pipeline: Pipeline, NoopSyncStateUpdate> = //! Pipeline::builder() //! .add_stages( -//! DefaultStages::new(consensus, headers_downloader, bodies_downloader, status_updater) +//! DefaultStages::new(consensus, headers_downloader, bodies_downloader, status_updater, factory) //! ) //! .build(); //! ``` @@ -55,11 +57,6 @@ mod pipeline; mod stage; mod util; -/// The real database type we use in Reth using MDBX. -pub type DefaultDB<'a> = LatestStateProviderRef<'a, 'a, Tx<'a, RW, WriteMap>>; -use reth_db::mdbx::{tx::Tx, WriteMap, RW}; -use reth_provider::LatestStateProviderRef; - #[allow(missing_docs)] #[cfg(any(test, feature = "test-utils"))] pub mod test_utils; diff --git a/crates/stages/src/sets.rs b/crates/stages/src/sets.rs index dd86ef121ca..54332e2f61f 100644 --- a/crates/stages/src/sets.rs +++ b/crates/stages/src/sets.rs @@ -14,18 +14,27 @@ //! # use reth_interfaces::sync::NoopSyncStateUpdate; //! # use reth_stages::Pipeline; //! # use reth_stages::sets::{OfflineStages}; +//! # use reth_executor::Factory; +//! # use reth_primitives::MAINNET; +//! # use std::sync::Arc; +//! +//! # let factory = Factory::new(Arc::new(MAINNET.clone())); //! // Build a pipeline with all offline stages. //! # let pipeline: Pipeline, NoopSyncStateUpdate> = -//! Pipeline::builder().add_stages(OfflineStages::default()).build(); +//! Pipeline::builder().add_stages(OfflineStages::new(factory)).build(); //! ``` //! //! ```ignore //! # use reth_stages::Pipeline; //! # use reth_stages::{StageSet, sets::OfflineStages}; +//! # use reth_executor::Factory; +//! # use reth_primitives::MAINNET; +//! # use std::sync::Arc; //! // Build a pipeline with all offline stages and a custom stage at the end. +//! # let factory = Factory::new(Arc::new(MAINNET.clone())); //! Pipeline::builder() //! .add_stages( -//! OfflineStages::default().builder().add_stage(MyCustomStage) +//! OfflineStages::new(factory).builder().add_stage(MyCustomStage) //! ) //! .build(); //! ``` @@ -45,7 +54,7 @@ use reth_interfaces::{ headers::{client::StatusUpdater, downloader::HeaderDownloader}, }, }; -use reth_primitives::ChainSpec; +use reth_provider::ExecutorFactory; use std::sync::Arc; /// A set containing all stages to run a fully syncing instance of reth. @@ -56,39 +65,47 @@ use std::sync::Arc; /// - [`OfflineStages`] /// - [`FinishStage`] #[derive(Debug)] -pub struct DefaultStages { +pub struct DefaultStages { /// Configuration for the online stages online: OnlineStages, + /// Executor factory needs for execution stage + executor_factory: EF, /// Configuration for the [`FinishStage`] stage. status_updater: S, } -impl DefaultStages { +impl DefaultStages { /// Create a new set of default stages with default values. pub fn new( consensus: Arc, header_downloader: H, body_downloader: B, status_updater: S, - ) -> Self { + executor_factory: EF, + ) -> Self + where + EF: ExecutorFactory, + { Self { online: OnlineStages::new(consensus, header_downloader, body_downloader), + executor_factory, status_updater, } } } -impl StageSet for DefaultStages +impl StageSet for DefaultStages where DB: Database, H: HeaderDownloader + 'static, B: BodyDownloader + 'static, S: StatusUpdater + 'static, + EF: ExecutorFactory, { fn builder(self) -> StageSetBuilder { self.online .builder() - .add_set(OfflineStages) + .add_set(OfflineStages::new(self.executor_factory)) .add_stage(FinishStage::new(self.status_updater)) } } @@ -137,40 +154,47 @@ where /// - [`HistoryIndexingStages`] #[derive(Debug, Default)] #[non_exhaustive] -pub struct OfflineStages; +pub struct OfflineStages { + /// Executor factory needs for execution stage + pub executor_factory: EF, +} + +impl OfflineStages { + /// Create a new set of ofline stages with default values. + pub fn new(executor_factory: EF) -> Self { + Self { executor_factory } + } +} -impl StageSet for OfflineStages { +impl StageSet for OfflineStages { fn builder(self) -> StageSetBuilder { - ExecutionStages::default().builder().add_set(HashingStages).add_set(HistoryIndexingStages) + ExecutionStages::new(self.executor_factory) + .builder() + .add_set(HashingStages) + .add_set(HistoryIndexingStages) } } /// A set containing all stages that are required to execute pre-existing block data. #[derive(Debug)] #[non_exhaustive] -pub struct ExecutionStages { - /// The chain specification to use for execution. - chain_spec: ChainSpec, -} - -impl Default for ExecutionStages { - fn default() -> Self { - Self { chain_spec: reth_primitives::MAINNET.clone() } - } +pub struct ExecutionStages { + /// Executor factory that will create executors. + executor_factory: EF, } -impl ExecutionStages { +impl ExecutionStages { /// Create a new set of execution stages with default values. - pub fn new(chain_spec: ChainSpec) -> Self { - Self { chain_spec } + pub fn new(executor_factory: EF) -> Self { + Self { executor_factory } } } -impl StageSet for ExecutionStages { +impl StageSet for ExecutionStages { fn builder(self) -> StageSetBuilder { StageSetBuilder::default() .add_stage(SenderRecoveryStage::default()) - .add_stage(ExecutionStage::>::from(self.chain_spec)) + .add_stage(ExecutionStage::new(self.executor_factory, 10_000)) } } diff --git a/crates/stages/src/stages/execution.rs b/crates/stages/src/stages/execution.rs index 633bfec4adf..4312a3ca059 100644 --- a/crates/stages/src/stages/execution.rs +++ b/crates/stages/src/stages/execution.rs @@ -1,6 +1,6 @@ use crate::{ - exec_or_return, DefaultDB, ExecAction, ExecInput, ExecOutput, Stage, StageError, StageId, - UnwindInput, UnwindOutput, + exec_or_return, ExecAction, ExecInput, ExecOutput, Stage, StageError, StageId, UnwindInput, + UnwindOutput, }; use reth_db::{ cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO}, @@ -9,11 +9,9 @@ use reth_db::{ tables, transaction::{DbTx, DbTxMut}, }; -use reth_executor::executor::Executor; use reth_interfaces::provider::ProviderError; -use reth_primitives::{Address, Block, ChainSpec, U256}; -use reth_provider::{LatestStateProviderRef, StateProvider, Transaction}; -use reth_revm::database::{State, SubState}; +use reth_primitives::{Address, Block, U256}; +use reth_provider::{BlockExecutor, ExecutorFactory, LatestStateProviderRef, Transaction}; use tracing::*; /// The [`StageId`] of the execution stage. @@ -49,33 +47,28 @@ pub const EXECUTION: StageId = StageId("Execution"); /// to [tables::PlainStorageState] // false positive, we cannot derive it if !DB: Debug. #[allow(missing_debug_implementations)] -pub struct ExecutionStage<'a, DB = DefaultDB<'a>> -where - DB: StateProvider, -{ +pub struct ExecutionStage { /// The stage's internal executor - pub executor: Executor<'a, DB>, + pub executor_factory: EF, /// Commit threshold pub commit_threshold: u64, } -impl<'a, DB: StateProvider> From> for ExecutionStage<'a, DB> { - fn from(executor: Executor<'a, DB>) -> Self { - Self { executor, commit_threshold: 1_000 } +impl ExecutionStage { + /// Create new execution stage with specified config. + pub fn new(executor_factory: EF, commit_threshold: u64) -> Self { + Self { executor_factory, commit_threshold } } -} -impl<'a, DB: StateProvider> From for ExecutionStage<'a, DB> { - fn from(chain_spec: ChainSpec) -> Self { - let executor = Executor::from(chain_spec); - Self::from(executor) + /// Create execution stage with executor factory and default commit threshold set to 10_000 + /// blocks + pub fn new_default_threshold(executor_factory: EF) -> Self { + Self { executor_factory, commit_threshold: 10_000 } } -} -impl<'a, S: StateProvider> ExecutionStage<'a, S> { /// Execute the stage. pub fn execute_inner( - &mut self, + &self, tx: &mut Transaction<'_, DB>, input: ExecInput, ) -> Result { @@ -116,7 +109,7 @@ impl<'a, S: StateProvider> ExecutionStage<'a, S> { // Create state provider with cached state - let mut state_provider = SubState::new(State::new(LatestStateProviderRef::new(&**tx))); + let mut executor = self.executor_factory.with_sp(LatestStateProviderRef::new(&**tx)); // Fetch transactions, execute them and generate results let mut changesets = Vec::with_capacity(block_batch.len()); @@ -154,7 +147,6 @@ impl<'a, S: StateProvider> ExecutionStage<'a, S> { trace!(target: "sync::stages::execution", number = block_number, txs = transactions.len(), "Executing block"); // Configure the executor to use the current state. - let mut executor = self.executor.with_db(&mut state_provider); let changeset = executor .execute_and_verify_receipt( &Block { header, body: transactions, ommers, withdrawals }, @@ -166,7 +158,7 @@ impl<'a, S: StateProvider> ExecutionStage<'a, S> { } // put execution results to database - tx.insert_execution_result(changesets, &self.executor.chain_spec, last_block)?; + tx.insert_execution_result(changesets, self.executor_factory.chain_spec(), last_block)?; let done = !capped; info!(target: "sync::stages::execution", stage_progress = end_block, done, "Sync iteration finished"); @@ -174,15 +166,8 @@ impl<'a, S: StateProvider> ExecutionStage<'a, S> { } } -impl<'a, DB: StateProvider> ExecutionStage<'a, DB> { - /// Create new execution stage with specified config. - pub fn new(executor: Executor<'a, DB>, commit_threshold: u64) -> Self { - Self { executor, commit_threshold } - } -} - #[async_trait::async_trait] -impl Stage for ExecutionStage<'_, State> { +impl Stage for ExecutionStage { /// Return the id of the stage fn id(&self) -> StageId { EXECUTION @@ -306,13 +291,23 @@ mod tests { mdbx::{test_utils::create_test_db, EnvKind, WriteMap}, models::AccountBeforeTx, }; + use reth_executor::Factory; use reth_primitives::{ hex_literal::hex, keccak256, Account, ChainSpecBuilder, SealedBlock, StorageEntry, H160, - H256, MAINNET, U256, + H256, U256, }; use reth_provider::insert_canonical_block; use reth_rlp::Decodable; - use std::ops::{Deref, DerefMut}; + use std::{ + ops::{Deref, DerefMut}, + sync::Arc, + }; + + fn stage() -> ExecutionStage { + let factory = + Factory::new(Arc::new(ChainSpecBuilder::mainnet().berlin_activated().build())); + ExecutionStage::new(factory, 100) + } #[tokio::test] async fn sanity_execution_of_block() { @@ -355,8 +350,7 @@ mod tests { db_tx.put::(code_hash, code.to_vec()).unwrap(); tx.commit().unwrap(); - let chain_spec = ChainSpecBuilder::mainnet().berlin_activated().build(); - let mut execution_stage = ExecutionStage::>::from(chain_spec); + let mut execution_stage = stage(); let output = execution_stage.execute(&mut tx, input).await.unwrap(); tx.commit().unwrap(); assert_eq!(output, ExecOutput { stage_progress: 1, done: true }); @@ -440,12 +434,12 @@ mod tests { tx.commit().unwrap(); // execute - let chain_spec = ChainSpecBuilder::mainnet().berlin_activated().build(); - let mut execution_stage = ExecutionStage::>::from(chain_spec); + let mut execution_stage = stage(); let _ = execution_stage.execute(&mut tx, input).await.unwrap(); tx.commit().unwrap(); - let o = ExecutionStage::>::from(MAINNET.clone()) + let mut stage = stage(); + let o = stage .unwind(&mut tx, UnwindInput { stage_progress: 1, unwind_to: 0, bad_block: None }) .await .unwrap(); @@ -526,8 +520,7 @@ mod tests { tx.commit().unwrap(); // execute - let chain_spec = ChainSpecBuilder::mainnet().berlin_activated().build(); - let mut execution_stage = ExecutionStage::>::from(chain_spec); + let mut execution_stage = stage(); let _ = execution_stage.execute(&mut tx, input).await.unwrap(); tx.commit().unwrap(); diff --git a/crates/storage/db/benches/utils.rs b/crates/storage/db/benches/utils.rs index 29561dfe629..8f159ec1daa 100644 --- a/crates/storage/db/benches/utils.rs +++ b/crates/storage/db/benches/utils.rs @@ -47,6 +47,7 @@ where /// Sets up a clear database at `bench_db_path`. #[allow(clippy::ptr_arg)] +#[allow(unused)] fn set_up_db( bench_db_path: &Path, pair: &Vec<(::Key, bytes::Bytes, ::Value, bytes::Bytes)>, diff --git a/crates/storage/provider/src/lib.rs b/crates/storage/provider/src/lib.rs index c9f2496dc3d..3e08932ec6e 100644 --- a/crates/storage/provider/src/lib.rs +++ b/crates/storage/provider/src/lib.rs @@ -11,8 +11,9 @@ /// Various provider traits. mod traits; pub use traits::{ - AccountProvider, BlockHashProvider, BlockIdProvider, BlockProvider, EvmEnvProvider, - HeaderProvider, StateProvider, StateProviderFactory, TransactionsProvider, WithdrawalsProvider, + AccountProvider, BlockExecutor, BlockHashProvider, BlockIdProvider, BlockProvider, + EvmEnvProvider, ExecutorFactory, HeaderProvider, StateProvider, StateProviderFactory, + TransactionsProvider, WithdrawalsProvider, }; /// Provider trait implementations. diff --git a/crates/storage/provider/src/traits/executor.rs b/crates/storage/provider/src/traits/executor.rs new file mode 100644 index 00000000000..98150d506f8 --- /dev/null +++ b/crates/storage/provider/src/traits/executor.rs @@ -0,0 +1,45 @@ +//! Executor Factory + +use crate::{execution_result::ExecutionResult, StateProvider}; +use reth_interfaces::executor::Error; +use reth_primitives::{Address, Block, ChainSpec, U256}; + +/// Executor factory that would create the EVM with particular state provider. +/// +/// It can be used to mock executor. +pub trait ExecutorFactory: Send + Sync + 'static { + /// The executor produced by the factory + type Executor: BlockExecutor; + + /// Executor with [`StateProvider`] + fn with_sp(&self, sp: SP) -> Self::Executor; + + /// Return internal chainspec + fn chain_spec(&self) -> &ChainSpec; +} + +/// An executor capable of executing a block. +pub trait BlockExecutor { + /// Execute a block. + /// + /// The number of `senders` should be equal to the number of transactions in the block. + /// + /// If no senders are specified, the `execute` function MUST recover the senders for the + /// provided block's transactions internally. We use this to allow for calculating senders in + /// parallel in e.g. staged sync, so that execution can happen without paying for sender + /// recovery costs. + fn execute( + &mut self, + block: &Block, + total_difficulty: U256, + senders: Option>, + ) -> Result; + + /// Executes the block and checks receipts + fn execute_and_verify_receipt( + &mut self, + block: &Block, + total_difficulty: U256, + senders: Option>, + ) -> Result; +} diff --git a/crates/storage/provider/src/traits/mod.rs b/crates/storage/provider/src/traits/mod.rs index 7552ca7f638..12d9adf4a8b 100644 --- a/crates/storage/provider/src/traits/mod.rs +++ b/crates/storage/provider/src/traits/mod.rs @@ -26,3 +26,6 @@ pub use transactions::TransactionsProvider; mod withdrawals; pub use withdrawals::WithdrawalsProvider; + +mod executor; +pub use executor::{BlockExecutor, ExecutorFactory}; From b449ac06dd1612e52f4c1f5e54212724a3dbd4f8 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 8 Mar 2023 16:11:55 +0100 Subject: [PATCH 067/191] chore: add from u256 impl to JsonU256 (#1673) --- crates/primitives/src/serde_helper/jsonu256.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/primitives/src/serde_helper/jsonu256.rs b/crates/primitives/src/serde_helper/jsonu256.rs index 58c71580615..3cf187f1f37 100644 --- a/crates/primitives/src/serde_helper/jsonu256.rs +++ b/crates/primitives/src/serde_helper/jsonu256.rs @@ -15,6 +15,12 @@ impl From for U256 { } } +impl From for JsonU256 { + fn from(value: U256) -> Self { + JsonU256(value) + } +} + impl Serialize for JsonU256 { fn serialize(&self, serializer: S) -> Result where From 161de9aadf64aea1af0b107662dd79cc9a2dcc1c Mon Sep 17 00:00:00 2001 From: Bjerg Date: Wed, 8 Mar 2023 16:25:14 +0100 Subject: [PATCH 068/191] feat: persist contract analysis in db (#1640) --- Cargo.lock | 8 +- bin/reth/src/test_eth_chain/runner.rs | 6 +- crates/executor/src/executor.rs | 10 +- crates/primitives/src/account.rs | 118 +++++++++++++++++- crates/primitives/src/lib.rs | 3 +- crates/revm/revm-primitives/src/env.rs | 2 +- crates/revm/revm-primitives/src/lib.rs | 1 - crates/revm/src/database.rs | 12 +- crates/rpc/rpc/src/eth/api/state.rs | 2 +- crates/stages/src/stages/execution.rs | 10 +- .../storage/db/src/tables/codecs/compact.rs | 3 +- crates/storage/db/src/tables/mod.rs | 5 +- .../src/providers/state/historical.rs | 6 +- .../provider/src/providers/state/latest.rs | 6 +- .../provider/src/providers/state/macros.rs | 2 +- .../storage/provider/src/test_utils/mock.rs | 11 +- .../storage/provider/src/test_utils/noop.rs | 6 +- crates/storage/provider/src/traits/state.rs | 6 +- crates/storage/provider/src/transaction.rs | 11 +- 19 files changed, 165 insertions(+), 63 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2389ae8b758..4447e5a43c1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5161,7 +5161,7 @@ dependencies = [ [[package]] name = "revm" version = "3.0.0" -source = "git+https://github.com/bluealloy/revm#7bb73da23d0278829f9669f175a6eede247c0538" +source = "git+https://github.com/bluealloy/revm#f2656b7eac6fab1cb872268bb7d0dda4b5d3aa85" dependencies = [ "auto_impl", "revm-interpreter", @@ -5171,7 +5171,7 @@ dependencies = [ [[package]] name = "revm-interpreter" version = "1.0.0" -source = "git+https://github.com/bluealloy/revm#7bb73da23d0278829f9669f175a6eede247c0538" +source = "git+https://github.com/bluealloy/revm#f2656b7eac6fab1cb872268bb7d0dda4b5d3aa85" dependencies = [ "bitvec 1.0.1", "derive_more", @@ -5183,7 +5183,7 @@ dependencies = [ [[package]] name = "revm-precompile" version = "2.0.0" -source = "git+https://github.com/bluealloy/revm#7bb73da23d0278829f9669f175a6eede247c0538" +source = "git+https://github.com/bluealloy/revm#f2656b7eac6fab1cb872268bb7d0dda4b5d3aa85" dependencies = [ "k256", "num", @@ -5199,7 +5199,7 @@ dependencies = [ [[package]] name = "revm-primitives" version = "1.0.0" -source = "git+https://github.com/bluealloy/revm#7bb73da23d0278829f9669f175a6eede247c0538" +source = "git+https://github.com/bluealloy/revm#f2656b7eac6fab1cb872268bb7d0dda4b5d3aa85" dependencies = [ "arbitrary", "auto_impl", diff --git a/bin/reth/src/test_eth_chain/runner.rs b/bin/reth/src/test_eth_chain/runner.rs index cae43ba4ef3..61025f6fa6e 100644 --- a/bin/reth/src/test_eth_chain/runner.rs +++ b/bin/reth/src/test_eth_chain/runner.rs @@ -10,8 +10,8 @@ use reth_db::{ Error as DbError, }; use reth_primitives::{ - keccak256, Account as RethAccount, Address, ChainSpec, ForkCondition, Hardfork, JsonU256, - SealedBlock, SealedHeader, StorageEntry, H256, U256, + keccak256, Account as RethAccount, Address, Bytecode, ChainSpec, ForkCondition, Hardfork, + JsonU256, SealedBlock, SealedHeader, StorageEntry, H256, U256, }; use reth_provider::Transaction; use reth_rlp::Decodable; @@ -162,7 +162,7 @@ pub async fn run_test(path: PathBuf) -> eyre::Result { }, )?; if let Some(code_hash) = code_hash { - tx.put::(code_hash, account.code.to_vec())?; + tx.put::(code_hash, Bytecode::new_raw(account.code.0))?; } account.storage.iter().try_for_each(|(k, v)| { trace!(target: "reth::cli", ?address, key = ?k.0, value = ?v.0, "Update storage"); diff --git a/crates/executor/src/executor.rs b/crates/executor/src/executor.rs index 03d28ee9c91..9f9ed262d2e 100644 --- a/crates/executor/src/executor.rs +++ b/crates/executor/src/executor.rs @@ -533,8 +533,8 @@ pub fn verify_receipt<'a>( mod tests { use super::*; use reth_primitives::{ - hex_literal::hex, keccak256, Account, Address, Bytes, ChainSpecBuilder, ForkCondition, - StorageKey, H256, MAINNET, U256, + hex_literal::hex, keccak256, Account, Address, Bytecode, Bytes, ChainSpecBuilder, + ForkCondition, StorageKey, H256, MAINNET, U256, }; use reth_provider::{AccountProvider, BlockHashProvider, StateProvider}; use reth_revm::database::State; @@ -544,7 +544,7 @@ mod tests { #[derive(Debug, Default, Clone, Eq, PartialEq)] struct StateProviderTest { accounts: HashMap, Account)>, - contracts: HashMap, + contracts: HashMap, block_hash: HashMap, } @@ -560,7 +560,7 @@ mod tests { if let Some(bytecode) = bytecode { let hash = keccak256(&bytecode); account.bytecode_hash = Some(hash); - self.contracts.insert(hash, bytecode); + self.contracts.insert(hash, Bytecode::new_raw(bytecode.into())); } self.accounts.insert(address, (storage, account)); } @@ -591,7 +591,7 @@ mod tests { .and_then(|(storage, _)| storage.get(&storage_key).cloned())) } - fn bytecode_by_hash(&self, code_hash: H256) -> reth_interfaces::Result> { + fn bytecode_by_hash(&self, code_hash: H256) -> reth_interfaces::Result> { Ok(self.contracts.get(&code_hash).cloned()) } } diff --git a/crates/primitives/src/account.rs b/crates/primitives/src/account.rs index eb77388a3f0..e822bc449e1 100644 --- a/crates/primitives/src/account.rs +++ b/crates/primitives/src/account.rs @@ -1,5 +1,10 @@ use crate::{H256, KECCAK_EMPTY, U256}; +use bytes::{Buf, BufMut, Bytes}; +use fixed_hash::byteorder::{BigEndian, ReadBytesExt}; use reth_codecs::{main_codec, Compact}; +use revm_primitives::{Bytecode as RevmBytecode, BytecodeState, JumpMap}; +use serde::{Deserialize, Serialize}; +use std::ops::Deref; /// An Ethereum account. #[main_codec] @@ -31,10 +36,97 @@ impl Account { } } +/// Bytecode for an account. +/// +/// A wrapper around [`revm::primitives::Bytecode`][RevmBytecode] with encoding/decoding support. +/// +/// Note: Upon decoding bytecode from the database, you *should* set the code hash using +/// [`Self::with_code_hash`]. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct Bytecode(pub RevmBytecode); + +impl Bytecode { + /// Create new bytecode from raw bytes. + /// + /// No analysis will be performed. + pub fn new_raw(bytes: Bytes) -> Self { + Self(RevmBytecode::new_raw(bytes)) + } + + /// Set the hash of the inner bytecode. + pub fn with_code_hash(mut self, code_hash: H256) -> Self { + self.0.hash = code_hash; + self + } +} + +impl Deref for Bytecode { + type Target = RevmBytecode; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl Compact for Bytecode { + fn to_compact(self, buf: &mut impl BufMut) -> usize { + buf.put_u32(self.0.bytecode.len() as u32); + buf.put_slice(self.0.bytecode.as_ref()); + let len = match self.0.state() { + BytecodeState::Raw => { + buf.put_u8(0); + 1 + } + BytecodeState::Checked { len } => { + buf.put_u8(1); + buf.put_u64(*len as u64); + 9 + } + BytecodeState::Analysed { len, jump_map } => { + buf.put_u8(2); + buf.put_u64(*len as u64); + let map = jump_map.as_slice(); + buf.put_slice(map); + 9 + map.len() + } + }; + len + self.0.bytecode.len() + 4 + } + + fn from_compact(mut buf: &[u8], _: usize) -> (Self, &[u8]) + where + Self: Sized, + { + let len = buf.read_u32::().expect("could not read bytecode length"); + let bytes = buf.copy_to_bytes(len as usize); + let variant = buf.read_u8().expect("could not read bytecode variant"); + let decoded = match variant { + 0 => Bytecode(RevmBytecode::new_raw(bytes)), + 1 => Bytecode(unsafe { + RevmBytecode::new_checked( + bytes, + buf.read_u64::().unwrap() as usize, + None, + ) + }), + 2 => Bytecode(RevmBytecode { + bytecode: bytes, + hash: KECCAK_EMPTY, + state: BytecodeState::Analysed { + len: buf.read_u64::().unwrap() as usize, + jump_map: JumpMap::from_slice(buf), + }, + }), + _ => unreachable!(), + }; + (decoded, &[]) + } +} + #[cfg(test)] mod tests { - use crate::{Account, U256}; - use reth_codecs::Compact; + use super::*; + use hex_literal::hex; #[test] fn test_account() { @@ -51,4 +143,26 @@ mod tests { let len = acc.to_compact(&mut buf); assert_eq!(len, 4); } + + #[test] + fn test_bytecode() { + let mut buf = vec![]; + let mut bytecode = Bytecode(RevmBytecode::new_raw(Bytes::default())); + let len = bytecode.clone().to_compact(&mut buf); + assert_eq!(len, 5); + + let mut buf = vec![]; + bytecode.0.bytecode = Bytes::from(hex!("ffff").as_ref()); + let len = bytecode.clone().to_compact(&mut buf); + assert_eq!(len, 7); + + let mut buf = vec![]; + bytecode.0.state = BytecodeState::Analysed { len: 2, jump_map: JumpMap::from_slice(&[0]) }; + let len = bytecode.clone().to_compact(&mut buf); + assert_eq!(len, 16); + + let (decoded, remainder) = Bytecode::from_compact(&buf, len); + assert_eq!(decoded, bytecode); + assert!(remainder.is_empty()); + } } diff --git a/crates/primitives/src/lib.rs b/crates/primitives/src/lib.rs index 52f349c3af6..59f969e184a 100644 --- a/crates/primitives/src/lib.rs +++ b/crates/primitives/src/lib.rs @@ -35,7 +35,7 @@ mod withdrawal; /// Helper function for calculating Merkle proofs and hashes pub mod proofs; -pub use account::Account; +pub use account::{Account, Bytecode}; pub use bits::H512; pub use block::{Block, BlockHashOrNumber, BlockId, BlockNumberOrTag, SealedBlock}; pub use bloom::Bloom; @@ -56,6 +56,7 @@ pub use log::Log; pub use net::NodeRecord; pub use peer::{PeerId, WithPeerId}; pub use receipt::Receipt; +pub use revm_primitives::JumpMap; pub use serde_helper::JsonU256; pub use storage::{StorageEntry, StorageTrieEntry}; pub use transaction::{ diff --git a/crates/revm/revm-primitives/src/env.rs b/crates/revm/revm-primitives/src/env.rs index 470684a7603..84465494eff 100644 --- a/crates/revm/revm-primitives/src/env.rs +++ b/crates/revm/revm-primitives/src/env.rs @@ -39,7 +39,7 @@ pub fn fill_cfg_env( cfg_env.chain_id = U256::from(chain_spec.chain().id()); cfg_env.spec_id = spec_id; cfg_env.perf_all_precompiles_have_balance = false; - cfg_env.perf_analyse_created_bytecodes = AnalysisKind::Raw; + cfg_env.perf_analyse_created_bytecodes = AnalysisKind::Analyse; } /// Fill block environment from Block. pub fn fill_block_env(block_env: &mut BlockEnv, header: &Header, after_merge: bool) { diff --git a/crates/revm/revm-primitives/src/lib.rs b/crates/revm/revm-primitives/src/lib.rs index a989c0b65e3..80537049460 100644 --- a/crates/revm/revm-primitives/src/lib.rs +++ b/crates/revm/revm-primitives/src/lib.rs @@ -6,7 +6,6 @@ ))] //! revm utils and implementations specific to reth. - pub mod config; /// Helpers for configuring revm [Env](revm::primitives::Env) diff --git a/crates/revm/src/database.rs b/crates/revm/src/database.rs index c90a7b111b0..d18d872f2cc 100644 --- a/crates/revm/src/database.rs +++ b/crates/revm/src/database.rs @@ -49,19 +49,9 @@ impl DatabaseRef for State { fn code_by_hash(&self, code_hash: H256) -> Result { let bytecode = self.0.bytecode_by_hash(code_hash)?; - // SAFETY: We are requesting the code by its hash, so it is almost tautological why this - // would be safe. If the bytecode is not found, we return an empty bytecode with the - // appropriate hash. - // - // In an ideal world we would return analysed bytecode here, but analysed bytecode in revm - // depends on the current active hard fork, since it calculates gas blocks... if let Some(bytecode) = bytecode { - Ok(unsafe { Bytecode::new_raw_with_hash(bytecode.0, code_hash) }) + Ok(bytecode.with_code_hash(code_hash).0) } else { - // NOTE(onbjerg): This corresponds to an empty analysed bytecode with a hash of - // `KECCAK_EMPTY`. In the case where the bytecode is not found, we would - // return empty bytes anyway: this simply skips the hashing and analysis steps, which - // would otherwise be present if we simply did an `.unwrap_or_default()` above. Ok(Bytecode::new()) } } diff --git a/crates/rpc/rpc/src/eth/api/state.rs b/crates/rpc/rpc/src/eth/api/state.rs index 935615ea4d2..fce654d13c2 100644 --- a/crates/rpc/rpc/src/eth/api/state.rs +++ b/crates/rpc/rpc/src/eth/api/state.rs @@ -15,7 +15,7 @@ where let state = self.state_at_block_id_or_latest(block_id)?.ok_or(EthApiError::UnknownBlockNumber)?; let code = state.account_code(address)?.unwrap_or_default(); - Ok(code) + Ok(code.original_bytes().into()) } pub(crate) fn balance(&self, address: Address, block_id: Option) -> EthResult { diff --git a/crates/stages/src/stages/execution.rs b/crates/stages/src/stages/execution.rs index 4312a3ca059..0f28ce87b42 100644 --- a/crates/stages/src/stages/execution.rs +++ b/crates/stages/src/stages/execution.rs @@ -293,8 +293,8 @@ mod tests { }; use reth_executor::Factory; use reth_primitives::{ - hex_literal::hex, keccak256, Account, ChainSpecBuilder, SealedBlock, StorageEntry, H160, - H256, U256, + hex_literal::hex, keccak256, Account, Bytecode, ChainSpecBuilder, SealedBlock, + StorageEntry, H160, H256, U256, }; use reth_provider::insert_canonical_block; use reth_rlp::Decodable; @@ -347,7 +347,7 @@ mod tests { Account { nonce: 0, balance, bytecode_hash: None }, ) .unwrap(); - db_tx.put::(code_hash, code.to_vec()).unwrap(); + db_tx.put::(code_hash, Bytecode::new_raw(code.to_vec().into())).unwrap(); tx.commit().unwrap(); let mut execution_stage = stage(); @@ -430,7 +430,7 @@ mod tests { db_tx.put::(acc1, acc1_info).unwrap(); db_tx.put::(acc2, acc2_info).unwrap(); - db_tx.put::(code_hash, code.to_vec()).unwrap(); + db_tx.put::(code_hash, Bytecode::new_raw(code.to_vec().into())).unwrap(); tx.commit().unwrap(); // execute @@ -502,7 +502,7 @@ mod tests { // set account db_tx.put::(caller_address, caller_info).unwrap(); db_tx.put::(destroyed_address, destroyed_info).unwrap(); - db_tx.put::(code_hash, code.to_vec()).unwrap(); + db_tx.put::(code_hash, Bytecode::new_raw(code.to_vec().into())).unwrap(); // set storage to check when account gets destroyed. db_tx .put::( diff --git a/crates/storage/db/src/tables/codecs/compact.rs b/crates/storage/db/src/tables/codecs/compact.rs index a43fe7d76e9..4c0a4136416 100644 --- a/crates/storage/db/src/tables/codecs/compact.rs +++ b/crates/storage/db/src/tables/codecs/compact.rs @@ -43,7 +43,8 @@ impl_compression_for_compact!( StorageTrieEntry, StoredBlockBody, StoredBlockOmmers, - StoredBlockWithdrawals + StoredBlockWithdrawals, + Bytecode ); impl_compression_for_compact!(AccountBeforeTx, TransactionSigned); impl_compression_for_compact!(CompactU256); diff --git a/crates/storage/db/src/tables/mod.rs b/crates/storage/db/src/tables/mod.rs index 43904cf483f..16fb6ac49b4 100644 --- a/crates/storage/db/src/tables/mod.rs +++ b/crates/storage/db/src/tables/mod.rs @@ -18,7 +18,7 @@ use crate::{ }, }; use reth_primitives::{ - Account, Address, BlockHash, BlockNumber, Header, IntegerList, Receipt, StorageEntry, + Account, Address, BlockHash, BlockNumber, Bytecode, Header, IntegerList, Receipt, StorageEntry, StorageTrieEntry, TransactionSigned, TransitionId, TxHash, TxNumber, H256, }; @@ -304,6 +304,3 @@ pub type StageId = Vec; // // TODO: Temporary types, until they're properly defined alongside with the Encode and Decode Trait // - -/// Temporary placeholder type for DB. -pub type Bytecode = Vec; diff --git a/crates/storage/provider/src/providers/state/historical.rs b/crates/storage/provider/src/providers/state/historical.rs index b5339469128..214f0ba2eb8 100644 --- a/crates/storage/provider/src/providers/state/historical.rs +++ b/crates/storage/provider/src/providers/state/historical.rs @@ -10,7 +10,7 @@ use reth_db::{ }; use reth_interfaces::Result; use reth_primitives::{ - Account, Address, Bytes, StorageKey, StorageValue, TransitionId, H256, U256, + Account, Address, Bytecode, StorageKey, StorageValue, TransitionId, H256, U256, }; use std::marker::PhantomData; @@ -116,8 +116,8 @@ impl<'a, 'b, TX: DbTx<'a>> StateProvider for HistoricalStateProviderRef<'a, 'b, } /// Get account code by its hash - fn bytecode_by_hash(&self, code_hash: H256) -> Result> { - self.tx.get::(code_hash).map_err(Into::into).map(|r| r.map(Bytes::from)) + fn bytecode_by_hash(&self, code_hash: H256) -> Result> { + self.tx.get::(code_hash).map_err(Into::into) } } diff --git a/crates/storage/provider/src/providers/state/latest.rs b/crates/storage/provider/src/providers/state/latest.rs index b1e5ea4833b..311f37e415d 100644 --- a/crates/storage/provider/src/providers/state/latest.rs +++ b/crates/storage/provider/src/providers/state/latest.rs @@ -4,7 +4,7 @@ use crate::{ }; use reth_db::{cursor::DbDupCursorRO, tables, transaction::DbTx}; use reth_interfaces::Result; -use reth_primitives::{Account, Address, Bytes, StorageKey, StorageValue, H256, U256}; +use reth_primitives::{Account, Address, Bytecode, StorageKey, StorageValue, H256, U256}; use std::marker::PhantomData; /// State provider over latest state that takes tx reference. @@ -49,8 +49,8 @@ impl<'a, 'b, TX: DbTx<'a>> StateProvider for LatestStateProviderRef<'a, 'b, TX> } /// Get account code by its hash - fn bytecode_by_hash(&self, code_hash: H256) -> Result> { - self.db.get::(code_hash).map_err(Into::into).map(|r| r.map(Bytes::from)) + fn bytecode_by_hash(&self, code_hash: H256) -> Result> { + self.db.get::(code_hash).map_err(Into::into) } } diff --git a/crates/storage/provider/src/providers/state/macros.rs b/crates/storage/provider/src/providers/state/macros.rs index b7d9a775a2f..d488010d713 100644 --- a/crates/storage/provider/src/providers/state/macros.rs +++ b/crates/storage/provider/src/providers/state/macros.rs @@ -38,7 +38,7 @@ macro_rules! delegate_provider_impls { } StateProvider $(where [$($generics)*])?{ fn storage(&self, account: reth_primitives::Address, storage_key: reth_primitives::StorageKey) -> reth_interfaces::Result>; - fn bytecode_by_hash(&self, code_hash: reth_primitives::H256) -> reth_interfaces::Result>; + fn bytecode_by_hash(&self, code_hash: reth_primitives::H256) -> reth_interfaces::Result>; } ); } diff --git a/crates/storage/provider/src/test_utils/mock.rs b/crates/storage/provider/src/test_utils/mock.rs index f6946f95ef9..7d448fd2a9c 100644 --- a/crates/storage/provider/src/test_utils/mock.rs +++ b/crates/storage/provider/src/test_utils/mock.rs @@ -5,8 +5,9 @@ use crate::{ use parking_lot::Mutex; use reth_interfaces::Result; use reth_primitives::{ - keccak256, Account, Address, Block, BlockHash, BlockId, BlockNumber, BlockNumberOrTag, Bytes, - ChainInfo, Header, StorageKey, StorageValue, TransactionSigned, TxHash, H256, U256, + keccak256, Account, Address, Block, BlockHash, BlockId, BlockNumber, BlockNumberOrTag, + Bytecode, Bytes, ChainInfo, Header, StorageKey, StorageValue, TransactionSigned, TxHash, H256, + U256, }; use revm_primitives::{BlockEnv, CfgEnv}; use std::{collections::HashMap, ops::RangeBounds, sync::Arc}; @@ -26,7 +27,7 @@ pub struct MockEthProvider { #[derive(Debug, Clone)] pub struct ExtendedAccount { account: Account, - bytecode: Option, + bytecode: Option, storage: HashMap, } @@ -44,7 +45,7 @@ impl ExtendedAccount { pub fn with_bytecode(mut self, bytecode: Bytes) -> Self { let hash = keccak256(&bytecode); self.account.bytecode_hash = Some(hash); - self.bytecode = Some(bytecode); + self.bytecode = Some(Bytecode::new_raw(bytecode.into())); self } } @@ -226,7 +227,7 @@ impl StateProvider for MockEthProvider { Ok(lock.get(&account).and_then(|account| account.storage.get(&storage_key)).cloned()) } - fn bytecode_by_hash(&self, code_hash: H256) -> Result> { + fn bytecode_by_hash(&self, code_hash: H256) -> Result> { let lock = self.accounts.lock(); Ok(lock.values().find_map(|account| { match (account.account.bytecode_hash.as_ref(), account.bytecode.as_ref()) { diff --git a/crates/storage/provider/src/test_utils/noop.rs b/crates/storage/provider/src/test_utils/noop.rs index 28eba66138d..c9f86b0cafa 100644 --- a/crates/storage/provider/src/test_utils/noop.rs +++ b/crates/storage/provider/src/test_utils/noop.rs @@ -4,8 +4,8 @@ use crate::{ }; use reth_interfaces::Result; use reth_primitives::{ - Account, Address, Block, BlockHash, BlockId, BlockNumber, Bytes, ChainInfo, Header, StorageKey, - StorageValue, TransactionSigned, TxHash, TxNumber, H256, U256, + Account, Address, Block, BlockHash, BlockId, BlockNumber, Bytecode, ChainInfo, Header, + StorageKey, StorageValue, TransactionSigned, TxHash, TxNumber, H256, U256, }; use revm_primitives::{BlockEnv, CfgEnv}; use std::ops::RangeBounds; @@ -96,7 +96,7 @@ impl StateProvider for NoopProvider { Ok(None) } - fn bytecode_by_hash(&self, _code_hash: H256) -> Result> { + fn bytecode_by_hash(&self, _code_hash: H256) -> Result> { Ok(None) } } diff --git a/crates/storage/provider/src/traits/state.rs b/crates/storage/provider/src/traits/state.rs index dd050c63328..f82f2ad5482 100644 --- a/crates/storage/provider/src/traits/state.rs +++ b/crates/storage/provider/src/traits/state.rs @@ -3,7 +3,7 @@ use crate::BlockHashProvider; use auto_impl::auto_impl; use reth_interfaces::Result; use reth_primitives::{ - Address, BlockHash, BlockNumber, Bytes, StorageKey, StorageValue, H256, KECCAK_EMPTY, U256, + Address, BlockHash, BlockNumber, Bytecode, StorageKey, StorageValue, H256, KECCAK_EMPTY, U256, }; /// An abstraction for a type that provides state data. @@ -13,12 +13,12 @@ pub trait StateProvider: BlockHashProvider + AccountProvider + Send + Sync { fn storage(&self, account: Address, storage_key: StorageKey) -> Result>; /// Get account code by its hash - fn bytecode_by_hash(&self, code_hash: H256) -> Result>; + fn bytecode_by_hash(&self, code_hash: H256) -> Result>; /// Get account code by its address. /// /// Returns `None` if the account doesn't exist or account is not a contract - fn account_code(&self, addr: Address) -> Result> { + fn account_code(&self, addr: Address) -> Result> { // Get basic account information // Returns None if acc doesn't exist let acc = match self.basic_account(addr)? { diff --git a/crates/storage/provider/src/transaction.rs b/crates/storage/provider/src/transaction.rs index 5da17221cf6..b3aabfa304f 100644 --- a/crates/storage/provider/src/transaction.rs +++ b/crates/storage/provider/src/transaction.rs @@ -14,8 +14,8 @@ use reth_db::{ }; use reth_interfaces::{db::Error as DbError, provider::ProviderError}; use reth_primitives::{ - keccak256, Account, Address, BlockHash, BlockNumber, ChainSpec, Hardfork, Header, SealedBlock, - StorageEntry, TransitionId, TxNumber, H256, U256, + keccak256, Account, Address, BlockHash, BlockNumber, Bytecode, ChainSpec, Hardfork, Header, + SealedBlock, StorageEntry, TransitionId, TxNumber, H256, U256, }; use reth_tracing::tracing::{info, trace}; use std::{ @@ -710,10 +710,9 @@ where for (hash, bytecode) in result.new_bytecodes.into_iter() { // make different types of bytecode. Checked and maybe even analyzed (needs to // be packed). Currently save only raw bytes. - let bytecode = bytecode.bytes(); - trace!(target: "sync::stages::execution", ?hash, ?bytecode, len = bytecode.len(), "Inserting bytecode"); - self.put::(hash, bytecode[..bytecode.len()].to_vec())?; - + let bytes = bytecode.bytes(); + trace!(target: "sync::stages::execution", ?hash, ?bytes, len = bytes.len(), "Inserting bytecode"); + self.put::(hash, Bytecode(bytecode))?; // NOTE: bytecode bytes are not inserted in change set and can be found in // separate table } From 9e1c341e166f931a561d608a99c7f52e44786606 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Wed, 8 Mar 2023 11:30:50 -0500 Subject: [PATCH 069/191] fix(primitives): write withdrawal hash at genesis (#1654) --- bin/reth/src/chain/init.rs | 5 +-- crates/primitives/src/chain/spec.rs | 52 +++++++++++++++++++++++++++-- crates/primitives/src/constants.rs | 4 +++ crates/primitives/src/genesis.rs | 2 +- 4 files changed, 57 insertions(+), 6 deletions(-) diff --git a/bin/reth/src/chain/init.rs b/bin/reth/src/chain/init.rs index 82d7d1f434c..4c1dbe2fd24 100644 --- a/bin/reth/src/chain/init.rs +++ b/bin/reth/src/chain/init.rs @@ -42,15 +42,16 @@ pub struct InitCommand { impl InitCommand { /// Execute the `init` command pub async fn execute(&self) -> eyre::Result<()> { - info!(target: "reth::cli", "reth import starting"); + info!(target: "reth::cli", "reth init starting"); info!(target: "reth::cli", path = %self.db, "Opening database"); let db = Arc::new(init_db(&self.db)?); info!(target: "reth::cli", "Database opened"); info!(target: "reth::cli", "Writing genesis block"); - init_genesis(db, self.chain.clone())?; + let hash = init_genesis(db, self.chain.clone())?; + info!(target: "reth::cli", hash = ?hash, "Genesis block written"); Ok(()) } } diff --git a/crates/primitives/src/chain/spec.rs b/crates/primitives/src/chain/spec.rs index 98ca08338eb..c04270cdff8 100644 --- a/crates/primitives/src/chain/spec.rs +++ b/crates/primitives/src/chain/spec.rs @@ -1,7 +1,10 @@ use crate::{ - constants::EIP1559_INITIAL_BASE_FEE, forkid::ForkFilterKey, header::Head, - proofs::genesis_state_root, BlockNumber, Chain, ForkFilter, ForkHash, ForkId, Genesis, - GenesisAccount, Hardfork, Header, H160, H256, U256, + constants::{EIP1559_INITIAL_BASE_FEE, EMPTY_WITHDRAWALS}, + forkid::ForkFilterKey, + header::Head, + proofs::genesis_state_root, + BlockNumber, Chain, ForkFilter, ForkHash, ForkId, Genesis, GenesisAccount, Hardfork, Header, + H160, H256, U256, }; use ethers_core::utils::Genesis as EthersGenesis; use hex_literal::hex; @@ -142,6 +145,15 @@ impl ChainSpec { None }; + // If shanghai is activated, initialize the header with an empty withdrawals hash, and + // empty withdrawals list. + let withdrawals_root = + if self.fork(Hardfork::Shanghai).active_at_timestamp(self.genesis.timestamp) { + Some(EMPTY_WITHDRAWALS) + } else { + None + }; + Header { gas_limit: self.genesis.gas_limit, difficulty: self.genesis.difficulty, @@ -152,6 +164,7 @@ impl ChainSpec { mix_hash: self.genesis.mix_hash, beneficiary: self.genesis.coinbase, base_fee_per_gas, + withdrawals_root, ..Default::default() } } @@ -587,7 +600,9 @@ mod tests { AllGenesisFormats, Chain, ChainSpec, ChainSpecBuilder, ForkCondition, ForkHash, ForkId, Genesis, Hardfork, Head, GOERLI, H256, MAINNET, SEPOLIA, U256, }; + use bytes::BytesMut; use ethers_core::types as EtherType; + use reth_rlp::Encodable; fn test_fork_ids(spec: &ChainSpec, cases: &[(Head, ForkId)]) { for (block, expected_id) in cases { let computed_id = spec.fork_id(block); @@ -1025,6 +1040,37 @@ mod tests { chainspec.hardforks.get(&Hardfork::Shanghai).unwrap(), &ForkCondition::Timestamp(0) ); + + // alloc key -> expected rlp mapping + let key_rlp = vec![ + (hex_literal::hex!("658bdf435d810c91414ec09147daa6db62406379"), "f84d8089487a9a304539440000a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"), + (hex_literal::hex!("aa00000000000000000000000000000000000000"), "f8440101a08afc95b7d18a226944b9c2070b6bda1c3a36afcc3730429d47579c94b9fe5850a0ce92c756baff35fa740c3557c1a971fd24d2d35b7c8e067880d50cd86bb0bc99"), + (hex_literal::hex!("bb00000000000000000000000000000000000000"), "f8440102a08afc95b7d18a226944b9c2070b6bda1c3a36afcc3730429d47579c94b9fe5850a0e25a53cbb501cec2976b393719c63d832423dd70a458731a0b64e4847bbca7d2"), + ]; + + for (key, expected_rlp) in key_rlp { + let account = chainspec.genesis.alloc.get(&key.into()).expect("account should exist"); + let mut account_rlp = BytesMut::new(); + account.encode(&mut account_rlp); + assert_eq!(hex::encode(account_rlp), expected_rlp) + } + + assert_eq!(chainspec.genesis_hash, None); + let expected_state_root: H256 = + hex_literal::hex!("078dc6061b1d8eaa8493384b59c9c65ceb917201221d08b80c4de6770b6ec7e7") + .into(); + assert_eq!(chainspec.genesis_header().state_root, expected_state_root); + + let expected_withdrawals_hash: H256 = + hex_literal::hex!("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") + .into(); + assert_eq!(chainspec.genesis_header().withdrawals_root, Some(expected_withdrawals_hash)); + + let expected_hash: H256 = + hex_literal::hex!("1fc027d65f820d3eef441ebeec139ebe09e471cf98516dce7b5643ccb27f418c") + .into(); + let hash = chainspec.genesis_hash(); + assert_eq!(hash, expected_hash); } #[test] diff --git a/crates/primitives/src/constants.rs b/crates/primitives/src/constants.rs index c38819ae35a..3004b8084b5 100644 --- a/crates/primitives/src/constants.rs +++ b/crates/primitives/src/constants.rs @@ -37,3 +37,7 @@ pub const KECCAK_EMPTY: H256 = /// Ommer root of empty list. pub const EMPTY_OMMER_ROOT: H256 = H256(hex!("1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347")); + +/// Withdrawals root of empty withdrawals set. +pub const EMPTY_WITHDRAWALS: H256 = + H256(hex!("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421")); diff --git a/crates/primitives/src/genesis.rs b/crates/primitives/src/genesis.rs index 8ae6bfce357..fde9db63f14 100644 --- a/crates/primitives/src/genesis.rs +++ b/crates/primitives/src/genesis.rs @@ -165,7 +165,7 @@ impl Encodable for GenesisAccount { return EMPTY_ROOT } let storage_values = - storage.iter().filter(|(_k, &v)| v != KECCAK_EMPTY).map(|(&k, v)| { + storage.iter().filter(|(_k, &v)| v != H256::zero()).map(|(&k, v)| { let value = U256::from_be_bytes(**v); (k, encode_fixed_size(&value)) }); From 279ad346f7a668d57feb44debdb2f634eb0f94ee Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Wed, 8 Mar 2023 18:17:28 -0500 Subject: [PATCH 070/191] fix: return zero latestValidHash pre-merge (#1677) --- crates/rpc/rpc-engine-api/src/engine_api.rs | 23 ++++++++++++++++++--- crates/rpc/rpc-types/src/eth/engine.rs | 5 +++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/crates/rpc/rpc-engine-api/src/engine_api.rs b/crates/rpc/rpc-engine-api/src/engine_api.rs index cc91f293d78..9e6d0b922c8 100644 --- a/crates/rpc/rpc-engine-api/src/engine_api.rs +++ b/crates/rpc/rpc-engine-api/src/engine_api.rs @@ -4,7 +4,7 @@ use reth_interfaces::consensus::ForkchoiceState; use reth_primitives::{ proofs::{self, EMPTY_LIST_HASH}, BlockHash, BlockId, BlockNumber, ChainSpec, Hardfork, Header, SealedBlock, TransactionSigned, - H64, U256, + H256, H64, U256, }; use reth_provider::{ BlockExecutor, BlockProvider, EvmEnvProvider, ExecutorFactory, HeaderProvider, @@ -257,6 +257,9 @@ impl EngineApiResult { let block = match self.try_construct_block(payload) { Ok(b) => b, @@ -289,9 +292,22 @@ impl assert_eq!(result, expected_result)); } diff --git a/crates/rpc/rpc-types/src/eth/engine.rs b/crates/rpc/rpc-types/src/eth/engine.rs index 7ef76d02b65..16e67d80344 100644 --- a/crates/rpc/rpc-types/src/eth/engine.rs +++ b/crates/rpc/rpc-types/src/eth/engine.rs @@ -145,6 +145,11 @@ impl PayloadStatus { pub fn from_status(status: PayloadStatusEnum) -> Self { Self { status, latest_valid_hash: None } } + + pub fn with_latest_valid_hash(mut self, latest_valid_hash: H256) -> Self { + self.latest_valid_hash = Some(latest_valid_hash); + self + } } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] From e49401181945730ff4801f02798ed287f5d8e75c Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Thu, 9 Mar 2023 10:08:05 +0000 Subject: [PATCH 071/191] chore(discv4): remove deprecated EF bootnodes (#1680) Co-authored-by: Roman Krasiuk --- crates/net/discv4/src/bootnodes.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/crates/net/discv4/src/bootnodes.rs b/crates/net/discv4/src/bootnodes.rs index 01c02017a92..a4b34b8d09b 100644 --- a/crates/net/discv4/src/bootnodes.rs +++ b/crates/net/discv4/src/bootnodes.rs @@ -5,13 +5,9 @@ use reth_primitives::NodeRecord; /// Ethereum Foundation Go Bootnodes -pub static MAINNET_BOOTNODES : [&str; 8] = [ +pub static MAINNET_BOOTNODES : [&str; 4] = [ "enode://d860a01f9722d78051619d1e2351aba3f43f943f6f00718d1b9baa4101932a1f5011f16bb2b1bb35db20d6fe28fa0bf09636d26a87d31de9ec6203eeedb1f666@18.138.108.67:30303", // bootnode-aws-ap-southeast-1-001 "enode://22a8232c3abc76a16ae9d6c3b164f98775fe226f0917b0ca871128a74a8e9630b458460865bab457221f1d448dd9791d24c4e5d88786180ac185df813a68d4de@3.209.45.79:30303", // bootnode-aws-us-east-1-001 - "enode://8499da03c47d637b20eee24eec3c356c9a2e6148d6fe25ca195c7949ab8ec2c03e3556126b0d7ed644675e78c4318b08691b7b57de10e5f0d40d05b09238fa0a@52.187.207.27:30303", // bootnode-azure-australiaeast-001 - "enode://103858bdb88756c71f15e9b5e09b56dc1be52f0a5021d46301dbbfb7e130029cc9d0d6f73f693bc29b665770fff7da4d34f3c6379fe12721b5d7a0bcb5ca1fc1@191.234.162.198:30303", // bootnode-azure-brazilsouth-001 - "enode://715171f50508aba88aecd1250af392a45a330af91d7b90701c436b618c86aaa1589c9184561907bebbb56439b8f8787bc01f49a7c77276c58c1b09822d75e8e8@52.231.165.108:30303", // bootnode-azure-koreasouth-001 - "enode://5d6d7cd20d6da4bb83a1d28cadb5d409b64edf314c0335df658c1a54e32c7c4a7ab7823d57c39b6a757556e68ff1df17c748b698544a55cb488b52479a92b60f@104.42.217.25:30303", // bootnode-azure-westus-001 "enode://2b252ab6a1d0f971d9722cb839a42cb81db019ba44c08754628ab4a823487071b5695317c8ccd085219c3a03af063495b2f1da8d18218da2d6a82981b45e6ffc@65.108.70.101:30303", // bootnode-hetzner-hel "enode://4aeb4ab6c14b23e2c4cfdce879c04b0748a20d8e9b59e25ded2a08143e265c6c25936e74cbc8e641e3312ca288673d91f2f93f8e277de3cfa444ecdaaf982052@157.90.35.166:30303", // bootnode-hetzner-fsn ]; From d0b1e0aa0263af35e39c1524da703f956c68a81c Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Thu, 9 Mar 2023 12:43:56 +0200 Subject: [PATCH 072/191] fix(dep): update and lock revm (#1681) --- Cargo.lock | 9 ++++----- Cargo.toml | 4 ++-- crates/revm/revm-inspectors/src/stack.rs | 4 ++-- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4447e5a43c1..fe8328691cc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5161,7 +5161,7 @@ dependencies = [ [[package]] name = "revm" version = "3.0.0" -source = "git+https://github.com/bluealloy/revm#f2656b7eac6fab1cb872268bb7d0dda4b5d3aa85" +source = "git+https://github.com/bluealloy/revm?rev=b1208fee#b1208feed9cb5957dc4f1d0bda6419bcbfc6e6e8" dependencies = [ "auto_impl", "revm-interpreter", @@ -5171,9 +5171,8 @@ dependencies = [ [[package]] name = "revm-interpreter" version = "1.0.0" -source = "git+https://github.com/bluealloy/revm#f2656b7eac6fab1cb872268bb7d0dda4b5d3aa85" +source = "git+https://github.com/bluealloy/revm?rev=b1208fee#b1208feed9cb5957dc4f1d0bda6419bcbfc6e6e8" dependencies = [ - "bitvec 1.0.1", "derive_more", "enumn", "revm-primitives", @@ -5183,7 +5182,7 @@ dependencies = [ [[package]] name = "revm-precompile" version = "2.0.0" -source = "git+https://github.com/bluealloy/revm#f2656b7eac6fab1cb872268bb7d0dda4b5d3aa85" +source = "git+https://github.com/bluealloy/revm?rev=b1208fee#b1208feed9cb5957dc4f1d0bda6419bcbfc6e6e8" dependencies = [ "k256", "num", @@ -5199,7 +5198,7 @@ dependencies = [ [[package]] name = "revm-primitives" version = "1.0.0" -source = "git+https://github.com/bluealloy/revm#f2656b7eac6fab1cb872268bb7d0dda4b5d3aa85" +source = "git+https://github.com/bluealloy/revm?rev=b1208fee#b1208feed9cb5957dc4f1d0bda6419bcbfc6e6e8" dependencies = [ "arbitrary", "auto_impl", diff --git a/Cargo.toml b/Cargo.toml index ad7253f8624..debc3602069 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,7 +47,7 @@ inherits = "release" debug = true [patch.crates-io] -revm = { git = "https://github.com/bluealloy/revm" } -revm-primitives = { git = "https://github.com/bluealloy/revm" } +revm = { git = "https://github.com/bluealloy/revm", rev = "b1208fee" } +revm-primitives = { git = "https://github.com/bluealloy/revm", rev = "b1208fee" } # patched for quantity U256 responses ruint = { git = "https://github.com/paradigmxyz/uint" } diff --git a/crates/revm/revm-inspectors/src/stack.rs b/crates/revm/revm-inspectors/src/stack.rs index 042dd339f9b..01d56c87039 100644 --- a/crates/revm/revm-inspectors/src/stack.rs +++ b/crates/revm/revm-inspectors/src/stack.rs @@ -232,9 +232,9 @@ where (ret, address, remaining_gas, out) } - fn selfdestruct(&mut self) { + fn selfdestruct(&mut self, contract: Address, target: Address) { call_inspectors!(inspector, [&mut self.custom_print_tracer], { - Inspector::::selfdestruct(inspector); + Inspector::::selfdestruct(inspector, contract, target); }); } } From 17e8ce48dc4ccc480a568c30ff678f616544fd1d Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Thu, 9 Mar 2023 12:49:33 +0200 Subject: [PATCH 073/191] chore(deps): clean up `reth-stages` deps (#1682) --- Cargo.lock | 3 --- crates/stages/Cargo.toml | 9 ++------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fe8328691cc..b1c71ef1912 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5100,16 +5100,13 @@ dependencies = [ "reth-metrics-derive", "reth-primitives", "reth-provider", - "reth-revm", "reth-rlp", - "reth-staged-sync", "serde", "tempfile", "thiserror", "tokio", "tokio-stream", "tracing", - "triehash", ] [[package]] diff --git a/crates/stages/Cargo.toml b/crates/stages/Cargo.toml index f08754b58af..3b1970bd710 100644 --- a/crates/stages/Cargo.toml +++ b/crates/stages/Cargo.toml @@ -14,11 +14,9 @@ normal = [ ] [dependencies] -# reth libs +# reth reth-primitives = { path = "../primitives" } reth-interfaces = { path = "../interfaces" } -reth-revm = { path = "../revm" } -reth-rlp = { path = "../rlp" } reth-db = { path = "../storage/db" } reth-provider = { path = "../storage/provider" } reth-metrics-derive = { path = "../metrics/metrics-derive" } @@ -54,6 +52,7 @@ reth-interfaces = { path = "../interfaces", features = ["test-utils"] } reth-downloaders = { path = "../net/downloaders" } reth-eth-wire = { path = "../net/eth-wire" } # TODO(onbjerg): We only need this for [BlockBody] reth-executor = { path = "../executor" } +reth-rlp = { path = "../rlp" } tokio = { version = "*", features = ["rt", "sync", "macros"] } tempfile = "3.3.0" assert_matches = "1.5.0" @@ -67,10 +66,6 @@ proptest = { version = "1.0" } arbitrary = { version = "1.1.7", features = ["derive"] } eyre = "0.6.8" -# trie -reth-staged-sync = { path = "../staged-sync" } -triehash = "0.8" - [features] default = ["serde"] serde = ["dep:serde"] From e07e9532c04b479532c92a8f5a68e6c4dbdcdf13 Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Thu, 9 Mar 2023 14:00:46 +0200 Subject: [PATCH 074/191] dep(revm): update (#1684) --- Cargo.lock | 8 ++++---- Cargo.toml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b1c71ef1912..4607cebde20 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5158,7 +5158,7 @@ dependencies = [ [[package]] name = "revm" version = "3.0.0" -source = "git+https://github.com/bluealloy/revm?rev=b1208fee#b1208feed9cb5957dc4f1d0bda6419bcbfc6e6e8" +source = "git+https://github.com/bluealloy/revm?rev=d8dc6526#d8dc6526b33d94af8ce46dd9c8d2559c04593504" dependencies = [ "auto_impl", "revm-interpreter", @@ -5168,7 +5168,7 @@ dependencies = [ [[package]] name = "revm-interpreter" version = "1.0.0" -source = "git+https://github.com/bluealloy/revm?rev=b1208fee#b1208feed9cb5957dc4f1d0bda6419bcbfc6e6e8" +source = "git+https://github.com/bluealloy/revm?rev=d8dc6526#d8dc6526b33d94af8ce46dd9c8d2559c04593504" dependencies = [ "derive_more", "enumn", @@ -5179,7 +5179,7 @@ dependencies = [ [[package]] name = "revm-precompile" version = "2.0.0" -source = "git+https://github.com/bluealloy/revm?rev=b1208fee#b1208feed9cb5957dc4f1d0bda6419bcbfc6e6e8" +source = "git+https://github.com/bluealloy/revm?rev=d8dc6526#d8dc6526b33d94af8ce46dd9c8d2559c04593504" dependencies = [ "k256", "num", @@ -5195,7 +5195,7 @@ dependencies = [ [[package]] name = "revm-primitives" version = "1.0.0" -source = "git+https://github.com/bluealloy/revm?rev=b1208fee#b1208feed9cb5957dc4f1d0bda6419bcbfc6e6e8" +source = "git+https://github.com/bluealloy/revm?rev=d8dc6526#d8dc6526b33d94af8ce46dd9c8d2559c04593504" dependencies = [ "arbitrary", "auto_impl", diff --git a/Cargo.toml b/Cargo.toml index debc3602069..eeb3b810ec3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,7 +47,7 @@ inherits = "release" debug = true [patch.crates-io] -revm = { git = "https://github.com/bluealloy/revm", rev = "b1208fee" } -revm-primitives = { git = "https://github.com/bluealloy/revm", rev = "b1208fee" } +revm = { git = "https://github.com/bluealloy/revm", rev = "d8dc6526" } +revm-primitives = { git = "https://github.com/bluealloy/revm", rev = "d8dc6526" } # patched for quantity U256 responses ruint = { git = "https://github.com/paradigmxyz/uint" } From d164a16be7481ebeeda70bad0069a84bb11163c3 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 9 Mar 2023 14:41:12 +0100 Subject: [PATCH 075/191] chore: rm duplicate TxHashNumber entry (#1685) --- bin/reth/src/db/mod.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/reth/src/db/mod.rs b/bin/reth/src/db/mod.rs index fb17e8cc829..2147954be8c 100644 --- a/bin/reth/src/db/mod.rs +++ b/bin/reth/src/db/mod.rs @@ -178,7 +178,6 @@ impl Command { BlockTransitionIndex, TxTransitionIndex, SyncStage, - TxHashNumber, Transactions ]); } From 1e4ab0e1ac94c7898dd43d589bf25f9db7e7c3ac Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 9 Mar 2023 16:20:40 +0100 Subject: [PATCH 076/191] chore: replace lru with schnellru (#1688) --- Cargo.lock | 2 +- crates/net/dns/Cargo.toml | 2 +- crates/net/dns/src/config.rs | 10 +++++++--- crates/net/dns/src/lib.rs | 8 ++++---- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4607cebde20..71b8ad0fe92 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4487,12 +4487,12 @@ dependencies = [ "data-encoding", "enr", "linked_hash_set", - "lru 0.9.0", "parking_lot 0.12.1", "reth-net-common", "reth-primitives", "reth-rlp", "reth-tracing", + "schnellru", "secp256k1 0.24.3", "serde", "serde_with", diff --git a/crates/net/dns/Cargo.toml b/crates/net/dns/Cargo.toml index 80bea744cb5..f8f673fd7f6 100644 --- a/crates/net/dns/Cargo.toml +++ b/crates/net/dns/Cargo.toml @@ -32,7 +32,7 @@ trust-dns-resolver = "0.22" data-encoding = "2" async-trait = "0.1" linked_hash_set = "0.1" -lru = "0.9" +schnellru = "0.2" thiserror = "1.0" tracing = "0.1" parking_lot = "0.12" diff --git a/crates/net/dns/src/config.rs b/crates/net/dns/src/config.rs index 092ddbfce54..6a31c36ad40 100644 --- a/crates/net/dns/src/config.rs +++ b/crates/net/dns/src/config.rs @@ -1,5 +1,9 @@ use crate::tree::LinkEntry; -use std::{collections::HashSet, num::NonZeroUsize, time::Duration}; +use std::{ + collections::HashSet, + num::{NonZeroU32, NonZeroUsize}, + time::Duration, +}; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; @@ -21,7 +25,7 @@ pub struct DnsDiscoveryConfig { /// Default: 30min pub recheck_interval: Duration, /// Maximum number of cached DNS records. - pub dns_record_cache_limit: NonZeroUsize, + pub dns_record_cache_limit: NonZeroU32, /// Links to the DNS networks to bootstrap. pub bootstrap_dns_networks: Option>, } @@ -32,7 +36,7 @@ impl Default for DnsDiscoveryConfig { lookup_timeout: Duration::from_secs(5), max_requests_per_sec: NonZeroUsize::new(3).unwrap(), recheck_interval: Duration::from_secs(60 * 30), - dns_record_cache_limit: NonZeroUsize::new(1_000).unwrap(), + dns_record_cache_limit: NonZeroU32::new(1_000).unwrap(), bootstrap_dns_networks: Some(Default::default()), } } diff --git a/crates/net/dns/src/lib.rs b/crates/net/dns/src/lib.rs index 77ea3e8762d..82cea25df99 100644 --- a/crates/net/dns/src/lib.rs +++ b/crates/net/dns/src/lib.rs @@ -16,8 +16,8 @@ use crate::{ pub use config::DnsDiscoveryConfig; use enr::Enr; use error::ParseDnsEntryError; -use lru::LruCache; use reth_primitives::{ForkId, NodeRecord, PeerId}; +use schnellru::{ByLength, LruMap}; use secp256k1::SecretKey; use std::{ collections::{hash_map::Entry, HashMap, HashSet, VecDeque}, @@ -95,7 +95,7 @@ pub struct DnsDiscoveryService { /// All queries currently in progress queries: QueryPool, /// Cached dns records - dns_record_cache: LruCache>, + dns_record_cache: LruMap>, /// all buffered events queued_events: VecDeque, /// The rate at which trees should be updated. @@ -133,7 +133,7 @@ impl DnsDiscoveryService { node_record_listeners: Default::default(), trees: Default::default(), queries, - dns_record_cache: LruCache::new(dns_record_cache_limit), + dns_record_cache: LruMap::new(ByLength::new(dns_record_cache_limit.get())), queued_events: Default::default(), recheck_interval, bootstrap_dns_networks: bootstrap_dns_networks.unwrap_or_default(), @@ -250,7 +250,7 @@ impl DnsDiscoveryService { } Some(Ok(entry)) => { // cache entry - self.dns_record_cache.push(hash.clone(), entry.clone()); + self.dns_record_cache.insert(hash.clone(), entry.clone()); match entry { DnsEntry::Root(root) => { From e913a536f0001b7bd7daa57c0a1a553491809653 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 9 Mar 2023 16:25:34 +0100 Subject: [PATCH 077/191] fix: serialize index as hex string (#1687) --- Cargo.lock | 1 + crates/rpc/rpc-types/Cargo.toml | 1 + crates/rpc/rpc-types/src/eth/index.rs | 19 ++++++++++++++++++- 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 71b8ad0fe92..8ce1a7824c3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5021,6 +5021,7 @@ version = "0.1.0" dependencies = [ "jsonrpsee-types", "lru 0.9.0", + "rand 0.8.5", "reth-interfaces", "reth-network-api", "reth-primitives", diff --git a/crates/rpc/rpc-types/Cargo.toml b/crates/rpc/rpc-types/Cargo.toml index d3d5a9d1c18..dcf39ef795a 100644 --- a/crates/rpc/rpc-types/Cargo.toml +++ b/crates/rpc/rpc-types/Cargo.toml @@ -27,4 +27,5 @@ jsonrpsee-types = { version = "0.16" } lru = "0.9" [dev-dependencies] +rand = "0.8" reth-interfaces = { path = "../../interfaces", features = ["test-utils"] } diff --git a/crates/rpc/rpc-types/src/eth/index.rs b/crates/rpc/rpc-types/src/eth/index.rs index 217a928e3ed..69e28b016d7 100644 --- a/crates/rpc/rpc-types/src/eth/index.rs +++ b/crates/rpc/rpc-types/src/eth/index.rs @@ -19,7 +19,7 @@ impl Serialize for Index { where S: Serializer, { - serializer.serialize_str(&format!("{:x}", self.0)) + serializer.serialize_str(&format!("0x{:x}", self.0)) } } @@ -71,3 +71,20 @@ impl<'a> Deserialize<'a> for Index { deserializer.deserialize_any(IndexVisitor) } } + +#[cfg(test)] +mod tests { + use super::*; + use rand::{thread_rng, Rng}; + + #[test] + fn test_serde_index_rand() { + let mut rng = thread_rng(); + for _ in 0..100 { + let index = Index(rng.gen()); + let val = serde_json::to_string(&index).unwrap(); + let de: Index = serde_json::from_str(&val).unwrap(); + assert_eq!(index, de); + } + } +} From 8ba0a190b85a3a5b027b7afcceb8799515b72bb1 Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Thu, 9 Mar 2023 19:14:36 +0200 Subject: [PATCH 078/191] fix(executor): preserve existing account state (#1691) --- Cargo.lock | 8 +++--- Cargo.toml | 4 +-- crates/executor/src/executor.rs | 48 +++++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8ce1a7824c3..501d69bc2f6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5159,7 +5159,7 @@ dependencies = [ [[package]] name = "revm" version = "3.0.0" -source = "git+https://github.com/bluealloy/revm?rev=d8dc6526#d8dc6526b33d94af8ce46dd9c8d2559c04593504" +source = "git+https://github.com/bluealloy/revm?rev=afc3066#afc30663270f77df9b4399ad9d4cfb0ad2b814ec" dependencies = [ "auto_impl", "revm-interpreter", @@ -5169,7 +5169,7 @@ dependencies = [ [[package]] name = "revm-interpreter" version = "1.0.0" -source = "git+https://github.com/bluealloy/revm?rev=d8dc6526#d8dc6526b33d94af8ce46dd9c8d2559c04593504" +source = "git+https://github.com/bluealloy/revm?rev=afc3066#afc30663270f77df9b4399ad9d4cfb0ad2b814ec" dependencies = [ "derive_more", "enumn", @@ -5180,7 +5180,7 @@ dependencies = [ [[package]] name = "revm-precompile" version = "2.0.0" -source = "git+https://github.com/bluealloy/revm?rev=d8dc6526#d8dc6526b33d94af8ce46dd9c8d2559c04593504" +source = "git+https://github.com/bluealloy/revm?rev=afc3066#afc30663270f77df9b4399ad9d4cfb0ad2b814ec" dependencies = [ "k256", "num", @@ -5196,7 +5196,7 @@ dependencies = [ [[package]] name = "revm-primitives" version = "1.0.0" -source = "git+https://github.com/bluealloy/revm?rev=d8dc6526#d8dc6526b33d94af8ce46dd9c8d2559c04593504" +source = "git+https://github.com/bluealloy/revm?rev=afc3066#afc30663270f77df9b4399ad9d4cfb0ad2b814ec" dependencies = [ "arbitrary", "auto_impl", diff --git a/Cargo.toml b/Cargo.toml index eeb3b810ec3..62904d8309d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,7 +47,7 @@ inherits = "release" debug = true [patch.crates-io] -revm = { git = "https://github.com/bluealloy/revm", rev = "d8dc6526" } -revm-primitives = { git = "https://github.com/bluealloy/revm", rev = "d8dc6526" } +revm = { git = "https://github.com/bluealloy/revm", rev = "afc3066" } +revm-primitives = { git = "https://github.com/bluealloy/revm", rev = "afc3066" } # patched for quantity U256 responses ruint = { git = "https://github.com/paradigmxyz/uint" } diff --git a/crates/executor/src/executor.rs b/crates/executor/src/executor.rs index 9f9ed262d2e..7b682b65f4c 100644 --- a/crates/executor/src/executor.rs +++ b/crates/executor/src/executor.rs @@ -195,6 +195,10 @@ where new_account.account_state = if account.storage_cleared { new_account.storage.clear(); AccountState::StorageCleared + } else if new_account.account_state.is_storage_cleared() { + // the account already exists and its storage was cleared, preserve its previous + // state + AccountState::StorageCleared } else { AccountState::Touched }; @@ -953,4 +957,48 @@ mod tests { }) ); } + + #[test] + fn test_account_state_preserved() { + let account = Address::from_str("c94f5374fce5edbc8e2a8697c15331677e6ebf0b").unwrap(); + + let mut db = StateProviderTest::default(); + db.insert_account(account, Account::default(), None, HashMap::default()); + + let chain_spec = Arc::new(ChainSpecBuilder::mainnet().istanbul_activated().build()); + let db = SubState::new(State::new(db)); + + let default_acc = RevmAccount { + info: AccountInfo::default(), + storage: hash_map::HashMap::default(), + is_destroyed: false, + is_touched: false, + storage_cleared: false, + is_not_existing: false, + }; + let mut executor = Executor::new(chain_spec, db); + // touch account + executor.commit_changes(hash_map::HashMap::from([( + account, + RevmAccount { ..default_acc.clone() }, + )])); + // destroy account + executor.commit_changes(hash_map::HashMap::from([( + account, + RevmAccount { is_destroyed: true, is_touched: true, ..default_acc.clone() }, + )])); + // re-create account + executor.commit_changes(hash_map::HashMap::from([( + account, + RevmAccount { is_touched: true, storage_cleared: true, ..default_acc.clone() }, + )])); + // touch account + executor + .commit_changes(hash_map::HashMap::from([(account, RevmAccount { ..default_acc })])); + + let db = executor.db(); + + let account = db.load_account(account).unwrap(); + assert_eq!(account.account_state, AccountState::StorageCleared); + } } From ce633c22e57f2bba58e0adf710529a30f3d396d6 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 9 Mar 2023 19:10:26 +0100 Subject: [PATCH 079/191] chore(rpc): move EthRpcErrorCode to rpc-types (#1690) --- crates/rpc/rpc-types/src/eth/error.rs | 23 +++++++++++++++++++++++ crates/rpc/rpc-types/src/eth/mod.rs | 1 + crates/rpc/rpc/src/eth/error.rs | 26 ++------------------------ 3 files changed, 26 insertions(+), 24 deletions(-) create mode 100644 crates/rpc/rpc-types/src/eth/error.rs diff --git a/crates/rpc/rpc-types/src/eth/error.rs b/crates/rpc/rpc-types/src/eth/error.rs new file mode 100644 index 00000000000..43a70feefb2 --- /dev/null +++ b/crates/rpc/rpc-types/src/eth/error.rs @@ -0,0 +1,23 @@ +//! Commonly used errors for the `eth_` namespace. + +/// List of JSON-RPC error codes +#[derive(Debug, Copy, PartialEq, Eq, Clone)] +pub enum EthRpcErrorCode { + /// Failed to send transaction, See also + TransactionRejected, + /// Custom geth error code, + ExecutionError, + /// + InvalidInput, +} + +impl EthRpcErrorCode { + /// Returns the error code as `i32` + pub const fn code(&self) -> i32 { + match *self { + EthRpcErrorCode::TransactionRejected => -32003, + EthRpcErrorCode::ExecutionError => 3, + EthRpcErrorCode::InvalidInput => -32000, + } + } +} diff --git a/crates/rpc/rpc-types/src/eth/mod.rs b/crates/rpc/rpc-types/src/eth/mod.rs index 60d40e999ce..3efa9750a79 100644 --- a/crates/rpc/rpc-types/src/eth/mod.rs +++ b/crates/rpc/rpc-types/src/eth/mod.rs @@ -4,6 +4,7 @@ mod account; mod block; mod call; pub mod engine; +pub mod error; mod fee; mod filter; mod index; diff --git a/crates/rpc/rpc/src/eth/error.rs b/crates/rpc/rpc/src/eth/error.rs index 49cecc7cd58..cca0dd39eba 100644 --- a/crates/rpc/rpc/src/eth/error.rs +++ b/crates/rpc/rpc/src/eth/error.rs @@ -1,37 +1,15 @@ -//! Error variants for the `eth_` namespace. +//! Implementation specific Errors for the `eth_` namespace. use crate::result::{internal_rpc_err, rpc_err}; use jsonrpsee::{core::Error as RpcError, types::error::INVALID_PARAMS_CODE}; use reth_primitives::{constants::SELECTOR_LEN, Address, U128, U256}; -use reth_rpc_types::BlockError; +use reth_rpc_types::{error::EthRpcErrorCode, BlockError}; use reth_transaction_pool::error::PoolError; use revm::primitives::{EVMError, Halt}; /// Result alias pub(crate) type EthResult = Result; -/// List of JSON-RPC error codes -#[derive(Debug, Copy, PartialEq, Eq, Clone)] -pub(crate) enum EthRpcErrorCode { - /// Failed to send transaction, See also - TransactionRejected, - /// Custom geth error code, - ExecutionError, - /// - InvalidInput, -} - -impl EthRpcErrorCode { - /// Returns the error code as `i32` - pub(crate) const fn code(&self) -> i32 { - match *self { - EthRpcErrorCode::TransactionRejected => -32003, - EthRpcErrorCode::ExecutionError => 3, - EthRpcErrorCode::InvalidInput => -32000, - } - } -} - /// Errors that can occur when interacting with the `eth_` namespace #[derive(Debug, thiserror::Error)] #[allow(missing_docs)] From 89305b883af35c539f92f069ea6f5192390ee000 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 9 Mar 2023 21:50:26 +0100 Subject: [PATCH 080/191] chore(rpc): add convenience from impl (#1686) --- crates/rpc/rpc-types/src/eth/index.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/crates/rpc/rpc-types/src/eth/index.rs b/crates/rpc/rpc-types/src/eth/index.rs index 69e28b016d7..4ef83b6ceab 100644 --- a/crates/rpc/rpc-types/src/eth/index.rs +++ b/crates/rpc/rpc-types/src/eth/index.rs @@ -1,10 +1,12 @@ +use reth_primitives::U256; use serde::{ de::{Error, Visitor}, Deserialize, Deserializer, Serialize, Serializer, }; use std::fmt; -/// A hex encoded or decimal index +/// A hex encoded or decimal index that's intended to be used as a rust index, hence it's +/// deserialized into a `usize`. #[derive(Debug, PartialEq, Eq, Hash, Clone, Copy, Default)] pub struct Index(usize); @@ -14,6 +16,12 @@ impl From for usize { } } +impl From for U256 { + fn from(idx: Index) -> Self { + U256::from(idx.0) + } +} + impl Serialize for Index { fn serialize(&self, serializer: S) -> Result where From ea11461126229a75532bc96d97e615d3b47ea343 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 9 Mar 2023 21:51:35 +0100 Subject: [PATCH 081/191] chore(rpc): add required fields to Trace handler (#1683) --- crates/rpc/rpc-builder/src/lib.rs | 5 ++++- crates/rpc/rpc/src/trace.rs | 27 +++++++++++++++++---------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/crates/rpc/rpc-builder/src/lib.rs b/crates/rpc/rpc-builder/src/lib.rs index c09caf4ca64..a9837adff9f 100644 --- a/crates/rpc/rpc-builder/src/lib.rs +++ b/crates/rpc/rpc-builder/src/lib.rs @@ -559,6 +559,7 @@ where namespaces: impl Iterator, ) -> Vec { let eth_api = self.eth_api(); + let eth_cache = self.eth_cache(); namespaces .map(|namespace| { self.modules @@ -572,7 +573,9 @@ where RethRpcModule::Net => { NetApi::new(self.network.clone(), eth_api.clone()).into_rpc().into() } - RethRpcModule::Trace => TraceApi::new().into_rpc().into(), + RethRpcModule::Trace => { + TraceApi::new(self.client.clone(), eth_cache.clone()).into_rpc().into() + } RethRpcModule::Web3 => Web3Api::new(self.network.clone()).into_rpc().into(), }) .clone() diff --git a/crates/rpc/rpc/src/trace.rs b/crates/rpc/rpc/src/trace.rs index 962129fbd10..2767d6bf99e 100644 --- a/crates/rpc/rpc/src/trace.rs +++ b/crates/rpc/rpc/src/trace.rs @@ -1,7 +1,8 @@ -use crate::result::internal_rpc_err; +use crate::{eth::cache::EthStateCache, result::internal_rpc_err}; use async_trait::async_trait; use jsonrpsee::core::RpcResult as Result; use reth_primitives::{BlockId, Bytes, H256}; +use reth_provider::{BlockProvider, EvmEnvProvider, StateProviderFactory}; use reth_rpc_api::TraceApiServer; use reth_rpc_types::{ trace::{filter::TraceFilter, parity::*}, @@ -12,22 +13,28 @@ use std::collections::HashSet; /// `trace` API implementation. /// /// This type provides the functionality for handling `trace` related requests. -#[non_exhaustive] -pub struct TraceApi {} +#[derive(Clone)] +pub struct TraceApi { + /// The client that can interact with the chain. + client: Client, + /// The async cache frontend for eth related data + eth_cache: EthStateCache, +} // === impl TraceApi === -impl TraceApi { +impl TraceApi { /// Create a new instance of the [TraceApi] - #[allow(clippy::new_without_default)] - // TODO add necessary types - pub fn new() -> Self { - Self {} + pub fn new(client: Client, eth_cache: EthStateCache) -> Self { + Self { client, eth_cache } } } #[async_trait] -impl TraceApiServer for TraceApi { +impl TraceApiServer for TraceApi +where + Client: BlockProvider + StateProviderFactory + EvmEnvProvider + 'static, +{ async fn call( &self, _call: CallRequest, @@ -91,7 +98,7 @@ impl TraceApiServer for TraceApi { } } -impl std::fmt::Debug for TraceApi { +impl std::fmt::Debug for TraceApi { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("TraceApi").finish_non_exhaustive() } From 6190eac9cea28d5fb87ab28a239fb6f1e74719b5 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 9 Mar 2023 22:43:24 +0100 Subject: [PATCH 082/191] feat: add Tracing inspector (#1618) --- Cargo.lock | 3 + crates/revm/revm-inspectors/Cargo.toml | 5 +- crates/revm/revm-inspectors/src/lib.rs | 3 + .../revm-inspectors/src/stack/maybe_owned.rs | 209 +++++++++ .../src/{stack.rs => stack/mod.rs} | 4 + .../revm/revm-inspectors/src/tracing/arena.rs | 160 +++++++ .../revm/revm-inspectors/src/tracing/mod.rs | 408 ++++++++++++++++++ .../revm/revm-inspectors/src/tracing/types.rs | 293 +++++++++++++ .../revm/revm-inspectors/src/tracing/utils.rs | 40 ++ crates/rpc/rpc-types/Cargo.toml | 3 + crates/rpc/rpc-types/src/eth/trace/mod.rs | 48 +++ crates/rpc/rpc-types/src/eth/trace/parity.rs | 17 +- 12 files changed, 1191 insertions(+), 2 deletions(-) create mode 100644 crates/revm/revm-inspectors/src/stack/maybe_owned.rs rename crates/revm/revm-inspectors/src/{stack.rs => stack/mod.rs} (98%) create mode 100644 crates/revm/revm-inspectors/src/tracing/arena.rs create mode 100644 crates/revm/revm-inspectors/src/tracing/mod.rs create mode 100644 crates/revm/revm-inspectors/src/tracing/types.rs create mode 100644 crates/revm/revm-inspectors/src/tracing/utils.rs diff --git a/Cargo.lock b/Cargo.lock index 501d69bc2f6..3abe264cc10 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4880,7 +4880,9 @@ version = "0.1.0" dependencies = [ "hashbrown 0.13.2", "reth-primitives", + "reth-rpc-types", "revm", + "serde", ] [[package]] @@ -5019,6 +5021,7 @@ dependencies = [ name = "reth-rpc-types" version = "0.1.0" dependencies = [ + "ethers-core", "jsonrpsee-types", "lru 0.9.0", "rand 0.8.5", diff --git a/crates/revm/revm-inspectors/Cargo.toml b/crates/revm/revm-inspectors/Cargo.toml index 5f21a007b20..19e5c34f70a 100644 --- a/crates/revm/revm-inspectors/Cargo.toml +++ b/crates/revm/revm-inspectors/Cargo.toml @@ -9,7 +9,10 @@ description = "revm inspector implementations used by reth" [dependencies] # reth reth-primitives = { path = "../../primitives" } +reth-rpc-types = { path = "../../rpc/rpc-types" } revm = { version = "3.0.0" } # remove from reth and reexport from revm -hashbrown = "0.13" \ No newline at end of file +hashbrown = "0.13" + +serde = { version = "1.0", features = ["derive"] } diff --git a/crates/revm/revm-inspectors/src/lib.rs b/crates/revm/revm-inspectors/src/lib.rs index b4d79c40603..a447c70951a 100644 --- a/crates/revm/revm-inspectors/src/lib.rs +++ b/crates/revm/revm-inspectors/src/lib.rs @@ -14,3 +14,6 @@ pub mod access_list; /// each inspector and allowing to hook on block/transaciton execution, /// used in the main RETH executor. pub mod stack; + +/// An inspector for recording traces +pub mod tracing; diff --git a/crates/revm/revm-inspectors/src/stack/maybe_owned.rs b/crates/revm/revm-inspectors/src/stack/maybe_owned.rs new file mode 100644 index 00000000000..897546f1672 --- /dev/null +++ b/crates/revm/revm-inspectors/src/stack/maybe_owned.rs @@ -0,0 +1,209 @@ +use revm::{ + interpreter::{CallInputs, CreateInputs, Gas, InstructionResult, Interpreter}, + primitives::{db::Database, Bytes, B160, B256}, + EVMData, Inspector, +}; +use std::{ + cell::{Ref, RefCell}, + rc::Rc, +}; + +/// An [Inspector] that is either owned by an individual [Inspector] or is shared as part of a +/// series of inspectors in a [InspectorStack](crate::stack::InspectorStack). +/// +/// Caution: if the [Inspector] is _stacked_ then it _must_ be called first. +#[derive(Debug)] +pub enum MaybeOwnedInspector { + /// Inspector is owned. + Owned(Rc>), + /// Inspector is shared and part of a stack + Stacked(Rc>), +} + +impl MaybeOwnedInspector { + /// Create a new _owned_ instance + pub fn new_owned(inspector: INSP) -> Self { + MaybeOwnedInspector::Owned(Rc::new(RefCell::new(inspector))) + } + + /// Creates a [MaybeOwnedInspector::Stacked] clone of this type. + pub fn clone_stacked(&self) -> Self { + match self { + MaybeOwnedInspector::Owned(gas) | MaybeOwnedInspector::Stacked(gas) => { + MaybeOwnedInspector::Stacked(Rc::clone(gas)) + } + } + } + + /// Returns a reference to the inspector. + pub fn as_ref(&self) -> Ref<'_, INSP> { + match self { + MaybeOwnedInspector::Owned(insp) => insp.borrow(), + MaybeOwnedInspector::Stacked(insp) => insp.borrow(), + } + } +} + +impl MaybeOwnedInspector { + /// Create a new _owned_ instance + pub fn owned() -> Self { + Self::new_owned(Default::default()) + } +} + +impl Default for MaybeOwnedInspector { + fn default() -> Self { + Self::owned() + } +} + +impl Clone for MaybeOwnedInspector { + fn clone(&self) -> Self { + self.clone_stacked() + } +} + +impl Inspector for MaybeOwnedInspector +where + DB: Database, + INSP: Inspector, +{ + fn initialize_interp( + &mut self, + interp: &mut Interpreter, + data: &mut EVMData<'_, DB>, + is_static: bool, + ) -> InstructionResult { + match self { + MaybeOwnedInspector::Owned(insp) => { + return insp.borrow_mut().initialize_interp(interp, data, is_static) + } + MaybeOwnedInspector::Stacked(_) => {} + } + + InstructionResult::Continue + } + + fn step( + &mut self, + interp: &mut Interpreter, + data: &mut EVMData<'_, DB>, + is_static: bool, + ) -> InstructionResult { + match self { + MaybeOwnedInspector::Owned(insp) => { + return insp.borrow_mut().step(interp, data, is_static) + } + MaybeOwnedInspector::Stacked(_) => {} + } + + InstructionResult::Continue + } + + fn log( + &mut self, + evm_data: &mut EVMData<'_, DB>, + address: &B160, + topics: &[B256], + data: &Bytes, + ) { + match self { + MaybeOwnedInspector::Owned(insp) => { + return insp.borrow_mut().log(evm_data, address, topics, data) + } + MaybeOwnedInspector::Stacked(_) => {} + } + } + + fn step_end( + &mut self, + interp: &mut Interpreter, + data: &mut EVMData<'_, DB>, + is_static: bool, + eval: InstructionResult, + ) -> InstructionResult { + match self { + MaybeOwnedInspector::Owned(insp) => { + return insp.borrow_mut().step_end(interp, data, is_static, eval) + } + MaybeOwnedInspector::Stacked(_) => {} + } + + InstructionResult::Continue + } + + fn call( + &mut self, + data: &mut EVMData<'_, DB>, + inputs: &mut CallInputs, + is_static: bool, + ) -> (InstructionResult, Gas, Bytes) { + match self { + MaybeOwnedInspector::Owned(insp) => { + return insp.borrow_mut().call(data, inputs, is_static) + } + MaybeOwnedInspector::Stacked(_) => {} + } + + (InstructionResult::Continue, Gas::new(0), Bytes::new()) + } + + fn call_end( + &mut self, + data: &mut EVMData<'_, DB>, + inputs: &CallInputs, + remaining_gas: Gas, + ret: InstructionResult, + out: Bytes, + is_static: bool, + ) -> (InstructionResult, Gas, Bytes) { + match self { + MaybeOwnedInspector::Owned(insp) => { + return insp.borrow_mut().call_end(data, inputs, remaining_gas, ret, out, is_static) + } + MaybeOwnedInspector::Stacked(_) => {} + } + (ret, remaining_gas, out) + } + + fn create( + &mut self, + data: &mut EVMData<'_, DB>, + inputs: &mut CreateInputs, + ) -> (InstructionResult, Option, Gas, Bytes) { + match self { + MaybeOwnedInspector::Owned(insp) => return insp.borrow_mut().create(data, inputs), + MaybeOwnedInspector::Stacked(_) => {} + } + + (InstructionResult::Continue, None, Gas::new(0), Bytes::default()) + } + + fn create_end( + &mut self, + data: &mut EVMData<'_, DB>, + inputs: &CreateInputs, + ret: InstructionResult, + address: Option, + remaining_gas: Gas, + out: Bytes, + ) -> (InstructionResult, Option, Gas, Bytes) { + match self { + MaybeOwnedInspector::Owned(insp) => { + return insp.borrow_mut().create_end(data, inputs, ret, address, remaining_gas, out) + } + MaybeOwnedInspector::Stacked(_) => {} + } + + (ret, address, remaining_gas, out) + } + + fn selfdestruct(&mut self, contract: B160, target: B160) { + match self { + MaybeOwnedInspector::Owned(insp) => { + return insp.borrow_mut().selfdestruct(contract, target) + } + MaybeOwnedInspector::Stacked(_) => {} + } + } +} diff --git a/crates/revm/revm-inspectors/src/stack.rs b/crates/revm/revm-inspectors/src/stack/mod.rs similarity index 98% rename from crates/revm/revm-inspectors/src/stack.rs rename to crates/revm/revm-inspectors/src/stack/mod.rs index 01d56c87039..5a37169e2f9 100644 --- a/crates/revm/revm-inspectors/src/stack.rs +++ b/crates/revm/revm-inspectors/src/stack/mod.rs @@ -6,6 +6,10 @@ use revm::{ Database, EVMData, Inspector, }; +/// A wrapped [Inspector](revm::Inspector) that can be reused in the stack +mod maybe_owned; +pub use maybe_owned::MaybeOwnedInspector; + /// One can hook on inspector execution in 3 ways: /// - Block: Hook on block execution /// - BlockWithIndex: Hook on block execution transaction index diff --git a/crates/revm/revm-inspectors/src/tracing/arena.rs b/crates/revm/revm-inspectors/src/tracing/arena.rs new file mode 100644 index 00000000000..08bfa65bf9b --- /dev/null +++ b/crates/revm/revm-inspectors/src/tracing/arena.rs @@ -0,0 +1,160 @@ +use crate::tracing::types::{CallTrace, CallTraceNode, LogCallOrder}; +use reth_primitives::{Address, JsonU256, H256, U256}; +use reth_rpc_types::trace::{ + geth::{DefaultFrame, GethDebugTracingOptions, StructLog}, + parity::{ActionType, TransactionTrace}, +}; +use revm::interpreter::{opcode, InstructionResult}; +use std::collections::{BTreeMap, HashMap}; + +/// An arena of recorded traces. +/// +/// This type will be populated via the [TracingInspector](crate::tracing::TracingInspector). +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct CallTraceArena { + /// The arena of recorded trace nodes + pub(crate) arena: Vec, +} + +impl CallTraceArena { + /// Pushes a new trace into the arena, returning the trace ID + pub(crate) fn push_trace(&mut self, entry: usize, new_trace: CallTrace) -> usize { + match new_trace.depth { + // The entry node, just update it + 0 => { + self.arena[0].trace = new_trace; + 0 + } + // We found the parent node, add the new trace as a child + _ if self.arena[entry].trace.depth == new_trace.depth - 1 => { + let id = self.arena.len(); + + let trace_location = self.arena[entry].children.len(); + self.arena[entry].ordering.push(LogCallOrder::Call(trace_location)); + let node = CallTraceNode { + parent: Some(entry), + trace: new_trace, + idx: id, + ..Default::default() + }; + self.arena.push(node); + self.arena[entry].children.push(id); + + id + } + // We haven't found the parent node, go deeper + _ => self.push_trace( + *self.arena[entry].children.last().expect("Disconnected trace"), + new_trace, + ), + } + } + + /// Returns the traces of the transaction for `trace_transaction` + pub fn parity_traces(&self) -> Vec { + let traces = Vec::with_capacity(self.arena.len()); + for (_idx, node) in self.arena.iter().cloned().enumerate() { + let _action = node.parity_action(); + let _result = node.parity_result(); + + let _action_type = if node.status() == InstructionResult::SelfDestruct { + ActionType::Selfdestruct + } else { + node.kind().into() + }; + + todo!() + + // let trace = TransactionTrace { + // action, + // result: Some(result), + // trace_address: self.info.trace_address(idx), + // subtraces: node.children.len(), + // }; + // traces.push(trace) + } + + traces + } + + /// Recursively fill in the geth trace by going through the traces + /// + /// TODO rewrite this iteratively + fn add_to_geth_trace( + &self, + storage: &mut HashMap>, + trace_node: &CallTraceNode, + struct_logs: &mut Vec, + opts: &GethDebugTracingOptions, + ) { + let mut child_id = 0; + // Iterate over the steps inside the given trace + for step in trace_node.trace.steps.iter() { + let mut log: StructLog = step.into(); + + // Fill in memory and storage depending on the options + if !opts.disable_storage.unwrap_or_default() { + let contract_storage = storage.entry(step.contract).or_default(); + if let Some((key, value)) = step.state_diff { + contract_storage.insert(key.into(), value.into()); + log.storage = Some(contract_storage.clone()); + } + } + if opts.disable_stack.unwrap_or_default() { + log.stack = None; + } + if !opts.enable_memory.unwrap_or_default() { + log.memory = None; + } + + // Add step to geth trace + struct_logs.push(log); + + // If the opcode is a call, the descend into child trace + match step.op.u8() { + opcode::CREATE | + opcode::CREATE2 | + opcode::DELEGATECALL | + opcode::CALL | + opcode::STATICCALL | + opcode::CALLCODE => { + self.add_to_geth_trace( + storage, + &self.arena[trace_node.children[child_id]], + struct_logs, + opts, + ); + child_id += 1; + } + _ => {} + } + } + } + + /// Generate a geth-style trace e.g. for `debug_traceTransaction` + pub fn geth_traces( + &self, + // TODO(mattsse): This should be the total gas used, or gas used by last CallTrace? + receipt_gas_used: U256, + opts: GethDebugTracingOptions, + ) -> DefaultFrame { + if self.arena.is_empty() { + return Default::default() + } + // Fetch top-level trace + let main_trace_node = &self.arena[0]; + let main_trace = &main_trace_node.trace; + + let mut struct_logs = Vec::new(); + let mut storage = HashMap::new(); + self.add_to_geth_trace(&mut storage, main_trace_node, &mut struct_logs, &opts); + + DefaultFrame { + // If the top-level trace succeeded, then it was a success + failed: !main_trace.success, + gas: JsonU256(receipt_gas_used), + return_value: main_trace.output.clone().into(), + struct_logs, + } + } +} diff --git a/crates/revm/revm-inspectors/src/tracing/mod.rs b/crates/revm/revm-inspectors/src/tracing/mod.rs new file mode 100644 index 00000000000..e5285a3e6b0 --- /dev/null +++ b/crates/revm/revm-inspectors/src/tracing/mod.rs @@ -0,0 +1,408 @@ +use crate::{ + stack::MaybeOwnedInspector, + tracing::{ + types::{CallKind, LogCallOrder, RawLog}, + utils::{gas_used, get_create_address}, + }, +}; +pub use arena::CallTraceArena; +use reth_primitives::{bytes::Bytes, Address, H256, U256}; +use revm::{ + inspectors::GasInspector, + interpreter::{ + opcode, return_ok, CallInputs, CallScheme, CreateInputs, Gas, InstructionResult, + Interpreter, OpCode, + }, + Database, EVMData, Inspector, JournalEntry, +}; +use types::{CallTrace, CallTraceStep}; + +mod arena; +mod types; +mod utils; + +/// An inspector that collects call traces. +/// +/// This [Inspector] can be hooked into the [EVM](revm::EVM) which then calls the inspector +/// functions, such as [Inspector::call] or [Inspector::call_end]. +/// +/// The [TracingInspector] keeps track of everything by: +/// 1. start tracking steps/calls on [Inspector::step] and [Inspector::call] +/// 2. complete steps/calls on [Inspector::step_end] and [Inspector::call_end] +#[derive(Default, Debug, Clone)] +pub struct TracingInspector { + /// Whether to include individual steps [Inspector::step] + record_steps: bool, + /// Records all call traces + traces: CallTraceArena, + trace_stack: Vec, + step_stack: Vec, + /// The gas inspector used to track remaining gas. + /// + /// This is either owned by this inspector directly or part of a stack of inspectors, in which + /// case all delegated functions are no-ops. + gas_inspector: MaybeOwnedInspector, +} + +// === impl TracingInspector === + +impl TracingInspector { + /// Consumes the Inspector and returns the recorded. + pub fn finalize(self) -> CallTraceArena { + self.traces + } + + /// Enables step recording and uses the configured [GasInspector] to report gas costs for each + /// step. + pub fn with_steps_recording(mut self) -> Self { + self.record_steps = true; + self + } + + /// Configures a [GasInspector] + /// + /// If this [TracingInspector] is part of a stack [InspectorStack](crate::stack::InspectorStack) + /// which already uses a [GasInspector], it can be reused as [MaybeOwnedInspector::Stacked] in + /// which case the `gas_inspector`'s usage will be a no-op within the context of this + /// [TracingInspector]. + pub fn with_stacked_gas_inspector( + mut self, + gas_inspector: MaybeOwnedInspector, + ) -> Self { + self.gas_inspector = gas_inspector; + self + } + + /// Returns the last trace [CallTrace] index from the stack. + /// + /// # Panics + /// + /// If no [CallTrace] was pushed + #[track_caller] + #[inline] + fn last_trace_idx(&self) -> usize { + self.trace_stack.last().copied().expect("can't start step without starting a trace first") + } + + /// _Removes_ the last trace [CallTrace] index from the stack. + /// + /// # Panics + /// + /// If no [CallTrace] was pushed + #[track_caller] + #[inline] + fn pop_trace_idx(&mut self) -> usize { + self.trace_stack.pop().expect("more traces were filled than started") + } + + /// Starts tracking a new trace. + /// + /// Invoked on [Inspector::call]. + fn start_trace_on_call( + &mut self, + depth: usize, + address: Address, + data: Bytes, + value: U256, + kind: CallKind, + caller: Address, + ) { + self.trace_stack.push(self.traces.push_trace( + 0, + CallTrace { + depth, + address, + kind, + data, + value, + status: InstructionResult::Continue, + caller, + ..Default::default() + }, + )); + } + + /// Fills the current trace with the outcome of a call. + /// + /// Invoked on [Inspector::call_end]. + /// + /// # Panics + /// + /// This expects an existing trace [Self::start_trace_on_call] + fn fill_trace_on_call_end( + &mut self, + status: InstructionResult, + gas_used: u64, + output: Bytes, + created_address: Option
, + ) { + let trace_idx = self.pop_trace_idx(); + let trace = &mut self.traces.arena[trace_idx].trace; + + let success = matches!(status, return_ok!()); + trace.status = status; + trace.success = success; + trace.gas_used = gas_used; + trace.output = output; + + if let Some(address) = created_address { + // A new contract was created via CREATE + trace.address = address; + } + } + + /// Starts tracking a step + /// + /// Invoked on [Inspector::step] + /// + /// # Panics + /// + /// This expects an existing [CallTrace], in other words, this panics if not within the context + /// of a call. + fn start_step(&mut self, interp: &mut Interpreter, data: &mut EVMData<'_, DB>) { + let trace_idx = self.last_trace_idx(); + let trace = &mut self.traces.arena[trace_idx]; + + self.step_stack.push(StackStep { trace_idx, step_idx: trace.trace.steps.len() }); + + let pc = interp.program_counter(); + + trace.trace.steps.push(CallTraceStep { + depth: data.journaled_state.depth(), + pc, + op: OpCode::try_from_u8(interp.contract.bytecode.bytecode()[pc]) + .expect("is valid opcode;"), + contract: interp.contract.address, + stack: interp.stack.clone(), + memory: interp.memory.clone(), + gas: self.gas_inspector.as_ref().gas_remaining(), + gas_refund_counter: interp.gas.refunded() as u64, + + // fields will be populated end of call + gas_cost: 0, + state_diff: None, + status: InstructionResult::Continue, + }); + } + + /// Fills the current trace with the output of a step. + /// + /// Invoked on [Inspector::step_end]. + fn fill_step_on_step_end( + &mut self, + interp: &mut Interpreter, + data: &mut EVMData<'_, DB>, + status: InstructionResult, + ) { + let StackStep { trace_idx, step_idx } = + self.step_stack.pop().expect("can't fill step without starting a step first"); + let step = &mut self.traces.arena[trace_idx].trace.steps[step_idx]; + + if let Some(pc) = interp.program_counter().checked_sub(1) { + let op = interp.contract.bytecode.bytecode()[pc]; + + let journal_entry = data + .journaled_state + .journal + .last() + // This should always work because revm initializes it as `vec![vec![]]` + // See [JournaledState::new](revm::JournaledState) + .expect("exists; initialized with vec") + .last(); + + step.state_diff = match (op, journal_entry) { + ( + opcode::SLOAD | opcode::SSTORE, + Some(JournalEntry::StorageChange { address, key, .. }), + ) => { + // SAFETY: (Address,key) exists if part if StorageChange + let value = data.journaled_state.state[address].storage[key].present_value(); + Some((*key, value)) + } + _ => None, + }; + + step.gas_cost = step.gas - self.gas_inspector.as_ref().gas_remaining(); + } + + // set the status + step.status = status; + } +} + +impl Inspector for TracingInspector +where + DB: Database, +{ + fn initialize_interp( + &mut self, + interp: &mut Interpreter, + data: &mut EVMData<'_, DB>, + is_static: bool, + ) -> InstructionResult { + self.gas_inspector.initialize_interp(interp, data, is_static) + } + + fn step( + &mut self, + interp: &mut Interpreter, + data: &mut EVMData<'_, DB>, + is_static: bool, + ) -> InstructionResult { + if self.record_steps { + self.gas_inspector.step(interp, data, is_static); + self.start_step(interp, data); + } + + InstructionResult::Continue + } + + fn log( + &mut self, + evm_data: &mut EVMData<'_, DB>, + address: &Address, + topics: &[H256], + data: &Bytes, + ) { + self.gas_inspector.log(evm_data, address, topics, data); + + let trace_idx = self.last_trace_idx(); + let trace = &mut self.traces.arena[trace_idx]; + trace.ordering.push(LogCallOrder::Log(trace.logs.len())); + trace.logs.push(RawLog { topics: topics.to_vec(), data: data.clone() }); + } + + fn step_end( + &mut self, + interp: &mut Interpreter, + data: &mut EVMData<'_, DB>, + is_static: bool, + eval: InstructionResult, + ) -> InstructionResult { + if self.record_steps { + self.gas_inspector.step_end(interp, data, is_static, eval); + self.fill_step_on_step_end(interp, data, eval); + return eval + } + InstructionResult::Continue + } + + fn call( + &mut self, + data: &mut EVMData<'_, DB>, + inputs: &mut CallInputs, + is_static: bool, + ) -> (InstructionResult, Gas, Bytes) { + self.gas_inspector.call(data, inputs, is_static); + + // determine correct `from` and `to` based on the call scheme + let (from, to) = match inputs.context.scheme { + CallScheme::DelegateCall | CallScheme::CallCode => { + (inputs.context.address, inputs.context.code_address) + } + _ => (inputs.context.caller, inputs.context.address), + }; + + self.start_trace_on_call( + data.journaled_state.depth() as usize, + to, + inputs.input.clone(), + inputs.transfer.value, + inputs.context.scheme.into(), + from, + ); + + (InstructionResult::Continue, Gas::new(0), Bytes::new()) + } + + fn call_end( + &mut self, + data: &mut EVMData<'_, DB>, + inputs: &CallInputs, + gas: Gas, + ret: InstructionResult, + out: Bytes, + is_static: bool, + ) -> (InstructionResult, Gas, Bytes) { + self.gas_inspector.call_end(data, inputs, gas, ret, out.clone(), is_static); + + self.fill_trace_on_call_end( + ret, + gas_used(data.env.cfg.spec_id, gas.spend(), gas.refunded() as u64), + out.clone(), + None, + ); + + (ret, gas, out) + } + + fn create( + &mut self, + data: &mut EVMData<'_, DB>, + inputs: &mut CreateInputs, + ) -> (InstructionResult, Option
, Gas, Bytes) { + self.gas_inspector.create(data, inputs); + + let _ = data.journaled_state.load_account(inputs.caller, data.db); + let nonce = data.journaled_state.account(inputs.caller).info.nonce; + self.start_trace_on_call( + data.journaled_state.depth() as usize, + get_create_address(inputs, nonce), + inputs.init_code.clone(), + inputs.value, + inputs.scheme.into(), + inputs.caller, + ); + + (InstructionResult::Continue, None, Gas::new(inputs.gas_limit), Bytes::default()) + } + + /// Called when a contract has been created. + /// + /// InstructionResulting anything other than the values passed to this function (`(ret, + /// remaining_gas, address, out)`) will alter the result of the create. + fn create_end( + &mut self, + data: &mut EVMData<'_, DB>, + inputs: &CreateInputs, + status: InstructionResult, + address: Option
, + gas: Gas, + retdata: Bytes, + ) -> (InstructionResult, Option
, Gas, Bytes) { + self.gas_inspector.create_end(data, inputs, status, address, gas, retdata.clone()); + + // get the code of the created contract + let code = address + .and_then(|address| { + data.journaled_state + .account(address) + .info + .code + .as_ref() + .map(|code| code.bytes()[..code.len()].to_vec()) + }) + .unwrap_or_default(); + + self.fill_trace_on_call_end( + status, + gas_used(data.env.cfg.spec_id, gas.spend(), gas.refunded() as u64), + code.into(), + address, + ); + + (status, address, gas, retdata) + } + + fn selfdestruct(&mut self, _contract: Address, target: Address) { + let trace_idx = self.last_trace_idx(); + let trace = &mut self.traces.arena[trace_idx].trace; + trace.selfdestruct_refund_target = Some(target) + } +} + +#[derive(Debug, Clone, Copy)] +struct StackStep { + trace_idx: usize, + step_idx: usize, +} diff --git a/crates/revm/revm-inspectors/src/tracing/types.rs b/crates/revm/revm-inspectors/src/tracing/types.rs new file mode 100644 index 00000000000..3c13d29135c --- /dev/null +++ b/crates/revm/revm-inspectors/src/tracing/types.rs @@ -0,0 +1,293 @@ +//! Types for representing call trace items. + +use crate::tracing::utils::convert_memory; +use reth_primitives::{bytes::Bytes, Address, H256, U256}; +use reth_rpc_types::trace::{ + geth::StructLog, + parity::{ + Action, ActionType, CallAction, CallOutput, CallType, CreateAction, CreateOutput, + SelfdestructAction, TraceOutput, + }, +}; +use revm::interpreter::{ + CallContext, CallScheme, CreateScheme, InstructionResult, Memory, OpCode, Stack, +}; +use serde::{Deserialize, Serialize}; + +/// A unified representation of a call +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "UPPERCASE")] +#[allow(missing_docs)] +pub enum CallKind { + #[default] + Call, + StaticCall, + CallCode, + DelegateCall, + Create, + Create2, +} + +impl From for CallKind { + fn from(scheme: CallScheme) -> Self { + match scheme { + CallScheme::Call => CallKind::Call, + CallScheme::StaticCall => CallKind::StaticCall, + CallScheme::CallCode => CallKind::CallCode, + CallScheme::DelegateCall => CallKind::DelegateCall, + } + } +} + +impl From for CallKind { + fn from(create: CreateScheme) -> Self { + match create { + CreateScheme::Create => CallKind::Create, + CreateScheme::Create2 { .. } => CallKind::Create2, + } + } +} + +impl From for ActionType { + fn from(kind: CallKind) -> Self { + match kind { + CallKind::Call | CallKind::StaticCall | CallKind::DelegateCall | CallKind::CallCode => { + ActionType::Call + } + CallKind::Create => ActionType::Create, + CallKind::Create2 => ActionType::Create, + } + } +} + +impl From for CallType { + fn from(ty: CallKind) -> Self { + match ty { + CallKind::Call => CallType::Call, + CallKind::StaticCall => CallType::StaticCall, + CallKind::CallCode => CallType::CallCode, + CallKind::DelegateCall => CallType::DelegateCall, + CallKind::Create => CallType::None, + CallKind::Create2 => CallType::None, + } + } +} + +/// A trace of a call. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct CallTrace { + /// The depth of the call + pub(crate) depth: usize, + /// Whether the call was successful + pub(crate) success: bool, + /// caller of this call + pub(crate) caller: Address, + /// The destination address of the call or the address from the created contract. + /// + /// In other words, this is the callee if the [CallKind::Call] or the address of the created + /// contract if [CallKind::Create]. + pub(crate) address: Address, + /// Holds the target for the selfdestruct refund target if `status` is + /// [InstructionResult::SelfDestruct] + pub(crate) selfdestruct_refund_target: Option
, + /// The kind of call this is + pub(crate) kind: CallKind, + /// The value transferred in the call + pub(crate) value: U256, + /// The calldata for the call, or the init code for contract creations + pub(crate) data: Bytes, + /// The return data of the call if this was not a contract creation, otherwise it is the + /// runtime bytecode of the created contract + pub(crate) output: Bytes, + /// The gas cost of the call + pub(crate) gas_used: u64, + /// The status of the trace's call + pub(crate) status: InstructionResult, + /// call context of the runtime + pub(crate) call_context: Option, + /// Opcode-level execution steps + pub(crate) steps: Vec, +} + +impl Default for CallTrace { + fn default() -> Self { + Self { + depth: Default::default(), + success: Default::default(), + caller: Default::default(), + address: Default::default(), + selfdestruct_refund_target: None, + kind: Default::default(), + value: Default::default(), + data: Default::default(), + output: Default::default(), + gas_used: Default::default(), + status: InstructionResult::Continue, + call_context: Default::default(), + steps: Default::default(), + } + } +} + +/// A node in the arena +#[derive(Default, Debug, Clone, PartialEq, Eq)] +pub(crate) struct CallTraceNode { + /// Parent node index in the arena + pub(crate) parent: Option, + /// Children node indexes in the arena + pub(crate) children: Vec, + /// This node's index in the arena + pub(crate) idx: usize, + /// The call trace + pub(crate) trace: CallTrace, + /// Logs + pub(crate) logs: Vec, + /// Ordering of child calls and logs + pub(crate) ordering: Vec, +} + +impl CallTraceNode { + /// Returns the kind of call the trace belongs to + pub(crate) fn kind(&self) -> CallKind { + self.trace.kind + } + + /// Returns the status of the call + pub(crate) fn status(&self) -> InstructionResult { + self.trace.status + } + + /// Returns the `Output` for a parity trace + pub(crate) fn parity_result(&self) -> TraceOutput { + match self.kind() { + CallKind::Call | CallKind::StaticCall | CallKind::CallCode | CallKind::DelegateCall => { + TraceOutput::Call(CallOutput { + gas_used: self.trace.gas_used.into(), + output: self.trace.output.clone().into(), + }) + } + CallKind::Create | CallKind::Create2 => TraceOutput::Create(CreateOutput { + gas_used: self.trace.gas_used.into(), + code: self.trace.output.clone().into(), + address: self.trace.address, + }), + } + } + + /// Returns the `Action` for a parity trace + pub(crate) fn parity_action(&self) -> Action { + if self.status() == InstructionResult::SelfDestruct { + return Action::Selfdestruct(SelfdestructAction { + address: self.trace.address, + refund_address: self.trace.selfdestruct_refund_target.unwrap_or_default(), + balance: self.trace.value, + }) + } + match self.kind() { + CallKind::Call | CallKind::StaticCall | CallKind::CallCode | CallKind::DelegateCall => { + Action::Call(CallAction { + from: self.trace.caller, + to: self.trace.address, + value: self.trace.value, + gas: self.trace.gas_used.into(), + input: self.trace.data.clone().into(), + call_type: self.kind().into(), + }) + } + CallKind::Create | CallKind::Create2 => Action::Create(CreateAction { + from: self.trace.caller, + value: self.trace.value, + gas: self.trace.gas_used.into(), + init: self.trace.data.clone().into(), + }), + } + } +} + +/// Ordering enum for calls and logs +/// +/// i.e. if Call 0 occurs before Log 0, it will be pushed into the `CallTraceNode`'s ordering before +/// the log. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum LogCallOrder { + Log(usize), + Call(usize), +} + +/// Ethereum log. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct RawLog { + /// Indexed event params are represented as log topics. + pub(crate) topics: Vec, + /// Others are just plain data. + pub(crate) data: Bytes, +} + +/// Represents a tracked call step during execution +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CallTraceStep { + // Fields filled in `step` + /// Call depth + pub depth: u64, + /// Program counter before step execution + pub pc: usize, + /// Opcode to be executed + pub op: OpCode, + /// Current contract address + pub contract: Address, + /// Stack before step execution + pub stack: Stack, + /// Memory before step execution + pub memory: Memory, + /// Remaining gas before step execution + pub gas: u64, + /// Gas refund counter before step execution + pub gas_refund_counter: u64, + // Fields filled in `step_end` + /// Gas cost of step execution + pub gas_cost: u64, + /// Change of the contract state after step execution (effect of the SLOAD/SSTORE instructions) + pub state_diff: Option<(U256, U256)>, + /// Final status of the call + pub status: InstructionResult, +} + +// === impl CallTraceStep === + +impl CallTraceStep { + // Returns true if the status code is an error or revert, See [InstructionResult::Revert] + pub fn is_error(&self) -> bool { + self.status as u8 >= InstructionResult::Revert as u8 + } + + /// Returns the error message if it is an erroneous result. + pub fn as_error(&self) -> Option { + if self.is_error() { + Some(format!("{:?}", self.status)) + } else { + None + } + } +} + +impl From<&CallTraceStep> for StructLog { + fn from(step: &CallTraceStep) -> Self { + StructLog { + depth: step.depth, + error: step.as_error(), + gas: step.gas, + gas_cost: step.gas_cost, + memory: Some(convert_memory(step.memory.data())), + op: step.op.to_string(), + pc: step.pc as u64, + refund_counter: if step.gas_refund_counter > 0 { + Some(step.gas_refund_counter) + } else { + None + }, + stack: Some(step.stack.data().clone()), + // Filled in `CallTraceArena::geth_trace` as a result of compounding all slot changes + storage: None, + } + } +} diff --git a/crates/revm/revm-inspectors/src/tracing/utils.rs b/crates/revm/revm-inspectors/src/tracing/utils.rs new file mode 100644 index 00000000000..64468f82502 --- /dev/null +++ b/crates/revm/revm-inspectors/src/tracing/utils.rs @@ -0,0 +1,40 @@ +//! Util functions for revm related ops + +use reth_primitives::{ + contract::{create2_address_from_code, create_address}, + hex, Address, +}; +use revm::{ + interpreter::CreateInputs, + primitives::{CreateScheme, SpecId}, +}; + +/// creates the memory data in 32byte chunks +/// see +#[inline] +pub(crate) fn convert_memory(data: &[u8]) -> Vec { + let mut memory = Vec::with_capacity((data.len() + 31) / 32); + for idx in (0..data.len()).step_by(32) { + let len = std::cmp::min(idx + 32, data.len()); + memory.push(hex::encode(&data[idx..len])); + } + memory +} + +/// Get the gas used, accounting for refunds +#[inline] +pub(crate) fn gas_used(spec: SpecId, spent: u64, refunded: u64) -> u64 { + let refund_quotient = if SpecId::enabled(spec, SpecId::LONDON) { 5 } else { 2 }; + spent - (refunded).min(spent / refund_quotient) +} + +/// Get the address of a contract creation +#[inline] +pub(crate) fn get_create_address(call: &CreateInputs, nonce: u64) -> Address { + match call.scheme { + CreateScheme::Create => create_address(call.caller, nonce), + CreateScheme::Create2 { salt } => { + create2_address_from_code(call.caller, call.init_code.clone(), salt) + } + } +} diff --git a/crates/rpc/rpc-types/Cargo.toml b/crates/rpc/rpc-types/Cargo.toml index dcf39ef795a..e1abd62a9f6 100644 --- a/crates/rpc/rpc-types/Cargo.toml +++ b/crates/rpc/rpc-types/Cargo.toml @@ -14,6 +14,9 @@ reth-primitives = { path = "../../primitives" } reth-rlp = { path = "../../rlp" } reth-network-api = { path = "../../net/network-api"} +# for geth tracing types +ethers-core = { git = "https://github.com/gakonst/ethers-rs", default-features = false } + # errors thiserror = "1.0" diff --git a/crates/rpc/rpc-types/src/eth/trace/mod.rs b/crates/rpc/rpc-types/src/eth/trace/mod.rs index 9270dbe1e57..cc07e21cb37 100644 --- a/crates/rpc/rpc-types/src/eth/trace/mod.rs +++ b/crates/rpc/rpc-types/src/eth/trace/mod.rs @@ -2,3 +2,51 @@ pub mod filter; pub mod parity; + +/// Geth tracing types +pub mod geth { + #![allow(missing_docs)] + + use reth_primitives::{Bytes, JsonU256, H256, U256}; + use serde::{Deserialize, Serialize}; + use std::collections::BTreeMap; + + // re-exported for geth tracing types + pub use ethers_core::types::GethDebugTracingOptions; + + /// Geth Default trace frame + /// + /// + #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] + #[serde(rename_all = "camelCase")] + pub struct DefaultFrame { + pub failed: bool, + pub gas: JsonU256, + pub return_value: Bytes, + pub struct_logs: Vec, + } + + /// Represents a struct log entry in a trace + /// + /// + #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] + pub struct StructLog { + pub depth: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error: Option, + pub gas: u64, + #[serde(rename = "gasCost")] + pub gas_cost: u64, + /// ref + #[serde(default, skip_serializing_if = "Option::is_none")] + pub memory: Option>, + pub op: String, + pub pc: u64, + #[serde(default, rename = "refund", skip_serializing_if = "Option::is_none")] + pub refund_counter: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stack: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub storage: Option>, + } +} diff --git a/crates/rpc/rpc-types/src/eth/trace/parity.rs b/crates/rpc/rpc-types/src/eth/trace/parity.rs index 80c4359794c..174f1f13d71 100644 --- a/crates/rpc/rpc-types/src/eth/trace/parity.rs +++ b/crates/rpc/rpc-types/src/eth/trace/parity.rs @@ -75,11 +75,26 @@ pub enum Action { Reward(RewardAction), } +/// An external action type. +#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum ActionType { + /// Contract call. + Call, + /// Contract creation. + Create, + /// Contract suicide/selfdestruct. + Selfdestruct, + /// A block reward. + Reward, +} + /// Call type. -#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "lowercase")] pub enum CallType { /// None + #[default] None, /// Call Call, From dd1a0b495f4e23fa795e8dcb52906fcaeb9d037a Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Thu, 9 Mar 2023 22:09:34 -0500 Subject: [PATCH 083/191] chore: remove redundant handshake traces (#1694) --- crates/net/eth-wire/src/ethstream.rs | 1 - crates/net/eth-wire/src/p2pstream.rs | 2 -- 2 files changed, 3 deletions(-) diff --git a/crates/net/eth-wire/src/ethstream.rs b/crates/net/eth-wire/src/ethstream.rs index 605b8e6a675..ae5d60edf3e 100644 --- a/crates/net/eth-wire/src/ethstream.rs +++ b/crates/net/eth-wire/src/ethstream.rs @@ -66,7 +66,6 @@ where let our_status_bytes = our_status_bytes.freeze(); self.inner.send(our_status_bytes).await?; - tracing::trace!("waiting for eth status from peer"); let their_msg_res = self.inner.next().await; let their_msg = match their_msg_res { diff --git a/crates/net/eth-wire/src/p2pstream.rs b/crates/net/eth-wire/src/p2pstream.rs index 9a487cc3783..d3cc751da3d 100644 --- a/crates/net/eth-wire/src/p2pstream.rs +++ b/crates/net/eth-wire/src/p2pstream.rs @@ -90,8 +90,6 @@ where P2PMessage::Hello(hello.clone()).encode(&mut raw_hello_bytes); self.inner.send(raw_hello_bytes.into()).await?; - tracing::trace!("waiting for p2p hello from peer"); - let first_message_bytes = tokio::time::timeout(HANDSHAKE_TIMEOUT, self.inner.next()) .await .or(Err(P2PStreamError::HandshakeError(P2PHandshakeError::Timeout)))? From 9d7975a07a9c7c46679fce95811d872e07ab9c70 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Fri, 10 Mar 2023 03:46:49 -0500 Subject: [PATCH 084/191] fix: error on pow block in forkchoiceUpdated (#1692) --- crates/rpc/rpc-engine-api/src/engine_api.rs | 120 +++++++++++++++++++- 1 file changed, 116 insertions(+), 4 deletions(-) diff --git a/crates/rpc/rpc-engine-api/src/engine_api.rs b/crates/rpc/rpc-engine-api/src/engine_api.rs index 9e6d0b922c8..2e85330f69d 100644 --- a/crates/rpc/rpc-engine-api/src/engine_api.rs +++ b/crates/rpc/rpc-engine-api/src/engine_api.rs @@ -336,6 +336,9 @@ impl finalized.difficulty); + + // set the difficulty so we know it is post-merge + head.difficulty = ttd - U256::from(1) - finalized.difficulty; + let head = head.seal_slow(); + handle.client.extend_headers([ + (head.hash(), head.clone().unseal()), + (finalized.hash(), finalized.clone().unseal()), + ]); + + let state = ForkchoiceState { + head_block_hash: head.hash(), + finalized_block_hash: finalized.hash(), + ..Default::default() + }; + + let (result_tx, result_rx) = oneshot::channel(); + handle.send_message(EngineApiMessage::ForkchoiceUpdated( + EngineApiMessageVersion::V1, + state.clone(), + None, + result_tx, + )); + + let expected_result = ForkchoiceUpdated { + payload_id: None, + payload_status: PayloadStatus { + status: PayloadStatusEnum::Invalid { + validation_error: EngineApiError::PayloadPreMerge.to_string(), + }, + latest_valid_hash: Some(H256::zero()), + }, + }; + assert_matches!(result_rx.await, Ok(Ok(result)) => assert_eq!(result, expected_result)); + + // From the engine API spec: + // + // Additionally, if this validation fails, client software MUST NOT update the + // forkchoice state and MUST NOT begin a payload build process. + assert!(!handle.forkchoice_state_has_changed()); + } } // https://github.com/ethereum/execution-apis/blob/main/src/engine/paris.md#specification-3 From 45c6be4e3d977298fc468fdf6e9937dd85657aee Mon Sep 17 00:00:00 2001 From: Bjerg Date: Fri, 10 Mar 2023 10:13:48 +0100 Subject: [PATCH 085/191] ci: delete project workflow (#1695) --- .github/workflows/project.yml | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 .github/workflows/project.yml diff --git a/.github/workflows/project.yml b/.github/workflows/project.yml deleted file mode 100644 index 4315ebf1aaf..00000000000 --- a/.github/workflows/project.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: project - -on: - issues: - types: - - opened - -jobs: - add: - name: add issue/pr - runs-on: ubuntu-latest - steps: - - uses: actions/add-to-project@v0.4.0 - with: - project-url: https://github.com/orgs/paradigmxyz/projects/1 - github-token: ${{ secrets.GH_PROJECT_TOKEN }} From ba96b9d1656a4460824af94ed459926be47d531d Mon Sep 17 00:00:00 2001 From: chirag-bgh <76247491+chirag-bgh@users.noreply.github.com> Date: Fri, 10 Mar 2023 15:05:59 +0530 Subject: [PATCH 086/191] feat: Add Latest State TransactionValidator implementation (#1498) --- Cargo.lock | 2 + crates/interfaces/src/consensus.rs | 37 ++++- crates/rpc/rpc/src/eth/error.rs | 2 + crates/transaction-pool/Cargo.toml | 2 + crates/transaction-pool/src/error.rs | 10 ++ crates/transaction-pool/src/lib.rs | 48 +++++- crates/transaction-pool/src/pool/mod.rs | 4 +- .../transaction-pool/src/test_utils/mock.rs | 4 + crates/transaction-pool/src/test_utils/mod.rs | 7 +- crates/transaction-pool/src/traits.rs | 8 + crates/transaction-pool/src/validate.rs | 150 +++++++++++++++++- 11 files changed, 257 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3abe264cc10..d66d9ac35c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5149,8 +5149,10 @@ dependencies = [ "parking_lot 0.12.1", "paste", "rand 0.8.5", + "reth-interfaces", "reth-metrics-derive", "reth-primitives", + "reth-provider", "reth-rlp", "ruint", "serde", diff --git a/crates/interfaces/src/consensus.rs b/crates/interfaces/src/consensus.rs index c940733b64b..2c926c89eb6 100644 --- a/crates/interfaces/src/consensus.rs +++ b/crates/interfaces/src/consensus.rs @@ -97,7 +97,7 @@ pub enum Error { #[error("Transaction nonce is not consistent.")] TransactionNonceNotConsistent, #[error("Account does not have enough funds ({available_funds:?}) to cover transaction max fee: {max_fee:?}.")] - InsufficientFunds { max_fee: u128, available_funds: u128 }, + InsufficientFunds { max_fee: u128, available_funds: U256 }, #[error("Eip2930 transaction is enabled after berlin hardfork.")] TransactionEip2930Disabled, #[error("Old legacy transaction before Spurious Dragon should not have chain_id.")] @@ -130,4 +130,39 @@ pub enum Error { WithdrawalIndexInvalid { got: u64, expected: u64 }, #[error("Missing withdrawals")] BodyWithdrawalsMissing, + /// Thrown when calculating gas usage + #[error("gas uint64 overflow")] + GasUintOverflow, + /// returned if the transaction is specified to use less gas than required to start the + /// invocation. + #[error("intrinsic gas too low")] + GasTooLow, + /// returned if the transaction gas exceeds the limit + #[error("intrinsic gas too high")] + GasTooHigh, + /// thrown if a transaction is not supported in the current network configuration. + #[error("transaction type not supported")] + TxTypeNotSupported, + /// Thrown to ensure no one is able to specify a transaction with a tip higher than the total + /// fee cap. + #[error("max priority fee per gas higher than max fee per gas")] + TipAboveFeeCap, + /// A sanity error to avoid huge numbers specified in the tip field. + #[error("max priority fee per gas higher than 2^256-1")] + TipVeryHigh, + /// A sanity error to avoid huge numbers specified in the fee cap field. + #[error("max fee per gas higher than 2^256-1")] + FeeCapVeryHigh, + /// Thrown post London if the transaction's fee is less than the base fee of the block + #[error("max fee per gas less than block base fee")] + FeeCapTooLow, + /// Thrown if the sender of a transaction is a contract. + #[error("sender not an eoa")] + SenderNoEOA, +} + +impl From for Error { + fn from(_: crate::error::Error) -> Self { + Error::TransactionSignerRecoveryError + } } diff --git a/crates/rpc/rpc/src/eth/error.rs b/crates/rpc/rpc/src/eth/error.rs index cca0dd39eba..8fe9eff1f4b 100644 --- a/crates/rpc/rpc/src/eth/error.rs +++ b/crates/rpc/rpc/src/eth/error.rs @@ -296,6 +296,8 @@ impl From for GethTxPoolError { PoolError::DiscardedOnInsert(_) => GethTxPoolError::TxPoolOverflow, PoolError::TxExceedsGasLimit(_, _, _) => GethTxPoolError::GasLimit, PoolError::TxExceedsMaxInitCodeSize(_, _, _) => GethTxPoolError::OversizedData, + PoolError::AccountNotFound(_) => GethTxPoolError::InvalidSender, + PoolError::OversizedData(_, _, _) => GethTxPoolError::OversizedData, } } } diff --git a/crates/transaction-pool/Cargo.toml b/crates/transaction-pool/Cargo.toml index 34a09589b3b..bde4c1da72c 100644 --- a/crates/transaction-pool/Cargo.toml +++ b/crates/transaction-pool/Cargo.toml @@ -19,6 +19,8 @@ normal = [ # reth reth-primitives = { path = "../primitives" } +reth-provider = { path = "../storage/provider" } +reth-interfaces = { path = "../interfaces" } reth-rlp = { path = "../rlp" } # async/futures diff --git a/crates/transaction-pool/src/error.rs b/crates/transaction-pool/src/error.rs index ee8133787aa..4048a37c417 100644 --- a/crates/transaction-pool/src/error.rs +++ b/crates/transaction-pool/src/error.rs @@ -29,6 +29,14 @@ pub enum PoolError { /// respect the max_init_code_size. #[error("[{0:?}] Transaction's size {1} exceeds max_init_code_size {2}.")] TxExceedsMaxInitCodeSize(TxHash, usize, usize), + /// Thrown if the transaction contains an invalid signature + #[error("[{0:?}]: Invalid sender")] + AccountNotFound(TxHash), + /// Thrown if the input data of a transaction is greater + /// than some meaningful limit a user might use. This is not a consensus error + /// making the transaction invalid, rather a DOS protection. + #[error("[{0:?}]: Input data too large")] + OversizedData(TxHash, usize, usize), } // === impl PoolError === @@ -43,6 +51,8 @@ impl PoolError { PoolError::DiscardedOnInsert(hash) => hash, PoolError::TxExceedsGasLimit(hash, _, _) => hash, PoolError::TxExceedsMaxInitCodeSize(hash, _, _) => hash, + PoolError::AccountNotFound(hash) => hash, + PoolError::OversizedData(hash, _, _) => hash, } } } diff --git a/crates/transaction-pool/src/lib.rs b/crates/transaction-pool/src/lib.rs index 3dd3ee29287..11ef1812f6f 100644 --- a/crates/transaction-pool/src/lib.rs +++ b/crates/transaction-pool/src/lib.rs @@ -93,6 +93,8 @@ use crate::{ pool::PoolInner, traits::{NewTransactionEvent, PoolSize}, }; + +use reth_interfaces::consensus::Error; use reth_primitives::{TxHash, U256}; use std::{collections::HashMap, sync::Arc}; use tokio::sync::mpsc::Receiver; @@ -110,6 +112,24 @@ mod validate; /// Common test helpers for mocking A pool pub mod test_utils; +// TX_SLOT_SIZE is used to calculate how many data slots a single transaction +// takes up based on its size. The slots are used as DoS protection, ensuring +// that validating a new transaction remains a constant operation (in reality +// O(maxslots), where max slots are 4 currently). +pub(crate) const TX_SLOT_SIZE: usize = 32 * 1024; + +// TX_MAX_SIZE is the maximum size a single transaction can have. This field has +// non-trivial consequences: larger transactions are significantly harder and +// more expensive to propagate; larger transactions also take more resources +// to validate whether they fit into the pool or not. +pub(crate) const TX_MAX_SIZE: usize = 4 * TX_SLOT_SIZE; //128KB + +// Maximum bytecode to permit for a contract +pub(crate) const MAX_CODE_SIZE: usize = 24576; + +// Maximum initcode to permit in a creation transaction and create instructions +pub(crate) const MAX_INIT_CODE_SIZE: usize = 2 * MAX_CODE_SIZE; + /// A shareable, generic, customizable `TransactionPool` implementation. #[derive(Debug)] pub struct Pool { @@ -144,7 +164,8 @@ where &self, origin: TransactionOrigin, transactions: impl IntoIterator, - ) -> PoolResult>> { + ) -> PoolResult, Error>>> + { let outcome = futures_util::future::join_all( transactions.into_iter().map(|tx| self.validate(origin, tx)), ) @@ -160,10 +181,12 @@ where &self, origin: TransactionOrigin, transaction: V::Transaction, - ) -> (TxHash, TransactionValidationOutcome) { + ) -> (TxHash, Result, Error>) { let hash = *transaction.hash(); + // TODO(mattsse): this is where additional validate checks would go, like banned senders // etc... + let outcome = self.pool.validator().validate_transaction(origin, transaction).await; (hash, outcome) @@ -203,7 +226,22 @@ where transaction: Self::Transaction, ) -> PoolResult { let (_, tx) = self.validate(origin, transaction).await; - self.pool.add_transactions(origin, std::iter::once(tx)).pop().expect("exists; qed") + + match tx { + Ok(TransactionValidationOutcome::Valid { + balance: _, + state_nonce: _, + transaction: _, + }) => self + .pool + .add_transactions(origin, std::iter::once(tx.unwrap())) + .pop() + .expect("exists; qed"), + Ok(TransactionValidationOutcome::Invalid(_transaction, error)) => Err(error), + Err(_err) => { + unimplemented!() + } + } } async fn add_transactions( @@ -212,7 +250,9 @@ where transactions: Vec, ) -> PoolResult>> { let validated = self.validate_all(origin, transactions).await?; - let transactions = self.pool.add_transactions(origin, validated.into_values()); + + let transactions = + self.pool.add_transactions(origin, validated.into_values().map(|x| x.unwrap())); Ok(transactions) } diff --git a/crates/transaction-pool/src/pool/mod.rs b/crates/transaction-pool/src/pool/mod.rs index 334dec5eb29..6ef8fc53128 100644 --- a/crates/transaction-pool/src/pool/mod.rs +++ b/crates/transaction-pool/src/pool/mod.rs @@ -1,7 +1,7 @@ //! Transaction Pool internals. //! -//! Incoming transactions are before they enter the pool first. The validation outcome can have 3 -//! states: +//! Incoming transactions validated are before they enter the pool first. The validation outcome can +//! have 3 states: //! //! 1. Transaction can _never_ be valid //! 2. Transaction is _currently_ valid diff --git a/crates/transaction-pool/src/test_utils/mock.rs b/crates/transaction-pool/src/test_utils/mock.rs index da03552b807..c9d1e5a2393 100644 --- a/crates/transaction-pool/src/test_utils/mock.rs +++ b/crates/transaction-pool/src/test_utils/mock.rs @@ -356,6 +356,10 @@ impl PoolTransaction for MockTransaction { fn encoded_length(&self) -> usize { 0 } + + fn chain_id(&self) -> Option { + Some(1) + } } impl FromRecoveredTransaction for MockTransaction { diff --git a/crates/transaction-pool/src/test_utils/mod.rs b/crates/transaction-pool/src/test_utils/mod.rs index 7fd1da50bbd..52e6ff2c210 100644 --- a/crates/transaction-pool/src/test_utils/mod.rs +++ b/crates/transaction-pool/src/test_utils/mod.rs @@ -9,6 +9,7 @@ use crate::{ }; use async_trait::async_trait; pub use mock::*; +use reth_interfaces::consensus::Error; use std::{marker::PhantomData, sync::Arc}; /// A [Pool] used for testing @@ -36,12 +37,12 @@ impl TransactionValidator for NoopTransactionValidator { &self, origin: TransactionOrigin, transaction: Self::Transaction, - ) -> TransactionValidationOutcome { - TransactionValidationOutcome::Valid { + ) -> Result, Error> { + Ok(TransactionValidationOutcome::Valid { balance: Default::default(), state_nonce: 0, transaction, - } + }) } } diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index 300699e8616..697dbafc713 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -295,6 +295,9 @@ pub trait PoolTransaction: fmt::Debug + Send + Sync + FromRecoveredTransaction { /// Returns the length of the rlp encoded object fn encoded_length(&self) -> usize; + + /// Returns chain_id + fn chain_id(&self) -> Option; } /// The default [PoolTransaction] for the [Pool](crate::Pool). @@ -397,6 +400,11 @@ impl PoolTransaction for PooledTransaction { fn encoded_length(&self) -> usize { self.transaction.length() } + + /// Returns chain_id + fn chain_id(&self) -> Option { + self.transaction.chain_id() + } } impl FromRecoveredTransaction for PooledTransaction { diff --git a/crates/transaction-pool/src/validate.rs b/crates/transaction-pool/src/validate.rs index 7200d4f8501..b2ebf339da7 100644 --- a/crates/transaction-pool/src/validate.rs +++ b/crates/transaction-pool/src/validate.rs @@ -4,8 +4,14 @@ use crate::{ error::PoolError, identifier::{SenderId, TransactionId}, traits::{PoolTransaction, TransactionOrigin}, + MAX_INIT_CODE_SIZE, TX_MAX_SIZE, }; -use reth_primitives::{Address, TransactionKind, TxHash, U256}; +use reth_interfaces::consensus::Error; +use reth_primitives::{ + Address, TransactionKind, TxHash, EIP1559_TX_TYPE_ID, EIP2930_TX_TYPE_ID, LEGACY_TX_TYPE_ID, + U256, +}; +use reth_provider::AccountProvider; use std::{fmt, time::Instant}; /// A Result type returned after checking a transaction's validity. @@ -53,7 +59,7 @@ pub trait TransactionValidator: Send + Sync { &self, origin: TransactionOrigin, transaction: Self::Transaction, - ) -> TransactionValidationOutcome; + ) -> Result, Error>; /// Ensure that the code size is not greater than `max_init_code_size`. /// `max_init_code_size` should be configurable so this will take it as an argument. @@ -62,11 +68,6 @@ pub trait TransactionValidator: Send + Sync { transaction: Self::Transaction, max_init_code_size: usize, ) -> Result<(), PoolError> { - // TODO check whether we are in the Shanghai stage. - // if !self.shanghai { - // return Ok(()) - // } - if *transaction.kind() == TransactionKind::Create && transaction.size() > max_init_code_size { Err(PoolError::TxExceedsMaxInitCodeSize( @@ -80,6 +81,141 @@ pub trait TransactionValidator: Send + Sync { } } +/// TODO: Add docs and make this public +pub(crate) struct EthTransactionValidatorConfig { + /// Chain id + chain_id: u64, + /// This type fetches account info from the db + client: Client, + /// Fork indicator whether we are in the Shanghai stage. + shanghai: bool, + /// Fork indicator whether we are using EIP-2718 type transactions. + eip2718: bool, + /// Fork indicator whether we are using EIP-1559 type transactions. + eip1559: bool, + /// The current max gas limit + current_max_gas_limit: u64, + /// gasprice + gas_price: Option, +} + +#[async_trait::async_trait] +impl TransactionValidator + for EthTransactionValidatorConfig +{ + type Transaction = T; + + async fn validate_transaction( + &self, + origin: TransactionOrigin, + transaction: Self::Transaction, + ) -> Result, Error> { + // Checks for tx_type + match transaction.tx_type() { + LEGACY_TX_TYPE_ID => { + // Accept legacy transactions + } + + EIP2930_TX_TYPE_ID => { + // Accept only legacy transactions until EIP-2718/2930 activates + if !self.eip2718 { + return Err(Error::TransactionEip2930Disabled) + } + } + + EIP1559_TX_TYPE_ID => { + // Reject dynamic fee transactions until EIP-1559 activates. + if !self.eip1559 { + return Err(Error::TransactionEip1559Disabled) + } + } + + _ => return Err(Error::TxTypeNotSupported), + }; + + // Reject transactions over defined size to prevent DOS attacks + if transaction.size() > TX_MAX_SIZE { + return Ok(TransactionValidationOutcome::Invalid( + transaction.clone(), + PoolError::OversizedData(*transaction.hash(), transaction.size(), TX_MAX_SIZE), + )) + } + + // Check whether the init code size has been exceeded. + if self.shanghai { + match self.ensure_max_init_code_size(transaction.clone(), MAX_INIT_CODE_SIZE) { + Ok(_) => {} + Err(e) => return Ok(TransactionValidationOutcome::Invalid(transaction, e)), + } + } + + // Checks for gas limit + if transaction.gas_limit() > self.current_max_gas_limit { + return Ok(TransactionValidationOutcome::Invalid( + transaction.clone(), + PoolError::TxExceedsGasLimit( + *transaction.hash(), + transaction.gas_limit(), + self.current_max_gas_limit, + ), + )) + } + + // Ensure max_fee_per_gas is greater than or equal to max_priority_fee_per_gas. + if transaction.max_fee_per_gas() <= transaction.max_priority_fee_per_gas() { + return Err(Error::TipAboveFeeCap) + } + + // Drop non-local transactions under our own minimal accepted gas price or tip + if !origin.is_local() && transaction.max_fee_per_gas() < self.gas_price { + return Err(Error::TransactionMaxFeeLessThenBaseFee) + } + + // Checks for chainid + if transaction.chain_id() != Some(self.chain_id) { + return Err(Error::TransactionChainId) + } + + let account = match self.client.basic_account(transaction.sender())? { + Some(account) => { + // Signer account shouldn't have bytecode. Presence of bytecode means this is a + // smartcontract. + if account.has_bytecode() { + return Err(Error::SignerAccountHasBytecode) + } else { + account + } + } + None => { + return Ok(TransactionValidationOutcome::Invalid( + transaction.clone(), + PoolError::AccountNotFound(*transaction.hash()), + )) + } + }; + + // Checks for nonce + if transaction.nonce() < account.nonce { + return Err(Error::TransactionNonceNotConsistent) + } + + // Checks for max cost + if transaction.cost() > account.balance { + return Err(Error::InsufficientFunds { + max_fee: transaction.max_fee_per_gas().unwrap_or_default(), + available_funds: account.balance, + }) + } + + // Return the valid transaction + Ok(TransactionValidationOutcome::Valid { + balance: account.balance, + state_nonce: account.nonce, + transaction, + }) + } +} + /// A valid transaction in the pool. pub struct ValidPoolTransaction { /// The transaction From ad5f9aa78ce04d263e93eddcccc13db64763e0c8 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 10 Mar 2023 13:43:04 +0100 Subject: [PATCH 087/191] refactor: extract transaction consensus errors to standalone type (#1697) --- .../consensus/src/beacon/beacon_consensus.rs | 18 ++-- crates/consensus/src/validation.rs | 97 +++++++++++-------- crates/interfaces/src/consensus.rs | 76 +++------------ crates/interfaces/src/error.rs | 2 +- crates/interfaces/src/p2p/error.rs | 4 +- crates/interfaces/src/test_utils/headers.rs | 18 ++-- crates/net/network/src/import.rs | 2 +- crates/primitives/src/lib.rs | 6 +- crates/primitives/src/transaction/error.rs | 47 +++++++++ crates/primitives/src/transaction/mod.rs | 2 + crates/stages/src/error.rs | 2 +- crates/stages/src/pipeline/mod.rs | 2 +- crates/stages/src/stages/merkle.rs | 7 +- crates/stages/src/stages/total_difficulty.rs | 8 +- crates/transaction-pool/src/lib.rs | 9 +- crates/transaction-pool/src/test_utils/mod.rs | 4 +- crates/transaction-pool/src/validate.rs | 33 ++++--- 17 files changed, 183 insertions(+), 154 deletions(-) create mode 100644 crates/primitives/src/transaction/error.rs diff --git a/crates/consensus/src/beacon/beacon_consensus.rs b/crates/consensus/src/beacon/beacon_consensus.rs index 779073d5031..b2efe58b093 100644 --- a/crates/consensus/src/beacon/beacon_consensus.rs +++ b/crates/consensus/src/beacon/beacon_consensus.rs @@ -1,6 +1,6 @@ //! Consensus for ethereum network use crate::validation; -use reth_interfaces::consensus::{Consensus, Error, ForkchoiceState}; +use reth_interfaces::consensus::{Consensus, ConsensusError, ForkchoiceState}; use reth_primitives::{ChainSpec, Hardfork, SealedBlock, SealedHeader, EMPTY_OMMER_ROOT, U256}; use tokio::sync::watch; @@ -42,28 +42,32 @@ impl Consensus for BeaconConsensus { &self, header: &SealedHeader, parent: &SealedHeader, - ) -> Result<(), Error> { + ) -> Result<(), ConsensusError> { validation::validate_header_standalone(header, &self.chain_spec)?; validation::validate_header_regarding_parent(parent, header, &self.chain_spec)?; Ok(()) } - fn validate_header(&self, header: &SealedHeader, total_difficulty: U256) -> Result<(), Error> { + fn validate_header( + &self, + header: &SealedHeader, + total_difficulty: U256, + ) -> Result<(), ConsensusError> { if self.chain_spec.fork(Hardfork::Paris).active_at_ttd(total_difficulty, header.difficulty) { // EIP-3675: Upgrade consensus to Proof-of-Stake: // https://eips.ethereum.org/EIPS/eip-3675#replacing-difficulty-with-0 if header.difficulty != U256::ZERO { - return Err(Error::TheMergeDifficultyIsNotZero) + return Err(ConsensusError::TheMergeDifficultyIsNotZero) } if header.nonce != 0 { - return Err(Error::TheMergeNonceIsNotZero) + return Err(ConsensusError::TheMergeNonceIsNotZero) } if header.ommers_hash != EMPTY_OMMER_ROOT { - return Err(Error::TheMergeOmmerRootIsNotEmpty) + return Err(ConsensusError::TheMergeOmmerRootIsNotEmpty) } // mixHash is used instead of difficulty inside EVM @@ -77,7 +81,7 @@ impl Consensus for BeaconConsensus { Ok(()) } - fn pre_validate_block(&self, block: &SealedBlock) -> Result<(), Error> { + fn pre_validate_block(&self, block: &SealedBlock) -> Result<(), ConsensusError> { validation::validate_block_standalone(block, &self.chain_spec) } diff --git a/crates/consensus/src/validation.rs b/crates/consensus/src/validation.rs index b786fd618be..ff4310757ba 100644 --- a/crates/consensus/src/validation.rs +++ b/crates/consensus/src/validation.rs @@ -1,8 +1,8 @@ //! Collection of methods for block validation. -use reth_interfaces::{consensus::Error, Result as RethResult}; +use reth_interfaces::{consensus::ConsensusError, Result as RethResult}; use reth_primitives::{ - BlockNumber, ChainSpec, Hardfork, Header, SealedBlock, SealedHeader, Transaction, - TransactionSignedEcRecovered, TxEip1559, TxEip2930, TxLegacy, + BlockNumber, ChainSpec, Hardfork, Header, InvalidTransactionError, SealedBlock, SealedHeader, + Transaction, TransactionSignedEcRecovered, TxEip1559, TxEip2930, TxLegacy, }; use reth_provider::{AccountProvider, HeaderProvider}; use std::{ @@ -16,10 +16,10 @@ use reth_primitives::constants; pub fn validate_header_standalone( header: &SealedHeader, chain_spec: &ChainSpec, -) -> Result<(), Error> { +) -> Result<(), ConsensusError> { // Gas used needs to be less then gas limit. Gas used is going to be check after execution. if header.gas_used > header.gas_limit { - return Err(Error::HeaderGasUsedExceedsGasLimit { + return Err(ConsensusError::HeaderGasUsedExceedsGasLimit { gas_used: header.gas_used, gas_limit: header.gas_limit, }) @@ -29,31 +29,34 @@ pub fn validate_header_standalone( let present_timestamp = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH).unwrap().as_secs(); if header.timestamp > present_timestamp { - return Err(Error::TimestampIsInFuture { timestamp: header.timestamp, present_timestamp }) + return Err(ConsensusError::TimestampIsInFuture { + timestamp: header.timestamp, + present_timestamp, + }) } // From yellow paper: extraData: An arbitrary byte array containing data // relevant to this block. This must be 32 bytes or fewer; formally Hx. if header.extra_data.len() > 32 { - return Err(Error::ExtraDataExceedsMax { len: header.extra_data.len() }) + return Err(ConsensusError::ExtraDataExceedsMax { len: header.extra_data.len() }) } // Check if base fee is set. if chain_spec.fork(Hardfork::London).active_at_block(header.number) && header.base_fee_per_gas.is_none() { - return Err(Error::BaseFeeMissing) + return Err(ConsensusError::BaseFeeMissing) } // EIP-4895: Beacon chain push withdrawals as operations if chain_spec.fork(Hardfork::Shanghai).active_at_timestamp(header.timestamp) && header.withdrawals_root.is_none() { - return Err(Error::WithdrawalsRootMissing) + return Err(ConsensusError::WithdrawalsRootMissing) } else if !chain_spec.fork(Hardfork::Shanghai).active_at_timestamp(header.timestamp) && header.withdrawals_root.is_some() { - return Err(Error::WithdrawalsRootUnexpected) + return Err(ConsensusError::WithdrawalsRootUnexpected) } Ok(()) @@ -67,21 +70,21 @@ pub fn validate_transaction_regarding_header( chain_spec: &ChainSpec, at_block_number: BlockNumber, base_fee: Option, -) -> Result<(), Error> { +) -> Result<(), ConsensusError> { let chain_id = match transaction { Transaction::Legacy(TxLegacy { chain_id, .. }) => { // EIP-155: Simple replay attack protection: https://eips.ethereum.org/EIPS/eip-155 if chain_spec.fork(Hardfork::SpuriousDragon).active_at_block(at_block_number) && chain_id.is_some() { - return Err(Error::TransactionOldLegacyChainId) + return Err(InvalidTransactionError::OldLegacyChainId.into()) } *chain_id } Transaction::Eip2930(TxEip2930 { chain_id, .. }) => { // EIP-2930: Optional access lists: https://eips.ethereum.org/EIPS/eip-2930 (New transaction type) if !chain_spec.fork(Hardfork::Berlin).active_at_block(at_block_number) { - return Err(Error::TransactionEip2930Disabled) + return Err(InvalidTransactionError::Eip2930Disabled.into()) } Some(*chain_id) } @@ -93,13 +96,13 @@ pub fn validate_transaction_regarding_header( }) => { // EIP-1559: Fee market change for ETH 1.0 chain https://eips.ethereum.org/EIPS/eip-1559 if !chain_spec.fork(Hardfork::Berlin).active_at_block(at_block_number) { - return Err(Error::TransactionEip1559Disabled) + return Err(InvalidTransactionError::Eip1559Disabled.into()) } // EIP-1559: add more constraints to the tx validation // https://github.com/ethereum/EIPs/pull/3594 if max_priority_fee_per_gas > max_fee_per_gas { - return Err(Error::TransactionPriorityFeeMoreThenMaxFee) + return Err(InvalidTransactionError::PriorityFeeMoreThenMaxFee.into()) } Some(*chain_id) @@ -107,14 +110,14 @@ pub fn validate_transaction_regarding_header( }; if let Some(chain_id) = chain_id { if chain_id != chain_spec.chain().id() { - return Err(Error::TransactionChainId) + return Err(InvalidTransactionError::ChainIdMismatch.into()) } } // Check basefee and few checks that are related to that. // https://github.com/ethereum/EIPs/pull/3594 if let Some(base_fee_per_gas) = base_fee { if transaction.max_fee_per_gas() < base_fee_per_gas as u128 { - return Err(Error::TransactionMaxFeeLessThenBaseFee) + return Err(InvalidTransactionError::MaxFeeLessThenBaseFee.into()) } } @@ -155,7 +158,10 @@ pub fn validate_all_transaction_regarding_block_and_nonces< // Signer account shouldn't have bytecode. Presence of bytecode means this is a // smartcontract. if account.has_bytecode() { - return Err(Error::SignerAccountHasBytecode.into()) + return Err(ConsensusError::from( + InvalidTransactionError::SignerAccountHasBytecode, + ) + .into()) } let nonce = account.nonce; entry.insert(account.nonce + 1); @@ -165,7 +171,7 @@ pub fn validate_all_transaction_regarding_block_and_nonces< // check nonce if transaction.nonce() != nonce { - return Err(Error::TransactionNonceNotConsistent.into()) + return Err(ConsensusError::from(InvalidTransactionError::NonceNotConsistent).into()) } } @@ -178,13 +184,16 @@ pub fn validate_all_transaction_regarding_block_and_nonces< /// - Compares the transactions root in the block header to the block body /// - Pre-execution transaction validation /// - (Optionally) Compares the receipts root in the block header to the block body -pub fn validate_block_standalone(block: &SealedBlock, chain_spec: &ChainSpec) -> Result<(), Error> { +pub fn validate_block_standalone( + block: &SealedBlock, + chain_spec: &ChainSpec, +) -> Result<(), ConsensusError> { // Check ommers hash // TODO(onbjerg): This should probably be accessible directly on [Block] let ommers_hash = reth_primitives::proofs::calculate_ommers_root(block.ommers.iter().map(|h| h.as_ref())); if block.header.ommers_hash != ommers_hash { - return Err(Error::BodyOmmersHashDiff { + return Err(ConsensusError::BodyOmmersHashDiff { got: ommers_hash, expected: block.header.ommers_hash, }) @@ -194,7 +203,7 @@ pub fn validate_block_standalone(block: &SealedBlock, chain_spec: &ChainSpec) -> // TODO(onbjerg): This should probably be accessible directly on [Block] let transaction_root = reth_primitives::proofs::calculate_transaction_root(block.body.iter()); if block.header.transactions_root != transaction_root { - return Err(Error::BodyTransactionRootDiff { + return Err(ConsensusError::BodyTransactionRootDiff { got: transaction_root, expected: block.header.transactions_root, }) @@ -202,13 +211,14 @@ pub fn validate_block_standalone(block: &SealedBlock, chain_spec: &ChainSpec) -> // EIP-4895: Beacon chain push withdrawals as operations if chain_spec.fork(Hardfork::Shanghai).active_at_timestamp(block.timestamp) { - let withdrawals = block.withdrawals.as_ref().ok_or(Error::BodyWithdrawalsMissing)?; + let withdrawals = + block.withdrawals.as_ref().ok_or(ConsensusError::BodyWithdrawalsMissing)?; let withdrawals_root = reth_primitives::proofs::calculate_withdrawals_root(withdrawals.iter()); let header_withdrawals_root = - block.withdrawals_root.as_ref().ok_or(Error::WithdrawalsRootMissing)?; + block.withdrawals_root.as_ref().ok_or(ConsensusError::WithdrawalsRootMissing)?; if withdrawals_root != *header_withdrawals_root { - return Err(Error::BodyWithdrawalsRootDiff { + return Err(ConsensusError::BodyWithdrawalsRootDiff { got: withdrawals_root, expected: *header_withdrawals_root, }) @@ -220,7 +230,10 @@ pub fn validate_block_standalone(block: &SealedBlock, chain_spec: &ChainSpec) -> for withdrawal in withdrawals.iter().skip(1) { let expected = prev_index + 1; if expected != withdrawal.index { - return Err(Error::WithdrawalIndexInvalid { got: withdrawal.index, expected }) + return Err(ConsensusError::WithdrawalIndexInvalid { + got: withdrawal.index, + expected, + }) } prev_index = withdrawal.index; } @@ -261,10 +274,10 @@ pub fn validate_header_regarding_parent( parent: &SealedHeader, child: &SealedHeader, chain_spec: &ChainSpec, -) -> Result<(), Error> { +) -> Result<(), ConsensusError> { // Parent number is consistent. if parent.number + 1 != child.number { - return Err(Error::ParentBlockNumberMismatch { + return Err(ConsensusError::ParentBlockNumberMismatch { parent_block_number: parent.number, block_number: child.number, }) @@ -272,7 +285,7 @@ pub fn validate_header_regarding_parent( // timestamp in past check if child.timestamp < parent.timestamp { - return Err(Error::TimestampIsInPast { + return Err(ConsensusError::TimestampIsInPast { parent_timestamp: parent.timestamp, timestamp: child.timestamp, }) @@ -295,13 +308,13 @@ pub fn validate_header_regarding_parent( // Check gas limit, max diff between child/parent gas_limit should be max_diff=parent_gas/1024 if child.gas_limit > parent_gas_limit { if child.gas_limit - parent_gas_limit >= parent_gas_limit / 1024 { - return Err(Error::GasLimitInvalidIncrease { + return Err(ConsensusError::GasLimitInvalidIncrease { parent_gas_limit, child_gas_limit: child.gas_limit, }) } } else if parent_gas_limit - child.gas_limit >= parent_gas_limit / 1024 { - return Err(Error::GasLimitInvalidDecrease { + return Err(ConsensusError::GasLimitInvalidDecrease { parent_gas_limit, child_gas_limit: child.gas_limit, }) @@ -309,7 +322,7 @@ pub fn validate_header_regarding_parent( // EIP-1559 check base fee if chain_spec.fork(Hardfork::London).active_at_block(child.number) { - let base_fee = child.base_fee_per_gas.ok_or(Error::BaseFeeMissing)?; + let base_fee = child.base_fee_per_gas.ok_or(ConsensusError::BaseFeeMissing)?; let expected_base_fee = if chain_spec.fork(Hardfork::London).transitions_at_block(child.number) { @@ -319,11 +332,11 @@ pub fn validate_header_regarding_parent( calculate_next_block_base_fee( parent.gas_used, parent.gas_limit, - parent.base_fee_per_gas.ok_or(Error::BaseFeeMissing)?, + parent.base_fee_per_gas.ok_or(ConsensusError::BaseFeeMissing)?, ) }; if expected_base_fee != base_fee { - return Err(Error::BaseFeeDiff { expected: expected_base_fee, got: base_fee }) + return Err(ConsensusError::BaseFeeDiff { expected: expected_base_fee, got: base_fee }) } } @@ -345,13 +358,13 @@ pub fn validate_block_regarding_chain( // Check if block is known. if provider.is_known(&hash)? { - return Err(Error::BlockKnown { hash, number: block.header.number }.into()) + return Err(ConsensusError::BlockKnown { hash, number: block.header.number }.into()) } // Check if parent is known. let parent = provider .header(&block.parent_hash)? - .ok_or(Error::ParentUnknown { hash: block.parent_hash })?; + .ok_or(ConsensusError::ParentUnknown { hash: block.parent_hash })?; // Return parent header. Ok(parent.seal(block.parent_hash)) @@ -372,7 +385,7 @@ pub fn full_validation( let transactions = block .body .iter() - .map(|tx| tx.try_ecrecovered().ok_or(Error::TransactionSignerRecoveryError)) + .map(|tx| tx.try_ecrecovered().ok_or(ConsensusError::TransactionSignerRecoveryError)) .collect::, _>>()?; validate_all_transaction_regarding_block_and_nonces( @@ -543,7 +556,7 @@ mod tests { assert_eq!( full_validation(&block, provider, &MAINNET), - Err(Error::BlockKnown { hash: block.hash(), number: block.number }.into()), + Err(ConsensusError::BlockKnown { hash: block.hash(), number: block.number }.into()), "Should fail with error" ); } @@ -579,7 +592,7 @@ mod tests { provider, &MAINNET, ), - Err(Error::TransactionNonceNotConsistent.into()) + Err(ConsensusError::from(InvalidTransactionError::NonceNotConsistent).into()) ) } @@ -598,7 +611,7 @@ mod tests { provider, &MAINNET, ), - Err(Error::TransactionNonceNotConsistent.into()) + Err(ConsensusError::from(InvalidTransactionError::NonceNotConsistent).into()) ); } @@ -636,12 +649,12 @@ mod tests { let block = create_block_with_withdrawals(&[100, 102]); assert_matches!( validate_block_standalone(&block, &chain_spec), - Err(Error::WithdrawalIndexInvalid { .. }) + Err(ConsensusError::WithdrawalIndexInvalid { .. }) ); let block = create_block_with_withdrawals(&[5, 6, 7, 9]); assert_matches!( validate_block_standalone(&block, &chain_spec), - Err(Error::WithdrawalIndexInvalid { .. }) + Err(ConsensusError::WithdrawalIndexInvalid { .. }) ); } diff --git a/crates/interfaces/src/consensus.rs b/crates/interfaces/src/consensus.rs index 2c926c89eb6..6886da85e44 100644 --- a/crates/interfaces/src/consensus.rs +++ b/crates/interfaces/src/consensus.rs @@ -1,5 +1,7 @@ use async_trait::async_trait; -use reth_primitives::{BlockHash, BlockNumber, SealedBlock, SealedHeader, H256, U256}; +use reth_primitives::{ + BlockHash, BlockNumber, InvalidTransactionError, SealedBlock, SealedHeader, H256, U256, +}; use std::fmt::Debug; use tokio::sync::watch::Receiver; @@ -23,13 +25,17 @@ pub trait Consensus: Debug + Send + Sync { &self, header: &SealedHeader, parent: &SealedHeader, - ) -> Result<(), Error>; + ) -> Result<(), ConsensusError>; /// Validate if the header is correct and follows the consensus specification, including /// computed properties (like total difficulty). /// /// Some consensus engines may want to do additional checks here. - fn validate_header(&self, header: &SealedHeader, total_difficulty: U256) -> Result<(), Error>; + fn validate_header( + &self, + header: &SealedHeader, + total_difficulty: U256, + ) -> Result<(), ConsensusError>; /// Validate a block disregarding world state, i.e. things that can be checked before sender /// recovery and execution. @@ -38,7 +44,7 @@ pub trait Consensus: Debug + Send + Sync { /// 11.1 "Ommer Validation". /// /// **This should not be called for the genesis block**. - fn pre_validate_block(&self, block: &SealedBlock) -> Result<(), Error>; + fn pre_validate_block(&self, block: &SealedBlock) -> Result<(), ConsensusError>; /// After the Merge (aka Paris) block rewards became obsolete. /// @@ -51,7 +57,7 @@ pub trait Consensus: Debug + Send + Sync { /// Consensus Errors #[allow(missing_docs)] #[derive(thiserror::Error, Debug, PartialEq, Eq, Clone)] -pub enum Error { +pub enum ConsensusError { #[error("Block used gas ({gas_used:?}) is greater than gas limit ({gas_limit:?}).")] HeaderGasUsedExceedsGasLimit { gas_used: u64, gas_limit: u64 }, #[error("Block ommer hash ({got:?}) is different from expected: ({expected:?})")] @@ -71,7 +77,7 @@ pub enum Error { #[error("Block number {block_number:?} is mismatch with parent block number {parent_block_number:?}")] ParentBlockNumberMismatch { parent_block_number: BlockNumber, block_number: BlockNumber }, #[error( - "Block timestamp {timestamp:?} is in past in comparison with parent timestamp {parent_timestamp:?}." + "Block timestamp {timestamp:?} is in past in comparison with parent timestamp {parent_timestamp:?}." )] TimestampIsInPast { parent_timestamp: u64, timestamp: u64 }, #[error("Block timestamp {timestamp:?} is in future in comparison of our clock time {present_timestamp:?}.")] @@ -84,26 +90,6 @@ pub enum Error { BaseFeeMissing, #[error("Block base fee ({got:?}) is different then expected: ({expected:?}).")] BaseFeeDiff { expected: u64, got: u64 }, - #[error("Transaction eip1559 priority fee is more then max fee.")] - TransactionPriorityFeeMoreThenMaxFee, - #[error("Transaction chain_id does not match.")] - TransactionChainId, - #[error("Transaction max fee is less them block base fee.")] - TransactionMaxFeeLessThenBaseFee, - #[error("Transaction signer does not have account.")] - SignerAccountNotExisting, - #[error("Transaction signer has bytecode set.")] - SignerAccountHasBytecode, - #[error("Transaction nonce is not consistent.")] - TransactionNonceNotConsistent, - #[error("Account does not have enough funds ({available_funds:?}) to cover transaction max fee: {max_fee:?}.")] - InsufficientFunds { max_fee: u128, available_funds: U256 }, - #[error("Eip2930 transaction is enabled after berlin hardfork.")] - TransactionEip2930Disabled, - #[error("Old legacy transaction before Spurious Dragon should not have chain_id.")] - TransactionOldLegacyChainId, - #[error("Eip2930 transaction is enabled after london hardfork.")] - TransactionEip1559Disabled, #[error("Transaction signer recovery error.")] TransactionSignerRecoveryError, #[error( @@ -130,39 +116,7 @@ pub enum Error { WithdrawalIndexInvalid { got: u64, expected: u64 }, #[error("Missing withdrawals")] BodyWithdrawalsMissing, - /// Thrown when calculating gas usage - #[error("gas uint64 overflow")] - GasUintOverflow, - /// returned if the transaction is specified to use less gas than required to start the - /// invocation. - #[error("intrinsic gas too low")] - GasTooLow, - /// returned if the transaction gas exceeds the limit - #[error("intrinsic gas too high")] - GasTooHigh, - /// thrown if a transaction is not supported in the current network configuration. - #[error("transaction type not supported")] - TxTypeNotSupported, - /// Thrown to ensure no one is able to specify a transaction with a tip higher than the total - /// fee cap. - #[error("max priority fee per gas higher than max fee per gas")] - TipAboveFeeCap, - /// A sanity error to avoid huge numbers specified in the tip field. - #[error("max priority fee per gas higher than 2^256-1")] - TipVeryHigh, - /// A sanity error to avoid huge numbers specified in the fee cap field. - #[error("max fee per gas higher than 2^256-1")] - FeeCapVeryHigh, - /// Thrown post London if the transaction's fee is less than the base fee of the block - #[error("max fee per gas less than block base fee")] - FeeCapTooLow, - /// Thrown if the sender of a transaction is a contract. - #[error("sender not an eoa")] - SenderNoEOA, -} - -impl From for Error { - fn from(_: crate::error::Error) -> Self { - Error::TransactionSignerRecoveryError - } + /// Error for a transaction that violates consensus. + #[error(transparent)] + InvalidTransaction(#[from] InvalidTransactionError), } diff --git a/crates/interfaces/src/error.rs b/crates/interfaces/src/error.rs index 6b27e906bba..746dffcb559 100644 --- a/crates/interfaces/src/error.rs +++ b/crates/interfaces/src/error.rs @@ -9,7 +9,7 @@ pub enum Error { Execution(#[from] crate::executor::Error), #[error(transparent)] - Consensus(#[from] crate::consensus::Error), + Consensus(#[from] crate::consensus::ConsensusError), #[error(transparent)] Database(#[from] crate::db::Error), diff --git a/crates/interfaces/src/p2p/error.rs b/crates/interfaces/src/p2p/error.rs index 92922e40b00..78623328a62 100644 --- a/crates/interfaces/src/p2p/error.rs +++ b/crates/interfaces/src/p2p/error.rs @@ -125,7 +125,7 @@ pub enum DownloadError { hash: H256, /// The details of validation failure #[source] - error: consensus::Error, + error: consensus::ConsensusError, }, /// Error when checking that the current [`Header`] has the parent's hash as the parent_hash /// field, and that they have sequential block numbers. @@ -172,7 +172,7 @@ pub enum DownloadError { hash: H256, /// The details of validation failure #[source] - error: consensus::Error, + error: consensus::ConsensusError, }, /// Received more bodies than requested. #[error("Received more bodies than requested. Expected: {expected}. Received: {received}")] diff --git a/crates/interfaces/src/test_utils/headers.rs b/crates/interfaces/src/test_utils/headers.rs index eaa8f804de3..d4f7fce563b 100644 --- a/crates/interfaces/src/test_utils/headers.rs +++ b/crates/interfaces/src/test_utils/headers.rs @@ -1,6 +1,6 @@ //! Testing support for headers related interfaces. use crate::{ - consensus::{self, Consensus, Error}, + consensus::{self, Consensus, ConsensusError}, p2p::{ download::DownloadClient, error::{DownloadError, DownloadResult, PeerRequestResult, RequestError}, @@ -331,25 +331,29 @@ impl Consensus for TestConsensus { &self, header: &SealedHeader, parent: &SealedHeader, - ) -> Result<(), Error> { + ) -> Result<(), ConsensusError> { if self.fail_validation() { - Err(consensus::Error::BaseFeeMissing) + Err(consensus::ConsensusError::BaseFeeMissing) } else { Ok(()) } } - fn validate_header(&self, header: &SealedHeader, total_difficulty: U256) -> Result<(), Error> { + fn validate_header( + &self, + header: &SealedHeader, + total_difficulty: U256, + ) -> Result<(), ConsensusError> { if self.fail_validation() { - Err(consensus::Error::BaseFeeMissing) + Err(consensus::ConsensusError::BaseFeeMissing) } else { Ok(()) } } - fn pre_validate_block(&self, _block: &SealedBlock) -> Result<(), consensus::Error> { + fn pre_validate_block(&self, _block: &SealedBlock) -> Result<(), consensus::ConsensusError> { if self.fail_validation() { - Err(consensus::Error::BaseFeeMissing) + Err(consensus::ConsensusError::BaseFeeMissing) } else { Ok(()) } diff --git a/crates/net/network/src/import.rs b/crates/net/network/src/import.rs index 87f7e30ded1..2146772c2ae 100644 --- a/crates/net/network/src/import.rs +++ b/crates/net/network/src/import.rs @@ -47,7 +47,7 @@ pub enum BlockValidation { pub enum BlockImportError { /// Consensus error #[error(transparent)] - Consensus(#[from] reth_interfaces::consensus::Error), + Consensus(#[from] reth_interfaces::consensus::ConsensusError), } /// An implementation of `BlockImport` used in Proof-of-Stake consensus that does nothing. diff --git a/crates/primitives/src/lib.rs b/crates/primitives/src/lib.rs index 59f969e184a..c4b8a8a1c4d 100644 --- a/crates/primitives/src/lib.rs +++ b/crates/primitives/src/lib.rs @@ -61,9 +61,9 @@ pub use serde_helper::JsonU256; pub use storage::{StorageEntry, StorageTrieEntry}; pub use transaction::{ AccessList, AccessListItem, AccessListWithGasUsed, FromRecoveredTransaction, - IntoRecoveredTransaction, Signature, Transaction, TransactionKind, TransactionSigned, - TransactionSignedEcRecovered, TxEip1559, TxEip2930, TxLegacy, TxType, EIP1559_TX_TYPE_ID, - EIP2930_TX_TYPE_ID, LEGACY_TX_TYPE_ID, + IntoRecoveredTransaction, InvalidTransactionError, Signature, Transaction, TransactionKind, + TransactionSigned, TransactionSignedEcRecovered, TxEip1559, TxEip2930, TxLegacy, TxType, + EIP1559_TX_TYPE_ID, EIP2930_TX_TYPE_ID, LEGACY_TX_TYPE_ID, }; pub use withdrawal::Withdrawal; diff --git a/crates/primitives/src/transaction/error.rs b/crates/primitives/src/transaction/error.rs new file mode 100644 index 00000000000..649ecbe36b4 --- /dev/null +++ b/crates/primitives/src/transaction/error.rs @@ -0,0 +1,47 @@ +use crate::U256; + +/// Represents error variants that can happen when trying to validate a +/// [Transaction](crate::Transaction) +#[allow(missing_docs)] +#[derive(Debug, Clone, Eq, PartialEq, thiserror::Error)] +pub enum InvalidTransactionError { + #[error("Transaction eip1559 priority fee is more then max fee.")] + PriorityFeeMoreThenMaxFee, + #[error("Account does not have enough funds ({available_funds:?}) to cover transaction max fee: {max_fee:?}.")] + InsufficientFunds { max_fee: u128, available_funds: U256 }, + #[error("Transaction nonce is not consistent.")] + NonceNotConsistent, + #[error("Old legacy transaction before Spurious Dragon should not have chain_id.")] + OldLegacyChainId, + #[error("Transaction chain_id does not match.")] + ChainIdMismatch, + #[error("Transaction max fee is less them block base fee.")] + MaxFeeLessThenBaseFee, + #[error("Eip2930 transaction is enabled after berlin hardfork.")] + Eip2930Disabled, + #[error("Eip2930 transaction is enabled after london hardfork.")] + Eip1559Disabled, + /// Thrown when calculating gas usage + #[error("gas uint64 overflow")] + GasUintOverflow, + /// returned if the transaction is specified to use less gas than required to start the + /// invocation. + #[error("intrinsic gas too low")] + GasTooLow, + /// returned if the transaction gas exceeds the limit + #[error("intrinsic gas too high")] + GasTooHigh, + /// thrown if a transaction is not supported in the current network configuration. + #[error("transaction type not supported")] + TxTypeNotSupported, + /// Thrown to ensure no one is able to specify a transaction with a tip higher than the total + /// fee cap. + #[error("max priority fee per gas higher than max fee per gas")] + TipAboveFeeCap, + /// Thrown post London if the transaction's fee is less than the base fee of the block + #[error("max fee per gas less than block base fee")] + FeeCapTooLow, + /// Thrown if the sender of a transaction is a contract. + #[error("Transaction signer has bytecode set.")] + SignerAccountHasBytecode, +} diff --git a/crates/primitives/src/transaction/mod.rs b/crates/primitives/src/transaction/mod.rs index 45708a048c5..23885b8b0cd 100644 --- a/crates/primitives/src/transaction/mod.rs +++ b/crates/primitives/src/transaction/mod.rs @@ -2,6 +2,7 @@ use crate::{keccak256, Address, Bytes, ChainId, TxHash, H256}; pub use access_list::{AccessList, AccessListItem, AccessListWithGasUsed}; use bytes::{Buf, BytesMut}; use derive_more::{AsRef, Deref}; +pub use error::InvalidTransactionError; use reth_codecs::{add_arbitrary_tests, main_codec, Compact}; use reth_rlp::{ length_of_length, Decodable, DecodeError, Encodable, Header, EMPTY_LIST_CODE, EMPTY_STRING_CODE, @@ -10,6 +11,7 @@ pub use signature::Signature; pub use tx_type::{TxType, EIP1559_TX_TYPE_ID, EIP2930_TX_TYPE_ID, LEGACY_TX_TYPE_ID}; mod access_list; +mod error; mod signature; mod tx_type; mod util; diff --git a/crates/stages/src/error.rs b/crates/stages/src/error.rs index 0698bf092f4..bd1c91bc122 100644 --- a/crates/stages/src/error.rs +++ b/crates/stages/src/error.rs @@ -17,7 +17,7 @@ pub enum StageError { block: BlockNumber, /// The underlying consensus error. #[source] - error: consensus::Error, + error: consensus::ConsensusError, }, /// The stage encountered a database error. #[error("An internal database error occurred: {0}")] diff --git a/crates/stages/src/pipeline/mod.rs b/crates/stages/src/pipeline/mod.rs index 794503bc582..cd8bbbac438 100644 --- a/crates/stages/src/pipeline/mod.rs +++ b/crates/stages/src/pipeline/mod.rs @@ -620,7 +620,7 @@ mod tests { TestStage::new(StageId("B")) .add_exec(Err(StageError::Validation { block: 5, - error: consensus::Error::BaseFeeMissing, + error: consensus::ConsensusError::BaseFeeMissing, })) .add_unwind(Ok(UnwindOutput { stage_progress: 0 })) .add_exec(Ok(ExecOutput { stage_progress: 10, done: true })), diff --git a/crates/stages/src/stages/merkle.rs b/crates/stages/src/stages/merkle.rs index ee29dd74d91..ea310fc46d4 100644 --- a/crates/stages/src/stages/merkle.rs +++ b/crates/stages/src/stages/merkle.rs @@ -124,7 +124,10 @@ impl Stage for MerkleStage { warn!(target: "sync::stages::merkle::exec", ?previous_stage_progress, got = ?block_root, expected = ?trie_root, "Block's root state failed verification"); return Err(StageError::Validation { block: previous_stage_progress, - error: consensus::Error::BodyStateRootDiff { got: trie_root, expected: block_root }, + error: consensus::ConsensusError::BodyStateRootDiff { + got: trie_root, + expected: block_root, + }, }) } @@ -167,7 +170,7 @@ impl Stage for MerkleStage { warn!(target: "sync::stages::merkle::unwind", ?unwind_to, got = ?block_root, expected = ?target_root, "Block's root state failed verification"); return Err(StageError::Validation { block: unwind_to, - error: consensus::Error::BodyStateRootDiff { + error: consensus::ConsensusError::BodyStateRootDiff { got: block_root, expected: target_root, }, diff --git a/crates/stages/src/stages/total_difficulty.rs b/crates/stages/src/stages/total_difficulty.rs index 6556bf81df2..a02b368fa8a 100644 --- a/crates/stages/src/stages/total_difficulty.rs +++ b/crates/stages/src/stages/total_difficulty.rs @@ -8,7 +8,7 @@ use reth_db::{ tables, transaction::{DbTx, DbTxMut}, }; -use reth_interfaces::{consensus::Error, provider::ProviderError}; +use reth_interfaces::{consensus::ConsensusError, provider::ProviderError}; use reth_primitives::{ChainSpec, Hardfork, EMPTY_OMMER_ROOT, MAINNET, U256}; use reth_provider::Transaction; use tracing::*; @@ -78,21 +78,21 @@ impl Stage for TotalDifficultyStage { if header.difficulty != U256::ZERO { return Err(StageError::Validation { block: header.number, - error: Error::TheMergeDifficultyIsNotZero, + error: ConsensusError::TheMergeDifficultyIsNotZero, }) } if header.nonce != 0 { return Err(StageError::Validation { block: header.number, - error: Error::TheMergeNonceIsNotZero, + error: ConsensusError::TheMergeNonceIsNotZero, }) } if header.ommers_hash != EMPTY_OMMER_ROOT { return Err(StageError::Validation { block: header.number, - error: Error::TheMergeOmmerRootIsNotEmpty, + error: ConsensusError::TheMergeOmmerRootIsNotEmpty, }) } } diff --git a/crates/transaction-pool/src/lib.rs b/crates/transaction-pool/src/lib.rs index 11ef1812f6f..1304d607a20 100644 --- a/crates/transaction-pool/src/lib.rs +++ b/crates/transaction-pool/src/lib.rs @@ -94,7 +94,7 @@ use crate::{ traits::{NewTransactionEvent, PoolSize}, }; -use reth_interfaces::consensus::Error; +use reth_interfaces::consensus::ConsensusError; use reth_primitives::{TxHash, U256}; use std::{collections::HashMap, sync::Arc}; use tokio::sync::mpsc::Receiver; @@ -164,8 +164,9 @@ where &self, origin: TransactionOrigin, transactions: impl IntoIterator, - ) -> PoolResult, Error>>> - { + ) -> PoolResult< + HashMap, ConsensusError>>, + > { let outcome = futures_util::future::join_all( transactions.into_iter().map(|tx| self.validate(origin, tx)), ) @@ -181,7 +182,7 @@ where &self, origin: TransactionOrigin, transaction: V::Transaction, - ) -> (TxHash, Result, Error>) { + ) -> (TxHash, Result, ConsensusError>) { let hash = *transaction.hash(); // TODO(mattsse): this is where additional validate checks would go, like banned senders diff --git a/crates/transaction-pool/src/test_utils/mod.rs b/crates/transaction-pool/src/test_utils/mod.rs index 52e6ff2c210..a20cf98dcd0 100644 --- a/crates/transaction-pool/src/test_utils/mod.rs +++ b/crates/transaction-pool/src/test_utils/mod.rs @@ -9,7 +9,7 @@ use crate::{ }; use async_trait::async_trait; pub use mock::*; -use reth_interfaces::consensus::Error; +use reth_interfaces::consensus::ConsensusError; use std::{marker::PhantomData, sync::Arc}; /// A [Pool] used for testing @@ -37,7 +37,7 @@ impl TransactionValidator for NoopTransactionValidator { &self, origin: TransactionOrigin, transaction: Self::Transaction, - ) -> Result, Error> { + ) -> Result, ConsensusError> { Ok(TransactionValidationOutcome::Valid { balance: Default::default(), state_nonce: 0, diff --git a/crates/transaction-pool/src/validate.rs b/crates/transaction-pool/src/validate.rs index b2ebf339da7..5a5f9a553e1 100644 --- a/crates/transaction-pool/src/validate.rs +++ b/crates/transaction-pool/src/validate.rs @@ -6,10 +6,10 @@ use crate::{ traits::{PoolTransaction, TransactionOrigin}, MAX_INIT_CODE_SIZE, TX_MAX_SIZE, }; -use reth_interfaces::consensus::Error; +use reth_interfaces::consensus::ConsensusError; use reth_primitives::{ - Address, TransactionKind, TxHash, EIP1559_TX_TYPE_ID, EIP2930_TX_TYPE_ID, LEGACY_TX_TYPE_ID, - U256, + Address, InvalidTransactionError, TransactionKind, TxHash, EIP1559_TX_TYPE_ID, + EIP2930_TX_TYPE_ID, LEGACY_TX_TYPE_ID, U256, }; use reth_provider::AccountProvider; use std::{fmt, time::Instant}; @@ -59,7 +59,7 @@ pub trait TransactionValidator: Send + Sync { &self, origin: TransactionOrigin, transaction: Self::Transaction, - ) -> Result, Error>; + ) -> Result, ConsensusError>; /// Ensure that the code size is not greater than `max_init_code_size`. /// `max_init_code_size` should be configurable so this will take it as an argument. @@ -109,7 +109,7 @@ impl TransactionValidator &self, origin: TransactionOrigin, transaction: Self::Transaction, - ) -> Result, Error> { + ) -> Result, ConsensusError> { // Checks for tx_type match transaction.tx_type() { LEGACY_TX_TYPE_ID => { @@ -119,18 +119,18 @@ impl TransactionValidator EIP2930_TX_TYPE_ID => { // Accept only legacy transactions until EIP-2718/2930 activates if !self.eip2718 { - return Err(Error::TransactionEip2930Disabled) + return Err(InvalidTransactionError::Eip2930Disabled.into()) } } EIP1559_TX_TYPE_ID => { // Reject dynamic fee transactions until EIP-1559 activates. if !self.eip1559 { - return Err(Error::TransactionEip1559Disabled) + return Err(InvalidTransactionError::Eip1559Disabled.into()) } } - _ => return Err(Error::TxTypeNotSupported), + _ => return Err(InvalidTransactionError::TxTypeNotSupported.into()), }; // Reject transactions over defined size to prevent DOS attacks @@ -163,25 +163,25 @@ impl TransactionValidator // Ensure max_fee_per_gas is greater than or equal to max_priority_fee_per_gas. if transaction.max_fee_per_gas() <= transaction.max_priority_fee_per_gas() { - return Err(Error::TipAboveFeeCap) + return Err(InvalidTransactionError::TipAboveFeeCap.into()) } // Drop non-local transactions under our own minimal accepted gas price or tip if !origin.is_local() && transaction.max_fee_per_gas() < self.gas_price { - return Err(Error::TransactionMaxFeeLessThenBaseFee) + return Err(InvalidTransactionError::MaxFeeLessThenBaseFee.into()) } // Checks for chainid if transaction.chain_id() != Some(self.chain_id) { - return Err(Error::TransactionChainId) + return Err(InvalidTransactionError::ChainIdMismatch.into()) } - let account = match self.client.basic_account(transaction.sender())? { + let account = match self.client.basic_account(transaction.sender()).unwrap() { Some(account) => { // Signer account shouldn't have bytecode. Presence of bytecode means this is a // smartcontract. if account.has_bytecode() { - return Err(Error::SignerAccountHasBytecode) + return Err(InvalidTransactionError::SignerAccountHasBytecode.into()) } else { account } @@ -196,15 +196,16 @@ impl TransactionValidator // Checks for nonce if transaction.nonce() < account.nonce { - return Err(Error::TransactionNonceNotConsistent) + return Err(InvalidTransactionError::NonceNotConsistent.into()) } // Checks for max cost if transaction.cost() > account.balance { - return Err(Error::InsufficientFunds { + return Err(InvalidTransactionError::InsufficientFunds { max_fee: transaction.max_fee_per_gas().unwrap_or_default(), available_funds: account.balance, - }) + } + .into()) } // Return the valid transaction From 3ed9ef49d0e0be1569d86cb535fd8a0027916e05 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 10 Mar 2023 14:27:33 +0100 Subject: [PATCH 088/191] feat(rpc): add missing geth trace calls (#1700) --- crates/rpc/rpc-api/src/debug.rs | 78 +++++++++++++++++++- crates/rpc/rpc-types/src/eth/trace/common.rs | 14 ++++ crates/rpc/rpc-types/src/eth/trace/geth.rs | 59 +++++++++++++++ crates/rpc/rpc-types/src/eth/trace/mod.rs | 50 +------------ crates/rpc/rpc-types/src/eth/trace/parity.rs | 10 +-- crates/rpc/rpc/src/debug.rs | 74 +++++++++++++++++-- 6 files changed, 220 insertions(+), 65 deletions(-) create mode 100644 crates/rpc/rpc-types/src/eth/trace/common.rs create mode 100644 crates/rpc/rpc-types/src/eth/trace/geth.rs diff --git a/crates/rpc/rpc-api/src/debug.rs b/crates/rpc/rpc-api/src/debug.rs index 4a78a635a62..dc594e0275f 100644 --- a/crates/rpc/rpc-api/src/debug.rs +++ b/crates/rpc/rpc-api/src/debug.rs @@ -1,6 +1,9 @@ use jsonrpsee::{core::RpcResult as Result, proc_macros::rpc}; -use reth_primitives::{BlockId, Bytes, H256}; -use reth_rpc_types::RichBlock; +use reth_primitives::{BlockId, BlockNumberOrTag, Bytes, H256}; +use reth_rpc_types::{ + trace::geth::{BlockTraceResult, GethDebugTracingOptions, GethTraceFrame, TraceResult}, + CallRequest, RichBlock, +}; /// Debug rpc interface. #[cfg_attr(not(feature = "client"), rpc(server))] @@ -25,4 +28,75 @@ pub trait DebugApi { /// Returns an array of recent bad blocks that the client has seen on the network. #[method(name = "debug_getBadBlocks")] async fn bad_blocks(&self) -> Result>; + + /// Returns the structured logs created during the execution of EVM between two blocks + /// (excluding start) as a JSON object. + #[method(name = "debug_traceChain")] + async fn debug_trace_chain( + &self, + start_exclusive: BlockNumberOrTag, + end_inclusive: BlockNumberOrTag, + ) -> Result>; + + /// The `debug_traceBlock` method will return a full stack trace of all invoked opcodes of all + /// transaction that were included in this block. + /// + /// This expects an rlp encoded block + /// + /// Note, the parent of this block must be present, or it will fail. For the second parameter + /// see [GethDebugTracingOptions] reference. + #[method(name = "debug_traceBlock")] + async fn debug_trace_block( + &self, + rlp_block: Bytes, + opts: GethDebugTracingOptions, + ) -> Result>; + + /// Similar to `debug_traceBlock`, `debug_traceBlockByHash` accepts a block hash and will replay + /// the block that is already present in the database. For the second parameter see + /// [GethDebugTracingOptions]. + #[method(name = "debug_traceBlockByHash")] + async fn debug_trace_block_by_hash( + &self, + block: H256, + opts: GethDebugTracingOptions, + ) -> Result>; + + /// Similar to `debug_traceBlockByNumber`, `debug_traceBlockByHash` accepts a block number + /// [BlockNumberOrTag] and will replay the block that is already present in the database. + /// For the second parameter see [GethDebugTracingOptions]. + #[method(name = "debug_traceBlockByNumber")] + async fn debug_trace_block_by_number( + &self, + block: BlockNumberOrTag, + opts: GethDebugTracingOptions, + ) -> Result>; + + /// The `debug_traceTransaction` debugging method will attempt to run the transaction in the + /// exact same manner as it was executed on the network. It will replay any transaction that + /// may have been executed prior to this one before it will finally attempt to execute the + /// transaction that corresponds to the given hash. + #[method(name = "debug_traceTransaction")] + async fn debug_trace_transaction( + &self, + tx_hash: H256, + opts: GethDebugTracingOptions, + ) -> Result; + + /// The debug_traceCall method lets you run an `eth_call` within the context of the given block + /// execution using the final state of parent block as the base. + /// + /// The first argument (just as in`eth_call`) is a transaction request. + /// The block can be specified either by hash or by number as + /// the second argument. + /// The trace can be configured similar to `debug_traceTransaction`, + /// see [GethDebugTracingOptions]. The method returns the same output as + /// `debug_traceTransaction`. + #[method(name = "debug_traceCall")] + async fn debug_trace_call( + &self, + request: CallRequest, + block_number: Option, + opts: GethDebugTracingOptions, + ) -> Result; } diff --git a/crates/rpc/rpc-types/src/eth/trace/common.rs b/crates/rpc/rpc-types/src/eth/trace/common.rs new file mode 100644 index 00000000000..78ee088ce28 --- /dev/null +++ b/crates/rpc/rpc-types/src/eth/trace/common.rs @@ -0,0 +1,14 @@ +//! Types used by tracing backends + +use serde::{Deserialize, Serialize}; + +/// The result of a single transaction trace. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +#[allow(missing_docs)] +pub enum TraceResult { + /// Untagged success variant + Success { result: Ok }, + /// Untagged error variant + Error { error: Err }, +} diff --git a/crates/rpc/rpc-types/src/eth/trace/geth.rs b/crates/rpc/rpc-types/src/eth/trace/geth.rs new file mode 100644 index 00000000000..6e38ba81f38 --- /dev/null +++ b/crates/rpc/rpc-types/src/eth/trace/geth.rs @@ -0,0 +1,59 @@ +#![allow(missing_docs)] +/// Geth tracing types +use reth_primitives::{Bytes, JsonU256, H256, U256}; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +// re-exported for geth tracing types +pub use ethers_core::types::{GethDebugTracingOptions, GethTraceFrame}; + +/// Result type for geth style transaction trace +pub type TraceResult = crate::trace::common::TraceResult; + +/// blockTraceResult represents the results of tracing a single block when an entire chain is being +/// traced. ref +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct BlockTraceResult { + /// Block number corresponding to the trace task + pub block: U256, + /// Block hash corresponding to the trace task + pub hash: H256, + /// Trace results produced by the trace task + pub traces: Vec, +} + +/// Geth Default trace frame +/// +/// +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DefaultFrame { + pub failed: bool, + pub gas: JsonU256, + pub return_value: Bytes, + pub struct_logs: Vec, +} + +/// Represents a struct log entry in a trace +/// +/// +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct StructLog { + pub depth: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error: Option, + pub gas: u64, + #[serde(rename = "gasCost")] + pub gas_cost: u64, + /// ref + #[serde(default, skip_serializing_if = "Option::is_none")] + pub memory: Option>, + pub op: String, + pub pc: u64, + #[serde(default, rename = "refund", skip_serializing_if = "Option::is_none")] + pub refund_counter: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stack: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub storage: Option>, +} diff --git a/crates/rpc/rpc-types/src/eth/trace/mod.rs b/crates/rpc/rpc-types/src/eth/trace/mod.rs index cc07e21cb37..d4ec6960f0d 100644 --- a/crates/rpc/rpc-types/src/eth/trace/mod.rs +++ b/crates/rpc/rpc-types/src/eth/trace/mod.rs @@ -1,52 +1,6 @@ //! Types for tracing +pub mod common; pub mod filter; +pub mod geth; pub mod parity; - -/// Geth tracing types -pub mod geth { - #![allow(missing_docs)] - - use reth_primitives::{Bytes, JsonU256, H256, U256}; - use serde::{Deserialize, Serialize}; - use std::collections::BTreeMap; - - // re-exported for geth tracing types - pub use ethers_core::types::GethDebugTracingOptions; - - /// Geth Default trace frame - /// - /// - #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] - #[serde(rename_all = "camelCase")] - pub struct DefaultFrame { - pub failed: bool, - pub gas: JsonU256, - pub return_value: Bytes, - pub struct_logs: Vec, - } - - /// Represents a struct log entry in a trace - /// - /// - #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] - pub struct StructLog { - pub depth: u64, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub error: Option, - pub gas: u64, - #[serde(rename = "gasCost")] - pub gas_cost: u64, - /// ref - #[serde(default, skip_serializing_if = "Option::is_none")] - pub memory: Option>, - pub op: String, - pub pc: u64, - #[serde(default, rename = "refund", skip_serializing_if = "Option::is_none")] - pub refund_counter: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub stack: Option>, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub storage: Option>, - } -} diff --git a/crates/rpc/rpc-types/src/eth/trace/parity.rs b/crates/rpc/rpc-types/src/eth/trace/parity.rs index 174f1f13d71..14df5808984 100644 --- a/crates/rpc/rpc-types/src/eth/trace/parity.rs +++ b/crates/rpc/rpc-types/src/eth/trace/parity.rs @@ -7,6 +7,9 @@ use reth_primitives::{Address, Bytes, H256, U256, U64}; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; +/// Result type for parity style transaction trace +pub type TraceResult = crate::trace::common::TraceResult; + #[derive(Clone, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub enum TraceType { @@ -171,13 +174,6 @@ pub enum TraceOutput { Create(CreateOutput), } -#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] -#[serde(untagged)] -pub enum TraceResult { - Success { result: TraceOutput }, - Error { error: String }, -} - #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct TransactionTrace { diff --git a/crates/rpc/rpc/src/debug.rs b/crates/rpc/rpc/src/debug.rs index de3d11398c9..3fc9c23c464 100644 --- a/crates/rpc/rpc/src/debug.rs +++ b/crates/rpc/rpc/src/debug.rs @@ -1,9 +1,12 @@ use crate::{result::internal_rpc_err, EthApiSpec}; use async_trait::async_trait; -use jsonrpsee::core::RpcResult as Result; -use reth_primitives::{BlockId, Bytes, H256}; +use jsonrpsee::core::RpcResult; +use reth_primitives::{BlockId, BlockNumberOrTag, Bytes, H256}; use reth_rpc_api::DebugApiServer; -use reth_rpc_types::RichBlock; +use reth_rpc_types::{ + trace::geth::{BlockTraceResult, GethDebugTracingOptions, GethTraceFrame, TraceResult}, + CallRequest, RichBlock, +}; /// `debug` API implementation. /// @@ -28,24 +31,79 @@ impl DebugApiServer for DebugApi where Eth: EthApiSpec + 'static, { - async fn raw_header(&self, _block_id: BlockId) -> Result { + async fn raw_header(&self, _block_id: BlockId) -> RpcResult { Err(internal_rpc_err("unimplemented")) } - async fn raw_block(&self, _block_id: BlockId) -> Result { + async fn raw_block(&self, _block_id: BlockId) -> RpcResult { Err(internal_rpc_err("unimplemented")) } /// Returns the bytes of the transaction for the given hash. - async fn raw_transaction(&self, _hash: H256) -> Result { + async fn raw_transaction(&self, _hash: H256) -> RpcResult { Err(internal_rpc_err("unimplemented")) } - async fn raw_receipts(&self, _block_id: BlockId) -> Result> { + async fn raw_receipts(&self, _block_id: BlockId) -> RpcResult> { Err(internal_rpc_err("unimplemented")) } - async fn bad_blocks(&self) -> Result> { + async fn bad_blocks(&self) -> RpcResult> { + Err(internal_rpc_err("unimplemented")) + } + + /// Handler for `debug_traceChain` + async fn debug_trace_chain( + &self, + _start_exclusive: BlockNumberOrTag, + _end_inclusive: BlockNumberOrTag, + ) -> RpcResult> { + Err(internal_rpc_err("unimplemented")) + } + + /// Handler for `debug_traceBlock` + async fn debug_trace_block( + &self, + _rlp_block: Bytes, + _opts: GethDebugTracingOptions, + ) -> RpcResult> { + Err(internal_rpc_err("unimplemented")) + } + + /// Handler for `debug_traceBlockByHash` + async fn debug_trace_block_by_hash( + &self, + _block: H256, + _opts: GethDebugTracingOptions, + ) -> RpcResult> { + Err(internal_rpc_err("unimplemented")) + } + + /// Handler for `debug_traceBlockByNumber` + async fn debug_trace_block_by_number( + &self, + _block: BlockNumberOrTag, + _opts: GethDebugTracingOptions, + ) -> RpcResult> { + Err(internal_rpc_err("unimplemented")) + } + + /// Handler for `debug_traceTransaction` + async fn debug_trace_transaction( + &self, + _tx_hash: H256, + _opts: GethDebugTracingOptions, + ) -> RpcResult { + Err(internal_rpc_err("unimplemented")) + } + + /// Handler for `debug_traceCall` + async fn debug_trace_call( + &self, + _request: CallRequest, + _block_number: Option, + _opts: GethDebugTracingOptions, + ) -> RpcResult { Err(internal_rpc_err("unimplemented")) } } From 89514d70f20297dd42e3bb05fee7f3a6b4a1ccb1 Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Fri, 10 Mar 2023 16:49:26 +0200 Subject: [PATCH 089/191] fix(txpool): track promoted transactions (#1702) --- crates/transaction-pool/src/pool/txpool.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/transaction-pool/src/pool/txpool.rs b/crates/transaction-pool/src/pool/txpool.rs index eeeeb087231..2dcf6f5d5bd 100644 --- a/crates/transaction-pool/src/pool/txpool.rs +++ b/crates/transaction-pool/src/pool/txpool.rs @@ -290,6 +290,9 @@ impl TxPool { Destination::Pool(move_to) => { debug_assert!(!move_to.eq(¤t), "destination must be different"); self.move_transaction(current, move_to, &id); + if matches!(move_to, SubPool::Pending) { + outcome.promoted.push(hash); + } } } } From fdfeeb42dcd1a039a5f5d5f223095eeb4887cfa7 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 10 Mar 2023 15:57:12 +0100 Subject: [PATCH 090/191] refactor(txpool): use transaction error type (#1698) --- Cargo.lock | 1 - crates/rpc/rpc/src/eth/error.rs | 32 ++--- crates/transaction-pool/Cargo.toml | 1 - crates/transaction-pool/src/error.rs | 57 +++++---- crates/transaction-pool/src/lib.rs | 36 +++--- crates/transaction-pool/src/pool/mod.rs | 7 +- crates/transaction-pool/src/pool/txpool.rs | 9 +- crates/transaction-pool/src/test_utils/mod.rs | 7 +- crates/transaction-pool/src/validate.rs | 118 ++++++++++++------ 9 files changed, 158 insertions(+), 110 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d66d9ac35c2..6417ea64c8b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5149,7 +5149,6 @@ dependencies = [ "parking_lot 0.12.1", "paste", "rand 0.8.5", - "reth-interfaces", "reth-metrics-derive", "reth-primitives", "reth-provider", diff --git a/crates/rpc/rpc/src/eth/error.rs b/crates/rpc/rpc/src/eth/error.rs index 8fe9eff1f4b..2d5466417c3 100644 --- a/crates/rpc/rpc/src/eth/error.rs +++ b/crates/rpc/rpc/src/eth/error.rs @@ -4,7 +4,7 @@ use crate::result::{internal_rpc_err, rpc_err}; use jsonrpsee::{core::Error as RpcError, types::error::INVALID_PARAMS_CODE}; use reth_primitives::{constants::SELECTOR_LEN, Address, U128, U256}; use reth_rpc_types::{error::EthRpcErrorCode, BlockError}; -use reth_transaction_pool::error::PoolError; +use reth_transaction_pool::error::{InvalidPoolTransactionError, PoolError}; use revm::primitives::{EVMError, Halt}; /// Result alias @@ -22,7 +22,7 @@ pub(crate) enum EthApiError { #[error("Invalid transaction signature")] InvalidTransactionSignature, #[error(transparent)] - PoolError(GethTxPoolError), + PoolError(RpcPoolError), #[error("Unknown block number")] UnknownBlockNumber, #[error("Invalid block range")] @@ -266,9 +266,9 @@ impl std::fmt::Display for RevertError { impl std::error::Error for RevertError {} -/// A helper error type that mirrors `geth` Txpool's error messages +/// A helper error type that's mainly used to mirror `geth` Txpool's error messages #[derive(Debug, thiserror::Error)] -pub(crate) enum GethTxPoolError { +pub(crate) enum RpcPoolError { #[error("already known")] AlreadyKnown, #[error("invalid sender")] @@ -285,26 +285,28 @@ pub(crate) enum GethTxPoolError { NegativeValue, #[error("oversized data")] OversizedData, + #[error(transparent)] + Invalid(#[from] InvalidPoolTransactionError), + #[error(transparent)] + Other(Box), } -impl From for GethTxPoolError { - fn from(err: PoolError) -> GethTxPoolError { +impl From for RpcPoolError { + fn from(err: PoolError) -> RpcPoolError { match err { - PoolError::ReplacementUnderpriced(_) => GethTxPoolError::ReplaceUnderpriced, - PoolError::ProtocolFeeCapTooLow(_, _) => GethTxPoolError::Underpriced, - PoolError::SpammerExceededCapacity(_, _) => GethTxPoolError::TxPoolOverflow, - PoolError::DiscardedOnInsert(_) => GethTxPoolError::TxPoolOverflow, - PoolError::TxExceedsGasLimit(_, _, _) => GethTxPoolError::GasLimit, - PoolError::TxExceedsMaxInitCodeSize(_, _, _) => GethTxPoolError::OversizedData, - PoolError::AccountNotFound(_) => GethTxPoolError::InvalidSender, - PoolError::OversizedData(_, _, _) => GethTxPoolError::OversizedData, + PoolError::ReplacementUnderpriced(_) => RpcPoolError::ReplaceUnderpriced, + PoolError::ProtocolFeeCapTooLow(_, _) => RpcPoolError::Underpriced, + PoolError::SpammerExceededCapacity(_, _) => RpcPoolError::TxPoolOverflow, + PoolError::DiscardedOnInsert(_) => RpcPoolError::TxPoolOverflow, + PoolError::InvalidTransaction(_, err) => err.into(), + PoolError::Other(_, err) => RpcPoolError::Other(err), } } } impl From for EthApiError { fn from(err: PoolError) -> Self { - EthApiError::PoolError(GethTxPoolError::from(err)) + EthApiError::PoolError(RpcPoolError::from(err)) } } diff --git a/crates/transaction-pool/Cargo.toml b/crates/transaction-pool/Cargo.toml index bde4c1da72c..e04c502ec83 100644 --- a/crates/transaction-pool/Cargo.toml +++ b/crates/transaction-pool/Cargo.toml @@ -20,7 +20,6 @@ normal = [ # reth reth-primitives = { path = "../primitives" } reth-provider = { path = "../storage/provider" } -reth-interfaces = { path = "../interfaces" } reth-rlp = { path = "../rlp" } # async/futures diff --git a/crates/transaction-pool/src/error.rs b/crates/transaction-pool/src/error.rs index 4048a37c417..1a9a1e6b04b 100644 --- a/crates/transaction-pool/src/error.rs +++ b/crates/transaction-pool/src/error.rs @@ -1,6 +1,6 @@ //! Transaction pool errors -use reth_primitives::{Address, TxHash}; +use reth_primitives::{Address, InvalidTransactionError, TxHash}; /// Transaction pool result type. pub type PoolResult = Result; @@ -21,22 +21,13 @@ pub enum PoolError { /// respect the size limits of the pool. #[error("[{0:?}] Transaction discarded outright due to pool size constraints.")] DiscardedOnInsert(TxHash), - /// Thrown when a new transaction is added to the pool, but then immediately discarded to - /// respect the size limits of the pool. - #[error("[{0:?}] Transaction's gas limit {1} exceeds block's gas limit {2}.")] - TxExceedsGasLimit(TxHash, u64, u64), - /// Thrown when a new transaction is added to the pool, but then immediately discarded to - /// respect the max_init_code_size. - #[error("[{0:?}] Transaction's size {1} exceeds max_init_code_size {2}.")] - TxExceedsMaxInitCodeSize(TxHash, usize, usize), - /// Thrown if the transaction contains an invalid signature - #[error("[{0:?}]: Invalid sender")] - AccountNotFound(TxHash), - /// Thrown if the input data of a transaction is greater - /// than some meaningful limit a user might use. This is not a consensus error - /// making the transaction invalid, rather a DOS protection. - #[error("[{0:?}]: Input data too large")] - OversizedData(TxHash, usize, usize), + /// Thrown when the transaction is considered invalid. + #[error("[{0:?}] {1:?}")] + InvalidTransaction(TxHash, InvalidPoolTransactionError), + /// Any other error that occurred while inserting/validating a transaction. e.g. IO database + /// error + #[error("[{0:?}] {1:?}")] + Other(TxHash, Box), } // === impl PoolError === @@ -49,10 +40,34 @@ impl PoolError { PoolError::ProtocolFeeCapTooLow(hash, _) => hash, PoolError::SpammerExceededCapacity(_, hash) => hash, PoolError::DiscardedOnInsert(hash) => hash, - PoolError::TxExceedsGasLimit(hash, _, _) => hash, - PoolError::TxExceedsMaxInitCodeSize(hash, _, _) => hash, - PoolError::AccountNotFound(hash) => hash, - PoolError::OversizedData(hash, _, _) => hash, + PoolError::InvalidTransaction(hash, _) => hash, + PoolError::Other(hash, _) => hash, } } } + +/// Represents errors that can happen when validating transactions for the pool +/// +/// See [TransactionValidator](crate::TransactionValidator). +#[derive(Debug, Clone, thiserror::Error)] +pub enum InvalidPoolTransactionError { + /// Hard consensus errors + #[error(transparent)] + Consensus(#[from] InvalidTransactionError), + /// Thrown when a new transaction is added to the pool, but then immediately discarded to + /// respect the size limits of the pool. + #[error("Transaction's gas limit {0} exceeds block's gas limit {1}.")] + ExceedsGasLimit(u64, u64), + /// Thrown when a new transaction is added to the pool, but then immediately discarded to + /// respect the max_init_code_size. + #[error("Transaction's size {0} exceeds max_init_code_size {1}.")] + ExceedsMaxInitCodeSize(usize, usize), + /// Thrown if the transaction contains an invalid signature + #[error("Invalid sender")] + AccountNotFound, + /// Thrown if the input data of a transaction is greater + /// than some meaningful limit a user might use. This is not a consensus error + /// making the transaction invalid, rather a DOS protection. + #[error("Input data too large")] + OversizedData(usize, usize), +} diff --git a/crates/transaction-pool/src/lib.rs b/crates/transaction-pool/src/lib.rs index 1304d607a20..d355e34d248 100644 --- a/crates/transaction-pool/src/lib.rs +++ b/crates/transaction-pool/src/lib.rs @@ -86,7 +86,10 @@ pub use crate::{ BestTransactions, OnNewBlockEvent, PoolTransaction, PooledTransaction, PropagateKind, PropagatedTransactions, TransactionOrigin, TransactionPool, }, - validate::{TransactionValidationOutcome, TransactionValidator, ValidPoolTransaction}, + validate::{ + EthTransactionValidator, TransactionValidationOutcome, TransactionValidator, + ValidPoolTransaction, + }, }; use crate::{ error::PoolResult, @@ -94,7 +97,7 @@ use crate::{ traits::{NewTransactionEvent, PoolSize}, }; -use reth_interfaces::consensus::ConsensusError; +use crate::error::PoolError; use reth_primitives::{TxHash, U256}; use std::{collections::HashMap, sync::Arc}; use tokio::sync::mpsc::Receiver; @@ -164,9 +167,7 @@ where &self, origin: TransactionOrigin, transactions: impl IntoIterator, - ) -> PoolResult< - HashMap, ConsensusError>>, - > { + ) -> PoolResult>> { let outcome = futures_util::future::join_all( transactions.into_iter().map(|tx| self.validate(origin, tx)), ) @@ -182,7 +183,7 @@ where &self, origin: TransactionOrigin, transaction: V::Transaction, - ) -> (TxHash, Result, ConsensusError>) { + ) -> (TxHash, TransactionValidationOutcome) { let hash = *transaction.hash(); // TODO(mattsse): this is where additional validate checks would go, like banned senders @@ -229,18 +230,14 @@ where let (_, tx) = self.validate(origin, transaction).await; match tx { - Ok(TransactionValidationOutcome::Valid { - balance: _, - state_nonce: _, - transaction: _, - }) => self - .pool - .add_transactions(origin, std::iter::once(tx.unwrap())) - .pop() - .expect("exists; qed"), - Ok(TransactionValidationOutcome::Invalid(_transaction, error)) => Err(error), - Err(_err) => { - unimplemented!() + TransactionValidationOutcome::Valid { .. } => { + self.pool.add_transactions(origin, std::iter::once(tx)).pop().expect("exists; qed") + } + TransactionValidationOutcome::Invalid(transaction, error) => { + Err(PoolError::InvalidTransaction(*transaction.hash(), error)) + } + TransactionValidationOutcome::Error(transaction, error) => { + Err(PoolError::Other(*transaction.hash(), error)) } } } @@ -252,8 +249,7 @@ where ) -> PoolResult>> { let validated = self.validate_all(origin, transactions).await?; - let transactions = - self.pool.add_transactions(origin, validated.into_values().map(|x| x.unwrap())); + let transactions = self.pool.add_transactions(origin, validated.into_values()); Ok(transactions) } diff --git a/crates/transaction-pool/src/pool/mod.rs b/crates/transaction-pool/src/pool/mod.rs index 6ef8fc53128..7ae9ee48af5 100644 --- a/crates/transaction-pool/src/pool/mod.rs +++ b/crates/transaction-pool/src/pool/mod.rs @@ -232,7 +232,12 @@ where TransactionValidationOutcome::Invalid(tx, err) => { let mut listener = self.event_listener.write(); listener.discarded(tx.hash()); - Err(err) + Err(PoolError::InvalidTransaction(*tx.hash(), err)) + } + TransactionValidationOutcome::Error(tx, err) => { + let mut listener = self.event_listener.write(); + listener.discarded(tx.hash()); + Err(PoolError::Other(*tx.hash(), err)) } } } diff --git a/crates/transaction-pool/src/pool/txpool.rs b/crates/transaction-pool/src/pool/txpool.rs index 2dcf6f5d5bd..9f43dafacd5 100644 --- a/crates/transaction-pool/src/pool/txpool.rs +++ b/crates/transaction-pool/src/pool/txpool.rs @@ -1,7 +1,7 @@ //! The internal transaction pool implementation. use crate::{ config::MAX_ACCOUNT_SLOTS_PER_SENDER, - error::PoolError, + error::{InvalidPoolTransactionError, PoolError}, identifier::{SenderId, TransactionId}, metrics::TxPoolMetrics, pool::{ @@ -220,8 +220,6 @@ impl TxPool { .or_default() .update(on_chain_nonce, on_chain_balance); - let _hash = *tx.hash(); - match self.all_transactions.insert_tx(tx, on_chain_balance, on_chain_nonce) { Ok(InsertOk { transaction, move_to, replaced_tx, updates, .. }) => { self.add_new_transaction(transaction.clone(), replaced_tx, move_to); @@ -263,10 +261,9 @@ impl TxPool { transaction, block_gas_limit, tx_gas_limit, - } => Err(PoolError::TxExceedsGasLimit( + } => Err(PoolError::InvalidTransaction( *transaction.hash(), - block_gas_limit, - tx_gas_limit, + InvalidPoolTransactionError::ExceedsGasLimit(block_gas_limit, tx_gas_limit), )), } } diff --git a/crates/transaction-pool/src/test_utils/mod.rs b/crates/transaction-pool/src/test_utils/mod.rs index a20cf98dcd0..7fd1da50bbd 100644 --- a/crates/transaction-pool/src/test_utils/mod.rs +++ b/crates/transaction-pool/src/test_utils/mod.rs @@ -9,7 +9,6 @@ use crate::{ }; use async_trait::async_trait; pub use mock::*; -use reth_interfaces::consensus::ConsensusError; use std::{marker::PhantomData, sync::Arc}; /// A [Pool] used for testing @@ -37,12 +36,12 @@ impl TransactionValidator for NoopTransactionValidator { &self, origin: TransactionOrigin, transaction: Self::Transaction, - ) -> Result, ConsensusError> { - Ok(TransactionValidationOutcome::Valid { + ) -> TransactionValidationOutcome { + TransactionValidationOutcome::Valid { balance: Default::default(), state_nonce: 0, transaction, - }) + } } } diff --git a/crates/transaction-pool/src/validate.rs b/crates/transaction-pool/src/validate.rs index 5a5f9a553e1..cafab6aaf7c 100644 --- a/crates/transaction-pool/src/validate.rs +++ b/crates/transaction-pool/src/validate.rs @@ -1,12 +1,11 @@ //! Transaction validation abstractions. use crate::{ - error::PoolError, + error::InvalidPoolTransactionError, identifier::{SenderId, TransactionId}, traits::{PoolTransaction, TransactionOrigin}, MAX_INIT_CODE_SIZE, TX_MAX_SIZE, }; -use reth_interfaces::consensus::ConsensusError; use reth_primitives::{ Address, InvalidTransactionError, TransactionKind, TxHash, EIP1559_TX_TYPE_ID, EIP2930_TX_TYPE_ID, LEGACY_TX_TYPE_ID, U256, @@ -28,7 +27,9 @@ pub enum TransactionValidationOutcome { }, /// The transaction is considered invalid indefinitely: It violates constraints that prevent /// this transaction from ever becoming valid. - Invalid(T, PoolError), + Invalid(T, InvalidPoolTransactionError), + /// An error occurred while trying to validate the transaction + Error(T, Box), } /// Provides support for validating transaction at any given state of the chain @@ -59,7 +60,7 @@ pub trait TransactionValidator: Send + Sync { &self, origin: TransactionOrigin, transaction: Self::Transaction, - ) -> Result, ConsensusError>; + ) -> TransactionValidationOutcome; /// Ensure that the code size is not greater than `max_init_code_size`. /// `max_init_code_size` should be configurable so this will take it as an argument. @@ -67,11 +68,10 @@ pub trait TransactionValidator: Send + Sync { &self, transaction: Self::Transaction, max_init_code_size: usize, - ) -> Result<(), PoolError> { + ) -> Result<(), InvalidPoolTransactionError> { if *transaction.kind() == TransactionKind::Create && transaction.size() > max_init_code_size { - Err(PoolError::TxExceedsMaxInitCodeSize( - *transaction.hash(), + Err(InvalidPoolTransactionError::ExceedsMaxInitCodeSize( transaction.size(), max_init_code_size, )) @@ -81,8 +81,9 @@ pub trait TransactionValidator: Send + Sync { } } -/// TODO: Add docs and make this public -pub(crate) struct EthTransactionValidatorConfig { +/// A [TransactionValidator] implementation that validates ethereum transaction. +#[derive(Debug, Clone)] +pub struct EthTransactionValidator { /// Chain id chain_id: u64, /// This type fetches account info from the db @@ -101,7 +102,7 @@ pub(crate) struct EthTransactionValidatorConfig { #[async_trait::async_trait] impl TransactionValidator - for EthTransactionValidatorConfig + for EthTransactionValidator { type Transaction = T; @@ -109,7 +110,7 @@ impl TransactionValidator &self, origin: TransactionOrigin, transaction: Self::Transaction, - ) -> Result, ConsensusError> { + ) -> TransactionValidationOutcome { // Checks for tx_type match transaction.tx_type() { LEGACY_TX_TYPE_ID => { @@ -119,101 +120,136 @@ impl TransactionValidator EIP2930_TX_TYPE_ID => { // Accept only legacy transactions until EIP-2718/2930 activates if !self.eip2718 { - return Err(InvalidTransactionError::Eip2930Disabled.into()) + return TransactionValidationOutcome::Invalid( + transaction, + InvalidTransactionError::Eip1559Disabled.into(), + ) } } EIP1559_TX_TYPE_ID => { // Reject dynamic fee transactions until EIP-1559 activates. if !self.eip1559 { - return Err(InvalidTransactionError::Eip1559Disabled.into()) + return TransactionValidationOutcome::Invalid( + transaction, + InvalidTransactionError::Eip1559Disabled.into(), + ) } } - _ => return Err(InvalidTransactionError::TxTypeNotSupported.into()), + _ => { + return TransactionValidationOutcome::Invalid( + transaction, + InvalidTransactionError::TxTypeNotSupported.into(), + ) + } }; // Reject transactions over defined size to prevent DOS attacks if transaction.size() > TX_MAX_SIZE { - return Ok(TransactionValidationOutcome::Invalid( + return TransactionValidationOutcome::Invalid( transaction.clone(), - PoolError::OversizedData(*transaction.hash(), transaction.size(), TX_MAX_SIZE), - )) + InvalidPoolTransactionError::OversizedData(transaction.size(), TX_MAX_SIZE), + ) } // Check whether the init code size has been exceeded. if self.shanghai { - match self.ensure_max_init_code_size(transaction.clone(), MAX_INIT_CODE_SIZE) { - Ok(_) => {} - Err(e) => return Ok(TransactionValidationOutcome::Invalid(transaction, e)), + if let Err(err) = + self.ensure_max_init_code_size(transaction.clone(), MAX_INIT_CODE_SIZE) + { + return TransactionValidationOutcome::Invalid(transaction, err) } } // Checks for gas limit if transaction.gas_limit() > self.current_max_gas_limit { - return Ok(TransactionValidationOutcome::Invalid( + return TransactionValidationOutcome::Invalid( transaction.clone(), - PoolError::TxExceedsGasLimit( - *transaction.hash(), + InvalidPoolTransactionError::ExceedsGasLimit( transaction.gas_limit(), self.current_max_gas_limit, ), - )) + ) } // Ensure max_fee_per_gas is greater than or equal to max_priority_fee_per_gas. if transaction.max_fee_per_gas() <= transaction.max_priority_fee_per_gas() { - return Err(InvalidTransactionError::TipAboveFeeCap.into()) + return TransactionValidationOutcome::Invalid( + transaction, + InvalidTransactionError::TipAboveFeeCap.into(), + ) } // Drop non-local transactions under our own minimal accepted gas price or tip if !origin.is_local() && transaction.max_fee_per_gas() < self.gas_price { - return Err(InvalidTransactionError::MaxFeeLessThenBaseFee.into()) + return TransactionValidationOutcome::Invalid( + transaction, + InvalidTransactionError::MaxFeeLessThenBaseFee.into(), + ) } // Checks for chainid if transaction.chain_id() != Some(self.chain_id) { - return Err(InvalidTransactionError::ChainIdMismatch.into()) + return TransactionValidationOutcome::Invalid( + transaction, + InvalidTransactionError::ChainIdMismatch.into(), + ) } - let account = match self.client.basic_account(transaction.sender()).unwrap() { + let account = match self.client.basic_account(transaction.sender()) { + Ok(account) => account, + Err(err) => return TransactionValidationOutcome::Error(transaction, Box::new(err)), + }; + + let account = match account { Some(account) => { // Signer account shouldn't have bytecode. Presence of bytecode means this is a // smartcontract. if account.has_bytecode() { - return Err(InvalidTransactionError::SignerAccountHasBytecode.into()) + return TransactionValidationOutcome::Invalid( + transaction, + InvalidTransactionError::SignerAccountHasBytecode.into(), + ) } else { account } } None => { - return Ok(TransactionValidationOutcome::Invalid( - transaction.clone(), - PoolError::AccountNotFound(*transaction.hash()), - )) + return TransactionValidationOutcome::Invalid( + transaction, + InvalidPoolTransactionError::AccountNotFound, + ) } }; // Checks for nonce if transaction.nonce() < account.nonce { - return Err(InvalidTransactionError::NonceNotConsistent.into()) + return TransactionValidationOutcome::Invalid( + transaction, + InvalidTransactionError::NonceNotConsistent.into(), + ) } // Checks for max cost if transaction.cost() > account.balance { - return Err(InvalidTransactionError::InsufficientFunds { - max_fee: transaction.max_fee_per_gas().unwrap_or_default(), - available_funds: account.balance, - } - .into()) + let max_fee = transaction.max_fee_per_gas().unwrap_or_default(); + return TransactionValidationOutcome::Invalid( + transaction, + InvalidTransactionError::InsufficientFunds { + max_fee, + available_funds: account.balance, + } + .into(), + ) } // Return the valid transaction - Ok(TransactionValidationOutcome::Valid { + TransactionValidationOutcome::Valid { balance: account.balance, state_nonce: account.nonce, transaction, - }) + } } } From 99a821aa78c03f4397a0d0f35ee55eb9e7f8b465 Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Fri, 10 Mar 2023 17:07:35 +0200 Subject: [PATCH 091/191] chore(txpool): delete unused `removed` updates (#1703) --- crates/transaction-pool/src/pool/mod.rs | 9 +------- crates/transaction-pool/src/pool/txpool.rs | 24 ++++++---------------- 2 files changed, 7 insertions(+), 26 deletions(-) diff --git a/crates/transaction-pool/src/pool/mod.rs b/crates/transaction-pool/src/pool/mod.rs index 7ae9ee48af5..1090feedfff 100644 --- a/crates/transaction-pool/src/pool/mod.rs +++ b/crates/transaction-pool/src/pool/mod.rs @@ -422,19 +422,12 @@ pub struct AddedPendingTransaction { promoted: Vec, /// transaction that failed and became discarded discarded: Vec, - /// Transactions removed from the Ready pool - removed: Vec>>, } impl AddedPendingTransaction { /// Create a new, empty transaction. fn new(transaction: Arc>) -> Self { - Self { - transaction, - promoted: Default::default(), - discarded: Default::default(), - removed: Default::default(), - } + Self { transaction, promoted: Default::default(), discarded: Default::default() } } } diff --git a/crates/transaction-pool/src/pool/txpool.rs b/crates/transaction-pool/src/pool/txpool.rs index 9f43dafacd5..41bbd959ff7 100644 --- a/crates/transaction-pool/src/pool/txpool.rs +++ b/crates/transaction-pool/src/pool/txpool.rs @@ -178,7 +178,7 @@ impl TxPool { self.all_transactions.update(event.pending_block_base_fee, &event.state_changes); // Process the sub-pool updates - let UpdateOutcome { promoted, discarded, .. } = self.process_updates(updates); + let UpdateOutcome { promoted, discarded } = self.process_updates(updates); OnNewBlockOutcome { block_hash: event.hash, @@ -225,7 +225,7 @@ impl TxPool { self.add_new_transaction(transaction.clone(), replaced_tx, move_to); // Update inserted transactions metric self.metrics.inserted_transactions.increment(1); - let UpdateOutcome { promoted, discarded, removed } = self.process_updates(updates); + let UpdateOutcome { promoted, discarded } = self.process_updates(updates); // This transaction was moved to the pending pool. let res = if move_to.is_pending() { @@ -233,7 +233,6 @@ impl TxPool { transaction, promoted, discarded, - removed, }) } else { AddedTransaction::Parked { transaction, subpool: move_to } @@ -273,10 +272,7 @@ impl TxPool { /// Maintenance task to apply a series of updates. /// /// This will move/discard the given transaction according to the `PoolUpdate` - fn process_updates( - &mut self, - updates: impl IntoIterator, - ) -> UpdateOutcome { + fn process_updates(&mut self, updates: impl IntoIterator) -> UpdateOutcome { let mut outcome = UpdateOutcome::default(); for update in updates { let PoolUpdate { id, hash, current, destination } = update; @@ -1083,27 +1079,19 @@ impl PoolInternalTransaction { } /// Tracks the result after updating the pool -#[derive(Debug)] -pub struct UpdateOutcome { +#[derive(Default, Debug)] +pub struct UpdateOutcome { /// transactions promoted to the ready queue promoted: Vec, /// transaction that failed and became discarded discarded: Vec, - /// Transactions removed from the Ready pool - removed: Vec>>, -} - -impl Default for UpdateOutcome { - fn default() -> Self { - Self { promoted: vec![], discarded: vec![], removed: vec![] } - } } /// Represents the outcome of a prune pub struct PruneResult { /// A list of added transactions that a pruned marker satisfied pub promoted: Vec>, - /// all transactions that failed to be promoted and now are discarded + /// all transactions that failed to be promoted and now are discarded pub failed: Vec, /// all transactions that were pruned from the ready pool pub pruned: Vec>>, From a0f195590c5b0bc5d7b32a9f449db80aac7018e7 Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Fri, 10 Mar 2023 18:30:36 +0200 Subject: [PATCH 092/191] feat(txpool): default transaction ordering (#1704) --- crates/transaction-pool/src/lib.rs | 2 +- crates/transaction-pool/src/ordering.rs | 23 ++++++++++++++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/crates/transaction-pool/src/lib.rs b/crates/transaction-pool/src/lib.rs index d355e34d248..54297988241 100644 --- a/crates/transaction-pool/src/lib.rs +++ b/crates/transaction-pool/src/lib.rs @@ -81,7 +81,7 @@ pub use crate::{ config::PoolConfig, - ordering::TransactionOrdering, + ordering::{CostOrdering, TransactionOrdering}, traits::{ BestTransactions, OnNewBlockEvent, PoolTransaction, PooledTransaction, PropagateKind, PropagatedTransactions, TransactionOrigin, TransactionPool, diff --git a/crates/transaction-pool/src/ordering.rs b/crates/transaction-pool/src/ordering.rs index f5bf68eb953..b96db095480 100644 --- a/crates/transaction-pool/src/ordering.rs +++ b/crates/transaction-pool/src/ordering.rs @@ -1,5 +1,6 @@ use crate::traits::PoolTransaction; -use std::fmt; +use reth_primitives::U256; +use std::{fmt, marker::PhantomData}; /// Transaction ordering trait to determine the order of transactions. /// @@ -18,3 +19,23 @@ pub trait TransactionOrdering: Send + Sync + 'static { /// Returns the priority score for the given transaction. fn priority(&self, transaction: &Self::Transaction) -> Self::Priority; } + +/// Default ordering for the pool. +/// +/// The transactions are ordered by their cost. The higher the cost, +/// the higher the priority of this transaction is. +#[derive(Debug, Default)] +#[non_exhaustive] +pub struct CostOrdering(PhantomData); + +impl TransactionOrdering for CostOrdering +where + T: PoolTransaction + 'static, +{ + type Priority = U256; + type Transaction = T; + + fn priority(&self, transaction: &Self::Transaction) -> Self::Priority { + transaction.cost() + } +} From 853da85c44e5c05eba8a4b629bc6b93a44d77281 Mon Sep 17 00:00:00 2001 From: sambukowski Date: Sat, 11 Mar 2023 02:54:11 -0700 Subject: [PATCH 093/191] feat(rpc): `eth_transactionBy` impls (#1664) Co-authored-by: Sam Bukowski --- .../primitives/src/transaction/signature.rs | 2 +- crates/rpc/rpc-builder/tests/it/http.rs | 18 ++---- .../rpc/rpc-types/src/eth/transaction/mod.rs | 4 +- crates/rpc/rpc/src/eth/api/server.rs | 19 +++---- crates/rpc/rpc/src/eth/api/transactions.rs | 57 ++++++++++++++++++- 5 files changed, 71 insertions(+), 29 deletions(-) diff --git a/crates/primitives/src/transaction/signature.rs b/crates/primitives/src/transaction/signature.rs index b20c4f79b2e..468855b906b 100644 --- a/crates/primitives/src/transaction/signature.rs +++ b/crates/primitives/src/transaction/signature.rs @@ -6,7 +6,7 @@ use reth_rlp::{Decodable, DecodeError, Encodable}; /// transaction and used to determine the sender of /// the transaction; formally Tr and Ts. This is expanded in Appendix F of yellow paper. #[main_codec] -#[derive(Debug, Clone, PartialEq, Eq, Hash, Default)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, Default)] pub struct Signature { /// The R field of the signature; the point on the curve. pub r: U256, diff --git a/crates/rpc/rpc-builder/tests/it/http.rs b/crates/rpc/rpc-builder/tests/it/http.rs index 79783781483..ad109d084e7 100644 --- a/crates/rpc/rpc-builder/tests/it/http.rs +++ b/crates/rpc/rpc-builder/tests/it/http.rs @@ -9,7 +9,8 @@ use jsonrpsee::{ types::error::{CallError, ErrorCode}, }; use reth_primitives::{ - hex_literal::hex, Address, BlockId, BlockNumberOrTag, Bytes, NodeRecord, H256, H64, U256, + hex_literal::hex, Address, BlockId, BlockNumberOrTag, Bytes, NodeRecord, TxHash, H256, H64, + U256, }; use reth_rpc_api::{ clients::{AdminApiClient, EthApiClient}, @@ -50,6 +51,7 @@ where let address = Address::default(); let index = Index::default(); let hash = H256::default(); + let tx_hash = TxHash::default(); let block_number = BlockNumberOrTag::default(); let call_request = CallRequest::default(); let transaction_request = TransactionRequest::default(); @@ -75,21 +77,13 @@ where EthApiClient::block_uncles_count_by_number(client, block_number).await.unwrap(); EthApiClient::uncle_by_block_hash_and_index(client, hash, index).await.unwrap(); EthApiClient::uncle_by_block_number_and_index(client, block_number, index).await.unwrap(); - EthApiClient::create_access_list(client, call_request.clone(), None).await.unwrap(); + EthApiClient::transaction_by_hash(client, tx_hash).await.unwrap(); + EthApiClient::transaction_by_block_hash_and_index(client, hash, index).await.unwrap(); + EthApiClient::transaction_by_block_number_and_index(client, block_number, index).await.unwrap(); // Unimplemented assert!(is_unimplemented(EthApiClient::syncing(client).await.err().unwrap())); assert!(is_unimplemented(EthApiClient::author(client).await.err().unwrap())); - assert!(is_unimplemented(EthApiClient::transaction_by_hash(client, hash).await.err().unwrap())); - assert!(is_unimplemented( - EthApiClient::transaction_by_block_hash_and_index(client, hash, index).await.err().unwrap() - )); - assert!(is_unimplemented( - EthApiClient::transaction_by_block_number_and_index(client, block_number, index) - .await - .err() - .unwrap() - )); assert!(is_unimplemented(EthApiClient::transaction_receipt(client, hash).await.err().unwrap())); assert!(is_unimplemented( EthApiClient::call(client, call_request.clone(), None, None).await.err().unwrap() diff --git a/crates/rpc/rpc-types/src/eth/transaction/mod.rs b/crates/rpc/rpc-types/src/eth/transaction/mod.rs index a04a3cbd785..f6a2f421333 100644 --- a/crates/rpc/rpc-types/src/eth/transaction/mod.rs +++ b/crates/rpc/rpc-types/src/eth/transaction/mod.rs @@ -74,7 +74,7 @@ impl Transaction { /// /// The block hash, number, and tx index fields should be from the original block where the /// transaction was mined. - pub(crate) fn from_recovered_with_block_context( + pub fn from_recovered_with_block_context( tx: TransactionSignedEcRecovered, block_hash: H256, block_number: BlockNumber, @@ -142,7 +142,7 @@ impl Transaction { max_fee_per_gas, max_priority_fee_per_gas: signed_tx.max_priority_fee_per_gas().map(U128::from), signature: Some(Signature::from_primitive_signature( - signed_tx.signature().clone(), + *signed_tx.signature(), signed_tx.chain_id(), )), gas: U256::from(signed_tx.gas_limit()), diff --git a/crates/rpc/rpc/src/eth/api/server.rs b/crates/rpc/rpc/src/eth/api/server.rs index 3c8a3f96246..0ff4310e0f6 100644 --- a/crates/rpc/rpc/src/eth/api/server.rs +++ b/crates/rpc/rpc/src/eth/api/server.rs @@ -117,29 +117,26 @@ where } /// Handler for: `eth_getTransactionByHash` - async fn transaction_by_hash( - &self, - _hash: H256, - ) -> Result> { - Err(internal_rpc_err("unimplemented")) + async fn transaction_by_hash(&self, hash: H256) -> Result> { + Ok(EthApi::transaction_by_hash(self, hash).await?) } /// Handler for: `eth_getTransactionByBlockHashAndIndex` async fn transaction_by_block_hash_and_index( &self, - _hash: H256, - _index: Index, + hash: H256, + index: Index, ) -> Result> { - Err(internal_rpc_err("unimplemented")) + Ok(EthApi::transaction_by_block_and_tx_index(self, hash, index).await?) } /// Handler for: `eth_getTransactionByBlockNumberAndIndex` async fn transaction_by_block_number_and_index( &self, - _number: BlockNumberOrTag, - _index: Index, + number: BlockNumberOrTag, + index: Index, ) -> Result> { - Err(internal_rpc_err("unimplemented")) + Ok(EthApi::transaction_by_block_and_tx_index(self, number, index).await?) } /// Handler for: `eth_getTransactionReceipt` diff --git a/crates/rpc/rpc/src/eth/api/transactions.rs b/crates/rpc/rpc/src/eth/api/transactions.rs index f861c0585b4..72ce26a63f8 100644 --- a/crates/rpc/rpc/src/eth/api/transactions.rs +++ b/crates/rpc/rpc/src/eth/api/transactions.rs @@ -1,13 +1,12 @@ //! Contains RPC handler implementations specific to transactions - use crate::{ eth::error::{EthApiError, EthResult}, EthApi, }; -use reth_primitives::{Bytes, FromRecoveredTransaction, TransactionSigned, H256}; +use reth_primitives::{BlockId, Bytes, FromRecoveredTransaction, TransactionSigned, H256, U256}; use reth_provider::{BlockProvider, EvmEnvProvider, StateProviderFactory}; use reth_rlp::Decodable; -use reth_rpc_types::TransactionRequest; +use reth_rpc_types::{Index, Transaction, TransactionRequest}; use reth_transaction_pool::{TransactionOrigin, TransactionPool}; impl EthApi @@ -20,6 +19,58 @@ where unimplemented!() } + /// Finds a given trasaction by its hash. + pub(crate) async fn transaction_by_hash( + &self, + hash: H256, + ) -> EthResult> { + let tx_by_hash = match self.client().transaction_by_hash(hash)? { + Some(item) => item, + None => return Ok(None), + }; + + if let Some(tx) = tx_by_hash.into_ecrecovered() { + Ok(Some(Transaction::from_recovered_with_block_context( + tx, + // TODO: this is just stubbed out for now still need to fully implement tx => block + H256::default(), + u64::default(), + U256::from(usize::from(Index::default())), + ))) + } else { + Ok(None) + } + } + + /// Get Transaction by Block and tx index within that Block. + /// Used for both: + /// transaction_by_block_hash_and_index() & + /// transaction_by_block_number_and_index() + pub(crate) async fn transaction_by_block_and_tx_index( + &self, + block_id: impl Into, + index: Index, + ) -> EthResult> { + let block_id = block_id.into(); + let Some(block) = self.client().block(block_id)? else { + return Ok(None); + }; + let block_hash = + self.client().block_hash_for_id(block_id)?.ok_or(EthApiError::UnknownBlockNumber)?; + let Some(tx_signed) = block.body.into_iter().nth(usize::from(index)) else { + return Ok(None); + }; + let Some(tx) = tx_signed.into_ecrecovered() else { + return Ok(None); + }; + Ok(Some(Transaction::from_recovered_with_block_context( + tx, + block_hash, + block.header.number, + U256::from(usize::from(index)), + ))) + } + /// Decodes and recovers the transaction and submits it to the pool. /// /// Returns the hash of the transaction. From e5a0fc655034f82af572d2d7cccdf2dfd9b0e141 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sat, 11 Mar 2023 15:27:24 +0100 Subject: [PATCH 094/191] chore(clippy): make clippy happy (#1710) --- bin/reth/src/node/mod.rs | 6 +-- bin/reth/src/p2p/mod.rs | 2 +- crates/net/discv4/src/lib.rs | 4 +- crates/net/discv4/src/test_utils.rs | 4 +- crates/net/dns/src/query.rs | 9 ++-- crates/net/downloaders/src/bodies/task.rs | 2 +- crates/net/downloaders/src/headers/task.rs | 2 +- crates/net/network/src/session/mod.rs | 54 +++++++++----------- crates/net/network/src/test_utils/testnet.rs | 2 +- crates/net/network/src/transactions.rs | 2 + crates/stages/src/pipeline/mod.rs | 4 +- 11 files changed, 43 insertions(+), 48 deletions(-) diff --git a/bin/reth/src/node/mod.rs b/bin/reth/src/node/mod.rs index 1cd2a6b2a6c..74f5a473875 100644 --- a/bin/reth/src/node/mod.rs +++ b/bin/reth/src/node/mod.rs @@ -342,11 +342,11 @@ impl Command { NetworkManager::builder(config).await?.request_handler(client).split_with_handle(); let known_peers_file = self.network.persistent_peers_file(); - task_executor.spawn_critical_with_signal("p2p network task", |shutdown| async move { - run_network_until_shutdown(shutdown, network, known_peers_file).await + task_executor.spawn_critical_with_signal("p2p network task", |shutdown| { + run_network_until_shutdown(shutdown, network, known_peers_file) }); - task_executor.spawn_critical("p2p eth request handler", async move { eth.await }); + task_executor.spawn_critical("p2p eth request handler", eth); // TODO spawn pool diff --git a/bin/reth/src/p2p/mod.rs b/bin/reth/src/p2p/mod.rs index d8b27777ba5..268168de4cc 100644 --- a/bin/reth/src/p2p/mod.rs +++ b/bin/reth/src/p2p/mod.rs @@ -137,7 +137,7 @@ impl Command { }; let (_, result) = (move || { let client = fetch_client.clone(); - async move { client.get_block_bodies(vec![hash]).await } + client.get_block_bodies(vec![hash]) }) .retry(&backoff) .notify(|err, _| println!("Error requesting block: {err}. Retrying...")) diff --git a/crates/net/discv4/src/lib.rs b/crates/net/discv4/src/lib.rs index d0fadddac83..c5bee30030c 100644 --- a/crates/net/discv4/src/lib.rs +++ b/crates/net/discv4/src/lib.rs @@ -406,10 +406,10 @@ impl Discv4Service { let mut tasks = JoinSet::<()>::new(); let udp = Arc::clone(&socket); - tasks.spawn(async move { receive_loop(udp, ingress_tx, local_node_record.id).await }); + tasks.spawn(receive_loop(udp, ingress_tx, local_node_record.id)); let udp = Arc::clone(&socket); - tasks.spawn(async move { send_loop(udp, egress_rx).await }); + tasks.spawn(send_loop(udp, egress_rx)); let kbuckets = KBucketsTable::new( NodeKey::from(&local_node_record).into(), diff --git a/crates/net/discv4/src/test_utils.rs b/crates/net/discv4/src/test_utils.rs index 0657ec98ab6..d3de9fc2622 100644 --- a/crates/net/discv4/src/test_utils.rs +++ b/crates/net/discv4/src/test_utils.rs @@ -65,10 +65,10 @@ impl MockDiscovery { let mut tasks = JoinSet::<()>::new(); let udp = Arc::clone(&socket); - tasks.spawn(async move { receive_loop(udp, ingress_tx, local_enr.id).await }); + tasks.spawn(receive_loop(udp, ingress_tx, local_enr.id)); let udp = Arc::clone(&socket); - tasks.spawn(async move { send_loop(udp, egress_rx).await }); + tasks.spawn(send_loop(udp, egress_rx)); let (tx, command_rx) = mpsc::channel(128); let this = Self { diff --git a/crates/net/dns/src/query.rs b/crates/net/dns/src/query.rs index 34b5deccc6d..9ef45bceca5 100644 --- a/crates/net/dns/src/query.rs +++ b/crates/net/dns/src/query.rs @@ -59,18 +59,15 @@ impl QueryPool { pub(crate) fn resolve_root(&mut self, link: LinkEntry) { let resolver = Arc::clone(&self.resolver); let timeout = self.lookup_timeout; - self.queued_queries.push_back(Query::Root(Box::pin(async move { - resolve_root(resolver, link, timeout).await - }))) + self.queued_queries.push_back(Query::Root(Box::pin(resolve_root(resolver, link, timeout)))) } /// Resolves the [DnsEntry] for `` pub(crate) fn resolve_entry(&mut self, link: LinkEntry, hash: String, kind: ResolveKind) { let resolver = Arc::clone(&self.resolver); let timeout = self.lookup_timeout; - self.queued_queries.push_back(Query::Entry(Box::pin(async move { - resolve_entry(resolver, link, hash, kind, timeout).await - }))) + self.queued_queries + .push_back(Query::Entry(Box::pin(resolve_entry(resolver, link, hash, kind, timeout)))) } /// Advances the state of the queries diff --git a/crates/net/downloaders/src/bodies/task.rs b/crates/net/downloaders/src/bodies/task.rs index a59c7d9e406..bf7ced932ab 100644 --- a/crates/net/downloaders/src/bodies/task.rs +++ b/crates/net/downloaders/src/bodies/task.rs @@ -76,7 +76,7 @@ impl TaskDownloader { downloader, }; - spawner.spawn(async move { downloader.await }.boxed()); + spawner.spawn(downloader.boxed()); Self { from_downloader: UnboundedReceiverStream::new(bodies_rx), to_downloader } } diff --git a/crates/net/downloaders/src/headers/task.rs b/crates/net/downloaders/src/headers/task.rs index 1d8a4ab1af4..bce7a8d0f87 100644 --- a/crates/net/downloaders/src/headers/task.rs +++ b/crates/net/downloaders/src/headers/task.rs @@ -68,7 +68,7 @@ impl TaskDownloader { updates: UnboundedReceiverStream::new(updates_rx), downloader, }; - spawner.spawn(async move { downloader.await }.boxed()); + spawner.spawn(downloader.boxed()); Self { from_downloader: UnboundedReceiverStream::new(headers_rx), to_downloader } } diff --git a/crates/net/network/src/session/mod.rs b/crates/net/network/src/session/mod.rs index f617c9516a1..03d67f915f5 100644 --- a/crates/net/network/src/session/mod.rs +++ b/crates/net/network/src/session/mod.rs @@ -169,7 +169,7 @@ impl SessionManager { where F: Future + Send + 'static, { - self.executor.spawn(async move { f.await }.boxed()); + self.executor.spawn(f.boxed()); } /// Invoked on a received status update. @@ -209,20 +209,17 @@ impl SessionManager { let hello_message = self.hello_message.clone(); let status = self.status; let fork_filter = self.fork_filter.clone(); - self.spawn(async move { - start_pending_incoming_session( - disconnect_rx, - session_id, - metered_stream, - pending_events, - remote_addr, - secret_key, - hello_message, - status, - fork_filter, - ) - .await - }); + self.spawn(start_pending_incoming_session( + disconnect_rx, + session_id, + metered_stream, + pending_events, + remote_addr, + secret_key, + hello_message, + status, + fork_filter, + )); let handle = PendingSessionHandle { disconnect_tx: Some(disconnect_tx), @@ -243,21 +240,18 @@ impl SessionManager { let fork_filter = self.fork_filter.clone(); let status = self.status; let band_with_meter = self.bandwidth_meter.clone(); - self.spawn(async move { - start_pending_outbound_session( - disconnect_rx, - pending_events, - session_id, - remote_addr, - remote_peer_id, - secret_key, - hello_message, - status, - fork_filter, - band_with_meter, - ) - .await - }); + self.spawn(start_pending_outbound_session( + disconnect_rx, + pending_events, + session_id, + remote_addr, + remote_peer_id, + secret_key, + hello_message, + status, + fork_filter, + band_with_meter, + )); let handle = PendingSessionHandle { disconnect_tx: Some(disconnect_tx), diff --git a/crates/net/network/src/test_utils/testnet.rs b/crates/net/network/src/test_utils/testnet.rs index b31e4bb32e1..bfd2645a47a 100644 --- a/crates/net/network/src/test_utils/testnet.rs +++ b/crates/net/network/src/test_utils/testnet.rs @@ -77,7 +77,7 @@ where &mut self, configs: impl IntoIterator>, ) -> Result<(), NetworkError> { - let peers = configs.into_iter().map(|c| async { c.launch().await }).collect::>(); + let peers = configs.into_iter().map(|c| c.launch()).collect::>(); let peers = futures::future::join_all(peers).await; for peer in peers { self.peers.push(peer?); diff --git a/crates/net/network/src/transactions.rs b/crates/net/network/src/transactions.rs index 1cf1f0afe9a..109aec0e1a5 100644 --- a/crates/net/network/src/transactions.rs +++ b/crates/net/network/src/transactions.rs @@ -431,6 +431,8 @@ where let pool_transaction = ::from_recovered_transaction(tx); let pool = self.pool.clone(); + + #[allow(clippy::redundant_async_block)] let import = Box::pin(async move { pool.add_external_transaction(pool_transaction).await }); diff --git a/crates/stages/src/pipeline/mod.rs b/crates/stages/src/pipeline/mod.rs index cd8bbbac438..b8db1146291 100644 --- a/crates/stages/src/pipeline/mod.rs +++ b/crates/stages/src/pipeline/mod.rs @@ -490,7 +490,9 @@ mod tests { let events = pipeline.events(); // Run pipeline - tokio::spawn(async move { pipeline.run(db).await }); + tokio::spawn(async move { + pipeline.run(db).await.unwrap(); + }); // Check that the stages were run in order assert_eq!( From a4a80a713ef04112f428e0329d9d2cda97b2d362 Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Sun, 12 Mar 2023 10:33:24 +0200 Subject: [PATCH 095/191] refactor(sync): td stage consensus checks (#1706) --- bin/reth/src/chain/import.rs | 8 +- bin/reth/src/node/mod.rs | 8 +- crates/stages/src/sets.rs | 2 +- crates/stages/src/stages/total_difficulty.rs | 79 ++++++++++---------- 4 files changed, 49 insertions(+), 48 deletions(-) diff --git a/bin/reth/src/chain/import.rs b/bin/reth/src/chain/import.rs index a694ba2001a..1e96c71ab3f 100644 --- a/bin/reth/src/chain/import.rs +++ b/bin/reth/src/chain/import.rs @@ -152,10 +152,10 @@ impl ImportCommand { NoopStatusUpdater::default(), factory.clone(), ) - .set(TotalDifficultyStage { - chain_spec: self.chain.clone(), - commit_threshold: config.stages.total_difficulty.commit_threshold, - }) + .set( + TotalDifficultyStage::new(consensus.clone()) + .with_commit_threshold(config.stages.total_difficulty.commit_threshold), + ) .set(SenderRecoveryStage { commit_threshold: config.stages.sender_recovery.commit_threshold, }) diff --git a/bin/reth/src/node/mod.rs b/bin/reth/src/node/mod.rs index 74f5a473875..daeeb5461ef 100644 --- a/bin/reth/src/node/mod.rs +++ b/bin/reth/src/node/mod.rs @@ -454,10 +454,10 @@ impl Command { updater, factory.clone(), ) - .set(TotalDifficultyStage { - chain_spec: self.chain.clone(), - commit_threshold: stage_conf.total_difficulty.commit_threshold, - }) + .set( + TotalDifficultyStage::new(consensus.clone()) + .with_commit_threshold(stage_conf.total_difficulty.commit_threshold), + ) .set(SenderRecoveryStage { commit_threshold: stage_conf.sender_recovery.commit_threshold, }) diff --git a/crates/stages/src/sets.rs b/crates/stages/src/sets.rs index 54332e2f61f..aca8bca593d 100644 --- a/crates/stages/src/sets.rs +++ b/crates/stages/src/sets.rs @@ -140,7 +140,7 @@ where fn builder(self) -> StageSetBuilder { StageSetBuilder::default() .add_stage(HeaderStage::new(self.header_downloader, self.consensus.clone())) - .add_stage(TotalDifficultyStage::default()) + .add_stage(TotalDifficultyStage::new(self.consensus.clone())) .add_stage(BodyStage { downloader: self.body_downloader, consensus: self.consensus }) } } diff --git a/crates/stages/src/stages/total_difficulty.rs b/crates/stages/src/stages/total_difficulty.rs index a02b368fa8a..69f1ccad94e 100644 --- a/crates/stages/src/stages/total_difficulty.rs +++ b/crates/stages/src/stages/total_difficulty.rs @@ -8,9 +8,10 @@ use reth_db::{ tables, transaction::{DbTx, DbTxMut}, }; -use reth_interfaces::{consensus::ConsensusError, provider::ProviderError}; -use reth_primitives::{ChainSpec, Hardfork, EMPTY_OMMER_ROOT, MAINNET, U256}; +use reth_interfaces::{consensus::Consensus, provider::ProviderError}; +use reth_primitives::U256; use reth_provider::Transaction; +use std::sync::Arc; use tracing::*; /// The [`StageId`] of the total difficulty stage. @@ -23,15 +24,22 @@ pub const TOTAL_DIFFICULTY: StageId = StageId("TotalDifficulty"); /// table. #[derive(Debug, Clone)] pub struct TotalDifficultyStage { - /// The chain specification. - pub chain_spec: ChainSpec, + /// Consensus client implementation + consensus: Arc, /// The number of table entries to commit at once - pub commit_threshold: u64, + commit_threshold: u64, } -impl Default for TotalDifficultyStage { - fn default() -> Self { - Self { chain_spec: MAINNET.clone(), commit_threshold: 100_000 } +impl TotalDifficultyStage { + /// Create a new total difficulty stage + pub fn new(consensus: Arc) -> Self { + Self { consensus, commit_threshold: 100_000 } + } + + /// Set a commit threshold on total difficulty stage + pub fn with_commit_threshold(mut self, commit_threshold: u64) -> Self { + self.commit_threshold = commit_threshold; + self } } @@ -55,6 +63,7 @@ impl Stage for TotalDifficultyStage { // Acquire cursor over total difficulty and headers tables let mut cursor_td = tx.cursor_write::()?; + let mut cursor_canonical = tx.cursor_read::()?; let mut cursor_headers = tx.cursor_read::()?; // Get latest total difficulty @@ -66,38 +75,22 @@ impl Stage for TotalDifficultyStage { let mut td: U256 = last_entry.1.into(); debug!(target: "sync::stages::total_difficulty", ?td, block_number = last_header_number, "Last total difficulty entry"); - let walker = cursor_headers - .walk(Some(start_block))? - .take_while(|e| e.as_ref().map(|(_, h)| h.number <= end_block).unwrap_or_default()); + // Acquire canonical walker + let walker = cursor_canonical.walk_range(start_block..=end_block)?; + // Walk over newly inserted headers, update & insert td for entry in walker { - let (key, header) = entry?; + let (number, hash) = entry?; + let (_, header) = + cursor_headers.seek_exact(number)?.ok_or(ProviderError::Header { number })?; + let header = header.seal(hash); td += header.difficulty; - if self.chain_spec.fork(Hardfork::Paris).active_at_ttd(td, header.difficulty) { - if header.difficulty != U256::ZERO { - return Err(StageError::Validation { - block: header.number, - error: ConsensusError::TheMergeDifficultyIsNotZero, - }) - } - - if header.nonce != 0 { - return Err(StageError::Validation { - block: header.number, - error: ConsensusError::TheMergeNonceIsNotZero, - }) - } + self.consensus + .validate_header(&header, td) + .map_err(|error| StageError::Validation { block: header.number, error })?; - if header.ommers_hash != EMPTY_OMMER_ROOT { - return Err(StageError::Validation { - block: header.number, - error: ConsensusError::TheMergeOmmerRootIsNotEmpty, - }) - } - } - - cursor_td.append(key, td.into())?; + cursor_td.append(number, td.into())?; } let done = !capped; @@ -120,8 +113,11 @@ impl Stage for TotalDifficultyStage { #[cfg(test)] mod tests { use reth_db::transaction::DbTx; - use reth_interfaces::test_utils::generators::{random_header, random_header_range}; - use reth_primitives::{BlockNumber, SealedHeader, MAINNET}; + use reth_interfaces::test_utils::{ + generators::{random_header, random_header_range}, + TestConsensus, + }; + use reth_primitives::{BlockNumber, SealedHeader}; use super::*; use crate::test_utils::{ @@ -173,12 +169,17 @@ mod tests { struct TotalDifficultyTestRunner { tx: TestTransaction, + consensus: Arc, commit_threshold: u64, } impl Default for TotalDifficultyTestRunner { fn default() -> Self { - Self { tx: Default::default(), commit_threshold: 500 } + Self { + tx: Default::default(), + consensus: Arc::new(TestConsensus::default()), + commit_threshold: 500, + } } } @@ -191,7 +192,7 @@ mod tests { fn stage(&self) -> Self::S { TotalDifficultyStage { - chain_spec: MAINNET.clone(), + consensus: self.consensus.clone(), commit_threshold: self.commit_threshold, } } From 1846f2d73c7793423f5987d36905e1b13100428f Mon Sep 17 00:00:00 2001 From: chirag-bgh <76247491+chirag-bgh@users.noreply.github.com> Date: Sun, 12 Mar 2023 14:46:47 +0530 Subject: [PATCH 096/191] feat: Implement is_syncing subscription handler (#1562) Co-authored-by: Matthias Seitz --- crates/rpc/rpc-builder/src/eth.rs | 2 +- crates/rpc/rpc/src/eth/pubsub.rs | 92 ++++++++++++++++++++++++------- 2 files changed, 74 insertions(+), 20 deletions(-) diff --git a/crates/rpc/rpc-builder/src/eth.rs b/crates/rpc/rpc-builder/src/eth.rs index 45c136e91b8..eec235cdd8c 100644 --- a/crates/rpc/rpc-builder/src/eth.rs +++ b/crates/rpc/rpc-builder/src/eth.rs @@ -14,7 +14,7 @@ pub struct EthHandlers { /// Polling based filter handler available on all transports pub filter: EthFilter, /// Handler for subscriptions only available for transports that support it (ws, ipc) - pub pubsub: Option>, + pub pubsub: Option>, } /// Additional config values for the eth namespace diff --git a/crates/rpc/rpc/src/eth/pubsub.rs b/crates/rpc/rpc/src/eth/pubsub.rs index 57f66f974f9..01064fdb219 100644 --- a/crates/rpc/rpc/src/eth/pubsub.rs +++ b/crates/rpc/rpc/src/eth/pubsub.rs @@ -1,12 +1,15 @@ //! `eth_` PubSub RPC handler implementation use jsonrpsee::{types::SubscriptionResult, SubscriptionSink}; -use reth_interfaces::events::ChainEventSubscriptions; +use reth_interfaces::{events::ChainEventSubscriptions, sync::SyncStateProvider}; use reth_primitives::{rpc::FilteredParams, TxHash}; use reth_provider::{BlockProvider, EvmEnvProvider}; use reth_rpc_api::EthPubSubApiServer; use reth_rpc_types::{ - pubsub::{Params, SubscriptionKind, SubscriptionResult as EthSubscriptionResult}, + pubsub::{ + Params, PubSubSyncStatus, SubscriptionKind, SubscriptionResult as EthSubscriptionResult, + SyncStatusMetadata, + }, Header, }; use reth_tasks::{TaskSpawner, TokioTaskExecutor}; @@ -20,21 +23,21 @@ use tokio_stream::{ /// /// This handles `eth_subscribe` RPC calls. #[derive(Clone)] -pub struct EthPubSub { +pub struct EthPubSub { /// All nested fields bundled together. - inner: EthPubSubInner, + inner: EthPubSubInner, /// The type that's used to spawn subscription tasks. subscription_task_spawner: Box, } // === impl EthPubSub === -impl EthPubSub { +impl EthPubSub { /// Creates a new, shareable instance. /// /// Subscription tasks are spawned via [tokio::task::spawn] - pub fn new(client: Client, pool: Pool, chain_events: Events) -> Self { - Self::with_spawner(client, pool, chain_events, Box::::default()) + pub fn new(client: Client, pool: Pool, chain_events: Events, network: Network) -> Self { + Self::with_spawner(client, pool, chain_events, network, Box::::default()) } /// Creates a new, shareable instance. @@ -42,18 +45,20 @@ impl EthPubSub { client: Client, pool: Pool, chain_events: Events, + network: Network, subscription_task_spawner: Box, ) -> Self { - let inner = EthPubSubInner { client, pool, chain_events }; + let inner = EthPubSubInner { client, pool, chain_events, network }; Self { inner, subscription_task_spawner } } } -impl EthPubSubApiServer for EthPubSub +impl EthPubSubApiServer for EthPubSub where Client: BlockProvider + EvmEnvProvider + Clone + 'static, Pool: TransactionPool + 'static, Events: ChainEventSubscriptions + Clone + 'static, + Network: SyncStateProvider + Clone + 'static, { fn subscribe( &self, @@ -65,7 +70,7 @@ where let pubsub = self.inner.clone(); self.subscription_task_spawner.spawn(Box::pin(async move { - handle_accepted(pubsub, sink, kind, params).await; + handle_accepted(pubsub, sink, kind, params, Box::::default()).await; })); Ok(()) @@ -73,15 +78,17 @@ where } /// The actual handler for and accepted [`EthPubSub::subscribe`] call. -async fn handle_accepted( - pubsub: EthPubSubInner, +async fn handle_accepted( + pubsub: EthPubSubInner, mut accepted_sink: SubscriptionSink, kind: SubscriptionKind, params: Option, + subscription_task_spawner: Box, ) where - Client: BlockProvider + EvmEnvProvider + 'static, + Client: BlockProvider + EvmEnvProvider + Clone + 'static, Pool: TransactionPool + 'static, - Events: ChainEventSubscriptions + 'static, + Events: ChainEventSubscriptions + Clone + 'static, + Network: SyncStateProvider + Clone + 'static, { // if no params are provided, used default filter params let _params = match params { @@ -106,12 +113,35 @@ async fn handle_accepted( accepted_sink.pipe_from_stream(stream).await; } SubscriptionKind::Syncing => { - // TODO subscribe new blocks -> read is_syncing from network + subscription_task_spawner.spawn(Box::pin(async move { + // get new block subscription + let mut new_blocks = + UnboundedReceiverStream::new(pubsub.chain_events.subscribe_new_blocks()); + // get current sync status + let mut initial_sync_status = pubsub.network.is_syncing(); + let current_sub_res = pubsub.sync_status(initial_sync_status).await; + + // send the current status immediately + let _ = accepted_sink.send(¤t_sub_res); + + while (new_blocks.next().await).is_some() { + let current_syncing = pubsub.network.is_syncing(); + // Only send a new response if the sync status has changed + if current_syncing != initial_sync_status { + // Update the sync status on each new block + initial_sync_status = current_syncing; + + // send a new message now that the status changed + let sync_status = pubsub.sync_status(current_syncing).await; + let _ = accepted_sink.send(&sync_status); + } + } + })); } } } -impl std::fmt::Debug for EthPubSub { +impl std::fmt::Debug for EthPubSub { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("EthPubSub").finish_non_exhaustive() } @@ -119,18 +149,41 @@ impl std::fmt::Debug for EthPubSub { /// Container type `EthPubSub` #[derive(Clone)] -struct EthPubSubInner { +struct EthPubSubInner { /// The transaction pool. pool: Pool, /// The client that can interact with the chain. client: Client, /// A type that allows to create new event subscriptions, chain_events: Events, + /// The network. + network: Network, } // == impl EthPubSubInner === -impl EthPubSubInner +impl EthPubSubInner +where + Client: BlockProvider + 'static, +{ + /// Returns the current sync status for the `syncing` subscription + async fn sync_status(&self, is_syncing: bool) -> EthSubscriptionResult { + let current_block = + self.client.chain_info().map(|info| info.best_number).unwrap_or_default(); + if is_syncing { + EthSubscriptionResult::SyncState(PubSubSyncStatus::Detailed(SyncStatusMetadata { + syncing: true, + starting_block: 0, + current_block, + highest_block: Some(current_block), + })) + } else { + EthSubscriptionResult::SyncState(PubSubSyncStatus::Simple(false)) + } + } +} + +impl EthPubSubInner where Pool: TransactionPool + 'static, { @@ -140,10 +193,11 @@ where } } -impl EthPubSubInner +impl EthPubSubInner where Client: BlockProvider + EvmEnvProvider + 'static, Events: ChainEventSubscriptions + 'static, + Network: SyncStateProvider + 'static, { /// Returns a stream that yields all new RPC blocks. fn into_new_headers_stream(self) -> impl Stream { From 12e94029acb24b750711262e8e4dc0e15f412510 Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Sun, 12 Mar 2023 14:33:55 +0200 Subject: [PATCH 097/191] fix(executor): reset access list (#1716) --- crates/revm/revm-primitives/src/env.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/revm/revm-primitives/src/env.rs b/crates/revm/revm-primitives/src/env.rs index 84465494eff..a9ceac12dec 100644 --- a/crates/revm/revm-primitives/src/env.rs +++ b/crates/revm/revm-primitives/src/env.rs @@ -81,6 +81,7 @@ pub fn fill_tx_env(tx_env: &mut TxEnv, transaction: &TransactionSigned, sender: tx_env.data = input.0.clone(); tx_env.chain_id = *chain_id; tx_env.nonce = Some(*nonce); + tx_env.access_list.clear(); } Transaction::Eip2930(TxEip2930 { nonce, From ff34c0ec2a3df1f7c81ea6148a9160f915e97851 Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Sun, 12 Mar 2023 19:38:29 +0200 Subject: [PATCH 098/191] chore(executor): add tx hash to evm error (#1714) --- crates/executor/src/executor.rs | 7 ++++--- crates/interfaces/src/executor.rs | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/crates/executor/src/executor.rs b/crates/executor/src/executor.rs index 7b682b65f4c..01a291b3d9a 100644 --- a/crates/executor/src/executor.rs +++ b/crates/executor/src/executor.rs @@ -376,12 +376,13 @@ where // Fill revm structure. fill_tx_env(&mut self.evm.env.tx, transaction, sender); - let out = if self.stack.should_inspect(&self.evm.env, transaction.hash()) { + let hash = transaction.hash(); + let out = if self.stack.should_inspect(&self.evm.env, hash) { // execution with inspector. let output = self.evm.inspect(&mut self.stack); tracing::trace!( target: "evm", - hash = ?transaction.hash(), ?output, ?transaction, env = ?self.evm.env, + ?hash, ?output, ?transaction, env = ?self.evm.env, "Executed transaction" ); output @@ -389,7 +390,7 @@ where // main execution. self.evm.transact() }; - out.map_err(|e| Error::EVM(format!("{e:?}"))) + out.map_err(|e| Error::EVM { hash, message: format!("{e:?}") }) } /// Runs the provided transactions and commits their state. Will proceed diff --git a/crates/interfaces/src/executor.rs b/crates/interfaces/src/executor.rs index 0165708da30..ad5f4e000bf 100644 --- a/crates/interfaces/src/executor.rs +++ b/crates/interfaces/src/executor.rs @@ -5,8 +5,8 @@ use thiserror::Error; #[allow(missing_docs)] #[derive(Error, Debug, Clone, PartialEq, Eq)] pub enum Error { - #[error("EVM reported invalid transaction:{0}")] - EVM(String), + #[error("EVM reported invalid transaction ({hash:?}): {message}")] + EVM { hash: H256, message: String }, #[error("Example of error.")] VerificationFailed, #[error("Fatal internal error")] From 29a2e1ab3a81f75cf6b5fefd41ed66c3381a2af1 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 13 Mar 2023 03:36:19 +0100 Subject: [PATCH 099/191] feat(rpc): add EthSubscriptionIdProvider (#1721) --- crates/rpc/ipc/src/server/mod.rs | 1 + crates/rpc/rpc-builder/src/lib.rs | 45 ++++++++++++++++-- crates/rpc/rpc/src/eth/id_provider.rs | 67 +++++++++++++++++++++++++++ crates/rpc/rpc/src/eth/mod.rs | 2 + crates/rpc/rpc/src/lib.rs | 2 +- 5 files changed, 111 insertions(+), 6 deletions(-) create mode 100644 crates/rpc/rpc/src/eth/id_provider.rs diff --git a/crates/rpc/ipc/src/server/mod.rs b/crates/rpc/ipc/src/server/mod.rs index e6f46fe1c52..f4718fd5cc3 100644 --- a/crates/rpc/ipc/src/server/mod.rs +++ b/crates/rpc/ipc/src/server/mod.rs @@ -379,6 +379,7 @@ pub struct Builder { settings: Settings, resources: Resources, logger: L, + /// Subscription ID provider. id_provider: Arc, service_builder: tower::ServiceBuilder, } diff --git a/crates/rpc/rpc-builder/src/lib.rs b/crates/rpc/rpc-builder/src/lib.rs index a9837adff9f..5d66e238703 100644 --- a/crates/rpc/rpc-builder/src/lib.rs +++ b/crates/rpc/rpc-builder/src/lib.rs @@ -58,13 +58,15 @@ use jsonrpsee::{ server::{host_filtering::AllowHosts, rpc_module::Methods}, Error as RpcError, }, - server::{Server, ServerHandle}, + server::{IdProvider, Server, ServerHandle}, RpcModule, }; use reth_ipc::server::IpcServer; use reth_network_api::{NetworkInfo, Peers}; use reth_provider::{BlockProvider, EvmEnvProvider, HeaderProvider, StateProviderFactory}; -use reth_rpc::{AdminApi, DebugApi, EthApi, EthFilter, NetApi, TraceApi, Web3Api}; +use reth_rpc::{ + AdminApi, DebugApi, EthApi, EthFilter, EthSubscriptionIdProvider, NetApi, TraceApi, Web3Api, +}; use reth_rpc_api::servers::*; use reth_transaction_pool::TransactionPool; use serde::{Deserialize, Serialize, Serializer}; @@ -670,11 +672,17 @@ impl RpcServerConfig { pub fn ipc(config: IpcServerBuilder) -> Self { Self::default().with_ipc(config) } + /// Configures the http server + /// + /// Note: this always configures an [EthSubscriptionIdProvider] [IdProvider] for convenience. + /// To set a custom [IdProvider], please use [Self::with_id_provider]. pub fn with_http(mut self, config: ServerBuilder) -> Self { - self.http_server_config = Some(config.http_only()); + self.http_server_config = + Some(config.http_only().set_id_provider(EthSubscriptionIdProvider::default())); self } + /// Configure the corsdomains pub fn with_cors(mut self, cors_domain: String) -> Self { self.http_cors_domains = Some(cors_domain); @@ -682,8 +690,12 @@ impl RpcServerConfig { } /// Configures the ws server + /// + /// Note: this always configures an [EthSubscriptionIdProvider] [IdProvider] for convenience. + /// To set a custom [IdProvider], please use [Self::with_id_provider]. pub fn with_ws(mut self, config: ServerBuilder) -> Self { - self.ws_server_config = Some(config.ws_only()); + self.ws_server_config = + Some(config.ws_only().set_id_provider(EthSubscriptionIdProvider::default())); self } @@ -704,8 +716,31 @@ impl RpcServerConfig { } /// Configures the ipc server + /// + /// Note: this always configures an [EthSubscriptionIdProvider] [IdProvider] for convenience. + /// To set a custom [IdProvider], please use [Self::with_id_provider]. pub fn with_ipc(mut self, mut config: IpcServerBuilder) -> Self { - self.ipc_server_config = Some(config); + self.ipc_server_config = Some(config.set_id_provider(EthSubscriptionIdProvider::default())); + self + } + + /// Sets a custom [IdProvider] for all configured transports. + /// + /// By default all transports use [EthSubscriptionIdProvider] + pub fn with_id_provider(mut self, id_provider: I) -> Self + where + I: IdProvider + Clone + 'static, + { + if let Some(http) = self.http_server_config { + self.http_server_config = Some(http.set_id_provider(id_provider.clone())); + } + if let Some(ws) = self.ws_server_config { + self.ws_server_config = Some(ws.set_id_provider(id_provider.clone())); + } + if let Some(ipc) = self.ipc_server_config { + self.ipc_server_config = Some(ipc.set_id_provider(id_provider)); + } + self } diff --git a/crates/rpc/rpc/src/eth/id_provider.rs b/crates/rpc/rpc/src/eth/id_provider.rs new file mode 100644 index 00000000000..0355d714d72 --- /dev/null +++ b/crates/rpc/rpc/src/eth/id_provider.rs @@ -0,0 +1,67 @@ +use jsonrpsee::types::SubscriptionId; +use std::fmt::Write; + +/// An [IdProvider](jsonrpsee::core::traits::IdProvider) for ethereum subscription ids. +/// +/// Returns new hex-string [QUANTITY](https://ethereum.org/en/developers/docs/apis/json-rpc/#quantities-encoding) ids +#[derive(Debug, Clone, Copy, Default)] +#[non_exhaustive] +pub struct EthSubscriptionIdProvider; + +impl jsonrpsee::core::traits::IdProvider for EthSubscriptionIdProvider { + fn next_id(&self) -> SubscriptionId<'static> { + to_quantity(rand::random::()) + } +} + +/// Returns a hex quantity string for the given value +/// +/// Strips all leading zeros, `0` is returned as `0x0` +#[inline(always)] +fn to_quantity(val: u128) -> SubscriptionId<'static> { + let bytes = val.to_be_bytes(); + let b = bytes.as_slice(); + let non_zero = b.iter().take_while(|b| **b == 0).count(); + let b = &b[non_zero..]; + if b.is_empty() { + return SubscriptionId::Str("0x0".into()) + } + + let mut id = String::with_capacity(2 * b.len() + 2); + id.push_str("0x"); + let first_byte = b[0]; + write!(id, "{first_byte:x}").unwrap(); + + for byte in &b[1..] { + write!(id, "{byte:02x}").unwrap(); + } + id.into() +} + +#[cfg(test)] +mod tests { + use super::*; + use reth_primitives::U128; + + #[test] + fn test_id_provider_quantity() { + let id = to_quantity(0); + assert_eq!(id, SubscriptionId::Str("0x0".into())); + let id = to_quantity(1); + assert_eq!(id, SubscriptionId::Str("0x1".into())); + + for _ in 0..1000 { + let val = rand::random::(); + let id = to_quantity(val); + match id { + SubscriptionId::Str(id) => { + let from_hex: U128 = id.parse().unwrap(); + assert_eq!(from_hex, U128::from(val)); + } + SubscriptionId::Num(_) => { + unreachable!() + } + } + } + } +} diff --git a/crates/rpc/rpc/src/eth/mod.rs b/crates/rpc/rpc/src/eth/mod.rs index 42500801a36..c4eea7869d8 100644 --- a/crates/rpc/rpc/src/eth/mod.rs +++ b/crates/rpc/rpc/src/eth/mod.rs @@ -4,9 +4,11 @@ mod api; pub mod cache; pub(crate) mod error; mod filter; +mod id_provider; mod pubsub; mod signer; pub use api::{EthApi, EthApiSpec}; pub use filter::EthFilter; +pub use id_provider::EthSubscriptionIdProvider; pub use pubsub::EthPubSub; diff --git a/crates/rpc/rpc/src/lib.rs b/crates/rpc/rpc/src/lib.rs index f5c60506da7..53a37d4dfd2 100644 --- a/crates/rpc/rpc/src/lib.rs +++ b/crates/rpc/rpc/src/lib.rs @@ -23,7 +23,7 @@ mod web3; pub use admin::AdminApi; pub use debug::DebugApi; pub use engine::EngineApi; -pub use eth::{EthApi, EthApiSpec, EthFilter, EthPubSub}; +pub use eth::{EthApi, EthApiSpec, EthFilter, EthPubSub, EthSubscriptionIdProvider}; pub use layers::{AuthLayer, AuthValidator, JwtAuthValidator, JwtError, JwtSecret}; pub use net::NetApi; pub use trace::TraceApi; From 3ba2398f9586bb41f78e6c94b7ef87d8275b265a Mon Sep 17 00:00:00 2001 From: rakita Date: Mon, 13 Mar 2023 10:35:02 +0100 Subject: [PATCH 100/191] bug: Increment transition_id if block changeset is present (#1718) --- crates/storage/provider/src/transaction.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/crates/storage/provider/src/transaction.rs b/crates/storage/provider/src/transaction.rs index b3aabfa304f..eb7b4518d33 100644 --- a/crates/storage/provider/src/transaction.rs +++ b/crates/storage/provider/src/transaction.rs @@ -719,6 +719,8 @@ where current_transition_id += 1; } + let have_block_changeset = !results.block_changesets.is_empty(); + // If there are any post block changes, we will add account changesets to db. for (address, changeset) in results.block_changesets.into_iter() { trace!(target: "sync::stages::execution", ?address, current_transition_id, "Applying block reward"); @@ -729,7 +731,13 @@ where spurious_dragon_active, )?; } - current_transition_id += 1; + + // Transition is incremeneted every time before Paris hardfork and after + // Shanghai only if there are Withdrawals in the block. So it is correct to + // to increment transition id every time there is a block changeset present. + if have_block_changeset { + current_transition_id += 1; + } } Ok(()) } From a98dc43cb09fee347b7d02f7dc6dda69edc43a16 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 13 Mar 2023 12:13:56 +0100 Subject: [PATCH 101/191] docs: add api handler docs (#1727) --- crates/rpc/rpc/src/trace.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/crates/rpc/rpc/src/trace.rs b/crates/rpc/rpc/src/trace.rs index 2767d6bf99e..5b5663e8b04 100644 --- a/crates/rpc/rpc/src/trace.rs +++ b/crates/rpc/rpc/src/trace.rs @@ -35,6 +35,7 @@ impl TraceApiServer for TraceApi where Client: BlockProvider + StateProviderFactory + EvmEnvProvider + 'static, { + /// Handler for `trace_call` async fn call( &self, _call: CallRequest, @@ -44,6 +45,7 @@ where Err(internal_rpc_err("unimplemented")) } + /// Handler for `trace_callMany` async fn call_many( &self, _calls: Vec<(CallRequest, HashSet)>, @@ -52,6 +54,7 @@ where Err(internal_rpc_err("unimplemented")) } + /// Handler for `trace_rawTransaction` async fn raw_transaction( &self, _data: Bytes, @@ -61,6 +64,7 @@ where Err(internal_rpc_err("unimplemented")) } + /// Handler for `trace_replayBlockTransactions` async fn replay_block_transactions( &self, _block_id: BlockId, @@ -69,6 +73,7 @@ where Err(internal_rpc_err("unimplemented")) } + /// Handler for `trace_replayTransaction` async fn replay_transaction( &self, _transaction: H256, @@ -77,14 +82,17 @@ where Err(internal_rpc_err("unimplemented")) } + /// Handler for `trace_block` async fn block(&self, _block_id: BlockId) -> Result>> { Err(internal_rpc_err("unimplemented")) } + /// Handler for `trace_filter` async fn filter(&self, _filter: TraceFilter) -> Result> { Err(internal_rpc_err("unimplemented")) } + /// Handler for `trace_get` fn trace( &self, _hash: H256, @@ -93,6 +101,7 @@ where Err(internal_rpc_err("unimplemented")) } + /// Handler for `trace_transaction` fn transaction_traces(&self, _hash: H256) -> Result>> { Err(internal_rpc_err("unimplemented")) } From 102a5a378f80af7d4c4a527b4f288dd04bd1d8d6 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 13 Mar 2023 13:44:43 +0100 Subject: [PATCH 102/191] chore: rename trace handler functions (#1731) --- crates/rpc/rpc-api/src/trace.rs | 21 ++++++++++++++------- crates/rpc/rpc-builder/tests/it/http.rs | 18 +++++++++++------- crates/rpc/rpc/src/trace.rs | 17 ++++++++++------- 3 files changed, 35 insertions(+), 21 deletions(-) diff --git a/crates/rpc/rpc-api/src/trace.rs b/crates/rpc/rpc-api/src/trace.rs index 7ff742f237c..431ab04bb9c 100644 --- a/crates/rpc/rpc-api/src/trace.rs +++ b/crates/rpc/rpc-api/src/trace.rs @@ -12,7 +12,7 @@ use std::collections::HashSet; pub trait TraceApi { /// Executes the given call and returns a number of possible traces for it. #[method(name = "trace_call")] - async fn call( + async fn trace_call( &self, call: CallRequest, trace_types: HashSet, @@ -23,7 +23,7 @@ pub trait TraceApi { /// on top of a pending block with all n-1 transactions applied (traced) first. Allows to trace /// dependent transactions. #[method(name = "trace_callMany")] - async fn call_many( + async fn trace_call_many( &self, calls: Vec<(CallRequest, HashSet)>, block_id: Option, @@ -33,7 +33,7 @@ pub trait TraceApi { /// /// Expects a raw transaction data #[method(name = "trace_rawTransaction")] - async fn raw_transaction( + async fn trace_raw_transaction( &self, data: Bytes, trace_types: HashSet, @@ -58,17 +58,24 @@ pub trait TraceApi { /// Returns traces created at given block. #[method(name = "trace_block")] - async fn block(&self, block_id: BlockId) -> Result>>; + async fn trace_block( + &self, + block_id: BlockId, + ) -> Result>>; /// Returns traces matching given filter #[method(name = "trace_filter")] - async fn filter(&self, filter: TraceFilter) -> Result>; + async fn trace_filter(&self, filter: TraceFilter) -> Result>; /// Returns transaction trace at given index. #[method(name = "trace_get")] - fn trace(&self, hash: H256, indices: Vec) -> Result>; + fn trace_get( + &self, + hash: H256, + indices: Vec, + ) -> Result>; /// Returns all traces of given transaction. #[method(name = "trace_transaction")] - fn transaction_traces(&self, hash: H256) -> Result>>; + fn trace_transaction(&self, hash: H256) -> Result>>; } diff --git a/crates/rpc/rpc-builder/tests/it/http.rs b/crates/rpc/rpc-builder/tests/it/http.rs index ad109d084e7..898beae6d84 100644 --- a/crates/rpc/rpc-builder/tests/it/http.rs +++ b/crates/rpc/rpc-builder/tests/it/http.rs @@ -167,14 +167,16 @@ where }; assert!(is_unimplemented( - TraceApiClient::call(client, CallRequest::default(), HashSet::default(), None) + TraceApiClient::trace_call(client, CallRequest::default(), HashSet::default(), None) .await .err() .unwrap() )); - assert!(is_unimplemented(TraceApiClient::call_many(client, vec![], None).await.err().unwrap())); assert!(is_unimplemented( - TraceApiClient::raw_transaction(client, Bytes::default(), HashSet::default(), None) + TraceApiClient::trace_call_many(client, vec![], None).await.err().unwrap() + )); + assert!(is_unimplemented( + TraceApiClient::trace_raw_transaction(client, Bytes::default(), HashSet::default(), None) .await .err() .unwrap() @@ -191,13 +193,15 @@ where .err() .unwrap() )); - assert!(is_unimplemented(TraceApiClient::block(client, block_id).await.err().unwrap())); - assert!(is_unimplemented(TraceApiClient::filter(client, trace_filter).await.err().unwrap())); + assert!(is_unimplemented(TraceApiClient::trace_block(client, block_id).await.err().unwrap())); + assert!(is_unimplemented( + TraceApiClient::trace_filter(client, trace_filter).await.err().unwrap() + )); assert!(is_unimplemented( - TraceApiClient::trace(client, H256::default(), vec![]).await.err().unwrap() + TraceApiClient::trace_get(client, H256::default(), vec![]).await.err().unwrap() )); assert!(is_unimplemented( - TraceApiClient::transaction_traces(client, H256::default()).await.err().unwrap() + TraceApiClient::trace_transaction(client, H256::default()).await.err().unwrap() )); } diff --git a/crates/rpc/rpc/src/trace.rs b/crates/rpc/rpc/src/trace.rs index 5b5663e8b04..b6b20e35148 100644 --- a/crates/rpc/rpc/src/trace.rs +++ b/crates/rpc/rpc/src/trace.rs @@ -36,7 +36,7 @@ where Client: BlockProvider + StateProviderFactory + EvmEnvProvider + 'static, { /// Handler for `trace_call` - async fn call( + async fn trace_call( &self, _call: CallRequest, _trace_types: HashSet, @@ -46,7 +46,7 @@ where } /// Handler for `trace_callMany` - async fn call_many( + async fn trace_call_many( &self, _calls: Vec<(CallRequest, HashSet)>, _block_id: Option, @@ -55,7 +55,7 @@ where } /// Handler for `trace_rawTransaction` - async fn raw_transaction( + async fn trace_raw_transaction( &self, _data: Bytes, _trace_types: HashSet, @@ -83,17 +83,20 @@ where } /// Handler for `trace_block` - async fn block(&self, _block_id: BlockId) -> Result>> { + async fn trace_block( + &self, + _block_id: BlockId, + ) -> Result>> { Err(internal_rpc_err("unimplemented")) } /// Handler for `trace_filter` - async fn filter(&self, _filter: TraceFilter) -> Result> { + async fn trace_filter(&self, _filter: TraceFilter) -> Result> { Err(internal_rpc_err("unimplemented")) } /// Handler for `trace_get` - fn trace( + fn trace_get( &self, _hash: H256, _indices: Vec, @@ -102,7 +105,7 @@ where } /// Handler for `trace_transaction` - fn transaction_traces(&self, _hash: H256) -> Result>> { + fn trace_transaction(&self, _hash: H256) -> Result>> { Err(internal_rpc_err("unimplemented")) } } From e33e93c9a0c116190e2b66693aaf8d38db664ce0 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 13 Mar 2023 14:00:47 +0100 Subject: [PATCH 103/191] chore(txpool): remove Arc for internals (#1733) --- crates/transaction-pool/src/lib.rs | 6 +++--- crates/transaction-pool/src/pool/best.rs | 4 ++-- crates/transaction-pool/src/pool/mod.rs | 6 +++--- crates/transaction-pool/src/pool/pending.rs | 4 ++-- crates/transaction-pool/src/pool/txpool.rs | 2 +- crates/transaction-pool/src/test_utils/mock.rs | 2 +- crates/transaction-pool/src/test_utils/mod.rs | 6 +----- crates/transaction-pool/src/test_utils/pool.rs | 2 +- 8 files changed, 14 insertions(+), 18 deletions(-) diff --git a/crates/transaction-pool/src/lib.rs b/crates/transaction-pool/src/lib.rs index 54297988241..cd006c14c62 100644 --- a/crates/transaction-pool/src/lib.rs +++ b/crates/transaction-pool/src/lib.rs @@ -148,8 +148,8 @@ where T: TransactionOrdering::Transaction>, { /// Create a new transaction pool instance. - pub fn new(client: Arc, ordering: Arc, config: PoolConfig) -> Self { - Self { pool: Arc::new(PoolInner::new(client, ordering, config)) } + pub fn new(validator: V, ordering: T, config: PoolConfig) -> Self { + Self { pool: Arc::new(PoolInner::new(validator, ordering, config)) } } /// Returns the wrapped pool. @@ -302,7 +302,7 @@ where } } -impl Clone for Pool { +impl Clone for Pool { fn clone(&self) -> Self { Self { pool: Arc::clone(&self.pool) } } diff --git a/crates/transaction-pool/src/pool/best.rs b/crates/transaction-pool/src/pool/best.rs index 078e68a58a4..ce68fc5d99c 100644 --- a/crates/transaction-pool/src/pool/best.rs +++ b/crates/transaction-pool/src/pool/best.rs @@ -83,7 +83,7 @@ mod tests { #[test] fn test_best_iter() { - let mut pool = PendingPool::new(Arc::new(MockOrdering::default())); + let mut pool = PendingPool::new(MockOrdering::default()); let mut f = MockTransactionFactory::default(); let num_tx = 10; @@ -109,7 +109,7 @@ mod tests { #[test] fn test_best_iter_invalid() { - let mut pool = PendingPool::new(Arc::new(MockOrdering::default())); + let mut pool = PendingPool::new(MockOrdering::default()); let mut f = MockTransactionFactory::default(); let num_tx = 10; diff --git a/crates/transaction-pool/src/pool/mod.rs b/crates/transaction-pool/src/pool/mod.rs index 1090feedfff..bbcd5aa9d97 100644 --- a/crates/transaction-pool/src/pool/mod.rs +++ b/crates/transaction-pool/src/pool/mod.rs @@ -100,7 +100,7 @@ pub struct PoolInner { /// Internal mapping of addresses to plain ints. identifiers: RwLock, /// Transaction validation. - validator: Arc, + validator: V, /// The internal pool that manages all transactions. pool: RwLock>, /// Pool settings. @@ -115,13 +115,13 @@ pub struct PoolInner { // === impl PoolInner === -impl PoolInner +impl PoolInner where V: TransactionValidator, T: TransactionOrdering::Transaction>, { /// Create a new transaction pool instance. - pub(crate) fn new(validator: Arc, ordering: Arc, config: PoolConfig) -> Self { + pub(crate) fn new(validator: V, ordering: T, config: PoolConfig) -> Self { Self { identifiers: Default::default(), validator, diff --git a/crates/transaction-pool/src/pool/pending.rs b/crates/transaction-pool/src/pool/pending.rs index e5751c6877a..1efdde6dece 100644 --- a/crates/transaction-pool/src/pool/pending.rs +++ b/crates/transaction-pool/src/pool/pending.rs @@ -22,7 +22,7 @@ use std::{ /// is also pending, then this will be moved to the `independent` queue. pub(crate) struct PendingPool { /// How to order transactions. - ordering: Arc, + ordering: T, /// Keeps track of transactions inserted in the pool. /// /// This way we can determine when transactions where submitted to the pool. @@ -46,7 +46,7 @@ pub(crate) struct PendingPool { impl PendingPool { /// Create a new pool instance. - pub(crate) fn new(ordering: Arc) -> Self { + pub(crate) fn new(ordering: T) -> Self { Self { ordering, submission_id: 0, diff --git a/crates/transaction-pool/src/pool/txpool.rs b/crates/transaction-pool/src/pool/txpool.rs index 41bbd959ff7..5458d7ddc05 100644 --- a/crates/transaction-pool/src/pool/txpool.rs +++ b/crates/transaction-pool/src/pool/txpool.rs @@ -98,7 +98,7 @@ pub struct TxPool { impl TxPool { /// Create a new graph pool instance. - pub(crate) fn new(ordering: Arc, config: PoolConfig) -> Self { + pub(crate) fn new(ordering: T, config: PoolConfig) -> Self { Self { sender_info: Default::default(), pending_pool: PendingPool::new(ordering), diff --git a/crates/transaction-pool/src/test_utils/mock.rs b/crates/transaction-pool/src/test_utils/mock.rs index c9d1e5a2393..348c184cd93 100644 --- a/crates/transaction-pool/src/test_utils/mock.rs +++ b/crates/transaction-pool/src/test_utils/mock.rs @@ -23,7 +23,7 @@ pub type MockValidTx = ValidPoolTransaction; /// Create an empty `TxPool` pub(crate) fn mock_tx_pool() -> MockTxPool { - MockTxPool::new(Arc::new(Default::default()), Default::default()) + MockTxPool::new(Default::default(), Default::default()) } /// Sets the value for the field diff --git a/crates/transaction-pool/src/test_utils/mod.rs b/crates/transaction-pool/src/test_utils/mod.rs index 7fd1da50bbd..4a0d67534d7 100644 --- a/crates/transaction-pool/src/test_utils/mod.rs +++ b/crates/transaction-pool/src/test_utils/mod.rs @@ -16,11 +16,7 @@ pub type TestPool = Pool, MockOrdering /// Returns a new [Pool] used for testing purposes pub fn testing_pool() -> TestPool { - Pool::new( - Arc::new(NoopTransactionValidator::default()), - Arc::new(MockOrdering::default()), - Default::default(), - ) + Pool::new(NoopTransactionValidator::default(), MockOrdering::default(), Default::default()) } // A [`TransactionValidator`] that does nothing. diff --git a/crates/transaction-pool/src/test_utils/pool.rs b/crates/transaction-pool/src/test_utils/pool.rs index 30217a86724..c8d79fb16ec 100644 --- a/crates/transaction-pool/src/test_utils/pool.rs +++ b/crates/transaction-pool/src/test_utils/pool.rs @@ -41,7 +41,7 @@ impl MockPool { impl Default for MockPool { fn default() -> Self { - Self { pool: TxPool::new(Arc::new(MockOrdering::default()), Default::default()) } + Self { pool: TxPool::new(MockOrdering::default(), Default::default()) } } } From 139c0c45627e7787ba460b361a3e55c4d984cb5b Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 13 Mar 2023 14:02:52 +0100 Subject: [PATCH 104/191] refactor(rpc): move revm utils to module (#1732) --- crates/rpc/rpc/src/eth/api/call.rs | 192 ++------------------------- crates/rpc/rpc/src/eth/mod.rs | 1 + crates/rpc/rpc/src/eth/revm_utils.rs | 174 ++++++++++++++++++++++++ 3 files changed, 187 insertions(+), 180 deletions(-) create mode 100644 crates/rpc/rpc/src/eth/revm_utils.rs diff --git a/crates/rpc/rpc/src/eth/api/call.rs b/crates/rpc/rpc/src/eth/api/call.rs index dc6e4146ab6..cf92895e3ca 100644 --- a/crates/rpc/rpc/src/eth/api/call.rs +++ b/crates/rpc/rpc/src/eth/api/call.rs @@ -1,16 +1,14 @@ //! Contains RPC handler implementations specific to endpoints that call/execute within evm. -#![allow(unused)] // TODO rm later - use crate::{ - eth::error::{EthApiError, EthResult, InvalidTransactionError, RevertError}, + eth::{ + error::{EthApiError, EthResult, InvalidTransactionError, RevertError}, + revm_utils::{build_call_evm_env, get_precompiles, inspect, transact}, + }, EthApi, }; use ethers_core::utils::get_contract_address; -use reth_primitives::{ - AccessList, AccessListItem, AccessListWithGasUsed, Address, BlockId, BlockNumberOrTag, Bytes, - TransactionKind, H256, U128, U256, -}; +use reth_primitives::{AccessList, Address, BlockId, BlockNumberOrTag, U256}; use reth_provider::{BlockProvider, EvmEnvProvider, StateProvider, StateProviderFactory}; use reth_revm::{ access_list::AccessListInspector, @@ -22,12 +20,10 @@ use reth_rpc_types::{ }; use revm::{ db::{CacheDB, DatabaseRef}, - precompile::{Precompiles, SpecId as PrecompilesSpecId}, primitives::{ - ruint::Uint, BlockEnv, Bytecode, CfgEnv, Env, ExecutionResult, Halt, ResultAndState, - SpecId, TransactTo, TxEnv, + BlockEnv, Bytecode, CfgEnv, Env, ExecutionResult, Halt, ResultAndState, TransactTo, }, - Database, Inspector, + Database, }; // Gas per transaction not creating a contract. @@ -80,7 +76,7 @@ where &self, mut cfg: CfgEnv, block: BlockEnv, - mut request: CallRequest, + request: CallRequest, state: S, state_overrides: Option, ) -> EthResult<(ResultAndState, Env)> @@ -91,7 +87,7 @@ where // impls and providers cfg.disable_block_gas_limit = true; - let mut env = build_call_evm_env(cfg, block, request)?; + let env = build_call_evm_env(cfg, block, request)?; let mut db = SubState::new(State::new(state)); // apply state overrides @@ -120,7 +116,7 @@ where &self, cfg: CfgEnv, block: BlockEnv, - mut request: CallRequest, + request: CallRequest, state: S, ) -> EthResult where @@ -146,7 +142,7 @@ where let no_code_callee = code.map(|code| code.is_empty()).unwrap_or(true); if no_code_callee { // simple transfer, check if caller has sufficient funds - let mut available_funds = + let available_funds = db.basic(env.tx.caller)?.map(|acc| acc.balance).unwrap_or_default(); if env.tx.value > available_funds { return Err(InvalidTransactionError::InsufficientFundsForTransfer.into()) @@ -298,7 +294,7 @@ where // impls and providers cfg.disable_block_gas_limit = true; - let mut env = build_call_evm_env(cfg, block, request.clone())?; + let env = build_call_evm_env(cfg, block, request.clone())?; let mut db = SubState::new(State::new(state)); let from = request.from.unwrap_or_default(); @@ -329,170 +325,6 @@ where } } -/// Returns the addresses of the precompiles corresponding to the SpecId. -fn get_precompiles(spec_id: &SpecId) -> Vec { - let spec = match spec_id { - SpecId::FRONTIER | SpecId::FRONTIER_THAWING => return vec![], - SpecId::HOMESTEAD | SpecId::DAO_FORK | SpecId::TANGERINE | SpecId::SPURIOUS_DRAGON => { - PrecompilesSpecId::HOMESTEAD - } - SpecId::BYZANTIUM | SpecId::CONSTANTINOPLE | SpecId::PETERSBURG => { - PrecompilesSpecId::BYZANTIUM - } - SpecId::ISTANBUL | SpecId::MUIR_GLACIER => PrecompilesSpecId::ISTANBUL, - SpecId::BERLIN | - SpecId::LONDON | - SpecId::ARROW_GLACIER | - SpecId::GRAY_GLACIER | - SpecId::MERGE | - SpecId::SHANGHAI | - SpecId::CANCUN => PrecompilesSpecId::BERLIN, - SpecId::LATEST => PrecompilesSpecId::LATEST, - }; - Precompiles::new(spec).addresses().into_iter().map(Address::from).collect() -} - -/// Executes the [Env] against the given [Database] without committing state changes. -pub(crate) fn transact(db: S, env: Env) -> EthResult<(ResultAndState, Env)> -where - S: Database, - ::Error: Into, -{ - let mut evm = revm::EVM::with_env(env); - evm.database(db); - let res = evm.transact()?; - Ok((res, evm.env)) -} - -/// Executes the [Env] against the given [Database] without committing state changes. -pub(crate) fn inspect(db: S, env: Env, inspector: I) -> EthResult<(ResultAndState, Env)> -where - S: Database, - ::Error: Into, - I: Inspector, -{ - let mut evm = revm::EVM::with_env(env); - evm.database(db); - let res = evm.inspect(inspector)?; - Ok((res, evm.env)) -} - -/// Creates a new [Env] to be used for executing the [CallRequest] in `eth_call` -pub(crate) fn build_call_evm_env( - mut cfg: CfgEnv, - block: BlockEnv, - request: CallRequest, -) -> EthResult { - let tx = create_txn_env(&block, request)?; - Ok(Env { cfg, block, tx }) -} - -/// Configures a new [TxEnv] for the [CallRequest] -fn create_txn_env(block_env: &BlockEnv, request: CallRequest) -> EthResult { - let CallRequest { - from, - to, - gas_price, - max_fee_per_gas, - max_priority_fee_per_gas, - gas, - value, - data, - nonce, - access_list, - chain_id, - } = request; - - let CallFees { max_priority_fee_per_gas, gas_price } = - CallFees::ensure_fees(gas_price, max_fee_per_gas, max_priority_fee_per_gas)?; - - let gas_limit = gas.unwrap_or(block_env.gas_limit.min(U256::from(u64::MAX))); - - let env = TxEnv { - gas_limit: gas_limit.try_into().map_err(|_| InvalidTransactionError::GasUintOverflow)?, - nonce: nonce - .map(|n| n.try_into().map_err(|n| InvalidTransactionError::NonceTooHigh)) - .transpose()?, - caller: from.unwrap_or_default(), - gas_price, - gas_priority_fee: max_priority_fee_per_gas, - transact_to: to.map(TransactTo::Call).unwrap_or_else(TransactTo::create), - value: value.unwrap_or_default(), - data: data.map(|data| data.0).unwrap_or_default(), - chain_id: chain_id.map(|c| c.as_u64()), - access_list: access_list.map(AccessList::flattened).unwrap_or_default(), - }; - - Ok(env) -} - -/// Helper type for representing the fees of a [CallRequest] -struct CallFees { - /// EIP-1559 priority fee - max_priority_fee_per_gas: Option, - /// Unified gas price setting - /// - /// Will be `0` if unset in request - /// - /// `gasPrice` for legacy, - /// `maxFeePerGas` for EIP-1559 - gas_price: U256, -} - -// === impl CallFees === - -impl CallFees { - /// Ensures the fields of a [CallRequest] are not conflicting - fn ensure_fees( - call_gas_price: Option, - call_max_fee: Option, - call_priority_fee: Option, - ) -> EthResult { - match (call_gas_price, call_max_fee, call_priority_fee) { - (gas_price, None, None) => { - // request for a legacy transaction - // set everything to zero - let gas_price = gas_price.unwrap_or_default(); - Ok(CallFees { gas_price: U256::from(gas_price), max_priority_fee_per_gas: None }) - } - (None, max_fee_per_gas, max_priority_fee_per_gas) => { - // request for eip-1559 transaction - let max_fee = max_fee_per_gas.unwrap_or_default(); - - if let Some(max_priority) = max_priority_fee_per_gas { - if max_priority > max_fee { - // Fail early - return Err( - // `max_priority_fee_per_gas` is greater than the `max_fee_per_gas` - InvalidTransactionError::TipAboveFeeCap.into(), - ) - } - } - Ok(CallFees { - gas_price: U256::from(max_fee), - max_priority_fee_per_gas: max_priority_fee_per_gas.map(U256::from), - }) - } - (Some(gas_price), Some(max_fee_per_gas), Some(max_priority_fee_per_gas)) => { - Err(EthApiError::ConflictingRequestGasPriceAndTipSet { - gas_price, - max_fee_per_gas, - max_priority_fee_per_gas, - }) - } - (Some(gas_price), Some(max_fee_per_gas), None) => { - Err(EthApiError::ConflictingRequestGasPrice { gas_price, max_fee_per_gas }) - } - (Some(gas_price), None, Some(max_priority_fee_per_gas)) => { - Err(EthApiError::RequestLegacyGasPriceAndTipSet { - gas_price, - max_priority_fee_per_gas, - }) - } - } - } -} - /// Applies the given state overrides (a set of [AccountOverride]) to the [CacheDB]. fn apply_state_overrides(overrides: StateOverride, db: &mut CacheDB) -> EthResult<()> where diff --git a/crates/rpc/rpc/src/eth/mod.rs b/crates/rpc/rpc/src/eth/mod.rs index c4eea7869d8..da465c50a55 100644 --- a/crates/rpc/rpc/src/eth/mod.rs +++ b/crates/rpc/rpc/src/eth/mod.rs @@ -6,6 +6,7 @@ pub(crate) mod error; mod filter; mod id_provider; mod pubsub; +pub(crate) mod revm_utils; mod signer; pub use api::{EthApi, EthApiSpec}; diff --git a/crates/rpc/rpc/src/eth/revm_utils.rs b/crates/rpc/rpc/src/eth/revm_utils.rs new file mode 100644 index 00000000000..63b7ac9b8ff --- /dev/null +++ b/crates/rpc/rpc/src/eth/revm_utils.rs @@ -0,0 +1,174 @@ +//! utilities for working with revm + +use crate::eth::error::{EthApiError, EthResult, InvalidTransactionError}; +use reth_primitives::{AccessList, Address, U128, U256}; +use reth_rpc_types::CallRequest; +use revm::{ + precompile::{Precompiles, SpecId as PrecompilesSpecId}, + primitives::{BlockEnv, CfgEnv, Env, ResultAndState, SpecId, TransactTo, TxEnv}, + Database, Inspector, +}; + +/// Returns the addresses of the precompiles corresponding to the SpecId. +pub(crate) fn get_precompiles(spec_id: &SpecId) -> Vec { + let spec = match spec_id { + SpecId::FRONTIER | SpecId::FRONTIER_THAWING => return vec![], + SpecId::HOMESTEAD | SpecId::DAO_FORK | SpecId::TANGERINE | SpecId::SPURIOUS_DRAGON => { + PrecompilesSpecId::HOMESTEAD + } + SpecId::BYZANTIUM | SpecId::CONSTANTINOPLE | SpecId::PETERSBURG => { + PrecompilesSpecId::BYZANTIUM + } + SpecId::ISTANBUL | SpecId::MUIR_GLACIER => PrecompilesSpecId::ISTANBUL, + SpecId::BERLIN | + SpecId::LONDON | + SpecId::ARROW_GLACIER | + SpecId::GRAY_GLACIER | + SpecId::MERGE | + SpecId::SHANGHAI | + SpecId::CANCUN => PrecompilesSpecId::BERLIN, + SpecId::LATEST => PrecompilesSpecId::LATEST, + }; + Precompiles::new(spec).addresses().into_iter().map(Address::from).collect() +} + +/// Executes the [Env] against the given [Database] without committing state changes. +pub(crate) fn transact(db: S, env: Env) -> EthResult<(ResultAndState, Env)> +where + S: Database, + ::Error: Into, +{ + let mut evm = revm::EVM::with_env(env); + evm.database(db); + let res = evm.transact()?; + Ok((res, evm.env)) +} + +/// Executes the [Env] against the given [Database] without committing state changes. +pub(crate) fn inspect(db: S, env: Env, inspector: I) -> EthResult<(ResultAndState, Env)> +where + S: Database, + ::Error: Into, + I: Inspector, +{ + let mut evm = revm::EVM::with_env(env); + evm.database(db); + let res = evm.inspect(inspector)?; + Ok((res, evm.env)) +} + +/// Creates a new [Env] to be used for executing the [CallRequest] in `eth_call` +pub(crate) fn build_call_evm_env( + cfg: CfgEnv, + block: BlockEnv, + request: CallRequest, +) -> EthResult { + let tx = create_txn_env(&block, request)?; + Ok(Env { cfg, block, tx }) +} + +/// Configures a new [TxEnv] for the [CallRequest] +pub(crate) fn create_txn_env(block_env: &BlockEnv, request: CallRequest) -> EthResult { + let CallRequest { + from, + to, + gas_price, + max_fee_per_gas, + max_priority_fee_per_gas, + gas, + value, + data, + nonce, + access_list, + chain_id, + } = request; + + let CallFees { max_priority_fee_per_gas, gas_price } = + CallFees::ensure_fees(gas_price, max_fee_per_gas, max_priority_fee_per_gas)?; + + let gas_limit = gas.unwrap_or(block_env.gas_limit.min(U256::from(u64::MAX))); + + let env = TxEnv { + gas_limit: gas_limit.try_into().map_err(|_| InvalidTransactionError::GasUintOverflow)?, + nonce: nonce + .map(|n| n.try_into().map_err(|_| InvalidTransactionError::NonceTooHigh)) + .transpose()?, + caller: from.unwrap_or_default(), + gas_price, + gas_priority_fee: max_priority_fee_per_gas, + transact_to: to.map(TransactTo::Call).unwrap_or_else(TransactTo::create), + value: value.unwrap_or_default(), + data: data.map(|data| data.0).unwrap_or_default(), + chain_id: chain_id.map(|c| c.as_u64()), + access_list: access_list.map(AccessList::flattened).unwrap_or_default(), + }; + + Ok(env) +} + +/// Helper type for representing the fees of a [CallRequest] +pub(crate) struct CallFees { + /// EIP-1559 priority fee + max_priority_fee_per_gas: Option, + /// Unified gas price setting + /// + /// Will be `0` if unset in request + /// + /// `gasPrice` for legacy, + /// `maxFeePerGas` for EIP-1559 + gas_price: U256, +} + +// === impl CallFees === + +impl CallFees { + /// Ensures the fields of a [CallRequest] are not conflicting + fn ensure_fees( + call_gas_price: Option, + call_max_fee: Option, + call_priority_fee: Option, + ) -> EthResult { + match (call_gas_price, call_max_fee, call_priority_fee) { + (gas_price, None, None) => { + // request for a legacy transaction + // set everything to zero + let gas_price = gas_price.unwrap_or_default(); + Ok(CallFees { gas_price: U256::from(gas_price), max_priority_fee_per_gas: None }) + } + (None, max_fee_per_gas, max_priority_fee_per_gas) => { + // request for eip-1559 transaction + let max_fee = max_fee_per_gas.unwrap_or_default(); + + if let Some(max_priority) = max_priority_fee_per_gas { + if max_priority > max_fee { + // Fail early + return Err( + // `max_priority_fee_per_gas` is greater than the `max_fee_per_gas` + InvalidTransactionError::TipAboveFeeCap.into(), + ) + } + } + Ok(CallFees { + gas_price: U256::from(max_fee), + max_priority_fee_per_gas: max_priority_fee_per_gas.map(U256::from), + }) + } + (Some(gas_price), Some(max_fee_per_gas), Some(max_priority_fee_per_gas)) => { + Err(EthApiError::ConflictingRequestGasPriceAndTipSet { + gas_price, + max_fee_per_gas, + max_priority_fee_per_gas, + }) + } + (Some(gas_price), Some(max_fee_per_gas), None) => { + Err(EthApiError::ConflictingRequestGasPrice { gas_price, max_fee_per_gas }) + } + (Some(gas_price), None, Some(max_priority_fee_per_gas)) => { + Err(EthApiError::RequestLegacyGasPriceAndTipSet { + gas_price, + max_priority_fee_per_gas, + }) + } + } + } +} From c110c9162969bef39188e633f776d3ecc967d027 Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Mon, 13 Mar 2023 15:23:47 +0200 Subject: [PATCH 105/191] refactor(provider): trie provider (#1723) --- crates/stages/src/stages/merkle.rs | 31 ++-- crates/storage/provider/src/transaction.rs | 4 +- crates/storage/provider/src/trie/mod.rs | 177 +++++++++++---------- 3 files changed, 116 insertions(+), 96 deletions(-) diff --git a/crates/stages/src/stages/merkle.rs b/crates/stages/src/stages/merkle.rs index ea310fc46d4..20ebab25b59 100644 --- a/crates/stages/src/stages/merkle.rs +++ b/crates/stages/src/stages/merkle.rs @@ -108,15 +108,15 @@ impl Stage for MerkleStage { } else if to_transition - from_transition > threshold || stage_progress == 0 { debug!(target: "sync::stages::merkle::exec", current = ?stage_progress, target = ?previous_stage_progress, "Rebuilding trie"); // if there are more blocks than threshold it is faster to rebuild the trie - let loader = DBTrieLoader::default(); - loader.calculate_root(tx).map_err(|e| StageError::Fatal(Box::new(e)))? + DBTrieLoader::::new(tx) + .calculate_root() + .map_err(|e| StageError::Fatal(Box::new(e)))? } else { debug!(target: "sync::stages::merkle::exec", current = ?stage_progress, target = ?previous_stage_progress, "Updating trie"); // Iterate over changeset (similar to Hashing stages) and take new values let current_root = tx.get_header(stage_progress)?.state_root; - let loader = DBTrieLoader::default(); - loader - .update_root(tx, current_root, from_transition..to_transition) + DBTrieLoader::::new(tx) + .update_root(current_root, from_transition..to_transition) .map_err(|e| StageError::Fatal(Box::new(e)))? }; @@ -155,14 +155,13 @@ impl Stage for MerkleStage { return Ok(UnwindOutput { stage_progress: input.unwind_to }) } - let loader = DBTrieLoader::default(); let current_root = tx.get_header(input.stage_progress)?.state_root; let from_transition = tx.get_block_transition(input.unwind_to)?; let to_transition = tx.get_block_transition(input.stage_progress)?; - let block_root = loader - .update_root(tx, current_root, from_transition..to_transition) + let block_root = DBTrieLoader::::new(tx) + .update_root(current_root, from_transition..to_transition) .map_err(|e| StageError::Fatal(Box::new(e)))?; if block_root != target_root { @@ -192,6 +191,7 @@ mod tests { use assert_matches::assert_matches; use reth_db::{ cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO, DbDupCursorRW}, + mdbx::{Env, WriteMap}, tables, transaction::{DbTx, DbTxMut}, }; @@ -260,6 +260,12 @@ mod tests { assert!(runner.validate_execution(input, result.ok()).is_ok(), "execution validation"); } + fn create_trie_loader<'tx, 'db>( + tx: &'tx Transaction<'db, Env>, + ) -> DBTrieLoader<'tx, 'db, Env> { + DBTrieLoader::>::new(tx) + } + struct MerkleTestRunner { tx: TestTransaction, clean_threshold: u64, @@ -440,7 +446,7 @@ mod tests { impl MerkleTestRunner { fn state_root(&self) -> Result { - Ok(DBTrieLoader::default().calculate_root(&self.tx.inner()).unwrap()) + Ok(create_trie_loader(&self.tx.inner()).calculate_root().unwrap()) } pub(crate) fn generate_initial_trie( @@ -451,10 +457,9 @@ mod tests { accounts.into_iter().map(|(addr, acc)| (addr, (acc, std::iter::empty()))), )?; - let loader = DBTrieLoader::default(); - let mut tx = self.tx.inner(); - let root = loader.calculate_root(&tx).expect("couldn't create initial trie"); + let root = + create_trie_loader(&tx).calculate_root().expect("couldn't create initial trie"); tx.commit()?; @@ -465,7 +470,7 @@ mod tests { if previous_stage_progress != 0 { let block_root = self.tx.inner().get_header(previous_stage_progress).unwrap().state_root; - let root = DBTrieLoader::default().calculate_root(&self.tx.inner()).unwrap(); + let root = create_trie_loader(&self.tx.inner()).calculate_root().unwrap(); assert_eq!(block_root, root); } Ok(()) diff --git a/crates/storage/provider/src/transaction.rs b/crates/storage/provider/src/transaction.rs index eb7b4518d33..a1bf8bbdf19 100644 --- a/crates/storage/provider/src/transaction.rs +++ b/crates/storage/provider/src/transaction.rs @@ -308,8 +308,8 @@ where // merkle tree { let current_root = self.get_header(parent_block_number)?.state_root; - let loader = DBTrieLoader::default(); - let root = loader.update_root(self, current_root, from..to)?; + let loader = DBTrieLoader::::new(self); + let root = loader.update_root(current_root, from..to)?; if root != block.state_root { return Err(TransactionError::StateTrieRootMismatch { got: root, diff --git a/crates/storage/provider/src/trie/mod.rs b/crates/storage/provider/src/trie/mod.rs index 4b8f5d744ab..1821aa7e16b 100644 --- a/crates/storage/provider/src/trie/mod.rs +++ b/crates/storage/provider/src/trie/mod.rs @@ -1,9 +1,8 @@ -use crate::Transaction; use cita_trie::{PatriciaTrie, Trie}; use hasher::HasherKeccak; use reth_db::{ cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO}, - database::Database, + database::{Database, DatabaseGAT}, models::{AccountBeforeTx, TransitionIdAddress}, tables, transaction::{DbTx, DbTxMut}, @@ -38,11 +37,11 @@ pub enum TrieError { } /// Database wrapper implementing HashDB trait. -struct HashDatabase<'tx, 'itx, DB: Database> { - tx: &'tx Transaction<'itx, DB>, +pub struct HashDatabase<'tx, 'db, DB: Database> { + tx: &'tx >::TXMut, } -impl<'tx, 'itx, DB> cita_trie::DB for HashDatabase<'tx, 'itx, DB> +impl<'tx, 'db, DB> cita_trie::DB for HashDatabase<'tx, 'db, DB> where DB: Database, { @@ -88,9 +87,9 @@ where } } -impl<'tx, 'itx, DB: Database> HashDatabase<'tx, 'itx, DB> { +impl<'tx, 'db, DB: Database> HashDatabase<'tx, 'db, DB> { /// Instantiates a new Database for the accounts trie, with an empty root - fn new(tx: &'tx Transaction<'itx, DB>) -> Result { + pub fn new(tx: &'tx >::TXMut) -> Result { let root = EMPTY_ROOT; if tx.get::(root)?.is_none() { tx.put::(root, [EMPTY_STRING_CODE].to_vec())?; @@ -99,7 +98,10 @@ impl<'tx, 'itx, DB: Database> HashDatabase<'tx, 'itx, DB> { } /// Instantiates a new Database for the accounts trie, with an existing root - fn from_root(tx: &'tx Transaction<'itx, DB>, root: H256) -> Result { + pub fn from_root( + tx: &'tx >::TXMut, + root: H256, + ) -> Result { if root == EMPTY_ROOT { return Self::new(tx) } @@ -109,12 +111,12 @@ impl<'tx, 'itx, DB: Database> HashDatabase<'tx, 'itx, DB> { } /// Database wrapper implementing HashDB trait. -struct DupHashDatabase<'tx, 'itx, DB: Database> { - tx: &'tx Transaction<'itx, DB>, +pub struct DupHashDatabase<'tx, 'db, DB: Database> { + tx: &'tx >::TXMut, key: H256, } -impl<'tx, 'itx, DB> cita_trie::DB for DupHashDatabase<'tx, 'itx, DB> +impl<'tx, 'db, DB> cita_trie::DB for DupHashDatabase<'tx, 'db, DB> where DB: Database, { @@ -170,9 +172,9 @@ where } } -impl<'tx, 'itx, DB: Database> DupHashDatabase<'tx, 'itx, DB> { +impl<'tx, 'db, DB: Database> DupHashDatabase<'tx, 'db, DB> { /// Instantiates a new Database for the storage trie, with an empty root - fn new(tx: &'tx Transaction<'itx, DB>, key: H256) -> Result { + pub fn new(tx: &'tx >::TXMut, key: H256) -> Result { let root = EMPTY_ROOT; let mut cursor = tx.cursor_dup_write::()?; if cursor.seek_by_key_subkey(key, root)?.filter(|entry| entry.hash == root).is_none() { @@ -185,7 +187,11 @@ impl<'tx, 'itx, DB: Database> DupHashDatabase<'tx, 'itx, DB> { } /// Instantiates a new Database for the storage trie, with an existing root - fn from_root(tx: &'tx Transaction<'itx, DB>, key: H256, root: H256) -> Result { + pub fn from_root( + tx: &'tx >::TXMut, + key: H256, + root: H256, + ) -> Result { if root == EMPTY_ROOT { return Self::new(tx, key) } @@ -199,7 +205,7 @@ impl<'tx, 'itx, DB: Database> DupHashDatabase<'tx, 'itx, DB> { /// An Ethereum account, for RLP encoding traits deriving. #[derive(Clone, Copy, Debug, PartialEq, Eq, Default, RlpEncodable, RlpDecodable)] -pub(crate) struct EthAccount { +pub struct EthAccount { /// Account nonce. nonce: u64, /// Account balance. @@ -222,39 +228,47 @@ impl From for EthAccount { } impl EthAccount { - pub(crate) fn from_with_root(acc: Account, storage_root: H256) -> EthAccount { - Self { storage_root, ..Self::from(acc) } + /// Set storage root on account. + pub fn with_storage_root(mut self, storage_root: H256) -> Self { + self.storage_root = storage_root; + self + } + + /// Get account's storage root. + pub fn storage_root(&self) -> H256 { + self.storage_root } } /// Struct for calculating the root of a merkle patricia tree, /// while populating the database with intermediate hashes. -#[derive(Debug, Default)] -pub struct DBTrieLoader; +pub struct DBTrieLoader<'tx, 'db, DB: Database> { + tx: &'tx >::TXMut, +} + +impl<'tx, 'db, DB: Database> DBTrieLoader<'tx, 'db, DB> { + /// Create new instance of trie loader. + pub fn new(tx: &'tx >::TXMut) -> Self { + Self { tx } + } -impl DBTrieLoader { /// Calculates the root of the state trie, saving intermediate hashes in the database. - pub fn calculate_root( - &self, - tx: &Transaction<'_, DB>, - ) -> Result { - tx.clear::()?; - tx.clear::()?; + pub fn calculate_root(&self) -> Result { + self.tx.clear::()?; + self.tx.clear::()?; - let mut accounts_cursor = tx.cursor_read::()?; + let mut accounts_cursor = self.tx.cursor_read::()?; let mut walker = accounts_cursor.walk(None)?; - let db = Arc::new(HashDatabase::new(tx)?); + let db = Arc::new(HashDatabase::::new(self.tx)?); let hasher = Arc::new(HasherKeccak::new()); let mut trie = PatriciaTrie::new(Arc::clone(&db), Arc::clone(&hasher)); while let Some((hashed_address, account)) = walker.next().transpose()? { - let value = EthAccount::from_with_root( - account, - self.calculate_storage_root(tx, hashed_address)?, - ); + let value = EthAccount::from(account) + .with_storage_root(self.calculate_storage_root(hashed_address)?); let mut out = Vec::new(); Encodable::encode(&value, &mut out); @@ -265,18 +279,15 @@ impl DBTrieLoader { Ok(root) } - fn calculate_storage_root( - &self, - tx: &Transaction<'_, DB>, - address: H256, - ) -> Result { - let db = Arc::new(DupHashDatabase::new(tx, address)?); + /// Calculate the accounts storage root. + pub fn calculate_storage_root(&self, address: H256) -> Result { + let db = Arc::new(DupHashDatabase::::new(self.tx, address)?); let hasher = Arc::new(HasherKeccak::new()); let mut trie = PatriciaTrie::new(Arc::clone(&db), Arc::clone(&hasher)); - let mut storage_cursor = tx.cursor_dup_read::()?; + let mut storage_cursor = self.tx.cursor_dup_read::()?; // Should be able to use walk_dup, but any call to next() causes an assert fail in mdbx.c // let mut walker = storage_cursor.walk_dup(address, H256::zero())?; @@ -292,24 +303,23 @@ impl DBTrieLoader { // if root is empty remove it from db if root == EMPTY_ROOT { - tx.delete::(address, None)?; + self.tx.delete::(address, None)?; } Ok(root) } /// Calculates the root of the state trie by updating an existing trie. - pub fn update_root( + pub fn update_root( &self, - tx: &Transaction<'_, DB>, root: H256, tid_range: Range, ) -> Result { - let mut accounts_cursor = tx.cursor_read::()?; + let mut accounts_cursor = self.tx.cursor_read::()?; - let changed_accounts = self.gather_changes(tx, tid_range)?; + let changed_accounts = self.gather_changes(tid_range)?; - let db = Arc::new(HashDatabase::from_root(tx, root)?); + let db = Arc::new(HashDatabase::::from_root(self.tx, root)?); let hasher = Arc::new(HasherKeccak::new()); @@ -320,13 +330,13 @@ impl DBTrieLoader { trie.remove(address.as_bytes())?; let storage_root = EthAccount::decode(&mut account.as_slice())?.storage_root; - self.update_storage_root(tx, storage_root, address, changed_storages)? + self.update_storage_root(storage_root, address, changed_storages)? } else { - self.calculate_storage_root(tx, address)? + self.calculate_storage_root(address)? }; if let Some((_, account)) = accounts_cursor.seek_exact(address)? { - let value = EthAccount::from_with_root(account, storage_root); + let value = EthAccount::from(account).with_storage_root(storage_root); let mut out = Vec::new(); Encodable::encode(&value, &mut out); @@ -336,7 +346,7 @@ impl DBTrieLoader { let new_root = H256::from_slice(trie.root()?.as_slice()); if new_root != root { - let mut cursor = tx.cursor_write::()?; + let mut cursor = self.tx.cursor_write::()?; if cursor.seek_exact(root)?.is_some() { cursor.delete_current()?; } @@ -345,19 +355,19 @@ impl DBTrieLoader { Ok(new_root) } - fn update_storage_root( + /// Update the account's storage root + pub fn update_storage_root( &self, - tx: &Transaction<'_, DB>, root: H256, address: H256, changed_storages: BTreeSet, ) -> Result { - let db = Arc::new(DupHashDatabase::from_root(tx, address, root)?); + let db = Arc::new(DupHashDatabase::::from_root(self.tx, address, root)?); let hasher = Arc::new(HasherKeccak::new()); let mut trie = PatriciaTrie::from(Arc::clone(&db), Arc::clone(&hasher), root.as_bytes())?; - let mut storage_cursor = tx.cursor_dup_read::()?; + let mut storage_cursor = self.tx.cursor_dup_read::()?; for key in changed_storages { if let Some(StorageEntry { value, .. }) = @@ -372,7 +382,7 @@ impl DBTrieLoader { let new_root = H256::from_slice(trie.root()?.as_slice()); if new_root != root { - let mut cursor = tx.cursor_dup_write::()?; + let mut cursor = self.tx.cursor_dup_write::()?; if cursor .seek_by_key_subkey(address, root)? .filter(|entry| entry.hash == root) @@ -384,18 +394,17 @@ impl DBTrieLoader { // if root is empty remove it from db if new_root == EMPTY_ROOT { - tx.delete::(address, None)?; + self.tx.delete::(address, None)?; } Ok(new_root) } - fn gather_changes( + fn gather_changes( &self, - tx: &Transaction<'_, DB>, tid_range: Range, ) -> Result>, TrieError> { - let mut account_cursor = tx.cursor_read::()?; + let mut account_cursor = self.tx.cursor_read::()?; let mut account_changes: BTreeMap> = BTreeMap::new(); @@ -405,7 +414,7 @@ impl DBTrieLoader { account_changes.insert(address, Default::default()); } - let mut storage_cursor = tx.cursor_dup_read::()?; + let mut storage_cursor = self.tx.cursor_dup_read::()?; let start = TransitionIdAddress((tid_range.start, Address::zero())); let end = TransitionIdAddress((tid_range.end, Address::zero())); @@ -430,30 +439,43 @@ impl DBTrieLoader { #[cfg(test)] mod tests { + use crate::Transaction; + use super::*; use assert_matches::assert_matches; use proptest::{prelude::ProptestConfig, proptest}; - use reth_db::{mdbx::test_utils::create_test_rw_db, tables, transaction::DbTxMut}; + use reth_db::{ + mdbx::{test_utils::create_test_rw_db, Env, WriteMap}, + tables, + transaction::DbTxMut, + }; use reth_primitives::{ hex_literal::hex, keccak256, proofs::{genesis_state_root, KeccakHasher, EMPTY_ROOT}, Address, ChainSpec, MAINNET, }; - use std::{collections::HashMap, str::FromStr}; + use std::{collections::HashMap, ops::Deref, str::FromStr}; use triehash::sec_trie_root; + fn create_test_loader<'tx, 'db>( + tx: &'tx Transaction<'db, Env>, + ) -> DBTrieLoader<'tx, 'db, Arc>> { + DBTrieLoader::new(tx.deref()) + } + #[test] fn empty_trie() { - let trie = DBTrieLoader::default(); let db = create_test_rw_db(); let tx = Transaction::new(db.as_ref()).unwrap(); - assert_matches!(trie.calculate_root(&tx), Ok(got) if got == EMPTY_ROOT); + assert_matches!( + create_test_loader(&tx).calculate_root(), + Ok(got) if got == EMPTY_ROOT + ); } #[test] fn single_account_trie() { - let trie = DBTrieLoader::default(); let db = create_test_rw_db(); let tx = Transaction::new(db.as_ref()).unwrap(); let address = Address::from_str("9fe4abd71ad081f091bd06dd1c16f7e92927561e").unwrap(); @@ -463,14 +485,13 @@ mod tests { EthAccount::from(account).encode(&mut encoded_account); let expected = H256(sec_trie_root::([(address, encoded_account)]).0); assert_matches!( - trie.calculate_root(&tx), + create_test_loader(&tx).calculate_root(), Ok(got) if got == expected ); } #[test] fn two_accounts_trie() { - let trie = DBTrieLoader::default(); let db = create_test_rw_db(); let tx = Transaction::new(db.as_ref()).unwrap(); @@ -494,14 +515,13 @@ mod tests { }); let expected = H256(sec_trie_root::(encoded_accounts).0); assert_matches!( - trie.calculate_root(&tx), + create_test_loader(&tx).calculate_root(), Ok(got) if got == expected ); } #[test] fn single_storage_trie() { - let trie = DBTrieLoader::default(); let db = create_test_rw_db(); let tx = Transaction::new(db.as_ref()).unwrap(); @@ -522,14 +542,13 @@ mod tests { }); let expected = H256(sec_trie_root::(encoded_storage).0); assert_matches!( - trie.calculate_storage_root(&tx, hashed_address), + create_test_loader(&tx).calculate_storage_root(hashed_address), Ok(got) if got == expected ); } #[test] fn single_account_with_storage_trie() { - let trie = DBTrieLoader::default(); let db = create_test_rw_db(); let tx = Transaction::new(db.as_ref()).unwrap(); @@ -562,22 +581,19 @@ mod tests { (k, out) }); - let eth_account = EthAccount::from_with_root( - account, - H256(sec_trie_root::(encoded_storage).0), - ); + let storage_root = H256(sec_trie_root::(encoded_storage).0); + let eth_account = EthAccount::from(account).with_storage_root(storage_root); eth_account.encode(&mut out); let expected = H256(sec_trie_root::([(address, out)]).0); assert_matches!( - trie.calculate_root(&tx), + create_test_loader(&tx).calculate_root(), Ok(got) if got == expected ); } #[test] fn verify_genesis() { - let trie = DBTrieLoader::default(); let db = create_test_rw_db(); let mut tx = Transaction::new(db.as_ref()).unwrap(); let ChainSpec { genesis, .. } = MAINNET.clone(); @@ -599,7 +615,7 @@ mod tests { let state_root = genesis_state_root(&genesis.alloc); assert_matches!( - trie.calculate_root(&tx), + create_test_loader(&tx).calculate_root(), Ok(got) if got == state_root ); } @@ -643,13 +659,12 @@ mod tests { BTreeSet::from([keccak256(H256::zero()), keccak256(H256::from_low_u64_be(2))]), )]); assert_matches!( - DBTrieLoader::default().gather_changes(&tx, 32..33), + create_test_loader(&tx).gather_changes(32..33), Ok(got) if got == expected ); } fn test_with_accounts(accounts: BTreeMap)>) { - let trie = DBTrieLoader::default(); let db = create_test_rw_db(); let tx = Transaction::new(db.as_ref()).unwrap(); @@ -675,14 +690,14 @@ mod tests { EMPTY_ROOT }; let mut out = Vec::new(); - EthAccount::from_with_root(account, storage_root).encode(&mut out); + EthAccount::from(account).with_storage_root(storage_root).encode(&mut out); (address, out) }) .collect::)>>(); let expected = H256(sec_trie_root::(encoded_accounts).0); assert_matches!( - trie.calculate_root(&tx), + create_test_loader(&tx).calculate_root(), Ok(got) if got == expected , "where expected is {expected:?}"); } From 31411681ed758419629f1c90133e7102c5919008 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 13 Mar 2023 14:35:28 +0100 Subject: [PATCH 106/191] feat: add ReceiptProvider (#1735) --- crates/storage/provider/src/lib.rs | 4 +- crates/storage/provider/src/providers/mod.rs | 46 ++++++++++++++++++- .../storage/provider/src/test_utils/mock.rs | 22 +++++++-- .../storage/provider/src/test_utils/noop.rs | 20 ++++++-- crates/storage/provider/src/traits/mod.rs | 3 ++ .../storage/provider/src/traits/receipts.rs | 15 ++++++ 6 files changed, 99 insertions(+), 11 deletions(-) create mode 100644 crates/storage/provider/src/traits/receipts.rs diff --git a/crates/storage/provider/src/lib.rs b/crates/storage/provider/src/lib.rs index 3e08932ec6e..a980f2f15a1 100644 --- a/crates/storage/provider/src/lib.rs +++ b/crates/storage/provider/src/lib.rs @@ -12,8 +12,8 @@ mod traits; pub use traits::{ AccountProvider, BlockExecutor, BlockHashProvider, BlockIdProvider, BlockProvider, - EvmEnvProvider, ExecutorFactory, HeaderProvider, StateProvider, StateProviderFactory, - TransactionsProvider, WithdrawalsProvider, + EvmEnvProvider, ExecutorFactory, HeaderProvider, ReceiptProvider, StateProvider, + StateProviderFactory, TransactionsProvider, WithdrawalsProvider, }; /// Provider trait implementations. diff --git a/crates/storage/provider/src/providers/mod.rs b/crates/storage/provider/src/providers/mod.rs index cc843246f49..e8807110c77 100644 --- a/crates/storage/provider/src/providers/mod.rs +++ b/crates/storage/provider/src/providers/mod.rs @@ -10,7 +10,7 @@ use reth_db::{ }; use reth_interfaces::Result; use reth_primitives::{ - Block, BlockHash, BlockId, BlockNumber, ChainInfo, ChainSpec, Hardfork, Head, Header, + Block, BlockHash, BlockId, BlockNumber, ChainInfo, ChainSpec, Hardfork, Head, Header, Receipt, TransactionSigned, TxHash, TxNumber, Withdrawal, H256, U256, }; use reth_revm_primitives::{ @@ -21,6 +21,7 @@ use revm_primitives::{BlockEnv, CfgEnv, SpecId}; use std::{ops::RangeBounds, sync::Arc}; mod state; +use crate::traits::ReceiptProvider; pub use state::{ chain::ChainState, historical::{HistoricalStateProvider, HistoricalStateProviderRef}, @@ -176,7 +177,7 @@ impl TransactionsProvider for ShareableDatabase { if let Some(body) = tx.get::(number)? { let tx_range = body.tx_id_range(); if tx_range.is_empty() { - Ok(Some(Vec::default())) + Ok(Some(Vec::new())) } else { let mut tx_cursor = tx.cursor_read::()?; let transactions = tx_cursor @@ -219,6 +220,47 @@ impl TransactionsProvider for ShareableDatabase { } } +impl ReceiptProvider for ShareableDatabase { + fn receipt(&self, id: TxNumber) -> Result> { + self.db.view(|tx| tx.get::(id))?.map_err(Into::into) + } + + fn receipt_by_hash(&self, hash: TxHash) -> Result> { + self.db + .view(|tx| { + if let Some(id) = tx.get::(hash)? { + tx.get::(id) + } else { + Ok(None) + } + })? + .map_err(Into::into) + } + + fn receipts_by_block(&self, block: BlockId) -> Result>> { + if let Some(number) = self.block_number_for_id(block)? { + let tx = self.db.tx()?; + if let Some(body) = tx.get::(number)? { + let tx_range = body.tx_id_range(); + if tx_range.is_empty() { + Ok(Some(Vec::new())) + } else { + let mut tx_cursor = tx.cursor_read::()?; + let transactions = tx_cursor + .walk_range(tx_range)? + .map(|result| result.map(|(_, tx)| tx)) + .collect::, _>>()?; + Ok(Some(transactions)) + } + } else { + Ok(None) + } + } else { + Ok(None) + } + } +} + impl WithdrawalsProvider for ShareableDatabase { fn withdrawals_by_block(&self, id: BlockId, timestamp: u64) -> Result>> { if self.chain_spec.fork(Hardfork::Shanghai).active_at_timestamp(timestamp) { diff --git a/crates/storage/provider/src/test_utils/mock.rs b/crates/storage/provider/src/test_utils/mock.rs index 7d448fd2a9c..31430ae84a1 100644 --- a/crates/storage/provider/src/test_utils/mock.rs +++ b/crates/storage/provider/src/test_utils/mock.rs @@ -1,13 +1,13 @@ use crate::{ - AccountProvider, BlockHashProvider, BlockIdProvider, BlockProvider, EvmEnvProvider, - HeaderProvider, StateProvider, StateProviderFactory, TransactionsProvider, + traits::ReceiptProvider, AccountProvider, BlockHashProvider, BlockIdProvider, BlockProvider, + EvmEnvProvider, HeaderProvider, StateProvider, StateProviderFactory, TransactionsProvider, }; use parking_lot::Mutex; use reth_interfaces::Result; use reth_primitives::{ keccak256, Account, Address, Block, BlockHash, BlockId, BlockNumber, BlockNumberOrTag, - Bytecode, Bytes, ChainInfo, Header, StorageKey, StorageValue, TransactionSigned, TxHash, H256, - U256, + Bytecode, Bytes, ChainInfo, Header, Receipt, StorageKey, StorageValue, TransactionSigned, + TxHash, TxNumber, H256, U256, }; use revm_primitives::{BlockEnv, CfgEnv}; use std::{collections::HashMap, ops::RangeBounds, sync::Arc}; @@ -156,6 +156,20 @@ impl TransactionsProvider for MockEthProvider { } } +impl ReceiptProvider for MockEthProvider { + fn receipt(&self, _id: TxNumber) -> Result> { + Ok(None) + } + + fn receipt_by_hash(&self, _hash: TxHash) -> Result> { + Ok(None) + } + + fn receipts_by_block(&self, _block: BlockId) -> Result>> { + Ok(None) + } +} + impl BlockHashProvider for MockEthProvider { fn block_hash(&self, number: U256) -> Result> { let lock = self.blocks.lock(); diff --git a/crates/storage/provider/src/test_utils/noop.rs b/crates/storage/provider/src/test_utils/noop.rs index c9f86b0cafa..c7bd3f80207 100644 --- a/crates/storage/provider/src/test_utils/noop.rs +++ b/crates/storage/provider/src/test_utils/noop.rs @@ -1,10 +1,10 @@ use crate::{ - AccountProvider, BlockHashProvider, BlockIdProvider, BlockProvider, EvmEnvProvider, - HeaderProvider, StateProvider, StateProviderFactory, TransactionsProvider, + traits::ReceiptProvider, AccountProvider, BlockHashProvider, BlockIdProvider, BlockProvider, + EvmEnvProvider, HeaderProvider, StateProvider, StateProviderFactory, TransactionsProvider, }; use reth_interfaces::Result; use reth_primitives::{ - Account, Address, Block, BlockHash, BlockId, BlockNumber, Bytecode, ChainInfo, Header, + Account, Address, Block, BlockHash, BlockId, BlockNumber, Bytecode, ChainInfo, Header, Receipt, StorageKey, StorageValue, TransactionSigned, TxHash, TxNumber, H256, U256, }; use revm_primitives::{BlockEnv, CfgEnv}; @@ -63,6 +63,20 @@ impl TransactionsProvider for NoopProvider { } } +impl ReceiptProvider for NoopProvider { + fn receipt(&self, _id: TxNumber) -> Result> { + Ok(None) + } + + fn receipt_by_hash(&self, _hash: TxHash) -> Result> { + Ok(None) + } + + fn receipts_by_block(&self, _block: BlockId) -> Result>> { + Ok(None) + } +} + impl HeaderProvider for NoopProvider { fn header(&self, _block_hash: &BlockHash) -> Result> { Ok(None) diff --git a/crates/storage/provider/src/traits/mod.rs b/crates/storage/provider/src/traits/mod.rs index 12d9adf4a8b..93b3ae303c5 100644 --- a/crates/storage/provider/src/traits/mod.rs +++ b/crates/storage/provider/src/traits/mod.rs @@ -18,6 +18,9 @@ pub use evm_env::EvmEnvProvider; mod header; pub use header::HeaderProvider; +mod receipts; +pub use receipts::ReceiptProvider; + mod state; pub use state::{StateProvider, StateProviderFactory}; diff --git a/crates/storage/provider/src/traits/receipts.rs b/crates/storage/provider/src/traits/receipts.rs new file mode 100644 index 00000000000..fa5604b2811 --- /dev/null +++ b/crates/storage/provider/src/traits/receipts.rs @@ -0,0 +1,15 @@ +use reth_interfaces::Result; +use reth_primitives::{BlockId, Receipt, TxHash, TxNumber}; + +/// Client trait for fetching [Receipt] data . +#[auto_impl::auto_impl(&, Arc)] +pub trait ReceiptProvider: Send + Sync { + /// Get receipt by transaction number + fn receipt(&self, id: TxNumber) -> Result>; + + /// Get receipt by transaction hash. + fn receipt_by_hash(&self, hash: TxHash) -> Result>; + + /// Get receipts by block id. + fn receipts_by_block(&self, block: BlockId) -> Result>>; +} From 2bd73199c25e7eca0c1c0fbbc97f9f92d00de05d Mon Sep 17 00:00:00 2001 From: chirag-bgh <76247491+chirag-bgh@users.noreply.github.com> Date: Mon, 13 Mar 2023 19:07:39 +0530 Subject: [PATCH 107/191] chore(docs): add method name to RPC handler implementation (#1737) --- crates/rpc/rpc/src/admin.rs | 6 ++++++ crates/rpc/rpc/src/debug.rs | 5 +++++ crates/rpc/rpc/src/engine/mod.rs | 10 ++++++++++ crates/rpc/rpc/src/eth/filter.rs | 7 +++++++ crates/rpc/rpc/src/eth/pubsub.rs | 1 + crates/rpc/rpc/src/net.rs | 3 +++ crates/rpc/rpc/src/web3.rs | 2 ++ 7 files changed, 34 insertions(+) diff --git a/crates/rpc/rpc/src/admin.rs b/crates/rpc/rpc/src/admin.rs index eb0bbe8dfa8..7921ecad8d7 100644 --- a/crates/rpc/rpc/src/admin.rs +++ b/crates/rpc/rpc/src/admin.rs @@ -26,26 +26,31 @@ impl AdminApiServer for AdminApi where N: NetworkInfo + Peers + 'static, { + /// Handler for `admin_addPeer` fn add_peer(&self, record: NodeRecord) -> RpcResult { self.network.add_peer(record.id, record.tcp_addr()); Ok(true) } + /// Handler for `admin_removePeer` fn remove_peer(&self, record: NodeRecord) -> RpcResult { self.network.remove_peer(record.id, PeerKind::Basic); Ok(true) } + /// Handler for `admin_addTrustedPeer` fn add_trusted_peer(&self, record: NodeRecord) -> RpcResult { self.network.add_trusted_peer(record.id, record.tcp_addr()); Ok(true) } + /// Handler for `admin_removeTrustedPeer` fn remove_trusted_peer(&self, record: NodeRecord) -> RpcResult { self.network.remove_peer(record.id, PeerKind::Trusted); Ok(true) } + /// Handler for `admin_peerEvents` fn subscribe_peer_events( &self, _subscription_sink: jsonrpsee::SubscriptionSink, @@ -53,6 +58,7 @@ where todo!() } + /// Handler for `admin_nodeInfo` async fn node_info(&self) -> RpcResult { let enr = self.network.local_node_record(); let status = self.network.network_status().await.to_rpc_result()?; diff --git a/crates/rpc/rpc/src/debug.rs b/crates/rpc/rpc/src/debug.rs index 3fc9c23c464..8554072a1e5 100644 --- a/crates/rpc/rpc/src/debug.rs +++ b/crates/rpc/rpc/src/debug.rs @@ -31,23 +31,28 @@ impl DebugApiServer for DebugApi where Eth: EthApiSpec + 'static, { + /// Handler for `debug_getRawHeader` async fn raw_header(&self, _block_id: BlockId) -> RpcResult { Err(internal_rpc_err("unimplemented")) } + /// Handler for `debug_getRawBlock` async fn raw_block(&self, _block_id: BlockId) -> RpcResult { Err(internal_rpc_err("unimplemented")) } + /// Handler for `debug_getRawTransaction` /// Returns the bytes of the transaction for the given hash. async fn raw_transaction(&self, _hash: H256) -> RpcResult { Err(internal_rpc_err("unimplemented")) } + /// Handler for `debug_getRawReceipts` async fn raw_receipts(&self, _block_id: BlockId) -> RpcResult> { Err(internal_rpc_err("unimplemented")) } + /// Handler for `debug_getBadBlocks` async fn bad_blocks(&self) -> RpcResult> { Err(internal_rpc_err("unimplemented")) } diff --git a/crates/rpc/rpc/src/engine/mod.rs b/crates/rpc/rpc/src/engine/mod.rs index 2b917d394bc..1b943fd07f7 100644 --- a/crates/rpc/rpc/src/engine/mod.rs +++ b/crates/rpc/rpc/src/engine/mod.rs @@ -58,6 +58,7 @@ impl EngineApi { #[async_trait] impl EngineApiServer for EngineApi { + /// Handler for `engine_getPayloadV1` /// See also /// Caution: This should not accept the `withdrawals` field async fn new_payload_v1(&self, payload: ExecutionPayload) -> Result { @@ -69,6 +70,7 @@ impl EngineApiServer for EngineApi { .await } + /// Handler for `engine_getPayloadV2` /// See also async fn new_payload_v2(&self, payload: ExecutionPayload) -> Result { let (tx, rx) = oneshot::channel(); @@ -79,6 +81,7 @@ impl EngineApiServer for EngineApi { .await } + /// Handler for `engine_forkchoiceUpdatedV1` /// See also /// /// Caution: This should not accept the `withdrawals` field @@ -100,6 +103,7 @@ impl EngineApiServer for EngineApi { .await } + /// Handler for `engine_forkchoiceUpdatedV2` /// See also async fn fork_choice_updated_v2( &self, @@ -119,6 +123,7 @@ impl EngineApiServer for EngineApi { .await } + /// Handler for `engine_getPayloadV1` /// See also /// /// Caution: This should not return the `withdrawals` field @@ -127,12 +132,14 @@ impl EngineApiServer for EngineApi { self.delegate_request(EngineApiMessage::GetPayload(payload_id, tx), rx).await } + /// Handler for `engine_getPayloadV2` /// See also async fn get_payload_v2(&self, payload_id: H64) -> Result { let (tx, rx) = oneshot::channel(); self.delegate_request(EngineApiMessage::GetPayload(payload_id, tx), rx).await } + /// Handler for `engine_getPayloadBodiesByHashV1` /// See also async fn get_payload_bodies_by_hash_v1( &self, @@ -142,6 +149,7 @@ impl EngineApiServer for EngineApi { self.delegate_request(EngineApiMessage::GetPayloadBodiesByHash(block_hashes, tx), rx).await } + /// Handler for `engine_getPayloadBodiesByRangeV1` /// See also async fn get_payload_bodies_by_range_v1( &self, @@ -152,6 +160,7 @@ impl EngineApiServer for EngineApi { self.delegate_request(EngineApiMessage::GetPayloadBodiesByRange(start, count, tx), rx).await } + /// Handler for `engine_exchangeTransitionConfigurationV1` /// See also async fn exchange_transition_configuration( &self, @@ -162,6 +171,7 @@ impl EngineApiServer for EngineApi { .await } + /// Handler for `engine_exchangeCapabilitiesV1` /// See also async fn exchange_capabilities(&self, _capabilities: Vec) -> Result> { Ok(CAPABILITIES.into_iter().map(str::to_owned).collect()) diff --git a/crates/rpc/rpc/src/eth/filter.rs b/crates/rpc/rpc/src/eth/filter.rs index edaf73c97d5..a835d28f253 100644 --- a/crates/rpc/rpc/src/eth/filter.rs +++ b/crates/rpc/rpc/src/eth/filter.rs @@ -54,18 +54,22 @@ where Client: BlockProvider + EvmEnvProvider + 'static, Pool: TransactionPool + 'static, { + /// Handler for `eth_newFilter` async fn new_filter(&self, filter: Filter) -> RpcResult { self.inner.install_filter(FilterKind::Log(Box::new(filter))).await } + /// Handler for `eth_newBlockFilter` async fn new_block_filter(&self) -> RpcResult { self.inner.install_filter(FilterKind::Block).await } + /// Handler for `eth_newPendingTransactionFilter` async fn new_pending_transaction_filter(&self) -> RpcResult { self.inner.install_filter(FilterKind::PendingTransaction).await } + /// Handler for `eth_getFilterChanges` async fn filter_changes(&self, id: FilterId) -> RpcResult { let info = self.inner.client.chain_info().to_rpc_result()?; let best_number = info.best_number; @@ -136,10 +140,12 @@ where } } + /// Handler for `eth_getFilterLogs` async fn filter_logs(&self, _id: FilterId) -> RpcResult> { todo!() } + /// Handler for `eth_uninstallFilter` async fn uninstall_filter(&self, id: FilterId) -> RpcResult { let mut filters = self.inner.active_filters.inner.lock().await; if filters.remove(&id).is_some() { @@ -150,6 +156,7 @@ where } } + /// Handler for `eth_getLogs` async fn logs(&self, _filter: Filter) -> RpcResult> { todo!() } diff --git a/crates/rpc/rpc/src/eth/pubsub.rs b/crates/rpc/rpc/src/eth/pubsub.rs index 01064fdb219..fa08bf95ef9 100644 --- a/crates/rpc/rpc/src/eth/pubsub.rs +++ b/crates/rpc/rpc/src/eth/pubsub.rs @@ -60,6 +60,7 @@ where Events: ChainEventSubscriptions + Clone + 'static, Network: SyncStateProvider + Clone + 'static, { + /// Handler for `eth_subscribe` fn subscribe( &self, mut sink: SubscriptionSink, diff --git a/crates/rpc/rpc/src/net.rs b/crates/rpc/rpc/src/net.rs index c0e9e03861a..6f6ec7d38be 100644 --- a/crates/rpc/rpc/src/net.rs +++ b/crates/rpc/rpc/src/net.rs @@ -29,14 +29,17 @@ where Net: PeersInfo + 'static, Eth: EthApiSpec + 'static, { + /// Handler for `net_version` fn version(&self) -> Result { Ok(self.eth.chain_id().to_string()) } + /// Handler for `net_peerCount` fn peer_count(&self) -> Result { Ok(PeerCount::Hex(self.network.num_connected_peers().into())) } + /// Handler for `net_listening` fn is_listening(&self) -> Result { Ok(true) } diff --git a/crates/rpc/rpc/src/web3.rs b/crates/rpc/rpc/src/web3.rs index e5476f5bcd1..e3b600f945c 100644 --- a/crates/rpc/rpc/src/web3.rs +++ b/crates/rpc/rpc/src/web3.rs @@ -25,11 +25,13 @@ impl Web3ApiServer for Web3Api where N: NetworkInfo + 'static, { + /// Handler for `web3_clientVersion` async fn client_version(&self) -> RpcResult { let status = self.network.network_status().await.to_rpc_result()?; Ok(status.client_version) } + /// Handler for `web3_sha3` fn sha3(&self, input: Bytes) -> RpcResult { Ok(keccak256(input)) } From ff9f02d524130b0d456e6f9c6da234356755d2ea Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 13 Mar 2023 14:46:13 +0100 Subject: [PATCH 108/191] style(rpc): some transaction by touch ups (#1711) --- crates/primitives/src/transaction/mod.rs | 4 +- crates/rpc/rpc/src/eth/api/transactions.rs | 85 +++++++++++----------- 2 files changed, 44 insertions(+), 45 deletions(-) diff --git a/crates/primitives/src/transaction/mod.rs b/crates/primitives/src/transaction/mod.rs index 23885b8b0cd..f9d43f6a303 100644 --- a/crates/primitives/src/transaction/mod.rs +++ b/crates/primitives/src/transaction/mod.rs @@ -551,7 +551,7 @@ impl TransactionSigned { /// Recover signer from signature and hash. /// - /// Returns `None` if the transaction's signature is invalid. + /// Returns `None` if the transaction's signature is invalid, see also [Self::recover_signer]. pub fn recover_signer(&self) -> Option
{ let signature_hash = self.signature_hash(); self.signature.recover_signer(signature_hash) @@ -559,7 +559,7 @@ impl TransactionSigned { /// Devour Self, recover signer and return [`TransactionSignedEcRecovered`] /// - /// Returns `None` if the transaction's signature is invalid. + /// Returns `None` if the transaction's signature is invalid, see also [Self::recover_signer]. pub fn into_ecrecovered(self) -> Option { let signer = self.recover_signer()?; Some(TransactionSignedEcRecovered { signed_transaction: self, signer }) diff --git a/crates/rpc/rpc/src/eth/api/transactions.rs b/crates/rpc/rpc/src/eth/api/transactions.rs index 72ce26a63f8..bece1772f19 100644 --- a/crates/rpc/rpc/src/eth/api/transactions.rs +++ b/crates/rpc/rpc/src/eth/api/transactions.rs @@ -3,7 +3,7 @@ use crate::{ eth::error::{EthApiError, EthResult}, EthApi, }; -use reth_primitives::{BlockId, Bytes, FromRecoveredTransaction, TransactionSigned, H256, U256}; +use reth_primitives::{BlockId, Bytes, FromRecoveredTransaction, TransactionSigned, H256}; use reth_provider::{BlockProvider, EvmEnvProvider, StateProviderFactory}; use reth_rlp::Decodable; use reth_rpc_types::{Index, Transaction, TransactionRequest}; @@ -19,56 +19,55 @@ where unimplemented!() } - /// Finds a given trasaction by its hash. - pub(crate) async fn transaction_by_hash( - &self, - hash: H256, - ) -> EthResult> { - let tx_by_hash = match self.client().transaction_by_hash(hash)? { - Some(item) => item, - None => return Ok(None), - }; - - if let Some(tx) = tx_by_hash.into_ecrecovered() { - Ok(Some(Transaction::from_recovered_with_block_context( - tx, - // TODO: this is just stubbed out for now still need to fully implement tx => block - H256::default(), - u64::default(), - U256::from(usize::from(Index::default())), - ))) - } else { - Ok(None) + /// Finds a given [Transaction] by its hash. + /// + /// Returns `Ok(None)` if no matching transaction was found. + pub(crate) async fn transaction_by_hash(&self, hash: H256) -> EthResult> { + match self.client().transaction_by_hash(hash)? { + None => Ok(None), + Some(tx) => { + let tx = tx.into_ecrecovered().ok_or(EthApiError::InvalidTransactionSignature)?; + + let tx = Transaction::from_recovered_with_block_context( + tx, + // TODO: this is just stubbed out for now still need to fully implement tx => + // block + H256::default(), + u64::default(), + Index::default().into(), + ); + Ok(Some(tx)) + } } } - /// Get Transaction by Block and tx index within that Block. - /// Used for both: - /// transaction_by_block_hash_and_index() & - /// transaction_by_block_number_and_index() + /// Get Transaction by [BlockId] and the index of the transaction within that Block. + /// + /// Returns `Ok(None)` if the block does not exist, or the block as fewer transactions pub(crate) async fn transaction_by_block_and_tx_index( &self, block_id: impl Into, index: Index, - ) -> EthResult> { + ) -> EthResult> { let block_id = block_id.into(); - let Some(block) = self.client().block(block_id)? else { - return Ok(None); - }; - let block_hash = - self.client().block_hash_for_id(block_id)?.ok_or(EthApiError::UnknownBlockNumber)?; - let Some(tx_signed) = block.body.into_iter().nth(usize::from(index)) else { - return Ok(None); - }; - let Some(tx) = tx_signed.into_ecrecovered() else { - return Ok(None); - }; - Ok(Some(Transaction::from_recovered_with_block_context( - tx, - block_hash, - block.header.number, - U256::from(usize::from(index)), - ))) + if let Some(block) = self.client().block(block_id)? { + let block_hash = self + .client() + .block_hash_for_id(block_id)? + .ok_or(EthApiError::UnknownBlockNumber)?; + if let Some(tx_signed) = block.body.into_iter().nth(index.into()) { + let tx = + tx_signed.into_ecrecovered().ok_or(EthApiError::InvalidTransactionSignature)?; + return Ok(Some(Transaction::from_recovered_with_block_context( + tx, + block_hash, + block.header.number, + index.into(), + ))) + } + } + + Ok(None) } /// Decodes and recovers the transaction and submits it to the pool. From 5e0535c27ce690396e0cd041dc5418fdffaab04a Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 13 Mar 2023 15:28:47 +0100 Subject: [PATCH 109/191] chore: add ReceiptProvider trait bound (#1738) --- crates/storage/provider/src/traits/block.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/storage/provider/src/traits/block.rs b/crates/storage/provider/src/traits/block.rs index 2b04b601e0b..044bb5fce7e 100644 --- a/crates/storage/provider/src/traits/block.rs +++ b/crates/storage/provider/src/traits/block.rs @@ -1,11 +1,11 @@ -use crate::{BlockIdProvider, HeaderProvider, TransactionsProvider}; +use crate::{BlockIdProvider, HeaderProvider, ReceiptProvider, TransactionsProvider}; use reth_interfaces::Result; use reth_primitives::{Block, BlockId, BlockNumberOrTag, Header, H256}; /// Api trait for fetching `Block` related data. #[auto_impl::auto_impl(&, Arc)] pub trait BlockProvider: - BlockIdProvider + HeaderProvider + TransactionsProvider + Send + Sync + BlockIdProvider + HeaderProvider + TransactionsProvider + ReceiptProvider + Send + Sync { /// Returns the block. /// From 043098aa70a1438fe1bfad261bd21df86af8a424 Mon Sep 17 00:00:00 2001 From: pistomat Date: Mon, 13 Mar 2023 16:12:44 +0100 Subject: [PATCH 110/191] fix: default to correct ws port in rpc server args (#1739) --- bin/reth/src/args/rpc_server_args.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/reth/src/args/rpc_server_args.rs b/bin/reth/src/args/rpc_server_args.rs index 54c16749f0f..5c593b84e6f 100644 --- a/bin/reth/src/args/rpc_server_args.rs +++ b/bin/reth/src/args/rpc_server_args.rs @@ -206,7 +206,7 @@ impl RpcServerArgs { if self.ws { let socket_address = SocketAddr::new( self.ws_addr.unwrap_or(IpAddr::V4(Ipv4Addr::UNSPECIFIED)), - self.ws_port.unwrap_or(constants::DEFAULT_HTTP_RPC_PORT), + self.ws_port.unwrap_or(constants::DEFAULT_WS_RPC_PORT), ); config = config.with_ws_address(socket_address).with_http(ServerBuilder::new()); } From bc0ce636248ec4a5531caa7eaaf32f526266712c Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 13 Mar 2023 18:46:36 +0100 Subject: [PATCH 111/191] style: get rid of some else branches (#1736) --- crates/storage/provider/src/providers/mod.rs | 21 ++++++-------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/crates/storage/provider/src/providers/mod.rs b/crates/storage/provider/src/providers/mod.rs index e8807110c77..a7539fa7330 100644 --- a/crates/storage/provider/src/providers/mod.rs +++ b/crates/storage/provider/src/providers/mod.rs @@ -176,7 +176,7 @@ impl TransactionsProvider for ShareableDatabase { let tx = self.db.tx()?; if let Some(body) = tx.get::(number)? { let tx_range = body.tx_id_range(); - if tx_range.is_empty() { + return if tx_range.is_empty() { Ok(Some(Vec::new())) } else { let mut tx_cursor = tx.cursor_read::()?; @@ -186,12 +186,9 @@ impl TransactionsProvider for ShareableDatabase { .collect::, _>>()?; Ok(Some(transactions)) } - } else { - Ok(None) } - } else { - Ok(None) } + Ok(None) } fn transactions_by_block_range( @@ -242,7 +239,7 @@ impl ReceiptProvider for ShareableDatabase { let tx = self.db.tx()?; if let Some(body) = tx.get::(number)? { let tx_range = body.tx_id_range(); - if tx_range.is_empty() { + return if tx_range.is_empty() { Ok(Some(Vec::new())) } else { let mut tx_cursor = tx.cursor_read::()?; @@ -252,12 +249,9 @@ impl ReceiptProvider for ShareableDatabase { .collect::, _>>()?; Ok(Some(transactions)) } - } else { - Ok(None) } - } else { - Ok(None) } + Ok(None) } } @@ -265,16 +259,13 @@ impl WithdrawalsProvider for ShareableDatabase { fn withdrawals_by_block(&self, id: BlockId, timestamp: u64) -> Result>> { if self.chain_spec.fork(Hardfork::Shanghai).active_at_timestamp(timestamp) { if let Some(number) = self.block_number_for_id(id)? { - Ok(self + return Ok(self .db .view(|tx| tx.get::(number))?? .map(|w| w.withdrawals)) - } else { - Ok(None) } - } else { - Ok(None) } + Ok(None) } } From 5d9237c2e8ee9a6144c108b8560c0889ab788bb4 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 13 Mar 2023 20:57:50 +0100 Subject: [PATCH 112/191] fix(rpc): use eth id provider for filter (#1740) --- crates/rpc/rpc/src/eth/filter.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/crates/rpc/rpc/src/eth/filter.rs b/crates/rpc/rpc/src/eth/filter.rs index a835d28f253..9c7bdda56de 100644 --- a/crates/rpc/rpc/src/eth/filter.rs +++ b/crates/rpc/rpc/src/eth/filter.rs @@ -1,12 +1,10 @@ use crate::{ eth::error::EthApiError, result::{internal_rpc_err, rpc_error_with_code, ToRpcResult}, + EthSubscriptionIdProvider, }; use async_trait::async_trait; -use jsonrpsee::{ - core::RpcResult, - server::{IdProvider, RandomIntegerIdProvider}, -}; +use jsonrpsee::{core::RpcResult, server::IdProvider}; use reth_primitives::{ filter::{Filter, FilterBlockOption, FilteredParams}, Block, U256, @@ -36,7 +34,7 @@ impl EthFilter { client, active_filters: Default::default(), pool, - id_provider: Arc::new(RandomIntegerIdProvider), + id_provider: Arc::new(EthSubscriptionIdProvider::default()), max_logs_in_response: DEFAULT_MAX_LOGS_IN_RESPONSE, }; Self { inner: Arc::new(inner) } From 4c1a35b2b80e54e1507d8364d16458cdfef1b79a Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 13 Mar 2023 23:02:02 +0100 Subject: [PATCH 113/191] feat: add trace inspector config (#1729) --- .../revm-inspectors/src/tracing/config.rs | 75 ++++++++++++++++ .../revm/revm-inspectors/src/tracing/mod.rs | 86 +++++++++++-------- 2 files changed, 125 insertions(+), 36 deletions(-) create mode 100644 crates/revm/revm-inspectors/src/tracing/config.rs diff --git a/crates/revm/revm-inspectors/src/tracing/config.rs b/crates/revm/revm-inspectors/src/tracing/config.rs new file mode 100644 index 00000000000..14abb668a29 --- /dev/null +++ b/crates/revm/revm-inspectors/src/tracing/config.rs @@ -0,0 +1,75 @@ +/// Gives guidance to the [TracingInspector](crate::tracing::TracingInspector). +/// +/// Use [TraceInspectorConfig::default_parity] or [TraceInspectorConfig::default_geth] to get the +/// default configs for specific styles of traces. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub struct TraceInspectorConfig { + /// Whether to record every individual opcode level step. + pub record_steps: bool, + /// Whether to record individual memory snapshots. + pub record_memory_snapshots: bool, + /// Whether to record individual stack snapshots. + pub record_stack_snapshots: bool, + /// Whether to record state diffs. + pub record_state_diff: bool, +} + +impl TraceInspectorConfig { + /// Returns a config with everything enabled. + pub const fn all() -> Self { + Self { + record_steps: true, + record_memory_snapshots: true, + record_stack_snapshots: true, + record_state_diff: false, + } + } + + /// Returns a config for parity style traces. + /// + /// This config does _not_ record opcode level traces and is suited for `trace_transaction` + pub const fn default_parity() -> Self { + Self { + record_steps: false, + record_memory_snapshots: false, + record_stack_snapshots: false, + record_state_diff: false, + } + } + + /// Returns a config for geth style traces. + /// + /// This config does _not_ record opcode level traces and is suited for `debug_traceTransaction` + pub const fn default_geth() -> Self { + Self { + record_steps: true, + record_memory_snapshots: true, + record_stack_snapshots: true, + record_state_diff: true, + } + } + + /// Configure whether individual opcode level steps should be recorded + pub fn set_steps(mut self, record_steps: bool) -> Self { + self.record_steps = record_steps; + self + } + + /// Configure whether the tracer should record memory snapshots + pub fn set_memory_snapshots(mut self, record_memory_snapshots: bool) -> Self { + self.record_memory_snapshots = record_memory_snapshots; + self + } + + /// Configure whether the tracer should record stack snapshots + pub fn set_stack_snapshots(mut self, record_stack_snapshots: bool) -> Self { + self.record_stack_snapshots = record_stack_snapshots; + self + } + + /// Configure whether the tracer should record state diffs + pub fn set_state_diffs(mut self, record_state_diff: bool) -> Self { + self.record_state_diff = record_state_diff; + self + } +} diff --git a/crates/revm/revm-inspectors/src/tracing/mod.rs b/crates/revm/revm-inspectors/src/tracing/mod.rs index e5285a3e6b0..958fc2c64c5 100644 --- a/crates/revm/revm-inspectors/src/tracing/mod.rs +++ b/crates/revm/revm-inspectors/src/tracing/mod.rs @@ -18,8 +18,10 @@ use revm::{ use types::{CallTrace, CallTraceStep}; mod arena; +mod config; mod types; mod utils; +pub use config::TraceInspectorConfig; /// An inspector that collects call traces. /// @@ -29,10 +31,10 @@ mod utils; /// The [TracingInspector] keeps track of everything by: /// 1. start tracking steps/calls on [Inspector::step] and [Inspector::call] /// 2. complete steps/calls on [Inspector::step_end] and [Inspector::call_end] -#[derive(Default, Debug, Clone)] +#[derive(Debug, Clone)] pub struct TracingInspector { - /// Whether to include individual steps [Inspector::step] - record_steps: bool, + /// Configures what and how the inspector records traces. + config: TraceInspectorConfig, /// Records all call traces traces: CallTraceArena, trace_stack: Vec, @@ -47,18 +49,22 @@ pub struct TracingInspector { // === impl TracingInspector === impl TracingInspector { + /// Returns a new instance for the given config + pub fn new(config: TraceInspectorConfig) -> Self { + Self { + config, + traces: Default::default(), + trace_stack: vec![], + step_stack: vec![], + gas_inspector: Default::default(), + } + } + /// Consumes the Inspector and returns the recorded. pub fn finalize(self) -> CallTraceArena { self.traces } - /// Enables step recording and uses the configured [GasInspector] to report gas costs for each - /// step. - pub fn with_steps_recording(mut self) -> Self { - self.record_steps = true; - self - } - /// Configures a [GasInspector] /// /// If this [TracingInspector] is part of a stack [InspectorStack](crate::stack::InspectorStack) @@ -167,14 +173,19 @@ impl TracingInspector { let pc = interp.program_counter(); + let memory = + self.config.record_memory_snapshots.then(|| interp.memory.clone()).unwrap_or_default(); + let stack = + self.config.record_stack_snapshots.then(|| interp.stack.clone()).unwrap_or_default(); + trace.trace.steps.push(CallTraceStep { depth: data.journaled_state.depth(), pc, op: OpCode::try_from_u8(interp.contract.bytecode.bytecode()[pc]) .expect("is valid opcode;"), contract: interp.contract.address, - stack: interp.stack.clone(), - memory: interp.memory.clone(), + stack, + memory, gas: self.gas_inspector.as_ref().gas_remaining(), gas_refund_counter: interp.gas.refunded() as u64, @@ -199,28 +210,31 @@ impl TracingInspector { let step = &mut self.traces.arena[trace_idx].trace.steps[step_idx]; if let Some(pc) = interp.program_counter().checked_sub(1) { - let op = interp.contract.bytecode.bytecode()[pc]; - - let journal_entry = data - .journaled_state - .journal - .last() - // This should always work because revm initializes it as `vec![vec![]]` - // See [JournaledState::new](revm::JournaledState) - .expect("exists; initialized with vec") - .last(); - - step.state_diff = match (op, journal_entry) { - ( - opcode::SLOAD | opcode::SSTORE, - Some(JournalEntry::StorageChange { address, key, .. }), - ) => { - // SAFETY: (Address,key) exists if part if StorageChange - let value = data.journaled_state.state[address].storage[key].present_value(); - Some((*key, value)) - } - _ => None, - }; + if self.config.record_state_diff { + let op = interp.contract.bytecode.bytecode()[pc]; + + let journal_entry = data + .journaled_state + .journal + .last() + // This should always work because revm initializes it as `vec![vec![]]` + // See [JournaledState::new](revm::JournaledState) + .expect("exists; initialized with vec") + .last(); + + step.state_diff = match (op, journal_entry) { + ( + opcode::SLOAD | opcode::SSTORE, + Some(JournalEntry::StorageChange { address, key, .. }), + ) => { + // SAFETY: (Address,key) exists if part if StorageChange + let value = + data.journaled_state.state[address].storage[key].present_value(); + Some((*key, value)) + } + _ => None, + }; + } step.gas_cost = step.gas - self.gas_inspector.as_ref().gas_remaining(); } @@ -249,7 +263,7 @@ where data: &mut EVMData<'_, DB>, is_static: bool, ) -> InstructionResult { - if self.record_steps { + if self.config.record_steps { self.gas_inspector.step(interp, data, is_static); self.start_step(interp, data); } @@ -279,7 +293,7 @@ where is_static: bool, eval: InstructionResult, ) -> InstructionResult { - if self.record_steps { + if self.config.record_steps { self.gas_inspector.step_end(interp, data, is_static, eval); self.fill_step_on_step_end(interp, data, eval); return eval From 54aab533c2052ed350fc8a078812d026e663d101 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 13 Mar 2023 21:49:36 -0300 Subject: [PATCH 114/191] feat(rpc): add partial `eth_getProof` implementation (#1515) Co-authored-by: lambdaclass-user --- crates/executor/src/executor.rs | 8 + crates/interfaces/src/provider.rs | 3 + crates/primitives/src/account.rs | 9 + crates/rpc/rpc-builder/tests/it/http.rs | 4 +- crates/rpc/rpc/src/eth/api/call.rs | 7 +- crates/rpc/rpc/src/eth/api/mod.rs | 65 ++- crates/rpc/rpc/src/eth/api/server.rs | 19 +- crates/rpc/rpc/src/eth/api/state.rs | 56 ++- crates/stages/Cargo.toml | 10 +- crates/stages/src/stages/merkle.rs | 15 +- crates/storage/provider/Cargo.toml | 4 +- crates/storage/provider/src/lib.rs | 2 +- .../src/providers/state/historical.rs | 11 +- .../provider/src/providers/state/latest.rs | 53 ++- .../provider/src/providers/state/macros.rs | 3 +- .../storage/provider/src/test_utils/mock.rs | 8 + .../storage/provider/src/test_utils/noop.rs | 13 +- crates/storage/provider/src/traits/state.rs | 9 +- crates/storage/provider/src/transaction.rs | 2 +- crates/storage/provider/src/trie/mod.rs | 382 +++++++++++++++--- 20 files changed, 578 insertions(+), 105 deletions(-) diff --git a/crates/executor/src/executor.rs b/crates/executor/src/executor.rs index 01a291b3d9a..22cc7c391dc 100644 --- a/crates/executor/src/executor.rs +++ b/crates/executor/src/executor.rs @@ -599,6 +599,14 @@ mod tests { fn bytecode_by_hash(&self, code_hash: H256) -> reth_interfaces::Result> { Ok(self.contracts.get(&code_hash).cloned()) } + + fn proof( + &self, + _address: Address, + _keys: &[H256], + ) -> reth_interfaces::Result<(Vec, H256, Vec>)> { + todo!() + } } #[test] diff --git a/crates/interfaces/src/provider.rs b/crates/interfaces/src/provider.rs index ac24be67a6c..e86d2dc5946 100644 --- a/crates/interfaces/src/provider.rs +++ b/crates/interfaces/src/provider.rs @@ -68,6 +68,9 @@ pub enum ProviderError { /// Reached the end of the transaction sender table. #[error("Got to the end of the transaction sender table")] EndOfTransactionSenderTable, + /// Some error occurred while interacting with the state tree. + #[error("Unknown error occurred while interacting with the state tree.")] + StateTree, /// Thrown when required header related data was not found but was required. #[error("requested data not found")] HeaderNotFound, diff --git a/crates/primitives/src/account.rs b/crates/primitives/src/account.rs index e822bc449e1..b91796129c2 100644 --- a/crates/primitives/src/account.rs +++ b/crates/primitives/src/account.rs @@ -34,6 +34,15 @@ impl Account { self.nonce == 0 && self.balance == U256::ZERO && is_bytecode_empty } + + /// Returns an account bytecode's hash. + /// In case of no bytecode, returns [`KECCAK_EMPTY`]. + pub fn get_bytecode_hash(&self) -> H256 { + match self.bytecode_hash { + Some(hash) => hash, + None => KECCAK_EMPTY, + } + } } /// Bytecode for an account. diff --git a/crates/rpc/rpc-builder/tests/it/http.rs b/crates/rpc/rpc-builder/tests/it/http.rs index 898beae6d84..f55d4bcbf0f 100644 --- a/crates/rpc/rpc-builder/tests/it/http.rs +++ b/crates/rpc/rpc-builder/tests/it/http.rs @@ -73,6 +73,7 @@ where EthApiClient::block_by_number(client, block_number, false).await.unwrap(); EthApiClient::block_transaction_count_by_number(client, block_number).await.unwrap(); EthApiClient::block_transaction_count_by_hash(client, hash).await.unwrap(); + EthApiClient::get_proof(client, address, vec![], None).await.unwrap(); EthApiClient::block_uncles_count_by_hash(client, hash).await.unwrap(); EthApiClient::block_uncles_count_by_number(client, block_number).await.unwrap(); EthApiClient::uncle_by_block_hash_and_index(client, hash, index).await.unwrap(); @@ -123,9 +124,6 @@ where .err() .unwrap() )); - assert!(is_unimplemented( - EthApiClient::get_proof(client, address, vec![], None).await.err().unwrap() - )); } async fn test_basic_debug_calls(client: &C) diff --git a/crates/rpc/rpc/src/eth/api/call.rs b/crates/rpc/rpc/src/eth/api/call.rs index cf92895e3ca..6c43da516f6 100644 --- a/crates/rpc/rpc/src/eth/api/call.rs +++ b/crates/rpc/rpc/src/eth/api/call.rs @@ -25,6 +25,7 @@ use revm::{ }, Database, }; +use std::ops::Deref; // Gas per transaction not creating a contract. const MIN_TRANSACTION_GAS: u64 = 21_000u64; @@ -66,7 +67,7 @@ where ) -> EthResult<(ResultAndState, Env)> { let (cfg, block_env, at) = self.evm_env_at(at).await?; let state = self.state_at_block_id(at)?.ok_or_else(|| EthApiError::UnknownBlockNumber)?; - self.call_with(cfg, block_env, request, state, state_overrides) + self.call_with(cfg, block_env, request, &*state, state_overrides) } /// Executes the call request using the given environment against the state provider @@ -106,7 +107,7 @@ where ) -> EthResult { let (cfg, block_env, at) = self.evm_env_at(at).await?; let state = self.state_at_block_id(at)?.ok_or_else(|| EthApiError::UnknownBlockNumber)?; - self.estimate_gas_with(cfg, block_env, request, state) + self.estimate_gas_with(cfg, block_env, request, &*state) } /// Estimates the gas usage of the `request` with the state. @@ -295,7 +296,7 @@ where cfg.disable_block_gas_limit = true; let env = build_call_evm_env(cfg, block, request.clone())?; - let mut db = SubState::new(State::new(state)); + let mut db = SubState::new(State::new(state.deref())); let from = request.from.unwrap_or_default(); let to = if let Some(to) = request.to { diff --git a/crates/rpc/rpc/src/eth/api/mod.rs b/crates/rpc/rpc/src/eth/api/mod.rs index 29f321b482c..97911adf5c1 100644 --- a/crates/rpc/rpc/src/eth/api/mod.rs +++ b/crates/rpc/rpc/src/eth/api/mod.rs @@ -10,8 +10,10 @@ use reth_network_api::NetworkInfo; use reth_primitives::{ Address, BlockId, BlockNumberOrTag, ChainInfo, TransactionSigned, H256, U64, }; -use reth_provider::{BlockProvider, EvmEnvProvider, StateProviderFactory}; -use std::num::NonZeroUsize; +use reth_provider::{ + BlockProvider, EvmEnvProvider, StateProvider as StateProviderTrait, StateProviderFactory, +}; +use std::{num::NonZeroUsize, ops::Deref}; use crate::eth::{cache::EthStateCache, error::EthResult}; use reth_provider::providers::ChainState; @@ -97,6 +99,37 @@ impl EthApi { } } +// Transparent wrapper to enable state access helpers +// returning latest state provider when appropiate +pub(crate) enum StateProvider<'a, H, L> { + History(H), + Latest(L), + _Unreachable(&'a ()), // like a PhantomData for 'a +} + +type HistoryOrLatest<'a, Client> = StateProvider< + 'a, + ::HistorySP<'a>, + ::LatestSP<'a>, +>; + +impl<'a, H, L> Deref for StateProvider<'a, H, L> +where + Self: 'a, + H: StateProviderTrait + 'a, + L: StateProviderTrait + 'a, +{ + type Target = dyn StateProviderTrait + 'a; + + fn deref(&self) -> &Self::Target { + match self { + StateProvider::History(h) => h, + StateProvider::Latest(l) => l, + StateProvider::_Unreachable(()) => unreachable!(), + } + } +} + // === State access helpers === impl EthApi @@ -119,11 +152,11 @@ where pub(crate) fn state_at_block_id_or_latest( &self, block_id: Option, - ) -> Result::HistorySP<'_>>> { + ) -> Result>> { if let Some(block_id) = block_id { self.state_at_block_id(block_id) } else { - self.latest_state() + self.latest_state().map(|v| Some(StateProvider::Latest(v))) } } @@ -131,9 +164,11 @@ where pub(crate) fn state_at_block_id( &self, block_id: BlockId, - ) -> Result::HistorySP<'_>>> { + ) -> Result>> { match block_id { - BlockId::Hash(hash) => self.state_at_hash(hash.into()).map(Some), + BlockId::Hash(hash) => { + self.state_at_hash(hash.into()).map(|s| Some(StateProvider::History(s))) + } BlockId::Number(num) => self.state_at_block_number(num), } } @@ -144,7 +179,7 @@ where pub(crate) fn state_at_block_number( &self, num: BlockNumberOrTag, - ) -> Result::HistorySP<'_>>> { + ) -> Result>> { if let Some(number) = self.convert_block_number(num)? { self.state_at_number(number).map(Some) } else { @@ -161,18 +196,16 @@ where } /// Returns the state at the given block number - pub(crate) fn state_at_number( - &self, - block_number: u64, - ) -> Result<::HistorySP<'_>> { - self.client().history_by_block_number(block_number) + pub(crate) fn state_at_number(&self, block_number: u64) -> Result> { + match self.convert_block_number(BlockNumberOrTag::Latest)? { + Some(num) if num == block_number => self.latest_state().map(StateProvider::Latest), + _ => self.client().history_by_block_number(block_number).map(StateProvider::History), + } } /// Returns the _latest_ state - pub(crate) fn latest_state( - &self, - ) -> Result::HistorySP<'_>>> { - self.state_at_block_number(BlockNumberOrTag::Latest) + pub(crate) fn latest_state(&self) -> Result<::LatestSP<'_>> { + self.client().latest() } } diff --git a/crates/rpc/rpc/src/eth/api/server.rs b/crates/rpc/rpc/src/eth/api/server.rs index 0ff4310e0f6..f59f25f5d9b 100644 --- a/crates/rpc/rpc/src/eth/api/server.rs +++ b/crates/rpc/rpc/src/eth/api/server.rs @@ -374,17 +374,24 @@ where /// Handler for: `eth_getProof` async fn get_proof( &self, - _address: Address, - _keys: Vec, - _block_number: Option, + address: Address, + keys: Vec, + block_number: Option, ) -> Result { - Err(internal_rpc_err("unimplemented")) + let res = EthApi::get_proof(self, address, keys, block_number); + + Ok(res.map_err(|e| match e { + EthApiError::InvalidBlockRange => { + internal_rpc_err("eth_getProof is unimplemented for historical blocks") + } + _ => e.into(), + })?) } } #[cfg(test)] mod tests { - use crate::eth::cache::EthStateCache; + use crate::{eth::cache::EthStateCache, EthApi}; use jsonrpsee::{ core::{error::Error as RpcError, RpcResult}, types::error::{CallError, INVALID_PARAMS_CODE}, @@ -396,8 +403,6 @@ mod tests { use reth_rpc_api::EthApiServer; use reth_transaction_pool::test_utils::testing_pool; - use crate::EthApi; - #[tokio::test] /// Handler for: `eth_test_fee_history` async fn test_fee_history() { diff --git a/crates/rpc/rpc/src/eth/api/state.rs b/crates/rpc/rpc/src/eth/api/state.rs index fce654d13c2..42638bcdb9d 100644 --- a/crates/rpc/rpc/src/eth/api/state.rs +++ b/crates/rpc/rpc/src/eth/api/state.rs @@ -1,11 +1,15 @@ //! Contains RPC handler implementations specific to state. use crate::{ - eth::error::{EthApiError, EthResult}, + eth::{ + api::StateProvider, + error::{EthApiError, EthResult}, + }, EthApi, }; -use reth_primitives::{Address, BlockId, Bytes, H256, U256}; -use reth_provider::{BlockProvider, EvmEnvProvider, StateProvider, StateProviderFactory}; +use reth_primitives::{Address, BlockId, Bytes, H256, KECCAK_EMPTY, U256}; +use reth_provider::{BlockProvider, EvmEnvProvider, StateProviderFactory}; +use reth_rpc_types::{EIP1186AccountProofResponse, StorageProof}; impl EthApi where @@ -48,4 +52,50 @@ where let value = state.storage(address, storage_key)?.unwrap_or_default(); Ok(H256(value.to_be_bytes())) } + + pub(crate) fn get_proof( + &self, + address: Address, + keys: Vec, + block_id: Option, + ) -> EthResult { + let state = + self.state_at_block_id_or_latest(block_id)?.ok_or(EthApiError::UnknownBlockNumber)?; + + // TODO: remove when HistoricalStateProviderRef::proof is implemented + if matches!(state, StateProvider::History(_)) { + return Err(EthApiError::InvalidBlockRange) + } + + let (account_proof, storage_hash, stg_proofs) = state.proof(address, &keys)?; + + let storage_proof = keys + .into_iter() + .zip(stg_proofs) + .map(|(key, proof)| { + state.storage(address, key).map(|op| StorageProof { + key: U256::from_be_bytes(*key.as_fixed_bytes()), + value: op.unwrap_or_default(), + proof, + }) + }) + .collect::>()?; + + let mut proof = EIP1186AccountProofResponse { + address, + code_hash: KECCAK_EMPTY, + account_proof, + storage_hash, + storage_proof, + ..Default::default() + }; + + if let Some(account) = state.basic_account(address)? { + proof.balance = account.balance; + proof.nonce = account.nonce.into(); + proof.code_hash = account.get_bytecode_hash(); + } + + Ok(proof) + } } diff --git a/crates/stages/Cargo.toml b/crates/stages/Cargo.toml index 3b1970bd710..507ed4cfdf9 100644 --- a/crates/stages/Cargo.toml +++ b/crates/stages/Cargo.toml @@ -10,7 +10,7 @@ description = "Staged syncing primitives used in reth." [package.metadata.cargo-udeps.ignore] normal = [ # Used for diagrams in docs - "aquamarine" + "aquamarine", ] [dependencies] @@ -46,7 +46,7 @@ proptest = { version = "1.0", optional = true } [dev-dependencies] # reth -reth-primitives = { path = "../primitives", features = ["arbitrary"]} +reth-primitives = { path = "../primitives", features = ["arbitrary"] } reth-db = { path = "../storage/db", features = ["test-utils", "mdbx"] } reth-interfaces = { path = "../interfaces", features = ["test-utils"] } reth-downloaders = { path = "../net/downloaders" } @@ -60,7 +60,11 @@ rand = "0.8.5" paste = "1.0" # Stage benchmarks -pprof = { version = "0.11", features = ["flamegraph", "frame-pointer", "criterion"] } +pprof = { version = "0.11", features = [ + "flamegraph", + "frame-pointer", + "criterion", +] } criterion = { version = "0.4.0", features = ["async_futures"] } proptest = { version = "1.0" } arbitrary = { version = "1.1.7", features = ["derive"] } diff --git a/crates/stages/src/stages/merkle.rs b/crates/stages/src/stages/merkle.rs index 20ebab25b59..3078742fd11 100644 --- a/crates/stages/src/stages/merkle.rs +++ b/crates/stages/src/stages/merkle.rs @@ -2,7 +2,7 @@ use crate::{ExecInput, ExecOutput, Stage, StageError, StageId, UnwindInput, Unwi use reth_db::{database::Database, tables, transaction::DbTx}; use reth_interfaces::consensus; use reth_provider::{trie::DBTrieLoader, Transaction}; -use std::fmt::Debug; +use std::{fmt::Debug, ops::DerefMut}; use tracing::*; /// The [`StageId`] of the merkle hashing execution stage. @@ -108,14 +108,14 @@ impl Stage for MerkleStage { } else if to_transition - from_transition > threshold || stage_progress == 0 { debug!(target: "sync::stages::merkle::exec", current = ?stage_progress, target = ?previous_stage_progress, "Rebuilding trie"); // if there are more blocks than threshold it is faster to rebuild the trie - DBTrieLoader::::new(tx) + DBTrieLoader::new(tx.deref_mut()) .calculate_root() .map_err(|e| StageError::Fatal(Box::new(e)))? } else { debug!(target: "sync::stages::merkle::exec", current = ?stage_progress, target = ?previous_stage_progress, "Updating trie"); // Iterate over changeset (similar to Hashing stages) and take new values let current_root = tx.get_header(stage_progress)?.state_root; - DBTrieLoader::::new(tx) + DBTrieLoader::new(tx.deref_mut()) .update_root(current_root, from_transition..to_transition) .map_err(|e| StageError::Fatal(Box::new(e)))? }; @@ -160,7 +160,7 @@ impl Stage for MerkleStage { let from_transition = tx.get_block_transition(input.unwind_to)?; let to_transition = tx.get_block_transition(input.stage_progress)?; - let block_root = DBTrieLoader::::new(tx) + let block_root = DBTrieLoader::new(tx.deref_mut()) .update_root(current_root, from_transition..to_transition) .map_err(|e| StageError::Fatal(Box::new(e)))?; @@ -191,6 +191,7 @@ mod tests { use assert_matches::assert_matches; use reth_db::{ cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO, DbDupCursorRW}, + database::DatabaseGAT, mdbx::{Env, WriteMap}, tables, transaction::{DbTx, DbTxMut}, @@ -199,7 +200,7 @@ mod tests { random_block, random_block_range, random_contract_account_range, random_transition_range, }; use reth_primitives::{keccak256, Account, Address, SealedBlock, StorageEntry, H256, U256}; - use std::collections::BTreeMap; + use std::{collections::BTreeMap, ops::Deref}; stage_test_suite_ext!(MerkleTestRunner, merkle); @@ -262,8 +263,8 @@ mod tests { fn create_trie_loader<'tx, 'db>( tx: &'tx Transaction<'db, Env>, - ) -> DBTrieLoader<'tx, 'db, Env> { - DBTrieLoader::>::new(tx) + ) -> DBTrieLoader<'tx, as DatabaseGAT<'db>>::TXMut> { + DBTrieLoader::new(tx.deref()) } struct MerkleTestRunner { diff --git a/crates/storage/provider/Cargo.toml b/crates/storage/provider/Cargo.toml index 0a1802d14e8..c0e26f28791 100644 --- a/crates/storage/provider/Cargo.toml +++ b/crates/storage/provider/Cargo.toml @@ -13,8 +13,8 @@ reth-primitives = { path = "../../primitives" } reth-interfaces = { path = "../../interfaces" } reth-revm-primitives = { path = "../../revm/revm-primitives" } reth-db = { path = "../db" } -reth-tracing = {path = "../../tracing"} -reth-rlp = {path = "../../rlp"} +reth-tracing = { path = "../../tracing" } +reth-rlp = { path = "../../rlp" } revm-primitives = "1.0.0" diff --git a/crates/storage/provider/src/lib.rs b/crates/storage/provider/src/lib.rs index a980f2f15a1..77882f42b4b 100644 --- a/crates/storage/provider/src/lib.rs +++ b/crates/storage/provider/src/lib.rs @@ -23,7 +23,7 @@ pub use providers::{ LatestStateProviderRef, ShareableDatabase, }; -/// Merkle trie +/// Helper type for loading Merkle Patricia Trees from the database pub mod trie; /// Execution result diff --git a/crates/storage/provider/src/providers/state/historical.rs b/crates/storage/provider/src/providers/state/historical.rs index 214f0ba2eb8..fbfc248da1a 100644 --- a/crates/storage/provider/src/providers/state/historical.rs +++ b/crates/storage/provider/src/providers/state/historical.rs @@ -10,7 +10,7 @@ use reth_db::{ }; use reth_interfaces::Result; use reth_primitives::{ - Account, Address, Bytecode, StorageKey, StorageValue, TransitionId, H256, U256, + Account, Address, Bytecode, Bytes, StorageKey, StorageValue, TransitionId, H256, U256, }; use std::marker::PhantomData; @@ -119,6 +119,15 @@ impl<'a, 'b, TX: DbTx<'a>> StateProvider for HistoricalStateProviderRef<'a, 'b, fn bytecode_by_hash(&self, code_hash: H256) -> Result> { self.tx.get::(code_hash).map_err(Into::into) } + + /// Get account and storage proofs. + fn proof( + &self, + _address: Address, + _keys: &[H256], + ) -> Result<(Vec, H256, Vec>)> { + todo!("this should retrieve past state info and generate proof") + } } /// State provider for a given transition diff --git a/crates/storage/provider/src/providers/state/latest.rs b/crates/storage/provider/src/providers/state/latest.rs index 311f37e415d..e600eacb294 100644 --- a/crates/storage/provider/src/providers/state/latest.rs +++ b/crates/storage/provider/src/providers/state/latest.rs @@ -1,10 +1,17 @@ use crate::{ - providers::state::macros::delegate_provider_impls, AccountProvider, BlockHashProvider, - StateProvider, + providers::state::macros::delegate_provider_impls, trie::DBTrieLoader, AccountProvider, + BlockHashProvider, StateProvider, +}; +use reth_db::{ + cursor::{DbCursorRO, DbDupCursorRO}, + tables, + transaction::DbTx, +}; +use reth_interfaces::{provider::ProviderError, Result}; +use reth_primitives::{ + keccak256, Account, Address, Bytecode, Bytes, StorageKey, StorageValue, H256, KECCAK_EMPTY, + U256, }; -use reth_db::{cursor::DbDupCursorRO, tables, transaction::DbTx}; -use reth_interfaces::Result; -use reth_primitives::{Account, Address, Bytecode, StorageKey, StorageValue, H256, U256}; use std::marker::PhantomData; /// State provider over latest state that takes tx reference. @@ -52,6 +59,42 @@ impl<'a, 'b, TX: DbTx<'a>> StateProvider for LatestStateProviderRef<'a, 'b, TX> fn bytecode_by_hash(&self, code_hash: H256) -> Result> { self.db.get::(code_hash).map_err(Into::into) } + + fn proof( + &self, + address: Address, + keys: &[H256], + ) -> Result<(Vec, H256, Vec>)> { + let hashed_address = keccak256(address); + let loader = DBTrieLoader::new(self.db); + let root = self + .db + .cursor_read::()? + .last()? + .ok_or(ProviderError::Header { number: 0 })? + .1 + .state_root; + + let (account_proof, storage_root) = loader + .generate_acount_proof(self.db, root, hashed_address) + .map_err(|_| ProviderError::StateTree)?; + let account_proof = account_proof.into_iter().map(Bytes::from).collect(); + + let storage_proof = if storage_root == KECCAK_EMPTY { + // if there isn't storage, we return empty storage proofs + (0..keys.len()).map(|_| Vec::new()).collect() + } else { + let hashed_keys: Vec = keys.iter().map(keccak256).collect(); + loader + .generate_storage_proofs(self.db, storage_root, hashed_address, &hashed_keys) + .map_err(|_| ProviderError::StateTree)? + .into_iter() + .map(|v| v.into_iter().map(Bytes::from).collect()) + .collect() + }; + + Ok((account_proof, storage_root, storage_proof)) + } } /// State provider for the latest state. diff --git a/crates/storage/provider/src/providers/state/macros.rs b/crates/storage/provider/src/providers/state/macros.rs index d488010d713..ab4ae7b7bfb 100644 --- a/crates/storage/provider/src/providers/state/macros.rs +++ b/crates/storage/provider/src/providers/state/macros.rs @@ -5,7 +5,7 @@ /// /// Used to implement provider traits. macro_rules! delegate_impls_to_as_ref { - (for $target:ty => $($trait:ident $(where [$($generics:tt)*])? { $(fn $func:ident(&self, $($arg:ident: $argty:path),*) -> $ret:path;)* })* ) => { + (for $target:ty => $($trait:ident $(where [$($generics:tt)*])? { $(fn $func:ident(&self, $($arg:ident: $argty:ty),*) -> $ret:path;)* })* ) => { $( impl<'a, $($($generics)*)?> $trait for $target { @@ -38,6 +38,7 @@ macro_rules! delegate_provider_impls { } StateProvider $(where [$($generics)*])?{ fn storage(&self, account: reth_primitives::Address, storage_key: reth_primitives::StorageKey) -> reth_interfaces::Result>; + fn proof(&self, address: reth_primitives::Address, keys: &[reth_primitives::H256]) -> reth_interfaces::Result<(Vec, reth_primitives::H256, Vec>)>; fn bytecode_by_hash(&self, code_hash: reth_primitives::H256) -> reth_interfaces::Result>; } ); diff --git a/crates/storage/provider/src/test_utils/mock.rs b/crates/storage/provider/src/test_utils/mock.rs index 31430ae84a1..83e5885ea29 100644 --- a/crates/storage/provider/src/test_utils/mock.rs +++ b/crates/storage/provider/src/test_utils/mock.rs @@ -252,6 +252,14 @@ impl StateProvider for MockEthProvider { } })) } + + fn proof( + &self, + _address: Address, + _keys: &[H256], + ) -> Result<(Vec, H256, Vec>)> { + todo!() + } } impl EvmEnvProvider for MockEthProvider { diff --git a/crates/storage/provider/src/test_utils/noop.rs b/crates/storage/provider/src/test_utils/noop.rs index c7bd3f80207..d79970eaa71 100644 --- a/crates/storage/provider/src/test_utils/noop.rs +++ b/crates/storage/provider/src/test_utils/noop.rs @@ -4,8 +4,9 @@ use crate::{ }; use reth_interfaces::Result; use reth_primitives::{ - Account, Address, Block, BlockHash, BlockId, BlockNumber, Bytecode, ChainInfo, Header, Receipt, - StorageKey, StorageValue, TransactionSigned, TxHash, TxNumber, H256, U256, + Account, Address, Block, BlockHash, BlockId, BlockNumber, Bytecode, Bytes, ChainInfo, Header, + Receipt, StorageKey, StorageValue, TransactionSigned, TxHash, TxNumber, H256, KECCAK_EMPTY, + U256, }; use revm_primitives::{BlockEnv, CfgEnv}; use std::ops::RangeBounds; @@ -113,6 +114,14 @@ impl StateProvider for NoopProvider { fn bytecode_by_hash(&self, _code_hash: H256) -> Result> { Ok(None) } + + fn proof( + &self, + _address: Address, + _keys: &[H256], + ) -> Result<(Vec, H256, Vec>)> { + Ok((vec![], KECCAK_EMPTY, vec![])) + } } impl EvmEnvProvider for NoopProvider { diff --git a/crates/storage/provider/src/traits/state.rs b/crates/storage/provider/src/traits/state.rs index f82f2ad5482..e2f502593c6 100644 --- a/crates/storage/provider/src/traits/state.rs +++ b/crates/storage/provider/src/traits/state.rs @@ -3,18 +3,23 @@ use crate::BlockHashProvider; use auto_impl::auto_impl; use reth_interfaces::Result; use reth_primitives::{ - Address, BlockHash, BlockNumber, Bytecode, StorageKey, StorageValue, H256, KECCAK_EMPTY, U256, + Address, BlockHash, BlockNumber, Bytecode, Bytes, StorageKey, StorageValue, H256, KECCAK_EMPTY, + U256, }; /// An abstraction for a type that provides state data. #[auto_impl(&, Box)] pub trait StateProvider: BlockHashProvider + AccountProvider + Send + Sync { - /// Get storage. + /// Get storage of given account. fn storage(&self, account: Address, storage_key: StorageKey) -> Result>; /// Get account code by its hash fn bytecode_by_hash(&self, code_hash: H256) -> Result>; + /// Get account and storage proofs. + fn proof(&self, address: Address, keys: &[H256]) + -> Result<(Vec, H256, Vec>)>; + /// Get account code by its address. /// /// Returns `None` if the account doesn't exist or account is not a contract diff --git a/crates/storage/provider/src/transaction.rs b/crates/storage/provider/src/transaction.rs index a1bf8bbdf19..0c1d39b362b 100644 --- a/crates/storage/provider/src/transaction.rs +++ b/crates/storage/provider/src/transaction.rs @@ -308,7 +308,7 @@ where // merkle tree { let current_root = self.get_header(parent_block_number)?.state_root; - let loader = DBTrieLoader::::new(self); + let loader = DBTrieLoader::new(self.deref_mut()); let root = loader.update_root(current_root, from..to)?; if root != block.state_root { return Err(TransactionError::StateTrieRootMismatch { diff --git a/crates/storage/provider/src/trie/mod.rs b/crates/storage/provider/src/trie/mod.rs index 1821aa7e16b..ffb9661d189 100644 --- a/crates/storage/provider/src/trie/mod.rs +++ b/crates/storage/provider/src/trie/mod.rs @@ -2,7 +2,6 @@ use cita_trie::{PatriciaTrie, Trie}; use hasher::HasherKeccak; use reth_db::{ cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO}, - database::{Database, DatabaseGAT}, models::{AccountBeforeTx, TransitionIdAddress}, tables, transaction::{DbTx, DbTxMut}, @@ -18,6 +17,7 @@ use reth_rlp::{ use reth_tracing::tracing::*; use std::{ collections::{BTreeMap, BTreeSet}, + marker::PhantomData, ops::Range, sync::Arc, }; @@ -26,24 +26,28 @@ use std::{ #[allow(missing_docs)] #[derive(Debug, thiserror::Error)] pub enum TrieError { + /// Error returned by the underlying implementation. #[error("Some error occurred: {0}")] InternalError(#[from] cita_trie::TrieError), + /// The database doesn't contain the root of the trie. #[error("The root node wasn't found in the DB")] MissingRoot(H256), + /// Error returned by the database. #[error("{0:?}")] DatabaseError(#[from] reth_db::Error), + /// Error when encoding/decoding a value. #[error("{0:?}")] DecodeError(#[from] DecodeError), } -/// Database wrapper implementing HashDB trait. -pub struct HashDatabase<'tx, 'db, DB: Database> { - tx: &'tx >::TXMut, +/// Database wrapper implementing HashDB trait, with a read-write transaction. +pub struct HashDatabaseMut<'tx, TX> { + tx: &'tx TX, } -impl<'tx, 'db, DB> cita_trie::DB for HashDatabase<'tx, 'db, DB> +impl<'tx, 'db, TX> cita_trie::DB for HashDatabaseMut<'tx, TX> where - DB: Database, + TX: DbTxMut<'db> + DbTx<'db> + Send + Sync, { type Error = TrieError; @@ -87,9 +91,12 @@ where } } -impl<'tx, 'db, DB: Database> HashDatabase<'tx, 'db, DB> { +impl<'tx, 'db, TX> HashDatabaseMut<'tx, TX> +where + TX: DbTxMut<'db> + DbTx<'db> + Send + Sync, +{ /// Instantiates a new Database for the accounts trie, with an empty root - pub fn new(tx: &'tx >::TXMut) -> Result { + pub fn new(tx: &'tx TX) -> Result { let root = EMPTY_ROOT; if tx.get::(root)?.is_none() { tx.put::(root, [EMPTY_STRING_CODE].to_vec())?; @@ -98,10 +105,7 @@ impl<'tx, 'db, DB: Database> HashDatabase<'tx, 'db, DB> { } /// Instantiates a new Database for the accounts trie, with an existing root - pub fn from_root( - tx: &'tx >::TXMut, - root: H256, - ) -> Result { + pub fn from_root(tx: &'tx TX, root: H256) -> Result { if root == EMPTY_ROOT { return Self::new(tx) } @@ -110,15 +114,15 @@ impl<'tx, 'db, DB: Database> HashDatabase<'tx, 'db, DB> { } } -/// Database wrapper implementing HashDB trait. -pub struct DupHashDatabase<'tx, 'db, DB: Database> { - tx: &'tx >::TXMut, +/// Database wrapper implementing HashDB trait, with a read-write transaction. +pub struct DupHashDatabaseMut<'tx, TX> { + tx: &'tx TX, key: H256, } -impl<'tx, 'db, DB> cita_trie::DB for DupHashDatabase<'tx, 'db, DB> +impl<'tx, 'db, TX> cita_trie::DB for DupHashDatabaseMut<'tx, TX> where - DB: Database, + TX: DbTxMut<'db> + DbTx<'db> + Send + Sync, { type Error = TrieError; @@ -172,9 +176,12 @@ where } } -impl<'tx, 'db, DB: Database> DupHashDatabase<'tx, 'db, DB> { +impl<'tx, 'db, TX> DupHashDatabaseMut<'tx, TX> +where + TX: DbTxMut<'db> + DbTx<'db> + Send + Sync, +{ /// Instantiates a new Database for the storage trie, with an empty root - pub fn new(tx: &'tx >::TXMut, key: H256) -> Result { + pub fn new(tx: &'tx TX, key: H256) -> Result { let root = EMPTY_ROOT; let mut cursor = tx.cursor_dup_write::()?; if cursor.seek_by_key_subkey(key, root)?.filter(|entry| entry.hash == root).is_none() { @@ -187,11 +194,7 @@ impl<'tx, 'db, DB: Database> DupHashDatabase<'tx, 'db, DB> { } /// Instantiates a new Database for the storage trie, with an existing root - pub fn from_root( - tx: &'tx >::TXMut, - key: H256, - root: H256, - ) -> Result { + pub fn from_root(tx: &'tx TX, key: H256, root: H256) -> Result { if root == EMPTY_ROOT { return Self::new(tx, key) } @@ -203,6 +206,95 @@ impl<'tx, 'db, DB: Database> DupHashDatabase<'tx, 'db, DB> { } } +/// Database wrapper implementing HashDB trait, with a read-only transaction. +struct HashDatabase<'tx, 'itx, TX: DbTx<'itx>> { + tx: &'tx TX, + _p: PhantomData<&'itx ()>, // to suppress "unused" lifetime 'itx +} + +impl<'tx, 'itx, TX> cita_trie::DB for HashDatabase<'tx, 'itx, TX> +where + TX: DbTx<'itx>, +{ + type Error = TrieError; + + fn get(&self, key: &[u8]) -> Result>, Self::Error> { + Ok(self.tx.get::(H256::from_slice(key))?) + } + + fn contains(&self, key: &[u8]) -> Result { + Ok(::get(self, key)?.is_some()) + } + + fn insert(&self, _key: Vec, _value: Vec) -> Result<(), Self::Error> { + // this could be avoided if cita_trie::DB was split into two traits + // with read and write operations respectively + unimplemented!("insert isn't valid for read-only transaction"); + } + + fn remove(&self, _key: &[u8]) -> Result<(), Self::Error> { + unimplemented!("remove isn't valid for read-only transaction"); + } + + fn flush(&self) -> Result<(), Self::Error> { + Ok(()) + } +} + +impl<'tx, 'itx, TX: DbTx<'itx>> HashDatabase<'tx, 'itx, TX> { + /// Instantiates a new Database for the accounts trie, with an existing root + fn from_root(tx: &'tx TX, root: H256) -> Result { + tx.get::(root)?.ok_or(TrieError::MissingRoot(root))?; + Ok(Self { tx, _p: Default::default() }) + } +} + +/// Database wrapper implementing HashDB trait, with a read-only transaction. +struct DupHashDatabase<'tx, 'itx, TX: DbTx<'itx>> { + tx: &'tx TX, + key: H256, + _p: PhantomData<&'itx ()>, // to suppress "unused" lifetime 'itx +} + +impl<'tx, 'itx, TX> cita_trie::DB for DupHashDatabase<'tx, 'itx, TX> +where + TX: DbTx<'itx>, +{ + type Error = TrieError; + + fn get(&self, key: &[u8]) -> Result>, Self::Error> { + let mut cursor = self.tx.cursor_dup_read::()?; + Ok(cursor.seek_by_key_subkey(self.key, H256::from_slice(key))?.map(|entry| entry.node)) + } + + fn contains(&self, key: &[u8]) -> Result { + Ok(::get(self, key)?.is_some()) + } + + fn insert(&self, _key: Vec, _value: Vec) -> Result<(), Self::Error> { + // Caching and bulk inserting shouldn't be needed, as the data is ordered + unimplemented!("insert isn't valid for read-only transaction"); + } + + fn remove(&self, _key: &[u8]) -> Result<(), Self::Error> { + unimplemented!("remove isn't valid for read-only transaction"); + } + + fn flush(&self) -> Result<(), Self::Error> { + Ok(()) + } +} + +impl<'tx, 'itx, TX: DbTx<'itx>> DupHashDatabase<'tx, 'itx, TX> { + /// Instantiates a new Database for the storage trie, with an existing root + fn from_root(tx: &'tx TX, key: H256, root: H256) -> Result { + tx.cursor_dup_read::()? + .seek_by_key_subkey(key, root)? + .ok_or(TrieError::MissingRoot(root))?; + Ok(Self { tx, key, _p: Default::default() }) + } +} + /// An Ethereum account, for RLP encoding traits deriving. #[derive(Clone, Copy, Debug, PartialEq, Eq, Default, RlpEncodable, RlpDecodable)] pub struct EthAccount { @@ -240,18 +332,28 @@ impl EthAccount { } } +/// A merkle proof of existence (or nonexistence) of a leaf value. Consists +/// of a the encoded nodes in the path from the root of the tree to the leaf. +pub type MerkleProof = Vec>; + /// Struct for calculating the root of a merkle patricia tree, /// while populating the database with intermediate hashes. -pub struct DBTrieLoader<'tx, 'db, DB: Database> { - tx: &'tx >::TXMut, +pub struct DBTrieLoader<'tx, TX> { + tx: &'tx TX, } -impl<'tx, 'db, DB: Database> DBTrieLoader<'tx, 'db, DB> { +impl<'tx, TX> DBTrieLoader<'tx, TX> { /// Create new instance of trie loader. - pub fn new(tx: &'tx >::TXMut) -> Self { + pub fn new(tx: &'tx TX) -> Self { Self { tx } } +} +// Read-write impls +impl<'tx, 'db, TX> DBTrieLoader<'tx, TX> +where + TX: DbTxMut<'db> + DbTx<'db> + Send + Sync, +{ /// Calculates the root of the state trie, saving intermediate hashes in the database. pub fn calculate_root(&self) -> Result { self.tx.clear::()?; @@ -260,7 +362,7 @@ impl<'tx, 'db, DB: Database> DBTrieLoader<'tx, 'db, DB> { let mut accounts_cursor = self.tx.cursor_read::()?; let mut walker = accounts_cursor.walk(None)?; - let db = Arc::new(HashDatabase::::new(self.tx)?); + let db = Arc::new(HashDatabaseMut::new(self.tx)?); let hasher = Arc::new(HasherKeccak::new()); @@ -281,7 +383,7 @@ impl<'tx, 'db, DB: Database> DBTrieLoader<'tx, 'db, DB> { /// Calculate the accounts storage root. pub fn calculate_storage_root(&self, address: H256) -> Result { - let db = Arc::new(DupHashDatabase::::new(self.tx, address)?); + let db = Arc::new(DupHashDatabaseMut::new(self.tx, address)?); let hasher = Arc::new(HasherKeccak::new()); @@ -319,7 +421,7 @@ impl<'tx, 'db, DB: Database> DBTrieLoader<'tx, 'db, DB> { let changed_accounts = self.gather_changes(tid_range)?; - let db = Arc::new(HashDatabase::::from_root(self.tx, root)?); + let db = Arc::new(HashDatabaseMut::from_root(self.tx, root)?); let hasher = Arc::new(HasherKeccak::new()); @@ -362,7 +464,7 @@ impl<'tx, 'db, DB: Database> DBTrieLoader<'tx, 'db, DB> { address: H256, changed_storages: BTreeSet, ) -> Result { - let db = Arc::new(DupHashDatabase::::from_root(self.tx, address, root)?); + let db = Arc::new(DupHashDatabaseMut::from_root(self.tx, address, root)?); let hasher = Arc::new(HasherKeccak::new()); @@ -437,6 +539,52 @@ impl<'tx, 'db, DB: Database> DBTrieLoader<'tx, 'db, DB> { } } +// Read-only impls +impl<'tx, 'db, TX> DBTrieLoader<'tx, TX> +where + TX: DbTx<'db> + Send + Sync, +{ + /// Returns a Merkle proof of the given account, plus its storage root hash. + pub fn generate_acount_proof<'itx>( + &self, + tx: &'tx impl DbTx<'itx>, + root: H256, + address: H256, + ) -> Result<(MerkleProof, H256), TrieError> { + let db = Arc::new(HashDatabase::from_root(tx, root)?); + let hasher = Arc::new(HasherKeccak::new()); + + let trie = PatriciaTrie::from(Arc::clone(&db), Arc::clone(&hasher), root.as_bytes())?; + let proof = trie.get_proof(address.as_bytes())?; + + let Some(account) = trie.get(address.as_slice())? else { return Ok((proof, KECCAK_EMPTY)) }; + + let storage_root = EthAccount::decode(&mut account.as_slice())?.storage_root; + + Ok((proof, storage_root)) + } + + /// Returns a Merkle proof of the given storage keys, starting at the given root hash. + pub fn generate_storage_proofs<'itx>( + &self, + tx: &'tx impl DbTx<'itx>, + storage_root: H256, + address: H256, + keys: &[H256], + ) -> Result, TrieError> { + let db = Arc::new(DupHashDatabase::from_root(tx, address, storage_root)?); + let hasher = Arc::new(HasherKeccak::new()); + + let trie = + PatriciaTrie::from(Arc::clone(&db), Arc::clone(&hasher), storage_root.as_bytes())?; + + let proof = + keys.iter().map(|key| trie.get_proof(key.as_bytes())).collect::, _>>()?; + + Ok(proof) + } +} + #[cfg(test)] mod tests { use crate::Transaction; @@ -445,6 +593,7 @@ mod tests { use assert_matches::assert_matches; use proptest::{prelude::ProptestConfig, proptest}; use reth_db::{ + database::{Database, DatabaseGAT}, mdbx::{test_utils::create_test_rw_db, Env, WriteMap}, tables, transaction::DbTxMut, @@ -453,14 +602,43 @@ mod tests { hex_literal::hex, keccak256, proofs::{genesis_state_root, KeccakHasher, EMPTY_ROOT}, - Address, ChainSpec, MAINNET, + Address, Bytes, ChainSpec, Genesis, MAINNET, }; use std::{collections::HashMap, ops::Deref, str::FromStr}; use triehash::sec_trie_root; + fn load_mainnet_genesis_root(tx: &mut Transaction<'_, DB>) -> Genesis { + let ChainSpec { genesis, .. } = MAINNET.clone(); + + // Insert account state + for (address, account) in &genesis.alloc { + tx.put::( + *address, + Account { + nonce: account.nonce.unwrap_or_default(), + balance: account.balance, + bytecode_hash: None, + }, + ) + .unwrap(); + tx.put::( + keccak256(address), + Account { + nonce: account.nonce.unwrap_or_default(), + balance: account.balance, + bytecode_hash: None, + }, + ) + .unwrap(); + } + tx.commit().unwrap(); + + genesis + } + fn create_test_loader<'tx, 'db>( tx: &'tx Transaction<'db, Env>, - ) -> DBTrieLoader<'tx, 'db, Arc>> { + ) -> DBTrieLoader<'tx, > as DatabaseGAT<'db>>::TXMut> { DBTrieLoader::new(tx.deref()) } @@ -596,21 +774,8 @@ mod tests { fn verify_genesis() { let db = create_test_rw_db(); let mut tx = Transaction::new(db.as_ref()).unwrap(); - let ChainSpec { genesis, .. } = MAINNET.clone(); - // Insert account state - for (address, account) in &genesis.alloc { - tx.put::( - keccak256(address), - Account { - nonce: account.nonce.unwrap_or_default(), - balance: account.balance, - bytecode_hash: None, - }, - ) - .unwrap(); - } - tx.commit().unwrap(); + let genesis = load_mainnet_genesis_root(&mut tx); let state_root = genesis_state_root(&genesis.alloc); @@ -708,4 +873,125 @@ mod tests { test_with_accounts(accounts); }); } + + #[test] + fn get_proof() { + let db = create_test_rw_db(); + let mut tx = Transaction::new(db.as_ref()).unwrap(); + + load_mainnet_genesis_root(&mut tx); + + let root = { + let trie = create_test_loader(&tx); + trie.calculate_root().expect("should be able to load trie") + }; + + tx.commit().unwrap(); + + let address = Address::from(hex!("000d836201318ec6899a67540690382780743280")); + + let trie = create_test_loader(&tx); + let (proof, storage_root) = trie + .generate_acount_proof(&tx.inner().tx().unwrap(), root, keccak256(address)) + .expect("failed to generate proof"); + + // values extracted from geth via rpc: + // { + // "method": "eth_getProof", + // "params": ["0x000d836201318ec6899a67540690382780743280", [], "0x0"] + // } + let expected = [ + hex!("f90211a090dcaf88c40c7bbc95a912cbdde67c175767b31173df9ee4b0d733bfdd511c43a0babe369f6b12092f49181ae04ca173fb68d1a5456f18d20fa32cba73954052bda0473ecf8a7e36a829e75039a3b055e51b8332cbf03324ab4af2066bbd6fbf0021a0bbda34753d7aa6c38e603f360244e8f59611921d9e1f128372fec0d586d4f9e0a04e44caecff45c9891f74f6a2156735886eedf6f1a733628ebc802ec79d844648a0a5f3f2f7542148c973977c8a1e154c4300fec92f755f7846f1b734d3ab1d90e7a0e823850f50bf72baae9d1733a36a444ab65d0a6faaba404f0583ce0ca4dad92da0f7a00cbe7d4b30b11faea3ae61b7f1f2b315b61d9f6bd68bfe587ad0eeceb721a07117ef9fc932f1a88e908eaead8565c19b5645dc9e5b1b6e841c5edbdfd71681a069eb2de283f32c11f859d7bcf93da23990d3e662935ed4d6b39ce3673ec84472a0203d26456312bbc4da5cd293b75b840fc5045e493d6f904d180823ec22bfed8ea09287b5c21f2254af4e64fca76acc5cd87399c7f1ede818db4326c98ce2dc2208a06fc2d754e304c48ce6a517753c62b1a9c1d5925b89707486d7fc08919e0a94eca07b1c54f15e299bd58bdfef9741538c7828b5d7d11a489f9c20d052b3471df475a051f9dd3739a927c89e357580a4c97b40234aa01ed3d5e0390dc982a7975880a0a089d613f26159af43616fd9455bb461f4869bfede26f2130835ed067a8b967bfb80").as_slice(), + hex!("f90211a0dae48f5b47930c28bb116fbd55e52cd47242c71bf55373b55eb2805ee2e4a929a00f1f37f337ec800e2e5974e2e7355f10f1a4832b39b846d916c3597a460e0676a0da8f627bb8fbeead17b318e0a8e4f528db310f591bb6ab2deda4a9f7ca902ab5a0971c662648d58295d0d0aa4b8055588da0037619951217c22052802549d94a2fa0ccc701efe4b3413fd6a61a6c9f40e955af774649a8d9fd212d046a5a39ddbb67a0d607cdb32e2bd635ee7f2f9e07bc94ddbd09b10ec0901b66628e15667aec570ba05b89203dc940e6fa70ec19ad4e01d01849d3a5baa0a8f9c0525256ed490b159fa0b84227d48df68aecc772939a59afa9e1a4ab578f7b698bdb1289e29b6044668ea0fd1c992070b94ace57e48cbf6511a16aa770c645f9f5efba87bbe59d0a042913a0e16a7ccea6748ae90de92f8aef3b3dc248a557b9ac4e296934313f24f7fced5fa042373cf4a00630d94de90d0a23b8f38ced6b0f7cb818b8925fee8f0c2a28a25aa05f89d2161c1741ff428864f7889866484cef622de5023a46e795dfdec336319fa07597a017664526c8c795ce1da27b8b72455c49657113e0455552dbc068c5ba31a0d5be9089012fda2c585a1b961e988ea5efcd3a06988e150a8682091f694b37c5a0f7b0352e38c315b2d9a14d51baea4ddee1770974c806e209355233c3c89dce6ea049bf6e8df0acafd0eff86defeeb305568e44d52d2235cf340ae15c6034e2b24180").as_slice(), + hex!("f901f1a0cf67e0f5d5f8d70e53a6278056a14ddca46846f5ef69c7bde6810d058d4a9eda80a06732ada65afd192197fe7ce57792a7f25d26978e64e954b7b84a1f7857ac279da05439f8d011683a6fc07efb90afca198fd7270c795c835c7c85d91402cda992eaa0449b93033b6152d289045fdb0bf3f44926f831566faa0e616b7be1abaad2cb2da031be6c3752bcd7afb99b1bb102baf200f8567c394d464315323a363697646616a0a40e3ed11d906749aa501279392ffde868bd35102db41364d9c601fd651f974aa0044bfa4fe8dd1a58e6c7144da79326e94d1331c0b00373f6ae7f3662f45534b7a098005e3e48db68cb1dc9b9f034ff74d2392028ddf718b0f2084133017da2c2e7a02a62bc40414ee95b02e202a9e89babbabd24bef0abc3fc6dcd3e9144ceb0b725a0239facd895bbf092830390a8676f34b35b29792ae561f196f86614e0448a5792a0a4080f88925daff6b4ce26d188428841bd65655d8e93509f2106020e76d41eefa04918987904be42a6894256ca60203283d1b89139cf21f09f5719c44b8cdbb8f7a06201fc3ef0827e594d953b5e3165520af4fceb719e11cc95fd8d3481519bfd8ca05d0e353d596bd725b09de49c01ede0f29023f0153d7b6d401556aeb525b2959ba0cd367d0679950e9c5f2aa4298fd4b081ade2ea429d71ff390c50f8520e16e30880").as_slice(), + hex!("f87180808080808080a0dbee8b33c73b86df839f309f7ac92eee19836e08b39302ffa33921b3c6a09f66a06068b283d51aeeee682b8fb5458354315d0b91737441ede5e137c18b4775174a8080808080a0fe7779c7d58c2fda43eba0a6644043c86ebb9ceb4836f89e30831f23eb059ece8080").as_slice(), + hex!("f8719f20b71c90b0d523dd5004cf206f325748da347685071b34812e21801f5270c4b84ff84d80890ad78ebc5ac6200000a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421a0c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").as_slice(), + ]; + + assert_eq!(storage_root, EMPTY_ROOT); + + assert_eq!(proof.len(), 5); + + for (node, expected) in proof.into_iter().zip(expected.into_iter()) { + assert_eq!(Bytes::from(node.as_slice()), Bytes::from(expected)); + } + } + + #[test] + fn get_storage_proofs() { + let db = create_test_rw_db(); + let mut tx = Transaction::new(db.as_ref()).unwrap(); + + let address = Address::from_str("9fe4abd71ad081f091bd06dd1c16f7e92927561e").unwrap(); + let hashed_address = keccak256(address); + + let storage = HashMap::from([ + (H256::zero(), U256::from(3)), + (H256::from_low_u64_be(2), U256::from(1)), + ]); + + let code = "el buen fla"; + let account = Account { + nonce: 155, + balance: U256::from(414241124u32), + bytecode_hash: Some(keccak256(code)), + }; + tx.put::(hashed_address, account).unwrap(); + + for (k, v) in storage.clone() { + tx.put::( + hashed_address, + StorageEntry { key: keccak256(k), value: v }, + ) + .unwrap(); + } + + let root = { + let trie = create_test_loader(&tx); + trie.calculate_root().expect("should be able to load trie") + }; + + tx.commit().unwrap(); + + let trie = create_test_loader(&tx); + let (account_proof, storage_root) = trie + .generate_acount_proof(&tx.inner().tx().unwrap(), root, hashed_address) + .expect("failed to generate proof"); + + // values extracted from geth via rpc: + let expected_account = hex!("f86fa1205126413e7857595763591580306b3f228f999498c4c5dfa74f633364936e7651b84bf849819b8418b0d164a029ff6f4d518044318d75b118cf439d8d3d7249c8afcba06ba9ecdf8959410571a02ce1a85814ad94a94ed2a1abaf7c57e9b64326622c1b8c21b4ba4d0e7df61392").as_slice(); + let expected_storage = [ + [ + // 0x0000000000000000000000000000000000000000000000000000000000000002 + hex!("f8518080a04355bd3061ad2d17e0782413925b4fd81a56bd162d91eedb2a00d6c87611471480a015503e91f9250654cf72906e38a7cb14c3f1cc06658379d37f0c5b5c32482880808080808080808080808080").as_slice(), + hex!("e2a0305787fa12a823e0f2b7631cc41b3ba8828b3321ca811111fa75cd3aa3bb5ace01").as_slice(), + ], + [ + // 0x0000000000000000000000000000000000000000000000000000000000000000 + hex!("f8518080a04355bd3061ad2d17e0782413925b4fd81a56bd162d91eedb2a00d6c87611471480a015503e91f9250654cf72906e38a7cb14c3f1cc06658379d37f0c5b5c32482880808080808080808080808080").as_slice(), + hex!("e2a0390decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e56303").as_slice(), + ] + ]; + + assert!(storage_root != EMPTY_ROOT); + + assert_eq!(account_proof.len(), 1); + assert_eq!(account_proof[0], expected_account); + + let storage_proofs = trie + .generate_storage_proofs( + &tx.inner().tx().unwrap(), + storage_root, + hashed_address, + &[keccak256(H256::from_low_u64_be(2)), keccak256(H256::zero())], + ) + .expect("couldn't generate storage proof"); + + for (proof, expected) in storage_proofs.into_iter().zip(expected_storage) { + assert_eq!(proof.len(), expected.len()); + for (got_node, expected_node) in proof.into_iter().zip(expected) { + assert_eq!(got_node, expected_node); + } + } + } } From c5cd236e1a664d2e66db18e2c6642faf91ec13ac Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 14 Mar 2023 04:00:58 +0100 Subject: [PATCH 115/191] feat(rpc): implement log subscription (#1719) --- crates/rpc/rpc-types/src/eth/pubsub.rs | 2 +- crates/rpc/rpc/src/eth/pubsub.rs | 92 +++++++++++++++++++++++--- 2 files changed, 83 insertions(+), 11 deletions(-) diff --git a/crates/rpc/rpc-types/src/eth/pubsub.rs b/crates/rpc/rpc-types/src/eth/pubsub.rs index 7fbd1e46bd6..5166e64368a 100644 --- a/crates/rpc/rpc-types/src/eth/pubsub.rs +++ b/crates/rpc/rpc-types/src/eth/pubsub.rs @@ -1,7 +1,7 @@ //! Ethereum types for pub-sub use crate::{Log, RichHeader}; -use reth_primitives::{rpc::Filter, H256}; +use reth_primitives::{filter::Filter, H256}; use serde::{de::Error, Deserialize, Deserializer, Serialize, Serializer}; /// Subscription result. diff --git a/crates/rpc/rpc/src/eth/pubsub.rs b/crates/rpc/rpc/src/eth/pubsub.rs index fa08bf95ef9..5a2ee798efb 100644 --- a/crates/rpc/rpc/src/eth/pubsub.rs +++ b/crates/rpc/rpc/src/eth/pubsub.rs @@ -1,8 +1,9 @@ //! `eth_` PubSub RPC handler implementation +use futures::StreamExt; use jsonrpsee::{types::SubscriptionResult, SubscriptionSink}; use reth_interfaces::{events::ChainEventSubscriptions, sync::SyncStateProvider}; -use reth_primitives::{rpc::FilteredParams, TxHash}; +use reth_primitives::{filter::FilteredParams, BlockId, TxHash, H256, U256}; use reth_provider::{BlockProvider, EvmEnvProvider}; use reth_rpc_api::EthPubSubApiServer; use reth_rpc_types::{ @@ -10,13 +11,13 @@ use reth_rpc_types::{ Params, PubSubSyncStatus, SubscriptionKind, SubscriptionResult as EthSubscriptionResult, SyncStatusMetadata, }, - Header, + Header, Log, }; use reth_tasks::{TaskSpawner, TokioTaskExecutor}; use reth_transaction_pool::TransactionPool; use tokio_stream::{ wrappers::{ReceiverStream, UnboundedReceiverStream}, - Stream, StreamExt, + Stream, }; /// `Eth` pubsub RPC implementation. @@ -91,12 +92,6 @@ async fn handle_accepted( Events: ChainEventSubscriptions + Clone + 'static, Network: SyncStateProvider + Clone + 'static, { - // if no params are provided, used default filter params - let _params = match params { - Some(Params::Logs(filter)) => FilteredParams::new(Some(*filter)), - _ => FilteredParams::default(), - }; - match kind { SubscriptionKind::NewHeads => { let stream = pubsub @@ -105,7 +100,14 @@ async fn handle_accepted( accepted_sink.pipe_from_stream(stream).await; } SubscriptionKind::Logs => { - // TODO subscribe new blocks -> fetch logs via bloom + // if no params are provided, used default filter params + let filter = match params { + Some(Params::Logs(filter)) => FilteredParams::new(Some(*filter)), + _ => FilteredParams::default(), + }; + let stream = + pubsub.into_log_stream(filter).map(|log| EthSubscriptionResult::Log(Box::new(log))); + accepted_sink.pipe_from_stream(stream).await; } SubscriptionKind::NewPendingTransactions => { let stream = pubsub @@ -206,4 +208,74 @@ where Header::from_primitive_with_hash(new_block.header.as_ref().clone(), new_block.hash) }) } + + /// Returns a stream that yields all logs that match the given filter. + fn into_log_stream(self, filter: FilteredParams) -> impl Stream { + UnboundedReceiverStream::new(self.chain_events.subscribe_new_blocks()) + .filter_map(move |new_block| { + let block_id: BlockId = new_block.hash.into(); + let txs = self.client.transactions_by_block(block_id).ok().flatten(); + let receipts = self.client.receipts_by_block(block_id).ok().flatten(); + match (txs, receipts) { + (Some(txs), Some(receipts)) => { + futures::future::ready(Some((new_block, txs, receipts))) + } + _ => futures::future::ready(None), + } + }) + .flat_map(move |(new_block, transactions, receipts)| { + let block_hash = new_block.hash; + let block_number = new_block.header.number; + let mut all_logs: Vec = Vec::new(); + + // tracks the index of a log in the entire block + let mut log_index: u32 = 0; + for (transaction_idx, (tx, receipt)) in + transactions.into_iter().zip(receipts).enumerate() + { + let logs = receipt.logs; + + // tracks the index of the log in the transaction + let transaction_hash = tx.hash; + + for (transaction_log_idx, log) in logs.into_iter().enumerate() { + if matches_filter(block_hash, block_number, &log, &filter) { + let log = Log { + address: log.address, + topics: log.topics, + data: log.data, + block_hash: Some(block_hash), + block_number: Some(U256::from(block_number)), + transaction_hash: Some(transaction_hash), + transaction_index: Some(U256::from(transaction_idx)), + log_index: Some(U256::from(log_index)), + transaction_log_index: Some(U256::from(transaction_log_idx)), + removed: false, + }; + all_logs.push(log); + } + log_index += 1; + } + } + futures::stream::iter(all_logs) + }) + } +} + +/// Returns true if the log matches the filter and should be included +fn matches_filter( + block_hash: H256, + block_number: u64, + log: &reth_primitives::Log, + params: &FilteredParams, +) -> bool { + if params.filter.is_some() && + (!params.filter_block_range(block_number) || + !params.filter_block_hash(block_hash) || + !params.filter_address(log) || + !params.filter_topics(log)) + { + return false + } + true } From 5b90cbc411d5c21add0f1be1613ed6c7eda023c0 Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Tue, 14 Mar 2023 12:47:16 +0800 Subject: [PATCH 116/191] fix(stages): add commit threshold to merkle stage v2 (#1656) Co-authored-by: Georgios Konstantopoulos --- Cargo.lock | 1 + bin/reth/src/dump_stage/merkle.rs | 35 +- crates/primitives/src/lib.rs | 1 + crates/primitives/src/proofs.rs | 15 + crates/stages/benches/setup/mod.rs | 6 +- crates/stages/src/stages/merkle.rs | 79 ++- .../storage/db/src/tables/codecs/compact.rs | 3 +- crates/storage/db/src/tables/mod.rs | 8 +- crates/storage/provider/Cargo.toml | 5 +- .../provider/src/providers/state/latest.rs | 4 +- crates/storage/provider/src/transaction.rs | 4 +- crates/storage/provider/src/trie/mod.rs | 485 +++++++++++++----- 12 files changed, 471 insertions(+), 175 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6417ea64c8b..f3130d2b86f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4851,6 +4851,7 @@ dependencies = [ "itertools 0.10.5", "parking_lot 0.12.1", "proptest", + "reth-codecs", "reth-db", "reth-interfaces", "reth-primitives", diff --git a/bin/reth/src/dump_stage/merkle.rs b/bin/reth/src/dump_stage/merkle.rs index a9e8a8f6204..6e6ce370ff3 100644 --- a/bin/reth/src/dump_stage/merkle.rs +++ b/bin/reth/src/dump_stage/merkle.rs @@ -44,10 +44,7 @@ pub(crate) async fn dump_merkle_stage( unwind_and_copy::(db_tool, (from, to), tip_block_number, &output_db).await?; if should_run { - println!( - "\n# Merkle stage does not support dry run, so it will actually be committing changes." - ); - run(output_db, to, from).await?; + dry_run(output_db, to, from).await?; } Ok(()) @@ -113,7 +110,7 @@ async fn unwind_and_copy( } /// Try to re-execute the stage straightaway -async fn run( +async fn dry_run( output_db: reth_db::mdbx::Env, to: u64, from: u64, @@ -121,18 +118,24 @@ async fn run( info!(target: "reth::cli", "Executing stage."); let mut tx = Transaction::new(&output_db)?; - - MerkleStage::Execution { - clean_threshold: u64::MAX, // Forces updating the root instead of calculating from scratch + let mut exec_output = false; + while !exec_output { + exec_output = MerkleStage::Execution { + clean_threshold: u64::MAX, /* Forces updating the root instead of calculating from + * scratch */ + } + .execute( + &mut tx, + reth_stages::ExecInput { + previous_stage: Some((StageId("Another"), to)), + stage_progress: Some(from), + }, + ) + .await? + .done; } - .execute( - &mut tx, - reth_stages::ExecInput { - previous_stage: Some((StageId("Another"), to)), - stage_progress: Some(from), - }, - ) - .await?; + + tx.drop()?; info!(target: "reth::cli", "Success."); diff --git a/crates/primitives/src/lib.rs b/crates/primitives/src/lib.rs index c4b8a8a1c4d..13d68dbfc03 100644 --- a/crates/primitives/src/lib.rs +++ b/crates/primitives/src/lib.rs @@ -34,6 +34,7 @@ mod withdrawal; /// Helper function for calculating Merkle proofs and hashes pub mod proofs; +pub use proofs::ProofCheckpoint; pub use account::{Account, Bytecode}; pub use bits::H512; diff --git a/crates/primitives/src/proofs.rs b/crates/primitives/src/proofs.rs index bfba3e50507..c94a68aad33 100644 --- a/crates/primitives/src/proofs.rs +++ b/crates/primitives/src/proofs.rs @@ -8,6 +8,7 @@ use bytes::BytesMut; use hash_db::Hasher; use hex_literal::hex; use plain_hasher::PlainHasher; +use reth_codecs::{main_codec, Compact}; use reth_rlp::Encodable; use triehash::{ordered_trie_root, sec_trie_root}; @@ -34,6 +35,20 @@ impl Hasher for KeccakHasher { } } +/// Saves the progress of MerkleStage +#[main_codec] +#[derive(Default, Debug, Copy, Clone, PartialEq)] +pub struct ProofCheckpoint { + /// The next hashed account to insert into the trie. + pub hashed_address: Option, + /// The next storage entry to insert into the trie. + pub storage_key: Option, + /// Current intermediate root for `AccountsTrie`. + pub account_root: Option, + /// Current intermediate storage root from an account. + pub storage_root: Option, +} + /// Calculate a transaction root. /// /// Iterates over the given transactions and the merkle merkle trie root of diff --git a/crates/stages/benches/setup/mod.rs b/crates/stages/benches/setup/mod.rs index aaed854bd82..08b30de0905 100644 --- a/crates/stages/benches/setup/mod.rs +++ b/crates/stages/benches/setup/mod.rs @@ -123,7 +123,8 @@ pub(crate) fn txs_testdata(num_blocks: u64) -> PathBuf { tx.insert_accounts_and_storages(start_state.clone()).unwrap(); // make first block after genesis have valid state root - let root = DBTrieLoader::default().calculate_root(&tx.inner()).unwrap(); + let root = + DBTrieLoader::default().calculate_root(&tx.inner()).and_then(|e| e.root()).unwrap(); let second_block = blocks.get_mut(1).unwrap(); let cloned_second = second_block.clone(); let mut updated_header = cloned_second.header.unseal(); @@ -144,7 +145,8 @@ pub(crate) fn txs_testdata(num_blocks: u64) -> PathBuf { // make last block have valid state root let root = { let mut tx_mut = tx.inner(); - let root = DBTrieLoader::default().calculate_root(&tx_mut).unwrap(); + let root = + DBTrieLoader::default().calculate_root(&tx_mut).and_then(|e| e.root()).unwrap(); tx_mut.commit().unwrap(); root }; diff --git a/crates/stages/src/stages/merkle.rs b/crates/stages/src/stages/merkle.rs index 3078742fd11..d2b8df05b2d 100644 --- a/crates/stages/src/stages/merkle.rs +++ b/crates/stages/src/stages/merkle.rs @@ -1,7 +1,10 @@ use crate::{ExecInput, ExecOutput, Stage, StageError, StageId, UnwindInput, UnwindOutput}; use reth_db::{database::Database, tables, transaction::DbTx}; use reth_interfaces::consensus; -use reth_provider::{trie::DBTrieLoader, Transaction}; +use reth_provider::{ + trie::{DBTrieLoader, TrieProgress}, + Transaction, +}; use std::{fmt::Debug, ops::DerefMut}; use tracing::*; @@ -105,23 +108,32 @@ impl Stage for MerkleStage { let trie_root = if from_transition == to_transition { block_root - } else if to_transition - from_transition > threshold || stage_progress == 0 { - debug!(target: "sync::stages::merkle::exec", current = ?stage_progress, target = ?previous_stage_progress, "Rebuilding trie"); - // if there are more blocks than threshold it is faster to rebuild the trie - DBTrieLoader::new(tx.deref_mut()) - .calculate_root() - .map_err(|e| StageError::Fatal(Box::new(e)))? } else { - debug!(target: "sync::stages::merkle::exec", current = ?stage_progress, target = ?previous_stage_progress, "Updating trie"); - // Iterate over changeset (similar to Hashing stages) and take new values - let current_root = tx.get_header(stage_progress)?.state_root; - DBTrieLoader::new(tx.deref_mut()) - .update_root(current_root, from_transition..to_transition) - .map_err(|e| StageError::Fatal(Box::new(e)))? + let res = if to_transition - from_transition > threshold || stage_progress == 0 { + debug!(target: "sync::stages::merkle::exec", current = ?stage_progress, target = ?previous_stage_progress, "Rebuilding trie"); + // if there are more blocks than threshold it is faster to rebuild the trie + let mut loader = DBTrieLoader::new(tx.deref_mut()); + loader.calculate_root().map_err(|e| StageError::Fatal(Box::new(e)))? + } else { + debug!(target: "sync::stages::merkle::exec", current = ?stage_progress, target = ?previous_stage_progress, "Updating trie"); + // Iterate over changeset (similar to Hashing stages) and take new values + let current_root = tx.get_header(stage_progress)?.state_root; + let mut loader = DBTrieLoader::new(tx.deref_mut()); + loader + .update_root(current_root, from_transition..to_transition) + .map_err(|e| StageError::Fatal(Box::new(e)))? + }; + + match res { + TrieProgress::Complete(root) => root, + TrieProgress::InProgress(_) => { + return Ok(ExecOutput { stage_progress, done: false }) + } + } }; if block_root != trie_root { - warn!(target: "sync::stages::merkle::exec", ?previous_stage_progress, got = ?block_root, expected = ?trie_root, "Block's root state failed verification"); + warn!(target: "sync::stages::merkle::exec", ?previous_stage_progress, got = ?trie_root, expected = ?block_root, "Block's root state failed verification"); return Err(StageError::Validation { block: previous_stage_progress, error: consensus::ConsensusError::BodyStateRootDiff { @@ -156,13 +168,28 @@ impl Stage for MerkleStage { } let current_root = tx.get_header(input.stage_progress)?.state_root; - let from_transition = tx.get_block_transition(input.unwind_to)?; let to_transition = tx.get_block_transition(input.stage_progress)?; - let block_root = DBTrieLoader::new(tx.deref_mut()) - .update_root(current_root, from_transition..to_transition) - .map_err(|e| StageError::Fatal(Box::new(e)))?; + let mut loader = DBTrieLoader::new(tx.deref_mut()); + let block_root = loop { + match loader + .update_root(current_root, from_transition..to_transition) + .map_err(|e| StageError::Fatal(Box::new(e)))? + { + TrieProgress::Complete(root) => break root, + TrieProgress::InProgress(_) => { + // Save the loader's progress & drop it to allow committing to the database, + // otherwise we're hitting the borrow checker + let progress = loader.current; + let _ = loader; + tx.commit()?; + // Reinstantiate the loader from where it was left off. + loader = DBTrieLoader::new(tx.deref_mut()); + loader.current = progress; + } + } + }; if block_root != target_root { let unwind_to = input.unwind_to; @@ -447,7 +474,10 @@ mod tests { impl MerkleTestRunner { fn state_root(&self) -> Result { - Ok(create_trie_loader(&self.tx.inner()).calculate_root().unwrap()) + Ok(create_trie_loader(&self.tx.inner()) + .calculate_root() + .and_then(|e| e.root()) + .unwrap()) } pub(crate) fn generate_initial_trie( @@ -459,8 +489,10 @@ mod tests { )?; let mut tx = self.tx.inner(); - let root = - create_trie_loader(&tx).calculate_root().expect("couldn't create initial trie"); + let root = create_trie_loader(&tx) + .calculate_root() + .and_then(|e| e.root()) + .expect("couldn't create initial trie"); tx.commit()?; @@ -471,7 +503,10 @@ mod tests { if previous_stage_progress != 0 { let block_root = self.tx.inner().get_header(previous_stage_progress).unwrap().state_root; - let root = create_trie_loader(&self.tx.inner()).calculate_root().unwrap(); + let root = create_trie_loader(&self.tx().inner()) + .calculate_root() + .and_then(|e| e.root()) + .unwrap(); assert_eq!(block_root, root); } Ok(()) diff --git a/crates/storage/db/src/tables/codecs/compact.rs b/crates/storage/db/src/tables/codecs/compact.rs index 4c0a4136416..c1689a4d8b8 100644 --- a/crates/storage/db/src/tables/codecs/compact.rs +++ b/crates/storage/db/src/tables/codecs/compact.rs @@ -44,7 +44,8 @@ impl_compression_for_compact!( StoredBlockBody, StoredBlockOmmers, StoredBlockWithdrawals, - Bytecode + Bytecode, + ProofCheckpoint ); impl_compression_for_compact!(AccountBeforeTx, TransactionSigned); impl_compression_for_compact!(CompactU256); diff --git a/crates/storage/db/src/tables/mod.rs b/crates/storage/db/src/tables/mod.rs index 16fb6ac49b4..bb45b8bf4e0 100644 --- a/crates/storage/db/src/tables/mod.rs +++ b/crates/storage/db/src/tables/mod.rs @@ -32,7 +32,7 @@ pub enum TableType { } /// Default tables that should be present inside database. -pub const TABLES: [(TableType, &str); 26] = [ +pub const TABLES: [(TableType, &str); 27] = [ (TableType::Table, CanonicalHeaders::const_name()), (TableType::Table, HeaderTD::const_name()), (TableType::Table, HeaderNumbers::const_name()), @@ -59,6 +59,7 @@ pub const TABLES: [(TableType, &str); 26] = [ (TableType::DupSort, StoragesTrie::const_name()), (TableType::Table, TxSenders::const_name()), (TableType::Table, SyncStage::const_name()), + (TableType::Table, SyncStageProgress::const_name()), ]; #[macro_export] @@ -293,6 +294,11 @@ table!( ( SyncStage ) StageId | BlockNumber ); +table!( + /// Stores arbitrary data to keep track of a stage first-sync progress. + ( SyncStageProgress ) StageId | Vec +); + /// /// Alias Types diff --git a/crates/storage/provider/Cargo.toml b/crates/storage/provider/Cargo.toml index c0e26f28791..7cfe4eaffbf 100644 --- a/crates/storage/provider/Cargo.toml +++ b/crates/storage/provider/Cargo.toml @@ -13,8 +13,9 @@ reth-primitives = { path = "../../primitives" } reth-interfaces = { path = "../../interfaces" } reth-revm-primitives = { path = "../../revm/revm-primitives" } reth-db = { path = "../db" } -reth-tracing = { path = "../../tracing" } -reth-rlp = { path = "../../rlp" } +reth-codecs = { path = "../codecs" } +reth-tracing = {path = "../../tracing"} +reth-rlp = {path = "../../rlp"} revm-primitives = "1.0.0" diff --git a/crates/storage/provider/src/providers/state/latest.rs b/crates/storage/provider/src/providers/state/latest.rs index e600eacb294..7d277427771 100644 --- a/crates/storage/provider/src/providers/state/latest.rs +++ b/crates/storage/provider/src/providers/state/latest.rs @@ -76,7 +76,7 @@ impl<'a, 'b, TX: DbTx<'a>> StateProvider for LatestStateProviderRef<'a, 'b, TX> .state_root; let (account_proof, storage_root) = loader - .generate_acount_proof(self.db, root, hashed_address) + .generate_acount_proof(root, hashed_address) .map_err(|_| ProviderError::StateTree)?; let account_proof = account_proof.into_iter().map(Bytes::from).collect(); @@ -86,7 +86,7 @@ impl<'a, 'b, TX: DbTx<'a>> StateProvider for LatestStateProviderRef<'a, 'b, TX> } else { let hashed_keys: Vec = keys.iter().map(keccak256).collect(); loader - .generate_storage_proofs(self.db, storage_root, hashed_address, &hashed_keys) + .generate_storage_proofs(storage_root, hashed_address, &hashed_keys) .map_err(|_| ProviderError::StateTree)? .into_iter() .map(|v| v.into_iter().map(Bytes::from).collect()) diff --git a/crates/storage/provider/src/transaction.rs b/crates/storage/provider/src/transaction.rs index 0c1d39b362b..90a3848c96e 100644 --- a/crates/storage/provider/src/transaction.rs +++ b/crates/storage/provider/src/transaction.rs @@ -308,8 +308,8 @@ where // merkle tree { let current_root = self.get_header(parent_block_number)?.state_root; - let loader = DBTrieLoader::new(self.deref_mut()); - let root = loader.update_root(current_root, from..to)?; + let mut loader = DBTrieLoader::new(self.deref_mut()); + let root = loader.update_root(current_root, from..to).and_then(|e| e.root())?; if root != block.state_root { return Err(TransactionError::StateTrieRootMismatch { got: root, diff --git a/crates/storage/provider/src/trie/mod.rs b/crates/storage/provider/src/trie/mod.rs index ffb9661d189..953fccd0d14 100644 --- a/crates/storage/provider/src/trie/mod.rs +++ b/crates/storage/provider/src/trie/mod.rs @@ -1,5 +1,6 @@ use cita_trie::{PatriciaTrie, Trie}; use hasher::HasherKeccak; +use reth_codecs::Compact; use reth_db::{ cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO}, models::{AccountBeforeTx, TransitionIdAddress}, @@ -7,8 +8,8 @@ use reth_db::{ transaction::{DbTx, DbTxMut}, }; use reth_primitives::{ - keccak256, proofs::EMPTY_ROOT, Account, Address, StorageEntry, StorageTrieEntry, TransitionId, - H256, KECCAK_EMPTY, U256, + keccak256, proofs::EMPTY_ROOT, Account, Address, ProofCheckpoint, StorageEntry, + StorageTrieEntry, TransitionId, H256, KECCAK_EMPTY, U256, }; use reth_rlp::{ encode_fixed_size, Decodable, DecodeError, Encodable, RlpDecodable, RlpEncodable, @@ -38,6 +39,8 @@ pub enum TrieError { /// Error when encoding/decoding a value. #[error("{0:?}")] DecodeError(#[from] DecodeError), + #[error("Trie requires committing a checkpoint.")] + UnexpectedCheckpoint, } /// Database wrapper implementing HashDB trait, with a read-write transaction. @@ -207,11 +210,18 @@ where } /// Database wrapper implementing HashDB trait, with a read-only transaction. -struct HashDatabase<'tx, 'itx, TX: DbTx<'itx>> { +pub struct HashDatabase<'tx, 'itx, TX: DbTx<'itx>> { tx: &'tx TX, _p: PhantomData<&'itx ()>, // to suppress "unused" lifetime 'itx } +impl<'tx, 'itx, TX: DbTx<'itx>> HashDatabase<'tx, 'itx, TX> { + /// Creates a new Hash database with the given transaction + pub fn new(tx: &'tx TX) -> Self { + Self { tx, _p: Default::default() } + } +} + impl<'tx, 'itx, TX> cita_trie::DB for HashDatabase<'tx, 'itx, TX> where TX: DbTx<'itx>, @@ -250,12 +260,19 @@ impl<'tx, 'itx, TX: DbTx<'itx>> HashDatabase<'tx, 'itx, TX> { } /// Database wrapper implementing HashDB trait, with a read-only transaction. -struct DupHashDatabase<'tx, 'itx, TX: DbTx<'itx>> { +pub struct DupHashDatabase<'tx, 'itx, TX: DbTx<'itx>> { tx: &'tx TX, key: H256, _p: PhantomData<&'itx ()>, // to suppress "unused" lifetime 'itx } +impl<'tx, 'itx, TX: DbTx<'itx>> DupHashDatabase<'tx, 'itx, TX> { + /// Creates a new DupHash database with the given transaction and key. + pub fn new(tx: &'tx TX, key: H256) -> Self { + Self { tx, key, _p: Default::default() } + } +} + impl<'tx, 'itx, TX> cita_trie::DB for DupHashDatabase<'tx, 'itx, TX> where TX: DbTx<'itx>, @@ -338,14 +355,41 @@ pub type MerkleProof = Vec>; /// Struct for calculating the root of a merkle patricia tree, /// while populating the database with intermediate hashes. +#[derive(Debug)] pub struct DBTrieLoader<'tx, TX> { - tx: &'tx TX, + /// The maximum number of keys to insert before committing. Both from `AccountsTrie` and + /// `StoragesTrie`. + pub commit_threshold: u64, + /// The current number of inserted keys from both `AccountsTrie` and `StoragesTrie`. + pub current: u64, + /// The transaction to use for inserting the trie nodes. + pub tx: &'tx TX, +} + +/// Status of the trie calculation. +#[derive(Debug, PartialEq, Copy, Clone)] +pub enum TrieProgress { + /// Trie has finished with the passed root. + Complete(H256), + /// Trie has hit its commit threshold. + InProgress(ProofCheckpoint), +} + +impl TrieProgress { + /// Consumes the root from its `Complete` variant. If that's not possible, throw + /// `TrieError::UnexpectedCheckpoint`. + pub fn root(self) -> Result { + match self { + Self::Complete(root) => Ok(root), + _ => Err(TrieError::UnexpectedCheckpoint), + } + } } impl<'tx, TX> DBTrieLoader<'tx, TX> { /// Create new instance of trie loader. pub fn new(tx: &'tx TX) -> Self { - Self { tx } + Self { tx, commit_threshold: 500_000, current: 0 } } } @@ -355,151 +399,243 @@ where TX: DbTxMut<'db> + DbTx<'db> + Send + Sync, { /// Calculates the root of the state trie, saving intermediate hashes in the database. - pub fn calculate_root(&self) -> Result { - self.tx.clear::()?; - self.tx.clear::()?; - - let mut accounts_cursor = self.tx.cursor_read::()?; - let mut walker = accounts_cursor.walk(None)?; + pub fn calculate_root(&mut self) -> Result { + let mut checkpoint = self.get_checkpoint()?; - let db = Arc::new(HashDatabaseMut::new(self.tx)?); + if checkpoint.hashed_address.is_none() { + self.tx.clear::()?; + self.tx.clear::()?; + } + let previous_root = checkpoint.account_root.unwrap_or(EMPTY_ROOT); let hasher = Arc::new(HasherKeccak::new()); + let mut trie = if let Some(root) = checkpoint.account_root { + PatriciaTrie::from( + Arc::new(HashDatabaseMut::from_root(self.tx, root)?), + hasher, + root.as_bytes(), + )? + } else { + PatriciaTrie::new(Arc::new(HashDatabaseMut::new(self.tx)?), hasher) + }; - let mut trie = PatriciaTrie::new(Arc::clone(&db), Arc::clone(&hasher)); + let mut accounts_cursor = self.tx.cursor_read::()?; + let mut walker = accounts_cursor.walk(checkpoint.hashed_address.take())?; while let Some((hashed_address, account)) = walker.next().transpose()? { - let value = EthAccount::from(account) - .with_storage_root(self.calculate_storage_root(hashed_address)?); - - let mut out = Vec::new(); - Encodable::encode(&value, &mut out); - trie.insert(hashed_address.as_bytes().to_vec(), out)?; + match self.calculate_storage_root( + hashed_address, + checkpoint.storage_key.take(), + checkpoint.storage_root.take(), + )? { + TrieProgress::Complete(root) => { + let value = EthAccount::from(account).with_storage_root(root); + + let mut out = Vec::new(); + Encodable::encode(&value, &mut out); + trie.insert(hashed_address.as_bytes().to_vec(), out)?; + + if self.has_hit_threshold() { + return self.save_account_checkpoint( + ProofCheckpoint::default(), + self.replace_account_root(&mut trie, previous_root)?, + hashed_address, + ) + } + } + TrieProgress::InProgress(checkpoint) => { + return self.save_account_checkpoint( + checkpoint, + self.replace_account_root(&mut trie, previous_root)?, + hashed_address, + ) + } + } } - let root = H256::from_slice(trie.root()?.as_slice()); - Ok(root) + // Reset inner stage progress + self.save_checkpoint(ProofCheckpoint::default())?; + + Ok(TrieProgress::Complete(self.replace_account_root(&mut trie, previous_root)?)) } - /// Calculate the accounts storage root. - pub fn calculate_storage_root(&self, address: H256) -> Result { - let db = Arc::new(DupHashDatabaseMut::new(self.tx, address)?); + fn calculate_storage_root( + &mut self, + address: H256, + next_storage: Option, + previous_root: Option, + ) -> Result { + let mut storage_cursor = self.tx.cursor_dup_read::()?; let hasher = Arc::new(HasherKeccak::new()); + let (mut current_entry, mut trie) = if let Some(entry) = next_storage { + ( + storage_cursor.seek_by_key_subkey(address, entry)?.filter(|e| e.key == entry), + PatriciaTrie::from( + Arc::new(DupHashDatabaseMut::from_root( + self.tx, + address, + previous_root.expect("is some"), + )?), + hasher, + previous_root.expect("is some").as_bytes(), + )?, + ) + } else { + ( + storage_cursor.seek_by_key_subkey(address, H256::zero())?, + PatriciaTrie::new(Arc::new(DupHashDatabaseMut::new(self.tx, address)?), hasher), + ) + }; - let mut trie = PatriciaTrie::new(Arc::clone(&db), Arc::clone(&hasher)); - - let mut storage_cursor = self.tx.cursor_dup_read::()?; - - // Should be able to use walk_dup, but any call to next() causes an assert fail in mdbx.c - // let mut walker = storage_cursor.walk_dup(address, H256::zero())?; - let mut current = storage_cursor.seek_by_key_subkey(address, H256::zero())?; + let previous_root = previous_root.unwrap_or(EMPTY_ROOT); - while let Some(StorageEntry { key: storage_key, value }) = current { + while let Some(StorageEntry { key: storage_key, value }) = current_entry { let out = encode_fixed_size(&value).to_vec(); trie.insert(storage_key.to_vec(), out)?; - current = storage_cursor.next_dup()?.map(|(_, v)| v); - } - - let root = H256::from_slice(trie.root()?.as_slice()); - - // if root is empty remove it from db - if root == EMPTY_ROOT { - self.tx.delete::(address, None)?; + // Should be able to use walk_dup, but any call to next() causes an assert fail in + // mdbx.c + current_entry = storage_cursor.next_dup()?.map(|(_, v)| v); + let threshold = self.has_hit_threshold(); + if let Some(current_entry) = current_entry { + if threshold { + return Ok(TrieProgress::InProgress(ProofCheckpoint { + storage_root: Some(self.replace_storage_root( + trie, + address, + previous_root, + )?), + storage_key: Some(current_entry.key), + ..Default::default() + })) + } + } } - Ok(root) + Ok(TrieProgress::Complete(self.replace_storage_root(trie, address, previous_root)?)) } /// Calculates the root of the state trie by updating an existing trie. pub fn update_root( - &self, - root: H256, + &mut self, + mut previous_root: H256, tid_range: Range, - ) -> Result { - let mut accounts_cursor = self.tx.cursor_read::()?; + ) -> Result { + let mut checkpoint = self.get_checkpoint()?; - let changed_accounts = self.gather_changes(tid_range)?; + if let Some(account_root) = checkpoint.account_root.take() { + previous_root = account_root; + } - let db = Arc::new(HashDatabaseMut::from_root(self.tx, root)?); + let next_acc = checkpoint.hashed_address.take(); + let changed_accounts = self + .gather_changes(tid_range)? + .into_iter() + .skip_while(|(addr, _)| next_acc.is_some() && next_acc.expect("is some") != *addr); - let hasher = Arc::new(HasherKeccak::new()); + let mut trie = PatriciaTrie::from( + Arc::new(HashDatabaseMut::from_root(self.tx, previous_root)?), + Arc::new(HasherKeccak::new()), + previous_root.as_bytes(), + )?; - let mut trie = PatriciaTrie::from(Arc::clone(&db), Arc::clone(&hasher), root.as_bytes())?; + let mut accounts_cursor = self.tx.cursor_read::()?; - for (address, changed_storages) in changed_accounts { - let storage_root = if let Some(account) = trie.get(address.as_slice())? { - trie.remove(address.as_bytes())?; + for (hashed_address, changed_storages) in changed_accounts { + let res = if let Some(account) = trie.get(hashed_address.as_slice())? { + trie.remove(hashed_address.as_bytes())?; let storage_root = EthAccount::decode(&mut account.as_slice())?.storage_root; - self.update_storage_root(storage_root, address, changed_storages)? + self.update_storage_root( + checkpoint.storage_root.take().unwrap_or(storage_root), + hashed_address, + changed_storages, + checkpoint.storage_key.take(), + )? } else { - self.calculate_storage_root(address)? + self.calculate_storage_root( + hashed_address, + checkpoint.storage_key.take(), + checkpoint.storage_root.take(), + )? }; - if let Some((_, account)) = accounts_cursor.seek_exact(address)? { + let storage_root = match res { + TrieProgress::Complete(root) => root, + TrieProgress::InProgress(checkpoint) => { + return self.save_account_checkpoint( + checkpoint, + self.replace_account_root(&mut trie, previous_root)?, + hashed_address, + ) + } + }; + + if let Some((_, account)) = accounts_cursor.seek_exact(hashed_address)? { let value = EthAccount::from(account).with_storage_root(storage_root); let mut out = Vec::new(); Encodable::encode(&value, &mut out); - trie.insert(address.as_bytes().to_vec(), out)?; - } - } - let new_root = H256::from_slice(trie.root()?.as_slice()); - if new_root != root { - let mut cursor = self.tx.cursor_write::()?; - if cursor.seek_exact(root)?.is_some() { - cursor.delete_current()?; + trie.insert(hashed_address.as_bytes().to_vec(), out)?; + + if self.has_hit_threshold() { + return self.save_account_checkpoint( + ProofCheckpoint::default(), + self.replace_account_root(&mut trie, previous_root)?, + hashed_address, + ) + } } } - Ok(new_root) + // Reset inner stage progress + self.save_checkpoint(ProofCheckpoint::default())?; + + Ok(TrieProgress::Complete(self.replace_account_root(&mut trie, previous_root)?)) } /// Update the account's storage root - pub fn update_storage_root( - &self, - root: H256, + fn update_storage_root( + &mut self, + previous_root: H256, address: H256, changed_storages: BTreeSet, - ) -> Result { - let db = Arc::new(DupHashDatabaseMut::from_root(self.tx, address, root)?); - - let hasher = Arc::new(HasherKeccak::new()); + next_storage: Option, + ) -> Result { + let mut hashed_storage_cursor = self.tx.cursor_dup_read::()?; + let mut trie = PatriciaTrie::new( + Arc::new(DupHashDatabaseMut::from_root(self.tx, address, previous_root)?), + Arc::new(HasherKeccak::new()), + ); - let mut trie = PatriciaTrie::from(Arc::clone(&db), Arc::clone(&hasher), root.as_bytes())?; - let mut storage_cursor = self.tx.cursor_dup_read::()?; + let changed_storages = changed_storages + .into_iter() + .skip_while(|k| next_storage.is_some() && *k == next_storage.expect("is some")); for key in changed_storages { if let Some(StorageEntry { value, .. }) = - storage_cursor.seek_by_key_subkey(address, key)?.filter(|e| e.key == key) + hashed_storage_cursor.seek_by_key_subkey(address, key)?.filter(|e| e.key == key) { let out = encode_fixed_size(&value).to_vec(); trie.insert(key.as_bytes().to_vec(), out)?; + if self.has_hit_threshold() { + return Ok(TrieProgress::InProgress(ProofCheckpoint { + storage_root: Some(self.replace_storage_root( + trie, + address, + previous_root, + )?), + storage_key: Some(key), + ..Default::default() + })) + } } else { trie.remove(key.as_bytes())?; } } - let new_root = H256::from_slice(trie.root()?.as_slice()); - if new_root != root { - let mut cursor = self.tx.cursor_dup_write::()?; - if cursor - .seek_by_key_subkey(address, root)? - .filter(|entry| entry.hash == root) - .is_some() - { - cursor.delete_current()?; - } - } - - // if root is empty remove it from db - if new_root == EMPTY_ROOT { - self.tx.delete::(address, None)?; - } - - Ok(new_root) + Ok(TrieProgress::Complete(self.replace_storage_root(trie, address, previous_root)?)) } fn gather_changes( @@ -537,6 +673,102 @@ where Ok(hashed_changes) } + + fn save_account_checkpoint( + &mut self, + mut checkpoint: ProofCheckpoint, + root: H256, + hashed_address: H256, + ) -> Result { + checkpoint.account_root = Some(root); + checkpoint.hashed_address = Some(hashed_address); + + debug!(target: "sync::stages::merkle::exec", account = ?hashed_address, storage = ?checkpoint.storage_key, "Saving inner trie checkpoint"); + + self.save_checkpoint(checkpoint)?; + + Ok(TrieProgress::InProgress(checkpoint)) + } + + fn has_hit_threshold(&mut self) -> bool { + self.current += 1; + self.current >= self.commit_threshold + } + + /// Saves the trie progress + pub fn save_checkpoint(&mut self, checkpoint: ProofCheckpoint) -> Result<(), TrieError> { + let mut buf = vec![]; + checkpoint.to_compact(&mut buf); + + // It allows unwind (which commits), to reuse this instance. + self.current = 0; + + Ok(self.tx.put::("TrieLoader".into(), buf)?) + } + + /// Gets the trie progress + pub fn get_checkpoint(&self) -> Result { + let buf = + self.tx.get::("TrieLoader".into())?.unwrap_or_default(); + + if buf.is_empty() { + return Ok(ProofCheckpoint::default()) + } + + let (checkpoint, _) = ProofCheckpoint::from_compact(&buf, buf.len()); + + if checkpoint.account_root.is_some() { + debug!(target: "sync::stages::merkle::exec", checkpoint = ?checkpoint, "Continuing inner trie checkpoint"); + } + + Ok(checkpoint) + } + + /// Finds the most recent account trie root and removes the previous one if applicable. + fn replace_account_root( + &self, + trie: &mut PatriciaTrie, HasherKeccak>, + previous_root: H256, + ) -> Result { + let new_root = H256::from_slice(trie.root()?.as_slice()); + + if new_root != previous_root { + let mut cursor = self.tx.cursor_write::()?; + if cursor.seek_exact(previous_root)?.is_some() { + cursor.delete_current()?; + } + } + + Ok(new_root) + } + + /// Finds the most recent storage trie root and removes the previous one if applicable. + fn replace_storage_root( + &self, + mut trie: PatriciaTrie, HasherKeccak>, + address: H256, + previous_root: H256, + ) -> Result { + let new_root = H256::from_slice(trie.root()?.as_slice()); + + if new_root != previous_root { + let mut trie_cursor = self.tx.cursor_dup_write::()?; + + if trie_cursor + .seek_by_key_subkey(address, previous_root)? + .filter(|entry| entry.hash == previous_root) + .is_some() + { + trie_cursor.delete_current()?; + } + } + + if new_root == EMPTY_ROOT { + self.tx.delete::(address, None)?; + } + + Ok(new_root) + } } // Read-only impls @@ -545,13 +777,12 @@ where TX: DbTx<'db> + Send + Sync, { /// Returns a Merkle proof of the given account, plus its storage root hash. - pub fn generate_acount_proof<'itx>( + pub fn generate_acount_proof( &self, - tx: &'tx impl DbTx<'itx>, root: H256, address: H256, ) -> Result<(MerkleProof, H256), TrieError> { - let db = Arc::new(HashDatabase::from_root(tx, root)?); + let db = Arc::new(HashDatabase::from_root(self.tx, root)?); let hasher = Arc::new(HasherKeccak::new()); let trie = PatriciaTrie::from(Arc::clone(&db), Arc::clone(&hasher), root.as_bytes())?; @@ -565,14 +796,13 @@ where } /// Returns a Merkle proof of the given storage keys, starting at the given root hash. - pub fn generate_storage_proofs<'itx>( + pub fn generate_storage_proofs( &self, - tx: &'tx impl DbTx<'itx>, storage_root: H256, address: H256, keys: &[H256], ) -> Result, TrieError> { - let db = Arc::new(DupHashDatabase::from_root(tx, address, storage_root)?); + let db = Arc::new(DupHashDatabase::from_root(self.tx, address, storage_root)?); let hasher = Arc::new(HasherKeccak::new()); let trie = @@ -588,6 +818,7 @@ where #[cfg(test)] mod tests { use crate::Transaction; + use std::ops::DerefMut; use super::*; use assert_matches::assert_matches; @@ -648,7 +879,7 @@ mod tests { let tx = Transaction::new(db.as_ref()).unwrap(); assert_matches!( create_test_loader(&tx).calculate_root(), - Ok(got) if got == EMPTY_ROOT + Ok(got) if got.root().unwrap() == EMPTY_ROOT ); } @@ -664,14 +895,15 @@ mod tests { let expected = H256(sec_trie_root::([(address, encoded_account)]).0); assert_matches!( create_test_loader(&tx).calculate_root(), - Ok(got) if got == expected + Ok(got) if got.root().unwrap() == expected ); } #[test] fn two_accounts_trie() { let db = create_test_rw_db(); - let tx = Transaction::new(db.as_ref()).unwrap(); + let mut tx = Transaction::new(db.as_ref()).unwrap(); + let mut trie = DBTrieLoader::new(tx.deref_mut()); let accounts = [ ( @@ -684,7 +916,7 @@ mod tests { ), ]; for (address, account) in accounts { - tx.put::(keccak256(address), account).unwrap(); + trie.tx.put::(keccak256(address), account).unwrap(); } let encoded_accounts = accounts.iter().map(|(k, v)| { let mut out = Vec::new(); @@ -693,26 +925,28 @@ mod tests { }); let expected = H256(sec_trie_root::(encoded_accounts).0); assert_matches!( - create_test_loader(&tx).calculate_root(), - Ok(got) if got == expected + trie.calculate_root(), + Ok(got) if got.root().unwrap() == expected ); } #[test] fn single_storage_trie() { let db = create_test_rw_db(); - let tx = Transaction::new(db.as_ref()).unwrap(); + let mut tx = Transaction::new(db.as_ref()).unwrap(); + let mut trie = DBTrieLoader::new(tx.deref_mut()); let address = Address::from_str("9fe4abd71ad081f091bd06dd1c16f7e92927561e").unwrap(); let hashed_address = keccak256(address); let storage = Vec::from([(H256::from_low_u64_be(2), U256::from(1))]); for (k, v) in storage.clone() { - tx.put::( - hashed_address, - StorageEntry { key: keccak256(k), value: v }, - ) - .unwrap(); + trie.tx + .put::( + hashed_address, + StorageEntry { key: keccak256(k), value: v }, + ) + .unwrap(); } let encoded_storage = storage.iter().map(|(k, v)| { let out = encode_fixed_size(v).to_vec(); @@ -720,8 +954,8 @@ mod tests { }); let expected = H256(sec_trie_root::(encoded_storage).0); assert_matches!( - create_test_loader(&tx).calculate_storage_root(hashed_address), - Ok(got) if got == expected + trie.calculate_storage_root(hashed_address, None, None), + Ok(got) if got.root().unwrap() == expected ); } @@ -766,7 +1000,7 @@ mod tests { let expected = H256(sec_trie_root::([(address, out)]).0); assert_matches!( create_test_loader(&tx).calculate_root(), - Ok(got) if got == expected + Ok(got) if got.root().unwrap() == expected ); } @@ -781,7 +1015,7 @@ mod tests { assert_matches!( create_test_loader(&tx).calculate_root(), - Ok(got) if got == state_root + Ok(got) if got.root().unwrap() == state_root ); } @@ -863,7 +1097,7 @@ mod tests { let expected = H256(sec_trie_root::(encoded_accounts).0); assert_matches!( create_test_loader(&tx).calculate_root(), - Ok(got) if got == expected + Ok(got) if got.root().unwrap() == expected , "where expected is {expected:?}"); } @@ -882,8 +1116,8 @@ mod tests { load_mainnet_genesis_root(&mut tx); let root = { - let trie = create_test_loader(&tx); - trie.calculate_root().expect("should be able to load trie") + let mut trie = create_test_loader(&tx); + trie.calculate_root().expect("should be able to load trie").root().unwrap() }; tx.commit().unwrap(); @@ -891,9 +1125,8 @@ mod tests { let address = Address::from(hex!("000d836201318ec6899a67540690382780743280")); let trie = create_test_loader(&tx); - let (proof, storage_root) = trie - .generate_acount_proof(&tx.inner().tx().unwrap(), root, keccak256(address)) - .expect("failed to generate proof"); + let (proof, storage_root) = + trie.generate_acount_proof(root, keccak256(address)).expect("failed to generate proof"); // values extracted from geth via rpc: // { @@ -947,16 +1180,15 @@ mod tests { } let root = { - let trie = create_test_loader(&tx); - trie.calculate_root().expect("should be able to load trie") + let mut trie = create_test_loader(&tx); + trie.calculate_root().expect("should be able to load trie").root().unwrap() }; tx.commit().unwrap(); let trie = create_test_loader(&tx); - let (account_proof, storage_root) = trie - .generate_acount_proof(&tx.inner().tx().unwrap(), root, hashed_address) - .expect("failed to generate proof"); + let (account_proof, storage_root) = + trie.generate_acount_proof(root, hashed_address).expect("failed to generate proof"); // values extracted from geth via rpc: let expected_account = hex!("f86fa1205126413e7857595763591580306b3f228f999498c4c5dfa74f633364936e7651b84bf849819b8418b0d164a029ff6f4d518044318d75b118cf439d8d3d7249c8afcba06ba9ecdf8959410571a02ce1a85814ad94a94ed2a1abaf7c57e9b64326622c1b8c21b4ba4d0e7df61392").as_slice(); @@ -980,7 +1212,6 @@ mod tests { let storage_proofs = trie .generate_storage_proofs( - &tx.inner().tx().unwrap(), storage_root, hashed_address, &[keccak256(H256::from_low_u64_be(2)), keccak256(H256::zero())], From ee3a8af754e994362f8159103221a339fd727e50 Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Tue, 14 Mar 2023 14:41:33 +0800 Subject: [PATCH 117/191] fix(stages): clear pre-eip161 accounts (#1747) --- crates/executor/src/executor.rs | 4 ++-- crates/storage/provider/src/execution_result.rs | 11 ++++++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/crates/executor/src/executor.rs b/crates/executor/src/executor.rs index 22cc7c391dc..e6a7bef842a 100644 --- a/crates/executor/src/executor.rs +++ b/crates/executor/src/executor.rs @@ -185,7 +185,7 @@ where new: to_reth_acc(&account.info), } } else { - AccountInfoChangeSet::NoChange + AccountInfoChangeSet::NoChange { is_empty: account.is_empty() } }; entry.info = account.info.clone(); (account_changeset, entry) @@ -709,7 +709,7 @@ mod tests { assert_eq!( changesets.changeset.get(&account1).unwrap().account, - AccountInfoChangeSet::NoChange, + AccountInfoChangeSet::NoChange { is_empty: false }, "No change to account" ); assert_eq!( diff --git a/crates/storage/provider/src/execution_result.rs b/crates/storage/provider/src/execution_result.rs index 747f08eabc2..3c0fcc18dd2 100644 --- a/crates/storage/provider/src/execution_result.rs +++ b/crates/storage/provider/src/execution_result.rs @@ -53,7 +53,10 @@ pub enum AccountInfoChangeSet { old: Account, }, /// Nothing was changed for the account (nonce/balance). - NoChange, + NoChange { + /// Useful to clear existing empty accounts pre-EIP-161. + is_empty: bool, + }, } impl AccountInfoChangeSet { @@ -94,8 +97,10 @@ impl AccountInfoChangeSet { AccountBeforeTx { address, info: Some(old) }, )?; } - AccountInfoChangeSet::NoChange => { - // do nothing storage account didn't change + AccountInfoChangeSet::NoChange { is_empty } => { + if has_state_clear_eip && is_empty { + tx.delete::(address, None)?; + } } } Ok(()) From b121e4d8c7ec143f993170dc77d83139031d6168 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 14 Mar 2023 10:55:48 +0100 Subject: [PATCH 118/191] chore(rpc): make async (#1748) --- crates/rpc/rpc-api/src/trace.rs | 5 +++-- crates/rpc/rpc/src/trace.rs | 7 +++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/crates/rpc/rpc-api/src/trace.rs b/crates/rpc/rpc-api/src/trace.rs index 431ab04bb9c..f5670283fdf 100644 --- a/crates/rpc/rpc-api/src/trace.rs +++ b/crates/rpc/rpc-api/src/trace.rs @@ -69,7 +69,7 @@ pub trait TraceApi { /// Returns transaction trace at given index. #[method(name = "trace_get")] - fn trace_get( + async fn trace_get( &self, hash: H256, indices: Vec, @@ -77,5 +77,6 @@ pub trait TraceApi { /// Returns all traces of given transaction. #[method(name = "trace_transaction")] - fn trace_transaction(&self, hash: H256) -> Result>>; + async fn trace_transaction(&self, hash: H256) + -> Result>>; } diff --git a/crates/rpc/rpc/src/trace.rs b/crates/rpc/rpc/src/trace.rs index b6b20e35148..d2e46474df9 100644 --- a/crates/rpc/rpc/src/trace.rs +++ b/crates/rpc/rpc/src/trace.rs @@ -96,7 +96,7 @@ where } /// Handler for `trace_get` - fn trace_get( + async fn trace_get( &self, _hash: H256, _indices: Vec, @@ -105,7 +105,10 @@ where } /// Handler for `trace_transaction` - fn trace_transaction(&self, _hash: H256) -> Result>> { + async fn trace_transaction( + &self, + _hash: H256, + ) -> Result>> { Err(internal_rpc_err("unimplemented")) } } From c3a49aea727e269af63a9ae71fd49374c64dbbd5 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 14 Mar 2023 11:01:18 +0100 Subject: [PATCH 119/191] style: resort trait impl order (#1749) --- crates/primitives/src/transaction/mod.rs | 38 +++++++++++++----------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/crates/primitives/src/transaction/mod.rs b/crates/primitives/src/transaction/mod.rs index f9d43f6a303..41668749ba3 100644 --- a/crates/primitives/src/transaction/mod.rs +++ b/crates/primitives/src/transaction/mod.rs @@ -176,6 +176,8 @@ pub enum Transaction { Eip1559(TxEip1559), } +// === impl Transaction === + impl Transaction { /// Heavy operation that return signature hash over rlp encoded transaction. /// It is only for signature signing or signer recovery. @@ -856,6 +858,8 @@ pub struct TransactionSignedEcRecovered { signed_transaction: TransactionSigned, } +// === impl TransactionSignedEcRecovered === + impl TransactionSignedEcRecovered { /// Signer of transaction recovered from signature pub fn signer(&self) -> Address { @@ -874,23 +878,6 @@ impl TransactionSignedEcRecovered { } } -/// A transaction type that can be created from a [`TransactionSignedEcRecovered`] transaction. -/// -/// This is a conversion trait that'll ensure transactions received via P2P can be converted to the -/// transaction type that the transaction pool uses. -pub trait FromRecoveredTransaction { - /// Converts to this type from the given [`TransactionSignedEcRecovered`]. - fn from_recovered_transaction(tx: TransactionSignedEcRecovered) -> Self; -} - -// Noop conversion -impl FromRecoveredTransaction for TransactionSignedEcRecovered { - #[inline] - fn from_recovered_transaction(tx: TransactionSignedEcRecovered) -> Self { - tx - } -} - impl Encodable for TransactionSignedEcRecovered { fn encode(&self, out: &mut dyn bytes::BufMut) { self.signed_transaction.encode(out) @@ -911,6 +898,23 @@ impl Decodable for TransactionSignedEcRecovered { } } +/// A transaction type that can be created from a [`TransactionSignedEcRecovered`] transaction. +/// +/// This is a conversion trait that'll ensure transactions received via P2P can be converted to the +/// transaction type that the transaction pool uses. +pub trait FromRecoveredTransaction { + /// Converts to this type from the given [`TransactionSignedEcRecovered`]. + fn from_recovered_transaction(tx: TransactionSignedEcRecovered) -> Self; +} + +// Noop conversion +impl FromRecoveredTransaction for TransactionSignedEcRecovered { + #[inline] + fn from_recovered_transaction(tx: TransactionSignedEcRecovered) -> Self { + tx + } +} + /// The inverse of [`FromRecoveredTransaction`] that ensure the transaction can be sent over the /// network pub trait IntoRecoveredTransaction { From 6c12ccb6e15008c13e7b07d0806a78ff05fd2b2f Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 14 Mar 2023 11:14:32 +0100 Subject: [PATCH 120/191] chore(txpool): add IntoRecoveredTransaction to PoolTransaction (#1750) --- crates/transaction-pool/src/traits.rs | 4 +++- crates/transaction-pool/src/validate.rs | 11 +++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index 697dbafc713..645a59df2a5 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -250,7 +250,9 @@ impl BestTransactions for std::iter::Empty { } /// Trait for transaction types used inside the pool -pub trait PoolTransaction: fmt::Debug + Send + Sync + FromRecoveredTransaction { +pub trait PoolTransaction: + fmt::Debug + Send + Sync + FromRecoveredTransaction + IntoRecoveredTransaction +{ /// Hash of the transaction. fn hash(&self) -> &TxHash; diff --git a/crates/transaction-pool/src/validate.rs b/crates/transaction-pool/src/validate.rs index cafab6aaf7c..84770a1ba77 100644 --- a/crates/transaction-pool/src/validate.rs +++ b/crates/transaction-pool/src/validate.rs @@ -7,8 +7,9 @@ use crate::{ MAX_INIT_CODE_SIZE, TX_MAX_SIZE, }; use reth_primitives::{ - Address, InvalidTransactionError, TransactionKind, TxHash, EIP1559_TX_TYPE_ID, - EIP2930_TX_TYPE_ID, LEGACY_TX_TYPE_ID, U256, + Address, IntoRecoveredTransaction, InvalidTransactionError, TransactionKind, + TransactionSignedEcRecovered, TxHash, EIP1559_TX_TYPE_ID, EIP2930_TX_TYPE_ID, + LEGACY_TX_TYPE_ID, U256, }; use reth_provider::AccountProvider; use std::{fmt, time::Instant}; @@ -330,6 +331,12 @@ impl ValidPoolTransaction { } } +impl IntoRecoveredTransaction for ValidPoolTransaction { + fn to_recovered_transaction(&self) -> TransactionSignedEcRecovered { + self.transaction.to_recovered_transaction() + } +} + #[cfg(test)] impl Clone for ValidPoolTransaction { fn clone(&self) -> Self { From a0ff24b691aee859db10a11b48df56100fd5e1e2 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 14 Mar 2023 13:55:35 +0100 Subject: [PATCH 121/191] feat(rpc): add eth transactions trait (#1751) --- .../rpc/rpc-types/src/eth/transaction/mod.rs | 4 +- crates/rpc/rpc/src/eth/api/mod.rs | 25 ++--- crates/rpc/rpc/src/eth/api/server.rs | 7 +- crates/rpc/rpc/src/eth/api/transactions.rs | 102 ++++++++++++++---- crates/rpc/rpc/src/eth/error.rs | 8 +- crates/rpc/rpc/src/eth/mod.rs | 4 +- 6 files changed, 103 insertions(+), 47 deletions(-) diff --git a/crates/rpc/rpc-types/src/eth/transaction/mod.rs b/crates/rpc/rpc-types/src/eth/transaction/mod.rs index f6a2f421333..a5a169bf9b1 100644 --- a/crates/rpc/rpc-types/src/eth/transaction/mod.rs +++ b/crates/rpc/rpc-types/src/eth/transaction/mod.rs @@ -87,9 +87,9 @@ impl Transaction { tx } - /// Create a new rpc transaction result for a pending signed transaction, setting block + /// Create a new rpc transaction result for a _pending_ signed transaction, setting block /// environment related fields to `None`. - pub(crate) fn from_recovered(tx: TransactionSignedEcRecovered) -> Self { + pub fn from_recovered(tx: TransactionSignedEcRecovered) -> Self { let signer = tx.signer(); let signed_tx = tx.into_signed(); diff --git a/crates/rpc/rpc/src/eth/api/mod.rs b/crates/rpc/rpc/src/eth/api/mod.rs index 97911adf5c1..826ae34c90c 100644 --- a/crates/rpc/rpc/src/eth/api/mod.rs +++ b/crates/rpc/rpc/src/eth/api/mod.rs @@ -3,29 +3,25 @@ //! The entire implementation of the namespace is quite large, hence it is divided across several //! files. -use crate::eth::signer::EthSigner; +use crate::eth::{cache::EthStateCache, error::EthResult, signer::EthSigner}; use async_trait::async_trait; use reth_interfaces::Result; use reth_network_api::NetworkInfo; -use reth_primitives::{ - Address, BlockId, BlockNumberOrTag, ChainInfo, TransactionSigned, H256, U64, -}; +use reth_primitives::{Address, BlockId, BlockNumberOrTag, ChainInfo, H256, U64}; use reth_provider::{ - BlockProvider, EvmEnvProvider, StateProvider as StateProviderTrait, StateProviderFactory, + providers::ChainState, BlockProvider, EvmEnvProvider, StateProvider as StateProviderTrait, + StateProviderFactory, }; -use std::{num::NonZeroUsize, ops::Deref}; - -use crate::eth::{cache::EthStateCache, error::EthResult}; -use reth_provider::providers::ChainState; use reth_rpc_types::FeeHistoryCache; use reth_transaction_pool::TransactionPool; -use std::sync::Arc; +use std::{num::NonZeroUsize, ops::Deref, sync::Arc}; mod block; mod call; mod server; mod state; mod transactions; +pub use transactions::{EthTransactions, TransactionSource}; /// Cache limit of block-level fee history for `eth_feeHistory` RPC method. const FEE_HISTORY_CACHE_LIMIT: usize = 2048; @@ -34,7 +30,7 @@ const FEE_HISTORY_CACHE_LIMIT: usize = 2048; /// /// Defines core functionality of the `eth` API implementation. #[async_trait] -pub trait EthApiSpec: Send + Sync { +pub trait EthApiSpec: EthTransactions + Send + Sync { /// Returns the current ethereum protocol version. async fn protocol_version(&self) -> Result; @@ -46,9 +42,6 @@ pub trait EthApiSpec: Send + Sync { /// Returns a list of addresses owned by client. fn accounts(&self) -> Vec
; - - /// Returns the transaction by hash - async fn transaction_by_hash(&self, hash: H256) -> Result>; } /// `Eth` API implementation. @@ -243,10 +236,6 @@ where fn accounts(&self) -> Vec
{ self.inner.signers.iter().flat_map(|s| s.accounts()).collect() } - - async fn transaction_by_hash(&self, hash: H256) -> Result> { - self.client().transaction_by_hash(hash) - } } /// Container type `EthApi` diff --git a/crates/rpc/rpc/src/eth/api/server.rs b/crates/rpc/rpc/src/eth/api/server.rs index f59f25f5d9b..9a6f4f7d5e4 100644 --- a/crates/rpc/rpc/src/eth/api/server.rs +++ b/crates/rpc/rpc/src/eth/api/server.rs @@ -3,7 +3,10 @@ use super::EthApiSpec; use crate::{ - eth::{api::EthApi, error::EthApiError}, + eth::{ + api::{EthApi, EthTransactions}, + error::EthApiError, + }, result::{internal_rpc_err, ToRpcResult}, }; use jsonrpsee::core::RpcResult as Result; @@ -118,7 +121,7 @@ where /// Handler for: `eth_getTransactionByHash` async fn transaction_by_hash(&self, hash: H256) -> Result> { - Ok(EthApi::transaction_by_hash(self, hash).await?) + Ok(EthTransactions::transaction_by_hash(self, hash).await?.map(Into::into)) } /// Handler for: `eth_getTransactionByBlockHashAndIndex` diff --git a/crates/rpc/rpc/src/eth/api/transactions.rs b/crates/rpc/rpc/src/eth/api/transactions.rs index bece1772f19..416a97c1d66 100644 --- a/crates/rpc/rpc/src/eth/api/transactions.rs +++ b/crates/rpc/rpc/src/eth/api/transactions.rs @@ -3,43 +3,71 @@ use crate::{ eth::error::{EthApiError, EthResult}, EthApi, }; -use reth_primitives::{BlockId, Bytes, FromRecoveredTransaction, TransactionSigned, H256}; -use reth_provider::{BlockProvider, EvmEnvProvider, StateProviderFactory}; +use async_trait::async_trait; +use reth_primitives::{ + BlockId, Bytes, FromRecoveredTransaction, IntoRecoveredTransaction, TransactionSigned, + TransactionSignedEcRecovered, H256, U256, +}; +use reth_provider::{BlockProvider, EvmEnvProvider, StateProviderFactory, TransactionsProvider}; use reth_rlp::Decodable; use reth_rpc_types::{Index, Transaction, TransactionRequest}; use reth_transaction_pool::{TransactionOrigin, TransactionPool}; -impl EthApi +/// Commonly used transaction related functions for the [EthApi] type in the `eth_` namespace +#[async_trait::async_trait] +pub trait EthTransactions: Send + Sync { + /// Returns the transaction by hash. + /// + /// Checks the pool and state. + /// + /// Returns `Ok(None)` if no matching transaction was found. + async fn transaction_by_hash(&self, hash: H256) -> EthResult>; +} + +#[async_trait] +impl EthTransactions for EthApi where - Pool: TransactionPool + 'static, - Client: BlockProvider + StateProviderFactory + EvmEnvProvider + 'static, - Network: 'static, + Pool: TransactionPool + Clone + 'static, + Client: TransactionsProvider + 'static, + Network: Send + Sync + 'static, { - pub(crate) async fn send_transaction(&self, _request: TransactionRequest) -> EthResult { - unimplemented!() - } + async fn transaction_by_hash(&self, hash: H256) -> EthResult> { + if let Some(tx) = self.pool().get(&hash).map(|tx| tx.transaction.to_recovered_transaction()) + { + return Ok(Some(TransactionSource::Pool(tx))) + } - /// Finds a given [Transaction] by its hash. - /// - /// Returns `Ok(None)` if no matching transaction was found. - pub(crate) async fn transaction_by_hash(&self, hash: H256) -> EthResult> { match self.client().transaction_by_hash(hash)? { None => Ok(None), Some(tx) => { - let tx = tx.into_ecrecovered().ok_or(EthApiError::InvalidTransactionSignature)?; + let transaction = + tx.into_ecrecovered().ok_or(EthApiError::InvalidTransactionSignature)?; - let tx = Transaction::from_recovered_with_block_context( - tx, + let tx = TransactionSource::Database { + transaction, // TODO: this is just stubbed out for now still need to fully implement tx => // block - H256::default(), - u64::default(), - Index::default().into(), - ); + index: 0, + block_hash: Default::default(), + block_number: 0, + }; Ok(Some(tx)) } } } +} + +// === impl EthApi === + +impl EthApi +where + Pool: TransactionPool + 'static, + Client: BlockProvider + StateProviderFactory + EvmEnvProvider + 'static, + Network: 'static, +{ + pub(crate) async fn send_transaction(&self, _request: TransactionRequest) -> EthResult { + unimplemented!() + } /// Get Transaction by [BlockId] and the index of the transaction within that Block. /// @@ -94,6 +122,40 @@ where } } +/// Represents from where a transaction was fetched. +#[derive(Debug, Clone, Eq, PartialEq)] +pub enum TransactionSource { + /// Transaction exists in the pool (Pending) + Pool(TransactionSignedEcRecovered), + /// Transaction already executed + Database { + /// Transaction fetched via provider + transaction: TransactionSignedEcRecovered, + /// Index of the transaction in the block + index: usize, + /// Hash of the block. + block_hash: H256, + /// Number of the block. + block_number: u64, + }, +} + +impl From for Transaction { + fn from(value: TransactionSource) -> Self { + match value { + TransactionSource::Pool(tx) => Transaction::from_recovered(tx), + TransactionSource::Database { transaction, index, block_hash, block_number } => { + Transaction::from_recovered_with_block_context( + transaction, + block_hash, + block_number, + U256::from(index), + ) + } + } + } +} + #[cfg(test)] mod tests { use crate::eth::cache::EthStateCache; diff --git a/crates/rpc/rpc/src/eth/error.rs b/crates/rpc/rpc/src/eth/error.rs index 2d5466417c3..3aa76da5804 100644 --- a/crates/rpc/rpc/src/eth/error.rs +++ b/crates/rpc/rpc/src/eth/error.rs @@ -8,12 +8,12 @@ use reth_transaction_pool::error::{InvalidPoolTransactionError, PoolError}; use revm::primitives::{EVMError, Halt}; /// Result alias -pub(crate) type EthResult = Result; +pub type EthResult = Result; /// Errors that can occur when interacting with the `eth_` namespace #[derive(Debug, thiserror::Error)] #[allow(missing_docs)] -pub(crate) enum EthApiError { +pub enum EthApiError { /// When a raw transaction is empty #[error("Empty transaction data")] EmptyRawTransactionData, @@ -161,6 +161,7 @@ pub enum InvalidTransactionError { /// Unspecific evm halt error #[error("EVM error {0:?}")] EvmHalt(Halt), + /// Invalid chain id set for the transaction. #[error("Invalid chain id")] InvalidChainId, } @@ -268,7 +269,8 @@ impl std::error::Error for RevertError {} /// A helper error type that's mainly used to mirror `geth` Txpool's error messages #[derive(Debug, thiserror::Error)] -pub(crate) enum RpcPoolError { +#[allow(missing_docs)] +pub enum RpcPoolError { #[error("already known")] AlreadyKnown, #[error("invalid sender")] diff --git a/crates/rpc/rpc/src/eth/mod.rs b/crates/rpc/rpc/src/eth/mod.rs index da465c50a55..7dee1a12578 100644 --- a/crates/rpc/rpc/src/eth/mod.rs +++ b/crates/rpc/rpc/src/eth/mod.rs @@ -2,14 +2,14 @@ mod api; pub mod cache; -pub(crate) mod error; +pub mod error; mod filter; mod id_provider; mod pubsub; pub(crate) mod revm_utils; mod signer; -pub use api::{EthApi, EthApiSpec}; +pub use api::{EthApi, EthApiSpec, EthTransactions, TransactionSource}; pub use filter::EthFilter; pub use id_provider::EthSubscriptionIdProvider; pub use pubsub::EthPubSub; From 8e5644dac09c6e49806b24b3dd096a080be5ea5a Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 14 Mar 2023 17:03:36 +0100 Subject: [PATCH 122/191] feat(rpc): complete log filter (#1741) --- crates/rpc/rpc/src/eth/filter.rs | 64 ++++++++++++++----------- crates/rpc/rpc/src/eth/logs_utils.rs | 72 ++++++++++++++++++++++++++++ crates/rpc/rpc/src/eth/mod.rs | 1 + crates/rpc/rpc/src/eth/pubsub.rs | 58 ++++------------------ 4 files changed, 116 insertions(+), 79 deletions(-) create mode 100644 crates/rpc/rpc/src/eth/logs_utils.rs diff --git a/crates/rpc/rpc/src/eth/filter.rs b/crates/rpc/rpc/src/eth/filter.rs index 9c7bdda56de..064bf14dfc4 100644 --- a/crates/rpc/rpc/src/eth/filter.rs +++ b/crates/rpc/rpc/src/eth/filter.rs @@ -1,5 +1,5 @@ use crate::{ - eth::error::EthApiError, + eth::{error::EthApiError, logs_utils}, result::{internal_rpc_err, rpc_error_with_code, ToRpcResult}, EthSubscriptionIdProvider, }; @@ -7,7 +7,7 @@ use async_trait::async_trait; use jsonrpsee::{core::RpcResult, server::IdProvider}; use reth_primitives::{ filter::{Filter, FilterBlockOption, FilteredParams}, - Block, U256, + U256, }; use reth_provider::{BlockProvider, EvmEnvProvider}; use reth_rpc_api::EthFilterApiServer; @@ -150,7 +150,7 @@ where trace!(target: "rpc::eth::filter", ?id, "uninstalled filter"); Ok(true) } else { - Err(internal_rpc_err(format!("Filter id {id:?} does not exist."))) + Ok(false) } } @@ -201,9 +201,8 @@ where /// Returns an error if: /// - underlying database error /// - amount of matches exceeds configured limit - #[allow(dead_code)] fn filter_logs(&self, filter: &Filter, from_block: u64, to_block: u64) -> RpcResult> { - let mut logs = Vec::new(); + let mut all_logs = Vec::new(); let filter_params = FilteredParams::new(Some(filter.clone())); let topics = @@ -213,38 +212,41 @@ where let address_filter = FilteredParams::address_filter(&filter.address); let topics_filter = FilteredParams::topics_filter(&topics); + // loop over the range of new blocks and check logs if the filter matches the log's bloom + // filter for block_number in from_block..=to_block { if let Some(block) = self.client.block_by_number(block_number).to_rpc_result()? { // only if filter matches if FilteredParams::matches_address(block.header.logs_bloom, &address_filter) && FilteredParams::matches_topics(block.header.logs_bloom, &topics_filter) { - self.append_matching_block_logs(&mut logs, &filter_params, block); - - // TODO size check + // get receipts for the block + if let Some(receipts) = + self.client.receipts_by_block(block.number.into()).to_rpc_result()? + { + let block_hash = block.hash_slow(); + + logs_utils::append_matching_block_logs( + &mut all_logs, + &filter_params, + block_hash, + block_number, + block.body.into_iter().map(|tx| tx.hash).zip(receipts), + ); + + // size check + if all_logs.len() > self.max_logs_in_response { + return Err(FilterError::QueryExceedsMaxResults( + self.max_logs_in_response, + ) + .into()) + } + } } } } - Ok(logs) - } - - /// Appends all logs emitted in the `block` that match the `filter` to the `logs` vector. - #[allow(clippy::ptr_arg)] - fn append_matching_block_logs( - &self, - _logs: &mut Vec, - _filter: &FilteredParams, - block: Block, - ) { - let _block_log_index: u32 = 0; - let _block_hash = block.hash_slow(); - - // loop over all transactions in the block - for tx in block.body { - let _transaction_log_index: u32 = 0; - let _transaction_hash = tx.hash; - } + Ok(all_logs) } } @@ -266,7 +268,6 @@ struct ActiveFilter { } #[derive(Clone, Debug)] -#[allow(clippy::large_enum_variant)] enum FilterKind { Log(Box), Block, @@ -278,6 +279,8 @@ enum FilterKind { pub enum FilterError { #[error("filter not found")] FilterNotFound(FilterId), + #[error("Query exceeds max results {0}")] + QueryExceedsMaxResults(usize), } // convert the error @@ -285,9 +288,12 @@ impl From for jsonrpsee::core::Error { fn from(err: FilterError) -> Self { match err { FilterError::FilterNotFound(_) => rpc_error_with_code( - jsonrpsee::types::error::CALL_EXECUTION_FAILED_CODE, + jsonrpsee::types::error::INVALID_PARAMS_CODE, "filter not found", ), + err @ FilterError::QueryExceedsMaxResults(_) => { + rpc_error_with_code(jsonrpsee::types::error::INVALID_PARAMS_CODE, err.to_string()) + } } } } diff --git a/crates/rpc/rpc/src/eth/logs_utils.rs b/crates/rpc/rpc/src/eth/logs_utils.rs new file mode 100644 index 00000000000..41f3cdf87e4 --- /dev/null +++ b/crates/rpc/rpc/src/eth/logs_utils.rs @@ -0,0 +1,72 @@ +use reth_primitives::{filter::FilteredParams, Receipt, TxHash, U256}; +use reth_rpc_types::Log; +use revm::primitives::B256 as H256; + +/// Returns all matching logs of a block's receipts grouped with the hash of their transaction. +pub(crate) fn matching_block_logs( + filter: &FilteredParams, + block_hash: H256, + block_number: u64, + tx_and_receipts: I, +) -> Vec +where + I: IntoIterator, +{ + let mut all_logs = Vec::new(); + append_matching_block_logs(&mut all_logs, filter, block_hash, block_number, tx_and_receipts); + all_logs +} + +/// Appends all matching logs of a block's receipts grouped with the hash of their transaction +pub(crate) fn append_matching_block_logs( + all_logs: &mut Vec, + filter: &FilteredParams, + block_hash: H256, + block_number: u64, + tx_and_receipts: I, +) where + I: IntoIterator, +{ + let block_number_u256 = U256::from(block_number); + // tracks the index of a log in the entire block + let mut log_index: u32 = 0; + for (transaction_idx, (transaction_hash, receipt)) in tx_and_receipts.into_iter().enumerate() { + let logs = receipt.logs; + for (transaction_log_idx, log) in logs.into_iter().enumerate() { + if log_matches_filter(block_hash, block_number, &log, filter) { + let log = Log { + address: log.address, + topics: log.topics, + data: log.data, + block_hash: Some(block_hash), + block_number: Some(block_number_u256), + transaction_hash: Some(transaction_hash), + transaction_index: Some(U256::from(transaction_idx)), + log_index: Some(U256::from(log_index)), + transaction_log_index: Some(U256::from(transaction_log_idx)), + removed: false, + }; + all_logs.push(log); + } + log_index += 1; + } + } +} + +/// Returns true if the log matches the filter and should be included +pub(crate) fn log_matches_filter( + block_hash: H256, + block_number: u64, + log: &reth_primitives::Log, + params: &FilteredParams, +) -> bool { + if params.filter.is_some() && + (!params.filter_block_range(block_number) || + !params.filter_block_hash(block_hash) || + !params.filter_address(log) || + !params.filter_topics(log)) + { + return false + } + true +} diff --git a/crates/rpc/rpc/src/eth/mod.rs b/crates/rpc/rpc/src/eth/mod.rs index 7dee1a12578..6958b1238d5 100644 --- a/crates/rpc/rpc/src/eth/mod.rs +++ b/crates/rpc/rpc/src/eth/mod.rs @@ -5,6 +5,7 @@ pub mod cache; pub mod error; mod filter; mod id_provider; +mod logs_utils; mod pubsub; pub(crate) mod revm_utils; mod signer; diff --git a/crates/rpc/rpc/src/eth/pubsub.rs b/crates/rpc/rpc/src/eth/pubsub.rs index 5a2ee798efb..ff35b78ba93 100644 --- a/crates/rpc/rpc/src/eth/pubsub.rs +++ b/crates/rpc/rpc/src/eth/pubsub.rs @@ -1,9 +1,10 @@ //! `eth_` PubSub RPC handler implementation +use crate::eth::logs_utils; use futures::StreamExt; use jsonrpsee::{types::SubscriptionResult, SubscriptionSink}; use reth_interfaces::{events::ChainEventSubscriptions, sync::SyncStateProvider}; -use reth_primitives::{filter::FilteredParams, BlockId, TxHash, H256, U256}; +use reth_primitives::{filter::FilteredParams, BlockId, TxHash}; use reth_provider::{BlockProvider, EvmEnvProvider}; use reth_rpc_api::EthPubSubApiServer; use reth_rpc_types::{ @@ -226,56 +227,13 @@ where .flat_map(move |(new_block, transactions, receipts)| { let block_hash = new_block.hash; let block_number = new_block.header.number; - let mut all_logs: Vec = Vec::new(); - - // tracks the index of a log in the entire block - let mut log_index: u32 = 0; - for (transaction_idx, (tx, receipt)) in - transactions.into_iter().zip(receipts).enumerate() - { - let logs = receipt.logs; - - // tracks the index of the log in the transaction - let transaction_hash = tx.hash; - - for (transaction_log_idx, log) in logs.into_iter().enumerate() { - if matches_filter(block_hash, block_number, &log, &filter) { - let log = Log { - address: log.address, - topics: log.topics, - data: log.data, - block_hash: Some(block_hash), - block_number: Some(U256::from(block_number)), - transaction_hash: Some(transaction_hash), - transaction_index: Some(U256::from(transaction_idx)), - log_index: Some(U256::from(log_index)), - transaction_log_index: Some(U256::from(transaction_log_idx)), - removed: false, - }; - all_logs.push(log); - } - log_index += 1; - } - } + let all_logs = logs_utils::matching_block_logs( + &filter, + block_hash, + block_number, + transactions.into_iter().map(|tx| tx.hash).zip(receipts), + ); futures::stream::iter(all_logs) }) } } - -/// Returns true if the log matches the filter and should be included -fn matches_filter( - block_hash: H256, - block_number: u64, - log: &reth_primitives::Log, - params: &FilteredParams, -) -> bool { - if params.filter.is_some() && - (!params.filter_block_range(block_number) || - !params.filter_block_hash(block_hash) || - !params.filter_address(log) || - !params.filter_topics(log)) - { - return false - } - true -} From c78effcb3f4f9f4c576b29716b78984e7e3908c0 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 14 Mar 2023 17:07:08 +0100 Subject: [PATCH 123/191] chore(rpc): more trace prep (#1752) --- crates/rpc/rpc-builder/src/lib.rs | 4 +- crates/rpc/rpc-builder/tests/it/http.rs | 3 - crates/rpc/rpc/src/eth/api/call.rs | 27 ++------ crates/rpc/rpc/src/eth/api/server.rs | 4 +- crates/rpc/rpc/src/eth/api/transactions.rs | 81 ++++++++++++++++++++-- crates/rpc/rpc/src/trace.rs | 29 +++++--- 6 files changed, 107 insertions(+), 41 deletions(-) diff --git a/crates/rpc/rpc-builder/src/lib.rs b/crates/rpc/rpc-builder/src/lib.rs index 5d66e238703..c5b81b8a143 100644 --- a/crates/rpc/rpc-builder/src/lib.rs +++ b/crates/rpc/rpc-builder/src/lib.rs @@ -576,7 +576,9 @@ where NetApi::new(self.network.clone(), eth_api.clone()).into_rpc().into() } RethRpcModule::Trace => { - TraceApi::new(self.client.clone(), eth_cache.clone()).into_rpc().into() + TraceApi::new(self.client.clone(), eth_api.clone(), eth_cache.clone()) + .into_rpc() + .into() } RethRpcModule::Web3 => Web3Api::new(self.network.clone()).into_rpc().into(), }) diff --git a/crates/rpc/rpc-builder/tests/it/http.rs b/crates/rpc/rpc-builder/tests/it/http.rs index f55d4bcbf0f..6d4cc66df01 100644 --- a/crates/rpc/rpc-builder/tests/it/http.rs +++ b/crates/rpc/rpc-builder/tests/it/http.rs @@ -198,9 +198,6 @@ where assert!(is_unimplemented( TraceApiClient::trace_get(client, H256::default(), vec![]).await.err().unwrap() )); - assert!(is_unimplemented( - TraceApiClient::trace_transaction(client, H256::default()).await.err().unwrap() - )); } async fn test_basic_web3_calls(client: &C) diff --git a/crates/rpc/rpc/src/eth/api/call.rs b/crates/rpc/rpc/src/eth/api/call.rs index 6c43da516f6..db76bfb8123 100644 --- a/crates/rpc/rpc/src/eth/api/call.rs +++ b/crates/rpc/rpc/src/eth/api/call.rs @@ -4,6 +4,7 @@ use crate::{ eth::{ error::{EthApiError, EthResult, InvalidTransactionError, RevertError}, revm_utils::{build_call_evm_env, get_precompiles, inspect, transact}, + EthTransactions, }, EthApi, }; @@ -18,6 +19,7 @@ use reth_rpc_types::{ state::{AccountOverride, StateOverride}, CallRequest, }; +use reth_transaction_pool::TransactionPool; use revm::{ db::{CacheDB, DatabaseRef}, primitives::{ @@ -33,31 +35,10 @@ const MIN_CREATE_GAS: u64 = 53_000u64; impl EthApi where + Pool: TransactionPool + Clone + 'static, Client: BlockProvider + StateProviderFactory + EvmEnvProvider + 'static, + Network: Send + Sync + 'static, { - /// Returns the revm evm env for the requested [BlockId] - /// - /// If the [BlockId] this will return the [BlockId::Hash] of the block the env was configured - /// for. - async fn evm_env_at(&self, at: BlockId) -> EthResult<(CfgEnv, BlockEnv, BlockId)> { - // TODO handle Pending state's env - match at { - BlockId::Number(BlockNumberOrTag::Pending) => { - // This should perhaps use the latest env settings and update block specific - // settings like basefee/number - unimplemented!("support pending state env") - } - hash_or_num => { - let block_hash = self - .client() - .block_hash_for_id(hash_or_num)? - .ok_or_else(|| EthApiError::UnknownBlockNumber)?; - let (cfg, env) = self.cache().get_evm_env(block_hash).await?; - Ok((cfg, env, block_hash.into())) - } - } - } - /// Executes the call request at the given [BlockId] pub(crate) async fn call_at( &self, diff --git a/crates/rpc/rpc/src/eth/api/server.rs b/crates/rpc/rpc/src/eth/api/server.rs index 9a6f4f7d5e4..4e676713a4e 100644 --- a/crates/rpc/rpc/src/eth/api/server.rs +++ b/crates/rpc/rpc/src/eth/api/server.rs @@ -27,10 +27,10 @@ use std::collections::BTreeMap; #[async_trait::async_trait] impl EthApiServer for EthApi where - Self: EthApiSpec, + Self: EthApiSpec + EthTransactions, Pool: TransactionPool + 'static, Client: BlockProvider + HeaderProvider + StateProviderFactory + EvmEnvProvider + 'static, - Network: 'static, + Network: Send + Sync + 'static, { /// Handler for: `eth_protocolVersion` async fn protocol_version(&self) -> Result { diff --git a/crates/rpc/rpc/src/eth/api/transactions.rs b/crates/rpc/rpc/src/eth/api/transactions.rs index 416a97c1d66..16d898e48ac 100644 --- a/crates/rpc/rpc/src/eth/api/transactions.rs +++ b/crates/rpc/rpc/src/eth/api/transactions.rs @@ -5,32 +5,64 @@ use crate::{ }; use async_trait::async_trait; use reth_primitives::{ - BlockId, Bytes, FromRecoveredTransaction, IntoRecoveredTransaction, TransactionSigned, - TransactionSignedEcRecovered, H256, U256, + BlockId, BlockNumberOrTag, Bytes, FromRecoveredTransaction, IntoRecoveredTransaction, + TransactionSigned, TransactionSignedEcRecovered, H256, U256, }; -use reth_provider::{BlockProvider, EvmEnvProvider, StateProviderFactory, TransactionsProvider}; +use reth_provider::{BlockProvider, EvmEnvProvider, StateProviderFactory}; use reth_rlp::Decodable; use reth_rpc_types::{Index, Transaction, TransactionRequest}; use reth_transaction_pool::{TransactionOrigin, TransactionPool}; +use revm::primitives::{BlockEnv, CfgEnv}; /// Commonly used transaction related functions for the [EthApi] type in the `eth_` namespace #[async_trait::async_trait] pub trait EthTransactions: Send + Sync { + /// Returns the revm evm env for the requested [BlockId] + /// + /// If the [BlockId] this will return the [BlockId::Hash] of the block the env was configured + /// for. + async fn evm_env_at(&self, at: BlockId) -> EthResult<(CfgEnv, BlockEnv, BlockId)>; + /// Returns the transaction by hash. /// /// Checks the pool and state. /// /// Returns `Ok(None)` if no matching transaction was found. async fn transaction_by_hash(&self, hash: H256) -> EthResult>; + + /// Returns the transaction by including its corresponding [BlockId] + async fn transaction_by_hash_at( + &self, + hash: H256, + ) -> EthResult>; } #[async_trait] impl EthTransactions for EthApi where Pool: TransactionPool + Clone + 'static, - Client: TransactionsProvider + 'static, + Client: BlockProvider + StateProviderFactory + EvmEnvProvider + 'static, Network: Send + Sync + 'static, { + async fn evm_env_at(&self, at: BlockId) -> EthResult<(CfgEnv, BlockEnv, BlockId)> { + // TODO handle Pending state's env + match at { + BlockId::Number(BlockNumberOrTag::Pending) => { + // This should perhaps use the latest env settings and update block specific + // settings like basefee/number + unimplemented!("support pending state env") + } + hash_or_num => { + let block_hash = self + .client() + .block_hash_for_id(hash_or_num)? + .ok_or_else(|| EthApiError::UnknownBlockNumber)?; + let (cfg, env) = self.cache().get_evm_env(block_hash).await?; + Ok((cfg, env, block_hash.into())) + } + } + } + async fn transaction_by_hash(&self, hash: H256) -> EthResult> { if let Some(tx) = self.pool().get(&hash).map(|tx| tx.transaction.to_recovered_transaction()) { @@ -55,6 +87,38 @@ where } } } + + async fn transaction_by_hash_at( + &self, + hash: H256, + ) -> EthResult> { + match self.transaction_by_hash(hash).await? { + None => return Ok(None), + Some(tx) => { + let res = match tx { + tx @ TransactionSource::Pool(_) => { + (tx, BlockId::Number(BlockNumberOrTag::Pending)) + } + TransactionSource::Database { + transaction, + index, + block_hash, + block_number, + } => { + let at = BlockId::Hash(block_hash.into()); + let tx = TransactionSource::Database { + transaction, + index, + block_hash, + block_number, + }; + (tx, at) + } + }; + Ok(Some(res)) + } + } + } } // === impl EthApi === @@ -140,6 +204,15 @@ pub enum TransactionSource { }, } +impl From for TransactionSignedEcRecovered { + fn from(value: TransactionSource) -> Self { + match value { + TransactionSource::Pool(tx) => tx, + TransactionSource::Database { transaction, .. } => transaction, + } + } +} + impl From for Transaction { fn from(value: TransactionSource) -> Self { match value { diff --git a/crates/rpc/rpc/src/trace.rs b/crates/rpc/rpc/src/trace.rs index d2e46474df9..b33dbb41195 100644 --- a/crates/rpc/rpc/src/trace.rs +++ b/crates/rpc/rpc/src/trace.rs @@ -1,4 +1,7 @@ -use crate::{eth::cache::EthStateCache, result::internal_rpc_err}; +use crate::{ + eth::{cache::EthStateCache, EthTransactions}, + result::internal_rpc_err, +}; use async_trait::async_trait; use jsonrpsee::core::RpcResult as Result; use reth_primitives::{BlockId, Bytes, H256}; @@ -14,26 +17,29 @@ use std::collections::HashSet; /// /// This type provides the functionality for handling `trace` related requests. #[derive(Clone)] -pub struct TraceApi { +pub struct TraceApi { /// The client that can interact with the chain. client: Client, + /// Access to commonly used code of the `eth` namespace + eth_api: Eth, /// The async cache frontend for eth related data eth_cache: EthStateCache, } // === impl TraceApi === -impl TraceApi { +impl TraceApi { /// Create a new instance of the [TraceApi] - pub fn new(client: Client, eth_cache: EthStateCache) -> Self { - Self { client, eth_cache } + pub fn new(client: Client, eth_api: Eth, eth_cache: EthStateCache) -> Self { + Self { client, eth_api, eth_cache } } } #[async_trait] -impl TraceApiServer for TraceApi +impl TraceApiServer for TraceApi where Client: BlockProvider + StateProviderFactory + EvmEnvProvider + 'static, + Eth: EthTransactions + 'static, { /// Handler for `trace_call` async fn trace_call( @@ -107,13 +113,20 @@ where /// Handler for `trace_transaction` async fn trace_transaction( &self, - _hash: H256, + hash: H256, ) -> Result>> { + let (_transaction, at) = match self.eth_api.transaction_by_hash_at(hash).await? { + None => return Ok(None), + Some(res) => res, + }; + + let (_cfg, _block_env, _at) = self.eth_api.evm_env_at(at).await?; + Err(internal_rpc_err("unimplemented")) } } -impl std::fmt::Debug for TraceApi { +impl std::fmt::Debug for TraceApi { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("TraceApi").finish_non_exhaustive() } From 06db495d969ad3aeefc9577d1e50f58f18670223 Mon Sep 17 00:00:00 2001 From: Chen Kai <281165273grape@gmail.com> Date: Wed, 15 Mar 2023 00:57:19 +0800 Subject: [PATCH 124/191] fix: use string instead of Vec for stageId. (#1495) Signed-off-by: Chen Kai <281165273grape@gmail.com> --- crates/stages/src/id.rs | 4 ++-- crates/storage/db/src/tables/mod.rs | 2 +- crates/storage/db/src/tables/models/mod.rs | 13 +++++++++++++ crates/storage/provider/src/providers/mod.rs | 2 +- 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/crates/stages/src/id.rs b/crates/stages/src/id.rs index 8feb0222133..2d4bfc971dc 100644 --- a/crates/stages/src/id.rs +++ b/crates/stages/src/id.rs @@ -27,7 +27,7 @@ impl StageId { /// Get the last committed progress of this stage. pub fn get_progress<'db>(&self, tx: &impl DbTx<'db>) -> Result, DbError> { - tx.get::(self.0.as_bytes().to_vec()) + tx.get::(self.0.to_string()) } /// Save the progress of this stage. @@ -36,7 +36,7 @@ impl StageId { tx: &impl DbTxMut<'db>, block: BlockNumber, ) -> Result<(), DbError> { - tx.put::(self.0.as_bytes().to_vec(), block) + tx.put::(self.0.to_string(), block) } } diff --git a/crates/storage/db/src/tables/mod.rs b/crates/storage/db/src/tables/mod.rs index bb45b8bf4e0..7d058403df5 100644 --- a/crates/storage/db/src/tables/mod.rs +++ b/crates/storage/db/src/tables/mod.rs @@ -305,7 +305,7 @@ table!( /// List with transaction numbers. pub type TransitionList = IntegerList; /// Encoded stage id. -pub type StageId = Vec; +pub type StageId = String; // // TODO: Temporary types, until they're properly defined alongside with the Encode and Decode Trait diff --git a/crates/storage/db/src/tables/models/mod.rs b/crates/storage/db/src/tables/models/mod.rs index 545eca6366f..5d98e702579 100644 --- a/crates/storage/db/src/tables/models/mod.rs +++ b/crates/storage/db/src/tables/models/mod.rs @@ -83,3 +83,16 @@ impl Decode for H256 { Ok(H256::from_slice(&value.into()[..])) } } + +impl Encode for String { + type Encoded = Vec; + fn encode(self) -> Self::Encoded { + self.into_bytes() + } +} + +impl Decode for String { + fn decode>(value: B) -> Result { + String::from_utf8(value.into().to_vec()).map_err(|_| Error::DecodeError) + } +} diff --git a/crates/storage/provider/src/providers/mod.rs b/crates/storage/provider/src/providers/mod.rs index a7539fa7330..ce50dba0cd0 100644 --- a/crates/storage/provider/src/providers/mod.rs +++ b/crates/storage/provider/src/providers/mod.rs @@ -106,7 +106,7 @@ impl BlockIdProvider for ShareableDatabase { fn chain_info(&self) -> Result { let best_number = self .db - .view(|tx| tx.get::("Finish".as_bytes().to_vec()))? + .view(|tx| tx.get::("Finish".to_string()))? .map_err(Into::::into)? .unwrap_or_default(); let best_hash = self.block_hash(U256::from(best_number))?.unwrap_or_default(); From 237fd5ce6ea25cfba20d7a4f9adc3c72aae9c541 Mon Sep 17 00:00:00 2001 From: rakita Date: Tue, 14 Mar 2023 19:17:14 +0100 Subject: [PATCH 125/191] feat: BlockchainTree (#1212) Co-authored-by: Dragan Rakita --- Cargo.lock | 2 + bin/reth/src/args/network_args.rs | 8 +- bin/reth/src/chain/import.rs | 4 +- bin/reth/src/chain/init.rs | 2 +- bin/reth/src/db/mod.rs | 4 +- bin/reth/src/node/mod.rs | 4 +- bin/reth/src/p2p/mod.rs | 2 +- bin/reth/src/stage/mod.rs | 4 +- bin/reth/src/test_eth_chain/runner.rs | 4 +- .../consensus/src/beacon/beacon_consensus.rs | 12 +- crates/consensus/src/beacon/builder.rs | 2 +- crates/executor/Cargo.toml | 15 +- .../src/blockchain_tree/block_indices.rs | 312 ++++++ crates/executor/src/blockchain_tree/chain.rs | 441 +++++++++ crates/executor/src/blockchain_tree/mod.rs | 918 ++++++++++++++++++ crates/executor/src/lib.rs | 2 + crates/executor/src/substate.rs | 306 ++++++ crates/interfaces/src/executor.rs | 32 +- crates/interfaces/src/provider.rs | 15 +- crates/net/network/src/config.rs | 10 +- crates/primitives/Cargo.toml | 1 + crates/primitives/src/block.rs | 73 +- crates/primitives/src/header.rs | 12 +- crates/primitives/src/lib.rs | 7 +- crates/primitives/src/transaction/mod.rs | 5 + crates/rpc/rpc-engine-api/src/engine_api.rs | 4 +- crates/staged-sync/src/utils/chainspec.rs | 14 +- crates/staged-sync/src/utils/init.rs | 14 +- crates/staged-sync/tests/sync.rs | 4 +- crates/stages/Cargo.toml | 2 +- crates/stages/src/stages/execution.rs | 12 +- crates/stages/src/stages/hashing_account.rs | 40 +- crates/stages/src/stages/hashing_storage.rs | 51 +- .../src/stages/index_account_history.rs | 92 +- .../src/stages/index_storage_history.rs | 107 +- crates/stages/src/stages/tx_lookup.rs | 4 +- crates/stages/src/test_utils/test_db.rs | 4 +- crates/storage/db/src/abstraction/common.rs | 6 +- crates/storage/db/src/abstraction/cursor.rs | 36 + crates/storage/db/src/abstraction/database.rs | 2 +- .../db/src/implementation/mdbx/cursor.rs | 8 +- crates/storage/db/src/tables/models/blocks.rs | 12 +- crates/storage/provider/Cargo.toml | 1 + .../storage/provider/src/execution_result.rs | 38 +- crates/storage/provider/src/providers/mod.rs | 10 +- .../src/providers/state/historical.rs | 2 +- .../provider/src/providers/state/latest.rs | 4 +- .../storage/provider/src/test_utils/blocks.rs | 146 +++ crates/storage/provider/src/test_utils/mod.rs | 1 + crates/storage/provider/src/traits/account.rs | 2 +- .../storage/provider/src/traits/block_hash.rs | 2 +- crates/storage/provider/src/transaction.rs | 907 ++++++++++++++++- crates/storage/provider/src/trie/mod.rs | 12 +- crates/storage/provider/src/utils.rs | 42 +- 54 files changed, 3409 insertions(+), 367 deletions(-) create mode 100644 crates/executor/src/blockchain_tree/block_indices.rs create mode 100644 crates/executor/src/blockchain_tree/chain.rs create mode 100644 crates/executor/src/blockchain_tree/mod.rs create mode 100644 crates/executor/src/substate.rs create mode 100644 crates/storage/provider/src/test_utils/blocks.rs diff --git a/Cargo.lock b/Cargo.lock index f3130d2b86f..89b60ecad7a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4598,9 +4598,11 @@ dependencies = [ name = "reth-executor" version = "0.1.0" dependencies = [ + "aquamarine", "async-trait", "auto_impl", "hash-db", + "parking_lot 0.12.1", "plain_hasher", "reth-db", "reth-interfaces", diff --git a/bin/reth/src/args/network_args.rs b/bin/reth/src/args/network_args.rs index 6bc3212df0d..a0af02ef985 100644 --- a/bin/reth/src/args/network_args.rs +++ b/bin/reth/src/args/network_args.rs @@ -7,7 +7,7 @@ use reth_net_nat::NatResolver; use reth_network::NetworkConfigBuilder; use reth_primitives::{ChainSpec, NodeRecord}; use reth_staged_sync::Config; -use std::path::PathBuf; +use std::{path::PathBuf, sync::Arc}; /// Parameters for configuring the network more granularity via CLI #[derive(Debug, Args)] @@ -50,7 +50,11 @@ pub struct NetworkArgs { impl NetworkArgs { /// Build a [`NetworkConfigBuilder`] from a [`Config`] and a [`ChainSpec`], in addition to the /// values in this option struct. - pub fn network_config(&self, config: &Config, chain_spec: ChainSpec) -> NetworkConfigBuilder { + pub fn network_config( + &self, + config: &Config, + chain_spec: Arc, + ) -> NetworkConfigBuilder { let peers_file = (!self.no_persist_peers).then_some(&self.peers_file); let network_config_builder = config .network_config(self.nat, peers_file.map(|f| f.as_ref().to_path_buf())) diff --git a/bin/reth/src/chain/import.rs b/bin/reth/src/chain/import.rs index 1e96c71ab3f..40c382b278d 100644 --- a/bin/reth/src/chain/import.rs +++ b/bin/reth/src/chain/import.rs @@ -63,7 +63,7 @@ pub struct ImportCommand { default_value = "mainnet", value_parser = genesis_value_parser )] - chain: ChainSpec, + chain: Arc, /// The path to a block file for import. /// @@ -140,7 +140,7 @@ impl ImportCommand { .build(file_client.clone(), consensus.clone(), db) .into_task(); - let factory = reth_executor::Factory::new(Arc::new(self.chain.clone())); + let factory = reth_executor::Factory::new(self.chain.clone()); let mut pipeline = Pipeline::builder() .with_sync_state_updater(file_client) diff --git a/bin/reth/src/chain/init.rs b/bin/reth/src/chain/init.rs index 4c1dbe2fd24..0d3f395634f 100644 --- a/bin/reth/src/chain/init.rs +++ b/bin/reth/src/chain/init.rs @@ -36,7 +36,7 @@ pub struct InitCommand { default_value = "mainnet", value_parser = genesis_value_parser )] - chain: ChainSpec, + chain: Arc, } impl InitCommand { diff --git a/bin/reth/src/db/mod.rs b/bin/reth/src/db/mod.rs index 2147954be8c..5e95213997a 100644 --- a/bin/reth/src/db/mod.rs +++ b/bin/reth/src/db/mod.rs @@ -207,8 +207,8 @@ impl<'a, DB: Database> DbTool<'a, DB> { let chain = random_block_range(0..len, Default::default(), 0..64); self.db.update(|tx| { - chain.iter().try_for_each(|block| { - insert_canonical_block(tx, block, true)?; + chain.into_iter().try_for_each(|block| { + insert_canonical_block(tx, block, None, true)?; Ok::<_, eyre::Error>(()) }) })??; diff --git a/bin/reth/src/node/mod.rs b/bin/reth/src/node/mod.rs index daeeb5461ef..6c8a325a98e 100644 --- a/bin/reth/src/node/mod.rs +++ b/bin/reth/src/node/mod.rs @@ -90,7 +90,7 @@ pub struct Command { default_value = "mainnet", value_parser = genesis_value_parser )] - chain: ChainSpec, + chain: Arc, /// Enable Prometheus metrics. /// @@ -443,7 +443,7 @@ impl Command { builder = builder.with_max_block(max_block) } - let factory = reth_executor::Factory::new(Arc::new(self.chain.clone())); + let factory = reth_executor::Factory::new(self.chain.clone()); let pipeline = builder .with_sync_state_updater(updater.clone()) .add_stages( diff --git a/bin/reth/src/p2p/mod.rs b/bin/reth/src/p2p/mod.rs index 268168de4cc..44960d76c66 100644 --- a/bin/reth/src/p2p/mod.rs +++ b/bin/reth/src/p2p/mod.rs @@ -39,7 +39,7 @@ pub struct Command { default_value = "mainnet", value_parser = chain_spec_value_parser )] - chain: ChainSpec, + chain: Arc, /// Disable the discovery service. #[command(flatten)] diff --git a/bin/reth/src/stage/mod.rs b/bin/reth/src/stage/mod.rs index 5c9ab23fd83..bcb079abf21 100644 --- a/bin/reth/src/stage/mod.rs +++ b/bin/reth/src/stage/mod.rs @@ -54,7 +54,7 @@ pub struct Command { default_value = "mainnet", value_parser = chain_spec_value_parser )] - chain: ChainSpec, + chain: Arc, /// Enable Prometheus metrics. /// @@ -171,7 +171,7 @@ impl Command { stage.execute(&mut tx, input).await?; } StageEnum::Execution => { - let factory = reth_executor::Factory::new(Arc::new(self.chain.clone())); + let factory = reth_executor::Factory::new(self.chain.clone()); let mut stage = ExecutionStage::new(factory, 10_000); stage.commit_threshold = num_blocks; if !self.skip_unwind { diff --git a/bin/reth/src/test_eth_chain/runner.rs b/bin/reth/src/test_eth_chain/runner.rs index 61025f6fa6e..d65760fb55a 100644 --- a/bin/reth/src/test_eth_chain/runner.rs +++ b/bin/reth/src/test_eth_chain/runner.rs @@ -140,13 +140,13 @@ pub async fn run_test(path: PathBuf) -> eyre::Result { // insert genesis let header: SealedHeader = suite.genesis_block_header.into(); let genesis_block = SealedBlock { header, body: vec![], ommers: vec![], withdrawals: None }; - reth_provider::insert_canonical_block(&tx, &genesis_block, has_block_reward)?; + reth_provider::insert_canonical_block(&tx, genesis_block, None, has_block_reward)?; let mut last_block = None; suite.blocks.iter().try_for_each(|block| -> eyre::Result<()> { let decoded = SealedBlock::decode(&mut block.rlp.as_ref())?; - reth_provider::insert_canonical_block(&tx, &decoded, has_block_reward)?; last_block = Some(decoded.number); + reth_provider::insert_canonical_block(&tx, decoded, None, has_block_reward)?; Ok(()) })?; diff --git a/crates/consensus/src/beacon/beacon_consensus.rs b/crates/consensus/src/beacon/beacon_consensus.rs index b2efe58b093..d39895c3795 100644 --- a/crates/consensus/src/beacon/beacon_consensus.rs +++ b/crates/consensus/src/beacon/beacon_consensus.rs @@ -1,4 +1,6 @@ //! Consensus for ethereum network +use std::sync::Arc; + use crate::validation; use reth_interfaces::consensus::{Consensus, ConsensusError, ForkchoiceState}; use reth_primitives::{ChainSpec, Hardfork, SealedBlock, SealedHeader, EMPTY_OMMER_ROOT, U256}; @@ -15,13 +17,13 @@ pub struct BeaconConsensus { /// Watcher over the forkchoice state forkchoice_state_rx: watch::Receiver, /// Configuration - chain_spec: ChainSpec, + chain_spec: Arc, } impl BeaconConsensus { /// Create a new instance of [BeaconConsensus] pub fn new( - chain_spec: ChainSpec, + chain_spec: Arc, forkchoice_state_rx: watch::Receiver, ) -> Self { Self { chain_spec, forkchoice_state_rx } @@ -92,14 +94,14 @@ impl Consensus for BeaconConsensus { #[cfg(test)] mod test { + use super::BeaconConsensus; use reth_interfaces::consensus::Consensus; use reth_primitives::{ChainSpecBuilder, U256}; - - use super::BeaconConsensus; + use std::sync::Arc; #[test] fn test_has_block_reward_before_paris() { - let chain_spec = ChainSpecBuilder::mainnet().build(); + let chain_spec = Arc::new(ChainSpecBuilder::mainnet().build()); let (consensus, _) = BeaconConsensus::builder().build(chain_spec); assert!(consensus.has_block_reward(U256::ZERO, U256::ZERO)); } diff --git a/crates/consensus/src/beacon/builder.rs b/crates/consensus/src/beacon/builder.rs index d3f2bd97b45..3636e8f95a3 100644 --- a/crates/consensus/src/beacon/builder.rs +++ b/crates/consensus/src/beacon/builder.rs @@ -13,7 +13,7 @@ impl BeaconConsensusBuilder { /// [watch::channel] for updating the forkchoice state. pub fn build( self, - chain_spec: ChainSpec, + chain_spec: Arc, ) -> (Arc, watch::Sender) { let (forkchoice_state_tx, forkchoice_state_rx) = watch::channel(ForkchoiceState::default()); let inner = Arc::new(BeaconConsensus::new(chain_spec, forkchoice_state_rx)); diff --git a/crates/executor/Cargo.toml b/crates/executor/Cargo.toml index 095aa06084c..c56c86bb044 100644 --- a/crates/executor/Cargo.toml +++ b/crates/executor/Cargo.toml @@ -6,6 +6,12 @@ license = "MIT OR Apache-2.0" repository = "https://github.com/paradigmxyz/reth" readme = "README.md" +[package.metadata.cargo-udeps.ignore] +normal = [ + # Used for diagrams in docs + "aquamarine", +] + [dependencies] # reth reth-primitives = { path = "../primitives" } @@ -17,7 +23,7 @@ reth-db = { path = "../storage/db" } reth-provider = { path = "../storage/provider" } # revm -revm = { version = "3.0.0"} +revm = { version = "3.0.0" } # common async-trait = "0.1.57" @@ -26,6 +32,9 @@ auto_impl = "1.0" tracing = "0.1.37" tokio = { version = "1.21.2", features = ["sync"] } +# mics +aquamarine = "0.2.1" #docs + triehash = "0.8" # See to replace hashers to simplify libraries plain_hasher = "0.2" @@ -38,3 +47,7 @@ sha3 = { version = "0.10", default-features = false } [dev-dependencies] reth-db = { path = "../storage/db", features = ["test-utils"] } +reth-interfaces = { path = "../interfaces", features = ["test-utils"] } +reth-primitives = { path = "../primitives", features = ["test-utils"] } +reth-provider = { path = "../storage/provider", features = ["test-utils"] } +parking_lot = "0.12" diff --git a/crates/executor/src/blockchain_tree/block_indices.rs b/crates/executor/src/blockchain_tree/block_indices.rs new file mode 100644 index 00000000000..0a96fc89d9f --- /dev/null +++ b/crates/executor/src/blockchain_tree/block_indices.rs @@ -0,0 +1,312 @@ +//! Implementation of [`BlockIndices`] related to [`super::BlockchainTree`] + +use super::chain::{BlockChainId, Chain, ForkBlock}; +use reth_primitives::{BlockHash, BlockNumber, SealedBlockWithSenders}; +use std::collections::{hash_map::Entry, BTreeMap, BTreeSet, HashMap, HashSet}; + +/// Internal indices of the blocks and chains. This is main connection +/// between blocks, chains and canonical chain. +/// +/// It contains list of canonical block hashes, forks to childs blocks +/// and block hash to chain id. +pub struct BlockIndices { + /// Last finalized block. + last_finalized_block: BlockNumber, + /// For EVM's "BLOCKHASH" opcode we require last 256 block hashes. So we need to specify + /// at least `additional_canonical_block_hashes`+`max_reorg_depth`, for eth that would be + /// 256+64. + num_of_additional_canonical_block_hashes: u64, + /// Canonical chain. Contains N number (depends on `finalization_depth`) of blocks. + /// These blocks are found in fork_to_child but not inside `blocks_to_chain` or + /// `number_to_block` as those are chain specific indices. + canonical_chain: BTreeMap, + /// Index needed when discarding the chain, so we can remove connected chains from tree. + /// NOTE: It contains just a blocks that are forks as a key and not all blocks. + fork_to_child: HashMap>, + /// Block hashes and side chain they belong + blocks_to_chain: HashMap, + /// Utility index. Block number to block hash. Can be used for + /// RPC to fetch all pending block in chain by its number. + index_number_to_block: HashMap>, +} + +impl BlockIndices { + /// Create new block indices structure + pub fn new( + last_finalized_block: BlockNumber, + num_of_additional_canonical_block_hashes: u64, + canonical_chain: BTreeMap, + ) -> Self { + Self { + last_finalized_block, + num_of_additional_canonical_block_hashes, + fork_to_child: Default::default(), + canonical_chain, + blocks_to_chain: Default::default(), + index_number_to_block: Default::default(), + } + } + + /// Return number of additional canonical block hashes that we need + /// to have to be able to have enought information for EVM execution. + pub fn num_of_additional_canonical_block_hashes(&self) -> u64 { + self.num_of_additional_canonical_block_hashes + } + + /// Return fork to child indices + pub fn fork_to_child(&self) -> &HashMap> { + &self.fork_to_child + } + + /// Return block to chain id + pub fn blocks_to_chain(&self) -> &HashMap { + &self.blocks_to_chain + } + + /// Returns `true` if the Tree knowns the block hash. + pub fn contains_pending_block_hash(&self, block_hash: BlockHash) -> bool { + self.blocks_to_chain.contains_key(&block_hash) + } + + /// Check if block hash belongs to canonical chain. + pub fn is_block_hash_canonical(&self, block_hash: &BlockHash) -> bool { + self.canonical_chain.range(self.last_finalized_block..).any(|(_, &h)| h == *block_hash) + } + + /// Last finalized block + pub fn last_finalized_block(&self) -> BlockNumber { + self.last_finalized_block + } + + /// Insert non fork block. + pub fn insert_non_fork_block( + &mut self, + block_number: BlockNumber, + block_hash: BlockHash, + chain_id: BlockChainId, + ) { + self.index_number_to_block.entry(block_number).or_default().insert(block_hash); + self.blocks_to_chain.insert(block_hash, chain_id); + } + + /// Insert block to chain and fork child indices of the new chain + pub fn insert_chain(&mut self, chain_id: BlockChainId, chain: &Chain) { + for (number, block) in chain.blocks().iter() { + // add block -> chain_id index + self.blocks_to_chain.insert(block.hash(), chain_id); + // add number -> block + self.index_number_to_block.entry(*number).or_default().insert(block.hash()); + } + let first = chain.first(); + // add parent block -> block index + self.fork_to_child.entry(first.parent_hash).or_default().insert(first.hash()); + } + + /// Get the chain ID the block belongs to + pub fn get_blocks_chain_id(&self, block: &BlockHash) -> Option { + self.blocks_to_chain.get(block).cloned() + } + + /// Update all block hashes. iterate over present and new list of canonical hashes and compare + /// them. Remove all missmatches, disconnect them and return all chains that needs to be + /// removed. + pub fn update_block_hashes( + &mut self, + hashes: BTreeMap, + ) -> BTreeSet { + let mut new_hashes = hashes.iter(); + let mut old_hashes = self.canonical_chain().clone().into_iter(); + + let mut remove = Vec::new(); + + let mut new_hash = new_hashes.next(); + let mut old_hash = old_hashes.next(); + + loop { + let Some(old_block_value) = old_hash else { + // end of old_hashes canonical chain. New chain has more block then old chain. + break + }; + let Some(new_block_value) = new_hash else { + // Old canonical chain had more block than new chain. + // remove all present block. + // this is mostly not going to happen as reorg should make new chain in Tree. + while let Some(rem) = old_hash { + remove.push(rem); + old_hash = old_hashes.next(); + } + break; + }; + // compare old and new canonical block number + match new_block_value.0.cmp(&old_block_value.0) { + std::cmp::Ordering::Less => { + // new chain has more past blocks than old chain + new_hash = new_hashes.next(); + } + std::cmp::Ordering::Equal => { + if *new_block_value.1 != old_block_value.1 { + // remove block hash as it is different + remove.push(old_block_value); + } + new_hash = new_hashes.next(); + old_hash = old_hashes.next(); + } + std::cmp::Ordering::Greater => { + // old chain has more past blocks that new chain + remove.push(old_block_value); + old_hash = old_hashes.next() + } + } + } + self.canonical_chain = hashes; + + remove.into_iter().fold(BTreeSet::new(), |mut fold, (number, hash)| { + fold.extend(self.remove_block(number, hash)); + fold + }) + } + + /// Remove chain from indices and return dependent chains that needs to be removed. + /// Does the cleaning of the tree and removing blocks from the chain. + pub fn remove_chain(&mut self, chain: &Chain) -> BTreeSet { + let mut lose_chains = BTreeSet::new(); + for (block_number, block) in chain.blocks().iter() { + let block_hash = block.hash(); + lose_chains.extend(self.remove_block(*block_number, block_hash)) + } + lose_chains + } + + /// Remove Blocks from indices. + fn remove_block( + &mut self, + block_number: BlockNumber, + block_hash: BlockHash, + ) -> BTreeSet { + // rm number -> block + if let Entry::Occupied(mut entry) = self.index_number_to_block.entry(block_number) { + let set = entry.get_mut(); + set.remove(&block_hash); + // remove set if empty + if set.is_empty() { + entry.remove(); + } + } + + // rm block -> chain_id + self.blocks_to_chain.remove(&block_hash); + + // rm fork -> child + let removed_fork = self.fork_to_child.remove(&block_hash); + removed_fork + .map(|fork_blocks| { + fork_blocks + .into_iter() + .filter_map(|fork_child| self.blocks_to_chain.remove(&fork_child)) + .collect() + }) + .unwrap_or_default() + } + + /// Remove all blocks from canonical list and insert new blocks to it. + /// + /// It is assumed that blocks are interconnected and that they connect to canonical chain + pub fn canonicalize_blocks(&mut self, blocks: &BTreeMap) { + if blocks.is_empty() { + return + } + + // Remove all blocks from canonical chain + let first_number = *blocks.first_key_value().unwrap().0; + + // this will remove all blocks numbers that are going to be replaced. + self.canonical_chain.retain(|num, _| *num < first_number); + + // remove them from block to chain_id index + blocks.iter().map(|(_, b)| (b.number, b.hash(), b.parent_hash)).for_each( + |(number, hash, parent_hash)| { + // rm block -> chain_id + self.blocks_to_chain.remove(&hash); + + // rm number -> block + if let Entry::Occupied(mut entry) = self.index_number_to_block.entry(number) { + let set = entry.get_mut(); + set.remove(&hash); + // remove set if empty + if set.is_empty() { + entry.remove(); + } + } + // rm fork block -> hash + if let Entry::Occupied(mut entry) = self.fork_to_child.entry(parent_hash) { + let set = entry.get_mut(); + set.remove(&hash); + // remove set if empty + if set.is_empty() { + entry.remove(); + } + } + }, + ); + + // insert new canonical + self.canonical_chain.extend(blocks.iter().map(|(number, block)| (*number, block.hash()))) + } + + /// Used for finalization of block. + /// Return list of chains for removal that depend on finalized canonical chain. + pub fn finalize_canonical_blocks( + &mut self, + finalized_block: BlockNumber, + ) -> BTreeSet { + // get finalized chains. blocks between [self.last_finalized,finalized_block). + // Dont remove finalized_block, as sidechain can point to it. + let finalized_blocks: Vec = self + .canonical_chain + .iter() + .filter(|(&number, _)| number >= self.last_finalized_block && number < finalized_block) + .map(|(_, hash)| *hash) + .collect(); + + // remove unneeded canonical hashes. + let remove_until = + finalized_block.saturating_sub(self.num_of_additional_canonical_block_hashes); + self.canonical_chain.retain(|&number, _| number >= remove_until); + + let mut lose_chains = BTreeSet::new(); + + for block_hash in finalized_blocks.into_iter() { + // there is a fork block. + if let Some(fork_blocks) = self.fork_to_child.remove(&block_hash) { + lose_chains = fork_blocks.into_iter().fold(lose_chains, |mut fold, fork_child| { + if let Some(lose_chain) = self.blocks_to_chain.remove(&fork_child) { + fold.insert(lose_chain); + } + fold + }); + } + } + + // set last finalized block. + self.last_finalized_block = finalized_block; + + lose_chains + } + + /// get canonical hash + pub fn canonical_hash(&self, block_number: &BlockNumber) -> Option { + self.canonical_chain.get(block_number).cloned() + } + + /// get canonical tip + pub fn canonical_tip(&self) -> ForkBlock { + let (&number, &hash) = + self.canonical_chain.last_key_value().expect("There is always the canonical chain"); + ForkBlock { number, hash } + } + + /// Canonical chain needs for execution of EVM. It should contains last 256 block hashes. + pub fn canonical_chain(&self) -> &BTreeMap { + &self.canonical_chain + } +} diff --git a/crates/executor/src/blockchain_tree/chain.rs b/crates/executor/src/blockchain_tree/chain.rs new file mode 100644 index 00000000000..8267f8cc328 --- /dev/null +++ b/crates/executor/src/blockchain_tree/chain.rs @@ -0,0 +1,441 @@ +//! Handles substate and list of blocks. +//! have functions to split, branch and append the chain. +use crate::{ + execution_result::ExecutionResult, + substate::{SubStateData, SubStateWithProvider}, +}; +use reth_interfaces::{consensus::Consensus, executor::Error as ExecError, Error}; +use reth_primitives::{BlockHash, BlockNumber, SealedBlockWithSenders, SealedHeader, U256}; +use reth_provider::{BlockExecutor, ExecutorFactory, StateProvider}; +use std::collections::BTreeMap; + +/// Internal to BlockchainTree chain identification. +pub(crate) type BlockChainId = u64; + +/// Side chain that contain it state and connect to block found in canonical chain. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Chain { + /// Chain substate. Updated state after execution all blocks in chain. + substate: SubStateData, + /// Changesets for block and transaction. Will be used to update tables in database. + changesets: Vec, + /// Blocks in this chain + blocks: BTreeMap, +} + +/// Contains fork block and hash. +#[derive(Clone, Copy)] +pub struct ForkBlock { + /// Block number of block that chains branches from + pub number: u64, + /// Block hash of block that chains branches from + pub hash: BlockHash, +} + +impl ForkBlock { + /// Return the number hash tuple. + pub fn num_hash(&self) -> (BlockNumber, BlockHash) { + (self.number, self.hash) + } +} + +impl Chain { + /// Return blocks found in chain + pub fn blocks(&self) -> &BTreeMap { + &self.blocks + } + + /// Into inner components + pub fn into_inner( + self, + ) -> (BTreeMap, Vec, SubStateData) { + (self.blocks, self.changesets, self.substate) + } + + /// Return execution results of blocks + pub fn changesets(&self) -> &Vec { + &self.changesets + } + + /// Return fork block number and hash. + pub fn fork_block(&self) -> ForkBlock { + let tip = self.first(); + ForkBlock { number: tip.number.saturating_sub(1), hash: tip.parent_hash } + } + + /// Block fork number + pub fn fork_block_number(&self) -> BlockNumber { + self.first().number.saturating_sub(1) + } + + /// Block fork hash + pub fn fork_block_hash(&self) -> BlockHash { + self.first().parent_hash + } + + /// First block in chain. + pub fn first(&self) -> &SealedBlockWithSenders { + self.blocks.first_key_value().expect("Chain has at least one block for first").1 + } + + /// Return tip of the chain. Chain always have at least one block inside + pub fn tip(&self) -> &SealedBlockWithSenders { + self.last() + } + + /// Return tip of the chain. Chain always have at least one block inside + pub fn last(&self) -> &SealedBlockWithSenders { + self.blocks.last_key_value().expect("Chain has at least one block for last").1 + } + + /// Create new chain with given blocks and execution result. + pub fn new(blocks: Vec<(SealedBlockWithSenders, ExecutionResult)>) -> Self { + let (blocks, changesets): (Vec<_>, Vec<_>) = blocks.into_iter().unzip(); + + let blocks = blocks.into_iter().map(|b| (b.number, b)).collect::>(); + + let mut substate = SubStateData::default(); + substate.apply(&changesets); + + Self { substate, changesets, blocks } + } + + /// Create new chain that joins canonical block + /// If parent block is the tip mark chain fork. + pub fn new_canonical_fork( + block: &SealedBlockWithSenders, + parent_header: &SealedHeader, + canonical_block_hashes: &BTreeMap, + provider: &SP, + consensus: &C, + factory: &EF, + ) -> Result { + // substate + let substate = SubStateData::default(); + let empty = BTreeMap::new(); + + let substate_with_sp = + SubStateWithProvider::new(&substate, provider, &empty, canonical_block_hashes); + + let changeset = Self::validate_and_execute( + block.clone(), + parent_header, + substate_with_sp, + consensus, + factory, + )?; + + Ok(Self::new(vec![(block.clone(), changeset)])) + } + + /// Create new chain that branches out from existing side chain. + pub fn new_chain_fork( + &self, + block: SealedBlockWithSenders, + side_chain_block_hashes: BTreeMap, + canonical_block_hashes: &BTreeMap, + provider: &SP, + consensus: &C, + factory: &EF, + ) -> Result { + let parent_number = block.number - 1; + let parent = self + .blocks + .get(&parent_number) + .ok_or(ExecError::BlockNumberNotFoundInChain { block_number: parent_number })?; + + // revert changesets + let revert_from = self.changesets.len() - (self.tip().number - parent.number) as usize; + let mut substate = self.substate.clone(); + + // Revert changesets to get the state of the parent that we need to apply the change. + substate.revert(&self.changesets[revert_from..]); + + let substate_with_sp = SubStateWithProvider::new( + &substate, + provider, + &side_chain_block_hashes, + canonical_block_hashes, + ); + let changeset = Self::validate_and_execute( + block.clone(), + parent, + substate_with_sp, + consensus, + factory, + )?; + substate.apply_one(&changeset); + + let chain = Self { + substate, + changesets: vec![changeset], + blocks: BTreeMap::from([(block.number, block)]), + }; + + // if all is okay, return new chain back. Present chain is not modified. + Ok(chain) + } + + /// Validate and execute block and return execution result or error. + fn validate_and_execute( + block: SealedBlockWithSenders, + parent_block: &SealedHeader, + substate: SubStateWithProvider<'_, SP>, + consensus: &C, + factory: &EF, + ) -> Result { + consensus.validate_header(&block, U256::MAX)?; + consensus.pre_validate_header(&block, parent_block)?; + consensus.pre_validate_block(&block)?; + + let (unseal, senders) = block.into_components(); + let unseal = unseal.unseal(); + let res = factory.with_sp(substate).execute_and_verify_receipt( + &unseal, + U256::MAX, + Some(senders), + )?; + Ok(res) + } + + /// Append block to this chain + pub fn append_block( + &mut self, + block: SealedBlockWithSenders, + side_chain_block_hashes: BTreeMap, + canonical_block_hashes: &BTreeMap, + provider: &SP, + consensus: &C, + factory: &EF, + ) -> Result<(), Error> { + let (_, parent_block) = self.blocks.last_key_value().expect("Chain has at least one block"); + + let changeset = Self::validate_and_execute( + block.clone(), + parent_block, + SubStateWithProvider::new( + &self.substate, + provider, + &side_chain_block_hashes, + canonical_block_hashes, + ), + consensus, + factory, + )?; + self.substate.apply_one(&changeset); + self.changesets.push(changeset); + self.blocks.insert(block.number, block); + Ok(()) + } + + /// Merge two chains into one by appending received chain to the current one. + /// Take substate from newest one. + pub fn append_chain(&mut self, chain: Chain) -> Result<(), Error> { + let chain_tip = self.tip(); + if chain_tip.hash != chain.fork_block_hash() { + return Err(ExecError::AppendChainDoesntConnect { + chain_tip: chain_tip.num_hash(), + other_chain_fork: chain.fork_block().num_hash(), + } + .into()) + } + self.blocks.extend(chain.blocks.into_iter()); + self.changesets.extend(chain.changesets.into_iter()); + self.substate = chain.substate; + Ok(()) + } + + /// Split chain at the number or hash, block with given number will be included at first chain. + /// If any chain is empty (Does not have blocks) None will be returned. + /// + /// If block hash is not found ChainSplit::NoSplitPending is returned. + /// + /// Subtate state will be only found in second chain. First change substate will be + /// invalid. + pub fn split(mut self, split_at: SplitAt) -> ChainSplit { + let chain_tip = *self.blocks.last_entry().expect("chain is never empty").key(); + let block_number = match split_at { + SplitAt::Hash(block_hash) => { + let block_number = self.blocks.iter().find_map(|(num, block)| { + if block.hash() == block_hash { + Some(*num) + } else { + None + } + }); + let Some(block_number) = block_number else { return ChainSplit::NoSplitPending(self)}; + // If block number is same as tip whole chain is becoming canonical. + if block_number == chain_tip { + return ChainSplit::NoSplitCanonical(self) + } + block_number + } + SplitAt::Number(block_number) => { + if block_number >= chain_tip { + return ChainSplit::NoSplitCanonical(self) + } + if block_number < *self.blocks.first_entry().expect("chain is never empty").key() { + return ChainSplit::NoSplitPending(self) + } + block_number + } + }; + + let higher_number_blocks = self.blocks.split_off(&(block_number + 1)); + let (first_changesets, second_changeset) = self.changesets.split_at(self.blocks.len()); + + ChainSplit::Split { + canonical: Chain { + substate: SubStateData::default(), + changesets: first_changesets.to_vec(), + blocks: self.blocks, + }, + pending: Chain { + substate: self.substate, + changesets: second_changeset.to_vec(), + blocks: higher_number_blocks, + }, + } + } +} + +/// Used in spliting the chain. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SplitAt { + /// Split at block number. + Number(BlockNumber), + /// Split at block hash. + Hash(BlockHash), +} + +/// Result of spliting chain. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ChainSplit { + /// Chain is not splited. Pending chain is returned. + /// Given block split is higher than last block. + /// Or in case of split by hash when hash is unknown. + NoSplitPending(Chain), + /// Chain is not splited. Canonical chain is returned. + /// Given block split is lower than first block. + NoSplitCanonical(Chain), + /// Chain is splited in two. + /// Given block split is contained in first chain. + Split { + /// Left contains lower block number that get canonicalized. + /// And substate is empty and not usable. + canonical: Chain, + /// Right contains higher block number, that is still pending. + /// And substate from original chain is moved here. + pending: Chain, + }, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::substate::AccountSubState; + use reth_primitives::{H160, H256}; + use reth_provider::execution_result::AccountInfoChangeSet; + + #[test] + fn chain_apend() { + let block = SealedBlockWithSenders::default(); + let block1_hash = H256([0x01; 32]); + let block2_hash = H256([0x02; 32]); + let block3_hash = H256([0x03; 32]); + let block4_hash = H256([0x04; 32]); + + let mut block1 = block.clone(); + let mut block2 = block.clone(); + let mut block3 = block.clone(); + let mut block4 = block.clone(); + + block1.block.header.hash = block1_hash; + block2.block.header.hash = block2_hash; + block3.block.header.hash = block3_hash; + block4.block.header.hash = block4_hash; + + block3.block.header.header.parent_hash = block2_hash; + + let mut chain1 = Chain { + substate: Default::default(), + changesets: vec![], + blocks: BTreeMap::from([(1, block1.clone()), (2, block2.clone())]), + }; + + let chain2 = Chain { + substate: Default::default(), + changesets: vec![], + blocks: BTreeMap::from([(3, block3.clone()), (4, block4.clone())]), + }; + + assert_eq!(chain1.append_chain(chain2.clone()), Ok(())); + + // chain1 got changed so this will fail + assert!(chain1.append_chain(chain2).is_err()); + } + + #[test] + fn test_number_split() { + let mut substate = SubStateData::default(); + let mut account = AccountSubState::default(); + account.info.nonce = 10; + substate.accounts.insert(H160([1; 20]), account); + + let mut exec1 = ExecutionResult::default(); + exec1.block_changesets.insert(H160([2; 20]), AccountInfoChangeSet::default()); + let mut exec2 = ExecutionResult::default(); + exec2.block_changesets.insert(H160([3; 20]), AccountInfoChangeSet::default()); + + let mut block1 = SealedBlockWithSenders::default(); + let block1_hash = H256([15; 32]); + block1.hash = block1_hash; + block1.senders.push(H160([4; 20])); + + let mut block2 = SealedBlockWithSenders::default(); + let block2_hash = H256([16; 32]); + block2.hash = block2_hash; + block2.senders.push(H160([4; 20])); + + let chain = Chain { + substate: substate.clone(), + changesets: vec![exec1.clone(), exec2.clone()], + blocks: BTreeMap::from([(1, block1.clone()), (2, block2.clone())]), + }; + + let chain_split1 = Chain { + substate: SubStateData::default(), + changesets: vec![exec1], + blocks: BTreeMap::from([(1, block1.clone())]), + }; + + let chain_split2 = Chain { + substate, + changesets: vec![exec2.clone()], + blocks: BTreeMap::from([(2, block2.clone())]), + }; + + // split in two + assert_eq!( + chain.clone().split(SplitAt::Hash(block1_hash)), + ChainSplit::Split { canonical: chain_split1.clone(), pending: chain_split2.clone() } + ); + + // split at unknown block hash + assert_eq!( + chain.clone().split(SplitAt::Hash(H256([100; 32]))), + ChainSplit::NoSplitPending(chain.clone()) + ); + + // split at higher number + assert_eq!( + chain.clone().split(SplitAt::Number(10)), + ChainSplit::NoSplitCanonical(chain.clone()) + ); + // split at lower number + assert_eq!( + chain.clone().split(SplitAt::Number(0)), + ChainSplit::NoSplitPending(chain.clone()) + ); + } +} diff --git a/crates/executor/src/blockchain_tree/mod.rs b/crates/executor/src/blockchain_tree/mod.rs new file mode 100644 index 00000000000..ff3ee144d12 --- /dev/null +++ b/crates/executor/src/blockchain_tree/mod.rs @@ -0,0 +1,918 @@ +//! Implementation of [`BlockchainTree`] +pub mod block_indices; +pub mod chain; + +use self::{ + block_indices::BlockIndices, + chain::{ChainSplit, SplitAt}, +}; +use chain::{BlockChainId, Chain, ForkBlock}; +use reth_db::{cursor::DbCursorRO, database::Database, tables, transaction::DbTx}; +use reth_interfaces::{consensus::Consensus, executor::Error as ExecError, Error}; +use reth_primitives::{BlockHash, BlockNumber, ChainSpec, SealedBlock, SealedBlockWithSenders}; +use reth_provider::{ + ExecutorFactory, HeaderProvider, ShareableDatabase, StateProvider, StateProviderFactory, + Transaction, +}; +use std::{ + collections::{BTreeMap, HashMap}, + sync::Arc, +}; + +#[cfg_attr(doc, aquamarine::aquamarine)] +/// Tree of chains and its identifications. +/// +/// Mermaid flowchart represent all blocks that can appear in blockchain. +/// Green blocks belong to canonical chain and are saved inside database table, they are our main +/// chain. Pending blocks and sidechains are found in memory inside [`BlockchainTree`]. +/// Both pending and sidechains have same mechanisms only difference is when they got committed to +/// database. For pending it is just append operation but for sidechains they need to move current +/// canonical blocks to BlockchainTree flush sidechain to the database to become canonical chain. +/// ```mermaid +/// flowchart BT +/// subgraph canonical chain +/// CanonState:::state +/// block0canon:::canon -->block1canon:::canon -->block2canon:::canon -->block3canon:::canon --> block4canon:::canon --> block5canon:::canon +/// end +/// block5canon --> block6pending1:::pending +/// block5canon --> block6pending2:::pending +/// subgraph sidechain2 +/// S2State:::state +/// block3canon --> block4s2:::sidechain --> block5s2:::sidechain +/// end +/// subgraph sidechain1 +/// S1State:::state +/// block2canon --> block3s1:::sidechain --> block4s1:::sidechain --> block5s1:::sidechain --> block6s1:::sidechain +/// end +/// classDef state fill:#1882C4 +/// classDef canon fill:#8AC926 +/// classDef pending fill:#FFCA3A +/// classDef sidechain fill:#FF595E +/// ``` +/// +/// +/// main functions: +/// * insert_block: Connect block to chain, execute it and if valid insert block inside tree. +/// * finalize_block: Remove chains that join to now finalized block, as chain becomes invalid. +/// * make_canonical: Check if we have the hash of block that we want to finalize and commit it to +/// db. If we dont have the block, pipeline syncing should start to fetch the blocks from p2p. Do +/// reorg in tables if canonical chain if needed. + +pub struct BlockchainTree { + /// chains and present data + chains: HashMap, + /// Static blockchain id generator + block_chain_id_generator: u64, + /// Indices to block and their connection. + block_indices: BlockIndices, + /// Number of block after finalized block that we are storing. It should be more then + /// finalization window + max_blocks_in_chain: u64, + /// Finalization windows. Number of blocks that can be reorged + max_reorg_depth: u64, + /// Externals + externals: Externals, +} + +/// Container for external abstractions. +struct Externals { + /// Save sidechain, do reorgs and push new block to canonical chain that is inside db. + db: DB, + /// Consensus checks + consensus: C, + /// Create executor to execute blocks. + executor_factory: EF, + /// Chain spec + chain_spec: Arc, +} + +impl Externals { + /// Return sharable database helper structure. + fn sharable_db(&self) -> ShareableDatabase<&DB> { + ShareableDatabase::new(&self.db, self.chain_spec.clone()) + } +} + +/// Helper structure that wraps chains and indices to search for block hash accross the chains. +pub struct BlockHashes<'a> { + /// Chains + pub chains: &'a mut HashMap, + /// Indices + pub indices: &'a BlockIndices, +} + +impl BlockchainTree { + /// New blockchain tree + pub fn new( + db: DB, + consensus: C, + executor_factory: EF, + chain_spec: Arc, + max_reorg_depth: u64, + max_blocks_in_chain: u64, + num_of_additional_canonical_block_hashes: u64, + ) -> Result { + if max_reorg_depth > max_blocks_in_chain { + panic!("Side chain size should be more then finalization window"); + } + + let last_canonical_hashes = db + .tx()? + .cursor_read::()? + .walk_back(None)? + .take((max_reorg_depth + num_of_additional_canonical_block_hashes) as usize) + .collect::, _>>()?; + + // TODO(rakita) save last finalized block inside database but for now just take + // tip-max_reorg_depth + // task: https://github.com/paradigmxyz/reth/issues/1712 + let (last_finalized_block_number, _) = + if last_canonical_hashes.len() > max_reorg_depth as usize { + last_canonical_hashes[max_reorg_depth as usize] + } else { + // it is in reverse order from tip to N + last_canonical_hashes.last().cloned().unwrap_or_default() + }; + + let externals = Externals { db, consensus, executor_factory, chain_spec }; + + Ok(Self { + externals, + block_chain_id_generator: 0, + chains: Default::default(), + block_indices: BlockIndices::new( + last_finalized_block_number, + num_of_additional_canonical_block_hashes, + BTreeMap::from_iter(last_canonical_hashes.into_iter()), + ), + max_blocks_in_chain, + max_reorg_depth, + }) + } + + /// Fork side chain or append the block if parent is the top of the chain + fn fork_side_chain( + &mut self, + block: SealedBlockWithSenders, + chain_id: BlockChainId, + ) -> Result<(), Error> { + let block_hashes = self.all_chain_hashes(chain_id); + + // get canonical fork. + let canonical_fork = + self.canonical_fork(chain_id).ok_or(ExecError::BlockChainIdConsistency { chain_id })?; + + // get chain that block needs to join to. + let parent_chain = self + .chains + .get_mut(&chain_id) + .ok_or(ExecError::BlockChainIdConsistency { chain_id })?; + let chain_tip = parent_chain.tip().hash(); + + let canonical_block_hashes = self.block_indices.canonical_chain(); + + // get canonical tip + let (_, canonical_tip_hash) = + canonical_block_hashes.last_key_value().map(|(i, j)| (*i, *j)).unwrap_or_default(); + + let db = self.externals.sharable_db(); + let provider = if canonical_fork.hash == canonical_tip_hash { + Box::new(db.latest()?) as Box + } else { + Box::new(db.history_by_block_number(canonical_fork.number)?) as Box + }; + + // append the block if it is continuing the chain. + if chain_tip == block.parent_hash { + let block_hash = block.hash(); + let block_number = block.number; + parent_chain.append_block( + block, + block_hashes, + canonical_block_hashes, + &provider, + &self.externals.consensus, + &self.externals.executor_factory, + )?; + drop(provider); + self.block_indices.insert_non_fork_block(block_number, block_hash, chain_id) + } else { + let chain = parent_chain.new_chain_fork( + block, + block_hashes, + canonical_block_hashes, + &provider, + &self.externals.consensus, + &self.externals.executor_factory, + )?; + // release the lifetime with a drop + drop(provider); + self.insert_chain(chain); + } + + Ok(()) + } + + /// Fork canonical chain by creating new chain + pub fn fork_canonical_chain(&mut self, block: SealedBlockWithSenders) -> Result<(), Error> { + let canonical_block_hashes = self.block_indices.canonical_chain(); + let (_, canonical_tip) = + canonical_block_hashes.last_key_value().map(|(i, j)| (*i, *j)).unwrap_or_default(); + + // create state provider + let db = self.externals.sharable_db(); + let parent_header = db + .header(&block.parent_hash)? + .ok_or(ExecError::CanonicalChain { block_hash: block.parent_hash })?; + + let provider = if block.parent_hash == canonical_tip { + Box::new(db.latest()?) as Box + } else { + Box::new(db.history_by_block_number(block.number - 1)?) as Box + }; + + let parent_header = parent_header.seal(block.parent_hash); + let chain = Chain::new_canonical_fork( + &block, + &parent_header, + canonical_block_hashes, + &provider, + &self.externals.consensus, + &self.externals.executor_factory, + )?; + drop(provider); + self.insert_chain(chain); + Ok(()) + } + + /// Get all block hashes from chain that are not canonical. This is one time operation per + /// block. Reason why this is not caches is to save memory. + fn all_chain_hashes(&self, chain_id: BlockChainId) -> BTreeMap { + // find chain and iterate over it, + let mut chain_id = chain_id; + let mut hashes = BTreeMap::new(); + loop { + let Some(chain) = self.chains.get(&chain_id) else { return hashes }; + hashes.extend(chain.blocks().values().map(|b| (b.number, b.hash()))); + + let fork_block = chain.fork_block_hash(); + if let Some(next_chain_id) = self.block_indices.get_blocks_chain_id(&fork_block) { + chain_id = next_chain_id; + } else { + // if there is no fork block that point to other chains, break the loop. + // it means that this fork joins to canonical block. + break + } + } + hashes + } + + /// Getting the canonical fork would tell use what kind of Provider we should execute block on. + /// If it is latest state provider or history state provider + /// Return None if chain_id is not known. + fn canonical_fork(&self, chain_id: BlockChainId) -> Option { + let mut chain_id = chain_id; + let mut fork; + loop { + // chain fork block + fork = self.chains.get(&chain_id)?.fork_block(); + // get fork block chain + if let Some(fork_chain_id) = self.block_indices.get_blocks_chain_id(&fork.hash) { + chain_id = fork_chain_id; + continue + } + break + } + if self.block_indices.canonical_hash(&fork.number) == Some(fork.hash) { + Some(fork) + } else { + None + } + } + + /// Insert chain to tree and ties the blocks to it. + /// Helper function that handles indexing and inserting. + fn insert_chain(&mut self, chain: Chain) -> BlockChainId { + let chain_id = self.block_chain_id_generator; + self.block_chain_id_generator += 1; + self.block_indices.insert_chain(chain_id, &chain); + // add chain_id -> chain index + self.chains.insert(chain_id, chain); + chain_id + } + + /// Insert block inside tree. recover transaction signers and + /// internaly call [`BlockchainTree::insert_block_with_senders`] fn. + pub fn insert_block(&mut self, block: SealedBlock) -> Result { + let block = block.seal_with_senders().ok_or(ExecError::SenderRecoveryError)?; + self.insert_block_with_senders(&block) + } + + /// Insert block with senders inside tree. + /// Returns `true` if: + /// 1. It is part of the blockchain tree + /// 2. It is part of the canonical chain + /// 3. Its parent is part of the blockchain tree and we can fork at the parent + /// 4. Its parent is part of the canonical chain and we can fork at the parent + /// Otherwise will return `false`, indicating that neither the block nor its parent + /// is part of the chain or any sidechains. This means that if block becomes canonical + /// we need to fetch the missing blocks over p2p. + pub fn insert_block_with_senders( + &mut self, + block: &SealedBlockWithSenders, + ) -> Result { + // check if block number is inside pending block slide + let last_finalized_block = self.block_indices.last_finalized_block(); + if block.number <= last_finalized_block { + return Err(ExecError::PendingBlockIsFinalized { + block_number: block.number, + block_hash: block.hash(), + last_finalized: last_finalized_block, + } + .into()) + } + + // we will not even try to insert blocks that are too far in future. + if block.number > last_finalized_block + self.max_blocks_in_chain { + return Err(ExecError::PendingBlockIsInFuture { + block_number: block.number, + block_hash: block.hash(), + last_finalized: last_finalized_block, + } + .into()) + } + + // check if block is already inside Tree + if self.block_indices.contains_pending_block_hash(block.hash()) { + // block is known return that is inserted + return Ok(true) + } + + // check if block is part of canonical chain + if self.block_indices.canonical_hash(&block.number) == Some(block.hash()) { + // block is part of canonical chain + return Ok(true) + } + + // check if block parent can be found in Tree + if let Some(parent_chain) = self.block_indices.get_blocks_chain_id(&block.parent_hash) { + self.fork_side_chain(block.clone(), parent_chain)?; + // TODO save pending block to database + // https://github.com/paradigmxyz/reth/issues/1713 + return Ok(true) + } + + // if not found, check if the parent can be found inside canonical chain. + if Some(block.parent_hash) == self.block_indices.canonical_hash(&(block.number - 1)) { + // create new chain that points to that block + self.fork_canonical_chain(block.clone())?; + // TODO save pending block to database + // https://github.com/paradigmxyz/reth/issues/1713 + return Ok(true) + } + // NOTE: Block doesn't have a parent, and if we receive this block in `make_canonical` + // function this could be a trigger to initiate p2p syncing, as we are missing the + // parent. + Ok(false) + } + + /// Do finalization of blocks. Remove them from tree + pub fn finalize_block(&mut self, finalized_block: BlockNumber) { + let mut remove_chains = self.block_indices.finalize_canonical_blocks(finalized_block); + + while let Some(chain_id) = remove_chains.pop_first() { + if let Some(chain) = self.chains.remove(&chain_id) { + remove_chains.extend(self.block_indices.remove_chain(&chain)); + } + } + } + + /// Update canonical hashes. Reads last N canonical blocks from database and update all indices. + pub fn update_canonical_hashes( + &mut self, + last_finalized_block: BlockNumber, + ) -> Result<(), Error> { + self.finalize_block(last_finalized_block); + + let num_of_canonical_hashes = + self.max_reorg_depth + self.block_indices.num_of_additional_canonical_block_hashes(); + + let last_canonical_hashes = self + .externals + .db + .tx()? + .cursor_read::()? + .walk_back(None)? + .take(num_of_canonical_hashes as usize) + .collect::, _>>()?; + + let mut remove_chains = self.block_indices.update_block_hashes(last_canonical_hashes); + + // remove all chains that got discarded + while let Some(chain_id) = remove_chains.first() { + if let Some(chain) = self.chains.remove(chain_id) { + remove_chains.extend(self.block_indices.remove_chain(&chain)); + } + } + + Ok(()) + } + + /// Split chain and return canonical part of it. Pending part reinsert inside tree + /// with same chain_id. + fn split_chain(&mut self, chain_id: BlockChainId, chain: Chain, split_at: SplitAt) -> Chain { + match chain.split(split_at) { + ChainSplit::Split { canonical, pending } => { + // rest of splited chain is inserted back with same chain_id. + self.block_indices.insert_chain(chain_id, &pending); + self.chains.insert(chain_id, pending); + canonical + } + ChainSplit::NoSplitCanonical(canonical) => canonical, + ChainSplit::NoSplitPending(_) => { + panic!("Should not happen as block indices guarantee structure of blocks") + } + } + } + + /// Make block and its parent canonical. Unwind chains to database if necessary. + /// + /// If block is already part of canonical chain return Ok. + pub fn make_canonical(&mut self, block_hash: &BlockHash) -> Result<(), Error> { + let chain_id = if let Some(chain_id) = self.block_indices.get_blocks_chain_id(block_hash) { + chain_id + } else { + // If block is already canonical don't return error. + if self.block_indices.is_block_hash_canonical(block_hash) { + return Ok(()) + } + return Err(ExecError::BlockHashNotFoundInChain { block_hash: *block_hash }.into()) + }; + let chain = self.chains.remove(&chain_id).expect("To be present"); + + // we are spliting chain as there is possibility that only part of chain get canonicalized. + let canonical = self.split_chain(chain_id, chain, SplitAt::Hash(*block_hash)); + + let mut block_fork = canonical.fork_block(); + let mut block_fork_number = canonical.fork_block_number(); + let mut chains_to_promote = vec![canonical]; + + // loop while fork blocks are found in Tree. + while let Some(chain_id) = self.block_indices.get_blocks_chain_id(&block_fork.hash) { + let chain = self.chains.remove(&chain_id).expect("To fork to be present"); + block_fork = chain.fork_block(); + let canonical = self.split_chain(chain_id, chain, SplitAt::Number(block_fork_number)); + block_fork_number = canonical.fork_block_number(); + chains_to_promote.push(canonical); + } + + let old_tip = self.block_indices.canonical_tip(); + // Merge all chain into one chain. + let mut new_canon_chain = chains_to_promote.pop().expect("There is at least one block"); + for chain in chains_to_promote.into_iter().rev() { + new_canon_chain.append_chain(chain).expect("We have just build the chain."); + } + + // update canonical index + self.block_indices.canonicalize_blocks(new_canon_chain.blocks()); + + // if joins to the tip + if new_canon_chain.fork_block_hash() == old_tip.hash { + // append to database + self.commit_canonical(new_canon_chain)?; + } else { + // it forks to canonical block that is not the tip. + + let canon_fork = new_canon_chain.fork_block(); + // sanity check + if self.block_indices.canonical_hash(&canon_fork.number) != Some(canon_fork.hash) { + unreachable!("all chains should point to canonical chain."); + } + + // revert `N` blocks from current canonical chain and put them inside BlockchanTree + // This is main reorgs on tables. + let old_canon_chain = self.revert_canonical(canon_fork.number)?; + self.commit_canonical(new_canon_chain)?; + + // TODO we can potentially merge now reverted canonical chain with + // one of the chain from the tree. Low priority. + + // insert old canonical chain to BlockchainTree. + self.insert_chain(old_canon_chain); + } + + Ok(()) + } + + /// Commit chain for it to become canonical. Assume we are doing pending operation to db. + fn commit_canonical(&mut self, chain: Chain) -> Result<(), Error> { + let mut tx = Transaction::new(&self.externals.db)?; + + let new_tip = chain.tip().number; + let (blocks, changesets, _) = chain.into_inner(); + for item in blocks.into_iter().zip(changesets.into_iter()) { + let ((_, block), changeset) = item; + tx.insert_block(block, self.externals.chain_spec.as_ref(), changeset) + .map_err(|e| ExecError::CanonicalCommit { inner: e.to_string() })?; + } + // update pipeline progress. + tx.update_pipeline_stages(new_tip) + .map_err(|e| ExecError::PipelineStatusUpdate { inner: e.to_string() })?; + + tx.commit()?; + + Ok(()) + } + + /// Revert canonical blocks from database and insert them to pending table + /// Revert should be non inclusive, and revert_until should stay in db. + /// Return the chain that represent reverted canonical blocks. + fn revert_canonical(&mut self, revert_until: BlockNumber) -> Result { + // read data that is needed for new sidechain + + let mut tx = Transaction::new(&self.externals.db)?; + + // read block and execution result from database. and remove traces of block from tables. + let blocks_and_execution = tx + .take_block_and_execution_range( + self.externals.chain_spec.as_ref(), + (revert_until + 1).., + ) + .map_err(|e| ExecError::CanonicalRevert { inner: e.to_string() })?; + + // update pipeline progress. + tx.update_pipeline_stages(revert_until) + .map_err(|e| ExecError::PipelineStatusUpdate { inner: e.to_string() })?; + + tx.commit()?; + + let chain = Chain::new(blocks_and_execution); + + Ok(chain) + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashSet; + + use super::*; + use parking_lot::Mutex; + use reth_db::{ + mdbx::{test_utils::create_test_rw_db, Env, WriteMap}, + transaction::DbTxMut, + }; + use reth_interfaces::test_utils::TestConsensus; + use reth_primitives::{hex_literal::hex, proofs::EMPTY_ROOT, ChainSpecBuilder, H256, MAINNET}; + use reth_provider::{ + execution_result::ExecutionResult, insert_block, test_utils::blocks::BlockChainTestData, + BlockExecutor, + }; + + struct TestFactory { + exec_result: Arc>>, + chain_spec: Arc, + } + + impl TestFactory { + fn new(chain_spec: Arc) -> Self { + Self { exec_result: Arc::new(Mutex::new(Vec::new())), chain_spec } + } + + fn extend(&self, exec_res: Vec) { + self.exec_result.lock().extend(exec_res.into_iter()); + } + } + + struct TestExecutor(Option); + + impl BlockExecutor for TestExecutor { + fn execute( + &mut self, + _block: &reth_primitives::Block, + _total_difficulty: reth_primitives::U256, + _senders: Option>, + ) -> Result { + self.0.clone().ok_or(ExecError::VerificationFailed) + } + + fn execute_and_verify_receipt( + &mut self, + _block: &reth_primitives::Block, + _total_difficulty: reth_primitives::U256, + _senders: Option>, + ) -> Result { + self.0.clone().ok_or(ExecError::VerificationFailed) + } + } + + impl ExecutorFactory for TestFactory { + type Executor = TestExecutor; + + fn with_sp(&self, _sp: SP) -> Self::Executor { + let exec_res = self.exec_result.lock().pop(); + TestExecutor(exec_res) + } + + fn chain_spec(&self) -> &ChainSpec { + self.chain_spec.as_ref() + } + } + + type TestExternals = (Arc>, TestConsensus, TestFactory, Arc); + + fn externals(exec_res: Vec) -> TestExternals { + let db = create_test_rw_db(); + let consensus = TestConsensus::default(); + let chain_spec = Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis(MAINNET.genesis.clone()) + .shanghai_activated() + .build(), + ); + let executor_factory = TestFactory::new(chain_spec.clone()); + executor_factory.extend(exec_res); + + (db, consensus, executor_factory, chain_spec) + } + + fn setup(mut genesis: SealedBlock, externals: &TestExternals) { + // insert genesis to db. + + genesis.header.header.number = 10; + genesis.header.header.state_root = EMPTY_ROOT; + let tx_mut = externals.0.tx_mut().unwrap(); + + insert_block(&tx_mut, genesis.clone(), None, false, Some((0, 0))).unwrap(); + + // insert first 10 blocks + for i in 0..10 { + tx_mut.put::(i, H256([100 + i as u8; 32])).unwrap(); + } + tx_mut.commit().unwrap(); + } + + /// Test data structure that will check tree internals + #[derive(Default, Debug)] + struct TreeTester { + /// Number of chains + chain_num: Option, + /// Check block to chain index + block_to_chain: Option>, + /// Check fork to child index + fork_to_child: Option>>, + } + + impl TreeTester { + fn with_chain_num(mut self, chain_num: usize) -> Self { + self.chain_num = Some(chain_num); + self + } + fn with_block_to_chain(mut self, block_to_chain: HashMap) -> Self { + self.block_to_chain = Some(block_to_chain); + self + } + fn with_fork_to_child( + mut self, + fork_to_child: HashMap>, + ) -> Self { + self.fork_to_child = Some(fork_to_child); + self + } + + fn assert( + self, + tree: &BlockchainTree, + ) { + if let Some(chain_num) = self.chain_num { + assert_eq!(tree.chains.len(), chain_num); + } + if let Some(block_to_chain) = self.block_to_chain { + assert_eq!(*tree.block_indices.blocks_to_chain(), block_to_chain); + } + if let Some(fork_to_child) = self.fork_to_child { + assert_eq!(*tree.block_indices.fork_to_child(), fork_to_child); + } + } + } + + #[test] + fn sanity_path() { + let data = BlockChainTestData::default(); + let (mut block1, exec1) = data.blocks[0].clone(); + block1.number = 11; + block1.state_root = + H256(hex!("5d035ccb3e75a9057452ff060b773b213ec1fc353426174068edfc3971a0b6bd")); + let (mut block2, exec2) = data.blocks[1].clone(); + block2.number = 12; + block2.state_root = + H256(hex!("90101a13dd059fa5cca99ed93d1dc23657f63626c5b8f993a2ccbdf7446b64f8")); + + // test pops execution results from vector, so order is from last to first.ß + let externals = externals(vec![exec2.clone(), exec1.clone(), exec2.clone(), exec1.clone()]); + + // last finalized block would be number 9. + setup(data.genesis, &externals); + + // make tree + let (db, consensus, exec_factory, chain_spec) = externals; + let mut tree = + BlockchainTree::new(db, consensus, exec_factory, chain_spec, 1, 2, 3).unwrap(); + + // genesis block 10 is already canonical + assert_eq!(tree.make_canonical(&H256::zero()), Ok(())); + + // insert block2 hits max chain size + assert_eq!( + tree.insert_block_with_senders(&block2), + Err(ExecError::PendingBlockIsInFuture { + block_number: block2.number, + block_hash: block2.hash(), + last_finalized: 9, + } + .into()) + ); + + // make genesis block 10 as finalized + tree.finalize_block(10); + + // block 2 parent is not known. + assert_eq!(tree.insert_block_with_senders(&block2), Ok(false)); + + // insert block1 + assert_eq!(tree.insert_block_with_senders(&block1), Ok(true)); + // already inserted block will return true. + assert_eq!(tree.insert_block_with_senders(&block1), Ok(true)); + + // insert block2 + assert_eq!(tree.insert_block_with_senders(&block2), Ok(true)); + + // Trie state: + // b2 (pending block) + // | + // | + // b1 (pending block) + // / + // / + // g1 (canonical blocks) + // | + TreeTester::default() + .with_chain_num(1) + .with_block_to_chain(HashMap::from([(block1.hash, 0), (block2.hash, 0)])) + .with_fork_to_child(HashMap::from([(block1.parent_hash, HashSet::from([block1.hash]))])) + .assert(&tree); + + // make block1 canonical + assert_eq!(tree.make_canonical(&block1.hash()), Ok(())); + // make block2 canonical + assert_eq!(tree.make_canonical(&block2.hash()), Ok(())); + + // Trie state: + // b2 (canonical block) + // | + // | + // b1 (canonical block) + // | + // | + // g1 (canonical blocks) + // | + TreeTester::default() + .with_chain_num(0) + .with_block_to_chain(HashMap::from([])) + .with_fork_to_child(HashMap::from([])) + .assert(&tree); + + let mut block1a = block1.clone(); + let block1a_hash = H256([0x33; 32]); + block1a.hash = block1a_hash; + let mut block2a = block2.clone(); + let block2a_hash = H256([0x34; 32]); + block2a.hash = block2a_hash; + + // reinsert two blocks that point to canonical chain + assert_eq!(tree.insert_block_with_senders(&block1a), Ok(true)); + + TreeTester::default() + .with_chain_num(1) + .with_block_to_chain(HashMap::from([(block1a_hash, 1)])) + .with_fork_to_child(HashMap::from([( + block1.parent_hash, + HashSet::from([block1a_hash]), + )])) + .assert(&tree); + + assert_eq!(tree.insert_block_with_senders(&block2a), Ok(true)); + // Trie state: + // b2 b2a (side chain) + // | / + // | / + // b1 b1a (side chain) + // | / + // |/ + // g1 (10) + // | + TreeTester::default() + .with_chain_num(2) + .with_block_to_chain(HashMap::from([(block1a_hash, 1), (block2a_hash, 2)])) + .with_fork_to_child(HashMap::from([ + (block1.parent_hash, HashSet::from([block1a_hash])), + (block1.hash(), HashSet::from([block2a_hash])), + ])) + .assert(&tree); + + // make b2a canonical + assert_eq!(tree.make_canonical(&block2a_hash), Ok(())); + // Trie state: + // b2a b2 (side chain) + // | / + // | / + // b1 b1a (side chain) + // | / + // |/ + // g1 (10) + // | + TreeTester::default() + .with_chain_num(2) + .with_block_to_chain(HashMap::from([(block1a_hash, 1), (block2.hash, 3)])) + .with_fork_to_child(HashMap::from([ + (block1.parent_hash, HashSet::from([block1a_hash])), + (block1.hash(), HashSet::from([block2.hash])), + ])) + .assert(&tree); + + assert_eq!(tree.make_canonical(&block1a_hash), Ok(())); + // Trie state: + // b2a b2 (side chain) + // | / + // | / + // b1a b1 (side chain) + // | / + // |/ + // g1 (10) + // | + TreeTester::default() + .with_chain_num(2) + .with_block_to_chain(HashMap::from([ + (block1.hash, 4), + (block2a_hash, 4), + (block2.hash, 3), + ])) + .with_fork_to_child(HashMap::from([ + (block1.parent_hash, HashSet::from([block1.hash])), + (block1.hash(), HashSet::from([block2.hash])), + ])) + .assert(&tree); + + // make b2 canonical + assert_eq!(tree.make_canonical(&block2.hash()), Ok(())); + // Trie state: + // b2 b2a (side chain) + // | / + // | / + // b1 b1a (side chain) + // | / + // |/ + // g1 (10) + // | + TreeTester::default() + .with_chain_num(2) + .with_block_to_chain(HashMap::from([(block1a_hash, 5), (block2a_hash, 4)])) + .with_fork_to_child(HashMap::from([ + (block1.parent_hash, HashSet::from([block1a_hash])), + (block1.hash(), HashSet::from([block2a_hash])), + ])) + .assert(&tree); + + // finalize b1 that would make b1a removed from tree + tree.finalize_block(11); + // Trie state: + // b2 b2a (side chain) + // | / + // | / + // b1 (canon) + // | + // g1 (10) + // | + TreeTester::default() + .with_chain_num(1) + .with_block_to_chain(HashMap::from([(block2a_hash, 4)])) + .with_fork_to_child(HashMap::from([(block1.hash(), HashSet::from([block2a_hash]))])) + .assert(&tree); + + // update canonical block to b2, this would make b2a be removed + assert_eq!(tree.update_canonical_hashes(12), Ok(())); + // Trie state: + // b2 (canon) + // | + // b1 (canon) + // | + // g1 (10) + // | + TreeTester::default() + .with_chain_num(0) + .with_block_to_chain(HashMap::from([])) + .with_fork_to_child(HashMap::from([])) + .assert(&tree); + } +} diff --git a/crates/executor/src/lib.rs b/crates/executor/src/lib.rs index 4bf17e8dc52..d1b17d73972 100644 --- a/crates/executor/src/lib.rs +++ b/crates/executor/src/lib.rs @@ -8,9 +8,11 @@ //! Reth executor executes transaction in block of data. pub mod eth_dao_fork; +pub mod substate; /// Execution result types. pub use reth_provider::execution_result; +pub mod blockchain_tree; /// Executor pub mod executor; diff --git a/crates/executor/src/substate.rs b/crates/executor/src/substate.rs new file mode 100644 index 00000000000..41b5fe6483c --- /dev/null +++ b/crates/executor/src/substate.rs @@ -0,0 +1,306 @@ +//! Substate for blockchain trees + +use reth_interfaces::{provider::ProviderError, Result}; +use reth_primitives::{Account, Address, BlockHash, BlockNumber, Bytecode, Bytes, H256, U256}; +use reth_provider::{AccountProvider, BlockHashProvider, StateProvider}; +use std::collections::{hash_map::Entry, BTreeMap, HashMap}; + +use crate::execution_result::{AccountInfoChangeSet, ExecutionResult}; + +/// Memory backend, storing all state values in a `Map` in memory. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct SubStateData { + /// Account info where None means it is not existing. Not existing state is needed for Pre + /// TANGERINE forks. `code` is always `None`, and bytecode can be found in `contracts`. + pub accounts: HashMap, + /// New bytecodes + pub bytecodes: HashMap, +} + +impl SubStateData { + /// Apply changesets to substate. + pub fn apply(&mut self, changesets: &[ExecutionResult]) { + for changeset in changesets { + self.apply_one(changeset) + } + } + + /// Apply one changeset to substate. + pub fn apply_one(&mut self, changeset: &ExecutionResult) { + for tx_changeset in changeset.tx_changesets.iter() { + // apply accounts + for (address, account_change) in tx_changeset.changeset.iter() { + // revert account + self.apply_account(address, &account_change.account); + // revert its storage + self.apply_storage(address, &account_change.storage); + } + // apply bytecodes + for (hash, bytecode) in tx_changeset.new_bytecodes.iter() { + self.bytecodes.entry(*hash).or_insert((0, Bytecode(bytecode.clone()))).0 += 1; + } + } + // apply block reward + for (address, change) in changeset.block_changesets.iter() { + self.apply_account(address, change) + } + } + + /// Apply account changeset to substate + fn apply_account(&mut self, address: &Address, change: &AccountInfoChangeSet) { + match change { + AccountInfoChangeSet::Created { new } => match self.accounts.entry(*address) { + Entry::Vacant(entry) => { + entry.insert(AccountSubState::created_account(*new)); + } + Entry::Occupied(mut entry) => { + let account = entry.get_mut(); + // increment counter + account.inc_storage_counter(); + account.info = *new; + } + }, + AccountInfoChangeSet::Destroyed { .. } => { + // set selfdestructed account + let account = self.accounts.entry(*address).or_default(); + account.inc_storage_counter(); + account.info = Default::default(); + account.storage.clear(); + } + AccountInfoChangeSet::Changed { old, .. } => { + self.accounts.entry(*address).or_default().info = *old; + } + AccountInfoChangeSet::NoChange { is_empty } => { + if *is_empty { + self.accounts.entry(*address).or_default(); + } + } + } + } + + /// Apply storage changeset to substate + fn apply_storage(&mut self, address: &Address, storage: &BTreeMap) { + if let Entry::Occupied(mut entry) = self.accounts.entry(*address) { + let account_storage = &mut entry.get_mut().storage; + for (key, (_, new_value)) in storage { + let key = H256(key.to_be_bytes()); + account_storage.insert(key, *new_value); + } + } + } + + /// Revert to old state in substate. Changesets will be reverted in reverse order, + pub fn revert(&mut self, changesets: &[ExecutionResult]) { + for changeset in changesets.iter().rev() { + // revert block changeset + for (address, change) in changeset.block_changesets.iter() { + self.revert_account(address, change) + } + + for tx_changeset in changeset.tx_changesets.iter() { + // revert bytecodes + for (hash, _) in tx_changeset.new_bytecodes.iter() { + match self.bytecodes.entry(*hash) { + Entry::Vacant(_) => panic!("Bytecode should be present"), + Entry::Occupied(mut entry) => { + let (cnt, _) = entry.get_mut(); + *cnt -= 1; + if *cnt == 0 { + entry.remove_entry(); + } + } + } + } + // revert accounts + for (address, account_change) in tx_changeset.changeset.iter() { + // revert account + self.revert_account(address, &account_change.account); + // revert its storage + self.revert_storage(address, &account_change.storage); + } + } + } + } + + /// Revert storage + fn revert_storage(&mut self, address: &Address, storage: &BTreeMap) { + if let Entry::Occupied(mut entry) = self.accounts.entry(*address) { + let account_storage = &mut entry.get_mut().storage; + for (key, (old_value, _)) in storage { + let key = H256(key.to_be_bytes()); + account_storage.insert(key, *old_value); + } + } + } + + /// Revert account + fn revert_account(&mut self, address: &Address, change: &AccountInfoChangeSet) { + match change { + AccountInfoChangeSet::Created { .. } => { + match self.accounts.entry(*address) { + Entry::Vacant(_) => { + // We inserted this account in apply fn. + panic!("It should be present, something is broken"); + } + Entry::Occupied(mut entry) => { + let val = entry.get_mut(); + if val.decr_storage_counter() { + // remove account that we didn't change from substate + + entry.remove_entry(); + return + } + val.info = Account::default(); + val.storage.clear(); + } + }; + } + AccountInfoChangeSet::Destroyed { old } => match self.accounts.entry(*address) { + Entry::Vacant(_) => { + // We inserted this account in apply fn. + panic!("It should be present, something is broken"); + } + Entry::Occupied(mut entry) => { + let val = entry.get_mut(); + + // Contrary to Created we are not removing this account as we dont know if + // this account was changer or not by `Changed` changeset. + val.decr_storage_counter(); + val.info = *old; + } + }, + AccountInfoChangeSet::Changed { old, .. } => { + self.accounts.entry(*address).or_default().info = *old; + } + AccountInfoChangeSet::NoChange { is_empty: _ } => { + // do nothing + } + } + } +} +/// Account changes in substate +#[derive(Debug, Clone, Default, Eq, PartialEq)] +pub struct AccountSubState { + /// New account state + pub info: Account, + /// If account is selfdestructed or newly created, storage will be cleared. + /// and we dont need to ask the provider for data. + /// As we need to have as + pub storage_is_clear: Option, + /// storage slots + pub storage: HashMap, +} + +impl AccountSubState { + /// Increment storage counter to mark this storage was cleared + pub fn inc_storage_counter(&mut self) { + self.storage_is_clear = Some(self.storage_is_clear.unwrap_or_default() + 1); + } + + /// Decrement storage counter to represent that changeset that cleared storage was reverted. + pub fn decr_storage_counter(&mut self) -> bool { + let Some(cnt) = self.storage_is_clear else { return false}; + + if cnt == 1 { + self.storage_is_clear = None; + return true + } + false + } + /// Account is created + pub fn created_account(info: Account) -> Self { + Self { info, storage_is_clear: Some(1), storage: HashMap::new() } + } + /// Should we ask the provider for storage data + pub fn ask_provider(&self) -> bool { + self.storage_is_clear.is_none() + } +} + +/// Wrapper around substate and provider, it decouples the database that can be Latest or historical +/// with substate changes that happened previously. +pub struct SubStateWithProvider<'a, SP: StateProvider> { + /// Substate + substate: &'a SubStateData, + /// Provider + provider: SP, + /// side chain block hashes + sidechain_block_hashes: &'a BTreeMap, + /// Last N canonical hashes, + canonical_block_hashes: &'a BTreeMap, +} + +impl<'a, SP: StateProvider> SubStateWithProvider<'a, SP> { + /// Create new substate with provider + pub fn new( + substate: &'a SubStateData, + provider: SP, + sidechain_block_hashes: &'a BTreeMap, + canonical_block_hashes: &'a BTreeMap, + ) -> Self { + Self { substate, provider, sidechain_block_hashes, canonical_block_hashes } + } +} + +/* Implement StateProvider traits */ + +impl<'a, SP: StateProvider> BlockHashProvider for SubStateWithProvider<'a, SP> { + fn block_hash(&self, number: U256) -> Result> { + // All block numbers fit inside u64 and revm checks if it is last 256 block numbers. + let block_number = number.as_limbs()[0]; + if let Some(sidechain_block_hash) = self.sidechain_block_hashes.get(&block_number).cloned() + { + return Ok(Some(sidechain_block_hash)) + } + + Ok(Some( + self.canonical_block_hashes + .get(&block_number) + .cloned() + .ok_or(ProviderError::BlockchainTreeBlockHash { block_number })?, + )) + } +} + +impl<'a, SP: StateProvider> AccountProvider for SubStateWithProvider<'a, SP> { + fn basic_account(&self, address: Address) -> Result> { + if let Some(account) = self.substate.accounts.get(&address).map(|acc| acc.info) { + return Ok(Some(account)) + } + self.provider.basic_account(address) + } +} + +impl<'a, SP: StateProvider> StateProvider for SubStateWithProvider<'a, SP> { + fn storage( + &self, + account: Address, + storage_key: reth_primitives::StorageKey, + ) -> Result> { + if let Some(substate_account) = self.substate.accounts.get(&account) { + if let Some(storage) = substate_account.storage.get(&storage_key) { + return Ok(Some(*storage)) + } + if !substate_account.ask_provider() { + return Ok(Some(U256::ZERO)) + } + } + self.provider.storage(account, storage_key) + } + + /// Get account and storage proofs. + fn proof( + &self, + _address: Address, + _keys: &[H256], + ) -> Result<(Vec, H256, Vec>)> { + Err(ProviderError::HistoryStateRoot.into()) + } + + fn bytecode_by_hash(&self, code_hash: H256) -> Result> { + if let Some((_, bytecode)) = self.substate.bytecodes.get(&code_hash).cloned() { + return Ok(Some(bytecode)) + } + self.provider.bytecode_by_hash(code_hash) + } +} diff --git a/crates/interfaces/src/executor.rs b/crates/interfaces/src/executor.rs index ad5f4e000bf..a40f60a0e25 100644 --- a/crates/interfaces/src/executor.rs +++ b/crates/interfaces/src/executor.rs @@ -1,4 +1,4 @@ -use reth_primitives::{Bloom, H256}; +use reth_primitives::{BlockHash, BlockNumber, Bloom, H256}; use thiserror::Error; /// BlockExecutor Errors @@ -34,4 +34,34 @@ pub enum Error { BlockGasUsed { got: u64, expected: u64 }, #[error("Provider error")] ProviderError, + #[error("BlockChainId can't be found in BlockchainTree with internal index {chain_id}")] + BlockChainIdConsistency { chain_id: u64 }, + #[error( + "Appending chain on fork (other_chain_fork:?) is not possible as the tip is {chain_tip:?}" + )] + AppendChainDoesntConnect { chain_tip: (u64, H256), other_chain_fork: (u64, H256) }, + #[error("Canonical chain header #{block_hash} can't be found ")] + CanonicalChain { block_hash: BlockHash }, + #[error("Can't insert #{block_number} {block_hash} as last finalized block number is {last_finalized}")] + PendingBlockIsFinalized { + block_hash: BlockHash, + block_number: BlockNumber, + last_finalized: BlockNumber, + }, + #[error("Can't insert block #{block_number} {block_hash} to far in future, as last finalized block number is {last_finalized}")] + PendingBlockIsInFuture { + block_hash: BlockHash, + block_number: BlockNumber, + last_finalized: BlockNumber, + }, + #[error("Block number #{block_number} not found in blockchain tree chain")] + BlockNumberNotFoundInChain { block_number: BlockNumber }, + #[error("Block hash {block_hash} not found in blockchain tree chain")] + BlockHashNotFoundInChain { block_hash: BlockHash }, + #[error("Transaction error on revert: {inner:?}")] + CanonicalRevert { inner: String }, + #[error("Transaction error on commit: {inner:?}")] + CanonicalCommit { inner: String }, + #[error("Transaction error on pipeline status update: {inner:?}")] + PipelineStatusUpdate { inner: String }, } diff --git a/crates/interfaces/src/provider.rs b/crates/interfaces/src/provider.rs index e86d2dc5946..33ace26d1ea 100644 --- a/crates/interfaces/src/provider.rs +++ b/crates/interfaces/src/provider.rs @@ -68,12 +68,23 @@ pub enum ProviderError { /// Reached the end of the transaction sender table. #[error("Got to the end of the transaction sender table")] EndOfTransactionSenderTable, + /// Missing block hash in BlockchainTree + #[error("Missing block hash for block #{block_number:?} in blockchain tree")] + BlockchainTreeBlockHash { block_number: BlockNumber }, /// Some error occurred while interacting with the state tree. - #[error("Unknown error occurred while interacting with the state tree.")] - StateTree, + #[error("Unknown error occurred while interacting with the state trie.")] + StateTrie, + #[error("History state root, can't be calculated")] + HistoryStateRoot, /// Thrown when required header related data was not found but was required. #[error("requested data not found")] HeaderNotFound, + /// Mismatch of sender and transaction + #[error("Mismatch of sender and transaction id {tx_id}")] + MismatchOfTransactionAndSenderId { tx_id: TxNumber }, + /// Block body wrong transaction count + #[error("Stored block indices does not match transaction count")] + BlockBodyTransactionCount, /// Thrown when the cache service task dropped #[error("cache service task stopped")] CacheServiceUnavailable, diff --git a/crates/net/network/src/config.rs b/crates/net/network/src/config.rs index add23de286f..aed9bcb7af6 100644 --- a/crates/net/network/src/config.rs +++ b/crates/net/network/src/config.rs @@ -15,6 +15,7 @@ use secp256k1::{SecretKey, SECP256K1}; use std::{ collections::HashSet, net::{Ipv4Addr, SocketAddr, SocketAddrV4}, + sync::Arc, }; /// reexports for convenience @@ -54,7 +55,7 @@ pub struct NetworkConfig { /// How to configure the [SessionManager](crate::session::SessionManager). pub sessions_config: SessionsConfig, /// The chain spec - pub chain_spec: ChainSpec, + pub chain_spec: Arc, /// The [`ForkFilter`] to use at launch for authenticating sessions. /// /// See also @@ -139,7 +140,7 @@ pub struct NetworkConfigBuilder { /// How to configure the sessions manager sessions_config: Option, /// The network's chain spec - chain_spec: ChainSpec, + chain_spec: Arc, /// The default mode of the network. network_mode: NetworkMode, /// The executor to use for spawning tasks. @@ -165,7 +166,7 @@ impl NetworkConfigBuilder { listener_addr: None, peers_config: None, sessions_config: None, - chain_spec: MAINNET.clone(), + chain_spec: Arc::new(MAINNET.clone()), network_mode: Default::default(), executor: None, hello_message: None, @@ -179,7 +180,7 @@ impl NetworkConfigBuilder { } /// Sets the chain spec. - pub fn chain_spec(mut self, chain_spec: ChainSpec) -> Self { + pub fn chain_spec(mut self, chain_spec: Arc) -> Self { self.chain_spec = chain_spec; self } @@ -417,6 +418,7 @@ mod tests { // remove any `next` fields we would have by removing all hardforks chain_spec.hardforks = BTreeMap::new(); + let chain_spec = Arc::new(chain_spec); // check that the forkid is initialized with the genesis and no other forks let genesis_fork_hash = ForkHash::from(chain_spec.genesis_hash()); diff --git a/crates/primitives/Cargo.toml b/crates/primitives/Cargo.toml index 5a872aeab4d..9172b064660 100644 --- a/crates/primitives/Cargo.toml +++ b/crates/primitives/Cargo.toml @@ -90,6 +90,7 @@ arbitrary = [ "dep:proptest", "dep:proptest-derive", ] +test-utils = [] [[bench]] name = "recover_ecdsa_crit" diff --git a/crates/primitives/src/block.rs b/crates/primitives/src/block.rs index 13b8b9d548a..e158f7c2ed9 100644 --- a/crates/primitives/src/block.rs +++ b/crates/primitives/src/block.rs @@ -1,4 +1,4 @@ -use crate::{Header, SealedHeader, TransactionSigned, Withdrawal, H256}; +use crate::{Address, Header, SealedHeader, TransactionSigned, Withdrawal, H256}; use ethers_core::types::BlockNumber; use reth_codecs::derive_arbitrary; use reth_rlp::{Decodable, DecodeError, Encodable, RlpDecodable, RlpEncodable}; @@ -12,10 +12,10 @@ use std::{fmt, fmt::Formatter, ops::Deref, str::FromStr}; /// Ethereum full block. /// /// Withdrawals can be optionally included at the end of the RLP encoded message. -#[derive_arbitrary(rlp, 25)] #[derive( Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize, RlpEncodable, RlpDecodable, )] +#[derive_arbitrary(rlp, 25)] #[rlp(trailing)] pub struct Block { /// Block header. @@ -28,6 +28,18 @@ pub struct Block { pub withdrawals: Option>, } +impl Block { + /// Create SealedBLock that will create all header hashes. + pub fn seal_slow(self) -> SealedBlock { + SealedBlock { + header: self.header.seal_slow(), + body: self.body, + ommers: self.ommers.into_iter().map(|o| o.seal_slow()).collect(), + withdrawals: self.withdrawals, + } + } +} + impl Deref for Block { type Target = Header; fn deref(&self) -> &Self::Target { @@ -65,6 +77,17 @@ impl SealedBlock { (self.header, self.body, self.ommers) } + /// Expensive operation that recovers transaction signer. See [SealedBlockWithSenders]. + pub fn senders(&self) -> Option> { + self.body.iter().map(|tx| tx.recover_signer()).collect::>>() + } + + /// Seal sealed block with recovered transaction senders. + pub fn seal_with_senders(self) -> Option { + let senders = self.senders()?; + Some(SealedBlockWithSenders { block: self, senders }) + } + /// Unseal the block pub fn unseal(self) -> Block { Block { @@ -83,6 +106,52 @@ impl Deref for SealedBlock { } } +#[cfg(any(test, feature = "test-utils"))] +impl std::ops::DerefMut for SealedBlock { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.header + } +} + +/// Sealed block with senders recovered from transactions. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct SealedBlockWithSenders { + /// Sealed block + pub block: SealedBlock, + /// List of senders that match trasanctions from block. + pub senders: Vec
, +} + +impl SealedBlockWithSenders { + /// New sealed block with sender. Return none if len of tx and senders does not match + pub fn new(block: SealedBlock, senders: Vec
) -> Option { + if block.body.len() != senders.len() { + None + } else { + Some(Self { block, senders }) + } + } + + /// Split Structure to its components + pub fn into_components(self) -> (SealedBlock, Vec
) { + (self.block, self.senders) + } +} + +impl Deref for SealedBlockWithSenders { + type Target = SealedBlock; + fn deref(&self) -> &Self::Target { + &self.block + } +} + +#[cfg(any(test, feature = "test-utils"))] +impl std::ops::DerefMut for SealedBlockWithSenders { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.block + } +} + /// Either a block hash _or_ a block number #[derive_arbitrary(rlp)] #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] diff --git a/crates/primitives/src/header.rs b/crates/primitives/src/header.rs index 951d1f46ded..012a5ca4a46 100644 --- a/crates/primitives/src/header.rs +++ b/crates/primitives/src/header.rs @@ -8,7 +8,7 @@ use ethers_core::types::{Block, H256 as EthersH256, H64}; use reth_codecs::{add_arbitrary_tests, derive_arbitrary, main_codec, Compact}; use reth_rlp::{length_of_length, Decodable, Encodable, EMPTY_STRING_CODE}; use serde::{Deserialize, Serialize}; -use std::ops::Deref; +use std::ops::{Deref, DerefMut}; /// Describes the current head block. /// @@ -284,9 +284,9 @@ impl Decodable for Header { #[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct SealedHeader { /// Locked Header fields. - header: Header, + pub header: Header, /// Locked Header hash. - hash: BlockHash, + pub hash: BlockHash, } impl SealedHeader { @@ -404,6 +404,12 @@ impl Deref for SealedHeader { } } +impl DerefMut for SealedHeader { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.header + } +} + /// Represents the direction for a headers request depending on the `reverse` field of the request. /// > The response must contain a number of block headers, of rising number when reverse is 0, /// > falling when 1 diff --git a/crates/primitives/src/lib.rs b/crates/primitives/src/lib.rs index 13d68dbfc03..6c6585e067d 100644 --- a/crates/primitives/src/lib.rs +++ b/crates/primitives/src/lib.rs @@ -38,7 +38,9 @@ pub use proofs::ProofCheckpoint; pub use account::{Account, Bytecode}; pub use bits::H512; -pub use block::{Block, BlockHashOrNumber, BlockId, BlockNumberOrTag, SealedBlock}; +pub use block::{ + Block, BlockHashOrNumber, BlockId, BlockNumberOrTag, SealedBlock, SealedBlockWithSenders, +}; pub use bloom::Bloom; pub use chain::{ AllGenesisFormats, Chain, ChainInfo, ChainSpec, ChainSpecBuilder, ForkCondition, GOERLI, @@ -124,3 +126,6 @@ pub fn keccak256(data: impl AsRef<[u8]>) -> H256 { hasher.finalize(&mut buf); buf.into() } + +#[cfg(any(test, feature = "arbitrary"))] +pub use arbitrary; diff --git a/crates/primitives/src/transaction/mod.rs b/crates/primitives/src/transaction/mod.rs index 41668749ba3..15213623ba4 100644 --- a/crates/primitives/src/transaction/mod.rs +++ b/crates/primitives/src/transaction/mod.rs @@ -871,6 +871,11 @@ impl TransactionSignedEcRecovered { self.signed_transaction } + /// Desolve Self to its component + pub fn to_components(self) -> (TransactionSigned, Address) { + (self.signed_transaction, self.signer) + } + /// Create [`TransactionSignedEcRecovered`] from [`TransactionSigned`] and [`Address`] of the /// signer. pub fn from_signed_transaction(signed_transaction: TransactionSigned, signer: Address) -> Self { diff --git a/crates/rpc/rpc-engine-api/src/engine_api.rs b/crates/rpc/rpc-engine-api/src/engine_api.rs index 2e85330f69d..31044a96d7f 100644 --- a/crates/rpc/rpc-engine-api/src/engine_api.rs +++ b/crates/rpc/rpc-engine-api/src/engine_api.rs @@ -53,13 +53,13 @@ impl, message_rx: mpsc::UnboundedReceiver, forkchoice_state_tx: watch::Sender, ) -> Self { Self { client, - chain_spec: Arc::new(chain_spec), + chain_spec, message_rx: UnboundedReceiverStream::new(message_rx), forkchoice_state_tx, } diff --git a/crates/staged-sync/src/utils/chainspec.rs b/crates/staged-sync/src/utils/chainspec.rs index 27151bbbd9d..6d089741249 100644 --- a/crates/staged-sync/src/utils/chainspec.rs +++ b/crates/staged-sync/src/utils/chainspec.rs @@ -1,10 +1,10 @@ use reth_primitives::{AllGenesisFormats, ChainSpec, GOERLI, MAINNET, SEPOLIA}; -use std::path::PathBuf; +use std::{path::PathBuf, sync::Arc}; /// Clap value parser for [ChainSpec]s that takes either a built-in chainspec or the path /// to a custom one. -pub fn chain_spec_value_parser(s: &str) -> Result { - Ok(match s { +pub fn chain_spec_value_parser(s: &str) -> Result, eyre::Error> { + Ok(Arc::new(match s { "mainnet" => MAINNET.clone(), "goerli" => GOERLI.clone(), "sepolia" => SEPOLIA.clone(), @@ -12,13 +12,13 @@ pub fn chain_spec_value_parser(s: &str) -> Result { let raw = std::fs::read_to_string(PathBuf::from(shellexpand::full(s)?.into_owned()))?; serde_json::from_str(&raw)? } - }) + })) } /// Clap value parser for [ChainSpec]s that takes either a built-in genesis format or the path /// to a custom one. -pub fn genesis_value_parser(s: &str) -> Result { - Ok(match s { +pub fn genesis_value_parser(s: &str) -> Result, eyre::Error> { + Ok(Arc::new(match s { "mainnet" => MAINNET.clone(), "goerli" => GOERLI.clone(), "sepolia" => SEPOLIA.clone(), @@ -27,7 +27,7 @@ pub fn genesis_value_parser(s: &str) -> Result { let genesis: AllGenesisFormats = serde_json::from_str(&raw)?; genesis.into() } - }) + })) } #[cfg(test)] diff --git a/crates/staged-sync/src/utils/init.rs b/crates/staged-sync/src/utils/init.rs index a43514881c4..d911d5d2297 100644 --- a/crates/staged-sync/src/utils/init.rs +++ b/crates/staged-sync/src/utils/init.rs @@ -37,7 +37,7 @@ pub enum InitDatabaseError { #[allow(clippy::field_reassign_with_default)] pub fn init_genesis( db: Arc, - chain: ChainSpec, + chain: Arc, ) -> Result { let genesis = chain.genesis(); @@ -85,6 +85,8 @@ pub fn init_genesis( #[cfg(test)] mod tests { + use std::sync::Arc; + use super::{init_genesis, InitDatabaseError}; use reth_db::mdbx::test_utils::create_test_rw_db; use reth_primitives::{ @@ -94,7 +96,7 @@ mod tests { #[test] fn success_init_genesis_mainnet() { let db = create_test_rw_db(); - let genesis_hash = init_genesis(db, MAINNET.clone()).unwrap(); + let genesis_hash = init_genesis(db, Arc::new(MAINNET.clone())).unwrap(); // actual, expected assert_eq!(genesis_hash, MAINNET_GENESIS); @@ -103,7 +105,7 @@ mod tests { #[test] fn success_init_genesis_goerli() { let db = create_test_rw_db(); - let genesis_hash = init_genesis(db, GOERLI.clone()).unwrap(); + let genesis_hash = init_genesis(db, Arc::new(GOERLI.clone())).unwrap(); // actual, expected assert_eq!(genesis_hash, GOERLI_GENESIS); @@ -112,7 +114,7 @@ mod tests { #[test] fn success_init_genesis_sepolia() { let db = create_test_rw_db(); - let genesis_hash = init_genesis(db, SEPOLIA.clone()).unwrap(); + let genesis_hash = init_genesis(db, Arc::new(SEPOLIA.clone())).unwrap(); // actual, expected assert_eq!(genesis_hash, SEPOLIA_GENESIS); @@ -121,10 +123,10 @@ mod tests { #[test] fn fail_init_inconsistent_db() { let db = create_test_rw_db(); - init_genesis(db.clone(), SEPOLIA.clone()).unwrap(); + init_genesis(db.clone(), Arc::new(SEPOLIA.clone())).unwrap(); // Try to init db with a different genesis block - let genesis_hash = init_genesis(db, MAINNET.clone()); + let genesis_hash = init_genesis(db, Arc::new(MAINNET.clone())); assert_eq!( genesis_hash.unwrap_err(), diff --git a/crates/staged-sync/tests/sync.rs b/crates/staged-sync/tests/sync.rs index 4bb5d07f0fb..96170fab1a4 100644 --- a/crates/staged-sync/tests/sync.rs +++ b/crates/staged-sync/tests/sync.rs @@ -57,7 +57,7 @@ async fn can_peer_with_geth() { assert_eq!(geth_peer_id, peer_id); } -async fn init_geth() -> (CliqueGethInstance, ChainSpec) { +async fn init_geth() -> (CliqueGethInstance, Arc) { // first create a signer that we will fund so we can make transactions let chain_id = 13337u64; let data_dir = tempfile::tempdir().expect("should be able to create temp geth datadir"); @@ -123,5 +123,5 @@ async fn init_geth() -> (CliqueGethInstance, ChainSpec) { let block = clique.provider.get_block_number().await.unwrap(); assert!(block > U64::zero()); - (clique, chainspec) + (clique, Arc::new(chainspec)) } diff --git a/crates/stages/Cargo.toml b/crates/stages/Cargo.toml index 507ed4cfdf9..b0ec65911bb 100644 --- a/crates/stages/Cargo.toml +++ b/crates/stages/Cargo.toml @@ -35,7 +35,7 @@ metrics = "0.20.1" # misc serde = { version = "1.0", optional = true } thiserror = "1.0.37" -aquamarine = "0.2.1" +aquamarine = "0.2.1" #docs itertools = "0.10.5" rayon = "1.6.0" num-traits = "0.2.15" diff --git a/crates/stages/src/stages/execution.rs b/crates/stages/src/stages/execution.rs index 0f28ce87b42..e08caa69d7a 100644 --- a/crates/stages/src/stages/execution.rs +++ b/crates/stages/src/stages/execution.rs @@ -324,8 +324,8 @@ mod tests { let genesis = SealedBlock::decode(&mut genesis_rlp).unwrap(); let mut block_rlp = hex!("f90262f901f9a075c371ba45999d87f4542326910a11af515897aebce5265d3f6acd1f1161f82fa01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347942adc25665018aa1fe0e6bc666dac8fc2697ff9baa098f2dcd87c8ae4083e7017a05456c14eea4b1db2032126e27b3b1563d57d7cc0a08151d548273f6683169524b66ca9fe338b9ce42bc3540046c828fd939ae23bcba03f4e5c2ec5b2170b711d97ee755c160457bb58d8daa338e835ec02ae6860bbabb901000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000083020000018502540be40082a8798203e800a00000000000000000000000000000000000000000000000000000000000000000880000000000000000f863f861800a8405f5e10094100000000000000000000000000000000000000080801ba07e09e26678ed4fac08a249ebe8ed680bf9051a5e14ad223e4b2b9d26e0208f37a05f6e3f188e3e6eab7d7d3b6568f5eac7d687b08d307d3154ccd8c87b4630509bc0").as_slice(); let block = SealedBlock::decode(&mut block_rlp).unwrap(); - insert_canonical_block(tx.deref_mut(), &genesis, true).unwrap(); - insert_canonical_block(tx.deref_mut(), &block, true).unwrap(); + insert_canonical_block(tx.deref_mut(), genesis, None, true).unwrap(); + insert_canonical_block(tx.deref_mut(), block.clone(), None, true).unwrap(); tx.commit().unwrap(); // insert pre state @@ -413,8 +413,8 @@ mod tests { let genesis = SealedBlock::decode(&mut genesis_rlp).unwrap(); let mut block_rlp = hex!("f90262f901f9a075c371ba45999d87f4542326910a11af515897aebce5265d3f6acd1f1161f82fa01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347942adc25665018aa1fe0e6bc666dac8fc2697ff9baa098f2dcd87c8ae4083e7017a05456c14eea4b1db2032126e27b3b1563d57d7cc0a08151d548273f6683169524b66ca9fe338b9ce42bc3540046c828fd939ae23bcba03f4e5c2ec5b2170b711d97ee755c160457bb58d8daa338e835ec02ae6860bbabb901000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000083020000018502540be40082a8798203e800a00000000000000000000000000000000000000000000000000000000000000000880000000000000000f863f861800a8405f5e10094100000000000000000000000000000000000000080801ba07e09e26678ed4fac08a249ebe8ed680bf9051a5e14ad223e4b2b9d26e0208f37a05f6e3f188e3e6eab7d7d3b6568f5eac7d687b08d307d3154ccd8c87b4630509bc0").as_slice(); let block = SealedBlock::decode(&mut block_rlp).unwrap(); - insert_canonical_block(tx.deref_mut(), &genesis, true).unwrap(); - insert_canonical_block(tx.deref_mut(), &block, true).unwrap(); + insert_canonical_block(tx.deref_mut(), genesis, None, true).unwrap(); + insert_canonical_block(tx.deref_mut(), block.clone(), None, true).unwrap(); tx.commit().unwrap(); // variables @@ -480,8 +480,8 @@ mod tests { let genesis = SealedBlock::decode(&mut genesis_rlp).unwrap(); let mut block_rlp = hex!("f9025ff901f7a0c86e8cc0310ae7c531c758678ddbfd16fc51c8cef8cec650b032de9869e8b94fa01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347942adc25665018aa1fe0e6bc666dac8fc2697ff9baa050554882fbbda2c2fd93fdc466db9946ea262a67f7a76cc169e714f105ab583da00967f09ef1dfed20c0eacfaa94d5cd4002eda3242ac47eae68972d07b106d192a0e3c8b47fbfc94667ef4cceb17e5cc21e3b1eebd442cebb27f07562b33836290db90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008302000001830f42408238108203e800a00000000000000000000000000000000000000000000000000000000000000000880000000000000000f862f860800a83061a8094095e7baea6a6c7c4c2dfeb977efac326af552d8780801ba072ed817487b84ba367d15d2f039b5fc5f087d0a8882fbdf73e8cb49357e1ce30a0403d800545b8fc544f92ce8124e2255f8c3c6af93f28243a120585d4c4c6a2a3c0").as_slice(); let block = SealedBlock::decode(&mut block_rlp).unwrap(); - insert_canonical_block(tx.deref_mut(), &genesis, true).unwrap(); - insert_canonical_block(tx.deref_mut(), &block, true).unwrap(); + insert_canonical_block(tx.deref_mut(), genesis, None, true).unwrap(); + insert_canonical_block(tx.deref_mut(), block.clone(), None, true).unwrap(); tx.commit().unwrap(); // variables diff --git a/crates/stages/src/stages/hashing_account.rs b/crates/stages/src/stages/hashing_account.rs index ceaedf30c05..a6ea743759a 100644 --- a/crates/stages/src/stages/hashing_account.rs +++ b/crates/stages/src/stages/hashing_account.rs @@ -5,7 +5,7 @@ use reth_db::{ tables, transaction::{DbTx, DbTxMut}, }; -use reth_primitives::{keccak256, Account, Address}; +use reth_primitives::keccak256; use reth_provider::Transaction; use std::{collections::BTreeMap, fmt::Debug, ops::Range}; use tracing::*; @@ -63,12 +63,12 @@ impl AccountHashingStage { pub fn seed( tx: &mut Transaction<'_, DB>, opts: SeedOpts, - ) -> Result, StageError> { + ) -> Result, StageError> { use reth_db::models::AccountBeforeTx; use reth_interfaces::test_utils::generators::{ random_block_range, random_eoa_account_range, }; - use reth_primitives::{H256, U256}; + use reth_primitives::{Account, H256, U256}; use reth_provider::insert_canonical_block; let blocks = random_block_range(opts.blocks, H256::zero(), opts.txs); @@ -76,7 +76,7 @@ impl AccountHashingStage { let transitions = std::cmp::min(opts.transitions, num_transitions); for block in blocks { - insert_canonical_block(&**tx, &block, true).unwrap(); + insert_canonical_block(&**tx, block, None, true).unwrap(); } let mut accounts = random_eoa_account_range(opts.accounts); { @@ -203,37 +203,8 @@ impl Stage for AccountHashingStage { let from_transition_rev = tx.get_block_transition(input.unwind_to)?; let to_transition_rev = tx.get_block_transition(input.stage_progress)?; - let mut hashed_accounts = tx.cursor_write::()?; - // Aggregate all transition changesets and and make list of account that have been changed. - tx.cursor_read::()? - .walk_range(from_transition_rev..to_transition_rev)? - .collect::, _>>()? - .into_iter() - .rev() - // fold all account to get the old balance/nonces and account that needs to be removed - .fold( - BTreeMap::new(), - |mut accounts: BTreeMap>, (_, account_before)| { - accounts.insert(account_before.address, account_before.info); - accounts - }, - ) - .into_iter() - // hash addresses and collect it inside sorted BTreeMap. - // We are doing keccak only once per address. - .map(|(address, account)| (keccak256(address), account)) - .collect::>() - .into_iter() - // Apply values to HashedState (if Account is None remove it); - .try_for_each(|(hashed_address, account)| -> Result<(), StageError> { - if let Some(account) = account { - hashed_accounts.upsert(hashed_address, account)?; - } else if hashed_accounts.seek_exact(hashed_address)?.is_some() { - hashed_accounts.delete_current()?; - } - Ok(()) - })?; + tx.unwind_account_hashing(from_transition_rev..to_transition_rev)?; Ok(UnwindOutput { stage_progress: input.unwind_to }) } @@ -283,6 +254,7 @@ mod tests { ExecInput, ExecOutput, UnwindInput, }; use reth_db::{cursor::DbCursorRO, tables, transaction::DbTx}; + use reth_primitives::Address; pub(crate) struct AccountHashingTestRunner { pub(crate) tx: TestTransaction, diff --git a/crates/stages/src/stages/hashing_storage.rs b/crates/stages/src/stages/hashing_storage.rs index 7e54b37fcd7..22d3cac1cc0 100644 --- a/crates/stages/src/stages/hashing_storage.rs +++ b/crates/stages/src/stages/hashing_storage.rs @@ -1,13 +1,13 @@ use crate::{ExecInput, ExecOutput, Stage, StageError, StageId, UnwindInput, UnwindOutput}; use num_traits::Zero; use reth_db::{ - cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO}, + cursor::DbDupCursorRO, database::Database, models::TransitionIdAddress, tables, transaction::{DbTx, DbTxMut}, }; -use reth_primitives::{keccak256, Address, StorageEntry, H256, U256}; +use reth_primitives::{keccak256, Address, StorageEntry}; use reth_provider::Transaction; use std::{collections::BTreeMap, fmt::Debug}; use tracing::*; @@ -154,47 +154,10 @@ impl Stage for StorageHashingStage { let from_transition_rev = tx.get_block_transition(input.unwind_to)?; let to_transition_rev = tx.get_block_transition(input.stage_progress)?; - let mut hashed_storage = tx.cursor_dup_write::()?; - - // Aggregate all transition changesets and make list of accounts that have been changed. - tx.cursor_read::()? - .walk_range( - TransitionIdAddress((from_transition_rev, Address::zero())).. - TransitionIdAddress((to_transition_rev, Address::zero())), - )? - .collect::, _>>()? - .into_iter() - .rev() - // fold all account to get the old balance/nonces and account that needs to be removed - .fold( - BTreeMap::new(), - |mut accounts: BTreeMap<(Address, H256), U256>, - (TransitionIdAddress((_, address)), storage_entry)| { - accounts.insert((address, storage_entry.key), storage_entry.value); - accounts - }, - ) - .into_iter() - // hash addresses and collect it inside sorted BTreeMap. - // We are doing keccak only once per address. - .map(|((address, key), value)| ((keccak256(address), keccak256(key)), value)) - .collect::>() - .into_iter() - // Apply values to HashedStorage (if Value is zero just remove it); - .try_for_each(|((hashed_address, key), value)| -> Result<(), StageError> { - if hashed_storage - .seek_by_key_subkey(hashed_address, key)? - .filter(|entry| entry.key == key) - .is_some() - { - hashed_storage.delete_current()?; - } - - if value != U256::ZERO { - hashed_storage.upsert(hashed_address, StorageEntry { key, value })?; - } - Ok(()) - })?; + tx.unwind_storage_hashing( + TransitionIdAddress((from_transition_rev, Address::zero())).. + TransitionIdAddress((to_transition_rev, Address::zero())), + )?; Ok(UnwindOutput { stage_progress: input.unwind_to }) } @@ -209,7 +172,7 @@ mod tests { }; use assert_matches::assert_matches; use reth_db::{ - cursor::DbCursorRW, + cursor::{DbCursorRO, DbCursorRW}, mdbx::{tx::Tx, WriteMap, RW}, models::{StoredBlockBody, TransitionIdAddress}, }; diff --git a/crates/stages/src/stages/index_account_history.rs b/crates/stages/src/stages/index_account_history.rs index 7cfa4fdcd9d..55ea9a3233d 100644 --- a/crates/stages/src/stages/index_account_history.rs +++ b/crates/stages/src/stages/index_account_history.rs @@ -1,16 +1,7 @@ use crate::{ExecInput, ExecOutput, Stage, StageError, StageId, UnwindInput, UnwindOutput}; -use reth_db::{ - cursor::{DbCursorRO, DbCursorRW}, - database::{Database, DatabaseGAT}, - models::ShardedKey, - tables, - transaction::{DbTx, DbTxMut, DbTxMutGAT}, - TransitionList, -}; +use reth_db::database::Database; use reth_provider::Transaction; - -use reth_primitives::{Address, TransitionId}; -use std::{collections::BTreeMap, fmt::Debug}; +use std::fmt::Debug; use tracing::*; /// The [`StageId`] of the account history indexing stage. @@ -18,7 +9,7 @@ pub const INDEX_ACCOUNT_HISTORY: StageId = StageId("IndexAccountHistory"); /// Stage is indexing history the account changesets generated in /// [`ExecutionStage`][crate::stages::ExecutionStage]. For more information -/// on index sharding take a look at [`tables::AccountHistory`] +/// on index sharding take a look at [`reth_db::tables::AccountHistory`] #[derive(Debug)] pub struct IndexAccountHistoryStage { /// Number of blocks after which the control @@ -75,84 +66,25 @@ impl Stage for IndexAccountHistoryStage { let from_transition_rev = tx.get_block_transition(input.unwind_to)?; let to_transition_rev = tx.get_block_transition(input.stage_progress)?; - let mut cursor = tx.cursor_write::()?; - - let account_changeset = tx - .cursor_read::()? - .walk(Some(from_transition_rev))? - .take_while(|res| res.as_ref().map(|(k, _)| *k < to_transition_rev).unwrap_or_default()) - .collect::, _>>()?; + tx.unwind_account_history_indices(from_transition_rev..to_transition_rev)?; - let last_indices = account_changeset - .into_iter() - // reverse so we can get lowest transition id where we need to unwind account. - .rev() - // fold all account and get last transition index - .fold(BTreeMap::new(), |mut accounts: BTreeMap, (index, account)| { - // we just need address and lowest transition id. - accounts.insert(account.address, index); - accounts - }); - // try to unwind the index - for (address, rem_index) in last_indices { - let shard_part = unwind_account_history_shards::(&mut cursor, address, rem_index)?; - - // check last shard_part, if present, items needs to be reinserted. - if !shard_part.is_empty() { - // there are items in list - tx.put::( - ShardedKey::new(address, u64::MAX), - TransitionList::new(shard_part) - .expect("There is at least one element in list and it is sorted."), - )?; - } - } // from HistoryIndex higher than that number. Ok(UnwindOutput { stage_progress: input.unwind_to }) } } -/// Unwind all history shards. For boundary shard, remove it from database and -/// return last part of shard with still valid items. If all full shard were removed, return list -/// would be empty. -pub fn unwind_account_history_shards( - cursor: &mut <>::TXMut as DbTxMutGAT<'_>>::CursorMut< - tables::AccountHistory, - >, - address: Address, - transition_id: TransitionId, -) -> Result, StageError> { - let mut item = cursor.seek_exact(ShardedKey::new(address, u64::MAX))?; - - while let Some((sharded_key, list)) = item { - // there is no more shard for address - if sharded_key.key != address { - break - } - cursor.delete_current()?; - // check first item and if it is more and eq than `transition_id` delete current - // item. - let first = list.iter(0).next().expect("List can't empty"); - if first >= transition_id as usize { - item = cursor.prev()?; - continue - } else if transition_id <= sharded_key.highest_transition_id { - // if first element is in scope whole list would be removed. - // so at least this first element is present. - return Ok(list.iter(0).take_while(|i| *i < transition_id as usize).collect::>()) - } else { - let new_list = list.iter(0).collect::>(); - return Ok(new_list) - } - } - Ok(Vec::new()) -} - #[cfg(test)] mod tests { + use std::collections::BTreeMap; + use super::*; use crate::test_utils::{TestTransaction, PREV_STAGE_ID}; - use reth_db::models::{sharded_key::NUM_OF_INDICES_IN_SHARD, AccountBeforeTx}; + use reth_db::{ + models::{sharded_key::NUM_OF_INDICES_IN_SHARD, AccountBeforeTx, ShardedKey}, + tables, + transaction::DbTxMut, + TransitionList, + }; use reth_primitives::{hex_literal::hex, H160}; const ADDRESS: H160 = H160(hex!("0000000000000000000000000000000000000001")); diff --git a/crates/stages/src/stages/index_storage_history.rs b/crates/stages/src/stages/index_storage_history.rs index e29091030ec..6431d12f58f 100644 --- a/crates/stages/src/stages/index_storage_history.rs +++ b/crates/stages/src/stages/index_storage_history.rs @@ -1,15 +1,8 @@ use crate::{ExecInput, ExecOutput, Stage, StageError, StageId, UnwindInput, UnwindOutput}; -use reth_db::{ - cursor::{DbCursorRO, DbCursorRW}, - database::{Database, DatabaseGAT}, - models::storage_sharded_key::StorageShardedKey, - tables, - transaction::{DbTx, DbTxMut, DbTxMutGAT}, - TransitionList, -}; -use reth_primitives::{Address, TransitionId, H256}; +use reth_db::{database::Database, models::TransitionIdAddress}; +use reth_primitives::Address; use reth_provider::Transaction; -use std::{collections::BTreeMap, fmt::Debug}; +use std::fmt::Debug; use tracing::*; /// The [`StageId`] of the storage history indexing stage. @@ -17,7 +10,7 @@ pub const INDEX_STORAGE_HISTORY: StageId = StageId("IndexStorageHistory"); /// Stage is indexing history the account changesets generated in /// [`ExecutionStage`][crate::stages::ExecutionStage]. For more information -/// on index sharding take a look at [`tables::StorageHistory`]. +/// on index sharding take a look at [`reth_db::tables::StorageHistory`]. #[derive(Debug)] pub struct IndexStorageHistoryStage { /// Number of blocks after which the control @@ -74,94 +67,32 @@ impl Stage for IndexStorageHistoryStage { let from_transition_rev = tx.get_block_transition(input.unwind_to)?; let to_transition_rev = tx.get_block_transition(input.stage_progress)?; - let mut cursor = tx.cursor_write::()?; + tx.unwind_storage_history_indices( + TransitionIdAddress((from_transition_rev, Address::zero())).. + TransitionIdAddress((to_transition_rev, Address::zero())), + )?; - let storage_changesets = tx - .cursor_read::()? - .walk(Some((from_transition_rev, Address::zero()).into()))? - .take_while(|res| { - res.as_ref().map(|(k, _)| k.transition_id() < to_transition_rev).unwrap_or_default() - }) - .collect::, _>>()?; - let last_indices = storage_changesets - .into_iter() - // reverse so we can get lowest transition id where we need to unwind account. - .rev() - // fold all storages and get last transition index - .fold( - BTreeMap::new(), - |mut accounts: BTreeMap<(Address, H256), u64>, (index, storage)| { - // we just need address and lowest transition id. - accounts.insert((index.address(), storage.key), index.transition_id()); - accounts - }, - ); - for ((address, storage_key), rem_index) in last_indices { - let shard_part = - unwind_storage_history_shards::(&mut cursor, address, storage_key, rem_index)?; - - // check last shard_part, if present, items needs to be reinserted. - if !shard_part.is_empty() { - // there are items in list - tx.put::( - StorageShardedKey::new(address, storage_key, u64::MAX), - TransitionList::new(shard_part) - .expect("There is at least one element in list and it is sorted."), - )?; - } - } Ok(UnwindOutput { stage_progress: input.unwind_to }) } } -/// Unwind all history shards. For boundary shard, remove it from database and -/// return last part of shard with still valid items. If all full shard were removed, return list -/// would be empty but this does not mean that there is none shard left but that there is no -/// splitted shards. -pub fn unwind_storage_history_shards( - cursor: &mut <>::TXMut as DbTxMutGAT<'_>>::CursorMut< - tables::StorageHistory, - >, - address: Address, - storage_key: H256, - transition_id: TransitionId, -) -> Result, StageError> { - let mut item = cursor.seek_exact(StorageShardedKey::new(address, storage_key, u64::MAX))?; - - while let Some((storage_sharded_key, list)) = item { - // there is no more shard for address - if storage_sharded_key.address != address || - storage_sharded_key.sharded_key.key != storage_key - { - // there is no more shard for address and storage_key. - break - } - cursor.delete_current()?; - // check first item and if it is more and eq than `transition_id` delete current - // item. - let first = list.iter(0).next().expect("List can't empty"); - if first >= transition_id as usize { - item = cursor.prev()?; - continue - } else if transition_id <= storage_sharded_key.sharded_key.highest_transition_id { - // if first element is in scope whole list would be removed. - // so at least this first element is present. - return Ok(list.iter(0).take_while(|i| *i < transition_id as usize).collect::>()) - } else { - return Ok(list.iter(0).collect::>()) - } - } - Ok(Vec::new()) -} #[cfg(test)] mod tests { + use std::collections::BTreeMap; + use super::*; use crate::test_utils::{TestTransaction, PREV_STAGE_ID}; - use reth_db::models::{ - storage_sharded_key::NUM_OF_INDICES_IN_SHARD, ShardedKey, TransitionIdAddress, + use reth_db::{ + models::{ + storage_sharded_key::{StorageShardedKey, NUM_OF_INDICES_IN_SHARD}, + ShardedKey, TransitionIdAddress, + }, + tables, + transaction::DbTxMut, + TransitionList, }; - use reth_primitives::{hex_literal::hex, StorageEntry, H160, U256}; + use reth_primitives::{hex_literal::hex, StorageEntry, H160, H256, U256}; const ADDRESS: H160 = H160(hex!("0000000000000000000000000000000000000001")); const STORAGE_KEY: H256 = diff --git a/crates/stages/src/stages/tx_lookup.rs b/crates/stages/src/stages/tx_lookup.rs index 7921dbdedc6..54599ff6775 100644 --- a/crates/stages/src/stages/tx_lookup.rs +++ b/crates/stages/src/stages/tx_lookup.rs @@ -111,9 +111,9 @@ impl Stage for TransactionLookupStage { ) -> Result { info!(target: "sync::stages::transaction_lookup", to_block = input.unwind_to, "Unwinding"); // Cursors to unwind tx hash to number - let mut body_cursor = tx.cursor_write::()?; + let mut body_cursor = tx.cursor_read::()?; let mut tx_hash_number_cursor = tx.cursor_write::()?; - let mut transaction_cursor = tx.cursor_write::()?; + let mut transaction_cursor = tx.cursor_read::()?; let mut rev_walker = body_cursor.walk_back(None)?; while let Some((number, body)) = rev_walker.next().transpose()? { if number <= input.unwind_to { diff --git a/crates/stages/src/test_utils/test_db.rs b/crates/stages/src/test_utils/test_db.rs index bf1a0043f28..3d0f299f89e 100644 --- a/crates/stages/src/test_utils/test_db.rs +++ b/crates/stages/src/test_utils/test_db.rs @@ -1,4 +1,5 @@ use reth_db::{ + common::KeyValue, cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO}, mdbx::{ test_utils::{create_test_db, create_test_db_with_path}, @@ -88,9 +89,8 @@ impl TestTransaction { }) } - #[allow(clippy::type_complexity)] /// Return full table as Vec - pub fn table(&self) -> Result, DbError> + pub fn table(&self) -> Result>, DbError> where T::Key: Default + Ord, { diff --git a/crates/storage/db/src/abstraction/common.rs b/crates/storage/db/src/abstraction/common.rs index 2e990e99ee2..f654a69c092 100644 --- a/crates/storage/db/src/abstraction/common.rs +++ b/crates/storage/db/src/abstraction/common.rs @@ -1,7 +1,9 @@ +/// Alias type containing key value pairs. +pub type KeyValue = (::Key, ::Value); /// Alias type for a `(key, value)` result coming from a cursor. -pub type PairResult = Result::Key, ::Value)>, Error>; +pub type PairResult = Result>, Error>; /// Alias type for a `(key, value)` result coming from an iterator. -pub type IterPairResult = Option::Key, ::Value), Error>>; +pub type IterPairResult = Option, Error>>; /// Alias type for a value result coming from a cursor without its key. pub type ValueOnlyResult = Result::Value>, Error>; diff --git a/crates/storage/db/src/abstraction/cursor.rs b/crates/storage/db/src/abstraction/cursor.rs index 3e28b78c14a..16d6e147f6b 100644 --- a/crates/storage/db/src/abstraction/cursor.rs +++ b/crates/storage/db/src/abstraction/cursor.rs @@ -159,6 +159,15 @@ impl<'cursor, 'tx, T: Table, CURSOR: DbCursorRO<'tx, T>> Walker<'cursor, 'tx, T, } } +impl<'cursor, 'tx, T: Table, CURSOR: DbCursorRW<'tx, T> + DbCursorRO<'tx, T>> + Walker<'cursor, 'tx, T, CURSOR> +{ + /// Delete current item that walker points to. + pub fn delete_current(&mut self) -> Result<(), Error> { + self.cursor.delete_current() + } +} + /// Provides a reverse iterator to `Cursor` when handling `Table`. /// Also check [`Walker`] pub struct ReverseWalker<'cursor, 'tx, T: Table, CURSOR: DbCursorRO<'tx, T>> { @@ -183,6 +192,15 @@ impl<'cursor, 'tx, T: Table, CURSOR: DbCursorRO<'tx, T>> ReverseWalker<'cursor, } } +impl<'cursor, 'tx, T: Table, CURSOR: DbCursorRW<'tx, T> + DbCursorRO<'tx, T>> + ReverseWalker<'cursor, 'tx, T, CURSOR> +{ + /// Delete current item that walker points to. + pub fn delete_current(&mut self) -> Result<(), Error> { + self.cursor.delete_current() + } +} + impl<'cursor, 'tx, T: Table, CURSOR: DbCursorRO<'tx, T>> std::iter::Iterator for ReverseWalker<'cursor, 'tx, T, CURSOR> { @@ -268,6 +286,15 @@ impl<'cursor, 'tx, T: Table, CURSOR: DbCursorRO<'tx, T>> RangeWalker<'cursor, 't } } +impl<'cursor, 'tx, T: Table, CURSOR: DbCursorRW<'tx, T> + DbCursorRO<'tx, T>> + RangeWalker<'cursor, 'tx, T, CURSOR> +{ + /// Delete current item that walker points to. + pub fn delete_current(&mut self) -> Result<(), Error> { + self.cursor.delete_current() + } +} + /// Provides an iterator to `Cursor` when handling a `DupSort` table. /// /// Reason why we have two lifetimes is to distinguish between `'cursor` lifetime @@ -282,6 +309,15 @@ pub struct DupWalker<'cursor, 'tx, T: DupSort, CURSOR: DbDupCursorRO<'tx, T>> { pub _tx_phantom: PhantomData<&'tx T>, } +impl<'cursor, 'tx, T: DupSort, CURSOR: DbCursorRW<'tx, T> + DbDupCursorRO<'tx, T>> + DupWalker<'cursor, 'tx, T, CURSOR> +{ + /// Delete current item that walker points to. + pub fn delete_current(&mut self) -> Result<(), Error> { + self.cursor.delete_current() + } +} + impl<'cursor, 'tx, T: DupSort, CURSOR: DbDupCursorRO<'tx, T>> std::iter::Iterator for DupWalker<'cursor, 'tx, T, CURSOR> { diff --git a/crates/storage/db/src/abstraction/database.rs b/crates/storage/db/src/abstraction/database.rs index 6a8a9aac673..f6559ac2431 100644 --- a/crates/storage/db/src/abstraction/database.rs +++ b/crates/storage/db/src/abstraction/database.rs @@ -43,7 +43,7 @@ pub trait Database: for<'a> DatabaseGAT<'a> { /// the end of the execution. fn update(&self, f: F) -> Result where - F: Fn(&>::TXMut) -> T, + F: FnOnce(&>::TXMut) -> T, { let tx = self.tx_mut()?; diff --git a/crates/storage/db/src/implementation/mdbx/cursor.rs b/crates/storage/db/src/implementation/mdbx/cursor.rs index cce188f8411..64730b7550b 100644 --- a/crates/storage/db/src/implementation/mdbx/cursor.rs +++ b/crates/storage/db/src/implementation/mdbx/cursor.rs @@ -3,6 +3,7 @@ use std::{borrow::Cow, collections::Bound, marker::PhantomData, ops::RangeBounds}; use crate::{ + common::{PairResult, ValueOnlyResult}, cursor::{ DbCursorRO, DbCursorRW, DbDupCursorRO, DbDupCursorRW, DupWalker, RangeWalker, ReverseWalker, Walker, @@ -13,13 +14,6 @@ use crate::{ }; use reth_libmdbx::{self, Error as MDBXError, TransactionKind, WriteFlags, RO, RW}; -/// Alias type for a `(key, value)` result coming from a cursor. -pub type PairResult = Result::Key, ::Value)>, Error>; -/// Alias type for a `(key, value)` result coming from an iterator. -pub type IterPairResult = Option::Key, ::Value), Error>>; -/// Alias type for a value result coming from a cursor without its key. -pub type ValueOnlyResult = Result::Value>, Error>; - /// Read only Cursor. pub type CursorRO<'tx, T> = Cursor<'tx, RO, T>; /// Read write cursor. diff --git a/crates/storage/db/src/tables/models/blocks.rs b/crates/storage/db/src/tables/models/blocks.rs index 87226a2bd64..9d2e48b938a 100644 --- a/crates/storage/db/src/tables/models/blocks.rs +++ b/crates/storage/db/src/tables/models/blocks.rs @@ -23,7 +23,7 @@ pub type NumTransactions = u64; pub struct StoredBlockBody { /// The id of the first transaction in this block pub start_tx_id: TxNumber, - /// The total number of transactions + /// The total number of transactions in the block pub tx_count: NumTransactions, } @@ -40,10 +40,20 @@ impl StoredBlockBody { self.start_tx_id.saturating_add(self.tx_count).saturating_sub(1) } + /// First transaction index. + pub fn first_tx_index(&self) -> TxNumber { + self.start_tx_id + } + /// Return a flag whether the block is empty pub fn is_empty(&self) -> bool { self.tx_count == 0 } + + /// Return number of transaction inside block + pub fn tx_count(&self) -> NumTransactions { + self.tx_count + } } /// The storage representation of a block ommers. diff --git a/crates/storage/provider/Cargo.toml b/crates/storage/provider/Cargo.toml index 7cfe4eaffbf..8a060751c88 100644 --- a/crates/storage/provider/Cargo.toml +++ b/crates/storage/provider/Cargo.toml @@ -33,6 +33,7 @@ parking_lot = { version = "0.12", optional = true } [dev-dependencies] reth-db = { path = "../db", features = ["test-utils"] } +reth-primitives = { path = "../../primitives", features = ["arbitrary"] } parking_lot = "0.12" proptest = { version = "1.0" } assert_matches = "1.5" diff --git a/crates/storage/provider/src/execution_result.rs b/crates/storage/provider/src/execution_result.rs index 3c0fcc18dd2..5a5d3cc744d 100644 --- a/crates/storage/provider/src/execution_result.rs +++ b/crates/storage/provider/src/execution_result.rs @@ -7,7 +7,7 @@ use std::collections::BTreeMap; /// Execution Result containing vector of transaction changesets /// and block reward if present -#[derive(Debug)] +#[derive(Debug, Default, Eq, PartialEq, Clone)] pub struct ExecutionResult { /// Transaction changeset containing [Receipt], changed [Accounts][Account] and Storages. pub tx_changesets: Vec, @@ -20,7 +20,7 @@ pub struct ExecutionResult { /// transaction [Receipt] every change to state ([Account], Storage, [Bytecode]) /// that this transaction made and its old values /// so that history account table can be updated. -#[derive(Debug, Clone)] +#[derive(Debug, Eq, PartialEq, Clone)] pub struct TransactionChangeSet { /// Transaction receipt pub receipt: Receipt, @@ -33,7 +33,9 @@ pub struct TransactionChangeSet { /// Contains old/new account changes #[derive(Debug, Clone, Eq, PartialEq)] pub enum AccountInfoChangeSet { - /// The account is newly created. + /// The account is newly created. Account can be created by just by sending balance, + /// + /// Revert of this changeset is empty account, Created { /// The newly created account. new: Account, @@ -41,11 +43,15 @@ pub enum AccountInfoChangeSet { /// An account was deleted (selfdestructed) or we have touched /// an empty account and we need to remove/destroy it. /// (Look at state clearing [EIP-158](https://eips.ethereum.org/EIPS/eip-158)) + /// + /// Revert of this changeset is old account Destroyed { /// The account that was destroyed. old: Account, }, /// The account was changed. + /// + /// revert of this changeset is old account Changed { /// The account after the change. new: Account, @@ -54,12 +60,34 @@ pub enum AccountInfoChangeSet { }, /// Nothing was changed for the account (nonce/balance). NoChange { - /// Useful to clear existing empty accounts pre-EIP-161. + /// Used to clear existing empty accounts pre-EIP-161. is_empty: bool, }, } +impl Default for AccountInfoChangeSet { + fn default() -> Self { + AccountInfoChangeSet::NoChange { is_empty: false } + } +} + impl AccountInfoChangeSet { + /// Create new account info changeset + pub fn new(old: Option, new: Option) -> Self { + match (old, new) { + (Some(old), Some(new)) => { + if new != old { + Self::Changed { new, old } + } else { + if new.is_empty() {} + Self::NoChange { is_empty: true } + } + } + (None, Some(new)) => Self::Created { new }, + (Some(old), None) => Self::Destroyed { old }, + (None, None) => Self::NoChange { is_empty: false }, + } + } /// Apply the changes from the changeset to a database transaction. pub fn apply_to_db<'a, TX: DbTxMut<'a>>( self, @@ -108,7 +136,7 @@ impl AccountInfoChangeSet { } /// Diff change set that is needed for creating history index and updating current world state. -#[derive(Debug, Clone)] +#[derive(Debug, Default, Eq, PartialEq, Clone)] pub struct AccountChangeSet { /// Old and New account account change. pub account: AccountInfoChangeSet, diff --git a/crates/storage/provider/src/providers/mod.rs b/crates/storage/provider/src/providers/mod.rs index ce50dba0cd0..6b090dba425 100644 --- a/crates/storage/provider/src/providers/mod.rs +++ b/crates/storage/provider/src/providers/mod.rs @@ -40,8 +40,8 @@ pub struct ShareableDatabase { impl ShareableDatabase { /// create new database provider - pub fn new(db: DB, chain_spec: ChainSpec) -> Self { - Self { db, chain_spec: Arc::new(chain_spec) } + pub fn new(db: DB, chain_spec: Arc) -> Self { + Self { db, chain_spec } } } @@ -366,6 +366,8 @@ impl StateProviderFactory for ShareableDatabase { #[cfg(test)] mod tests { + use std::sync::Arc; + use super::ShareableDatabase; use crate::{BlockIdProvider, StateProviderFactory}; use reth_db::mdbx::{test_utils::create_test_db, EnvKind, WriteMap}; @@ -375,7 +377,7 @@ mod tests { fn common_history_provider() { let chain_spec = ChainSpecBuilder::mainnet().build(); let db = create_test_db::(EnvKind::RW); - let provider = ShareableDatabase::new(db, chain_spec); + let provider = ShareableDatabase::new(db, Arc::new(chain_spec)); let _ = provider.latest(); } @@ -383,7 +385,7 @@ mod tests { fn default_chain_info() { let chain_spec = ChainSpecBuilder::mainnet().build(); let db = create_test_db::(EnvKind::RW); - let provider = ShareableDatabase::new(db, chain_spec); + let provider = ShareableDatabase::new(db, Arc::new(chain_spec)); let chain_info = provider.chain_info().expect("should be ok"); assert_eq!(chain_info.best_number, 0); diff --git a/crates/storage/provider/src/providers/state/historical.rs b/crates/storage/provider/src/providers/state/historical.rs index fbfc248da1a..99819caebc1 100644 --- a/crates/storage/provider/src/providers/state/historical.rs +++ b/crates/storage/provider/src/providers/state/historical.rs @@ -126,7 +126,7 @@ impl<'a, 'b, TX: DbTx<'a>> StateProvider for HistoricalStateProviderRef<'a, 'b, _address: Address, _keys: &[H256], ) -> Result<(Vec, H256, Vec>)> { - todo!("this should retrieve past state info and generate proof") + Err(ProviderError::HistoryStateRoot.into()) } } diff --git a/crates/storage/provider/src/providers/state/latest.rs b/crates/storage/provider/src/providers/state/latest.rs index 7d277427771..cf20585beee 100644 --- a/crates/storage/provider/src/providers/state/latest.rs +++ b/crates/storage/provider/src/providers/state/latest.rs @@ -77,7 +77,7 @@ impl<'a, 'b, TX: DbTx<'a>> StateProvider for LatestStateProviderRef<'a, 'b, TX> let (account_proof, storage_root) = loader .generate_acount_proof(root, hashed_address) - .map_err(|_| ProviderError::StateTree)?; + .map_err(|_| ProviderError::StateTrie)?; let account_proof = account_proof.into_iter().map(Bytes::from).collect(); let storage_proof = if storage_root == KECCAK_EMPTY { @@ -87,7 +87,7 @@ impl<'a, 'b, TX: DbTx<'a>> StateProvider for LatestStateProviderRef<'a, 'b, TX> let hashed_keys: Vec = keys.iter().map(keccak256).collect(); loader .generate_storage_proofs(storage_root, hashed_address, &hashed_keys) - .map_err(|_| ProviderError::StateTree)? + .map_err(|_| ProviderError::StateTrie)? .into_iter() .map(|v| v.into_iter().map(Bytes::from).collect()) .collect() diff --git a/crates/storage/provider/src/test_utils/blocks.rs b/crates/storage/provider/src/test_utils/blocks.rs new file mode 100644 index 00000000000..9beffb33949 --- /dev/null +++ b/crates/storage/provider/src/test_utils/blocks.rs @@ -0,0 +1,146 @@ +//! Dummy blocks and data for tests + +use crate::{ + execution_result::{ + AccountChangeSet, AccountInfoChangeSet, ExecutionResult, TransactionChangeSet, + }, + Transaction, +}; +use reth_db::{database::Database, models::StoredBlockBody, tables}; +use reth_primitives::{ + hex_literal::hex, proofs::EMPTY_ROOT, Account, Header, Receipt, SealedBlock, + SealedBlockWithSenders, Withdrawal, H160, H256, U256, +}; +use reth_rlp::Decodable; +use std::collections::BTreeMap; + +/// Assert genesis block +pub fn assert_genesis_block(tx: &Transaction<'_, DB>, g: SealedBlock) { + let n = g.number; + let h = H256::zero(); + // check if all tables are empty + assert_eq!(tx.table::().unwrap(), vec![(g.number, g.header.clone().unseal())]); + + assert_eq!(tx.table::().unwrap(), vec![(h, n)]); + assert_eq!(tx.table::().unwrap(), vec![(n, h)]); + assert_eq!(tx.table::().unwrap(), vec![(n, g.difficulty.into())]); + assert_eq!(tx.table::().unwrap(), vec![(0, StoredBlockBody::default())]); + assert_eq!(tx.table::().unwrap(), vec![]); + assert_eq!(tx.table::().unwrap(), vec![]); + assert_eq!(tx.table::().unwrap(), vec![]); + assert_eq!(tx.table::().unwrap(), vec![]); + assert_eq!(tx.table::().unwrap(), vec![]); + assert_eq!(tx.table::().unwrap(), vec![]); + assert_eq!(tx.table::().unwrap(), vec![]); + assert_eq!(tx.table::().unwrap(), vec![]); + assert_eq!(tx.table::().unwrap(), vec![]); + // TODO check after this gets done: https://github.com/paradigmxyz/reth/issues/1588 + // Bytecodes are not reverted assert_eq!(tx.table::().unwrap(), vec![]); + assert_eq!(tx.table::().unwrap(), vec![(n, 0)]); + assert_eq!(tx.table::().unwrap(), vec![]); + assert_eq!(tx.table::().unwrap(), vec![]); + assert_eq!(tx.table::().unwrap(), vec![]); + assert_eq!(tx.table::().unwrap(), vec![]); + assert_eq!(tx.table::().unwrap(), vec![]); + assert_eq!(tx.table::().unwrap(), vec![(EMPTY_ROOT, vec![0x80])]); + assert_eq!(tx.table::().unwrap(), vec![]); + assert_eq!(tx.table::().unwrap(), vec![]); + // SyncStage is not updated in tests +} + +/// Test chain with genesis, blocks, execution results +/// that have correcte changesets. +pub struct BlockChainTestData { + /// Genesis + pub genesis: SealedBlock, + /// Blocks with its execution result + pub blocks: Vec<(SealedBlockWithSenders, ExecutionResult)>, +} + +impl Default for BlockChainTestData { + fn default() -> Self { + Self { genesis: genesis(), blocks: vec![block1(), block2()] } + } +} + +/// Genesis block +pub fn genesis() -> SealedBlock { + SealedBlock { + header: Header { number: 0, difficulty: U256::from(1), ..Default::default() } + .seal(H256::zero()), + body: vec![], + ommers: vec![], + withdrawals: Some(vec![]), + } +} + +/// Block one that points to genesis +fn block1() -> (SealedBlockWithSenders, ExecutionResult) { + let mut block_rlp = hex!("f9025ff901f7a0c86e8cc0310ae7c531c758678ddbfd16fc51c8cef8cec650b032de9869e8b94fa01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347942adc25665018aa1fe0e6bc666dac8fc2697ff9baa050554882fbbda2c2fd93fdc466db9946ea262a67f7a76cc169e714f105ab583da00967f09ef1dfed20c0eacfaa94d5cd4002eda3242ac47eae68972d07b106d192a0e3c8b47fbfc94667ef4cceb17e5cc21e3b1eebd442cebb27f07562b33836290db90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008302000001830f42408238108203e800a00000000000000000000000000000000000000000000000000000000000000000880000000000000000f862f860800a83061a8094095e7baea6a6c7c4c2dfeb977efac326af552d8780801ba072ed817487b84ba367d15d2f039b5fc5f087d0a8882fbdf73e8cb49357e1ce30a0403d800545b8fc544f92ce8124e2255f8c3c6af93f28243a120585d4c4c6a2a3c0").as_slice(); + let mut block = SealedBlock::decode(&mut block_rlp).unwrap(); + block.withdrawals = Some(vec![Withdrawal::default()]); + let mut header = block.header.clone().unseal(); + header.number = 1; + header.state_root = + H256(hex!("5d035ccb3e75a9057452ff060b773b213ec1fc353426174068edfc3971a0b6bd")); + header.parent_hash = H256::zero(); + block.header = header.seal_slow(); + + let mut account_changeset = AccountChangeSet { + account: AccountInfoChangeSet::Created { + new: Account { nonce: 1, balance: U256::from(10), bytecode_hash: None }, + }, + ..Default::default() + }; + account_changeset.storage.insert(U256::from(5), (U256::ZERO, U256::from(10))); + + let exec_res = ExecutionResult { + tx_changesets: vec![TransactionChangeSet { + receipt: Receipt::default(), /* receipts are not saved. */ + changeset: BTreeMap::from([(H160([0x60; 20]), account_changeset.clone())]), + new_bytecodes: BTreeMap::from([]), + }], + block_changesets: BTreeMap::from([(H160([0x61; 20]), account_changeset.account)]), + }; + + (SealedBlockWithSenders { block, senders: vec![H160([0x30; 20])] }, exec_res) +} + +/// Block two that points to block 1 +fn block2() -> (SealedBlockWithSenders, ExecutionResult) { + let mut block_rlp = hex!("f9025ff901f7a0c86e8cc0310ae7c531c758678ddbfd16fc51c8cef8cec650b032de9869e8b94fa01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347942adc25665018aa1fe0e6bc666dac8fc2697ff9baa050554882fbbda2c2fd93fdc466db9946ea262a67f7a76cc169e714f105ab583da00967f09ef1dfed20c0eacfaa94d5cd4002eda3242ac47eae68972d07b106d192a0e3c8b47fbfc94667ef4cceb17e5cc21e3b1eebd442cebb27f07562b33836290db90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008302000001830f42408238108203e800a00000000000000000000000000000000000000000000000000000000000000000880000000000000000f862f860800a83061a8094095e7baea6a6c7c4c2dfeb977efac326af552d8780801ba072ed817487b84ba367d15d2f039b5fc5f087d0a8882fbdf73e8cb49357e1ce30a0403d800545b8fc544f92ce8124e2255f8c3c6af93f28243a120585d4c4c6a2a3c0").as_slice(); + let mut block = SealedBlock::decode(&mut block_rlp).unwrap(); + block.withdrawals = Some(vec![Withdrawal::default()]); + let mut header = block.header.clone().unseal(); + header.number = 2; + header.state_root = + H256(hex!("90101a13dd059fa5cca99ed93d1dc23657f63626c5b8f993a2ccbdf7446b64f8")); + // parent_hash points to block1 hash + header.parent_hash = + H256(hex!("d846db2ab174c492cfe985c18fa75b154e20572bc33bb1c67cf5d2995791bdb7")); + block.header = header.seal_slow(); + + let mut account_changeset = AccountChangeSet::default(); + // storage will be moved + let info_changeset = AccountInfoChangeSet::Changed { + old: Account { nonce: 1, balance: U256::from(10), bytecode_hash: None }, + new: Account { nonce: 2, balance: U256::from(15), bytecode_hash: None }, + }; + account_changeset.account = info_changeset; + account_changeset.storage.insert(U256::from(5), (U256::from(10), U256::from(15))); + + let block_changeset = AccountInfoChangeSet::Changed { + old: Account { nonce: 2, balance: U256::from(15), bytecode_hash: None }, + new: Account { nonce: 3, balance: U256::from(20), bytecode_hash: None }, + }; + let exec_res = ExecutionResult { + tx_changesets: vec![TransactionChangeSet { + receipt: Receipt::default(), /* receipts are not saved. */ + changeset: BTreeMap::from([(H160([0x60; 20]), account_changeset.clone())]), + new_bytecodes: BTreeMap::from([]), + }], + block_changesets: BTreeMap::from([(H160([0x60; 20]), block_changeset)]), + }; + + (SealedBlockWithSenders { block, senders: vec![H160([0x31; 20])] }, exec_res) +} diff --git a/crates/storage/provider/src/test_utils/mod.rs b/crates/storage/provider/src/test_utils/mod.rs index d8fb2726181..b3097bb5ff9 100644 --- a/crates/storage/provider/src/test_utils/mod.rs +++ b/crates/storage/provider/src/test_utils/mod.rs @@ -1,3 +1,4 @@ +pub mod blocks; mod mock; mod noop; diff --git a/crates/storage/provider/src/traits/account.rs b/crates/storage/provider/src/traits/account.rs index bd6864f35ec..fb2ba3943b9 100644 --- a/crates/storage/provider/src/traits/account.rs +++ b/crates/storage/provider/src/traits/account.rs @@ -3,7 +3,7 @@ use reth_interfaces::Result; use reth_primitives::{Account, Address}; /// Account provider -#[auto_impl(&)] +#[auto_impl(&,Box)] pub trait AccountProvider: Send + Sync { /// Get basic account information. fn basic_account(&self, address: Address) -> Result>; diff --git a/crates/storage/provider/src/traits/block_hash.rs b/crates/storage/provider/src/traits/block_hash.rs index 6d4b99d5248..3bf3d9dfa85 100644 --- a/crates/storage/provider/src/traits/block_hash.rs +++ b/crates/storage/provider/src/traits/block_hash.rs @@ -3,7 +3,7 @@ use reth_interfaces::Result; use reth_primitives::{H256, U256}; /// Client trait for fetching block hashes by number. -#[auto_impl(&, Arc)] +#[auto_impl(&, Arc, Box)] pub trait BlockHashProvider: Send + Sync { /// Get the hash of the block with the given number. Returns `None` if no block with this number /// exists. diff --git a/crates/storage/provider/src/transaction.rs b/crates/storage/provider/src/transaction.rs index 90a3848c96e..f297d717fc4 100644 --- a/crates/storage/provider/src/transaction.rs +++ b/crates/storage/provider/src/transaction.rs @@ -1,5 +1,6 @@ -use itertools::Itertools; +use itertools::{izip, Itertools}; use reth_db::{ + common::KeyValue, cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO}, database::{Database, DatabaseGAT}, models::{ @@ -9,22 +10,24 @@ use reth_db::{ }, table::Table, tables, - transaction::{DbTx, DbTxMut}, + transaction::{DbTx, DbTxMut, DbTxMutGAT}, TransitionList, }; use reth_interfaces::{db::Error as DbError, provider::ProviderError}; use reth_primitives::{ - keccak256, Account, Address, BlockHash, BlockNumber, Bytecode, ChainSpec, Hardfork, Header, - SealedBlock, StorageEntry, TransitionId, TxNumber, H256, U256, + keccak256, proofs::EMPTY_ROOT, Account, Address, BlockHash, BlockNumber, Bytecode, ChainSpec, + Hardfork, Header, Receipt, SealedBlock, SealedBlockWithSenders, StorageEntry, + TransactionSignedEcRecovered, TransitionId, TxNumber, H256, U256, }; use reth_tracing::tracing::{info, trace}; use std::{ - collections::{BTreeMap, BTreeSet}, + collections::{btree_map::Entry, BTreeMap, BTreeSet}, fmt::Debug, - ops::{Deref, DerefMut}, + ops::{Bound, Deref, DerefMut, Range, RangeBounds}, }; use crate::{ + execution_result::{AccountInfoChangeSet, TransactionChangeSet}, insert_canonical_block, trie::{DBTrieLoader, TrieError}, }; @@ -274,19 +277,261 @@ impl<'this, DB> Transaction<'this, DB> where DB: Database, { + /// Get requested blocks transaction with signer + pub fn get_block_transaction_range( + &self, + range: impl RangeBounds + Clone, + ) -> Result)>, TransactionError> { + self.get_take_block_transaction_range::(range) + } + + /// Take requested blocks transaction with signer + pub fn take_block_transaction_range( + &self, + range: impl RangeBounds + Clone, + ) -> Result)>, TransactionError> { + self.get_take_block_transaction_range::(range) + } + + /// Return range of blocks and its execution result + pub fn get_block_range( + &self, + chain_spec: &ChainSpec, + range: impl RangeBounds + Clone, + ) -> Result, TransactionError> { + self.get_take_block_range::(chain_spec, range) + } + + /// Return range of blocks and its execution result + pub fn take_block_range( + &self, + chain_spec: &ChainSpec, + range: impl RangeBounds + Clone, + ) -> Result, TransactionError> { + self.get_take_block_range::(chain_spec, range) + } + + /// Transverse over changesets and plain state and recreated the execution results. + /// + /// Return results from database. + pub fn get_block_execution_result_range( + &self, + range: impl RangeBounds + Clone, + ) -> Result, TransactionError> { + self.get_take_block_execution_result_range::(range) + } + + /// Transverse over changesets and plain state and recreated the execution results. + /// + /// Get results and remove them from database + pub fn take_block_execution_result_range( + &self, + range: impl RangeBounds + Clone, + ) -> Result, TransactionError> { + self.get_take_block_execution_result_range::(range) + } + + /// Get range of blocks and its execution result + pub fn get_block_and_execution_range( + &self, + chain_spec: &ChainSpec, + range: impl RangeBounds + Clone, + ) -> Result, TransactionError> { + self.get_take_block_and_execution_range::(chain_spec, range) + } + + /// Take range of blocks and its execution result + pub fn take_block_and_execution_range( + &self, + chain_spec: &ChainSpec, + range: impl RangeBounds + Clone, + ) -> Result, TransactionError> { + self.get_take_block_and_execution_range::(chain_spec, range) + } + + /// Unwind and clear account hashing + pub fn unwind_account_hashing( + &self, + range: Range, + ) -> Result<(), TransactionError> { + let mut hashed_accounts = self.cursor_write::()?; + + // Aggregate all transition changesets and and make list of account that have been changed. + self.cursor_read::()? + .walk_range(range)? + .collect::, _>>()? + .into_iter() + .rev() + // fold all account to get the old balance/nonces and account that needs to be removed + .fold( + BTreeMap::new(), + |mut accounts: BTreeMap>, (_, account_before)| { + accounts.insert(account_before.address, account_before.info); + accounts + }, + ) + .into_iter() + // hash addresses and collect it inside sorted BTreeMap. + // We are doing keccak only once per address. + .map(|(address, account)| (keccak256(address), account)) + .collect::>() + .into_iter() + // Apply values to HashedState (if Account is None remove it); + .try_for_each(|(hashed_address, account)| -> Result<(), TransactionError> { + if let Some(account) = account { + hashed_accounts.upsert(hashed_address, account)?; + } else if hashed_accounts.seek_exact(hashed_address)?.is_some() { + hashed_accounts.delete_current()?; + } + Ok(()) + })?; + Ok(()) + } + + /// Unwind and clear storage hashing + pub fn unwind_storage_hashing( + &self, + range: Range, + ) -> Result<(), TransactionError> { + let mut hashed_storage = self.cursor_dup_write::()?; + + // Aggregate all transition changesets and make list of accounts that have been changed. + self.cursor_read::()? + .walk_range(range)? + .collect::, _>>()? + .into_iter() + .rev() + // fold all account to get the old balance/nonces and account that needs to be removed + .fold( + BTreeMap::new(), + |mut accounts: BTreeMap<(Address, H256), U256>, + (TransitionIdAddress((_, address)), storage_entry)| { + accounts.insert((address, storage_entry.key), storage_entry.value); + accounts + }, + ) + .into_iter() + // hash addresses and collect it inside sorted BTreeMap. + // We are doing keccak only once per address. + .map(|((address, key), value)| ((keccak256(address), keccak256(key)), value)) + .collect::>() + .into_iter() + // Apply values to HashedStorage (if Value is zero just remove it); + .try_for_each(|((hashed_address, key), value)| -> Result<(), TransactionError> { + if hashed_storage + .seek_by_key_subkey(hashed_address, key)? + .filter(|entry| entry.key == key) + .is_some() + { + hashed_storage.delete_current()?; + } + + if value != U256::ZERO { + hashed_storage.upsert(hashed_address, StorageEntry { key, value })?; + } + Ok(()) + })?; + Ok(()) + } + + /// Unwind and clear account history indices + pub fn unwind_account_history_indices( + &self, + range: Range, + ) -> Result<(), TransactionError> { + let mut cursor = self.cursor_write::()?; + + let account_changeset = self + .cursor_read::()? + .walk_range(range)? + .collect::, _>>()?; + + let last_indices = account_changeset + .into_iter() + // reverse so we can get lowest transition id where we need to unwind account. + .rev() + // fold all account and get last transition index + .fold(BTreeMap::new(), |mut accounts: BTreeMap, (index, account)| { + // we just need address and lowest transition id. + accounts.insert(account.address, index); + accounts + }); + // try to unwind the index + for (address, rem_index) in last_indices { + let shard_part = unwind_account_history_shards::(&mut cursor, address, rem_index)?; + + // check last shard_part, if present, items needs to be reinserted. + if !shard_part.is_empty() { + // there are items in list + self.put::( + ShardedKey::new(address, u64::MAX), + TransitionList::new(shard_part) + .expect("There is at least one element in list and it is sorted."), + )?; + } + } + Ok(()) + } + + /// Unwind and clear storage history indices + pub fn unwind_storage_history_indices( + &self, + range: Range, + ) -> Result<(), TransactionError> { + let mut cursor = self.cursor_write::()?; + + let storage_changesets = self + .cursor_read::()? + .walk_range(range)? + .collect::, _>>()?; + let last_indices = storage_changesets + .into_iter() + // reverse so we can get lowest transition id where we need to unwind account. + .rev() + // fold all storages and get last transition index + .fold( + BTreeMap::new(), + |mut accounts: BTreeMap<(Address, H256), u64>, (index, storage)| { + // we just need address and lowest transition id. + accounts.insert((index.address(), storage.key), index.transition_id()); + accounts + }, + ); + for ((address, storage_key), rem_index) in last_indices { + let shard_part = + unwind_storage_history_shards::(&mut cursor, address, storage_key, rem_index)?; + + // check last shard_part, if present, items needs to be reinserted. + if !shard_part.is_empty() { + // there are items in list + self.put::( + StorageShardedKey::new(address, storage_key, u64::MAX), + TransitionList::new(shard_part) + .expect("There is at least one element in list and it is sorted."), + )?; + } + } + Ok(()) + } + /// Insert full block and make it canonical /// /// This is atomic operation and transaction will do one commit at the end of the function. pub fn insert_block( &mut self, - block: &SealedBlock, + block: SealedBlockWithSenders, chain_spec: &ChainSpec, changeset: ExecutionResult, ) -> Result<(), TransactionError> { // Header, Body, SenderRecovery, TD, TxLookup stages - let (from, to) = insert_canonical_block(self.deref_mut(), block, false).unwrap(); + let (block, senders) = block.into_components(); + let block_number = block.number; + let block_state_root = block.state_root; + let block_hash = block.hash(); + let parent_block_number = block.number.saturating_sub(1); - let parent_block_number = block.number - 1; + let (from, to) = + insert_canonical_block(self.deref_mut(), block, Some(senders), false).unwrap(); // execution stage self.insert_execution_result(vec![changeset], chain_spec, parent_block_number)?; @@ -310,12 +555,12 @@ where let current_root = self.get_header(parent_block_number)?.state_root; let mut loader = DBTrieLoader::new(self.deref_mut()); let root = loader.update_root(current_root, from..to).and_then(|e| e.root())?; - if root != block.state_root { + if root != block_state_root { return Err(TransactionError::StateTrieRootMismatch { got: root, - expected: block.state_root, - block_number: block.number, - block_hash: block.hash(), + expected: block_state_root, + block_number, + block_hash, }) } } @@ -332,8 +577,488 @@ where self.insert_storage_history_index(indices)?; } - // commit block to database - self.commit()?; + Ok(()) + } + + /// Return list of entries from table + /// + /// If TAKE is true, opened cursor would be write and it would delete all values from db. + #[inline] + pub fn get_or_take( + &self, + range: impl RangeBounds, + ) -> Result>, DbError> { + if TAKE { + let mut cursor_write = self.cursor_write::()?; + let mut walker = cursor_write.walk_range(range)?; + let mut items = Vec::new(); + while let Some(i) = walker.next().transpose()? { + walker.delete_current()?; + items.push(i) + } + Ok(items) + } else { + self.cursor_read::()?.walk_range(range)?.collect::, _>>() + } + } + + /// Get requested blocks transaction with signer + fn get_take_block_transaction_range( + &self, + range: impl RangeBounds + Clone, + ) -> Result)>, TransactionError> { + // Just read block tx id from table. as it is needed to get execution results. + let block_bodies = self.get_or_take::(range)?; + + if block_bodies.is_empty() { + return Ok(Vec::new()) + } + + // iterate over and get all transaction and signers + let first_transaction = + block_bodies.first().expect("If we have headers").1.first_tx_index(); + let last_transaction = block_bodies.last().expect("Not empty").1.last_tx_index(); + + let transactions = + self.get_or_take::(first_transaction..=last_transaction)?; + let senders = + self.get_or_take::(first_transaction..=last_transaction)?; + + if TAKE { + // rm TxHashNumber + let mut tx_hash_cursor = self.cursor_write::()?; + for (_, tx) in transactions.iter() { + if tx_hash_cursor.seek_exact(tx.hash())?.is_some() { + tx_hash_cursor.delete_current()?; + } + } + // rm TxTransitionId + self.get_or_take::( + first_transaction..=last_transaction, + )?; + } + + // Merge transaction into blocks + let mut block_tx = Vec::new(); + let mut senders = senders.into_iter(); + let mut transactions = transactions.into_iter(); + for (block_number, block_body) in block_bodies { + let mut one_block_tx = Vec::new(); + for _ in block_body.tx_id_range() { + let tx = transactions.next(); + let sender = senders.next(); + + let recovered = match (tx, sender) { + (Some((tx_id, tx)), Some((sender_tx_id, sender))) => { + if tx_id != sender_tx_id { + Err(ProviderError::MismatchOfTransactionAndSenderId { tx_id }) + } else { + Ok(TransactionSignedEcRecovered::from_signed_transaction(tx, sender)) + } + } + (Some((tx_id, _)), _) | (_, Some((tx_id, _))) => { + Err(ProviderError::MismatchOfTransactionAndSenderId { tx_id }) + } + (None, None) => Err(ProviderError::BlockBodyTransactionCount), + }?; + one_block_tx.push(recovered) + } + block_tx.push((block_number, one_block_tx)); + } + + Ok(block_tx) + } + + /// Return range of blocks and its execution result + fn get_take_block_range( + &self, + chain_spec: &ChainSpec, + range: impl RangeBounds + Clone, + ) -> Result, TransactionError> { + // For block we need Headers, Bodies, Uncles, withdrawals, Transactions, Signers + + let block_headers = self.get_or_take::(range.clone())?; + if block_headers.is_empty() { + return Ok(Vec::new()) + } + + let block_header_hashes = + self.get_or_take::(range.clone())?; + let block_ommers = self.get_or_take::(range.clone())?; + let block_withdrawals = + self.get_or_take::(range.clone())?; + + let block_tx = self.get_take_block_transaction_range::(range.clone())?; + + if TAKE { + // rm HeaderTD + self.get_or_take::(range)?; + // rm HeaderNumbers + let mut header_number_cursor = self.cursor_write::()?; + for (_, hash) in block_header_hashes.iter() { + if header_number_cursor.seek_exact(*hash)?.is_some() { + header_number_cursor.delete_current()?; + } + } + } + + // merge all into block + let block_header_iter = block_headers.into_iter(); + let block_header_hashes_iter = block_header_hashes.into_iter(); + let block_tx_iter = block_tx.into_iter(); + + // can be not found in tables + let mut block_ommers_iter = block_ommers.into_iter(); + let mut block_withdrawals_iter = block_withdrawals.into_iter(); + let mut block_ommers = block_ommers_iter.next(); + let mut block_withdrawals = block_withdrawals_iter.next(); + + let mut blocks = Vec::new(); + for ((main_block_number, header), (_, header_hash), (_, tx)) in izip!( + block_header_iter.into_iter(), + block_header_hashes_iter.into_iter(), + block_tx_iter.into_iter() + ) { + let header = header.seal(header_hash); + + let (body, senders) = tx.into_iter().map(|tx| tx.to_components()).unzip(); + + // Ommers can be missing + let mut ommers = Vec::new(); + if let Some((block_number, _)) = block_ommers.as_ref() { + if *block_number == main_block_number { + // Seal ommers as they dont have hash. + ommers = block_ommers + .take() + .unwrap() + .1 + .ommers + .into_iter() + .map(|h| h.seal_slow()) + .collect(); + block_ommers = block_ommers_iter.next(); + } + }; + + // withdrawal can be missing + let shanghai_is_active = + chain_spec.fork(Hardfork::Paris).active_at_block(main_block_number); + let mut withdrawals = Some(Vec::new()); + if shanghai_is_active { + if let Some((block_number, _)) = block_withdrawals.as_ref() { + if *block_number == main_block_number { + withdrawals = Some(block_withdrawals.take().unwrap().1.withdrawals); + block_withdrawals = block_withdrawals_iter.next(); + } + } + } else { + withdrawals = None + } + + blocks.push(SealedBlockWithSenders { + block: SealedBlock { header, body, ommers, withdrawals }, + senders, + }) + } + + Ok(blocks) + } + + /// Transverse over changesets and plain state and recreated the execution results. + fn get_take_block_execution_result_range( + &self, + range: impl RangeBounds + Clone, + ) -> Result, TransactionError> { + let block_transition = + self.get_or_take::(range.clone())?; + + if block_transition.is_empty() { + return Ok(Vec::new()) + } + // get block transitions + let first_block_number = + block_transition.first().expect("Check for empty is already done").0; + + // get block transition of parent block. + let from = self.get_block_transition(first_block_number.saturating_sub(1))?; + let to = block_transition.last().expect("Check for empty is already done").1; + + // NOTE: Just get block bodies dont remove them + // it is connection point for bodies getter and execution result getter. + let block_bodies = self.get_or_take::(range)?; + + // get saved previous values + let from_storage: TransitionIdAddress = (from, Address::zero()).into(); + let to_storage: TransitionIdAddress = (to, Address::zero()).into(); + + let storage_changeset = + self.get_or_take::(from_storage..to_storage)?; + let account_changeset = self.get_or_take::(from..to)?; + + // iterate previous value and get plain state value to create changeset + // Double option around Account represent if Account state is know (first option) and + // account is removed (Second Option) + type LocalPlainState = BTreeMap>, BTreeMap)>; + type Changesets = BTreeMap< + TransitionId, + BTreeMap)>, + >; + + let mut local_plain_state: LocalPlainState = BTreeMap::new(); + + // iterate in reverse and get plain state. + + // Bundle execution changeset to its particular transaction and block + let mut all_changesets: Changesets = BTreeMap::new(); + + let mut plain_accounts_cursor = self.cursor_write::()?; + let mut plain_storage_cursor = self.cursor_dup_write::()?; + + // add account changeset changes + for (transition_id, account_before) in account_changeset.into_iter().rev() { + let new_info = match local_plain_state.entry(account_before.address) { + Entry::Vacant(entry) => { + let new_account = + plain_accounts_cursor.seek(account_before.address)?.map(|(_s, i)| i); + entry.insert((Some(account_before.info), BTreeMap::new())); + new_account + } + Entry::Occupied(mut entry) => { + let new_account = + std::mem::replace(&mut entry.get_mut().0, Some(account_before.info)); + new_account.expect("As we are stacking account first, account would always be Some(Some) or Some(None)") + } + }; + let account_info_changeset = AccountInfoChangeSet::new(account_before.info, new_info); + // insert changeset to transition id. Multiple account for same transition Id are not + // possible. + all_changesets + .entry(transition_id) + .or_default() + .entry(account_before.address) + .or_default() + .0 = account_info_changeset + } + + // add storage changeset changes + for (transition_and_address, storage_entry) in storage_changeset.into_iter().rev() { + let TransitionIdAddress((transition_id, address)) = transition_and_address; + let new_storage = + match local_plain_state.entry(address).or_default().1.entry(storage_entry.key) { + Entry::Vacant(entry) => { + let new_storage = plain_storage_cursor + .seek_by_key_subkey(address, storage_entry.key)? + .filter(|storage| storage.key == storage_entry.key) + .unwrap_or_default(); + entry.insert(storage_entry.value); + new_storage.value + } + Entry::Occupied(mut entry) => { + std::mem::replace(entry.get_mut(), storage_entry.value) + } + }; + all_changesets + .entry(transition_id) + .or_default() + .entry(address) + .or_default() + .1 + .insert(storage_entry.key, (storage_entry.value, new_storage)); + } + + if TAKE { + // iterate over local plain state remove all account and all storages. + for (address, (account, storage)) in local_plain_state.into_iter() { + // revert account + if let Some(account) = account { + plain_accounts_cursor.seek_exact(address)?; + if let Some(account) = account { + plain_accounts_cursor.upsert(address, account)?; + } else { + plain_accounts_cursor.delete_current()?; + } + } + // revert storages + for (storage_key, storage_value) in storage.into_iter() { + let storage_entry = StorageEntry { key: storage_key, value: storage_value }; + // delete previous value + if plain_storage_cursor + .seek_by_key_subkey(address, storage_key)? + .filter(|s| s.key == storage_key) + .is_some() + { + plain_storage_cursor.delete_current()? + } + // insert value if needed + if storage_value != U256::ZERO { + plain_storage_cursor.insert(address, storage_entry)?; + } + } + } + } + + // NOTE: Some storage changesets can be empty, + // all account changeset have at least beneficiary fee transfer. + + // iterate over block body and create ExecutionResult + let mut block_exec_results = Vec::new(); + + let mut changeset_iter = all_changesets.into_iter(); + let mut block_transition_iter = block_transition.into_iter(); + let mut next_transition_id = from; + + let mut next_changeset = changeset_iter.next().unwrap_or_default(); + // loop break if we are at the end of the blocks. + for (_, block_body) in block_bodies.into_iter() { + let mut block_exec_res = ExecutionResult::default(); + for _ in 0..block_body.tx_count { + // only if next_changeset + let changeset = if next_transition_id == next_changeset.0 { + let changeset = next_changeset + .1 + .into_iter() + .map(|(address, (account, storage))| { + ( + address, + AccountChangeSet { + account, + storage: storage + .into_iter() + .map(|(key, val)| (U256::from_be_bytes(key.0), val)) + .collect(), + wipe_storage: false, /* it is always false as all storage + * changesets for selfdestruct are + * already accounted. */ + }, + ) + }) + .collect(); + next_changeset = changeset_iter.next().unwrap_or_default(); + changeset + } else { + BTreeMap::new() + }; + + next_transition_id += 1; + block_exec_res.tx_changesets.push(TransactionChangeSet { + receipt: Receipt::default(), /* TODO(receipt) when they are saved, load them + * from db */ + changeset, + new_bytecodes: Default::default(), /* TODO(bytecode), bytecode is not cleared + * so it is same sa previous. */ + }); + } + + let Some((_,block_transition)) = block_transition_iter.next() else { break}; + // if block transition points to 1+next transition id it means that there is block + // changeset. + if block_transition == next_transition_id + 1 { + // assert last_transition_id == block_transition + if next_transition_id == next_changeset.0 { + // take block changeset + block_exec_res.block_changesets = next_changeset + .1 + .into_iter() + .map(|(address, (account, _))| (address, account)) + .collect(); + next_changeset = changeset_iter.next().unwrap_or_default(); + } + next_transition_id += 1; + } + block_exec_results.push(block_exec_res) + } + Ok(block_exec_results) + } + + /// Return range of blocks and its execution result + pub fn get_take_block_and_execution_range( + &self, + chain_spec: &ChainSpec, + range: impl RangeBounds + Clone, + ) -> Result, TransactionError> { + if TAKE { + let (from_transition, parent_number, parent_state_root) = match range.start_bound() { + Bound::Included(n) => { + let parent_number = n.saturating_sub(1); + let transition = self.get_block_transition(parent_number)?; + let parent = self.get_header(parent_number)?; + (transition, parent_number, parent.state_root) + } + Bound::Excluded(n) => { + let transition = self.get_block_transition(*n)?; + let parent = self.get_header(*n)?; + (transition, *n, parent.state_root) + } + Bound::Unbounded => (0, 0, EMPTY_ROOT), + }; + let to_transition = match range.end_bound() { + Bound::Included(n) => self.get_block_transition(*n)?, + Bound::Excluded(n) => self.get_block_transition(n.saturating_sub(1))?, + Bound::Unbounded => TransitionId::MAX, + }; + + let transition_range = from_transition..to_transition; + let zero = Address::zero(); + let transition_storage_range = + (from_transition, zero).into()..(to_transition, zero).into(); + + self.unwind_account_hashing(transition_range.clone())?; + self.unwind_account_history_indices(transition_range.clone())?; + self.unwind_storage_hashing(transition_storage_range.clone())?; + self.unwind_storage_history_indices(transition_storage_range)?; + + // merkle tree + let new_state_root; + { + let (tip_number, _) = + self.cursor_read::()?.last()?.unwrap_or_default(); + let current_root = self.get_header(tip_number)?.state_root; + let mut loader = DBTrieLoader::new(self.deref()); + new_state_root = + loader.update_root(current_root, transition_range).and_then(|e| e.root())?; + } + // state root should be always correct as we are reverting state. + // but for sake of double verification we will check it again. + if new_state_root != parent_state_root { + let parent_hash = self.get_block_hash(parent_number)?; + return Err(TransactionError::StateTrieRootMismatch { + got: new_state_root, + expected: parent_state_root, + block_number: parent_number, + block_hash: parent_hash, + }) + } + } + // get blocks + let blocks = self.get_take_block_range::(chain_spec, range.clone())?; + // get execution res + let execution_res = self.get_take_block_execution_result_range::(range.clone())?; + // combine them + let blocks_with_exec_result: Vec<_> = + blocks.into_iter().zip(execution_res.into_iter()).collect(); + + // remove block bodies it is needed for both get block range and get block execution results + // that is why it is deleted afterwards. + if TAKE { + // rm block bodies + self.get_or_take::(range)?; + } + + // return them + Ok(blocks_with_exec_result) + } + + /// Update all pipeline sync stage progress. + pub fn update_pipeline_stages( + &self, + block_number: BlockNumber, + ) -> Result<(), TransactionError> { + // iterate over + let mut cursor = self.cursor_write::()?; + while let Some((stage_name, _)) = cursor.next()? { + cursor.upsert(stage_name, block_number)? + } + Ok(()) } @@ -741,6 +1466,90 @@ where } Ok(()) } + + /// Return full table as Vec + pub fn table(&self) -> Result>, DbError> + where + T::Key: Default + Ord, + { + self.cursor_read::()?.walk(Some(T::Key::default()))?.collect::, DbError>>() + } +} + +/// Unwind all history shards. For boundary shard, remove it from database and +/// return last part of shard with still valid items. If all full shard were removed, return list +/// would be empty. +fn unwind_account_history_shards( + cursor: &mut <>::TXMut as DbTxMutGAT<'_>>::CursorMut< + tables::AccountHistory, + >, + address: Address, + transition_id: TransitionId, +) -> Result, TransactionError> { + let mut item = cursor.seek_exact(ShardedKey::new(address, u64::MAX))?; + + while let Some((sharded_key, list)) = item { + // there is no more shard for address + if sharded_key.key != address { + break + } + cursor.delete_current()?; + // check first item and if it is more and eq than `transition_id` delete current + // item. + let first = list.iter(0).next().expect("List can't empty"); + if first >= transition_id as usize { + item = cursor.prev()?; + continue + } else if transition_id <= sharded_key.highest_transition_id { + // if first element is in scope whole list would be removed. + // so at least this first element is present. + return Ok(list.iter(0).take_while(|i| *i < transition_id as usize).collect::>()) + } else { + let new_list = list.iter(0).collect::>(); + return Ok(new_list) + } + } + Ok(Vec::new()) +} + +/// Unwind all history shards. For boundary shard, remove it from database and +/// return last part of shard with still valid items. If all full shard were removed, return list +/// would be empty but this does not mean that there is none shard left but that there is no +/// splitted shards. +fn unwind_storage_history_shards( + cursor: &mut <>::TXMut as DbTxMutGAT<'_>>::CursorMut< + tables::StorageHistory, + >, + address: Address, + storage_key: H256, + transition_id: TransitionId, +) -> Result, TransactionError> { + let mut item = cursor.seek_exact(StorageShardedKey::new(address, storage_key, u64::MAX))?; + + while let Some((storage_sharded_key, list)) = item { + // there is no more shard for address + if storage_sharded_key.address != address || + storage_sharded_key.sharded_key.key != storage_key + { + // there is no more shard for address and storage_key. + break + } + cursor.delete_current()?; + // check first item and if it is more and eq than `transition_id` delete current + // item. + let first = list.iter(0).next().expect("List can't empty"); + if first >= transition_id as usize { + item = cursor.prev()?; + continue + } else if transition_id <= storage_sharded_key.sharded_key.highest_transition_id { + // if first element is in scope whole list would be removed. + // so at least this first element is present. + return Ok(list.iter(0).take_while(|i| *i < transition_id as usize).collect::>()) + } else { + return Ok(list.iter(0).collect::>()) + } + } + Ok(Vec::new()) } /// An error that can occur when using the transaction container @@ -756,7 +1565,7 @@ pub enum TransactionError { #[error("Merkle trie calculation error: {0}")] MerkleTrie(#[from] TrieError), /// Root mismatch - #[error("Merkle trie root mismatch on block: #{block_number:?} {block_hash:?}. got: {got:?} expected:{got:?}")] + #[error("Merkle trie root mismatch on block: #{block_number:?} {block_hash:?}. got: {got:?} expected:{expected:?}")] StateTrieRootMismatch { /// Expected root expected: H256, @@ -768,3 +1577,69 @@ pub enum TransactionError { block_hash: BlockHash, }, } + +#[cfg(test)] +mod test { + use crate::{insert_canonical_block, test_utils::blocks::*, Transaction}; + use reth_db::{mdbx::test_utils::create_test_rw_db, tables, transaction::DbTxMut}; + use reth_primitives::{proofs::EMPTY_ROOT, ChainSpecBuilder, MAINNET}; + use std::ops::DerefMut; + + #[test] + fn insert_get_take() { + let db = create_test_rw_db(); + + // setup + let mut tx = Transaction::new(db.as_ref()).unwrap(); + let chain_spec = ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis(MAINNET.genesis.clone()) + .shanghai_activated() + .build(); + + let data = BlockChainTestData::default(); + let genesis = data.genesis.clone(); + let (block1, exec_res1) = data.blocks[0].clone(); + let (block2, exec_res2) = data.blocks[1].clone(); + + insert_canonical_block(tx.deref_mut(), data.genesis.clone(), None, false).unwrap(); + + tx.put::(EMPTY_ROOT, vec![0x80]).unwrap(); + assert_genesis_block(&tx, data.genesis.clone()); + + tx.insert_block(block1.clone(), &chain_spec, exec_res1.clone()).unwrap(); + + // get one block + let get = tx.get_block_and_execution_range(&chain_spec, 1..=1).unwrap(); + assert_eq!(get, vec![(block1.clone(), exec_res1.clone())]); + + // take one block + let take = tx.take_block_and_execution_range(&chain_spec, 1..=1).unwrap(); + assert_eq!(take, vec![(block1.clone(), exec_res1.clone())]); + assert_genesis_block(&tx, genesis.clone()); + + tx.insert_block(block1.clone(), &chain_spec, exec_res1.clone()).unwrap(); + tx.insert_block(block2.clone(), &chain_spec, exec_res2.clone()).unwrap(); + + // get second block + let get = tx.get_block_and_execution_range(&chain_spec, 2..=2).unwrap(); + assert_eq!(get, vec![(block2.clone(), exec_res2.clone())]); + + // get two blocks + let get = tx.get_block_and_execution_range(&chain_spec, 1..=2).unwrap(); + assert_eq!( + get, + vec![(block1.clone(), exec_res1.clone()), (block2.clone(), exec_res2.clone())] + ); + + // take two blocks + let get = tx.take_block_and_execution_range(&chain_spec, 1..=2).unwrap(); + assert_eq!( + get, + vec![(block1.clone(), exec_res1.clone()), (block2.clone(), exec_res2.clone())] + ); + + // assert genesis state + assert_genesis_block(&tx, genesis); + } +} diff --git a/crates/storage/provider/src/trie/mod.rs b/crates/storage/provider/src/trie/mod.rs index 953fccd0d14..d218eb607dd 100644 --- a/crates/storage/provider/src/trie/mod.rs +++ b/crates/storage/provider/src/trie/mod.rs @@ -32,7 +32,9 @@ pub enum TrieError { InternalError(#[from] cita_trie::TrieError), /// The database doesn't contain the root of the trie. #[error("The root node wasn't found in the DB")] - MissingRoot(H256), + MissingAccountRoot(H256), + #[error("The storage root node wasn't found in the DB")] + MissingStorageRoot(H256), /// Error returned by the database. #[error("{0:?}")] DatabaseError(#[from] reth_db::Error), @@ -112,7 +114,7 @@ where if root == EMPTY_ROOT { return Self::new(tx) } - tx.get::(root)?.ok_or(TrieError::MissingRoot(root))?; + tx.get::(root)?.ok_or(TrieError::MissingAccountRoot(root))?; Ok(Self { tx }) } } @@ -204,7 +206,7 @@ where tx.cursor_dup_read::()? .seek_by_key_subkey(key, root)? .filter(|entry| entry.hash == root) - .ok_or(TrieError::MissingRoot(root))?; + .ok_or(TrieError::MissingStorageRoot(root))?; Ok(Self { tx, key }) } } @@ -254,7 +256,7 @@ where impl<'tx, 'itx, TX: DbTx<'itx>> HashDatabase<'tx, 'itx, TX> { /// Instantiates a new Database for the accounts trie, with an existing root fn from_root(tx: &'tx TX, root: H256) -> Result { - tx.get::(root)?.ok_or(TrieError::MissingRoot(root))?; + tx.get::(root)?.ok_or(TrieError::MissingAccountRoot(root))?; Ok(Self { tx, _p: Default::default() }) } } @@ -307,7 +309,7 @@ impl<'tx, 'itx, TX: DbTx<'itx>> DupHashDatabase<'tx, 'itx, TX> { fn from_root(tx: &'tx TX, key: H256, root: H256) -> Result { tx.cursor_dup_read::()? .seek_by_key_subkey(key, root)? - .ok_or(TrieError::MissingRoot(root))?; + .ok_or(TrieError::MissingAccountRoot(root))?; Ok(Self { tx, key, _p: Default::default() }) } } diff --git a/crates/storage/provider/src/utils.rs b/crates/storage/provider/src/utils.rs index 13888706205..1262328a81f 100644 --- a/crates/storage/provider/src/utils.rs +++ b/crates/storage/provider/src/utils.rs @@ -4,7 +4,7 @@ use reth_db::{ transaction::{DbTx, DbTxMut}, }; use reth_interfaces::{provider::ProviderError, Result}; -use reth_primitives::{SealedBlock, TransitionId, U256}; +use reth_primitives::{Address, SealedBlock, TransitionId}; /// Insert block data into corresponding tables. Used mainly for testing & internal tooling. /// @@ -18,10 +18,12 @@ use reth_primitives::{SealedBlock, TransitionId, U256}; /// Return [TransitionId] `(from,to)` pub fn insert_block<'a, TX: DbTxMut<'a> + DbTx<'a>>( tx: &TX, - block: &SealedBlock, + block: SealedBlock, + senders: Option>, has_block_reward: bool, parent_tx_num_transition_id: Option<(u64, u64)>, ) -> Result<(TransitionId, TransitionId)> { + let block_number = block.number; tx.put::(block.number, block.hash())?; // Put header with canonical hashes. tx.put::(block.number, block.header.as_ref().clone())?; @@ -29,7 +31,7 @@ pub fn insert_block<'a, TX: DbTxMut<'a> + DbTx<'a>>( // total difficulty let ttd = if block.number == 0 { - U256::ZERO + block.difficulty } else { let parent_block_number = block.number - 1; let parent_ttd = tx.get::(parent_block_number)?.unwrap_or_default(); @@ -68,11 +70,24 @@ pub fn insert_block<'a, TX: DbTxMut<'a> + DbTx<'a>>( StoredBlockBody { start_tx_id: current_tx_id, tx_count: block.body.len() as u64 }, )?; - for transaction in block.body.iter() { - let rec_tx = transaction.clone().into_ecrecovered().unwrap(); - let hash = rec_tx.hash(); - tx.put::(current_tx_id, rec_tx.signer())?; - tx.put::(current_tx_id, rec_tx.into())?; + let senders_len = senders.as_ref().map(|s| s.len()); + let tx_iter = if Some(block.body.len()) == senders_len { + block.body.into_iter().zip(senders.unwrap().into_iter()).collect::>() + } else { + block + .body + .into_iter() + .map(|tx| { + let signer = tx.recover_signer(); + (tx, signer.unwrap_or_default()) + }) + .collect::>() + }; + + for (transaction, sender) in tx_iter { + let hash = transaction.hash(); + tx.put::(current_tx_id, sender)?; + tx.put::(current_tx_id, transaction)?; tx.put::(current_tx_id, transition_id)?; tx.put::(hash, current_tx_id)?; transition_id += 1; @@ -80,11 +95,11 @@ pub fn insert_block<'a, TX: DbTxMut<'a> + DbTx<'a>>( } let mut has_withdrawals = false; - if let Some(withdrawals) = block.withdrawals.clone() { + if let Some(withdrawals) = block.withdrawals { if !withdrawals.is_empty() { has_withdrawals = true; tx.put::( - block.number, + block_number, StoredBlockWithdrawals { withdrawals }, )?; } @@ -93,7 +108,7 @@ pub fn insert_block<'a, TX: DbTxMut<'a> + DbTx<'a>>( if has_block_reward || has_withdrawals { transition_id += 1; } - tx.put::(block.number, transition_id)?; + tx.put::(block_number, transition_id)?; let to_transition = transition_id; Ok((from_transition, to_transition)) @@ -103,8 +118,9 @@ pub fn insert_block<'a, TX: DbTxMut<'a> + DbTx<'a>>( /// parent block in database. pub fn insert_canonical_block<'a, TX: DbTxMut<'a> + DbTx<'a>>( tx: &TX, - block: &SealedBlock, + block: SealedBlock, + senders: Option>, has_block_reward: bool, ) -> Result<(TransitionId, TransitionId)> { - insert_block(tx, block, has_block_reward, None) + insert_block(tx, block, senders, has_block_reward, None) } From 46c96a146616ffd440edd7cec1402dfcf2dc207c Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Wed, 15 Mar 2023 02:25:54 +0800 Subject: [PATCH 126/191] feat(stages): add table checkpoint to `AccountHashing` and `StorageHashing` (#1667) Co-authored-by: Georgios Konstantopoulos --- Cargo.lock | 1 + bin/reth/src/dump_stage/hashing_account.rs | 29 +-- bin/reth/src/dump_stage/hashing_storage.rs | 46 ++++- crates/primitives/src/checkpoints.rs | 42 ++++ crates/primitives/src/lib.rs | 3 +- crates/primitives/src/proofs.rs | 19 +- crates/stages/Cargo.toml | 1 + crates/stages/src/stages/hashing_account.rs | 125 ++++++++---- crates/stages/src/stages/hashing_storage.rs | 203 +++++++++++++------- 9 files changed, 325 insertions(+), 144 deletions(-) create mode 100644 crates/primitives/src/checkpoints.rs diff --git a/Cargo.lock b/Cargo.lock index 89b60ecad7a..5b4fb682dbd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5099,6 +5099,7 @@ dependencies = [ "proptest", "rand 0.8.5", "rayon", + "reth-codecs", "reth-db", "reth-downloaders", "reth-eth-wire", diff --git a/bin/reth/src/dump_stage/hashing_account.rs b/bin/reth/src/dump_stage/hashing_account.rs index e5f6ccf0197..5f43ce61abd 100644 --- a/bin/reth/src/dump_stage/hashing_account.rs +++ b/bin/reth/src/dump_stage/hashing_account.rs @@ -36,8 +36,7 @@ pub(crate) async fn dump_hashing_account_stage( unwind_and_copy::(db_tool, from, tip_block_number, &output_db).await?; if should_run { - println!("\n# AccountHashing stage does not support dry run, so it will actually be committing changes."); - run(output_db, to, from).await?; + dry_run(output_db, to, from).await?; } Ok(()) @@ -69,7 +68,7 @@ async fn unwind_and_copy( } /// Try to re-execute the stage straightaway -async fn run( +async fn dry_run( output_db: reth_db::mdbx::Env, to: u64, from: u64, @@ -82,15 +81,21 @@ async fn run( ..Default::default() }; - exec_stage - .execute( - &mut tx, - reth_stages::ExecInput { - previous_stage: Some((StageId("Another"), to)), - stage_progress: Some(from), - }, - ) - .await?; + let mut exec_output = false; + while !exec_output { + exec_output = exec_stage + .execute( + &mut tx, + reth_stages::ExecInput { + previous_stage: Some((StageId("Another"), to)), + stage_progress: Some(from), + }, + ) + .await? + .done; + } + + tx.drop()?; info!(target: "reth::cli", "Success."); diff --git a/bin/reth/src/dump_stage/hashing_storage.rs b/bin/reth/src/dump_stage/hashing_storage.rs index 557c6a956d7..03cbb74a3f9 100644 --- a/bin/reth/src/dump_stage/hashing_storage.rs +++ b/bin/reth/src/dump_stage/hashing_storage.rs @@ -6,8 +6,9 @@ use crate::{ use eyre::Result; use reth_db::{database::Database, table::TableImporter, tables}; use reth_provider::Transaction; -use reth_stages::{stages::StorageHashingStage, Stage, UnwindInput}; +use reth_stages::{stages::StorageHashingStage, Stage, StageId, UnwindInput}; use std::ops::DerefMut; +use tracing::info; pub(crate) async fn dump_hashing_storage_stage( db_tool: &mut DbTool<'_, DB>, @@ -16,14 +17,14 @@ pub(crate) async fn dump_hashing_storage_stage( output_db: &PlatformPath, should_run: bool, ) -> Result<()> { - if should_run { - eyre::bail!("StorageHashing stage does not support dry run.") - } - let (output_db, tip_block_number) = setup::(from, to, output_db, db_tool)?; unwind_and_copy::(db_tool, from, tip_block_number, &output_db).await?; + if should_run { + dry_run(output_db, to, from).await?; + } + Ok(()) } @@ -53,3 +54,38 @@ async fn unwind_and_copy( Ok(()) } + +/// Try to re-execute the stage straightaway +async fn dry_run( + output_db: reth_db::mdbx::Env, + to: u64, + from: u64, +) -> eyre::Result<()> { + info!(target: "reth::cli", "Executing stage."); + + let mut tx = Transaction::new(&output_db)?; + let mut exec_stage = StorageHashingStage { + clean_threshold: 1, // Forces hashing from scratch + ..Default::default() + }; + + let mut exec_output = false; + while !exec_output { + exec_output = exec_stage + .execute( + &mut tx, + reth_stages::ExecInput { + previous_stage: Some((StageId("Another"), to)), + stage_progress: Some(from), + }, + ) + .await? + .done; + } + + tx.drop()?; + + info!(target: "reth::cli", "Success."); + + Ok(()) +} diff --git a/crates/primitives/src/checkpoints.rs b/crates/primitives/src/checkpoints.rs new file mode 100644 index 00000000000..62a90bfa0a0 --- /dev/null +++ b/crates/primitives/src/checkpoints.rs @@ -0,0 +1,42 @@ +use crate::{Address, H256}; +use reth_codecs::{main_codec, Compact}; + +/// Saves the progress of MerkleStage +#[main_codec] +#[derive(Default, Debug, Copy, Clone, PartialEq)] +pub struct ProofCheckpoint { + /// The next hashed account to insert into the trie. + pub hashed_address: Option, + /// The next storage entry to insert into the trie. + pub storage_key: Option, + /// Current intermediate root for `AccountsTrie`. + pub account_root: Option, + /// Current intermediate storage root from an account. + pub storage_root: Option, +} + +/// Saves the progress of AccountHashing +#[main_codec] +#[derive(Default, Debug, Copy, Clone, PartialEq)] +pub struct AccountHashingCheckpoint { + /// The next account to start hashing from + pub address: Option
, + /// Start transition id + pub from: u64, + /// Last transition id + pub to: u64, +} + +/// Saves the progress of StorageHashing +#[main_codec] +#[derive(Default, Debug, Copy, Clone, PartialEq)] +pub struct StorageHashingCheckpoint { + /// The next account to start hashing from + pub address: Option
, + /// The next storage slot to start hashing from + pub storage: Option, + /// Start transition id + pub from: u64, + /// Last transition id + pub to: u64, +} diff --git a/crates/primitives/src/lib.rs b/crates/primitives/src/lib.rs index 6c6585e067d..b6b60c37626 100644 --- a/crates/primitives/src/lib.rs +++ b/crates/primitives/src/lib.rs @@ -14,6 +14,7 @@ mod bits; mod block; pub mod bloom; mod chain; +mod checkpoints; pub mod constants; pub mod contract; mod error; @@ -34,7 +35,6 @@ mod withdrawal; /// Helper function for calculating Merkle proofs and hashes pub mod proofs; -pub use proofs::ProofCheckpoint; pub use account::{Account, Bytecode}; pub use bits::H512; @@ -46,6 +46,7 @@ pub use chain::{ AllGenesisFormats, Chain, ChainInfo, ChainSpec, ChainSpecBuilder, ForkCondition, GOERLI, MAINNET, SEPOLIA, }; +pub use checkpoints::{AccountHashingCheckpoint, ProofCheckpoint, StorageHashingCheckpoint}; pub use constants::{ EMPTY_OMMER_ROOT, GOERLI_GENESIS, KECCAK_EMPTY, MAINNET_GENESIS, SEPOLIA_GENESIS, }; diff --git a/crates/primitives/src/proofs.rs b/crates/primitives/src/proofs.rs index c94a68aad33..28f7f6e8b2e 100644 --- a/crates/primitives/src/proofs.rs +++ b/crates/primitives/src/proofs.rs @@ -1,5 +1,3 @@ -use std::collections::HashMap; - use crate::{ keccak256, Address, Bytes, GenesisAccount, Header, Log, Receipt, TransactionSigned, Withdrawal, H256, @@ -8,8 +6,8 @@ use bytes::BytesMut; use hash_db::Hasher; use hex_literal::hex; use plain_hasher::PlainHasher; -use reth_codecs::{main_codec, Compact}; use reth_rlp::Encodable; +use std::collections::HashMap; use triehash::{ordered_trie_root, sec_trie_root}; /// Keccak-256 hash of the RLP of an empty list, KEC("\xc0"). @@ -35,23 +33,8 @@ impl Hasher for KeccakHasher { } } -/// Saves the progress of MerkleStage -#[main_codec] -#[derive(Default, Debug, Copy, Clone, PartialEq)] -pub struct ProofCheckpoint { - /// The next hashed account to insert into the trie. - pub hashed_address: Option, - /// The next storage entry to insert into the trie. - pub storage_key: Option, - /// Current intermediate root for `AccountsTrie`. - pub account_root: Option, - /// Current intermediate storage root from an account. - pub storage_root: Option, -} - /// Calculate a transaction root. /// -/// Iterates over the given transactions and the merkle merkle trie root of /// `(rlp(index), encoded(tx))` pairs. pub fn calculate_transaction_root<'a>( transactions: impl IntoIterator, diff --git a/crates/stages/Cargo.toml b/crates/stages/Cargo.toml index b0ec65911bb..85c10f4c5c6 100644 --- a/crates/stages/Cargo.toml +++ b/crates/stages/Cargo.toml @@ -18,6 +18,7 @@ normal = [ reth-primitives = { path = "../primitives" } reth-interfaces = { path = "../interfaces" } reth-db = { path = "../storage/db" } +reth-codecs = { path = "../storage/codecs" } reth-provider = { path = "../storage/provider" } reth-metrics-derive = { path = "../metrics/metrics-derive" } diff --git a/crates/stages/src/stages/hashing_account.rs b/crates/stages/src/stages/hashing_account.rs index a6ea743759a..a129a5bcbd9 100644 --- a/crates/stages/src/stages/hashing_account.rs +++ b/crates/stages/src/stages/hashing_account.rs @@ -1,11 +1,12 @@ use crate::{ExecInput, ExecOutput, Stage, StageError, StageId, UnwindInput, UnwindOutput}; +use reth_codecs::Compact; use reth_db::{ cursor::{DbCursorRO, DbCursorRW}, database::Database, tables, transaction::{DbTx, DbTxMut}, }; -use reth_primitives::keccak256; +use reth_primitives::{keccak256, AccountHashingCheckpoint}; use reth_provider::Transaction; use std::{collections::BTreeMap, fmt::Debug, ops::Range}; use tracing::*; @@ -30,6 +31,43 @@ impl Default for AccountHashingStage { } } +impl AccountHashingStage { + /// Saves the hashing progress + pub fn save_checkpoint( + &mut self, + tx: &Transaction<'_, DB>, + checkpoint: AccountHashingCheckpoint, + ) -> Result<(), StageError> { + debug!(target: "sync::stages::account_hashing::exec", checkpoint = ?checkpoint, "Saving inner account hashing checkpoint"); + + let mut buf = vec![]; + checkpoint.to_compact(&mut buf); + + Ok(tx.put::(ACCOUNT_HASHING.0.into(), buf)?) + } + + /// Gets the hashing progress + pub fn get_checkpoint( + &self, + tx: &Transaction<'_, DB>, + ) -> Result { + let buf = + tx.get::(ACCOUNT_HASHING.0.into())?.unwrap_or_default(); + + if buf.is_empty() { + return Ok(AccountHashingCheckpoint::default()) + } + + let (checkpoint, _) = AccountHashingCheckpoint::from_compact(&buf, buf.len()); + + if checkpoint.address.is_some() { + debug!(target: "sync::stages::account_hashing::exec", checkpoint = ?checkpoint, "Continuing inner account hashing checkpoint"); + } + + Ok(checkpoint) + } +} + #[derive(Clone, Debug)] /// `SeedOpts` provides configuration parameters for calling `AccountHashingStage::seed` /// in unit tests or benchmarks to generate an initial database state for running the @@ -137,43 +175,58 @@ impl Stage for AccountHashingStage { // AccountHashing table. Also, if we start from genesis, we need to hash from scratch, as // genesis accounts are not in changeset. if to_transition - from_transition > self.clean_threshold || stage_progress == 0 { - // clear table, load all accounts and hash it - tx.clear::()?; - tx.commit()?; - - let mut first_key = None; - loop { - let next_key = { - let mut accounts = tx.cursor_read::()?; - - let hashed_batch = accounts - .walk(first_key)? - .take(self.commit_threshold as usize) - .map(|res| res.map(|(address, account)| (keccak256(address), account))) - .collect::, _>>()?; - - let mut hashed_account_cursor = tx.cursor_write::()?; - - // iterate and put presorted hashed accounts - if first_key.is_none() { - hashed_batch - .into_iter() - .try_for_each(|(k, v)| hashed_account_cursor.append(k, v))?; - } else { - hashed_batch - .into_iter() - .try_for_each(|(k, v)| hashed_account_cursor.insert(k, v))?; - } + let mut checkpoint = self.get_checkpoint(tx)?; + + if checkpoint.address.is_none() || + // Checkpoint is no longer valid if the range of transitions changed. + // An already hashed account may have been changed with the new range, and therefore should be hashed again. + checkpoint.to != to_transition || + checkpoint.from != from_transition + { + // clear table, load all accounts and hash it + tx.clear::()?; + + checkpoint = AccountHashingCheckpoint::default(); + self.save_checkpoint(tx, checkpoint)?; + } - // next key of iterator - accounts.next()? - }; - tx.commit()?; - if let Some((next_key, _)) = next_key { - first_key = Some(next_key); - continue + let start_address = checkpoint.address.take(); + let next_address = { + let mut accounts = tx.cursor_read::()?; + + let hashed_batch = accounts + .walk(start_address)? + .take(self.commit_threshold as usize) + .map(|res| res.map(|(address, account)| (keccak256(address), account))) + .collect::, _>>()?; + + let mut hashed_account_cursor = tx.cursor_write::()?; + + // iterate and put presorted hashed accounts + if start_address.is_none() { + hashed_batch + .into_iter() + .try_for_each(|(k, v)| hashed_account_cursor.append(k, v))?; + } else { + hashed_batch + .into_iter() + .try_for_each(|(k, v)| hashed_account_cursor.insert(k, v))?; } - break + + // next key of iterator + accounts.next()? + }; + + if let Some((next_address, _)) = &next_address { + checkpoint.address = Some(*next_address); + checkpoint.from = from_transition; + checkpoint.to = to_transition; + } + + self.save_checkpoint(tx, checkpoint)?; + + if next_address.is_some() { + return Ok(ExecOutput { stage_progress, done: false }) } } else { // Aggregate all transition changesets and and make list of account that have been diff --git a/crates/stages/src/stages/hashing_storage.rs b/crates/stages/src/stages/hashing_storage.rs index 22d3cac1cc0..0480e693146 100644 --- a/crates/stages/src/stages/hashing_storage.rs +++ b/crates/stages/src/stages/hashing_storage.rs @@ -1,5 +1,6 @@ use crate::{ExecInput, ExecOutput, Stage, StageError, StageId, UnwindInput, UnwindOutput}; use num_traits::Zero; +use reth_codecs::Compact; use reth_db::{ cursor::DbDupCursorRO, database::Database, @@ -7,7 +8,7 @@ use reth_db::{ tables, transaction::{DbTx, DbTxMut}, }; -use reth_primitives::{keccak256, Address, StorageEntry}; +use reth_primitives::{keccak256, Address, StorageEntry, StorageHashingCheckpoint}; use reth_provider::Transaction; use std::{collections::BTreeMap, fmt::Debug}; use tracing::*; @@ -32,6 +33,43 @@ impl Default for StorageHashingStage { } } +impl StorageHashingStage { + /// Saves the hashing progress + pub fn save_checkpoint( + &mut self, + tx: &Transaction<'_, DB>, + checkpoint: StorageHashingCheckpoint, + ) -> Result<(), StageError> { + debug!(target: "sync::stages::storage_hashing::exec", checkpoint = ?checkpoint, "Saving inner storage hashing checkpoint"); + + let mut buf = vec![]; + checkpoint.to_compact(&mut buf); + + Ok(tx.put::(STORAGE_HASHING.0.into(), buf)?) + } + + /// Gets the hashing progress + pub fn get_checkpoint( + &self, + tx: &Transaction<'_, DB>, + ) -> Result { + let buf = + tx.get::(STORAGE_HASHING.0.into())?.unwrap_or_default(); + + if buf.is_empty() { + return Ok(StorageHashingCheckpoint::default()) + } + + let (checkpoint, _) = StorageHashingCheckpoint::from_compact(&buf, buf.len()); + + if checkpoint.address.is_some() { + debug!(target: "sync::stages::storage_hashing::exec", checkpoint = ?checkpoint, "Continuing inner storage hashing checkpoint"); + } + + Ok(checkpoint) + } +} + #[async_trait::async_trait] impl Stage for StorageHashingStage { /// Return the id of the stage @@ -57,77 +95,92 @@ impl Stage for StorageHashingStage { // AccountHashing table. Also, if we start from genesis, we need to hash from scratch, as // genesis accounts are not in changeset, along with their storages. if to_transition - from_transition > self.clean_threshold || stage_progress == 0 { - tx.clear::()?; - tx.commit()?; + let mut checkpoint = self.get_checkpoint(tx)?; + + if checkpoint.address.is_none() || + // Checkpoint is no longer valid if the range of transitions changed. + // An already hashed storage may have been changed with the new range, and therefore should be hashed again. + checkpoint.to != to_transition || + checkpoint.from != from_transition + { + tx.clear::()?; + + checkpoint = StorageHashingCheckpoint::default(); + self.save_checkpoint(tx, checkpoint)?; + } - let mut current_key = None; - let mut current_subkey = None; + let mut current_key = checkpoint.address.take(); + let mut current_subkey = checkpoint.storage.take(); let mut keccak_address = None; - loop { - let mut hashed_batch = BTreeMap::new(); - let mut remaining = self.commit_threshold as usize; - { - let mut storage = tx.cursor_dup_read::()?; - while !remaining.is_zero() { - hashed_batch.extend( - storage - .walk_dup(current_key, current_subkey)? - .take(remaining) - .map(|res| { - res.map(|(address, slot)| { - // Address caching for the first iteration when current_key - // is None - let keccak_address = - if let Some(keccak_address) = keccak_address { - keccak_address - } else { - keccak256(address) - }; - - // TODO cache map keccak256(slot.key) ? - ((keccak_address, keccak256(slot.key)), slot.value) - }) + let mut hashed_batch = BTreeMap::new(); + let mut remaining = self.commit_threshold as usize; + { + let mut storage = tx.cursor_dup_read::()?; + while !remaining.is_zero() { + hashed_batch.extend( + storage + .walk_dup(current_key, current_subkey)? + .take(remaining) + .map(|res| { + res.map(|(address, slot)| { + // Address caching for the first iteration when current_key + // is None + let keccak_address = + if let Some(keccak_address) = keccak_address { + keccak_address + } else { + keccak256(address) + }; + + // TODO cache map keccak256(slot.key) ? + ((keccak_address, keccak256(slot.key)), slot.value) }) - .collect::, _>>()?, - ); - - remaining = self.commit_threshold as usize - hashed_batch.len(); - - if let Some((address, slot)) = storage.next_dup()? { - // There's still some remaining elements on this key, so we need to save - // the cursor position for the next - // iteration - - current_key = Some(address); - current_subkey = Some(slot.key); + }) + .collect::, _>>()?, + ); + + remaining = self.commit_threshold as usize - hashed_batch.len(); + + if let Some((address, slot)) = storage.next_dup()? { + // There's still some remaining elements on this key, so we need to save + // the cursor position for the next + // iteration + + current_key = Some(address); + current_subkey = Some(slot.key); + } else { + // Go to the next key + current_key = storage.next_no_dup()?.map(|(key, _)| key); + current_subkey = None; + + // Cache keccak256(address) for the next key if it exists + if let Some(address) = current_key { + keccak_address = Some(keccak256(address)); } else { - // Go to the next key - current_key = storage.next_no_dup()?.map(|(key, _)| key); - current_subkey = None; - - // Cache keccak256(address) for the next key if it exists - if let Some(address) = current_key { - keccak_address = Some(keccak256(address)); - } else { - // We have reached the end of table - break - } + // We have reached the end of table + break } } } + } - // iterate and put presorted hashed slots - hashed_batch.into_iter().try_for_each(|((addr, key), value)| { - tx.put::(addr, StorageEntry { key, value }) - })?; + // iterate and put presorted hashed slots + hashed_batch.into_iter().try_for_each(|((addr, key), value)| { + tx.put::(addr, StorageEntry { key, value }) + })?; - tx.commit()?; + if let Some(address) = ¤t_key { + checkpoint.address = Some(*address); + checkpoint.storage = current_subkey; + checkpoint.from = from_transition; + checkpoint.to = to_transition; + } - // We have reached the end of table - if current_key.is_none() { - break - } + self.save_checkpoint(tx, checkpoint)?; + + if current_key.is_some() { + return Ok(ExecOutput { stage_progress, done: false }) } } else { // Aggregate all transition changesets and and make list of storages that have been @@ -170,7 +223,6 @@ mod tests { stage_test_suite_ext, ExecuteStageTestRunner, StageTestRunner, TestRunnerError, TestTransaction, UnwindStageTestRunner, PREV_STAGE_ID, }; - use assert_matches::assert_matches; use reth_db::{ cursor::{DbCursorRO, DbCursorRW}, mdbx::{tx::Tx, WriteMap, RW}, @@ -205,18 +257,25 @@ mod tests { runner.seed_execution(input).expect("failed to seed execution"); - let rx = runner.execute(input); + loop { + if let Ok(result) = runner.execute(input).await.unwrap() { + if !result.done { + // Continue from checkpoint + continue + } else { + assert!(result.stage_progress == previous_stage); - // Assert the successful result - let result = rx.await.unwrap(); - assert_matches!( - result, - Ok(ExecOutput { done, stage_progress }) - if done && stage_progress == previous_stage - ); + // Validate the stage execution + assert!( + runner.validate_execution(input, Some(result)).is_ok(), + "execution validation" + ); - // Validate the stage execution - assert!(runner.validate_execution(input, result.ok()).is_ok(), "execution validation"); + break + } + } + panic!("Failed execution"); + } } struct StorageHashingTestRunner { From 4725c4d776f5372de50460d24fd563801e5cdf4d Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 15 Mar 2023 01:58:14 +0100 Subject: [PATCH 127/191] fix: make ChainState provider work for Box (#1755) --- crates/executor/src/blockchain_tree/chain.rs | 13 +++++------- crates/executor/src/blockchain_tree/mod.rs | 21 +++++++++---------- .../provider/src/providers/state/chain.rs | 13 +++++++----- 3 files changed, 23 insertions(+), 24 deletions(-) diff --git a/crates/executor/src/blockchain_tree/chain.rs b/crates/executor/src/blockchain_tree/chain.rs index 8267f8cc328..c5dcf3a9340 100644 --- a/crates/executor/src/blockchain_tree/chain.rs +++ b/crates/executor/src/blockchain_tree/chain.rs @@ -348,7 +348,7 @@ mod tests { let mut block1 = block.clone(); let mut block2 = block.clone(); let mut block3 = block.clone(); - let mut block4 = block.clone(); + let mut block4 = block; block1.block.header.hash = block1_hash; block2.block.header.hash = block2_hash; @@ -360,13 +360,13 @@ mod tests { let mut chain1 = Chain { substate: Default::default(), changesets: vec![], - blocks: BTreeMap::from([(1, block1.clone()), (2, block2.clone())]), + blocks: BTreeMap::from([(1, block1), (2, block2)]), }; let chain2 = Chain { substate: Default::default(), changesets: vec![], - blocks: BTreeMap::from([(3, block3.clone()), (4, block4.clone())]), + blocks: BTreeMap::from([(3, block3), (4, block4)]), }; assert_eq!(chain1.append_chain(chain2.clone()), Ok(())); @@ -418,7 +418,7 @@ mod tests { // split in two assert_eq!( chain.clone().split(SplitAt::Hash(block1_hash)), - ChainSplit::Split { canonical: chain_split1.clone(), pending: chain_split2.clone() } + ChainSplit::Split { canonical: chain_split1, pending: chain_split2 } ); // split at unknown block hash @@ -433,9 +433,6 @@ mod tests { ChainSplit::NoSplitCanonical(chain.clone()) ); // split at lower number - assert_eq!( - chain.clone().split(SplitAt::Number(0)), - ChainSplit::NoSplitPending(chain.clone()) - ); + assert_eq!(chain.clone().split(SplitAt::Number(0)), ChainSplit::NoSplitPending(chain)); } } diff --git a/crates/executor/src/blockchain_tree/mod.rs b/crates/executor/src/blockchain_tree/mod.rs index ff3ee144d12..30347fc2a4c 100644 --- a/crates/executor/src/blockchain_tree/mod.rs +++ b/crates/executor/src/blockchain_tree/mod.rs @@ -11,8 +11,8 @@ use reth_db::{cursor::DbCursorRO, database::Database, tables, transaction::DbTx} use reth_interfaces::{consensus::Consensus, executor::Error as ExecError, Error}; use reth_primitives::{BlockHash, BlockNumber, ChainSpec, SealedBlock, SealedBlockWithSenders}; use reth_provider::{ - ExecutorFactory, HeaderProvider, ShareableDatabase, StateProvider, StateProviderFactory, - Transaction, + providers::ChainState, ExecutorFactory, HeaderProvider, ShareableDatabase, + StateProviderFactory, Transaction, }; use std::{ collections::{BTreeMap, HashMap}, @@ -177,9 +177,9 @@ impl BlockchainTree let db = self.externals.sharable_db(); let provider = if canonical_fork.hash == canonical_tip_hash { - Box::new(db.latest()?) as Box + ChainState::boxed(db.latest()?) } else { - Box::new(db.history_by_block_number(canonical_fork.number)?) as Box + ChainState::boxed(db.history_by_block_number(canonical_fork.number)?) }; // append the block if it is continuing the chain. @@ -226,9 +226,9 @@ impl BlockchainTree .ok_or(ExecError::CanonicalChain { block_hash: block.parent_hash })?; let provider = if block.parent_hash == canonical_tip { - Box::new(db.latest()?) as Box + ChainState::boxed(db.latest()?) } else { - Box::new(db.history_by_block_number(block.number - 1)?) as Box + ChainState::boxed(db.history_by_block_number(block.number - 1)?) }; let parent_header = parent_header.seal(block.parent_hash); @@ -554,8 +554,6 @@ impl BlockchainTree #[cfg(test)] mod tests { - use std::collections::HashSet; - use super::*; use parking_lot::Mutex; use reth_db::{ @@ -566,8 +564,9 @@ mod tests { use reth_primitives::{hex_literal::hex, proofs::EMPTY_ROOT, ChainSpecBuilder, H256, MAINNET}; use reth_provider::{ execution_result::ExecutionResult, insert_block, test_utils::blocks::BlockChainTestData, - BlockExecutor, + BlockExecutor, StateProvider, }; + use std::collections::HashSet; struct TestFactory { exec_result: Arc>>, @@ -644,7 +643,7 @@ mod tests { genesis.header.header.state_root = EMPTY_ROOT; let tx_mut = externals.0.tx_mut().unwrap(); - insert_block(&tx_mut, genesis.clone(), None, false, Some((0, 0))).unwrap(); + insert_block(&tx_mut, genesis, None, false, Some((0, 0))).unwrap(); // insert first 10 blocks for i in 0..10 { @@ -710,7 +709,7 @@ mod tests { H256(hex!("90101a13dd059fa5cca99ed93d1dc23657f63626c5b8f993a2ccbdf7446b64f8")); // test pops execution results from vector, so order is from last to first.ß - let externals = externals(vec![exec2.clone(), exec1.clone(), exec2.clone(), exec1.clone()]); + let externals = externals(vec![exec2.clone(), exec1.clone(), exec2, exec1]); // last finalized block would be number 9. setup(data.genesis, &externals); diff --git a/crates/storage/provider/src/providers/state/chain.rs b/crates/storage/provider/src/providers/state/chain.rs index 567a9f82747..697d6b1fe8c 100644 --- a/crates/storage/provider/src/providers/state/chain.rs +++ b/crates/storage/provider/src/providers/state/chain.rs @@ -2,7 +2,6 @@ use crate::{ providers::state::macros::delegate_provider_impls, AccountProvider, BlockHashProvider, StateProvider, }; -use std::marker::PhantomData; /// A type that can access the state at a specific access point (block number or tag) /// @@ -16,16 +15,20 @@ use std::marker::PhantomData; /// /// Note: The lifetime of this type is limited by the type that created it. pub struct ChainState<'a> { - inner: Box, - _phantom: PhantomData<&'a ()>, + inner: Box, } // == impl ChainState === impl<'a> ChainState<'a> { /// Wraps the given [StateProvider] - pub fn new(inner: Box) -> Self { - Self { inner, _phantom: Default::default() } + pub fn boxed(inner: S) -> Self { + Self::new(Box::new(inner)) + } + + /// Wraps the given [StateProvider] + pub fn new(inner: Box) -> Self { + Self { inner } } /// Returns a new provider that takes the `TX` as reference From a688fdb38d989f9e2ffc0b8d3edba7e4b056d28b Mon Sep 17 00:00:00 2001 From: Francisco Krause Arnim <56402156+fkrause98@users.noreply.github.com> Date: Tue, 14 Mar 2023 22:03:38 -0300 Subject: [PATCH 128/191] feat(rpc): eth_sign* (#1665) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: lambdaclass-user Co-authored-by: Matthias Seitz Co-authored-by: Tomás --- .../interfaces/src/test_utils/generators.rs | 21 +-- crates/primitives/src/lib.rs | 8 +- crates/primitives/src/transaction/mod.rs | 2 +- .../primitives/src/transaction/signature.rs | 11 ++ crates/primitives/src/transaction/util.rs | 22 ++- crates/rpc/rpc-builder/tests/it/http.rs | 14 +- crates/rpc/rpc/Cargo.toml | 4 +- crates/rpc/rpc/src/eth/api/mod.rs | 1 + crates/rpc/rpc/src/eth/api/server.rs | 8 +- crates/rpc/rpc/src/eth/api/sign.rs | 41 +++++ crates/rpc/rpc/src/eth/error.rs | 18 ++ crates/rpc/rpc/src/eth/signer.rs | 157 +++++++++++++++++- 12 files changed, 264 insertions(+), 43 deletions(-) create mode 100644 crates/rpc/rpc/src/eth/api/sign.rs diff --git a/crates/interfaces/src/test_utils/generators.rs b/crates/interfaces/src/test_utils/generators.rs index d7e17aee632..8d8e3bd4802 100644 --- a/crates/interfaces/src/test_utils/generators.rs +++ b/crates/interfaces/src/test_utils/generators.rs @@ -1,9 +1,9 @@ use rand::{distributions::uniform::SampleRange, seq::SliceRandom, thread_rng, Rng}; use reth_primitives::{ - proofs, Account, Address, Bytes, Header, SealedBlock, SealedHeader, Signature, StorageEntry, - Transaction, TransactionKind, TransactionSigned, TxLegacy, H160, H256, U256, + proofs, sign_message, Account, Address, Bytes, Header, SealedBlock, SealedHeader, Signature, + StorageEntry, Transaction, TransactionKind, TransactionSigned, TxLegacy, H160, H256, U256, }; -use secp256k1::{KeyPair, Message as SecpMessage, Secp256k1, SecretKey}; +use secp256k1::{KeyPair, Message as SecpMessage, Secp256k1, SecretKey, SECP256K1}; use std::{collections::BTreeMap, ops::Sub}; // TODO(onbjerg): Maybe we should split this off to its own crate, or move the helpers to the @@ -72,21 +72,6 @@ pub fn random_signed_tx() -> TransactionSigned { TransactionSigned::from_transaction_and_signature(tx, signature) } -/// Signs message with the given secret key. -/// Returns the corresponding signature. -pub fn sign_message(secret: H256, message: H256) -> Result { - let secp = Secp256k1::new(); - let sec = SecretKey::from_slice(secret.as_ref())?; - let s = secp.sign_ecdsa_recoverable(&SecpMessage::from_slice(&message[..])?, &sec); - let (rec_id, data) = s.serialize_compact(); - - Ok(Signature { - r: U256::try_from_be_slice(&data[..32]).unwrap(), - s: U256::try_from_be_slice(&data[32..64]).unwrap(), - odd_y_parity: rec_id.to_i32() != 0, - }) -} - /// Generate a random block filled with signed transactions (generated using /// [random_signed_tx]). If no transaction count is provided, the number of transactions /// will be random, otherwise the provided count will be used. diff --git a/crates/primitives/src/lib.rs b/crates/primitives/src/lib.rs index b6b60c37626..88eca2ff07d 100644 --- a/crates/primitives/src/lib.rs +++ b/crates/primitives/src/lib.rs @@ -64,10 +64,10 @@ pub use revm_primitives::JumpMap; pub use serde_helper::JsonU256; pub use storage::{StorageEntry, StorageTrieEntry}; pub use transaction::{ - AccessList, AccessListItem, AccessListWithGasUsed, FromRecoveredTransaction, - IntoRecoveredTransaction, InvalidTransactionError, Signature, Transaction, TransactionKind, - TransactionSigned, TransactionSignedEcRecovered, TxEip1559, TxEip2930, TxLegacy, TxType, - EIP1559_TX_TYPE_ID, EIP2930_TX_TYPE_ID, LEGACY_TX_TYPE_ID, + util::secp256k1::sign_message, AccessList, AccessListItem, AccessListWithGasUsed, + FromRecoveredTransaction, IntoRecoveredTransaction, InvalidTransactionError, Signature, + Transaction, TransactionKind, TransactionSigned, TransactionSignedEcRecovered, TxEip1559, + TxEip2930, TxLegacy, TxType, EIP1559_TX_TYPE_ID, EIP2930_TX_TYPE_ID, LEGACY_TX_TYPE_ID, }; pub use withdrawal::Withdrawal; diff --git a/crates/primitives/src/transaction/mod.rs b/crates/primitives/src/transaction/mod.rs index 15213623ba4..3ae06b947e7 100644 --- a/crates/primitives/src/transaction/mod.rs +++ b/crates/primitives/src/transaction/mod.rs @@ -14,7 +14,7 @@ mod access_list; mod error; mod signature; mod tx_type; -mod util; +pub(crate) mod util; /// Legacy transaction. #[main_codec] diff --git a/crates/primitives/src/transaction/signature.rs b/crates/primitives/src/transaction/signature.rs index 468855b906b..e6bde0e4a0a 100644 --- a/crates/primitives/src/transaction/signature.rs +++ b/crates/primitives/src/transaction/signature.rs @@ -99,6 +99,17 @@ impl Signature { // errors and we care only if recovery is passing or not. secp256k1::recover(&sig, hash.as_fixed_bytes()).ok() } + + /// Turn this signature into its byte + /// (hex) representation. + pub fn to_bytes(&self) -> [u8; 65] { + let mut sig = [0u8; 65]; + sig[..32].copy_from_slice(&self.r.to_be_bytes::<32>()); + sig[32..64].copy_from_slice(&self.s.to_be_bytes::<32>()); + let v = u8::from(self.odd_y_parity) + 27; + sig[64] = v; + sig + } } #[cfg(test)] diff --git a/crates/primitives/src/transaction/util.rs b/crates/primitives/src/transaction/util.rs index 5bb11d41c52..f2d471c3045 100644 --- a/crates/primitives/src/transaction/util.rs +++ b/crates/primitives/src/transaction/util.rs @@ -1,11 +1,16 @@ use crate::{keccak256, Address}; pub(crate) mod secp256k1 { + use crate::Signature; + use super::*; use ::secp256k1::{ ecdsa::{RecoverableSignature, RecoveryId}, - Error, Message, SECP256K1, + Message, SecretKey, SECP256K1, }; + use revm_primitives::{B256, U256}; + + pub(crate) use ::secp256k1::Error; /// secp256k1 signer recovery pub(crate) fn recover(sig: &[u8; 65], msg: &[u8; 32]) -> Result { @@ -16,6 +21,21 @@ pub(crate) mod secp256k1 { let hash = keccak256(&public.serialize_uncompressed()[1..]); Ok(Address::from_slice(&hash[12..])) } + + /// Signs message with the given secret key. + /// Returns the corresponding signature. + pub fn sign_message(secret: B256, message: B256) -> Result { + let sec = SecretKey::from_slice(secret.as_ref())?; + let s = SECP256K1.sign_ecdsa_recoverable(&Message::from_slice(&message[..])?, &sec); + let (rec_id, data) = s.serialize_compact(); + + let signature = Signature { + r: U256::try_from_be_slice(&data[..32]).expect("The slice has at most 32 bytes"), + s: U256::try_from_be_slice(&data[32..64]).expect("The slice has at most 32 bytes"), + odd_y_parity: rec_id.to_i32() != 0, + }; + Ok(signature) + } } #[cfg(test)] mod tests { diff --git a/crates/rpc/rpc-builder/tests/it/http.rs b/crates/rpc/rpc-builder/tests/it/http.rs index 6d4cc66df01..876f4ce901d 100644 --- a/crates/rpc/rpc-builder/tests/it/http.rs +++ b/crates/rpc/rpc-builder/tests/it/http.rs @@ -78,6 +78,11 @@ where EthApiClient::block_uncles_count_by_number(client, block_number).await.unwrap(); EthApiClient::uncle_by_block_hash_and_index(client, hash, index).await.unwrap(); EthApiClient::uncle_by_block_number_and_index(client, block_number, index).await.unwrap(); + EthApiClient::sign(client, address, bytes.clone()).await.unwrap_err(); + EthApiClient::sign_typed_data(client, address, jsonrpsee::core::JsonValue::Null) + .await + .unwrap_err(); + EthApiClient::create_access_list(client, call_request.clone(), None).await.unwrap(); EthApiClient::transaction_by_hash(client, tx_hash).await.unwrap(); EthApiClient::transaction_by_block_hash_and_index(client, hash, index).await.unwrap(); EthApiClient::transaction_by_block_number_and_index(client, block_number, index).await.unwrap(); @@ -112,18 +117,9 @@ where assert!(is_unimplemented( EthApiClient::send_transaction(client, transaction_request).await.err().unwrap() )); - assert!(is_unimplemented( - EthApiClient::sign(client, address, bytes.clone()).await.err().unwrap() - )); assert!(is_unimplemented( EthApiClient::sign_transaction(client, call_request.clone()).await.err().unwrap() )); - assert!(is_unimplemented( - EthApiClient::sign_typed_data(client, address, jsonrpsee::core::JsonValue::Null) - .await - .err() - .unwrap() - )); } async fn test_basic_debug_calls(client: &C) diff --git a/crates/rpc/rpc/Cargo.toml b/crates/rpc/rpc/Cargo.toml index 949fb3313ef..0323eac5662 100644 --- a/crates/rpc/rpc/Cargo.toml +++ b/crates/rpc/rpc/Cargo.toml @@ -26,7 +26,7 @@ reth-tasks = { path = "../../tasks" } # eth revm = { version = "3.0.0", features = ["optional_block_gas_limit"] } -ethers-core = { git = "https://github.com/gakonst/ethers-rs" } +ethers-core = { git = "https://github.com/gakonst/ethers-rs", features = ["eip712"] } # rpc jsonrpsee = { version = "0.16" } @@ -46,7 +46,7 @@ bytes = "1.4" secp256k1 = { version = "0.26.0", features = [ "global-context", "rand-std", - "recovery", + "recovery" ] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/crates/rpc/rpc/src/eth/api/mod.rs b/crates/rpc/rpc/src/eth/api/mod.rs index 826ae34c90c..dac35c88db4 100644 --- a/crates/rpc/rpc/src/eth/api/mod.rs +++ b/crates/rpc/rpc/src/eth/api/mod.rs @@ -19,6 +19,7 @@ use std::{num::NonZeroUsize, ops::Deref, sync::Arc}; mod block; mod call; mod server; +mod sign; mod state; mod transactions; pub use transactions::{EthTransactions, TransactionSource}; diff --git a/crates/rpc/rpc/src/eth/api/server.rs b/crates/rpc/rpc/src/eth/api/server.rs index 4e676713a4e..41f6383a3a1 100644 --- a/crates/rpc/rpc/src/eth/api/server.rs +++ b/crates/rpc/rpc/src/eth/api/server.rs @@ -360,8 +360,8 @@ where } /// Handler for: `eth_sign` - async fn sign(&self, _address: Address, _message: Bytes) -> Result { - Err(internal_rpc_err("unimplemented")) + async fn sign(&self, address: Address, message: Bytes) -> Result { + Ok(EthApi::sign(self, address, message).await?) } /// Handler for: `eth_signTransaction` @@ -370,8 +370,8 @@ where } /// Handler for: `eth_signTypedData` - async fn sign_typed_data(&self, _address: Address, _data: Value) -> Result { - Err(internal_rpc_err("unimplemented")) + async fn sign_typed_data(&self, address: Address, data: Value) -> Result { + Ok(EthApi::sign_typed_data(self, data, address).await?) } /// Handler for: `eth_getProof` diff --git a/crates/rpc/rpc/src/eth/api/sign.rs b/crates/rpc/rpc/src/eth/api/sign.rs new file mode 100644 index 00000000000..edd03e4655a --- /dev/null +++ b/crates/rpc/rpc/src/eth/api/sign.rs @@ -0,0 +1,41 @@ +//! Contains RPC handler implementations specific to sign endpoints +use crate::{ + eth::{ + error::{EthResult, SignError}, + signer::EthSigner, + }, + EthApi, +}; +use ethers_core::types::transaction::eip712::TypedData; +use reth_primitives::{Address, Bytes}; +use serde_json::Value; +use std::ops::Deref; + +impl EthApi { + pub(crate) async fn sign(&self, account: Address, message: Bytes) -> EthResult { + let signer = self.find_signer(&account)?; + let signature = signer.sign(account, &message).await?; + let bytes = hex::encode(signature.to_bytes()).as_bytes().into(); + Ok(bytes) + } + + pub(crate) async fn sign_typed_data(&self, data: Value, account: Address) -> EthResult { + let signer = self.find_signer(&account)?; + let data = serde_json::from_value::(data).map_err(|_| SignError::TypedData)?; + let signature = signer.sign_typed_data(account, &data)?; + let bytes = hex::encode(signature.to_bytes()).as_bytes().into(); + Ok(bytes) + } + + pub(crate) fn find_signer( + &self, + account: &Address, + ) -> Result<&(dyn EthSigner + 'static), SignError> { + self.inner + .signers + .iter() + .find(|signer| signer.is_signer_for(account)) + .map(|signer| signer.deref()) + .ok_or(SignError::NoAccount) + } +} diff --git a/crates/rpc/rpc/src/eth/error.rs b/crates/rpc/rpc/src/eth/error.rs index 3aa76da5804..c95abea25be 100644 --- a/crates/rpc/rpc/src/eth/error.rs +++ b/crates/rpc/rpc/src/eth/error.rs @@ -52,6 +52,9 @@ pub enum EthApiError { /// Other internal error #[error(transparent)] Internal(#[from] reth_interfaces::Error), + /// Error related to signing + #[error(transparent)] + Signing(#[from] SignError), } impl From for RpcError { @@ -65,6 +68,7 @@ impl From for RpcError { EthApiError::ConflictingRequestGasPrice { .. } | EthApiError::ConflictingRequestGasPriceAndTipSet { .. } | EthApiError::RequestLegacyGasPriceAndTipSet { .. } | + EthApiError::Signing(_) | EthApiError::BothStateAndStateDiffInOverride(_) => { rpc_err(INVALID_PARAMS_CODE, error.to_string(), None) } @@ -312,6 +316,20 @@ impl From for EthApiError { } } +/// Errors returned from a sign request. +#[derive(Debug, thiserror::Error)] +pub enum SignError { + /// Error occured while trying to sign data. + #[error("Could not sign")] + CouldNotSign, + /// Signer for requested account not found. + #[error("Unknown account")] + NoAccount, + /// TypedData has invalid format. + #[error("Given typed data is not valid")] + TypedData, +} + /// Returns the revert reason from the `revm::TransactOut` data, if it's an abi encoded String. /// /// **Note:** it's assumed the `out` buffer starts with the call's signature diff --git a/crates/rpc/rpc/src/eth/signer.rs b/crates/rpc/rpc/src/eth/signer.rs index 2c16e748660..90a2bb4deec 100644 --- a/crates/rpc/rpc/src/eth/signer.rs +++ b/crates/rpc/rpc/src/eth/signer.rs @@ -1,11 +1,17 @@ //! An abstraction over ethereum signers. -use jsonrpsee::core::RpcResult as Result; -use reth_primitives::{Address, Signature, TransactionSigned}; +use crate::eth::error::SignError; +use ethers_core::{ + types::transaction::eip712::{Eip712, TypedData}, + utils::hash_message, +}; +use reth_primitives::{sign_message, Address, Signature, TransactionSigned, H256}; use reth_rpc_types::TypedTransactionRequest; use secp256k1::SecretKey; use std::collections::HashMap; +type Result = std::result::Result; + /// An Ethereum Signer used via RPC. #[async_trait::async_trait] pub(crate) trait EthSigner: Send + Sync { @@ -26,6 +32,9 @@ pub(crate) trait EthSigner: Send + Sync { request: TypedTransactionRequest, address: &Address, ) -> Result; + + /// Encodes and signs the typed data according EIP-712. Payload must implement Eip712 trait. + fn sign_typed_data(&self, address: Address, payload: &TypedData) -> Result; } /// Holds developer keys @@ -34,6 +43,16 @@ pub(crate) struct DevSigner { accounts: HashMap, } +impl DevSigner { + fn get_key(&self, account: Address) -> Result<&SecretKey> { + self.accounts.get(&account).ok_or(SignError::NoAccount) + } + fn sign_hash(&self, hash: H256, account: Address) -> Result { + let secret = self.get_key(account)?; + let signature = sign_message(H256::from_slice(secret.as_ref()), hash); + signature.map_err(|_| SignError::CouldNotSign) + } +} #[async_trait::async_trait] impl EthSigner for DevSigner { fn accounts(&self) -> Vec
{ @@ -44,8 +63,11 @@ impl EthSigner for DevSigner { self.accounts.contains_key(addr) } - async fn sign(&self, _address: Address, _message: &[u8]) -> Result { - todo!() + async fn sign(&self, address: Address, message: &[u8]) -> Result { + // Hash message according to EIP 191: + // https://ethereum.org/es/developers/docs/apis/json-rpc/#eth_sign + let hash = hash_message(message).into(); + self.sign_hash(hash, address) } fn sign_transaction( @@ -55,4 +77,131 @@ impl EthSigner for DevSigner { ) -> Result { todo!() } + + fn sign_typed_data(&self, address: Address, payload: &TypedData) -> Result { + let encoded: H256 = payload.encode_eip712().map_err(|_| SignError::TypedData)?.into(); + self.sign_hash(encoded, address) + } +} +#[cfg(test)] +mod test { + use super::*; + use reth_primitives::U256; + use std::str::FromStr; + fn build_signer() -> DevSigner { + let addresses = vec![]; + let secret = + SecretKey::from_str("4646464646464646464646464646464646464646464646464646464646464646") + .unwrap(); + let accounts = HashMap::from([(Address::default(), secret)]); + DevSigner { addresses, accounts } + } + + #[tokio::test] + async fn test_sign_type_data() { + let eip_712_example = serde_json::json!( + r#"{ + "types": { + "EIP712Domain": [ + { + "name": "name", + "type": "string" + }, + { + "name": "version", + "type": "string" + }, + { + "name": "chainId", + "type": "uint256" + }, + { + "name": "verifyingContract", + "type": "address" + } + ], + "Person": [ + { + "name": "name", + "type": "string" + }, + { + "name": "wallet", + "type": "address" + } + ], + "Mail": [ + { + "name": "from", + "type": "Person" + }, + { + "name": "to", + "type": "Person" + }, + { + "name": "contents", + "type": "string" + } + ] + }, + "primaryType": "Mail", + "domain": { + "name": "Ether Mail", + "version": "1", + "chainId": 1, + "verifyingContract": "0xCcCCccccCCCCcCCCCCCcCcCccCcCCCcCcccccccC" + }, + "message": { + "from": { + "name": "Cow", + "wallet": "0xCD2a3d9F938E13CD947Ec05AbC7FE734Df8DD826" + }, + "to": { + "name": "Bob", + "wallet": "0xbBbBBBBbbBBBbbbBbbBbbbbBBbBbbbbBbBbbBBbB" + }, + "contents": "Hello, Bob!" + } + }"# + ); + let data: TypedData = serde_json::from_value(eip_712_example).unwrap(); + let signer = build_signer(); + let sig = signer.sign_typed_data(Address::default(), &data).unwrap(); + let expected = Signature { + r: U256::from_str_radix( + "5318aee9942b84885761bb20e768372b76e7ee454fc4d39b59ce07338d15a06c", + 16, + ) + .unwrap(), + s: U256::from_str_radix( + "5e585a2f4882ec3228a9303244798b47a9102e4be72f48159d890c73e4511d79", + 16, + ) + .unwrap(), + odd_y_parity: false, + }; + assert_eq!(sig, expected) + } + + #[tokio::test] + async fn test_signer() { + let message = b"Test message"; + let signer = build_signer(); + let sig = signer.sign(Address::default(), message).await.unwrap(); + let expected = Signature { + r: U256::from_str_radix( + "54313da7432e4058b8d22491b2e7dbb19c7186c35c24155bec0820a8a2bfe0c1", + 16, + ) + .unwrap(), + s: U256::from_str_radix( + "687250f11a3d4435004c04a4cb60e846bc27997271d67f21c6c8170f17a25e10", + 16, + ) + .unwrap(), + odd_y_parity: true, + }; + assert_eq!(sig, expected) + } } From 880759ac57232ce23414914392726ca0912cac77 Mon Sep 17 00:00:00 2001 From: Aditya Pandey Date: Wed, 15 Mar 2023 06:49:15 +0530 Subject: [PATCH 129/191] Add p2p listener port options for reth node (#1753) Co-authored-by: Georgios Konstantopoulos --- bin/reth/src/args/network_args.rs | 4 ++++ bin/reth/src/node/mod.rs | 11 ++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/bin/reth/src/args/network_args.rs b/bin/reth/src/args/network_args.rs index a0af02ef985..ff99f419ca2 100644 --- a/bin/reth/src/args/network_args.rs +++ b/bin/reth/src/args/network_args.rs @@ -91,6 +91,10 @@ pub struct DiscoveryArgs { /// Disable Discv4 discovery. #[arg(long, conflicts_with = "disable_discovery")] disable_discv4_discovery: bool, + + /// The UDP port to use for P2P discovery/networking. + #[arg(long = "discovery.port")] + pub port: Option, } impl DiscoveryArgs { diff --git a/bin/reth/src/node/mod.rs b/bin/reth/src/node/mod.rs index 6c8a325a98e..bb99109720e 100644 --- a/bin/reth/src/node/mod.rs +++ b/bin/reth/src/node/mod.rs @@ -20,6 +20,7 @@ use reth_db::{ tables, transaction::DbTx, }; +use reth_discv4::DEFAULT_DISCOVERY_PORT; use reth_downloaders::{ bodies::bodies::BodiesDownloaderBuilder, headers::reverse_headers::ReverseHeadersDownloaderBuilder, @@ -52,7 +53,11 @@ use reth_stages::{ stages::{ExecutionStage, SenderRecoveryStage, TotalDifficultyStage, FINISH}, }; use reth_tasks::TaskExecutor; -use std::{net::SocketAddr, path::PathBuf, sync::Arc}; +use std::{ + net::{Ipv4Addr, SocketAddr, SocketAddrV4}, + path::PathBuf, + sync::Arc, +}; use tokio::sync::{mpsc::unbounded_channel, watch}; use tracing::*; @@ -417,6 +422,10 @@ impl Command { .network_config(config, self.chain.clone()) .with_task_executor(Box::new(executor)) .set_head(head) + .listener_addr(SocketAddr::V4(SocketAddrV4::new( + Ipv4Addr::UNSPECIFIED, + self.network.discovery.port.unwrap_or(DEFAULT_DISCOVERY_PORT), + ))) .build(ShareableDatabase::new(db, self.chain.clone())) } From 91f26f455f7c07cd81a043ed01cde110881b39f6 Mon Sep 17 00:00:00 2001 From: Chen Kai <281165273grape@gmail.com> Date: Wed, 15 Mar 2023 09:34:11 +0800 Subject: [PATCH 130/191] feat: validate withdrawal when validating block. (#1521) Signed-off-by: grapebaba <281165273@qq.com> --- Cargo.lock | 97 ++++++++++++++ crates/consensus/Cargo.toml | 1 + crates/consensus/src/validation.rs | 119 ++++++++++++++++-- crates/storage/provider/src/providers/mod.rs | 11 ++ .../provider/src/traits/withdrawals.rs | 3 + 5 files changed, 222 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5b4fb682dbd..71f1994d0c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1369,6 +1369,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "difflib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + [[package]] name = "digest" version = "0.8.1" @@ -1492,6 +1498,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "downcast" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1" + [[package]] name = "dunce" version = "1.0.3" @@ -2032,6 +2044,15 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "float-cmp" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4" +dependencies = [ + "num-traits", +] + [[package]] name = "fnv" version = "1.0.7" @@ -2047,6 +2068,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fragile" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c2141d6d6c8512188a7891b4b01590a45f6dac67afb4f255c4124dbb86d4eaa" + [[package]] name = "funty" version = "2.0.0" @@ -3359,6 +3386,33 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "mockall" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50e4a1c770583dac7ab5e2f6c139153b783a53a1bbee9729613f193e59828326" +dependencies = [ + "cfg-if", + "downcast", + "fragile", + "lazy_static", + "mockall_derive", + "predicates", + "predicates-tree", +] + +[[package]] +name = "mockall_derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "832663583d5fa284ca8810bf7015e46c9fff9622d3cf34bd1eea5003fec06dd0" +dependencies = [ + "cfg-if", + "proc-macro2 1.0.51", + "quote 1.0.23", + "syn 1.0.109", +] + [[package]] name = "modular-bitfield" version = "0.11.2" @@ -3417,6 +3471,12 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "normalize-line-endings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -3935,6 +3995,36 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +[[package]] +name = "predicates" +version = "2.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59230a63c37f3e18569bdb90e4a89cbf5bf8b06fea0b84e65ea10cc4df47addd" +dependencies = [ + "difflib", + "float-cmp", + "itertools 0.10.5", + "normalize-line-endings", + "predicates-core", + "regex", +] + +[[package]] +name = "predicates-core" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f883590242d3c6fc5bf50299011695fa6590c2c70eac95ee1bdb9a733ad1a2" + +[[package]] +name = "predicates-tree" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54ff541861505aabf6ea722d2131ee980b8276e10a1297b94e896dd8b621850d" +dependencies = [ + "predicates-core", + "termtree", +] + [[package]] name = "prettyplease" version = "0.1.23" @@ -4414,6 +4504,7 @@ name = "reth-consensus" version = "0.1.0" dependencies = [ "assert_matches", + "mockall", "reth-interfaces", "reth-primitives", "reth-provider", @@ -6145,6 +6236,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "termtree" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95059e91184749cb66be6dc994f67f182b6d897cb3df74a5bf66b5e709295fd8" + [[package]] name = "test-fuzz" version = "3.0.5" diff --git a/crates/consensus/Cargo.toml b/crates/consensus/Cargo.toml index 1cd67f69d92..fbd98504de5 100644 --- a/crates/consensus/Cargo.toml +++ b/crates/consensus/Cargo.toml @@ -19,3 +19,4 @@ tokio = { version = "1", features = ["sync"] } reth-interfaces = { path = "../interfaces", features = ["test-utils"] } reth-provider = { path = "../storage/provider", features = ["test-utils"] } assert_matches = "1.5.0" +mockall = "0.11.3" diff --git a/crates/consensus/src/validation.rs b/crates/consensus/src/validation.rs index ff4310757ba..60364f3d8ad 100644 --- a/crates/consensus/src/validation.rs +++ b/crates/consensus/src/validation.rs @@ -4,7 +4,7 @@ use reth_primitives::{ BlockNumber, ChainSpec, Hardfork, Header, InvalidTransactionError, SealedBlock, SealedHeader, Transaction, TransactionSignedEcRecovered, TxEip1559, TxEip2930, TxLegacy, }; -use reth_provider::{AccountProvider, HeaderProvider}; +use reth_provider::{AccountProvider, HeaderProvider, WithdrawalsProvider}; use std::{ collections::{hash_map::Entry, HashMap}, time::SystemTime, @@ -348,9 +348,10 @@ pub fn validate_header_regarding_parent( /// Checks: /// If we already know the block. /// If parent is known +/// If withdarwals are valid /// /// Returns parent block header -pub fn validate_block_regarding_chain( +pub fn validate_block_regarding_chain( block: &SealedBlock, provider: &PROV, ) -> RethResult { @@ -366,12 +367,39 @@ pub fn validate_block_regarding_chain( .header(&block.parent_hash)? .ok_or(ConsensusError::ParentUnknown { hash: block.parent_hash })?; + // Check if withdrawals are valid. + if let Some(withdrawals) = &block.withdrawals { + if !withdrawals.is_empty() { + let latest_withdrawal = provider.latest_withdrawal()?; + match latest_withdrawal { + Some(withdrawal) => { + if withdrawal.index + 1 != withdrawals.first().unwrap().index { + return Err(ConsensusError::WithdrawalIndexInvalid { + got: withdrawals.first().unwrap().index, + expected: withdrawal.index + 1, + } + .into()) + } + } + None => { + if withdrawals.first().unwrap().index != 0 { + return Err(ConsensusError::WithdrawalIndexInvalid { + got: withdrawals.first().unwrap().index, + expected: 0, + } + .into()) + } + } + } + } + } + // Return parent header. Ok(parent.seal(block.parent_hash)) } /// Full validation of block before execution. -pub fn full_validation( +pub fn full_validation( block: &SealedBlock, provider: Provider, chain_spec: &ChainSpec, @@ -401,10 +429,11 @@ pub fn full_validation( mod tests { use super::*; use assert_matches::assert_matches; - use reth_interfaces::Result; + use mockall::mock; + use reth_interfaces::{Error::Consensus, Result}; use reth_primitives::{ - hex_literal::hex, proofs, Account, Address, BlockHash, Bytes, ChainSpecBuilder, Header, - Signature, TransactionKind, TransactionSigned, Withdrawal, MAINNET, U256, + hex_literal::hex, proofs, Account, Address, BlockHash, BlockId, Bytes, ChainSpecBuilder, + Header, Signature, TransactionKind, TransactionSigned, Withdrawal, MAINNET, U256, }; use std::ops::RangeBounds; @@ -435,20 +464,45 @@ mod tests { } } + mock! { + WithdrawalsProvider {} + + impl WithdrawalsProvider for WithdrawalsProvider { + fn latest_withdrawal(&self) -> Result> ; + + fn withdrawals_by_block( + &self, + _id: BlockId, + _timestamp: u64, + ) -> RethResult>> ; + } + } + struct Provider { is_known: bool, parent: Option
, account: Option, + withdrawals_provider: MockWithdrawalsProvider, } impl Provider { /// New provider with parent fn new(parent: Option
) -> Self { - Self { is_known: false, parent, account: None } + Self { + is_known: false, + parent, + account: None, + withdrawals_provider: MockWithdrawalsProvider::new(), + } } /// New provider where is_known is always true fn new_known() -> Self { - Self { is_known: true, parent: None, account: None } + Self { + is_known: true, + parent: None, + account: None, + withdrawals_provider: MockWithdrawalsProvider::new(), + } } } @@ -484,6 +538,20 @@ mod tests { } } + impl WithdrawalsProvider for Provider { + fn latest_withdrawal(&self) -> Result> { + self.withdrawals_provider.latest_withdrawal() + } + + fn withdrawals_by_block( + &self, + _id: BlockId, + _timestamp: u64, + ) -> RethResult>> { + self.withdrawals_provider.withdrawals_by_block(_id, _timestamp) + } + } + fn mock_tx(nonce: u64) -> TransactionSignedEcRecovered { let request = Transaction::Eip2930(TxEip2930 { chain_id: 1u64, @@ -525,7 +593,7 @@ mod tests { mix_hash: hex!("0000000000000000000000000000000000000000000000000000000000000000").into(), nonce: 0x0000000000000000, base_fee_per_gas: 0x28f0001df.into(), - withdrawals_root: None + withdrawals_root: None, }; // size: 0x9b5 @@ -656,6 +724,39 @@ mod tests { validate_block_standalone(&block, &chain_spec), Err(ConsensusError::WithdrawalIndexInvalid { .. }) ); + + let (_, parent) = mock_block(); + let mut provider = Provider::new(Some(parent.clone())); + // Withdrawal index should be 0 if there are no withdrawals in the chain + let block = create_block_with_withdrawals(&[1, 2, 3]); + provider.withdrawals_provider.expect_latest_withdrawal().return_const(Ok(None)); + assert_matches!( + validate_block_regarding_chain(&block, &provider), + Err(Consensus(ConsensusError::WithdrawalIndexInvalid { got: 1, expected: 0 })) + ); + let block = create_block_with_withdrawals(&[0, 1, 2]); + let res = validate_block_regarding_chain(&block, &provider); + assert!(res.is_ok()); + + // Withdrawal index should be the last withdrawal index + 1 + let mut provider = Provider::new(Some(parent.clone())); + let block = create_block_with_withdrawals(&[4, 5, 6]); + provider + .withdrawals_provider + .expect_latest_withdrawal() + .return_const(Ok(Some(Withdrawal { index: 2, ..Default::default() }))); + assert_matches!( + validate_block_regarding_chain(&block, &provider), + Err(Consensus(ConsensusError::WithdrawalIndexInvalid { got: 4, expected: 3 })) + ); + + let block = create_block_with_withdrawals(&[3, 4, 5]); + provider + .withdrawals_provider + .expect_latest_withdrawal() + .return_const(Ok(Some(Withdrawal { index: 2, ..Default::default() }))); + let res = validate_block_regarding_chain(&block, &provider); + assert!(res.is_ok()); } #[test] diff --git a/crates/storage/provider/src/providers/mod.rs b/crates/storage/provider/src/providers/mod.rs index 6b090dba425..bd5df0818b9 100644 --- a/crates/storage/provider/src/providers/mod.rs +++ b/crates/storage/provider/src/providers/mod.rs @@ -267,6 +267,17 @@ impl WithdrawalsProvider for ShareableDatabase { } Ok(None) } + + fn latest_withdrawal(&self) -> Result> { + let latest_block_withdrawal = + self.db.view(|tx| tx.cursor_read::()?.last())?; + latest_block_withdrawal + .map(|block_withdrawal_pair| { + block_withdrawal_pair + .and_then(|(_, block_withdrawal)| block_withdrawal.withdrawals.last().cloned()) + }) + .map_err(Into::into) + } } impl EvmEnvProvider for ShareableDatabase { diff --git a/crates/storage/provider/src/traits/withdrawals.rs b/crates/storage/provider/src/traits/withdrawals.rs index 2f37e3bc270..9a6fe804507 100644 --- a/crates/storage/provider/src/traits/withdrawals.rs +++ b/crates/storage/provider/src/traits/withdrawals.rs @@ -5,4 +5,7 @@ use reth_primitives::{BlockId, Withdrawal}; pub trait WithdrawalsProvider: Send + Sync { /// Get withdrawals by block id. fn withdrawals_by_block(&self, id: BlockId, timestamp: u64) -> Result>>; + + /// Get latest withdrawal from this block or earlier . + fn latest_withdrawal(&self) -> Result>; } From 15d79cedade20207e2537ad4a4bc3a78765bf9f1 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 15 Mar 2023 11:15:55 +0100 Subject: [PATCH 131/191] chore(deps): upgrade secp256k1+enr (#1715) --- Cargo.lock | 458 ++++++++++++++++--------- bin/reth/Cargo.toml | 2 +- crates/interfaces/Cargo.toml | 4 +- crates/net/discv4/Cargo.toml | 5 +- crates/net/dns/Cargo.toml | 5 +- crates/net/dns/src/lib.rs | 2 +- crates/net/dns/src/tree.rs | 35 +- crates/net/ecies/Cargo.toml | 2 +- crates/net/eth-wire/Cargo.toml | 2 +- crates/net/network/Cargo.toml | 6 +- crates/primitives/Cargo.toml | 6 +- crates/revm/revm-primitives/Cargo.toml | 2 +- crates/rlp/Cargo.toml | 4 +- crates/staged-sync/Cargo.toml | 4 +- crates/storage/db/Cargo.toml | 4 +- 15 files changed, 361 insertions(+), 180 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 71f1994d0c2..206793fd004 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -150,7 +150,7 @@ checksum = "5bf310f0dd77f453bc43ec61506ead8283ab49423686244e474d0cc16226400c" dependencies = [ "itertools 0.9.0", "proc-macro-error", - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "syn 1.0.109", ] @@ -216,7 +216,7 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4655ae1a7b0cdf149156f780c5bf3f1352bc53cbd9e0a361a7ef7b22947e965" dependencies = [ - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "syn 1.0.109", ] @@ -227,7 +227,7 @@ version = "0.1.65" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "095183a3539c7c7649b2beb87c2d3f0591f3a7fed07761cc546d244e27e0238c" dependencies = [ - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "syn 1.0.109", ] @@ -282,7 +282,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a8c1df849285fbacd587de7818cc7d13be6cd2cbcd47a04fb1801b0e2706e33" dependencies = [ "proc-macro-error", - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "syn 1.0.109", ] @@ -326,6 +326,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce" +[[package]] +name = "base16ct" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" + [[package]] name = "base58" version = "0.1.0" @@ -411,7 +417,7 @@ dependencies = [ "lazy_static", "lazycell", "peeking_take_while", - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "regex", "rustc-hash", @@ -622,7 +628,7 @@ checksum = "08a1ec454bc3eead8719cb56e15dbbfecdbc14e4b3a3ae4936cc6e31f5fc0d07" dependencies = [ "camino", "cargo-platform", - "semver 1.0.16", + "semver 1.0.17", "serde", "serde_json", "thiserror", @@ -777,7 +783,7 @@ checksum = "44bec8e5c9d09e439c4335b1af0abaab56dcf3b94999a936e1bb47b9134288f0" dependencies = [ "heck", "proc-macro-error", - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "syn 1.0.109", ] @@ -812,7 +818,7 @@ version = "0.1.0" dependencies = [ "convert_case 0.6.0", "parity-scale-codec", - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "serde", "syn 1.0.109", @@ -830,9 +836,9 @@ dependencies = [ [[package]] name = "coins-bip32" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634c509653de24b439672164bbf56f5f582a2ab0e313d3b0f6af0b7345cf2560" +checksum = "8a3b95d26eace980ade01e0ea8d996f7d5ae7031f9c5f258317ec82e400f33c1" dependencies = [ "bincode", "bs58", @@ -840,7 +846,7 @@ dependencies = [ "digest 0.10.6", "getrandom 0.2.8", "hmac", - "k256", + "k256 0.11.6", "lazy_static", "serde", "sha2 0.10.6", @@ -849,26 +855,28 @@ dependencies = [ [[package]] name = "coins-bip39" -version = "0.7.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a11892bcac83b4c6e95ab84b5b06c76d9d70ad73548dd07418269c5c7977171" +checksum = "a05ceda6ab4876de899fe23e8a171b200be9a346289bce12d41311e4bce2f104" dependencies = [ "bitvec 0.17.4", "coins-bip32", "getrandom 0.2.8", "hex", "hmac", + "once_cell", "pbkdf2", "rand 0.8.5", "sha2 0.10.6", "thiserror", + "tracing", ] [[package]] name = "coins-core" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c94090a6663f224feae66ab01e41a2555a8296ee07b5f20dab8888bdefc9f617" +checksum = "87637d08a1dd6bfa3bcd697f0a4de6c1b3a03e085b9d841a7e9cde4ccb61514b" dependencies = [ "base58check", "base64 0.12.3", @@ -1117,6 +1125,18 @@ dependencies = [ "zeroize", ] +[[package]] +name = "crypto-bigint" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071c0f5945634bc9ba7a452f492377dd6b1993665ddb58f28704119b32f07a9a" +dependencies = [ + "generic-array 0.14.6", + "rand_core 0.6.4", + "subtle", + "zeroize", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -1179,7 +1199,7 @@ dependencies = [ "cc", "codespan-reporting", "once_cell", - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "scratch", "syn 1.0.109", @@ -1197,7 +1217,7 @@ version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "086c685979a698443656e5cf7856c95c642295a38599f12fb1ff76fb28d19892" dependencies = [ - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "syn 1.0.109", ] @@ -1230,7 +1250,7 @@ checksum = "f0c960ae2da4de88a91b2d920c2a7233b400bc33cb28453a2987822d8392519b" dependencies = [ "fnv", "ident_case", - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "strsim 0.9.3", "syn 1.0.109", @@ -1244,7 +1264,7 @@ checksum = "001d80444f28e193f30c2f293455da62dcf9a6b29918a4253152ae2b1de592cb" dependencies = [ "fnv", "ident_case", - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "strsim 0.10.0", "syn 1.0.109", @@ -1320,13 +1340,23 @@ dependencies = [ "zeroize", ] +[[package]] +name = "der" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc906908ea6458456e5eaa160a9c08543ec3d1e6f71e2235cedd660cb65f9df0" +dependencies = [ + "const-oid", + "zeroize", +] + [[package]] name = "derive_arbitrary" version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8beee4701e2e229e8098bbdecdca12449bc3e322f137d269182fa1291e20bd00" dependencies = [ - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "syn 1.0.109", ] @@ -1339,7 +1369,7 @@ checksum = "a2658621297f2cf68762a6f7dc0bb7e1ff2cfd6583daef8ee0fed6f7ec468ec0" dependencies = [ "darling 0.10.2", "derive_builder_core", - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "syn 1.0.109", ] @@ -1351,7 +1381,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2791ea3e372c8495c0bc2033991d76b512cd799d07491fbd6890124db9458bef" dependencies = [ "darling 0.10.2", - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "syn 1.0.109", ] @@ -1363,7 +1393,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" dependencies = [ "convert_case 0.4.0", - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "rustc_version", "syn 1.0.109", @@ -1463,7 +1493,7 @@ dependencies = [ "aes-gcm", "arrayvec", "delay_map", - "enr", + "enr 0.7.0", "fnv", "futures", "hashlink", @@ -1522,10 +1552,22 @@ version = "0.14.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c" dependencies = [ - "der", - "elliptic-curve", - "rfc6979", - "signature", + "der 0.6.1", + "elliptic-curve 0.12.3", + "rfc6979 0.3.1", + "signature 1.6.4", +] + +[[package]] +name = "ecdsa" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1b0a1222f8072619e8a6b667a854020a03d363738303203c09468b3424a420a" +dependencies = [ + "der 0.7.1", + "elliptic-curve 0.13.2", + "rfc6979 0.4.0", + "signature 2.0.0", ] [[package]] @@ -1534,7 +1576,7 @@ version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91cff35c70bba8a626e3185d8cd48cc11b5437e1a5bcd15b9b5fa3c64b6dfee7" dependencies = [ - "signature", + "signature 1.6.4", ] [[package]] @@ -1558,7 +1600,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb0188e3c3ba8df5753894d54461f0e39bc91741dc5b22e1c46999ec2c71f4e4" dependencies = [ "enum-ordinalize", - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "syn 1.0.109", ] @@ -1575,16 +1617,35 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3" dependencies = [ - "base16ct", - "crypto-bigint", - "der", + "base16ct 0.1.1", + "crypto-bigint 0.4.9", + "der 0.6.1", "digest 0.10.6", - "ff", + "ff 0.12.1", "generic-array 0.14.6", - "group", - "pkcs8", + "group 0.12.1", + "pkcs8 0.9.0", "rand_core 0.6.4", - "sec1", + "sec1 0.3.0", + "subtle", + "zeroize", +] + +[[package]] +name = "elliptic-curve" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea5a92946e8614bb585254898bb7dd1ddad241ace60c52149e3765e34cc039d" +dependencies = [ + "base16ct 0.2.0", + "crypto-bigint 0.5.0", + "digest 0.10.6", + "ff 0.13.0", + "generic-array 0.14.6", + "group 0.13.0", + "pkcs8 0.10.1", + "rand_core 0.6.4", + "sec1 0.7.1", "subtle", "zeroize", ] @@ -1615,11 +1676,29 @@ dependencies = [ "bytes", "ed25519-dalek", "hex", - "k256", + "k256 0.11.6", "log", "rand 0.8.5", "rlp", - "secp256k1 0.24.3", + "serde", + "sha3", + "zeroize", +] + +[[package]] +name = "enr" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb4d5fbf6f56acecd38f5988eb2e4ae412008a2a30268c748c701ec6322f39d4" +dependencies = [ + "base64 0.13.1", + "bytes", + "hex", + "k256 0.13.0", + "log", + "rand 0.8.5", + "rlp", + "secp256k1", "serde", "sha3", "zeroize", @@ -1632,7 +1711,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "570d109b813e904becc80d8d5da38376818a143348413f7149f1340fe04754d4" dependencies = [ "heck", - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "syn 1.0.109", ] @@ -1644,7 +1723,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c9720bba047d567ffc8a3cba48bf19126600e249ab7f128e9233e6376976a116" dependencies = [ "heck", - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "syn 1.0.109", ] @@ -1657,7 +1736,7 @@ checksum = "a62bb1df8b45ecb7ffa78dca1c17a438fb193eb083db0b1b494d2a61bcb5096a" dependencies = [ "num-bigint", "num-traits", - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "rustc_version", "syn 1.0.109", @@ -1669,7 +1748,7 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e88bcb3a067a6555d577aba299e75eff9942da276e6506fc6274327daa026132" dependencies = [ - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "syn 1.0.109", ] @@ -1767,8 +1846,8 @@ dependencies = [ [[package]] name = "ethers-contract" -version = "1.0.2" -source = "git+https://github.com/gakonst/ethers-rs#e54666b467c9cc5848873935622930449075828a" +version = "2.0.0" +source = "git+https://github.com/gakonst/ethers-rs#18a049b4c49965fbb4efbb394891767dc0fa1fa9" dependencies = [ "ethers-contract-abigen", "ethers-contract-derive", @@ -1785,8 +1864,8 @@ dependencies = [ [[package]] name = "ethers-contract-abigen" -version = "1.0.2" -source = "git+https://github.com/gakonst/ethers-rs#e54666b467c9cc5848873935622930449075828a" +version = "2.0.0" +source = "git+https://github.com/gakonst/ethers-rs#18a049b4c49965fbb4efbb394891767dc0fa1fa9" dependencies = [ "Inflector", "cfg-if", @@ -1797,7 +1876,7 @@ dependencies = [ "getrandom 0.2.8", "hex", "prettyplease", - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "regex", "reqwest", @@ -1812,39 +1891,37 @@ dependencies = [ [[package]] name = "ethers-contract-derive" -version = "1.0.2" -source = "git+https://github.com/gakonst/ethers-rs#e54666b467c9cc5848873935622930449075828a" +version = "2.0.0" +source = "git+https://github.com/gakonst/ethers-rs#18a049b4c49965fbb4efbb394891767dc0fa1fa9" dependencies = [ "ethers-contract-abigen", "ethers-core", - "eyre", "hex", - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", - "serde_json", "syn 1.0.109", ] [[package]] name = "ethers-core" -version = "1.0.2" -source = "git+https://github.com/gakonst/ethers-rs#e54666b467c9cc5848873935622930449075828a" +version = "2.0.0" +source = "git+https://github.com/gakonst/ethers-rs#18a049b4c49965fbb4efbb394891767dc0fa1fa9" dependencies = [ "arrayvec", "bytes", "cargo_metadata", "chrono", "convert_case 0.6.0", - "elliptic-curve", + "elliptic-curve 0.13.2", "ethabi", "generic-array 0.14.6", "getrandom 0.2.8", "hex", - "k256", + "k256 0.13.0", "num_enum", "once_cell", "open-fastrlp", - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "rand 0.8.5", "rlp", "rlp-derive", @@ -1860,13 +1937,13 @@ dependencies = [ [[package]] name = "ethers-etherscan" -version = "1.0.2" -source = "git+https://github.com/gakonst/ethers-rs#e54666b467c9cc5848873935622930449075828a" +version = "2.0.0" +source = "git+https://github.com/gakonst/ethers-rs#18a049b4c49965fbb4efbb394891767dc0fa1fa9" dependencies = [ "ethers-core", "getrandom 0.2.8", "reqwest", - "semver 1.0.16", + "semver 1.0.17", "serde", "serde-aux", "serde_json", @@ -1876,8 +1953,8 @@ dependencies = [ [[package]] name = "ethers-middleware" -version = "1.0.2" -source = "git+https://github.com/gakonst/ethers-rs#e54666b467c9cc5848873935622930449075828a" +version = "2.0.0" +source = "git+https://github.com/gakonst/ethers-rs#18a049b4c49965fbb4efbb394891767dc0fa1fa9" dependencies = [ "async-trait", "auto_impl", @@ -1901,13 +1978,13 @@ dependencies = [ [[package]] name = "ethers-providers" -version = "1.0.2" -source = "git+https://github.com/gakonst/ethers-rs#e54666b467c9cc5848873935622930449075828a" +version = "2.0.0" +source = "git+https://github.com/gakonst/ethers-rs#18a049b4c49965fbb4efbb394891767dc0fa1fa9" dependencies = [ "async-trait", "auto_impl", "base64 0.21.0", - "enr", + "enr 0.8.0", "ethers-core", "futures-channel", "futures-core", @@ -1938,13 +2015,13 @@ dependencies = [ [[package]] name = "ethers-signers" -version = "1.0.2" -source = "git+https://github.com/gakonst/ethers-rs#e54666b467c9cc5848873935622930449075828a" +version = "2.0.0" +source = "git+https://github.com/gakonst/ethers-rs#18a049b4c49965fbb4efbb394891767dc0fa1fa9" dependencies = [ "async-trait", "coins-bip32", "coins-bip39", - "elliptic-curve", + "elliptic-curve 0.13.2", "eth-keystore", "ethers-core", "hex", @@ -2010,6 +2087,16 @@ dependencies = [ "subtle", ] +[[package]] +name = "ff" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ded41244b729663b1e574f1b4fb731469f69f79c17667b5d776b16cda0479449" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "findshlibs" version = "0.10.2" @@ -2144,7 +2231,7 @@ version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95a73af87da33b5acf53acfebdc339fe592ecf5357ac7c0a7734ab9d8c876a70" dependencies = [ - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "syn 1.0.109", ] @@ -2215,6 +2302,7 @@ checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9" dependencies = [ "typenum", "version_check", + "zeroize", ] [[package]] @@ -2327,7 +2415,18 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" dependencies = [ - "ff", + "ff 0.12.1", + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "group" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" +dependencies = [ + "ff 0.13.0", "rand_core 0.6.4", "subtle", ] @@ -2763,7 +2862,7 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "11d7a9f6330b71fea57921c9b61c47ee6e84f72d394754eff6163ae67e7395eb" dependencies = [ - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "syn 1.0.109", ] @@ -3005,7 +3104,7 @@ checksum = "baa6da1e4199c10d7b1d0a6e5e8bd8e55f351163b6f4b3cbb044672a69bd4c1c" dependencies = [ "heck", "proc-macro-crate", - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "syn 1.0.109", ] @@ -3090,12 +3189,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72c1e0b51e7ec0a97369623508396067a486bd0cbed95a2659a4b863d28cfc8b" dependencies = [ "cfg-if", - "ecdsa", - "elliptic-curve", + "ecdsa 0.14.8", + "elliptic-curve 0.12.3", "sha2 0.10.6", "sha3", ] +[[package]] +name = "k256" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "955890845095ccf31ef83ad41a05aabb4d8cc23dc3cac5a9f5c89cf26dd0da75" +dependencies = [ + "cfg-if", + "ecdsa 0.16.1", + "elliptic-curve 0.13.2", + "once_cell", + "sha2 0.10.6", + "signature 2.0.0", +] + [[package]] name = "keccak" version = "0.1.3" @@ -3317,7 +3430,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "731f8ecebd9f3a4aa847dfe75455e4757a45da40a7793d2f0b1f9b6ed18b23f3" dependencies = [ - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "syn 1.0.109", ] @@ -3408,7 +3521,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "832663583d5fa284ca8810bf7015e46c9fff9622d3cf34bd1eea5003fec06dd0" dependencies = [ "cfg-if", - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "syn 1.0.109", ] @@ -3429,7 +3542,7 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a7d5f7076603ebc68de2dc6a650ec331a062a13abaa346975be747bbfa4b789" dependencies = [ - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "syn 1.0.109", ] @@ -3600,7 +3713,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcbff9bc912032c62bf65ef1d5aea88983b420f4f839db1e9b0c281a25c9c799" dependencies = [ "proc-macro-crate", - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "syn 1.0.109", ] @@ -3658,7 +3771,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "003b2be5c6c53c1cfeb0a238b8a1c3915cd410feb684457a36c10038f764bb1c" dependencies = [ "bytes", - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "syn 1.0.109", ] @@ -3722,7 +3835,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86b26a931f824dd4eca30b3e43bb4f31cd5f0d3a403c5f5ff27106b805bfde7b" dependencies = [ "proc-macro-crate", - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "syn 1.0.109", ] @@ -3874,7 +3987,7 @@ version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" dependencies = [ - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "syn 1.0.109", ] @@ -3897,8 +4010,18 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba" dependencies = [ - "der", - "spki", + "der 0.6.1", + "spki 0.6.0", +] + +[[package]] +name = "pkcs8" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d2820d87d2b008616e5c27212dd9e0e694fb4c6b522de06094106813328cb49" +dependencies = [ + "der 0.7.1", + "spki 0.7.0", ] [[package]] @@ -4031,7 +4154,7 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e97e3215779627f01ee256d2fad52f3d95e8e1c11e9fc6fd08f7cd455d5d5c78" dependencies = [ - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "syn 1.0.109", ] @@ -4066,7 +4189,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" dependencies = [ "proc-macro-error-attr", - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "syn 1.0.109", "version_check", @@ -4078,7 +4201,7 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" dependencies = [ - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "version_check", ] @@ -4094,9 +4217,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.51" +version = "1.0.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6" +checksum = "1d0e1ae9e836cc3beddd63db0df682593d7e2d3d891ae8c9083d2113e1744224" dependencies = [ "unicode-ident", ] @@ -4206,7 +4329,7 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" dependencies = [ - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", ] [[package]] @@ -4537,7 +4660,7 @@ dependencies = [ "reth-interfaces", "reth-libmdbx", "reth-primitives", - "secp256k1 0.24.3", + "secp256k1", "serde", "serde_json", "tempfile", @@ -4552,7 +4675,7 @@ name = "reth-discv4" version = "0.1.0" dependencies = [ "discv5", - "enr", + "enr 0.8.0", "generic-array 0.14.6", "hex", "rand 0.8.5", @@ -4562,7 +4685,7 @@ dependencies = [ "reth-rlp", "reth-rlp-derive", "reth-tracing", - "secp256k1 0.24.3", + "secp256k1", "serde", "thiserror", "tokio", @@ -4576,7 +4699,7 @@ version = "0.1.0" dependencies = [ "async-trait", "data-encoding", - "enr", + "enr 0.8.0", "linked_hash_set", "parking_lot 0.12.1", "reth-net-common", @@ -4584,7 +4707,7 @@ dependencies = [ "reth-rlp", "reth-tracing", "schnellru", - "secp256k1 0.24.3", + "secp256k1", "serde", "serde_with", "thiserror", @@ -4641,7 +4764,7 @@ dependencies = [ "reth-net-common", "reth-primitives", "reth-rlp", - "secp256k1 0.24.3", + "secp256k1", "sha2 0.10.6", "sha3", "thiserror", @@ -4673,7 +4796,7 @@ dependencies = [ "reth-primitives", "reth-rlp", "reth-tracing", - "secp256k1 0.24.3", + "secp256k1", "serde", "smol_str", "snap", @@ -4730,7 +4853,7 @@ dependencies = [ "reth-primitives", "reth-rpc-types", "revm-primitives", - "secp256k1 0.24.3", + "secp256k1", "thiserror", "tokio", "tokio-stream", @@ -4799,7 +4922,7 @@ version = "0.1.0" dependencies = [ "metrics", "once_cell", - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "regex", "serial_test", @@ -4837,7 +4960,7 @@ dependencies = [ "aquamarine", "async-trait", "auto_impl", - "enr", + "enr 0.8.0", "ethers-core", "ethers-middleware", "ethers-providers", @@ -4869,7 +4992,7 @@ dependencies = [ "reth-tasks", "reth-tracing", "reth-transaction-pool", - "secp256k1 0.24.3", + "secp256k1", "serde", "serde_json", "serial_test", @@ -4920,7 +5043,7 @@ dependencies = [ "reth-rlp", "reth-rlp-derive", "revm-primitives", - "secp256k1 0.24.3", + "secp256k1", "serde", "serde_json", "serde_with", @@ -4995,7 +5118,7 @@ dependencies = [ "auto_impl", "bytes", "criterion", - "enr", + "enr 0.8.0", "ethereum-types", "ethnum", "hex-literal", @@ -5005,7 +5128,7 @@ dependencies = [ "reth-rlp-derive", "revm-primitives", "rlp", - "secp256k1 0.24.3", + "secp256k1", "smol_str", ] @@ -5013,7 +5136,7 @@ dependencies = [ name = "reth-rlp-derive" version = "0.1.1" dependencies = [ - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "syn 1.0.109", ] @@ -5047,7 +5170,7 @@ dependencies = [ "reth-transaction-pool", "revm", "schnellru", - "secp256k1 0.26.0", + "secp256k1", "serde", "serde_json", "thiserror", @@ -5135,7 +5258,7 @@ version = "0.1.0" dependencies = [ "async-trait", "confy", - "enr", + "enr 0.8.0", "ethers-core", "ethers-middleware", "ethers-providers", @@ -5157,7 +5280,7 @@ dependencies = [ "reth-stages", "reth-tasks", "reth-tracing", - "secp256k1 0.24.3", + "secp256k1", "serde", "serde_json", "shellexpand", @@ -5281,12 +5404,12 @@ name = "revm-precompile" version = "2.0.0" source = "git+https://github.com/bluealloy/revm?rev=afc3066#afc30663270f77df9b4399ad9d4cfb0ad2b814ec" dependencies = [ - "k256", + "k256 0.11.6", "num", "once_cell", "revm-primitives", "ripemd", - "secp256k1 0.26.0", + "secp256k1", "sha2 0.10.6", "sha3", "substrate-bn", @@ -5322,11 +5445,21 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb" dependencies = [ - "crypto-bigint", + "crypto-bigint 0.4.9", "hmac", "zeroize", ] +[[package]] +name = "rfc6979" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" +dependencies = [ + "hmac", + "subtle", +] + [[package]] name = "rgb" version = "0.8.36" @@ -5376,7 +5509,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e33d7b2abe0c340d8797fe2907d3f20d3b5ea5908683618bfe80df7f621f672a" dependencies = [ - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "syn 1.0.109", ] @@ -5426,7 +5559,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver 1.0.16", + "semver 1.0.17", ] [[package]] @@ -5537,7 +5670,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "303959cf613a6f6efd19ed4b4ad5bf79966a13352716299ad532cfb115f4205c" dependencies = [ "proc-macro-crate", - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "syn 1.0.109", ] @@ -5602,23 +5735,26 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928" dependencies = [ - "base16ct", - "der", + "base16ct 0.1.1", + "der 0.6.1", "generic-array 0.14.6", - "pkcs8", + "pkcs8 0.9.0", "subtle", "zeroize", ] [[package]] -name = "secp256k1" -version = "0.24.3" +name = "sec1" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b1629c9c557ef9b293568b338dddfc8208c98a18c59d722a9d53f859d9c9b62" +checksum = "48518a2b5775ba8ca5b46596aae011caa431e6ce7e4a67ead66d92f08884220e" dependencies = [ - "rand 0.8.5", - "secp256k1-sys 0.6.1", - "serde", + "base16ct 0.2.0", + "der 0.7.1", + "generic-array 0.14.6", + "pkcs8 0.10.1", + "subtle", + "zeroize", ] [[package]] @@ -5628,16 +5764,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4124a35fe33ae14259c490fd70fa199a32b9ce9502f2ee6bc4f81ec06fa65894" dependencies = [ "rand 0.8.5", - "secp256k1-sys 0.8.0", -] - -[[package]] -name = "secp256k1-sys" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83080e2c2fc1006e625be82e5d1eb6a43b7fd9578b617fcc55814daf286bba4b" -dependencies = [ - "cc", + "secp256k1-sys", + "serde", ] [[package]] @@ -5683,9 +5811,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.16" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58bc9567378fc7690d6b2addae4e60ac2eeea07becb2c64b9f218b53865cba2a" +checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" dependencies = [ "serde", ] @@ -5736,7 +5864,7 @@ version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" dependencies = [ - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "syn 1.0.109", ] @@ -5787,7 +5915,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1966009f3c05f095697c537312f5415d1e3ed31ce0a56942bac4c771c5c335e" dependencies = [ "darling 0.14.3", - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "syn 1.0.109", ] @@ -5812,7 +5940,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b64f9e531ce97c88b4778aad0ceee079216071cffec6ac9b904277f8f92e7fe3" dependencies = [ - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "syn 1.0.109", ] @@ -5962,6 +6090,16 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "signature" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fe458c98333f9c8152221191a77e2a44e8325d0193484af2e9421a53019e57d" +dependencies = [ + "digest 0.10.6", + "rand_core 0.6.4", +] + [[package]] name = "simple_asn1" version = "0.6.2" @@ -6058,7 +6196,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b" dependencies = [ "base64ct", - "der", + "der 0.6.1", +] + +[[package]] +name = "spki" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0445c905640145c7ea8c1993555957f65e7c46d0535b91ba501bc9bfc85522f" +dependencies = [ + "base64ct", + "der 0.7.1", ] [[package]] @@ -6107,7 +6255,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" dependencies = [ "heck", - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "rustversion", "syn 1.0.109", @@ -6191,7 +6339,7 @@ version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "unicode-ident", ] @@ -6202,7 +6350,7 @@ version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" dependencies = [ - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "syn 1.0.109", "unicode-xid 0.2.4", @@ -6261,7 +6409,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9186daca5c58cb307d09731e0ba06b13fd6c036c90672b9bfc31cecf76cf689" dependencies = [ "cargo_metadata", - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "serde", "strum_macros", @@ -6276,7 +6424,7 @@ dependencies = [ "darling 0.14.3", "if_chain", "lazy_static", - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "subprocess", "syn 1.0.109", @@ -6307,20 +6455,20 @@ checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" [[package]] name = "thiserror" -version = "1.0.38" +version = "1.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0" +checksum = "a5ab016db510546d856297882807df8da66a16fb8c4101cb8b30054b0d5b2d9c" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.38" +version = "1.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f" +checksum = "5420d42e90af0c38c3290abcca25b9b3bdf379fc9f55c528f53a269d9c9a267e" dependencies = [ - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "syn 1.0.109", ] @@ -6422,7 +6570,7 @@ version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" dependencies = [ - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "syn 1.0.109", ] @@ -6637,7 +6785,7 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" dependencies = [ - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "syn 1.0.109", ] @@ -6986,7 +7134,7 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2e7e85a0596447f0f2ac090e16bc4c516c6fe91771fb0c0ccf7fa3dae896b9c" dependencies = [ - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "syn 1.0.109", ] @@ -7106,7 +7254,7 @@ dependencies = [ "bumpalo", "log", "once_cell", - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "syn 1.0.109", "wasm-bindgen-shared", @@ -7140,7 +7288,7 @@ version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" dependencies = [ - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "syn 1.0.109", "wasm-bindgen-backend", @@ -7397,7 +7545,7 @@ version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44bf07cb3e50ea2003396695d58bf46bc9887a1f362260446fad6bc4e79bd36c" dependencies = [ - "proc-macro2 1.0.51", + "proc-macro2 1.0.52", "quote 1.0.23", "syn 1.0.109", "synstructure", diff --git a/bin/reth/Cargo.toml b/bin/reth/Cargo.toml index 11328d60c0d..0e220277567 100644 --- a/bin/reth/Cargo.toml +++ b/bin/reth/Cargo.toml @@ -59,4 +59,4 @@ comfy-table = "6.1.4" crossterm = "0.25.0" tui = "0.19.0" jsonrpsee = { version = "0.16", features = ["server"] } -human_bytes = "0.4.1" \ No newline at end of file +human_bytes = "0.4.1" diff --git a/crates/interfaces/Cargo.toml b/crates/interfaces/Cargo.toml index a1b2dfcf286..d8a81d11542 100644 --- a/crates/interfaces/Cargo.toml +++ b/crates/interfaces/Cargo.toml @@ -26,7 +26,7 @@ futures = "0.3" tokio-stream = "0.1.11" rand = "0.8.5" arbitrary = { version = "1.1.7", features = ["derive"], optional = true } -secp256k1 = { version = "0.24.2", default-features = false, features = [ +secp256k1 = { version = "0.26.0", default-features = false, features = [ "alloc", "recovery", "rand", @@ -39,7 +39,7 @@ tokio = { version = "1.21.2", features = ["full"] } tokio-stream = { version = "0.1.11", features = ["sync"] } arbitrary = { version = "1.1.7", features = ["derive"] } hex-literal = "0.3" -secp256k1 = { version = "0.24.2", default-features = false, features = [ +secp256k1 = { version = "0.26.0", default-features = false, features = [ "alloc", "recovery", "rand", diff --git a/crates/net/discv4/Cargo.toml b/crates/net/discv4/Cargo.toml index a2b9f30c8a4..fdc0421cd01 100644 --- a/crates/net/discv4/Cargo.toml +++ b/crates/net/discv4/Cargo.toml @@ -19,12 +19,13 @@ reth-net-nat = { path = "../nat" } # ethereum discv5 = { git = "https://github.com/sigp/discv5" } -secp256k1 = { version = "0.24", features = [ +secp256k1 = { version = "0.26.0", features = [ "global-context", "rand-std", "recovery", + "serde" ] } -enr = { version = "0.7.0", default-features = false, features = [ +enr = { version = "0.8.0", default-features = false, features = [ "rust-secp256k1", ] } diff --git a/crates/net/dns/Cargo.toml b/crates/net/dns/Cargo.toml index f8f673fd7f6..29c4a1c5620 100644 --- a/crates/net/dns/Cargo.toml +++ b/crates/net/dns/Cargo.toml @@ -14,12 +14,13 @@ reth-net-common = { path = "../common" } reth-rlp = { path = "../../rlp" } # ethereum -secp256k1 = { version = "0.24", features = [ +secp256k1 = { version = "0.26.0", features = [ "global-context", "rand-std", "recovery", + "serde" ] } -enr = { version = "0.7.0", default-features = false, features = ["rust-secp256k1"] } +enr = { version = "0.8.0", default-features = false, features = ["rust-secp256k1"] } # async/futures tokio = { version = "1", features = ["io-util", "net", "time"] } diff --git a/crates/net/dns/src/lib.rs b/crates/net/dns/src/lib.rs index 82cea25df99..d32dd00691e 100644 --- a/crates/net/dns/src/lib.rs +++ b/crates/net/dns/src/lib.rs @@ -1,4 +1,4 @@ -#![warn(missing_docs, unreachable_pub)] +#![warn(missing_docs, unreachable_pub, unused_crate_dependencies)] #![deny(unused_must_use, rust_2018_idioms)] #![doc(test( no_crate_inject, diff --git a/crates/net/dns/src/tree.rs b/crates/net/dns/src/tree.rs index 6f42af14679..d9e0408995f 100644 --- a/crates/net/dns/src/tree.rs +++ b/crates/net/dns/src/tree.rs @@ -27,7 +27,11 @@ use data_encoding::{BASE32_NOPAD, BASE64URL_NOPAD}; use enr::{Enr, EnrError, EnrKey, EnrKeyUnambiguous, EnrPublicKey}; use reth_primitives::{bytes::Bytes, hex}; use secp256k1::SecretKey; -use std::{fmt, str::FromStr}; +use std::{ + fmt, + hash::{Hash, Hasher}, + str::FromStr, +}; #[cfg(feature = "serde")] use serde_with::{DeserializeFromStr, SerializeDisplay}; @@ -222,7 +226,7 @@ impl fmt::Display for BranchEntry { } /// A link entry -#[derive(Debug, Clone, Hash, Eq, PartialEq)] +#[derive(Debug, Clone)] #[cfg_attr(feature = "serde", derive(SerializeDisplay, DeserializeFromStr))] pub struct LinkEntry { pub domain: String, @@ -248,6 +252,33 @@ impl LinkEntry { } } +impl PartialEq for LinkEntry +where + K: EnrKeyUnambiguous, + K::PublicKey: PartialEq, +{ + fn eq(&self, other: &Self) -> bool { + self.domain == other.domain && self.pubkey == other.pubkey + } +} + +impl Eq for LinkEntry +where + K: EnrKeyUnambiguous, + K::PublicKey: Eq + PartialEq, +{ +} +impl Hash for LinkEntry +where + K: EnrKeyUnambiguous, + K::PublicKey: Hash, +{ + fn hash(&self, state: &mut H) { + self.domain.hash(state); + self.pubkey.hash(state); + } +} + impl FromStr for LinkEntry { type Err = ParseDnsEntryError; diff --git a/crates/net/ecies/Cargo.toml b/crates/net/ecies/Cargo.toml index 9c61f8eef36..f5c76d68dda 100644 --- a/crates/net/ecies/Cargo.toml +++ b/crates/net/ecies/Cargo.toml @@ -30,7 +30,7 @@ byteorder = "1.4.3" rand = "0.8.5" ctr = "0.9.2" digest = "0.10.5" -secp256k1 = { version = "0.24.2", features = ["global-context", "rand-std", "recovery"] } +secp256k1 = { version = "0.26.0", features = ["global-context", "rand-std", "recovery"] } sha2 = "0.10.6" sha3 = "0.10.5" aes = "0.8.1" diff --git a/crates/net/eth-wire/Cargo.toml b/crates/net/eth-wire/Cargo.toml index b4cc8118d4f..831298a449b 100644 --- a/crates/net/eth-wire/Cargo.toml +++ b/crates/net/eth-wire/Cargo.toml @@ -47,7 +47,7 @@ tokio-util = { version = "0.7.4", features = ["io", "codec"] } hex-literal = "0.3" hex = "0.4" rand = "0.8" -secp256k1 = { version = "0.24.2", features = ["global-context", "rand-std", "recovery"] } +secp256k1 = { version = "0.26.0", features = ["global-context", "rand-std", "recovery"] } arbitrary = { version = "1.1.7", features = ["derive"] } proptest = { version = "1.0" } diff --git a/crates/net/network/Cargo.toml b/crates/net/network/Cargo.toml index 9c27701b12b..fa41578658d 100644 --- a/crates/net/network/Cargo.toml +++ b/crates/net/network/Cargo.toml @@ -59,13 +59,13 @@ async-trait = "0.1" linked_hash_set = "0.1" linked-hash-map = "0.5.6" rand = "0.8" -secp256k1 = { version = "0.24", features = [ +secp256k1 = { version = "0.26.0", features = [ "global-context", "rand-std", "recovery", ] } -enr = { version = "0.7.0", features = ["rust-secp256k1"], optional = true } +enr = { version = "0.8.0", features = ["rust-secp256k1"], optional = true } ethers-core = { git = "https://github.com/gakonst/ethers-rs", default-features = false, optional = true } tempfile = { version = "3.3", optional = true } @@ -87,7 +87,7 @@ ethers-providers = { git = "https://github.com/gakonst/ethers-rs", default-featu ethers-signers = { git = "https://github.com/gakonst/ethers-rs", default-features = false } ethers-middleware = { git = "https://github.com/gakonst/ethers-rs", default-features = false } -enr = { version = "0.7.0", features = ["serde", "rust-secp256k1"] } +enr = { version = "0.8.0", features = ["serde", "rust-secp256k1"] } # misc hex = "0.4" diff --git a/crates/primitives/Cargo.toml b/crates/primitives/Cargo.toml index 9172b064660..99f63d4ceae 100644 --- a/crates/primitives/Cargo.toml +++ b/crates/primitives/Cargo.toml @@ -26,7 +26,7 @@ fixed-hash = { version = "0.8", default-features = false, features = [ ] } # crypto -secp256k1 = { version = "0.24.2", default-features = false, features = [ +secp256k1 = { version = "0.26.0", default-features = false, features = [ "global-context", "alloc", "recovery", @@ -74,7 +74,7 @@ proptest-derive = "0.3" # necessary so we don't hit a "undeclared 'std'": # https://github.com/paradigmxyz/reth/pull/177#discussion_r1021172198 -secp256k1 = "0.24.2" +secp256k1 = "0.26.0" criterion = "0.4.0" pprof = { version = "0.11", features = [ "flamegraph", @@ -94,4 +94,4 @@ test-utils = [] [[bench]] name = "recover_ecdsa_crit" -harness = false \ No newline at end of file +harness = false diff --git a/crates/revm/revm-primitives/Cargo.toml b/crates/revm/revm-primitives/Cargo.toml index 57ccc160e70..eab09ed7606 100644 --- a/crates/revm/revm-primitives/Cargo.toml +++ b/crates/revm/revm-primitives/Cargo.toml @@ -10,4 +10,4 @@ description = "core reth specific revm utilities" # reth reth-primitives = { path = "../../primitives" } -revm = { version = "3.0.0" } \ No newline at end of file +revm = { version = "3.0.0" } diff --git a/crates/rlp/Cargo.toml b/crates/rlp/Cargo.toml index c3554660590..cf6ebe9d5b8 100644 --- a/crates/rlp/Cargo.toml +++ b/crates/rlp/Cargo.toml @@ -12,7 +12,7 @@ auto_impl = "1" bytes = { version = "1", default-features = false } ethnum = { version = "1", default-features = false, optional = true } smol_str = { version = "0.1", default-features = false, optional = true } -enr = { version = "0.7", default-features = false, optional = true } +enr = { version = "0.8.0", default-features = false, optional = true } rlp = { version = "0.5.2", default-features = false, optional = true } ethereum-types = { version = "0.14", features = ["codec"], optional = true } revm-primitives = {version = "1.0.0", features = ["serde"] } @@ -30,7 +30,7 @@ reth-rlp = { path = ".", package = "reth-rlp", features = [ criterion = "0.4.0" hex-literal = "0.3" rand = "0.8" -secp256k1 = { version = "0.24", features = [ +secp256k1 = { version = "0.26.0", features = [ "rand-std", ] } pprof = { version = "0.11", features = ["flamegraph", "frame-pointer", "criterion"] } diff --git a/crates/staged-sync/Cargo.toml b/crates/staged-sync/Cargo.toml index 5521c66634a..5ab9d6c58aa 100644 --- a/crates/staged-sync/Cargo.toml +++ b/crates/staged-sync/Cargo.toml @@ -44,7 +44,7 @@ rand = { version = "0.8", optional = true } thiserror = "1" # enr -enr = { version = "0.7.0", features = ["serde", "rust-secp256k1"], optional = true } +enr = { version = "0.8.0", features = ["serde", "rust-secp256k1"], optional = true } # ethers ethers-core = { git = "https://github.com/gakonst/ethers-rs", default-features = false, optional = true } @@ -74,7 +74,7 @@ tokio = { version = "1", features = ["io-util", "net", "macros", "rt-multi-threa tokio-stream = "0.1" # crypto -secp256k1 = { version = "0.24", features = [ +secp256k1 = { version = "0.26.0", features = [ "global-context", "rand-std", "recovery", diff --git a/crates/storage/db/Cargo.toml b/crates/storage/db/Cargo.toml index 69f8dbd4538..e6c7647d484 100644 --- a/crates/storage/db/Cargo.toml +++ b/crates/storage/db/Cargo.toml @@ -22,7 +22,7 @@ parity-scale-codec = { version = "3.2.1", features = ["bytes"] } futures = "0.3.25" tokio-stream = "0.1.11" rand = "0.8.5" -secp256k1 = { version = "0.24.2", default-features = false, features = [ +secp256k1 = { version = "0.26.0", default-features = false, features = [ "alloc", "recovery", "rand", @@ -59,7 +59,7 @@ tokio = { version = "1.21.2", features = ["full"] } reth-db = { path = ".", features = ["test-utils", "bench"] } # needed for test-fuzz to work properly, see https://github.com/paradigmxyz/reth/pull/177#discussion_r1021172198 -secp256k1 = "0.24.2" +secp256k1 = "0.26.0" async-trait = "0.1.58" From 43e3621115b951b3d1e60876d22fa75c4c99899a Mon Sep 17 00:00:00 2001 From: GianfrancoBazzani <55500596+GianfrancoBazzani@users.noreply.github.com> Date: Wed, 15 Mar 2023 11:49:58 +0100 Subject: [PATCH 132/191] Improve revm halt out of gas error handling (#1725) Co-authored-by: Matthias Seitz --- crates/rpc/rpc/src/eth/api/call.rs | 7 +++++-- crates/rpc/rpc/src/eth/error.rs | 26 ++++++++++++++++++++++++-- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/crates/rpc/rpc/src/eth/api/call.rs b/crates/rpc/rpc/src/eth/api/call.rs index db76bfb8123..4ca5c0748a0 100644 --- a/crates/rpc/rpc/src/eth/api/call.rs +++ b/crates/rpc/rpc/src/eth/api/call.rs @@ -168,7 +168,9 @@ where } ExecutionResult::Halt { reason, .. } => { return match reason { - Halt::OutOfGas(_) => Err(InvalidTransactionError::OutOfGas(gas_limit).into()), + Halt::OutOfGas(err) => { + Err(InvalidTransactionError::out_of_gas(err, gas_limit).into()) + } Halt::NonceOverflow => Err(InvalidTransactionError::NonceMaxValue.into()), err => Err(InvalidTransactionError::EvmHalt(err).into()), } @@ -184,7 +186,8 @@ where ExecutionResult::Success { .. } => { // transaction succeeded by manually increasing the gas limit to // highest, which means the caller lacks funds to pay for the tx - Err(InvalidTransactionError::OutOfGas(U256::from(req_gas_limit)).into()) + Err(InvalidTransactionError::BasicOutOfGas(U256::from(req_gas_limit)) + .into()) } ExecutionResult::Revert { .. } => { // reverted again after bumping the limit diff --git a/crates/rpc/rpc/src/eth/error.rs b/crates/rpc/rpc/src/eth/error.rs index c95abea25be..0ea30bfe5f4 100644 --- a/crates/rpc/rpc/src/eth/error.rs +++ b/crates/rpc/rpc/src/eth/error.rs @@ -5,7 +5,7 @@ use jsonrpsee::{core::Error as RpcError, types::error::INVALID_PARAMS_CODE}; use reth_primitives::{constants::SELECTOR_LEN, Address, U128, U256}; use reth_rpc_types::{error::EthRpcErrorCode, BlockError}; use reth_transaction_pool::error::{InvalidPoolTransactionError, PoolError}; -use revm::primitives::{EVMError, Halt}; +use revm::primitives::{EVMError, Halt, OutOfGasError}; /// Result alias pub type EthResult = Result; @@ -158,7 +158,16 @@ pub enum InvalidTransactionError { SenderNoEOA, /// Thrown during estimate if caller has insufficient funds to cover the tx. #[error("Out of gas: gas required exceeds allowance: {0:?}")] - OutOfGas(U256), + BasicOutOfGas(U256), + /// As BasicOutOfGas but thrown when gas exhausts during memory expansion. + #[error("Out of gas: gas exhausts during memory expansion: {0:?}")] + MemoryOutOfGas(U256), + /// As BasicOutOfGas but thrown when gas exhausts during precompiled contract execution. + #[error("Out of gas: gas exhausts during precompiled contract execution: {0:?}")] + PrecompileOutOfGas(U256), + /// revm's Type cast error, U256 casts down to a u64 with overflow + #[error("Out of gas: revm's Type cast error, U256 casts down to a u64 with overflow {0:?}")] + InvalidOperandOutOfGas(U256), /// Thrown if executing a transaction failed during estimate/call #[error("{0}")] Revert(RevertError), @@ -181,6 +190,19 @@ impl InvalidTransactionError { _ => EthRpcErrorCode::TransactionRejected.code(), } } + + /// Converts the out of gas error + pub(crate) fn out_of_gas(reason: OutOfGasError, gas_limit: U256) -> Self { + match reason { + OutOfGasError::BasicOutOfGas => InvalidTransactionError::BasicOutOfGas(gas_limit), + OutOfGasError::Memory => InvalidTransactionError::MemoryOutOfGas(gas_limit), + OutOfGasError::Precompile => InvalidTransactionError::PrecompileOutOfGas(gas_limit), + OutOfGasError::InvalidOperand => { + InvalidTransactionError::InvalidOperandOutOfGas(gas_limit) + } + OutOfGasError::MemoryLimit => InvalidTransactionError::MemoryOutOfGas(gas_limit), + } + } } impl From for RpcError { From 83aacf218c783e461f5c0174e784ad530fe110a5 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 15 Mar 2023 12:56:03 +0100 Subject: [PATCH 133/191] feat(rpc): add with_state_at closure (#1763) --- crates/rpc/rpc/src/eth/api/mod.rs | 13 ++----------- crates/rpc/rpc/src/eth/api/transactions.rs | 14 +++++++++++++- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/crates/rpc/rpc/src/eth/api/mod.rs b/crates/rpc/rpc/src/eth/api/mod.rs index dac35c88db4..20eb3e9daf2 100644 --- a/crates/rpc/rpc/src/eth/api/mod.rs +++ b/crates/rpc/rpc/src/eth/api/mod.rs @@ -3,14 +3,13 @@ //! The entire implementation of the namespace is quite large, hence it is divided across several //! files. -use crate::eth::{cache::EthStateCache, error::EthResult, signer::EthSigner}; +use crate::eth::{cache::EthStateCache, signer::EthSigner}; use async_trait::async_trait; use reth_interfaces::Result; use reth_network_api::NetworkInfo; use reth_primitives::{Address, BlockId, BlockNumberOrTag, ChainInfo, H256, U64}; use reth_provider::{ - providers::ChainState, BlockProvider, EvmEnvProvider, StateProvider as StateProviderTrait, - StateProviderFactory, + BlockProvider, EvmEnvProvider, StateProvider as StateProviderTrait, StateProviderFactory, }; use reth_rpc_types::FeeHistoryCache; use reth_transaction_pool::TransactionPool; @@ -134,14 +133,6 @@ where self.client().convert_block_number(num) } - /// Helper function to execute a closure with the database at a specific block. - pub(crate) fn with_state_at(&self, _at: BlockId, _f: F) -> EthResult - where - F: FnOnce(ChainState<'_>) -> T, - { - unimplemented!() - } - /// Returns the state at the given [BlockId] enum or the latest. pub(crate) fn state_at_block_id_or_latest( &self, diff --git a/crates/rpc/rpc/src/eth/api/transactions.rs b/crates/rpc/rpc/src/eth/api/transactions.rs index 16d898e48ac..c603226483a 100644 --- a/crates/rpc/rpc/src/eth/api/transactions.rs +++ b/crates/rpc/rpc/src/eth/api/transactions.rs @@ -8,7 +8,7 @@ use reth_primitives::{ BlockId, BlockNumberOrTag, Bytes, FromRecoveredTransaction, IntoRecoveredTransaction, TransactionSigned, TransactionSignedEcRecovered, H256, U256, }; -use reth_provider::{BlockProvider, EvmEnvProvider, StateProviderFactory}; +use reth_provider::{providers::ChainState, BlockProvider, EvmEnvProvider, StateProviderFactory}; use reth_rlp::Decodable; use reth_rpc_types::{Index, Transaction, TransactionRequest}; use reth_transaction_pool::{TransactionOrigin, TransactionPool}; @@ -17,6 +17,11 @@ use revm::primitives::{BlockEnv, CfgEnv}; /// Commonly used transaction related functions for the [EthApi] type in the `eth_` namespace #[async_trait::async_trait] pub trait EthTransactions: Send + Sync { + /// Executes the closure with the state that corresponds to the given [BlockId]. + fn with_state_at(&self, _at: BlockId, _f: F) -> EthResult + where + F: FnOnce(ChainState<'_>) -> EthResult; + /// Returns the revm evm env for the requested [BlockId] /// /// If the [BlockId] this will return the [BlockId::Hash] of the block the env was configured @@ -44,6 +49,13 @@ where Client: BlockProvider + StateProviderFactory + EvmEnvProvider + 'static, Network: Send + Sync + 'static, { + fn with_state_at(&self, _at: BlockId, _f: F) -> EthResult + where + F: FnOnce(ChainState<'_>) -> EthResult, + { + unimplemented!() + } + async fn evm_env_at(&self, at: BlockId) -> EthResult<(CfgEnv, BlockEnv, BlockId)> { // TODO handle Pending state's env match at { From ec7b9554f3c4f8e7d4ffb780f8ea3e7d29321c51 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 15 Mar 2023 12:56:46 +0100 Subject: [PATCH 134/191] fix: fix clippy and broken code (#1761) --- crates/consensus/src/validation.rs | 2 +- crates/stages/benches/criterion.rs | 5 +++-- crates/stages/benches/setup/mod.rs | 5 +++-- crates/storage/db/benches/hash_keys.rs | 1 + crates/storage/provider/src/transaction.rs | 7 ++----- crates/storage/provider/src/trie/mod.rs | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/crates/consensus/src/validation.rs b/crates/consensus/src/validation.rs index 60364f3d8ad..af283908597 100644 --- a/crates/consensus/src/validation.rs +++ b/crates/consensus/src/validation.rs @@ -739,7 +739,7 @@ mod tests { assert!(res.is_ok()); // Withdrawal index should be the last withdrawal index + 1 - let mut provider = Provider::new(Some(parent.clone())); + let mut provider = Provider::new(Some(parent)); let block = create_block_with_withdrawals(&[4, 5, 6]); provider .withdrawals_provider diff --git a/crates/stages/benches/criterion.rs b/crates/stages/benches/criterion.rs index 0a34cbb4360..0385099fa1c 100644 --- a/crates/stages/benches/criterion.rs +++ b/crates/stages/benches/criterion.rs @@ -4,12 +4,13 @@ use criterion::{ }; use pprof::criterion::{Output, PProfProfiler}; use reth_db::mdbx::{Env, WriteMap}; +use reth_interfaces::test_utils::TestConsensus; use reth_stages::{ stages::{MerkleStage, SenderRecoveryStage, TotalDifficultyStage, TransactionLookupStage}, test_utils::TestTransaction, ExecInput, Stage, StageId, UnwindInput, }; -use std::path::PathBuf; +use std::{path::PathBuf, sync::Arc}; mod setup; use setup::StageRange; @@ -76,7 +77,7 @@ fn total_difficulty(c: &mut Criterion) { group.warm_up_time(std::time::Duration::from_millis(2000)); // don't need to run each stage for that many times group.sample_size(10); - let stage = TotalDifficultyStage::default(); + let stage = TotalDifficultyStage::new(Arc::new(TestConsensus::default())); measure_stage( &mut group, diff --git a/crates/stages/benches/setup/mod.rs b/crates/stages/benches/setup/mod.rs index 08b30de0905..1d3fb143b64 100644 --- a/crates/stages/benches/setup/mod.rs +++ b/crates/stages/benches/setup/mod.rs @@ -18,6 +18,7 @@ use reth_stages::{ }; use std::{ collections::BTreeMap, + ops::Deref, path::{Path, PathBuf}, }; @@ -124,7 +125,7 @@ pub(crate) fn txs_testdata(num_blocks: u64) -> PathBuf { // make first block after genesis have valid state root let root = - DBTrieLoader::default().calculate_root(&tx.inner()).and_then(|e| e.root()).unwrap(); + DBTrieLoader::new(tx.inner().deref()).calculate_root().and_then(|e| e.root()).unwrap(); let second_block = blocks.get_mut(1).unwrap(); let cloned_second = second_block.clone(); let mut updated_header = cloned_second.header.unseal(); @@ -146,7 +147,7 @@ pub(crate) fn txs_testdata(num_blocks: u64) -> PathBuf { let root = { let mut tx_mut = tx.inner(); let root = - DBTrieLoader::default().calculate_root(&tx_mut).and_then(|e| e.root()).unwrap(); + DBTrieLoader::new(tx_mut.deref()).calculate_root().and_then(|e| e.root()).unwrap(); tx_mut.commit().unwrap(); root }; diff --git a/crates/storage/db/benches/hash_keys.rs b/crates/storage/db/benches/hash_keys.rs index b372dca41fd..078ae5bee46 100644 --- a/crates/storage/db/benches/hash_keys.rs +++ b/crates/storage/db/benches/hash_keys.rs @@ -134,6 +134,7 @@ where /// Generates two batches. The first is to be inserted into the database before running the /// benchmark. The second is to be benchmarked with. +#[allow(clippy::type_complexity)] fn generate_batches(size: usize) -> (Vec<(T::Key, T::Value)>, Vec<(T::Key, T::Value)>) where T: Table + Default, diff --git a/crates/storage/provider/src/transaction.rs b/crates/storage/provider/src/transaction.rs index f297d717fc4..8a842be8e1a 100644 --- a/crates/storage/provider/src/transaction.rs +++ b/crates/storage/provider/src/transaction.rs @@ -1605,7 +1605,7 @@ mod test { insert_canonical_block(tx.deref_mut(), data.genesis.clone(), None, false).unwrap(); tx.put::(EMPTY_ROOT, vec![0x80]).unwrap(); - assert_genesis_block(&tx, data.genesis.clone()); + assert_genesis_block(&tx, data.genesis); tx.insert_block(block1.clone(), &chain_spec, exec_res1.clone()).unwrap(); @@ -1634,10 +1634,7 @@ mod test { // take two blocks let get = tx.take_block_and_execution_range(&chain_spec, 1..=2).unwrap(); - assert_eq!( - get, - vec![(block1.clone(), exec_res1.clone()), (block2.clone(), exec_res2.clone())] - ); + assert_eq!(get, vec![(block1, exec_res1), (block2, exec_res2)]); // assert genesis state assert_genesis_block(&tx, genesis); diff --git a/crates/storage/provider/src/trie/mod.rs b/crates/storage/provider/src/trie/mod.rs index d218eb607dd..8965d6da7d2 100644 --- a/crates/storage/provider/src/trie/mod.rs +++ b/crates/storage/provider/src/trie/mod.rs @@ -1173,7 +1173,7 @@ mod tests { }; tx.put::(hashed_address, account).unwrap(); - for (k, v) in storage.clone() { + for (k, v) in storage { tx.put::( hashed_address, StorageEntry { key: keccak256(k), value: v }, From a00df7ffbb205298e39597eee60b7bb175594035 Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Wed, 15 Mar 2023 14:56:48 +0200 Subject: [PATCH 135/191] feat(cli): separate net dir (#1764) --- bin/reth/src/dirs.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/bin/reth/src/dirs.rs b/bin/reth/src/dirs.rs index c5ac09bbc0d..72e83e587b2 100644 --- a/bin/reth/src/dirs.rs +++ b/bin/reth/src/dirs.rs @@ -44,11 +44,18 @@ pub fn logs_dir() -> Option { /// Returns the path to the reth jwtsecret directory. /// -/// Refer to [dirs_next::cache_dir] for cross-platform behavior. +/// Refer to [dirs_next::data_dir] for cross-platform behavior. pub fn jwt_secret_dir() -> Option { data_dir().map(|root| root.join("jwtsecret")) } +/// Returns the path to the reth net directory. +/// +/// Refer to [dirs_next::data_dir] +pub fn net_dir() -> Option { + data_dir().map(|root| root.join("net")) +} + /// Returns the path to the reth database. /// /// Refer to [dirs_next::data_dir] for cross-platform behavior. @@ -90,14 +97,14 @@ impl XdgPath for ConfigPath { /// Returns the path to the default reth known peers file. /// -/// Refer to [dirs_next::config_dir] for cross-platform behavior. +/// Refer to [dirs_next::data_dir] for cross-platform behavior. #[derive(Default, Debug, Clone)] #[non_exhaustive] pub struct KnownPeersPath; impl XdgPath for KnownPeersPath { fn resolve() -> Option { - database_path().map(|p| p.join("known-peers.json")) + net_dir().map(|p| p.join("known-peers.json")) } } From bba61c0b617e4d2f9569b1524c583431792949ad Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 15 Mar 2023 14:06:53 +0100 Subject: [PATCH 136/191] refactor(rpc): use ChainState type (#1756) --- crates/primitives/src/block.rs | 10 +++++ crates/rpc/rpc/src/eth/api/call.rs | 7 ++-- crates/rpc/rpc/src/eth/api/mod.rs | 58 +++++------------------------ crates/rpc/rpc/src/eth/api/state.rs | 20 +++++----- 4 files changed, 34 insertions(+), 61 deletions(-) diff --git a/crates/primitives/src/block.rs b/crates/primitives/src/block.rs index e158f7c2ed9..ea84f713ae4 100644 --- a/crates/primitives/src/block.rs +++ b/crates/primitives/src/block.rs @@ -245,6 +245,16 @@ impl BlockId { BlockId::Number(_) => None, } } + + /// Returns true if this is [BlockNumberOrTag::Latest] + pub fn is_latest(&self) -> bool { + matches!(self, BlockId::Number(BlockNumberOrTag::Latest)) + } + + /// Returns true if this is [BlockNumberOrTag::Pending] + pub fn is_pending(&self) -> bool { + matches!(self, BlockId::Number(BlockNumberOrTag::Pending)) + } } impl From for BlockId { diff --git a/crates/rpc/rpc/src/eth/api/call.rs b/crates/rpc/rpc/src/eth/api/call.rs index 4ca5c0748a0..a4105ccabae 100644 --- a/crates/rpc/rpc/src/eth/api/call.rs +++ b/crates/rpc/rpc/src/eth/api/call.rs @@ -27,7 +27,6 @@ use revm::{ }, Database, }; -use std::ops::Deref; // Gas per transaction not creating a contract. const MIN_TRANSACTION_GAS: u64 = 21_000u64; @@ -48,7 +47,7 @@ where ) -> EthResult<(ResultAndState, Env)> { let (cfg, block_env, at) = self.evm_env_at(at).await?; let state = self.state_at_block_id(at)?.ok_or_else(|| EthApiError::UnknownBlockNumber)?; - self.call_with(cfg, block_env, request, &*state, state_overrides) + self.call_with(cfg, block_env, request, state, state_overrides) } /// Executes the call request using the given environment against the state provider @@ -88,7 +87,7 @@ where ) -> EthResult { let (cfg, block_env, at) = self.evm_env_at(at).await?; let state = self.state_at_block_id(at)?.ok_or_else(|| EthApiError::UnknownBlockNumber)?; - self.estimate_gas_with(cfg, block_env, request, &*state) + self.estimate_gas_with(cfg, block_env, request, state) } /// Estimates the gas usage of the `request` with the state. @@ -280,7 +279,7 @@ where cfg.disable_block_gas_limit = true; let env = build_call_evm_env(cfg, block, request.clone())?; - let mut db = SubState::new(State::new(state.deref())); + let mut db = SubState::new(State::new(state)); let from = request.from.unwrap_or_default(); let to = if let Some(to) = request.to { diff --git a/crates/rpc/rpc/src/eth/api/mod.rs b/crates/rpc/rpc/src/eth/api/mod.rs index 20eb3e9daf2..19453314996 100644 --- a/crates/rpc/rpc/src/eth/api/mod.rs +++ b/crates/rpc/rpc/src/eth/api/mod.rs @@ -8,12 +8,10 @@ use async_trait::async_trait; use reth_interfaces::Result; use reth_network_api::NetworkInfo; use reth_primitives::{Address, BlockId, BlockNumberOrTag, ChainInfo, H256, U64}; -use reth_provider::{ - BlockProvider, EvmEnvProvider, StateProvider as StateProviderTrait, StateProviderFactory, -}; +use reth_provider::{providers::ChainState, BlockProvider, EvmEnvProvider, StateProviderFactory}; use reth_rpc_types::FeeHistoryCache; use reth_transaction_pool::TransactionPool; -use std::{num::NonZeroUsize, ops::Deref, sync::Arc}; +use std::{num::NonZeroUsize, sync::Arc}; mod block; mod call; @@ -92,37 +90,6 @@ impl EthApi { } } -// Transparent wrapper to enable state access helpers -// returning latest state provider when appropiate -pub(crate) enum StateProvider<'a, H, L> { - History(H), - Latest(L), - _Unreachable(&'a ()), // like a PhantomData for 'a -} - -type HistoryOrLatest<'a, Client> = StateProvider< - 'a, - ::HistorySP<'a>, - ::LatestSP<'a>, ->; - -impl<'a, H, L> Deref for StateProvider<'a, H, L> -where - Self: 'a, - H: StateProviderTrait + 'a, - L: StateProviderTrait + 'a, -{ - type Target = dyn StateProviderTrait + 'a; - - fn deref(&self) -> &Self::Target { - match self { - StateProvider::History(h) => h, - StateProvider::Latest(l) => l, - StateProvider::_Unreachable(()) => unreachable!(), - } - } -} - // === State access helpers === impl EthApi @@ -137,23 +104,18 @@ where pub(crate) fn state_at_block_id_or_latest( &self, block_id: Option, - ) -> Result>> { + ) -> Result>> { if let Some(block_id) = block_id { self.state_at_block_id(block_id) } else { - self.latest_state().map(|v| Some(StateProvider::Latest(v))) + self.latest_state().map(ChainState::boxed).map(Some) } } /// Returns the state at the given [BlockId] enum. - pub(crate) fn state_at_block_id( - &self, - block_id: BlockId, - ) -> Result>> { + pub(crate) fn state_at_block_id(&self, block_id: BlockId) -> Result>> { match block_id { - BlockId::Hash(hash) => { - self.state_at_hash(hash.into()).map(|s| Some(StateProvider::History(s))) - } + BlockId::Hash(hash) => self.state_at_hash(hash.into()).map(ChainState::boxed).map(Some), BlockId::Number(num) => self.state_at_block_number(num), } } @@ -164,7 +126,7 @@ where pub(crate) fn state_at_block_number( &self, num: BlockNumberOrTag, - ) -> Result>> { + ) -> Result>> { if let Some(number) = self.convert_block_number(num)? { self.state_at_number(number).map(Some) } else { @@ -181,10 +143,10 @@ where } /// Returns the state at the given block number - pub(crate) fn state_at_number(&self, block_number: u64) -> Result> { + pub(crate) fn state_at_number(&self, block_number: u64) -> Result> { match self.convert_block_number(BlockNumberOrTag::Latest)? { - Some(num) if num == block_number => self.latest_state().map(StateProvider::Latest), - _ => self.client().history_by_block_number(block_number).map(StateProvider::History), + Some(num) if num == block_number => self.latest_state().map(ChainState::boxed), + _ => self.client().history_by_block_number(block_number).map(ChainState::boxed), } } diff --git a/crates/rpc/rpc/src/eth/api/state.rs b/crates/rpc/rpc/src/eth/api/state.rs index 42638bcdb9d..fe79fa795af 100644 --- a/crates/rpc/rpc/src/eth/api/state.rs +++ b/crates/rpc/rpc/src/eth/api/state.rs @@ -1,14 +1,13 @@ //! Contains RPC handler implementations specific to state. use crate::{ - eth::{ - api::StateProvider, - error::{EthApiError, EthResult}, - }, + eth::error::{EthApiError, EthResult}, EthApi, }; -use reth_primitives::{Address, BlockId, Bytes, H256, KECCAK_EMPTY, U256}; -use reth_provider::{BlockProvider, EvmEnvProvider, StateProviderFactory}; +use reth_primitives::{Address, BlockId, BlockNumberOrTag, Bytes, H256, KECCAK_EMPTY, U256}; +use reth_provider::{ + AccountProvider, BlockProvider, EvmEnvProvider, StateProvider, StateProviderFactory, +}; use reth_rpc_types::{EIP1186AccountProofResponse, StorageProof}; impl EthApi @@ -59,14 +58,17 @@ where keys: Vec, block_id: Option, ) -> EthResult { - let state = - self.state_at_block_id_or_latest(block_id)?.ok_or(EthApiError::UnknownBlockNumber)?; + let block_id = block_id.unwrap_or(BlockId::Number(BlockNumberOrTag::Latest)); // TODO: remove when HistoricalStateProviderRef::proof is implemented - if matches!(state, StateProvider::History(_)) { + if !block_id.is_latest() { return Err(EthApiError::InvalidBlockRange) } + let state = self + .state_at_block_id_or_latest(Some(block_id))? + .ok_or(EthApiError::UnknownBlockNumber)?; + let (account_proof, storage_hash, stg_proofs) = state.proof(address, &keys)?; let storage_proof = keys From 0c434e7916d37acf4cb57d26c5dfd89da138d4d8 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 15 Mar 2023 15:03:42 +0100 Subject: [PATCH 137/191] feat(rpc): impl parity trace_transaction (#1765) --- .../revm/revm-inspectors/src/tracing/arena.rs | 115 ------------------ .../src/tracing/builder/geth.rs | 104 ++++++++++++++++ .../src/tracing/builder/mod.rs | 4 + .../src/tracing/builder/parity.rs | 104 ++++++++++++++++ .../revm/revm-inspectors/src/tracing/mod.rs | 13 +- .../revm/revm-inspectors/src/tracing/types.rs | 2 +- crates/revm/revm-primitives/src/env.rs | 23 +++- crates/rpc/rpc-types/src/eth/trace/parity.rs | 26 +++- .../rpc-types/src/eth/transaction/common.rs | 17 +++ .../rpc/rpc-types/src/eth/transaction/mod.rs | 2 + crates/rpc/rpc/src/eth/api/transactions.rs | 41 ++++++- crates/rpc/rpc/src/trace.rs | 28 ++++- 12 files changed, 349 insertions(+), 130 deletions(-) create mode 100644 crates/revm/revm-inspectors/src/tracing/builder/geth.rs create mode 100644 crates/revm/revm-inspectors/src/tracing/builder/mod.rs create mode 100644 crates/revm/revm-inspectors/src/tracing/builder/parity.rs create mode 100644 crates/rpc/rpc-types/src/eth/transaction/common.rs diff --git a/crates/revm/revm-inspectors/src/tracing/arena.rs b/crates/revm/revm-inspectors/src/tracing/arena.rs index 08bfa65bf9b..e8e14075d44 100644 --- a/crates/revm/revm-inspectors/src/tracing/arena.rs +++ b/crates/revm/revm-inspectors/src/tracing/arena.rs @@ -1,11 +1,4 @@ use crate::tracing::types::{CallTrace, CallTraceNode, LogCallOrder}; -use reth_primitives::{Address, JsonU256, H256, U256}; -use reth_rpc_types::trace::{ - geth::{DefaultFrame, GethDebugTracingOptions, StructLog}, - parity::{ActionType, TransactionTrace}, -}; -use revm::interpreter::{opcode, InstructionResult}; -use std::collections::{BTreeMap, HashMap}; /// An arena of recorded traces. /// @@ -49,112 +42,4 @@ impl CallTraceArena { ), } } - - /// Returns the traces of the transaction for `trace_transaction` - pub fn parity_traces(&self) -> Vec { - let traces = Vec::with_capacity(self.arena.len()); - for (_idx, node) in self.arena.iter().cloned().enumerate() { - let _action = node.parity_action(); - let _result = node.parity_result(); - - let _action_type = if node.status() == InstructionResult::SelfDestruct { - ActionType::Selfdestruct - } else { - node.kind().into() - }; - - todo!() - - // let trace = TransactionTrace { - // action, - // result: Some(result), - // trace_address: self.info.trace_address(idx), - // subtraces: node.children.len(), - // }; - // traces.push(trace) - } - - traces - } - - /// Recursively fill in the geth trace by going through the traces - /// - /// TODO rewrite this iteratively - fn add_to_geth_trace( - &self, - storage: &mut HashMap>, - trace_node: &CallTraceNode, - struct_logs: &mut Vec, - opts: &GethDebugTracingOptions, - ) { - let mut child_id = 0; - // Iterate over the steps inside the given trace - for step in trace_node.trace.steps.iter() { - let mut log: StructLog = step.into(); - - // Fill in memory and storage depending on the options - if !opts.disable_storage.unwrap_or_default() { - let contract_storage = storage.entry(step.contract).or_default(); - if let Some((key, value)) = step.state_diff { - contract_storage.insert(key.into(), value.into()); - log.storage = Some(contract_storage.clone()); - } - } - if opts.disable_stack.unwrap_or_default() { - log.stack = None; - } - if !opts.enable_memory.unwrap_or_default() { - log.memory = None; - } - - // Add step to geth trace - struct_logs.push(log); - - // If the opcode is a call, the descend into child trace - match step.op.u8() { - opcode::CREATE | - opcode::CREATE2 | - opcode::DELEGATECALL | - opcode::CALL | - opcode::STATICCALL | - opcode::CALLCODE => { - self.add_to_geth_trace( - storage, - &self.arena[trace_node.children[child_id]], - struct_logs, - opts, - ); - child_id += 1; - } - _ => {} - } - } - } - - /// Generate a geth-style trace e.g. for `debug_traceTransaction` - pub fn geth_traces( - &self, - // TODO(mattsse): This should be the total gas used, or gas used by last CallTrace? - receipt_gas_used: U256, - opts: GethDebugTracingOptions, - ) -> DefaultFrame { - if self.arena.is_empty() { - return Default::default() - } - // Fetch top-level trace - let main_trace_node = &self.arena[0]; - let main_trace = &main_trace_node.trace; - - let mut struct_logs = Vec::new(); - let mut storage = HashMap::new(); - self.add_to_geth_trace(&mut storage, main_trace_node, &mut struct_logs, &opts); - - DefaultFrame { - // If the top-level trace succeeded, then it was a success - failed: !main_trace.success, - gas: JsonU256(receipt_gas_used), - return_value: main_trace.output.clone().into(), - struct_logs, - } - } } diff --git a/crates/revm/revm-inspectors/src/tracing/builder/geth.rs b/crates/revm/revm-inspectors/src/tracing/builder/geth.rs new file mode 100644 index 00000000000..305ac18bfa2 --- /dev/null +++ b/crates/revm/revm-inspectors/src/tracing/builder/geth.rs @@ -0,0 +1,104 @@ +//! Geth trace builder + +use crate::tracing::{types::CallTraceNode, TraceInspectorConfig}; +use reth_primitives::{Address, JsonU256, H256, U256}; +use reth_rpc_types::trace::geth::*; +use revm::interpreter::opcode; +use std::collections::{BTreeMap, HashMap}; + +/// A type for creating geth style traces +#[derive(Clone, Debug)] +pub struct GethTraceBuilder { + /// Recorded trace nodes. + nodes: Vec, + /// How the traces were recorded + _config: TraceInspectorConfig, +} + +impl GethTraceBuilder { + /// Returns a new instance of the builder + pub(crate) fn new(nodes: Vec, _config: TraceInspectorConfig) -> Self { + Self { nodes, _config } + } + + /// Recursively fill in the geth trace by going through the traces + /// + /// TODO rewrite this iteratively + fn add_to_geth_trace( + &self, + storage: &mut HashMap>, + trace_node: &CallTraceNode, + struct_logs: &mut Vec, + opts: &GethDebugTracingOptions, + ) { + let mut child_id = 0; + // Iterate over the steps inside the given trace + for step in trace_node.trace.steps.iter() { + let mut log: StructLog = step.into(); + + // Fill in memory and storage depending on the options + if !opts.disable_storage.unwrap_or_default() { + let contract_storage = storage.entry(step.contract).or_default(); + if let Some((key, value)) = step.state_diff { + contract_storage.insert(key.into(), value.into()); + log.storage = Some(contract_storage.clone()); + } + } + if opts.disable_stack.unwrap_or_default() { + log.stack = None; + } + if !opts.enable_memory.unwrap_or_default() { + log.memory = None; + } + + // Add step to geth trace + struct_logs.push(log); + + // If the opcode is a call, the descend into child trace + match step.op.u8() { + opcode::CREATE | + opcode::CREATE2 | + opcode::DELEGATECALL | + opcode::CALL | + opcode::STATICCALL | + opcode::CALLCODE => { + self.add_to_geth_trace( + storage, + &self.nodes[trace_node.children[child_id]], + struct_logs, + opts, + ); + child_id += 1; + } + _ => {} + } + } + } + + /// Generate a geth-style trace e.g. for `debug_traceTransaction` + pub fn geth_traces( + &self, + // TODO(mattsse): This should be the total gas used, or gas used by last CallTrace? + receipt_gas_used: U256, + opts: GethDebugTracingOptions, + ) -> DefaultFrame { + if self.nodes.is_empty() { + return Default::default() + } + // Fetch top-level trace + let main_trace_node = &self.nodes[0]; + let main_trace = &main_trace_node.trace; + + let mut struct_logs = Vec::new(); + let mut storage = HashMap::new(); + self.add_to_geth_trace(&mut storage, main_trace_node, &mut struct_logs, &opts); + + DefaultFrame { + // If the top-level trace succeeded, then it was a success + failed: !main_trace.success, + gas: JsonU256(receipt_gas_used), + return_value: main_trace.output.clone().into(), + struct_logs, + } + } +} diff --git a/crates/revm/revm-inspectors/src/tracing/builder/mod.rs b/crates/revm/revm-inspectors/src/tracing/builder/mod.rs new file mode 100644 index 00000000000..a39a31913ee --- /dev/null +++ b/crates/revm/revm-inspectors/src/tracing/builder/mod.rs @@ -0,0 +1,4 @@ +//! Builder types for building traces + +pub(crate) mod geth; +pub(crate) mod parity; diff --git a/crates/revm/revm-inspectors/src/tracing/builder/parity.rs b/crates/revm/revm-inspectors/src/tracing/builder/parity.rs new file mode 100644 index 00000000000..96940861402 --- /dev/null +++ b/crates/revm/revm-inspectors/src/tracing/builder/parity.rs @@ -0,0 +1,104 @@ +use crate::tracing::{types::CallTraceNode, TraceInspectorConfig}; +use reth_rpc_types::{trace::parity::*, TransactionInfo}; + +/// A type for creating parity style traces +#[derive(Clone, Debug)] +pub struct ParityTraceBuilder { + /// Recorded trace nodes + nodes: Vec, + /// How the traces were recorded + _config: TraceInspectorConfig, +} + +impl ParityTraceBuilder { + /// Returns a new instance of the builder + pub(crate) fn new(nodes: Vec, _config: TraceInspectorConfig) -> Self { + Self { nodes, _config } + } + + /// Returns the trace addresses of all transactions in the set + fn trace_addresses(&self) -> Vec> { + let mut all_addresses = Vec::with_capacity(self.nodes.len()); + for idx in 0..self.nodes.len() { + all_addresses.push(self.trace_address(idx)); + } + all_addresses + } + + /// Returns the `traceAddress` of the node in the arena + /// + /// The `traceAddress` field of all returned traces, gives the exact location in the call trace + /// [index in root, index in first CALL, index in second CALL, …]. + /// + /// # Panics + /// + /// if the `idx` does not belong to a node + fn trace_address(&self, idx: usize) -> Vec { + if idx == 0 { + // root call has empty traceAddress + return vec![] + } + let mut graph = vec![]; + let mut node = &self.nodes[idx]; + while let Some(parent) = node.parent { + // the index of the child call in the arena + let child_idx = node.idx; + node = &self.nodes[parent]; + // find the index of the child call in the parent node + let call_idx = node + .children + .iter() + .position(|child| *child == child_idx) + .expect("child exists in parent"); + graph.push(call_idx); + } + graph.reverse(); + graph + } + + /// Returns an iterator over all recorded traces for `trace_transaction` + pub fn into_localized_transaction_traces_iter( + self, + info: TransactionInfo, + ) -> impl Iterator { + self.into_transaction_traces_iter().map(move |trace| { + let TransactionInfo { hash, index, block_hash, block_number } = info; + LocalizedTransactionTrace { + trace, + transaction_position: index, + transaction_hash: hash, + block_number, + block_hash, + } + }) + } + + /// Returns an iterator over all recorded traces for `trace_transaction` + pub fn into_localized_transaction_traces( + self, + info: TransactionInfo, + ) -> Vec { + self.into_localized_transaction_traces_iter(info).collect() + } + + /// Returns an iterator over all recorded traces for `trace_transaction` + pub fn into_transaction_traces_iter(self) -> impl Iterator { + let trace_addresses = self.trace_addresses(); + + self.nodes.into_iter().zip(trace_addresses).map(|(node, trace_address)| { + let action = node.parity_action(); + let output = TraceResult::parity_success(node.parity_trace_output()); + TransactionTrace { + action, + result: Some(output), + trace_address, + subtraces: node.children.len(), + } + }) + } + + /// Returns the raw traces of the transaction + pub fn into_transaction_traces(self) -> Vec { + self.into_transaction_traces_iter().collect() + } +} diff --git a/crates/revm/revm-inspectors/src/tracing/mod.rs b/crates/revm/revm-inspectors/src/tracing/mod.rs index 958fc2c64c5..4dd8b53f4fa 100644 --- a/crates/revm/revm-inspectors/src/tracing/mod.rs +++ b/crates/revm/revm-inspectors/src/tracing/mod.rs @@ -18,9 +18,11 @@ use revm::{ use types::{CallTrace, CallTraceStep}; mod arena; +mod builder; mod config; mod types; mod utils; +pub use builder::{geth::GethTraceBuilder, parity::ParityTraceBuilder}; pub use config::TraceInspectorConfig; /// An inspector that collects call traces. @@ -60,9 +62,14 @@ impl TracingInspector { } } - /// Consumes the Inspector and returns the recorded. - pub fn finalize(self) -> CallTraceArena { - self.traces + /// Consumes the Inspector and returns a [ParityTraceBuilder]. + pub fn into_parity_builder(self) -> ParityTraceBuilder { + ParityTraceBuilder::new(self.traces.arena, self.config) + } + + /// Consumes the Inspector and returns a [GethTraceBuilder]. + pub fn into_geth_builder(self) -> GethTraceBuilder { + GethTraceBuilder::new(self.traces.arena, self.config) } /// Configures a [GasInspector] diff --git a/crates/revm/revm-inspectors/src/tracing/types.rs b/crates/revm/revm-inspectors/src/tracing/types.rs index 3c13d29135c..963eae53f30 100644 --- a/crates/revm/revm-inspectors/src/tracing/types.rs +++ b/crates/revm/revm-inspectors/src/tracing/types.rs @@ -158,7 +158,7 @@ impl CallTraceNode { } /// Returns the `Output` for a parity trace - pub(crate) fn parity_result(&self) -> TraceOutput { + pub(crate) fn parity_trace_output(&self) -> TraceOutput { match self.kind() { CallKind::Call | CallKind::StaticCall | CallKind::CallCode | CallKind::DelegateCall => { TraceOutput::Call(CallOutput { diff --git a/crates/revm/revm-primitives/src/env.rs b/crates/revm/revm-primitives/src/env.rs index a9ceac12dec..1f81ec4c780 100644 --- a/crates/revm/revm-primitives/src/env.rs +++ b/crates/revm/revm-primitives/src/env.rs @@ -1,7 +1,7 @@ use crate::config::revm_spec; use reth_primitives::{ - Address, ChainSpec, Head, Header, Transaction, TransactionKind, TransactionSigned, TxEip1559, - TxEip2930, TxLegacy, U256, + Address, ChainSpec, Head, Header, Transaction, TransactionKind, TransactionSignedEcRecovered, + TxEip1559, TxEip2930, TxLegacy, U256, }; use revm::primitives::{AnalysisKind, BlockEnv, CfgEnv, SpecId, TransactTo, TxEnv}; @@ -57,8 +57,23 @@ pub fn fill_block_env(block_env: &mut BlockEnv, header: &Header, after_merge: bo block_env.gas_limit = U256::from(header.gas_limit); } -/// Fill transaction environment from Transaction. -pub fn fill_tx_env(tx_env: &mut TxEnv, transaction: &TransactionSigned, sender: Address) { +/// Returns a new [TxEnv] filled with the transaction's data. +pub fn tx_env_with_recovered(transaction: &TransactionSignedEcRecovered) -> TxEnv { + let mut tx_env = TxEnv::default(); + fill_tx_env(&mut tx_env, transaction.as_ref(), transaction.signer()); + tx_env +} + +/// Fill transaction environment from [TransactionSignedEcRecovered]. +pub fn fill_tx_env_with_recovered(tx_env: &mut TxEnv, transaction: &TransactionSignedEcRecovered) { + fill_tx_env(tx_env, transaction.as_ref(), transaction.signer()) +} + +/// Fill transaction environment from a [Transaction] and the given sender address. +pub fn fill_tx_env(tx_env: &mut TxEnv, transaction: T, sender: Address) +where + T: AsRef, +{ tx_env.caller = sender; match transaction.as_ref() { Transaction::Legacy(TxLegacy { diff --git a/crates/rpc/rpc-types/src/eth/trace/parity.rs b/crates/rpc/rpc-types/src/eth/trace/parity.rs index 14df5808984..a6f865a1728 100644 --- a/crates/rpc/rpc-types/src/eth/trace/parity.rs +++ b/crates/rpc/rpc-types/src/eth/trace/parity.rs @@ -10,6 +10,20 @@ use std::collections::BTreeMap; /// Result type for parity style transaction trace pub type TraceResult = crate::trace::common::TraceResult; +// === impl TraceResult === + +impl TraceResult { + /// Wraps the result type in a [TraceResult::Success] variant + pub fn parity_success(result: TraceOutput) -> Self { + TraceResult::Success { result } + } + + /// Wraps the result type in a [TraceResult::Error] variant + pub fn parity_error(error: String) -> Self { + TraceResult::Error { error } + } +} + #[derive(Clone, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub enum TraceType { @@ -190,10 +204,18 @@ pub struct TransactionTrace { pub struct LocalizedTransactionTrace { #[serde(flatten)] pub trace: TransactionTrace, + /// Transaction index within the block, None if pending. pub transaction_position: Option, + /// Hash of the transaction pub transaction_hash: Option, - pub block_number: U64, - pub block_hash: H256, + /// Block number the transaction is included in, None if pending. + /// + /// Note: this deviates from which always returns a block number + pub block_number: Option, + /// Hash of the block, if not pending + /// + /// Note: this deviates from which always returns a block number + pub block_hash: Option, } #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] diff --git a/crates/rpc/rpc-types/src/eth/transaction/common.rs b/crates/rpc/rpc-types/src/eth/transaction/common.rs new file mode 100644 index 00000000000..fd02a66bc11 --- /dev/null +++ b/crates/rpc/rpc-types/src/eth/transaction/common.rs @@ -0,0 +1,17 @@ +//! Commonly used additional types that are not part of the JSON RPC spec but are often required +//! when working with RPC types, such as [Transaction](crate::Transaction) + +use reth_primitives::{TxHash, H256}; + +/// Additional fields in the context of a block that contains this transaction. +#[derive(Debug, Clone, Copy, Default, Eq, PartialEq)] +pub struct TransactionInfo { + /// Hash of the transaction. + pub hash: Option, + /// Index of the transaction in the block + pub index: Option, + /// Hash of the block. + pub block_hash: Option, + /// Number of the block. + pub block_number: Option, +} diff --git a/crates/rpc/rpc-types/src/eth/transaction/mod.rs b/crates/rpc/rpc-types/src/eth/transaction/mod.rs index a5a169bf9b1..8aa3e7be1d0 100644 --- a/crates/rpc/rpc-types/src/eth/transaction/mod.rs +++ b/crates/rpc/rpc-types/src/eth/transaction/mod.rs @@ -1,8 +1,10 @@ +mod common; mod receipt; mod request; mod signature; mod typed; +pub use common::TransactionInfo; pub use receipt::TransactionReceipt; pub use request::TransactionRequest; pub use signature::Signature; diff --git a/crates/rpc/rpc/src/eth/api/transactions.rs b/crates/rpc/rpc/src/eth/api/transactions.rs index c603226483a..ef92e75b636 100644 --- a/crates/rpc/rpc/src/eth/api/transactions.rs +++ b/crates/rpc/rpc/src/eth/api/transactions.rs @@ -10,7 +10,7 @@ use reth_primitives::{ }; use reth_provider::{providers::ChainState, BlockProvider, EvmEnvProvider, StateProviderFactory}; use reth_rlp::Decodable; -use reth_rpc_types::{Index, Transaction, TransactionRequest}; +use reth_rpc_types::{Index, Transaction, TransactionInfo, TransactionRequest}; use reth_transaction_pool::{TransactionOrigin, TransactionPool}; use revm::primitives::{BlockEnv, CfgEnv}; @@ -216,6 +216,45 @@ pub enum TransactionSource { }, } +// === impl TransactionSource === + +impl TransactionSource { + /// Consumes the type and returns the wrapped transaction. + pub fn into_recovered(self) -> TransactionSignedEcRecovered { + self.into() + } + + /// Returns the transaction and block related info, if not pending + pub fn split(self) -> (TransactionSignedEcRecovered, TransactionInfo) { + match self { + TransactionSource::Pool(tx) => { + let hash = tx.hash(); + ( + tx, + TransactionInfo { + hash: Some(hash), + index: None, + block_hash: None, + block_number: None, + }, + ) + } + TransactionSource::Database { transaction, index, block_hash, block_number } => { + let hash = transaction.hash(); + ( + transaction, + TransactionInfo { + hash: Some(hash), + index: Some(index), + block_hash: Some(block_hash), + block_number: Some(block_number), + }, + ) + } + } + } +} + impl From for TransactionSignedEcRecovered { fn from(value: TransactionSource) -> Self { match value { diff --git a/crates/rpc/rpc/src/trace.rs b/crates/rpc/rpc/src/trace.rs index b33dbb41195..ec9e2eb7047 100644 --- a/crates/rpc/rpc/src/trace.rs +++ b/crates/rpc/rpc/src/trace.rs @@ -1,16 +1,22 @@ use crate::{ - eth::{cache::EthStateCache, EthTransactions}, + eth::{cache::EthStateCache, revm_utils::inspect, EthTransactions}, result::internal_rpc_err, }; use async_trait::async_trait; use jsonrpsee::core::RpcResult as Result; use reth_primitives::{BlockId, Bytes, H256}; use reth_provider::{BlockProvider, EvmEnvProvider, StateProviderFactory}; +use reth_revm::{ + database::{State, SubState}, + env::tx_env_with_recovered, + tracing::{TraceInspectorConfig, TracingInspector}, +}; use reth_rpc_api::TraceApiServer; use reth_rpc_types::{ trace::{filter::TraceFilter, parity::*}, CallRequest, Index, }; +use revm::primitives::Env; use std::collections::HashSet; /// `trace` API implementation. @@ -115,14 +121,28 @@ where &self, hash: H256, ) -> Result>> { - let (_transaction, at) = match self.eth_api.transaction_by_hash_at(hash).await? { + let (transaction, at) = match self.eth_api.transaction_by_hash_at(hash).await? { None => return Ok(None), Some(res) => res, }; - let (_cfg, _block_env, _at) = self.eth_api.evm_env_at(at).await?; + let (cfg, block, at) = self.eth_api.evm_env_at(at).await?; - Err(internal_rpc_err("unimplemented")) + let (tx, tx_info) = transaction.split(); + + let traces = self.eth_api.with_state_at(at, |state| { + let tx = tx_env_with_recovered(&tx); + let env = Env { cfg, block, tx }; + let db = SubState::new(State::new(state)); + let mut inspector = TracingInspector::new(TraceInspectorConfig::default_parity()); + + inspect(db, env, &mut inspector)?; + + let traces = inspector.into_parity_builder().into_localized_transaction_traces(tx_info); + + Ok(traces) + })?; + Ok(Some(traces)) } } From 7f0e32dd0978fc2b01f31a1f200343eb413083bd Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 15 Mar 2023 15:04:49 +0100 Subject: [PATCH 138/191] chore(rpc): impl with_state_at closure (#1767) --- crates/rpc/rpc/src/eth/api/transactions.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/rpc/rpc/src/eth/api/transactions.rs b/crates/rpc/rpc/src/eth/api/transactions.rs index ef92e75b636..e97d15adb0a 100644 --- a/crates/rpc/rpc/src/eth/api/transactions.rs +++ b/crates/rpc/rpc/src/eth/api/transactions.rs @@ -49,11 +49,12 @@ where Client: BlockProvider + StateProviderFactory + EvmEnvProvider + 'static, Network: Send + Sync + 'static, { - fn with_state_at(&self, _at: BlockId, _f: F) -> EthResult + fn with_state_at(&self, at: BlockId, f: F) -> EthResult where F: FnOnce(ChainState<'_>) -> EthResult, { - unimplemented!() + let state = self.state_at_block_id(at)?.ok_or(EthApiError::UnknownBlockNumber)?; + f(state) } async fn evm_env_at(&self, at: BlockId) -> EthResult<(CfgEnv, BlockEnv, BlockId)> { From 0dfb940f1a78078250e9a88b0c7f7e06adf1bdf3 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 15 Mar 2023 15:51:46 +0100 Subject: [PATCH 139/191] feat(rpc): impl trace_get (#1769) --- crates/rpc/rpc-builder/tests/it/http.rs | 3 - crates/rpc/rpc/src/trace.rs | 84 +++++++++++++++++-------- 2 files changed, 58 insertions(+), 29 deletions(-) diff --git a/crates/rpc/rpc-builder/tests/it/http.rs b/crates/rpc/rpc-builder/tests/it/http.rs index 876f4ce901d..ba3d945648e 100644 --- a/crates/rpc/rpc-builder/tests/it/http.rs +++ b/crates/rpc/rpc-builder/tests/it/http.rs @@ -191,9 +191,6 @@ where assert!(is_unimplemented( TraceApiClient::trace_filter(client, trace_filter).await.err().unwrap() )); - assert!(is_unimplemented( - TraceApiClient::trace_get(client, H256::default(), vec![]).await.err().unwrap() - )); } async fn test_basic_web3_calls(client: &C) diff --git a/crates/rpc/rpc/src/trace.rs b/crates/rpc/rpc/src/trace.rs index ec9e2eb7047..4e871a6e0f8 100644 --- a/crates/rpc/rpc/src/trace.rs +++ b/crates/rpc/rpc/src/trace.rs @@ -1,5 +1,5 @@ use crate::{ - eth::{cache::EthStateCache, revm_utils::inspect, EthTransactions}, + eth::{cache::EthStateCache, error::EthResult, revm_utils::inspect, EthTransactions}, result::internal_rpc_err, }; use async_trait::async_trait; @@ -41,6 +41,58 @@ impl TraceApi { } } +// === impl TraceApi === + +impl TraceApi +where + Client: BlockProvider + StateProviderFactory + EvmEnvProvider + 'static, + Eth: EthTransactions + 'static, +{ + /// Returns transaction trace with the given address. + pub async fn trace_get( + &self, + hash: H256, + trace_address: Vec, + ) -> EthResult> { + match self.trace_transaction(hash).await? { + None => Ok(None), + Some(traces) => { + let trace = + traces.into_iter().find(|trace| trace.trace.trace_address == trace_address); + Ok(trace) + } + } + } + + /// Returns all traces for the given transaction hash + pub async fn trace_transaction( + &self, + hash: H256, + ) -> EthResult>> { + let (transaction, at) = match self.eth_api.transaction_by_hash_at(hash).await? { + None => return Ok(None), + Some(res) => res, + }; + + let (cfg, block, at) = self.eth_api.evm_env_at(at).await?; + + let (tx, tx_info) = transaction.split(); + + self.eth_api.with_state_at(at, |state| { + let tx = tx_env_with_recovered(&tx); + let env = Env { cfg, block, tx }; + let db = SubState::new(State::new(state)); + let mut inspector = TracingInspector::new(TraceInspectorConfig::default_parity()); + + inspect(db, env, &mut inspector)?; + + let traces = inspector.into_parity_builder().into_localized_transaction_traces(tx_info); + + Ok(Some(traces)) + }) + } +} + #[async_trait] impl TraceApiServer for TraceApi where @@ -107,13 +159,14 @@ where Err(internal_rpc_err("unimplemented")) } + /// Returns transaction trace at given index. /// Handler for `trace_get` async fn trace_get( &self, - _hash: H256, - _indices: Vec, + hash: H256, + indices: Vec, ) -> Result> { - Err(internal_rpc_err("unimplemented")) + Ok(TraceApi::trace_get(self, hash, indices.into_iter().map(Into::into).collect()).await?) } /// Handler for `trace_transaction` @@ -121,28 +174,7 @@ where &self, hash: H256, ) -> Result>> { - let (transaction, at) = match self.eth_api.transaction_by_hash_at(hash).await? { - None => return Ok(None), - Some(res) => res, - }; - - let (cfg, block, at) = self.eth_api.evm_env_at(at).await?; - - let (tx, tx_info) = transaction.split(); - - let traces = self.eth_api.with_state_at(at, |state| { - let tx = tx_env_with_recovered(&tx); - let env = Env { cfg, block, tx }; - let db = SubState::new(State::new(state)); - let mut inspector = TracingInspector::new(TraceInspectorConfig::default_parity()); - - inspect(db, env, &mut inspector)?; - - let traces = inspector.into_parity_builder().into_localized_transaction_traces(tx_info); - - Ok(traces) - })?; - Ok(Some(traces)) + Ok(TraceApi::trace_transaction(self, hash).await?) } } From ed9c6a61ee669d20c52231b28f596a57eb754efe Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 15 Mar 2023 17:29:37 +0100 Subject: [PATCH 140/191] refactor(rpc): simplify state_at functions (#1771) --- crates/rpc/rpc/src/eth/api/call.rs | 6 +++--- crates/rpc/rpc/src/eth/api/mod.rs | 23 ++++++++++++---------- crates/rpc/rpc/src/eth/api/state.rs | 16 +++++---------- crates/rpc/rpc/src/eth/api/transactions.rs | 9 ++++++++- 4 files changed, 29 insertions(+), 25 deletions(-) diff --git a/crates/rpc/rpc/src/eth/api/call.rs b/crates/rpc/rpc/src/eth/api/call.rs index a4105ccabae..74c0f8338fa 100644 --- a/crates/rpc/rpc/src/eth/api/call.rs +++ b/crates/rpc/rpc/src/eth/api/call.rs @@ -46,7 +46,7 @@ where state_overrides: Option, ) -> EthResult<(ResultAndState, Env)> { let (cfg, block_env, at) = self.evm_env_at(at).await?; - let state = self.state_at_block_id(at)?.ok_or_else(|| EthApiError::UnknownBlockNumber)?; + let state = self.state_at(at)?; self.call_with(cfg, block_env, request, state, state_overrides) } @@ -86,7 +86,7 @@ where at: BlockId, ) -> EthResult { let (cfg, block_env, at) = self.evm_env_at(at).await?; - let state = self.state_at_block_id(at)?.ok_or_else(|| EthApiError::UnknownBlockNumber)?; + let state = self.state_at(at)?; self.estimate_gas_with(cfg, block_env, request, state) } @@ -272,7 +272,7 @@ where ) -> EthResult { let block_id = at.unwrap_or(BlockId::Number(BlockNumberOrTag::Latest)); let (mut cfg, block, at) = self.evm_env_at(block_id).await?; - let state = self.state_at_block_id(at)?.ok_or_else(|| EthApiError::UnknownBlockNumber)?; + let state = self.state_at(at)?; // we want to disable this in eth_call, since this is common practice used by other node // impls and providers diff --git a/crates/rpc/rpc/src/eth/api/mod.rs b/crates/rpc/rpc/src/eth/api/mod.rs index 19453314996..274835e10ba 100644 --- a/crates/rpc/rpc/src/eth/api/mod.rs +++ b/crates/rpc/rpc/src/eth/api/mod.rs @@ -19,6 +19,7 @@ mod server; mod sign; mod state; mod transactions; +use crate::eth::error::{EthApiError, EthResult}; pub use transactions::{EthTransactions, TransactionSource}; /// Cache limit of block-level fee history for `eth_feeHistory` RPC method. @@ -100,23 +101,25 @@ where self.client().convert_block_number(num) } + /// Returns the state at the given [BlockId] enum. + pub(crate) fn state_at_block_id(&self, at: BlockId) -> EthResult> { + match at { + BlockId::Hash(hash) => Ok(self.state_at_hash(hash.into()).map(ChainState::boxed)?), + BlockId::Number(num) => { + self.state_at_block_number(num)?.ok_or(EthApiError::UnknownBlockNumber) + } + } + } + /// Returns the state at the given [BlockId] enum or the latest. pub(crate) fn state_at_block_id_or_latest( &self, block_id: Option, - ) -> Result>> { + ) -> EthResult> { if let Some(block_id) = block_id { self.state_at_block_id(block_id) } else { - self.latest_state().map(ChainState::boxed).map(Some) - } - } - - /// Returns the state at the given [BlockId] enum. - pub(crate) fn state_at_block_id(&self, block_id: BlockId) -> Result>> { - match block_id { - BlockId::Hash(hash) => self.state_at_hash(hash.into()).map(ChainState::boxed).map(Some), - BlockId::Number(num) => self.state_at_block_number(num), + Ok(self.latest_state().map(ChainState::boxed)?) } } diff --git a/crates/rpc/rpc/src/eth/api/state.rs b/crates/rpc/rpc/src/eth/api/state.rs index fe79fa795af..4227983499a 100644 --- a/crates/rpc/rpc/src/eth/api/state.rs +++ b/crates/rpc/rpc/src/eth/api/state.rs @@ -15,15 +15,13 @@ where Client: BlockProvider + StateProviderFactory + EvmEnvProvider + 'static, { pub(crate) fn get_code(&self, address: Address, block_id: Option) -> EthResult { - let state = - self.state_at_block_id_or_latest(block_id)?.ok_or(EthApiError::UnknownBlockNumber)?; + let state = self.state_at_block_id_or_latest(block_id)?; let code = state.account_code(address)?.unwrap_or_default(); Ok(code.original_bytes().into()) } pub(crate) fn balance(&self, address: Address, block_id: Option) -> EthResult { - let state = - self.state_at_block_id_or_latest(block_id)?.ok_or(EthApiError::UnknownBlockNumber)?; + let state = self.state_at_block_id_or_latest(block_id)?; let balance = state.account_balance(address)?.unwrap_or_default(); Ok(balance) } @@ -33,8 +31,7 @@ where address: Address, block_id: Option, ) -> EthResult { - let state = - self.state_at_block_id_or_latest(block_id)?.ok_or(EthApiError::UnknownBlockNumber)?; + let state = self.state_at_block_id_or_latest(block_id)?; let nonce = U256::from(state.account_nonce(address)?.unwrap_or_default()); Ok(nonce) } @@ -45,8 +42,7 @@ where index: U256, block_id: Option, ) -> EthResult { - let state = - self.state_at_block_id_or_latest(block_id)?.ok_or(EthApiError::UnknownBlockNumber)?; + let state = self.state_at_block_id_or_latest(block_id)?; let storage_key = H256(index.to_be_bytes()); let value = state.storage(address, storage_key)?.unwrap_or_default(); Ok(H256(value.to_be_bytes())) @@ -65,9 +61,7 @@ where return Err(EthApiError::InvalidBlockRange) } - let state = self - .state_at_block_id_or_latest(Some(block_id))? - .ok_or(EthApiError::UnknownBlockNumber)?; + let state = self.state_at_block_id(block_id)?; let (account_proof, storage_hash, stg_proofs) = state.proof(address, &keys)?; diff --git a/crates/rpc/rpc/src/eth/api/transactions.rs b/crates/rpc/rpc/src/eth/api/transactions.rs index e97d15adb0a..cc8ffeeb822 100644 --- a/crates/rpc/rpc/src/eth/api/transactions.rs +++ b/crates/rpc/rpc/src/eth/api/transactions.rs @@ -17,6 +17,9 @@ use revm::primitives::{BlockEnv, CfgEnv}; /// Commonly used transaction related functions for the [EthApi] type in the `eth_` namespace #[async_trait::async_trait] pub trait EthTransactions: Send + Sync { + /// Returns the state at the given [BlockId] + fn state_at(&self, at: BlockId) -> EthResult>; + /// Executes the closure with the state that corresponds to the given [BlockId]. fn with_state_at(&self, _at: BlockId, _f: F) -> EthResult where @@ -49,11 +52,15 @@ where Client: BlockProvider + StateProviderFactory + EvmEnvProvider + 'static, Network: Send + Sync + 'static, { + fn state_at(&self, at: BlockId) -> EthResult> { + self.state_at_block_id(at) + } + fn with_state_at(&self, at: BlockId, f: F) -> EthResult where F: FnOnce(ChainState<'_>) -> EthResult, { - let state = self.state_at_block_id(at)?.ok_or(EthApiError::UnknownBlockNumber)?; + let state = self.state_at(at)?; f(state) } From 7d30bd39416a8bb7e143c3e9a4b3974195c671e7 Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Wed, 15 Mar 2023 19:44:26 +0200 Subject: [PATCH 141/191] chore(sync): blockchain tree config and externals (#1760) --- .../src/blockchain_tree/block_indices.rs | 17 +-- crates/executor/src/blockchain_tree/config.rs | 59 ++++++++ .../executor/src/blockchain_tree/externals.rs | 32 ++++ crates/executor/src/blockchain_tree/mod.rs | 141 +++++++----------- 4 files changed, 152 insertions(+), 97 deletions(-) create mode 100644 crates/executor/src/blockchain_tree/config.rs create mode 100644 crates/executor/src/blockchain_tree/externals.rs diff --git a/crates/executor/src/blockchain_tree/block_indices.rs b/crates/executor/src/blockchain_tree/block_indices.rs index 0a96fc89d9f..586c2773642 100644 --- a/crates/executor/src/blockchain_tree/block_indices.rs +++ b/crates/executor/src/blockchain_tree/block_indices.rs @@ -12,10 +12,6 @@ use std::collections::{hash_map::Entry, BTreeMap, BTreeSet, HashMap, HashSet}; pub struct BlockIndices { /// Last finalized block. last_finalized_block: BlockNumber, - /// For EVM's "BLOCKHASH" opcode we require last 256 block hashes. So we need to specify - /// at least `additional_canonical_block_hashes`+`max_reorg_depth`, for eth that would be - /// 256+64. - num_of_additional_canonical_block_hashes: u64, /// Canonical chain. Contains N number (depends on `finalization_depth`) of blocks. /// These blocks are found in fork_to_child but not inside `blocks_to_chain` or /// `number_to_block` as those are chain specific indices. @@ -34,25 +30,17 @@ impl BlockIndices { /// Create new block indices structure pub fn new( last_finalized_block: BlockNumber, - num_of_additional_canonical_block_hashes: u64, canonical_chain: BTreeMap, ) -> Self { Self { last_finalized_block, - num_of_additional_canonical_block_hashes, - fork_to_child: Default::default(), canonical_chain, + fork_to_child: Default::default(), blocks_to_chain: Default::default(), index_number_to_block: Default::default(), } } - /// Return number of additional canonical block hashes that we need - /// to have to be able to have enought information for EVM execution. - pub fn num_of_additional_canonical_block_hashes(&self) -> u64 { - self.num_of_additional_canonical_block_hashes - } - /// Return fork to child indices pub fn fork_to_child(&self) -> &HashMap> { &self.fork_to_child @@ -258,6 +246,7 @@ impl BlockIndices { pub fn finalize_canonical_blocks( &mut self, finalized_block: BlockNumber, + num_of_additional_canonical_hashes_to_retain: u64, ) -> BTreeSet { // get finalized chains. blocks between [self.last_finalized,finalized_block). // Dont remove finalized_block, as sidechain can point to it. @@ -270,7 +259,7 @@ impl BlockIndices { // remove unneeded canonical hashes. let remove_until = - finalized_block.saturating_sub(self.num_of_additional_canonical_block_hashes); + finalized_block.saturating_sub(num_of_additional_canonical_hashes_to_retain); self.canonical_chain.retain(|&number, _| number >= remove_until); let mut lose_chains = BTreeSet::new(); diff --git a/crates/executor/src/blockchain_tree/config.rs b/crates/executor/src/blockchain_tree/config.rs new file mode 100644 index 00000000000..3de52c11049 --- /dev/null +++ b/crates/executor/src/blockchain_tree/config.rs @@ -0,0 +1,59 @@ +//! Blockchain tree configuration + +/// The configuration for the blockchain tree. +#[derive(Clone, Debug)] +pub struct BlockchainTreeConfig { + /// Finalization windows. Number of blocks that can be reorged + max_reorg_depth: u64, + /// Number of block after finalized block that we are storing. It should be more then + /// finalization window + max_blocks_in_chain: u64, + /// For EVM's "BLOCKHASH" opcode we require last 256 block hashes. So we need to specify + /// at least `additional_canonical_block_hashes`+`max_reorg_depth`, for eth that would be + /// 256+64. + num_of_additional_canonical_block_hashes: u64, +} + +impl Default for BlockchainTreeConfig { + fn default() -> Self { + // The defaults for Ethereum mainnet + Self { + // Gasper allows reorgs of any length from 1 to 64. + max_reorg_depth: 64, + // This default is just an assumption. Has to be greater than the `max_reorg_depth`. + max_blocks_in_chain: 65, + // EVM requires that last 256 block hashes are available. + num_of_additional_canonical_block_hashes: 256, + } + } +} + +impl BlockchainTreeConfig { + /// Create tree configuration. + pub fn new( + max_reorg_depth: u64, + max_blocks_in_chain: u64, + num_of_additional_canonical_block_hashes: u64, + ) -> Self { + if max_reorg_depth > max_blocks_in_chain { + panic!("Side chain size should be more then finalization window"); + } + Self { max_blocks_in_chain, max_reorg_depth, num_of_additional_canonical_block_hashes } + } + + /// Return the maximum reorg depth. + pub fn max_reorg_depth(&self) -> u64 { + self.max_reorg_depth + } + + /// Return the maximum number of blocks in one chain. + pub fn max_blocks_in_chain(&self) -> u64 { + self.max_blocks_in_chain + } + + /// Return number of additional canonical block hashes that we need to retain + /// in order to have enough information for EVM execution. + pub fn num_of_additional_canonical_block_hashes(&self) -> u64 { + self.num_of_additional_canonical_block_hashes + } +} diff --git a/crates/executor/src/blockchain_tree/externals.rs b/crates/executor/src/blockchain_tree/externals.rs new file mode 100644 index 00000000000..72e31f4fb39 --- /dev/null +++ b/crates/executor/src/blockchain_tree/externals.rs @@ -0,0 +1,32 @@ +//! Blockchain tree externals. + +use reth_db::database::Database; +use reth_primitives::ChainSpec; +use reth_provider::ShareableDatabase; +use std::sync::Arc; + +/// Container for external abstractions. +pub struct TreeExternals { + /// Save sidechain, do reorgs and push new block to canonical chain that is inside db. + pub db: DB, + /// Consensus checks + pub consensus: C, + /// Create executor to execute blocks. + pub executor_factory: EF, + /// Chain spec + pub chain_spec: Arc, +} + +impl TreeExternals { + /// Create new tree externals. + pub fn new(db: DB, consensus: C, executor_factory: EF, chain_spec: Arc) -> Self { + Self { db, consensus, executor_factory, chain_spec } + } +} + +impl TreeExternals { + /// Return shareable database helper structure. + pub fn shareable_db(&self) -> ShareableDatabase<&DB> { + ShareableDatabase::new(&self.db, self.chain_spec.clone()) + } +} diff --git a/crates/executor/src/blockchain_tree/mod.rs b/crates/executor/src/blockchain_tree/mod.rs index 30347fc2a4c..722c1c49953 100644 --- a/crates/executor/src/blockchain_tree/mod.rs +++ b/crates/executor/src/blockchain_tree/mod.rs @@ -1,23 +1,24 @@ //! Implementation of [`BlockchainTree`] -pub mod block_indices; -pub mod chain; - -use self::{ - block_indices::BlockIndices, - chain::{ChainSplit, SplitAt}, -}; use chain::{BlockChainId, Chain, ForkBlock}; use reth_db::{cursor::DbCursorRO, database::Database, tables, transaction::DbTx}; use reth_interfaces::{consensus::Consensus, executor::Error as ExecError, Error}; -use reth_primitives::{BlockHash, BlockNumber, ChainSpec, SealedBlock, SealedBlockWithSenders}; +use reth_primitives::{BlockHash, BlockNumber, SealedBlock, SealedBlockWithSenders}; use reth_provider::{ - providers::ChainState, ExecutorFactory, HeaderProvider, ShareableDatabase, - StateProviderFactory, Transaction, -}; -use std::{ - collections::{BTreeMap, HashMap}, - sync::Arc, + providers::ChainState, ExecutorFactory, HeaderProvider, StateProviderFactory, Transaction, }; +use std::collections::{BTreeMap, HashMap}; + +pub mod block_indices; +use block_indices::BlockIndices; + +pub mod chain; +use chain::{ChainSplit, SplitAt}; + +pub mod config; +use config::BlockchainTreeConfig; + +pub mod externals; +use externals::TreeExternals; #[cfg_attr(doc, aquamarine::aquamarine)] /// Tree of chains and its identifications. @@ -52,12 +53,13 @@ use std::{ /// /// /// main functions: -/// * insert_block: Connect block to chain, execute it and if valid insert block inside tree. -/// * finalize_block: Remove chains that join to now finalized block, as chain becomes invalid. -/// * make_canonical: Check if we have the hash of block that we want to finalize and commit it to -/// db. If we dont have the block, pipeline syncing should start to fetch the blocks from p2p. Do -/// reorg in tables if canonical chain if needed. - +/// * [BlockchainTree::insert_block]: Connect block to chain, execute it and if valid insert block +/// inside tree. +/// * [BlockchainTree::finalize_block]: Remove chains that join to now finalized block, as chain +/// becomes invalid. +/// * [BlockchainTree::make_canonical]: Check if we have the hash of block that we want to finalize +/// and commit it to db. If we dont have the block, pipeline syncing should start to fetch the +/// blocks from p2p. Do reorg in tables if canonical chain if needed. pub struct BlockchainTree { /// chains and present data chains: HashMap, @@ -65,32 +67,10 @@ pub struct BlockchainTree { block_chain_id_generator: u64, /// Indices to block and their connection. block_indices: BlockIndices, - /// Number of block after finalized block that we are storing. It should be more then - /// finalization window - max_blocks_in_chain: u64, - /// Finalization windows. Number of blocks that can be reorged - max_reorg_depth: u64, + /// Tree configuration. + config: BlockchainTreeConfig, /// Externals - externals: Externals, -} - -/// Container for external abstractions. -struct Externals { - /// Save sidechain, do reorgs and push new block to canonical chain that is inside db. - db: DB, - /// Consensus checks - consensus: C, - /// Create executor to execute blocks. - executor_factory: EF, - /// Chain spec - chain_spec: Arc, -} - -impl Externals { - /// Return sharable database helper structure. - fn sharable_db(&self) -> ShareableDatabase<&DB> { - ShareableDatabase::new(&self.db, self.chain_spec.clone()) - } + externals: TreeExternals, } /// Helper structure that wraps chains and indices to search for block hash accross the chains. @@ -104,23 +84,17 @@ pub struct BlockHashes<'a> { impl BlockchainTree { /// New blockchain tree pub fn new( - db: DB, - consensus: C, - executor_factory: EF, - chain_spec: Arc, - max_reorg_depth: u64, - max_blocks_in_chain: u64, - num_of_additional_canonical_block_hashes: u64, + externals: TreeExternals, + config: BlockchainTreeConfig, ) -> Result { - if max_reorg_depth > max_blocks_in_chain { - panic!("Side chain size should be more then finalization window"); - } + let max_reorg_depth = config.max_reorg_depth(); - let last_canonical_hashes = db + let last_canonical_hashes = externals + .db .tx()? .cursor_read::()? .walk_back(None)? - .take((max_reorg_depth + num_of_additional_canonical_block_hashes) as usize) + .take((max_reorg_depth + config.num_of_additional_canonical_block_hashes()) as usize) .collect::, _>>()?; // TODO(rakita) save last finalized block inside database but for now just take @@ -134,19 +108,15 @@ impl BlockchainTree last_canonical_hashes.last().cloned().unwrap_or_default() }; - let externals = Externals { db, consensus, executor_factory, chain_spec }; - Ok(Self { externals, block_chain_id_generator: 0, chains: Default::default(), block_indices: BlockIndices::new( last_finalized_block_number, - num_of_additional_canonical_block_hashes, BTreeMap::from_iter(last_canonical_hashes.into_iter()), ), - max_blocks_in_chain, - max_reorg_depth, + config, }) } @@ -175,7 +145,7 @@ impl BlockchainTree let (_, canonical_tip_hash) = canonical_block_hashes.last_key_value().map(|(i, j)| (*i, *j)).unwrap_or_default(); - let db = self.externals.sharable_db(); + let db = self.externals.shareable_db(); let provider = if canonical_fork.hash == canonical_tip_hash { ChainState::boxed(db.latest()?) } else { @@ -220,7 +190,7 @@ impl BlockchainTree canonical_block_hashes.last_key_value().map(|(i, j)| (*i, *j)).unwrap_or_default(); // create state provider - let db = self.externals.sharable_db(); + let db = self.externals.shareable_db(); let parent_header = db .header(&block.parent_hash)? .ok_or(ExecError::CanonicalChain { block_hash: block.parent_hash })?; @@ -333,7 +303,7 @@ impl BlockchainTree } // we will not even try to insert blocks that are too far in future. - if block.number > last_finalized_block + self.max_blocks_in_chain { + if block.number > last_finalized_block + self.config.max_blocks_in_chain() { return Err(ExecError::PendingBlockIsInFuture { block_number: block.number, block_hash: block.hash(), @@ -378,7 +348,10 @@ impl BlockchainTree /// Do finalization of blocks. Remove them from tree pub fn finalize_block(&mut self, finalized_block: BlockNumber) { - let mut remove_chains = self.block_indices.finalize_canonical_blocks(finalized_block); + let mut remove_chains = self.block_indices.finalize_canonical_blocks( + finalized_block, + self.config.num_of_additional_canonical_block_hashes(), + ); while let Some(chain_id) = remove_chains.pop_first() { if let Some(chain) = self.chains.remove(&chain_id) { @@ -388,14 +361,14 @@ impl BlockchainTree } /// Update canonical hashes. Reads last N canonical blocks from database and update all indices. - pub fn update_canonical_hashes( + pub fn restore_canonical_hashes( &mut self, last_finalized_block: BlockNumber, ) -> Result<(), Error> { self.finalize_block(last_finalized_block); let num_of_canonical_hashes = - self.max_reorg_depth + self.block_indices.num_of_additional_canonical_block_hashes(); + self.config.max_reorg_depth() + self.config.num_of_additional_canonical_block_hashes(); let last_canonical_hashes = self .externals @@ -561,13 +534,16 @@ mod tests { transaction::DbTxMut, }; use reth_interfaces::test_utils::TestConsensus; - use reth_primitives::{hex_literal::hex, proofs::EMPTY_ROOT, ChainSpecBuilder, H256, MAINNET}; + use reth_primitives::{ + hex_literal::hex, proofs::EMPTY_ROOT, ChainSpec, ChainSpecBuilder, H256, MAINNET, + }; use reth_provider::{ execution_result::ExecutionResult, insert_block, test_utils::blocks::BlockChainTestData, BlockExecutor, StateProvider, }; - use std::collections::HashSet; + use std::{collections::HashSet, sync::Arc}; + #[derive(Clone)] struct TestFactory { exec_result: Arc>>, chain_spec: Arc, @@ -618,11 +594,11 @@ mod tests { } } - type TestExternals = (Arc>, TestConsensus, TestFactory, Arc); - - fn externals(exec_res: Vec) -> TestExternals { + fn setup_externals( + exec_res: Vec, + ) -> TreeExternals>, Arc, TestFactory> { let db = create_test_rw_db(); - let consensus = TestConsensus::default(); + let consensus = Arc::new(TestConsensus::default()); let chain_spec = Arc::new( ChainSpecBuilder::default() .chain(MAINNET.chain) @@ -633,15 +609,15 @@ mod tests { let executor_factory = TestFactory::new(chain_spec.clone()); executor_factory.extend(exec_res); - (db, consensus, executor_factory, chain_spec) + TreeExternals::new(db, consensus, executor_factory, chain_spec) } - fn setup(mut genesis: SealedBlock, externals: &TestExternals) { + fn setup_genesis(db: DB, mut genesis: SealedBlock) { // insert genesis to db. genesis.header.header.number = 10; genesis.header.header.state_root = EMPTY_ROOT; - let tx_mut = externals.0.tx_mut().unwrap(); + let tx_mut = db.tx_mut().unwrap(); insert_block(&tx_mut, genesis, None, false, Some((0, 0))).unwrap(); @@ -709,15 +685,14 @@ mod tests { H256(hex!("90101a13dd059fa5cca99ed93d1dc23657f63626c5b8f993a2ccbdf7446b64f8")); // test pops execution results from vector, so order is from last to first.ß - let externals = externals(vec![exec2.clone(), exec1.clone(), exec2, exec1]); + let externals = setup_externals(vec![exec2.clone(), exec1.clone(), exec2, exec1]); // last finalized block would be number 9. - setup(data.genesis, &externals); + setup_genesis(externals.db.clone(), data.genesis); // make tree - let (db, consensus, exec_factory, chain_spec) = externals; - let mut tree = - BlockchainTree::new(db, consensus, exec_factory, chain_spec, 1, 2, 3).unwrap(); + let config = BlockchainTreeConfig::new(1, 2, 3); + let mut tree = BlockchainTree::new(externals, config).expect("failed to create tree"); // genesis block 10 is already canonical assert_eq!(tree.make_canonical(&H256::zero()), Ok(())); @@ -900,7 +875,7 @@ mod tests { .assert(&tree); // update canonical block to b2, this would make b2a be removed - assert_eq!(tree.update_canonical_hashes(12), Ok(())); + assert_eq!(tree.restore_canonical_hashes(12), Ok(())); // Trie state: // b2 (canon) // | From 47878184b9ce6ac9b7d00d5c0cf8436e208b8bc5 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 15 Mar 2023 22:47:14 +0100 Subject: [PATCH 142/191] chore(deps): remove unused deps (#1779) --- Cargo.lock | 5 ----- crates/stages/Cargo.toml | 14 ++------------ crates/stages/benches/setup/mod.rs | 4 ++-- 3 files changed, 4 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 206793fd004..3980ec6fb23 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5298,11 +5298,9 @@ name = "reth-stages" version = "0.1.0" dependencies = [ "aquamarine", - "arbitrary", "assert_matches", "async-trait", "criterion", - "eyre", "futures-util", "itertools 0.10.5", "metrics", @@ -5310,7 +5308,6 @@ dependencies = [ "paste", "pin-project", "pprof", - "proptest", "rand 0.8.5", "rayon", "reth-codecs", @@ -5323,8 +5320,6 @@ dependencies = [ "reth-primitives", "reth-provider", "reth-rlp", - "serde", - "tempfile", "thiserror", "tokio", "tokio-stream", diff --git a/crates/stages/Cargo.toml b/crates/stages/Cargo.toml index 85c10f4c5c6..456619bb2c6 100644 --- a/crates/stages/Cargo.toml +++ b/crates/stages/Cargo.toml @@ -34,17 +34,11 @@ tracing = "0.1.36" metrics = "0.20.1" # misc -serde = { version = "1.0", optional = true } thiserror = "1.0.37" aquamarine = "0.2.1" #docs -itertools = "0.10.5" rayon = "1.6.0" num-traits = "0.2.15" -# arbitrary utils -arbitrary = { version = "1.1.7", features = ["derive"], optional = true } -proptest = { version = "1.0", optional = true } - [dev-dependencies] # reth reth-primitives = { path = "../primitives", features = ["arbitrary"] } @@ -54,8 +48,9 @@ reth-downloaders = { path = "../net/downloaders" } reth-eth-wire = { path = "../net/eth-wire" } # TODO(onbjerg): We only need this for [BlockBody] reth-executor = { path = "../executor" } reth-rlp = { path = "../rlp" } + +itertools = "0.10.5" tokio = { version = "*", features = ["rt", "sync", "macros"] } -tempfile = "3.3.0" assert_matches = "1.5.0" rand = "0.8.5" paste = "1.0" @@ -67,13 +62,8 @@ pprof = { version = "0.11", features = [ "criterion", ] } criterion = { version = "0.4.0", features = ["async_futures"] } -proptest = { version = "1.0" } -arbitrary = { version = "1.1.7", features = ["derive"] } -eyre = "0.6.8" [features] -default = ["serde"] -serde = ["dep:serde"] test-utils = [] diff --git a/crates/stages/benches/setup/mod.rs b/crates/stages/benches/setup/mod.rs index 1d3fb143b64..eb587f893d2 100644 --- a/crates/stages/benches/setup/mod.rs +++ b/crates/stages/benches/setup/mod.rs @@ -45,10 +45,10 @@ pub(crate) fn stage_unwind>>( .unwind(&mut db_tx, unwind) .await .map_err(|e| { - eyre::eyre!(format!( + format!( "{e}\nMake sure your test database at `{}` isn't too old and incompatible with newer stage changes.", tx.path.as_ref().unwrap().display() - )) + ) }) .unwrap(); From 1cba25e6517cee6332fdba57bf0eea93f7a8c022 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 15 Mar 2023 23:19:25 +0100 Subject: [PATCH 143/191] chore(deps): bump aquamarine 0.3 (#1780) --- Cargo.lock | 27 +++++++++------------------ crates/executor/Cargo.toml | 2 +- crates/net/network/Cargo.toml | 2 +- crates/stages/Cargo.toml | 3 ++- crates/transaction-pool/Cargo.toml | 2 +- 5 files changed, 14 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3980ec6fb23..6a29470793e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -144,11 +144,11 @@ checksum = "224afbd727c3d6e4b90103ece64b8d1b67fbb1973b1046c2281eed3f3803f800" [[package]] name = "aquamarine" -version = "0.2.2" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bf310f0dd77f453bc43ec61506ead8283ab49423686244e474d0cc16226400c" +checksum = "759d98a5db12e9c9d98ef2b92f794ae5c7ded6ec18d21c3fa485c9c65bec237d" dependencies = [ - "itertools 0.9.0", + "itertools", "proc-macro-error", "proc-macro2 1.0.52", "quote 1.0.23", @@ -1009,7 +1009,7 @@ dependencies = [ "clap 3.2.23", "criterion-plot", "futures", - "itertools 0.10.5", + "itertools", "lazy_static", "num-traits", "oorandom", @@ -1030,7 +1030,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" dependencies = [ "cast", - "itertools 0.10.5", + "itertools", ] [[package]] @@ -2973,15 +2973,6 @@ dependencies = [ "windows-sys 0.45.0", ] -[[package]] -name = "itertools" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "284f18f85651fe11e8a991b2adb42cb078325c996ed026d994719efcfca1d54b" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.10.5" @@ -4126,7 +4117,7 @@ checksum = "59230a63c37f3e18569bdb90e4a89cbf5bf8b06fea0b84e65ea10cc4df47addd" dependencies = [ "difflib", "float-cmp", - "itertools 0.10.5", + "itertools", "normalize-line-endings", "predicates-core", "regex", @@ -4724,7 +4715,7 @@ dependencies = [ "assert_matches", "futures", "futures-util", - "itertools 0.10.5", + "itertools", "metrics", "pin-project", "rayon", @@ -5064,7 +5055,7 @@ dependencies = [ "auto_impl", "cita_trie", "hasher", - "itertools 0.10.5", + "itertools", "parking_lot 0.12.1", "proptest", "reth-codecs", @@ -5302,7 +5293,7 @@ dependencies = [ "async-trait", "criterion", "futures-util", - "itertools 0.10.5", + "itertools", "metrics", "num-traits", "paste", diff --git a/crates/executor/Cargo.toml b/crates/executor/Cargo.toml index c56c86bb044..8eb3a72f893 100644 --- a/crates/executor/Cargo.toml +++ b/crates/executor/Cargo.toml @@ -33,7 +33,7 @@ tracing = "0.1.37" tokio = { version = "1.21.2", features = ["sync"] } # mics -aquamarine = "0.2.1" #docs +aquamarine = "0.3.0" triehash = "0.8" # See to replace hashers to simplify libraries diff --git a/crates/net/network/Cargo.toml b/crates/net/network/Cargo.toml index fa41578658d..a7c3a68fcc8 100644 --- a/crates/net/network/Cargo.toml +++ b/crates/net/network/Cargo.toml @@ -50,7 +50,7 @@ reth-metrics-derive = { path = "../../metrics/metrics-derive" } # misc auto_impl = "1" -aquamarine = "0.2.1" # docs +aquamarine = "0.3.0" tracing = "0.1" fnv = "1.0" thiserror = "1.0" diff --git a/crates/stages/Cargo.toml b/crates/stages/Cargo.toml index 456619bb2c6..8eded1c5166 100644 --- a/crates/stages/Cargo.toml +++ b/crates/stages/Cargo.toml @@ -35,7 +35,8 @@ metrics = "0.20.1" # misc thiserror = "1.0.37" -aquamarine = "0.2.1" #docs +aquamarine = "0.3.0" +itertools = "0.10.5" rayon = "1.6.0" num-traits = "0.2.15" diff --git a/crates/transaction-pool/Cargo.toml b/crates/transaction-pool/Cargo.toml index e04c502ec83..2c0f2bf6bd2 100644 --- a/crates/transaction-pool/Cargo.toml +++ b/crates/transaction-pool/Cargo.toml @@ -33,7 +33,7 @@ metrics = "0.20.1" reth-metrics-derive = { path = "../metrics/metrics-derive" } # misc -aquamarine = "0.2.1" # docs +aquamarine = "0.3.0" thiserror = "1.0" tracing = "0.1" serde = { version = "1.0", features = ["derive", "rc"], optional = true } From 3b8c8765243b8bf168ffa04976502e95d5e95921 Mon Sep 17 00:00:00 2001 From: rakita Date: Thu, 16 Mar 2023 01:52:38 +0100 Subject: [PATCH 144/191] chore(BlockchainTree): unwind function (#1775) Co-authored-by: Georgios Konstantopoulos --- .../src/blockchain_tree/block_indices.rs | 11 ++++ crates/executor/src/blockchain_tree/mod.rs | 66 +++++++++++++++++-- 2 files changed, 70 insertions(+), 7 deletions(-) diff --git a/crates/executor/src/blockchain_tree/block_indices.rs b/crates/executor/src/blockchain_tree/block_indices.rs index 586c2773642..24eebff1163 100644 --- a/crates/executor/src/blockchain_tree/block_indices.rs +++ b/crates/executor/src/blockchain_tree/block_indices.rs @@ -9,6 +9,7 @@ use std::collections::{hash_map::Entry, BTreeMap, BTreeSet, HashMap, HashSet}; /// /// It contains list of canonical block hashes, forks to childs blocks /// and block hash to chain id. +#[derive(Debug)] pub struct BlockIndices { /// Last finalized block. last_finalized_block: BlockNumber, @@ -241,6 +242,16 @@ impl BlockIndices { self.canonical_chain.extend(blocks.iter().map(|(number, block)| (*number, block.hash()))) } + /// this is function that is going to remove N number of last canonical hashes. + /// + /// NOTE: This is not safe standalone, as it will not disconnect + /// blocks that deppends on unwinded canonical chain. And should be + /// used when canonical chain is reinserted inside Tree. + fn unwind_canonical_chain(&mut self, unwind_to: BlockNumber) { + // this will remove all blocks numbers that are going to be replaced. + self.canonical_chain.retain(|num, _| *num <= unwind_to); + } + /// Used for finalization of block. /// Return list of chains for removal that depend on finalized canonical chain. pub fn finalize_canonical_blocks( diff --git a/crates/executor/src/blockchain_tree/mod.rs b/crates/executor/src/blockchain_tree/mod.rs index 722c1c49953..7d3451755a8 100644 --- a/crates/executor/src/blockchain_tree/mod.rs +++ b/crates/executor/src/blockchain_tree/mod.rs @@ -462,15 +462,10 @@ impl BlockchainTree unreachable!("all chains should point to canonical chain."); } - // revert `N` blocks from current canonical chain and put them inside BlockchanTree - // This is main reorgs on tables. let old_canon_chain = self.revert_canonical(canon_fork.number)?; + // commit new canonical chain. self.commit_canonical(new_canon_chain)?; - - // TODO we can potentially merge now reverted canonical chain with - // one of the chain from the tree. Low priority. - - // insert old canonical chain to BlockchainTree. + // insert old canon chain self.insert_chain(old_canon_chain); } @@ -497,6 +492,26 @@ impl BlockchainTree Ok(()) } + /// Unwind tables and put it inside state + pub fn unwind(&mut self, unwind_to: BlockNumber) -> Result<(), Error> { + // nothing to be done if unwind_to is higher then the tip + if self.block_indices.canonical_tip().number <= unwind_to { + return Ok(()) + } + // revert `N` blocks from current canonical chain and put them inside BlockchanTree + let old_canon_chain = self.revert_canonical(unwind_to)?; + + // check if there is block in chain + if old_canon_chain.blocks().is_empty() { + return Ok(()) + } + self.block_indices.unwind_canonical_chain(unwind_to); + // insert old canonical chain to BlockchainTree. + self.insert_chain(old_canon_chain); + + Ok(()) + } + /// Revert canonical blocks from database and insert them to pending table /// Revert should be non inclusive, and revert_until should stay in db. /// Return the chain that represent reverted canonical blocks. @@ -874,6 +889,43 @@ mod tests { .with_fork_to_child(HashMap::from([(block1.hash(), HashSet::from([block2a_hash]))])) .assert(&tree); + // unwind canonical + assert_eq!(tree.unwind(block1.number), Ok(())); + // Trie state: + // b2 b2a (pending block) + // / / + // / / + // / / + // b1 (canonical block) + // | + // | + // g1 (canonical blocks) + // | + TreeTester::default() + .with_chain_num(2) + .with_block_to_chain(HashMap::from([(block2a_hash, 4), (block2.hash, 6)])) + .with_fork_to_child(HashMap::from([( + block1.hash(), + HashSet::from([block2a_hash, block2.hash]), + )])) + .assert(&tree); + + // commit b2a + assert_eq!(tree.make_canonical(&block2.hash), Ok(())); + // Trie state: + // b2 b2a (side chain) + // | / + // | / + // b1 (canon) + // | + // g1 (10) + // | + TreeTester::default() + .with_chain_num(1) + .with_block_to_chain(HashMap::from([(block2a_hash, 4)])) + .with_fork_to_child(HashMap::from([(block1.hash(), HashSet::from([block2a_hash]))])) + .assert(&tree); + // update canonical block to b2, this would make b2a be removed assert_eq!(tree.restore_canonical_hashes(12), Ok(())); // Trie state: From d6344e79fdc76c622a765ac8aa3ff0969820b333 Mon Sep 17 00:00:00 2001 From: rakita Date: Thu, 16 Mar 2023 01:55:38 +0100 Subject: [PATCH 145/191] chore(provider): Added explanation for get_take ExecutionResult fn (#1776) --- crates/storage/provider/src/transaction.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/crates/storage/provider/src/transaction.rs b/crates/storage/provider/src/transaction.rs index 8a842be8e1a..bf60a31b154 100644 --- a/crates/storage/provider/src/transaction.rs +++ b/crates/storage/provider/src/transaction.rs @@ -765,6 +765,22 @@ where } /// Transverse over changesets and plain state and recreated the execution results. + /// + /// Iterate over [tables::BlockTransitionIndex] and take all transitions. + /// Then iterate over all [tables::StorageChangeSet] and [tables::AccountChangeSet] in reverse + /// order and populate all changesets. To be able to populate changesets correctly and to + /// have both, new and old value of account/storage, we needs to have local state and access + /// to [tables::PlainAccountState] [tables::PlainStorageState]. + /// While iteration over acocunt/storage changesets. + /// At first instance of account/storage we are taking old value from changeset, + /// new value from plain state and saving old value to local state. + /// As second accounter of same account/storage we are again taking old value from changeset, + /// but new value is taken from local state and old value is again updated to local state. + /// + /// Now if TAKE is true we will use local state and update all old values to plain state tables. + /// + /// After that, iterate over [`tables::BlockBodies`] and pack created changesets in block chunks + /// taking care if block has block changesets or not. fn get_take_block_execution_result_range( &self, range: impl RangeBounds + Clone, From 67478ba779e851f0aff3945b407a6d18d46a1119 Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Thu, 16 Mar 2023 11:21:52 +0200 Subject: [PATCH 146/191] fix(tree): vis of unwind block indices (#1786) --- crates/executor/src/blockchain_tree/block_indices.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/executor/src/blockchain_tree/block_indices.rs b/crates/executor/src/blockchain_tree/block_indices.rs index 24eebff1163..7362316d32d 100644 --- a/crates/executor/src/blockchain_tree/block_indices.rs +++ b/crates/executor/src/blockchain_tree/block_indices.rs @@ -247,7 +247,7 @@ impl BlockIndices { /// NOTE: This is not safe standalone, as it will not disconnect /// blocks that deppends on unwinded canonical chain. And should be /// used when canonical chain is reinserted inside Tree. - fn unwind_canonical_chain(&mut self, unwind_to: BlockNumber) { + pub(crate) fn unwind_canonical_chain(&mut self, unwind_to: BlockNumber) { // this will remove all blocks numbers that are going to be replaced. self.canonical_chain.retain(|num, _| *num <= unwind_to); } From 00bcb210a0d33ec9d32e50edcc0466a835d5b7b9 Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Thu, 16 Mar 2023 12:11:46 +0200 Subject: [PATCH 147/191] fix(ci): downgrade nightly to `2023-03-14` (#1790) --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5a171f1177c..d56f7da792e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,8 +23,9 @@ jobs: - name: Checkout sources uses: actions/checkout@v3 - name: Install toolchain - uses: dtolnay/rust-toolchain@nightly + uses: dtolnay/rust-toolchain@master with: + toolchain: nightly-2023-03-14 components: rustfmt, clippy - uses: Swatinem/rust-cache@v2 with: From 498687b76162fb1117e34e3e35c3517b9593b04e Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 16 Mar 2023 14:11:16 +0100 Subject: [PATCH 148/191] feat(txpool): add EthTransactionValidator::new (#1792) --- crates/transaction-pool/src/validate.rs | 40 ++++++++++++++++++++----- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/crates/transaction-pool/src/validate.rs b/crates/transaction-pool/src/validate.rs index 84770a1ba77..37c1829be63 100644 --- a/crates/transaction-pool/src/validate.rs +++ b/crates/transaction-pool/src/validate.rs @@ -7,12 +7,12 @@ use crate::{ MAX_INIT_CODE_SIZE, TX_MAX_SIZE, }; use reth_primitives::{ - Address, IntoRecoveredTransaction, InvalidTransactionError, TransactionKind, + Address, ChainSpec, IntoRecoveredTransaction, InvalidTransactionError, TransactionKind, TransactionSignedEcRecovered, TxHash, EIP1559_TX_TYPE_ID, EIP2930_TX_TYPE_ID, LEGACY_TX_TYPE_ID, U256, }; use reth_provider::AccountProvider; -use std::{fmt, time::Instant}; +use std::{fmt, sync::Arc, time::Instant}; /// A Result type returned after checking a transaction's validity. #[derive(Debug)] @@ -85,8 +85,8 @@ pub trait TransactionValidator: Send + Sync { /// A [TransactionValidator] implementation that validates ethereum transaction. #[derive(Debug, Clone)] pub struct EthTransactionValidator { - /// Chain id - chain_id: u64, + /// Spec of the chain + chain_spec: Arc, /// This type fetches account info from the db client: Client, /// Fork indicator whether we are in the Shanghai stage. @@ -97,8 +97,32 @@ pub struct EthTransactionValidator { eip1559: bool, /// The current max gas limit current_max_gas_limit: u64, - /// gasprice - gas_price: Option, + /// Current base fee. + base_fee: Option, +} + +// === impl EthTransactionValidator === + +impl EthTransactionValidator { + /// Creates a new instance for the given [ChainSpec] + pub fn new(client: Client, chain_spec: Arc) -> Self { + // TODO(mattsse): improve these settings by checking against hardfork + // See [reth_consensus::validation::validate_transaction_regarding_header] + Self { + chain_spec, + client, + shanghai: true, + eip2718: true, + eip1559: true, + current_max_gas_limit: 30_000_000, + base_fee: None, + } + } + + /// Returns the configured chain id + pub fn chain_id(&self) -> u64 { + self.chain_spec.chain().id() + } } #[async_trait::async_trait] @@ -183,7 +207,7 @@ impl TransactionValidator } // Drop non-local transactions under our own minimal accepted gas price or tip - if !origin.is_local() && transaction.max_fee_per_gas() < self.gas_price { + if !origin.is_local() && transaction.max_fee_per_gas() < self.base_fee { return TransactionValidationOutcome::Invalid( transaction, InvalidTransactionError::MaxFeeLessThenBaseFee.into(), @@ -191,7 +215,7 @@ impl TransactionValidator } // Checks for chainid - if transaction.chain_id() != Some(self.chain_id) { + if transaction.chain_id() != Some(self.chain_id()) { return TransactionValidationOutcome::Invalid( transaction, InvalidTransactionError::ChainIdMismatch.into(), From 9f00d48402e00192e1b27a7afcd1574177985a01 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 16 Mar 2023 15:22:08 +0100 Subject: [PATCH 149/191] chore: rearrange impl order (#1796) --- crates/net/eth-wire/src/types/broadcast.rs | 8 ++-- crates/primitives/src/chain/mod.rs | 12 +++--- crates/primitives/src/genesis.rs | 12 +++--- crates/primitives/src/receipt.rs | 6 +-- crates/rlp/src/encode.rs | 38 +++++++++---------- .../rpc-types/src/eth/transaction/typed.rs | 12 +++--- 6 files changed, 44 insertions(+), 44 deletions(-) diff --git a/crates/net/eth-wire/src/types/broadcast.rs b/crates/net/eth-wire/src/types/broadcast.rs index b7de075db25..c31ce72445f 100644 --- a/crates/net/eth-wire/src/types/broadcast.rs +++ b/crates/net/eth-wire/src/types/broadcast.rs @@ -259,7 +259,7 @@ pub struct NewPooledTransactionHashes68 { } impl Encodable for NewPooledTransactionHashes68 { - fn length(&self) -> usize { + fn encode(&self, out: &mut dyn bytes::BufMut) { #[derive(RlpEncodable)] struct EncodableNewPooledTransactionHashes68<'a> { types: &'a [u8], @@ -273,9 +273,9 @@ impl Encodable for NewPooledTransactionHashes68 { hashes: &self.hashes, }; - encodable.length() + encodable.encode(out); } - fn encode(&self, out: &mut dyn bytes::BufMut) { + fn length(&self) -> usize { #[derive(RlpEncodable)] struct EncodableNewPooledTransactionHashes68<'a> { types: &'a [u8], @@ -289,7 +289,7 @@ impl Encodable for NewPooledTransactionHashes68 { hashes: &self.hashes, }; - encodable.encode(out); + encodable.length() } } diff --git a/crates/primitives/src/chain/mod.rs b/crates/primitives/src/chain/mod.rs index d03aff38aa1..c5ef094f4dc 100644 --- a/crates/primitives/src/chain/mod.rs +++ b/crates/primitives/src/chain/mod.rs @@ -154,18 +154,18 @@ impl FromStr for Chain { } impl Encodable for Chain { - fn length(&self) -> usize { - match self { - Self::Named(chain) => u64::from(*chain).length(), - Self::Id(id) => id.length(), - } - } fn encode(&self, out: &mut dyn reth_rlp::BufMut) { match self { Self::Named(chain) => u64::from(*chain).encode(out), Self::Id(id) => id.encode(out), } } + fn length(&self) -> usize { + match self { + Self::Named(chain) => u64::from(*chain).length(), + Self::Id(id) => id.length(), + } + } } impl Decodable for Chain { diff --git a/crates/primitives/src/genesis.rs b/crates/primitives/src/genesis.rs index fde9db63f14..c9a3f9abc41 100644 --- a/crates/primitives/src/genesis.rs +++ b/crates/primitives/src/genesis.rs @@ -146,12 +146,6 @@ impl GenesisAccount { } impl Encodable for GenesisAccount { - fn length(&self) -> usize { - let len = self.payload_len(); - // RLP header length + payload length - len + length_of_length(len) - } - fn encode(&self, out: &mut dyn bytes::BufMut) { let header = RlpHeader { list: true, payload_length: self.payload_len() }; header.encode(out); @@ -174,6 +168,12 @@ impl Encodable for GenesisAccount { .encode(out); self.code.as_ref().map_or(KECCAK_EMPTY, keccak256).encode(out); } + + fn length(&self) -> usize { + let len = self.payload_len(); + // RLP header length + payload length + len + length_of_length(len) + } } impl From for GenesisAccount { diff --git a/crates/primitives/src/receipt.rs b/crates/primitives/src/receipt.rs index 80761da980b..ae4303856f9 100644 --- a/crates/primitives/src/receipt.rs +++ b/crates/primitives/src/receipt.rs @@ -106,6 +106,9 @@ impl Receipt { } impl Encodable for Receipt { + fn encode(&self, out: &mut dyn BufMut) { + self.encode_inner(out, true) + } fn length(&self) -> usize { let mut payload_len = self.receipt_length(); // account for eip-2718 type prefix and set the list @@ -117,9 +120,6 @@ impl Encodable for Receipt { payload_len } - fn encode(&self, out: &mut dyn BufMut) { - self.encode_inner(out, true) - } } impl Decodable for Receipt { diff --git a/crates/rlp/src/encode.rs b/crates/rlp/src/encode.rs index 98c00d61e8b..0350a76b0cd 100644 --- a/crates/rlp/src/encode.rs +++ b/crates/rlp/src/encode.rs @@ -79,6 +79,13 @@ pub trait Encodable { } impl<'a> Encodable for &'a [u8] { + fn encode(&self, out: &mut dyn BufMut) { + if self.len() != 1 || self[0] >= EMPTY_STRING_CODE { + Header { list: false, payload_length: self.len() }.encode(out); + } + out.put_slice(self); + } + fn length(&self) -> usize { let mut len = self.len(); if self.len() != 1 || self[0] >= EMPTY_STRING_CODE { @@ -86,23 +93,16 @@ impl<'a> Encodable for &'a [u8] { } len } +} +impl Encodable for [u8; LEN] { fn encode(&self, out: &mut dyn BufMut) { - if self.len() != 1 || self[0] >= EMPTY_STRING_CODE { - Header { list: false, payload_length: self.len() }.encode(out); - } - out.put_slice(self); + (self as &[u8]).encode(out) } -} -impl Encodable for [u8; LEN] { fn length(&self) -> usize { (self as &[u8]).length() } - - fn encode(&self, out: &mut dyn BufMut) { - (self as &[u8]).encode(out) - } } unsafe impl MaxEncodedLenAssoc for [u8; LEN] { @@ -164,13 +164,13 @@ encodable_uint!(u128); max_encoded_len_uint!(u128); impl Encodable for bool { - fn length(&self) -> usize { - (*self as u8).length() - } - fn encode(&self, out: &mut dyn BufMut) { (*self as u8).encode(out) } + + fn length(&self) -> usize { + (*self as u8).length() + } } impl_max_encoded_len!(bool, { ::LEN }); @@ -340,13 +340,13 @@ mod alloc_support { where T: Encodable, { - fn length(&self) -> usize { - list_length(self) - } - fn encode(&self, out: &mut dyn BufMut) { encode_list(self, out) } + + fn length(&self) -> usize { + list_length(self) + } } impl Encodable for ::alloc::string::String { @@ -583,7 +583,7 @@ mod tests { "0100020003000400050006000700080009000A0B4B000C000D000E010100020003000400050006000700080009000A0B4B000C000D000E01", 16, ) - .unwrap(), + .unwrap(), &hex!("b8380100020003000400050006000700080009000A0B4B000C000D000E010100020003000400050006000700080009000A0B4B000C000D000E01")[..], )]) } diff --git a/crates/rpc/rpc-types/src/eth/transaction/typed.rs b/crates/rpc/rpc-types/src/eth/transaction/typed.rs index bcca7a0b758..f91903c6966 100644 --- a/crates/rpc/rpc-types/src/eth/transaction/typed.rs +++ b/crates/rpc/rpc-types/src/eth/transaction/typed.rs @@ -83,18 +83,18 @@ impl TransactionKind { } impl Encodable for TransactionKind { - fn length(&self) -> usize { - match self { - TransactionKind::Call(to) => to.length(), - TransactionKind::Create => ([]).length(), - } - } fn encode(&self, out: &mut dyn BufMut) { match self { TransactionKind::Call(to) => to.encode(out), TransactionKind::Create => ([]).encode(out), } } + fn length(&self) -> usize { + match self { + TransactionKind::Call(to) => to.length(), + TransactionKind::Create => ([]).length(), + } + } } impl Decodable for TransactionKind { From 6bd9a25980b53ed803ffe7027280b1ab7c6a6069 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 16 Mar 2023 17:07:55 +0100 Subject: [PATCH 150/191] docs: add some encode docs (#1797) --- crates/rlp/src/encode.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crates/rlp/src/encode.rs b/crates/rlp/src/encode.rs index 0350a76b0cd..0c11b5f8c01 100644 --- a/crates/rlp/src/encode.rs +++ b/crates/rlp/src/encode.rs @@ -70,7 +70,12 @@ macro_rules! impl_max_encoded_len { #[auto_impl(&)] #[cfg_attr(feature = "alloc", auto_impl(Box, Arc))] pub trait Encodable { + /// Appends the rlp encoded object to the specified output buffer. fn encode(&self, out: &mut dyn BufMut); + + /// Returns the length of the encoded object. + /// + /// NOTE: This includes the length of the rlp [Header]. fn length(&self) -> usize { let mut out = BytesMut::new(); self.encode(&mut out); From fe011f1f8cc49a353355d14a507fe1688c30a9d6 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 16 Mar 2023 17:41:16 +0100 Subject: [PATCH 151/191] feat(rpc): enable eth_call eth_estimateGas (#1799) --- crates/rpc/rpc-builder/tests/it/http.rs | 16 ++++++----- crates/rpc/rpc/src/eth/api/call.rs | 12 ++------ crates/rpc/rpc/src/eth/api/server.rs | 31 ++++++++++++++------ crates/rpc/rpc/src/eth/error.rs | 38 +++++++++++++++++++++++-- 4 files changed, 69 insertions(+), 28 deletions(-) diff --git a/crates/rpc/rpc-builder/tests/it/http.rs b/crates/rpc/rpc-builder/tests/it/http.rs index ba3d945648e..1516a97679e 100644 --- a/crates/rpc/rpc-builder/tests/it/http.rs +++ b/crates/rpc/rpc-builder/tests/it/http.rs @@ -82,21 +82,23 @@ where EthApiClient::sign_typed_data(client, address, jsonrpsee::core::JsonValue::Null) .await .unwrap_err(); - EthApiClient::create_access_list(client, call_request.clone(), None).await.unwrap(); EthApiClient::transaction_by_hash(client, tx_hash).await.unwrap(); EthApiClient::transaction_by_block_hash_and_index(client, hash, index).await.unwrap(); EthApiClient::transaction_by_block_number_and_index(client, block_number, index).await.unwrap(); + EthApiClient::create_access_list(client, call_request.clone(), Some(block_number.into())) + .await + .unwrap(); + EthApiClient::estimate_gas(client, call_request.clone(), Some(block_number.into())) + .await + .unwrap(); + EthApiClient::call(client, call_request.clone(), Some(block_number.into()), None) + .await + .unwrap(); // Unimplemented assert!(is_unimplemented(EthApiClient::syncing(client).await.err().unwrap())); assert!(is_unimplemented(EthApiClient::author(client).await.err().unwrap())); assert!(is_unimplemented(EthApiClient::transaction_receipt(client, hash).await.err().unwrap())); - assert!(is_unimplemented( - EthApiClient::call(client, call_request.clone(), None, None).await.err().unwrap() - )); - assert!(is_unimplemented( - EthApiClient::estimate_gas(client, call_request.clone(), None).await.err().unwrap() - )); assert!(is_unimplemented(EthApiClient::gas_price(client).await.err().unwrap())); assert!(is_unimplemented(EthApiClient::max_priority_fee_per_gas(client).await.err().unwrap())); assert!(is_unimplemented(EthApiClient::is_mining(client).await.err().unwrap())); diff --git a/crates/rpc/rpc/src/eth/api/call.rs b/crates/rpc/rpc/src/eth/api/call.rs index 74c0f8338fa..dabd21045ff 100644 --- a/crates/rpc/rpc/src/eth/api/call.rs +++ b/crates/rpc/rpc/src/eth/api/call.rs @@ -39,7 +39,7 @@ where Network: Send + Sync + 'static, { /// Executes the call request at the given [BlockId] - pub(crate) async fn call_at( + pub(crate) async fn execute_call_at( &self, request: CallRequest, at: BlockId, @@ -165,14 +165,8 @@ where ExecutionResult::Success { .. } => { // succeeded } - ExecutionResult::Halt { reason, .. } => { - return match reason { - Halt::OutOfGas(err) => { - Err(InvalidTransactionError::out_of_gas(err, gas_limit).into()) - } - Halt::NonceOverflow => Err(InvalidTransactionError::NonceMaxValue.into()), - err => Err(InvalidTransactionError::EvmHalt(err).into()), - } + ExecutionResult::Halt { reason, gas_used } => { + return Err(InvalidTransactionError::halt(reason, gas_used).into()) } ExecutionResult::Revert { output, .. } => { // if price or limit was included in the request then we can execute the request diff --git a/crates/rpc/rpc/src/eth/api/server.rs b/crates/rpc/rpc/src/eth/api/server.rs index 41f6383a3a1..9a9279f0b7d 100644 --- a/crates/rpc/rpc/src/eth/api/server.rs +++ b/crates/rpc/rpc/src/eth/api/server.rs @@ -5,7 +5,7 @@ use super::EthApiSpec; use crate::{ eth::{ api::{EthApi, EthTransactions}, - error::EthApiError, + error::{ensure_success, EthApiError}, }, result::{internal_rpc_err, ToRpcResult}, }; @@ -179,11 +179,19 @@ where /// Handler for: `eth_call` async fn call( &self, - _request: CallRequest, - _block_number: Option, - _state_overrides: Option, + request: CallRequest, + block_number: Option, + state_overrides: Option, ) -> Result { - Err(internal_rpc_err("unimplemented")) + let (res, _env) = self + .execute_call_at( + request, + block_number.unwrap_or(BlockId::Number(BlockNumberOrTag::Pending)), + state_overrides, + ) + .await?; + + Ok(ensure_success(res.result)?) } /// Handler for: `eth_createAccessList` @@ -192,7 +200,7 @@ where mut request: CallRequest, block_number: Option, ) -> Result { - let block_id = block_number.unwrap_or(BlockId::Number(BlockNumberOrTag::Latest)); + let block_id = block_number.unwrap_or(BlockId::Number(BlockNumberOrTag::Pending)); let access_list = self.create_access_list_at(request.clone(), block_number).await?; request.access_list = Some(access_list.clone()); let gas_used = self.estimate_gas_at(request, block_id).await?; @@ -202,10 +210,15 @@ where /// Handler for: `eth_estimateGas` async fn estimate_gas( &self, - _request: CallRequest, - _block_number: Option, + request: CallRequest, + block_number: Option, ) -> Result { - Err(internal_rpc_err("unimplemented")) + Ok(EthApi::estimate_gas_at( + self, + request, + block_number.unwrap_or(BlockId::Number(BlockNumberOrTag::Pending)), + ) + .await?) } /// Handler for: `eth_gasPrice` diff --git a/crates/rpc/rpc/src/eth/error.rs b/crates/rpc/rpc/src/eth/error.rs index 0ea30bfe5f4..149c3c84506 100644 --- a/crates/rpc/rpc/src/eth/error.rs +++ b/crates/rpc/rpc/src/eth/error.rs @@ -2,10 +2,10 @@ use crate::result::{internal_rpc_err, rpc_err}; use jsonrpsee::{core::Error as RpcError, types::error::INVALID_PARAMS_CODE}; -use reth_primitives::{constants::SELECTOR_LEN, Address, U128, U256}; +use reth_primitives::{constants::SELECTOR_LEN, Address, Bytes, U128, U256}; use reth_rpc_types::{error::EthRpcErrorCode, BlockError}; use reth_transaction_pool::error::{InvalidPoolTransactionError, PoolError}; -use revm::primitives::{EVMError, Halt, OutOfGasError}; +use revm::primitives::{EVMError, ExecutionResult, Halt, OutOfGasError, Output}; /// Result alias pub type EthResult = Result; @@ -191,8 +191,20 @@ impl InvalidTransactionError { } } + /// Converts the halt error + /// + /// Takes the configured gas limit of the transaction which is attached to the error + pub(crate) fn halt(reason: Halt, gas_limit: u64) -> Self { + match reason { + Halt::OutOfGas(err) => InvalidTransactionError::out_of_gas(err, gas_limit), + Halt::NonceOverflow => InvalidTransactionError::NonceMaxValue, + err => InvalidTransactionError::EvmHalt(err), + } + } + /// Converts the out of gas error - pub(crate) fn out_of_gas(reason: OutOfGasError, gas_limit: U256) -> Self { + pub(crate) fn out_of_gas(reason: OutOfGasError, gas_limit: u64) -> Self { + let gas_limit = U256::from(gas_limit); match reason { OutOfGasError::BasicOutOfGas => InvalidTransactionError::BasicOutOfGas(gas_limit), OutOfGasError::Memory => InvalidTransactionError::MemoryOutOfGas(gas_limit), @@ -352,6 +364,26 @@ pub enum SignError { TypedData, } +/// Converts the evm [ExecutionResult] into a result where `Ok` variant is the output bytes if it is +/// [ExecutionResult::Success]. +pub(crate) fn ensure_success(result: ExecutionResult) -> EthResult { + match result { + ExecutionResult::Success { output, .. } => { + let data = match output { + Output::Call(data) => data, + Output::Create(data, _) => data, + }; + Ok(data.into()) + } + ExecutionResult::Revert { output, .. } => { + Err(InvalidTransactionError::Revert(RevertError::new(output)).into()) + } + ExecutionResult::Halt { reason, gas_used } => { + Err(InvalidTransactionError::halt(reason, gas_used).into()) + } + } +} + /// Returns the revert reason from the `revm::TransactOut` data, if it's an abi encoded String. /// /// **Note:** it's assumed the `out` buffer starts with the call's signature From 670db0b433b1d456089b4f3fb6a13c64cd072dbc Mon Sep 17 00:00:00 2001 From: rakita Date: Thu, 16 Mar 2023 19:45:01 +0100 Subject: [PATCH 152/191] feat(BcTree): return inserted block status Accepted/Valid/Disconnected (#1798) --- .../src/blockchain_tree/block_indices.rs | 5 -- crates/executor/src/blockchain_tree/chain.rs | 2 +- crates/executor/src/blockchain_tree/mod.rs | 76 +++++++++++++------ 3 files changed, 52 insertions(+), 31 deletions(-) diff --git a/crates/executor/src/blockchain_tree/block_indices.rs b/crates/executor/src/blockchain_tree/block_indices.rs index 7362316d32d..f752bc873bd 100644 --- a/crates/executor/src/blockchain_tree/block_indices.rs +++ b/crates/executor/src/blockchain_tree/block_indices.rs @@ -52,11 +52,6 @@ impl BlockIndices { &self.blocks_to_chain } - /// Returns `true` if the Tree knowns the block hash. - pub fn contains_pending_block_hash(&self, block_hash: BlockHash) -> bool { - self.blocks_to_chain.contains_key(&block_hash) - } - /// Check if block hash belongs to canonical chain. pub fn is_block_hash_canonical(&self, block_hash: &BlockHash) -> bool { self.canonical_chain.range(self.last_finalized_block..).any(|(_, &h)| h == *block_hash) diff --git a/crates/executor/src/blockchain_tree/chain.rs b/crates/executor/src/blockchain_tree/chain.rs index c5dcf3a9340..cea64b88ccd 100644 --- a/crates/executor/src/blockchain_tree/chain.rs +++ b/crates/executor/src/blockchain_tree/chain.rs @@ -24,7 +24,7 @@ pub struct Chain { } /// Contains fork block and hash. -#[derive(Clone, Copy)] +#[derive(Clone, Copy, Eq, PartialEq)] pub struct ForkBlock { /// Block number of block that chains branches from pub number: u64, diff --git a/crates/executor/src/blockchain_tree/mod.rs b/crates/executor/src/blockchain_tree/mod.rs index 7d3451755a8..9beb70456d3 100644 --- a/crates/executor/src/blockchain_tree/mod.rs +++ b/crates/executor/src/blockchain_tree/mod.rs @@ -73,6 +73,23 @@ pub struct BlockchainTree { externals: TreeExternals, } +/// From Engine API spec, block inclusion can be valid, accepted or invalid. +/// Invalid case is already covered by error but we needs to make distinction +/// between if it is valid (extends canonical chain) or just accepted (is side chain). +/// If we dont know the block parent we are returning Disconnected status +/// as we can't make a claim if block is valid or not. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum BlockStatus { + /// If block validation is valid and block extends canonical chain. + /// In BlockchainTree sense it forks on canonical tip. + Valid, + /// If block validation is valid but block does not extend canonical chain + /// (It is side chain) or hasn't been fully validated but ancestors of a payload are known. + Accepted, + /// If blocks is not connected to canonical chain. + Disconnected, +} + /// Helper structure that wraps chains and indices to search for block hash accross the chains. pub struct BlockHashes<'a> { /// Chains @@ -125,7 +142,7 @@ impl BlockchainTree &mut self, block: SealedBlockWithSenders, chain_id: BlockChainId, - ) -> Result<(), Error> { + ) -> Result { let block_hashes = self.all_chain_hashes(chain_id); // get canonical fork. @@ -165,7 +182,8 @@ impl BlockchainTree &self.externals.executor_factory, )?; drop(provider); - self.block_indices.insert_non_fork_block(block_number, block_hash, chain_id) + self.block_indices.insert_non_fork_block(block_number, block_hash, chain_id); + Ok(BlockStatus::Valid) } else { let chain = parent_chain.new_chain_fork( block, @@ -178,13 +196,15 @@ impl BlockchainTree // release the lifetime with a drop drop(provider); self.insert_chain(chain); + Ok(BlockStatus::Accepted) } - - Ok(()) } /// Fork canonical chain by creating new chain - pub fn fork_canonical_chain(&mut self, block: SealedBlockWithSenders) -> Result<(), Error> { + fn fork_canonical_chain( + &mut self, + block: SealedBlockWithSenders, + ) -> Result { let canonical_block_hashes = self.block_indices.canonical_chain(); let (_, canonical_tip) = canonical_block_hashes.last_key_value().map(|(i, j)| (*i, *j)).unwrap_or_default(); @@ -195,9 +215,12 @@ impl BlockchainTree .header(&block.parent_hash)? .ok_or(ExecError::CanonicalChain { block_hash: block.parent_hash })?; + let block_status; let provider = if block.parent_hash == canonical_tip { + block_status = BlockStatus::Valid; ChainState::boxed(db.latest()?) } else { + block_status = BlockStatus::Accepted; ChainState::boxed(db.history_by_block_number(block.number - 1)?) }; @@ -212,7 +235,7 @@ impl BlockchainTree )?; drop(provider); self.insert_chain(chain); - Ok(()) + Ok(block_status) } /// Get all block hashes from chain that are not canonical. This is one time operation per @@ -273,7 +296,7 @@ impl BlockchainTree /// Insert block inside tree. recover transaction signers and /// internaly call [`BlockchainTree::insert_block_with_senders`] fn. - pub fn insert_block(&mut self, block: SealedBlock) -> Result { + pub fn insert_block(&mut self, block: SealedBlock) -> Result { let block = block.seal_with_senders().ok_or(ExecError::SenderRecoveryError)?; self.insert_block_with_senders(&block) } @@ -290,7 +313,7 @@ impl BlockchainTree pub fn insert_block_with_senders( &mut self, block: &SealedBlockWithSenders, - ) -> Result { + ) -> Result { // check if block number is inside pending block slide let last_finalized_block = self.block_indices.last_finalized_block(); if block.number <= last_finalized_block { @@ -312,38 +335,41 @@ impl BlockchainTree .into()) } - // check if block is already inside Tree - if self.block_indices.contains_pending_block_hash(block.hash()) { - // block is known return that is inserted - return Ok(true) + // check if block known and is already inside Tree + if let Some(chain_id) = self.block_indices.get_blocks_chain_id(&block.hash()) { + let canonical_fork = self.canonical_fork(chain_id).expect("Chain id is valid"); + // if block chain extends canonical chain + if canonical_fork == self.block_indices.canonical_tip() { + return Ok(BlockStatus::Valid) + } else { + return Ok(BlockStatus::Accepted) + } } // check if block is part of canonical chain if self.block_indices.canonical_hash(&block.number) == Some(block.hash()) { // block is part of canonical chain - return Ok(true) + return Ok(BlockStatus::Valid) } // check if block parent can be found in Tree if let Some(parent_chain) = self.block_indices.get_blocks_chain_id(&block.parent_hash) { - self.fork_side_chain(block.clone(), parent_chain)?; + return self.fork_side_chain(block.clone(), parent_chain) // TODO save pending block to database // https://github.com/paradigmxyz/reth/issues/1713 - return Ok(true) } // if not found, check if the parent can be found inside canonical chain. if Some(block.parent_hash) == self.block_indices.canonical_hash(&(block.number - 1)) { // create new chain that points to that block - self.fork_canonical_chain(block.clone())?; + return self.fork_canonical_chain(block.clone()) // TODO save pending block to database // https://github.com/paradigmxyz/reth/issues/1713 - return Ok(true) } // NOTE: Block doesn't have a parent, and if we receive this block in `make_canonical` // function this could be a trigger to initiate p2p syncing, as we are missing the // parent. - Ok(false) + Ok(BlockStatus::Disconnected) } /// Do finalization of blocks. Remove them from tree @@ -391,7 +417,7 @@ impl BlockchainTree Ok(()) } - /// Split chain and return canonical part of it. Pending part reinsert inside tree + /// Split chain and return canonical part of it. Pending part is reinserted inside tree /// with same chain_id. fn split_chain(&mut self, chain_id: BlockChainId, chain: Chain, split_at: SplitAt) -> Chain { match chain.split(split_at) { @@ -727,15 +753,15 @@ mod tests { tree.finalize_block(10); // block 2 parent is not known. - assert_eq!(tree.insert_block_with_senders(&block2), Ok(false)); + assert_eq!(tree.insert_block_with_senders(&block2), Ok(BlockStatus::Disconnected)); // insert block1 - assert_eq!(tree.insert_block_with_senders(&block1), Ok(true)); + assert_eq!(tree.insert_block_with_senders(&block1), Ok(BlockStatus::Valid)); // already inserted block will return true. - assert_eq!(tree.insert_block_with_senders(&block1), Ok(true)); + assert_eq!(tree.insert_block_with_senders(&block1), Ok(BlockStatus::Valid)); // insert block2 - assert_eq!(tree.insert_block_with_senders(&block2), Ok(true)); + assert_eq!(tree.insert_block_with_senders(&block2), Ok(BlockStatus::Valid)); // Trie state: // b2 (pending block) @@ -780,7 +806,7 @@ mod tests { block2a.hash = block2a_hash; // reinsert two blocks that point to canonical chain - assert_eq!(tree.insert_block_with_senders(&block1a), Ok(true)); + assert_eq!(tree.insert_block_with_senders(&block1a), Ok(BlockStatus::Accepted)); TreeTester::default() .with_chain_num(1) @@ -791,7 +817,7 @@ mod tests { )])) .assert(&tree); - assert_eq!(tree.insert_block_with_senders(&block2a), Ok(true)); + assert_eq!(tree.insert_block_with_senders(&block2a), Ok(BlockStatus::Accepted)); // Trie state: // b2 b2a (side chain) // | / From 7f5ac990eb0ac78975be45177513be72c3b5db93 Mon Sep 17 00:00:00 2001 From: Bjerg Date: Fri, 17 Mar 2023 00:42:59 +0100 Subject: [PATCH 153/191] perf: write friendly `ExecutionResult` (#1674) Co-authored-by: Georgios Konstantopoulos --- .../src/blockchain_tree/block_indices.rs | 9 +- crates/executor/src/blockchain_tree/chain.rs | 283 +++--- crates/executor/src/blockchain_tree/config.rs | 9 +- .../executor/src/blockchain_tree/externals.rs | 19 +- crates/executor/src/blockchain_tree/mod.rs | 196 +++-- crates/executor/src/executor.rs | 536 ++++++------ crates/executor/src/lib.rs | 4 +- crates/executor/src/substate.rs | 287 +----- crates/stages/src/stages/execution.rs | 11 +- .../storage/provider/src/execution_result.rs | 206 ----- crates/storage/provider/src/lib.rs | 2 +- crates/storage/provider/src/post_state.rs | 828 ++++++++++++++++++ .../storage/provider/src/test_utils/blocks.rs | 94 +- .../storage/provider/src/traits/executor.rs | 6 +- crates/storage/provider/src/transaction.rs | 473 +++++----- 15 files changed, 1703 insertions(+), 1260 deletions(-) delete mode 100644 crates/storage/provider/src/execution_result.rs create mode 100644 crates/storage/provider/src/post_state.rs diff --git a/crates/executor/src/blockchain_tree/block_indices.rs b/crates/executor/src/blockchain_tree/block_indices.rs index f752bc873bd..ba5c63fa9f2 100644 --- a/crates/executor/src/blockchain_tree/block_indices.rs +++ b/crates/executor/src/blockchain_tree/block_indices.rs @@ -4,11 +4,12 @@ use super::chain::{BlockChainId, Chain, ForkBlock}; use reth_primitives::{BlockHash, BlockNumber, SealedBlockWithSenders}; use std::collections::{hash_map::Entry, BTreeMap, BTreeSet, HashMap, HashSet}; -/// Internal indices of the blocks and chains. This is main connection -/// between blocks, chains and canonical chain. +/// Internal indices of the blocks and chains. /// -/// It contains list of canonical block hashes, forks to childs blocks -/// and block hash to chain id. +/// This is main connection between blocks, chains and canonical chain. +/// +/// It contains a list of canonical block hashes, forks to child blocks, and a mapping of block hash +/// to chain ID. #[derive(Debug)] pub struct BlockIndices { /// Last finalized block. diff --git a/crates/executor/src/blockchain_tree/chain.rs b/crates/executor/src/blockchain_tree/chain.rs index cea64b88ccd..3f8440d6457 100644 --- a/crates/executor/src/blockchain_tree/chain.rs +++ b/crates/executor/src/blockchain_tree/chain.rs @@ -1,29 +1,39 @@ -//! Handles substate and list of blocks. -//! have functions to split, branch and append the chain. -use crate::{ - execution_result::ExecutionResult, - substate::{SubStateData, SubStateWithProvider}, -}; +//! A chain in a [`BlockchainTree`][super::BlockchainTree]. +//! +//! A [`Chain`] contains the state of accounts for the chain after execution of its constituent +//! blocks, as well as a list of the blocks the chain is composed of. +use crate::{post_state::PostState, substate::PostStateProvider}; use reth_interfaces::{consensus::Consensus, executor::Error as ExecError, Error}; use reth_primitives::{BlockHash, BlockNumber, SealedBlockWithSenders, SealedHeader, U256}; use reth_provider::{BlockExecutor, ExecutorFactory, StateProvider}; use std::collections::BTreeMap; -/// Internal to BlockchainTree chain identification. +/// The ID of a sidechain internally in a [`BlockchainTree`][super::BlockchainTree]. pub(crate) type BlockChainId = u64; -/// Side chain that contain it state and connect to block found in canonical chain. +/// A side chain. +/// +/// The sidechain contains the state of accounts after execution of its blocks, +/// changesets for those blocks (and their transactions), as well as the blocks themselves. +/// +/// Each chain in the tree are identified using a unique ID. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct Chain { - /// Chain substate. Updated state after execution all blocks in chain. - substate: SubStateData, - /// Changesets for block and transaction. Will be used to update tables in database. - changesets: Vec, - /// Blocks in this chain + /// The state of accounts after execution of the blocks in this chain. + /// + /// This state also contains the individual changes that lead to the current state. + state: PostState, + /// The blocks in this chain. blocks: BTreeMap, + /// A mapping of each block number in the chain to the highest transition ID in the chain's + /// state after execution of the block. + /// + /// This is used to revert changes in the state until a certain block number when the chain is + /// split. + block_transitions: BTreeMap, } -/// Contains fork block and hash. +/// Describes a fork block by its number and hash. #[derive(Clone, Copy, Eq, PartialEq)] pub struct ForkBlock { /// Block number of block that chains branches from @@ -33,75 +43,68 @@ pub struct ForkBlock { } impl ForkBlock { - /// Return the number hash tuple. + /// Return the `(block_number, block_hash)` tuple for this fork block. pub fn num_hash(&self) -> (BlockNumber, BlockHash) { (self.number, self.hash) } } impl Chain { - /// Return blocks found in chain + /// Get the blocks in this chain. pub fn blocks(&self) -> &BTreeMap { &self.blocks } - /// Into inner components - pub fn into_inner( - self, - ) -> (BTreeMap, Vec, SubStateData) { - (self.blocks, self.changesets, self.substate) + /// Destructure the chain into its inner components, the blocks and the state. + pub fn into_inner(self) -> (BTreeMap, PostState) { + (self.blocks, self.state) } - /// Return execution results of blocks - pub fn changesets(&self) -> &Vec { - &self.changesets - } - - /// Return fork block number and hash. + /// Get the block at which this chain forked. pub fn fork_block(&self) -> ForkBlock { let tip = self.first(); ForkBlock { number: tip.number.saturating_sub(1), hash: tip.parent_hash } } - /// Block fork number + /// Get the block number at which this chain forked. pub fn fork_block_number(&self) -> BlockNumber { self.first().number.saturating_sub(1) } - /// Block fork hash + /// Get the block hash at which this chain forked. pub fn fork_block_hash(&self) -> BlockHash { self.first().parent_hash } - /// First block in chain. + /// Get the first block in this chain. pub fn first(&self) -> &SealedBlockWithSenders { self.blocks.first_key_value().expect("Chain has at least one block for first").1 } - /// Return tip of the chain. Chain always have at least one block inside + /// Get the tip of the chain. + /// + /// # Note + /// + /// Chains always have at least one block. pub fn tip(&self) -> &SealedBlockWithSenders { - self.last() - } - - /// Return tip of the chain. Chain always have at least one block inside - pub fn last(&self) -> &SealedBlockWithSenders { - self.blocks.last_key_value().expect("Chain has at least one block for last").1 + self.blocks.last_key_value().expect("Chain should have at least one block").1 } - /// Create new chain with given blocks and execution result. - pub fn new(blocks: Vec<(SealedBlockWithSenders, ExecutionResult)>) -> Self { - let (blocks, changesets): (Vec<_>, Vec<_>) = blocks.into_iter().unzip(); - - let blocks = blocks.into_iter().map(|b| (b.number, b)).collect::>(); - - let mut substate = SubStateData::default(); - substate.apply(&changesets); + /// Create new chain with given blocks and post state. + pub fn new(blocks: Vec<(SealedBlockWithSenders, PostState)>) -> Self { + let mut state = PostState::default(); + let mut block_transitions = BTreeMap::new(); + let mut block_num_hash = BTreeMap::new(); + for (block, block_state) in blocks.into_iter() { + state.extend(block_state); + block_transitions.insert(block.number, state.transitions_count()); + block_num_hash.insert(block.number, block); + } - Self { substate, changesets, blocks } + Self { state, block_transitions, blocks: block_num_hash } } - /// Create new chain that joins canonical block - /// If parent block is the tip mark chain fork. + /// Create a new chain that forks off of the canonical chain. pub fn new_canonical_fork( block: &SealedBlockWithSenders, parent_header: &SealedHeader, @@ -110,17 +113,16 @@ impl Chain { consensus: &C, factory: &EF, ) -> Result { - // substate - let substate = SubStateData::default(); + let state = PostState::default(); let empty = BTreeMap::new(); - let substate_with_sp = - SubStateWithProvider::new(&substate, provider, &empty, canonical_block_hashes); + let state_provider = + PostStateProvider::new(&state, provider, &empty, canonical_block_hashes); let changeset = Self::validate_and_execute( block.clone(), parent_header, - substate_with_sp, + state_provider, consensus, factory, )?; @@ -128,7 +130,7 @@ impl Chain { Ok(Self::new(vec![(block.clone(), changeset)])) } - /// Create new chain that branches out from existing side chain. + /// Create a new chain that forks off of an existing sidechain. pub fn new_chain_fork( &self, block: SealedBlockWithSenders, @@ -144,61 +146,58 @@ impl Chain { .get(&parent_number) .ok_or(ExecError::BlockNumberNotFoundInChain { block_number: parent_number })?; - // revert changesets - let revert_from = self.changesets.len() - (self.tip().number - parent.number) as usize; - let mut substate = self.substate.clone(); + let revert_to_transition_id = self + .block_transitions + .get(&parent.number) + .expect("Should have the transition ID for the parent block"); + let mut state = self.state.clone(); - // Revert changesets to get the state of the parent that we need to apply the change. - substate.revert(&self.changesets[revert_from..]); + // Revert state to the state after execution of the parent block + state.revert_to(*revert_to_transition_id); - let substate_with_sp = SubStateWithProvider::new( - &substate, + // Revert changesets to get the state of the parent that we need to apply the change. + let state_provider = PostStateProvider::new( + &state, provider, &side_chain_block_hashes, canonical_block_hashes, ); - let changeset = Self::validate_and_execute( - block.clone(), - parent, - substate_with_sp, - consensus, - factory, - )?; - substate.apply_one(&changeset); + let block_state = + Self::validate_and_execute(block.clone(), parent, state_provider, consensus, factory)?; + state.extend(block_state); let chain = Self { - substate, - changesets: vec![changeset], + block_transitions: BTreeMap::from([(block.number, state.transitions_count())]), + state, blocks: BTreeMap::from([(block.number, block)]), }; - // if all is okay, return new chain back. Present chain is not modified. + // If all is okay, return new chain back. Present chain is not modified. Ok(chain) } - /// Validate and execute block and return execution result or error. + /// Validate and execute the given block. fn validate_and_execute( block: SealedBlockWithSenders, parent_block: &SealedHeader, - substate: SubStateWithProvider<'_, SP>, + state_provider: PostStateProvider<'_, SP>, consensus: &C, factory: &EF, - ) -> Result { + ) -> Result { consensus.validate_header(&block, U256::MAX)?; consensus.pre_validate_header(&block, parent_block)?; consensus.pre_validate_block(&block)?; let (unseal, senders) = block.into_components(); let unseal = unseal.unseal(); - let res = factory.with_sp(substate).execute_and_verify_receipt( - &unseal, - U256::MAX, - Some(senders), - )?; - Ok(res) + + factory + .with_sp(state_provider) + .execute_and_verify_receipt(&unseal, U256::MAX, Some(senders)) + .map_err(Into::into) } - /// Append block to this chain + /// Validate and execute the given block, and append it to this chain. pub fn append_block( &mut self, block: SealedBlockWithSenders, @@ -210,11 +209,11 @@ impl Chain { ) -> Result<(), Error> { let (_, parent_block) = self.blocks.last_key_value().expect("Chain has at least one block"); - let changeset = Self::validate_and_execute( + let block_state = Self::validate_and_execute( block.clone(), parent_block, - SubStateWithProvider::new( - &self.substate, + PostStateProvider::new( + &self.state, provider, &side_chain_block_hashes, canonical_block_hashes, @@ -222,14 +221,15 @@ impl Chain { consensus, factory, )?; - self.substate.apply_one(&changeset); - self.changesets.push(changeset); + self.state.extend(block_state); + self.block_transitions.insert(block.number, self.state.transitions_count()); self.blocks.insert(block.number, block); Ok(()) } - /// Merge two chains into one by appending received chain to the current one. - /// Take substate from newest one. + /// Merge two chains by appending the given chain into the current one. + /// + /// The state of accounts for this chain is set to the state of the newest chain. pub fn append_chain(&mut self, chain: Chain) -> Result<(), Error> { let chain_tip = self.tip(); if chain_tip.hash != chain.fork_block_hash() { @@ -239,19 +239,36 @@ impl Chain { } .into()) } + + // Insert blocks from other chain self.blocks.extend(chain.blocks.into_iter()); - self.changesets.extend(chain.changesets.into_iter()); - self.substate = chain.substate; + let current_transition_count = self.state.transitions_count(); + self.state.extend(chain.state); + + // Update the block transition mapping, shifting the transition ID by the current number of + // transitions in *this* chain + for (block_number, transition_id) in chain.block_transitions.iter() { + self.block_transitions.insert(*block_number, transition_id + current_transition_count); + } Ok(()) } + /// Split this chain at the given block. + /// + /// The given block will be the first block in the first returned chain. + /// + /// If the given block is not found, [`ChainSplit::NoSplitPending`] is returned. /// Split chain at the number or hash, block with given number will be included at first chain. /// If any chain is empty (Does not have blocks) None will be returned. /// - /// If block hash is not found ChainSplit::NoSplitPending is returned. + /// # Note /// - /// Subtate state will be only found in second chain. First change substate will be - /// invalid. + /// The block number to transition ID mapping is only found in the second chain, making it + /// impossible to perform any state reverts on the first chain. + /// + /// The second chain only contains the changes that were reverted on the first chain; however, + /// it retains the up to date state as if the chains were one, i.e. the second chain is an + /// extension of the first. pub fn split(mut self, split_at: SplitAt) -> ChainSplit { let chain_tip = *self.blocks.last_entry().expect("chain is never empty").key(); let block_number = match split_at { @@ -282,17 +299,22 @@ impl Chain { }; let higher_number_blocks = self.blocks.split_off(&(block_number + 1)); - let (first_changesets, second_changeset) = self.changesets.split_at(self.blocks.len()); + + let mut canonical_state = std::mem::take(&mut self.state); + let new_state = canonical_state.split_at( + *self.block_transitions.get(&(block_number)).expect("Unknown block transition ID"), + ); + self.state = new_state; ChainSplit::Split { canonical: Chain { - substate: SubStateData::default(), - changesets: first_changesets.to_vec(), + state: canonical_state, + block_transitions: BTreeMap::new(), blocks: self.blocks, }, pending: Chain { - substate: self.substate, - changesets: second_changeset.to_vec(), + state: self.state, + block_transitions: self.block_transitions, blocks: higher_number_blocks, }, } @@ -333,12 +355,10 @@ pub enum ChainSplit { #[cfg(test)] mod tests { use super::*; - use crate::substate::AccountSubState; - use reth_primitives::{H160, H256}; - use reth_provider::execution_result::AccountInfoChangeSet; + use reth_primitives::{Account, H160, H256}; #[test] - fn chain_apend() { + fn chain_append() { let block = SealedBlockWithSenders::default(); let block1_hash = H256([0x01; 32]); let block2_hash = H256([0x02; 32]); @@ -357,17 +377,11 @@ mod tests { block3.block.header.header.parent_hash = block2_hash; - let mut chain1 = Chain { - substate: Default::default(), - changesets: vec![], - blocks: BTreeMap::from([(1, block1), (2, block2)]), - }; + let mut chain1 = + Chain { blocks: BTreeMap::from([(1, block1), (2, block2)]), ..Default::default() }; - let chain2 = Chain { - substate: Default::default(), - changesets: vec![], - blocks: BTreeMap::from([(3, block3), (4, block4)]), - }; + let chain2 = + Chain { blocks: BTreeMap::from([(3, block3), (4, block4)]), ..Default::default() }; assert_eq!(chain1.append_chain(chain2.clone()), Ok(())); @@ -377,41 +391,49 @@ mod tests { #[test] fn test_number_split() { - let mut substate = SubStateData::default(); - let mut account = AccountSubState::default(); - account.info.nonce = 10; - substate.accounts.insert(H160([1; 20]), account); + let mut base_state = PostState::default(); + let mut account = Account::default(); + account.nonce = 10; + base_state.create_account(H160([1; 20]), account); + base_state.finish_transition(); - let mut exec1 = ExecutionResult::default(); - exec1.block_changesets.insert(H160([2; 20]), AccountInfoChangeSet::default()); - let mut exec2 = ExecutionResult::default(); - exec2.block_changesets.insert(H160([3; 20]), AccountInfoChangeSet::default()); + let mut block_state1 = PostState::default(); + block_state1.create_account(H160([2; 20]), Account::default()); + block_state1.finish_transition(); + + let mut block_state2 = PostState::default(); + block_state2.create_account(H160([3; 20]), Account::default()); + block_state2.finish_transition(); let mut block1 = SealedBlockWithSenders::default(); let block1_hash = H256([15; 32]); + block1.number = 1; block1.hash = block1_hash; block1.senders.push(H160([4; 20])); let mut block2 = SealedBlockWithSenders::default(); let block2_hash = H256([16; 32]); + block2.number = 2; block2.hash = block2_hash; block2.senders.push(H160([4; 20])); - let chain = Chain { - substate: substate.clone(), - changesets: vec![exec1.clone(), exec2.clone()], - blocks: BTreeMap::from([(1, block1.clone()), (2, block2.clone())]), - }; + let chain = Chain::new(vec![ + (block1.clone(), block_state1.clone()), + (block2.clone(), block_state2.clone()), + ]); + + let mut split1_state = chain.state.clone(); + let split2_state = split1_state.split_at(*chain.block_transitions.get(&1).unwrap()); let chain_split1 = Chain { - substate: SubStateData::default(), - changesets: vec![exec1], + state: split1_state, + block_transitions: BTreeMap::new(), blocks: BTreeMap::from([(1, block1.clone())]), }; let chain_split2 = Chain { - substate, - changesets: vec![exec2.clone()], + state: split2_state, + block_transitions: chain.block_transitions.clone(), blocks: BTreeMap::from([(2, block2.clone())]), }; @@ -432,6 +454,7 @@ mod tests { chain.clone().split(SplitAt::Number(10)), ChainSplit::NoSplitCanonical(chain.clone()) ); + // split at lower number assert_eq!(chain.clone().split(SplitAt::Number(0)), ChainSplit::NoSplitPending(chain)); } diff --git a/crates/executor/src/blockchain_tree/config.rs b/crates/executor/src/blockchain_tree/config.rs index 3de52c11049..d830f066fd0 100644 --- a/crates/executor/src/blockchain_tree/config.rs +++ b/crates/executor/src/blockchain_tree/config.rs @@ -3,11 +3,12 @@ /// The configuration for the blockchain tree. #[derive(Clone, Debug)] pub struct BlockchainTreeConfig { - /// Finalization windows. Number of blocks that can be reorged - max_reorg_depth: u64, - /// Number of block after finalized block that we are storing. It should be more then - /// finalization window + /// Number of blocks after the last finalized block that we are storing. + /// + /// It should be more than the finalization window for the canonical chain. max_blocks_in_chain: u64, + /// The number of blocks that can be re-orged (finalization windows) + max_reorg_depth: u64, /// For EVM's "BLOCKHASH" opcode we require last 256 block hashes. So we need to specify /// at least `additional_canonical_block_hashes`+`max_reorg_depth`, for eth that would be /// 256+64. diff --git a/crates/executor/src/blockchain_tree/externals.rs b/crates/executor/src/blockchain_tree/externals.rs index 72e31f4fb39..8df3f590f9d 100644 --- a/crates/executor/src/blockchain_tree/externals.rs +++ b/crates/executor/src/blockchain_tree/externals.rs @@ -5,15 +5,24 @@ use reth_primitives::ChainSpec; use reth_provider::ShareableDatabase; use std::sync::Arc; -/// Container for external abstractions. +/// A container for external components. +/// +/// This is a simple container for external components used throughout the blockchain tree +/// implementation: +/// +/// - A handle to the database +/// - A handle to the consensus engine +/// - The executor factory to exexcute blocks with +/// - The chain spec +#[derive(Debug)] pub struct TreeExternals { - /// Save sidechain, do reorgs and push new block to canonical chain that is inside db. + /// The database, used to commit the canonical chain, or unwind it. pub db: DB, - /// Consensus checks + /// The consensus engine. pub consensus: C, - /// Create executor to execute blocks. + /// The executor factory to execute blocks with. pub executor_factory: EF, - /// Chain spec + /// The chain spec. pub chain_spec: Arc, } diff --git a/crates/executor/src/blockchain_tree/mod.rs b/crates/executor/src/blockchain_tree/mod.rs index 9beb70456d3..0f89b7684cf 100644 --- a/crates/executor/src/blockchain_tree/mod.rs +++ b/crates/executor/src/blockchain_tree/mod.rs @@ -6,7 +6,10 @@ use reth_primitives::{BlockHash, BlockNumber, SealedBlock, SealedBlockWithSender use reth_provider::{ providers::ChainState, ExecutorFactory, HeaderProvider, StateProviderFactory, Transaction, }; -use std::collections::{BTreeMap, HashMap}; +use std::{ + collections::{BTreeMap, HashMap}, + ops::DerefMut, +}; pub mod block_indices; use block_indices::BlockIndices; @@ -60,17 +63,18 @@ use externals::TreeExternals; /// * [BlockchainTree::make_canonical]: Check if we have the hash of block that we want to finalize /// and commit it to db. If we dont have the block, pipeline syncing should start to fetch the /// blocks from p2p. Do reorg in tables if canonical chain if needed. +#[derive(Debug)] pub struct BlockchainTree { - /// chains and present data + /// The tracked chains and their current data. chains: HashMap, - /// Static blockchain id generator + /// Static blockchain ID generator block_chain_id_generator: u64, - /// Indices to block and their connection. + /// Indices to block and their connection to the canonical chain. block_indices: BlockIndices, - /// Tree configuration. - config: BlockchainTreeConfig, - /// Externals + /// External components (the database, consensus engine etc.) externals: TreeExternals, + /// Tree configuration + config: BlockchainTreeConfig, } /// From Engine API spec, block inclusion can be valid, accepted or invalid. @@ -90,16 +94,17 @@ pub enum BlockStatus { Disconnected, } -/// Helper structure that wraps chains and indices to search for block hash accross the chains. +/// A container that wraps chains and block indices to allow searching for block hashes across all +/// sidechains. pub struct BlockHashes<'a> { - /// Chains + /// The current tracked chains. pub chains: &'a mut HashMap, - /// Indices + /// The block indices for all chains. pub indices: &'a BlockIndices, } impl BlockchainTree { - /// New blockchain tree + /// Create a new blockchain tree. pub fn new( externals: TreeExternals, config: BlockchainTreeConfig, @@ -137,7 +142,8 @@ impl BlockchainTree }) } - /// Fork side chain or append the block if parent is the top of the chain + /// Create a new sidechain by forking the given chain, or append the block if the parent block + /// is the top of the given chain. fn fork_side_chain( &mut self, block: SealedBlockWithSenders, @@ -200,8 +206,9 @@ impl BlockchainTree } } - /// Fork canonical chain by creating new chain - fn fork_canonical_chain( + /// Create a new sidechain by forking the canonical chain. + // TODO(onbjerg): Is this not a specialized case of [`fork_side_chain`]? If so, can we merge? + pub fn fork_canonical_chain( &mut self, block: SealedBlockWithSenders, ) -> Result { @@ -238,8 +245,13 @@ impl BlockchainTree Ok(block_status) } - /// Get all block hashes from chain that are not canonical. This is one time operation per - /// block. Reason why this is not caches is to save memory. + /// Get all block hashes from a sidechain that are not part of the canonical chain. + /// + /// This is a one time operation per block. + /// + /// # Note + /// + /// This is not cached in order to save memory. fn all_chain_hashes(&self, chain_id: BlockChainId) -> BTreeMap { // find chain and iterate over it, let mut chain_id = chain_id; @@ -260,9 +272,12 @@ impl BlockchainTree hashes } - /// Getting the canonical fork would tell use what kind of Provider we should execute block on. - /// If it is latest state provider or history state provider - /// Return None if chain_id is not known. + /// Get the block at which the given chain forked from the current canonical chain. + /// + /// This is used to figure out what kind of state provider the executor should use to execute + /// the block. + /// + /// Returns `None` if the chain is not known. fn canonical_fork(&self, chain_id: BlockChainId) -> Option { let mut chain_id = chain_id; let mut fork; @@ -283,8 +298,9 @@ impl BlockchainTree } } - /// Insert chain to tree and ties the blocks to it. - /// Helper function that handles indexing and inserting. + /// Insert a chain into the tree. + /// + /// Inserts a chain into the tree and builds the block indices. fn insert_chain(&mut self, chain: Chain) -> BlockChainId { let chain_id = self.block_chain_id_generator; self.block_chain_id_generator += 1; @@ -294,22 +310,35 @@ impl BlockchainTree chain_id } - /// Insert block inside tree. recover transaction signers and - /// internaly call [`BlockchainTree::insert_block_with_senders`] fn. + /// Insert a new block in the tree. + /// + /// # Note + /// + /// This recovers transaction signers (unlike [`BlockchainTree::insert_block_with_senders`]). pub fn insert_block(&mut self, block: SealedBlock) -> Result { let block = block.seal_with_senders().ok_or(ExecError::SenderRecoveryError)?; self.insert_block_with_senders(&block) } - /// Insert block with senders inside tree. + /// Insert a block (with senders recovered) in the tree. + /// /// Returns `true` if: - /// 1. It is part of the blockchain tree - /// 2. It is part of the canonical chain - /// 3. Its parent is part of the blockchain tree and we can fork at the parent - /// 4. Its parent is part of the canonical chain and we can fork at the parent - /// Otherwise will return `false`, indicating that neither the block nor its parent - /// is part of the chain or any sidechains. This means that if block becomes canonical - /// we need to fetch the missing blocks over p2p. + /// + /// - The block is already part of a sidechain in the tree, or + /// - The block is already part of the canonical chain, or + /// - The parent is part of a sidechain in the tree, and we can fork at this block, or + /// - The parent is part of the canonical chain, and we can fork at this block + /// + /// Otherwise `false` is returned, indicating that neither the block nor its parent is part of + /// the chain or any sidechains. + /// + /// This means that if the block becomes canonical, we need to fetch the missing blocks over + /// P2P. + /// + /// # Note + /// + /// If the senders have not already been recovered, call [`BlockchainTree::insert_block`] + /// instead. pub fn insert_block_with_senders( &mut self, block: &SealedBlockWithSenders, @@ -372,7 +401,7 @@ impl BlockchainTree Ok(BlockStatus::Disconnected) } - /// Do finalization of blocks. Remove them from tree + /// Finalize blocks up until and including `finalized_block`, and remove them from the tree. pub fn finalize_block(&mut self, finalized_block: BlockNumber) { let mut remove_chains = self.block_indices.finalize_canonical_blocks( finalized_block, @@ -386,7 +415,16 @@ impl BlockchainTree } } - /// Update canonical hashes. Reads last N canonical blocks from database and update all indices. + /// Reads the last `N` canonical hashes from the database and updates the block indices of the + /// tree. + /// + /// `N` is the `max_reorg_depth` plus the number of block hashes needed to satisfy the + /// `BLOCKHASH` opcode in the EVM. + /// + /// # Note + /// + /// This finalizes `last_finalized_block` prior to reading the canonical hashes (using + /// [`BlockchainTree::finalize_block`]). pub fn restore_canonical_hashes( &mut self, last_finalized_block: BlockNumber, @@ -417,8 +455,9 @@ impl BlockchainTree Ok(()) } - /// Split chain and return canonical part of it. Pending part is reinserted inside tree - /// with same chain_id. + /// Split a sidechain at the given point, and return the canonical part of it. + /// + /// The pending part of the chain is reinserted into the tree with the same `chain_id`. fn split_chain(&mut self, chain_id: BlockChainId, chain: Chain, split_at: SplitAt) -> Chain { match chain.split(split_at) { ChainSplit::Split { canonical, pending } => { @@ -434,9 +473,16 @@ impl BlockchainTree } } - /// Make block and its parent canonical. Unwind chains to database if necessary. + /// Make a block and its parent part of the canonical chain. + /// + /// # Note + /// + /// This unwinds the database if necessary, i.e. if parts of the canonical chain have been + /// re-orged. /// - /// If block is already part of canonical chain return Ok. + /// # Returns + /// + /// Returns `Ok` if the blocks were canonicalized, or if the blocks were already canonical. pub fn make_canonical(&mut self, block_hash: &BlockHash) -> Result<(), Error> { let chain_id = if let Some(chain_id) = self.block_indices.get_blocks_chain_id(block_hash) { chain_id @@ -498,19 +544,41 @@ impl BlockchainTree Ok(()) } - /// Commit chain for it to become canonical. Assume we are doing pending operation to db. + /// Canonicalize the given chain and commit it to the database. fn commit_canonical(&mut self, chain: Chain) -> Result<(), Error> { let mut tx = Transaction::new(&self.externals.db)?; - - let new_tip = chain.tip().number; - let (blocks, changesets, _) = chain.into_inner(); - for item in blocks.into_iter().zip(changesets.into_iter()) { - let ((_, block), changeset) = item; - tx.insert_block(block, self.externals.chain_spec.as_ref(), changeset) + let new_tip_number = chain.tip().number; + let new_tip_hash = chain.tip().hash; + let first_transition_id = + tx.get_block_transition(chain.first().number.saturating_sub(1)) + .map_err(|e| ExecError::CanonicalCommit { inner: e.to_string() })?; + let expected_state_root = chain.tip().state_root; + let fork_block = chain.fork_block_number(); + let (blocks, state) = chain.into_inner(); + let num_transitions = state.transitions_count(); + + // Write state and changesets to the database + state + .write_to_db(tx.deref_mut(), first_transition_id) + .map_err(|e| ExecError::CanonicalCommit { inner: e.to_string() })?; + + // Insert the blocks + for block in blocks.into_values() { + tx.insert_block(block) .map_err(|e| ExecError::CanonicalCommit { inner: e.to_string() })?; } - // update pipeline progress. - tx.update_pipeline_stages(new_tip) + tx.insert_hashes( + fork_block, + first_transition_id, + first_transition_id + num_transitions as u64, + new_tip_number, + new_tip_hash, + expected_state_root, + ) + .map_err(|e| ExecError::CanonicalCommit { inner: e.to_string() })?; + + // Update pipeline progress + tx.update_pipeline_stages(new_tip_number) .map_err(|e| ExecError::PipelineStatusUpdate { inner: e.to_string() })?; tx.commit()?; @@ -538,9 +606,9 @@ impl BlockchainTree Ok(()) } - /// Revert canonical blocks from database and insert them to pending table - /// Revert should be non inclusive, and revert_until should stay in db. - /// Return the chain that represent reverted canonical blocks. + /// Revert canonical blocks from the database and return them. + /// + /// The block, `revert_until`, is non-inclusive, i.e. `revert_until` stays in the database. fn revert_canonical(&mut self, revert_until: BlockNumber) -> Result { // read data that is needed for new sidechain @@ -560,9 +628,7 @@ impl BlockchainTree tx.commit()?; - let chain = Chain::new(blocks_and_execution); - - Ok(chain) + Ok(Chain::new(blocks_and_execution)) } } @@ -575,18 +641,16 @@ mod tests { transaction::DbTxMut, }; use reth_interfaces::test_utils::TestConsensus; - use reth_primitives::{ - hex_literal::hex, proofs::EMPTY_ROOT, ChainSpec, ChainSpecBuilder, H256, MAINNET, - }; + use reth_primitives::{proofs::EMPTY_ROOT, ChainSpec, ChainSpecBuilder, H256, MAINNET}; use reth_provider::{ - execution_result::ExecutionResult, insert_block, test_utils::blocks::BlockChainTestData, - BlockExecutor, StateProvider, + insert_block, post_state::PostState, test_utils::blocks::BlockChainTestData, BlockExecutor, + StateProvider, }; use std::{collections::HashSet, sync::Arc}; - #[derive(Clone)] + #[derive(Clone, Debug)] struct TestFactory { - exec_result: Arc>>, + exec_result: Arc>>, chain_spec: Arc, } @@ -595,12 +659,12 @@ mod tests { Self { exec_result: Arc::new(Mutex::new(Vec::new())), chain_spec } } - fn extend(&self, exec_res: Vec) { + fn extend(&self, exec_res: Vec) { self.exec_result.lock().extend(exec_res.into_iter()); } } - struct TestExecutor(Option); + struct TestExecutor(Option); impl BlockExecutor for TestExecutor { fn execute( @@ -608,7 +672,7 @@ mod tests { _block: &reth_primitives::Block, _total_difficulty: reth_primitives::U256, _senders: Option>, - ) -> Result { + ) -> Result { self.0.clone().ok_or(ExecError::VerificationFailed) } @@ -617,7 +681,7 @@ mod tests { _block: &reth_primitives::Block, _total_difficulty: reth_primitives::U256, _senders: Option>, - ) -> Result { + ) -> Result { self.0.clone().ok_or(ExecError::VerificationFailed) } } @@ -636,7 +700,7 @@ mod tests { } fn setup_externals( - exec_res: Vec, + exec_res: Vec, ) -> TreeExternals>, Arc, TestFactory> { let db = create_test_rw_db(); let consensus = Arc::new(TestConsensus::default()); @@ -718,12 +782,8 @@ mod tests { let data = BlockChainTestData::default(); let (mut block1, exec1) = data.blocks[0].clone(); block1.number = 11; - block1.state_root = - H256(hex!("5d035ccb3e75a9057452ff060b773b213ec1fc353426174068edfc3971a0b6bd")); let (mut block2, exec2) = data.blocks[1].clone(); block2.number = 12; - block2.state_root = - H256(hex!("90101a13dd059fa5cca99ed93d1dc23657f63626c5b8f993a2ccbdf7446b64f8")); // test pops execution results from vector, so order is from last to first.ß let externals = setup_externals(vec![exec2.clone(), exec1.clone(), exec2, exec1]); diff --git a/crates/executor/src/executor.rs b/crates/executor/src/executor.rs index e6a7bef842a..80edf42a863 100644 --- a/crates/executor/src/executor.rs +++ b/crates/executor/src/executor.rs @@ -1,10 +1,8 @@ -use crate::execution_result::{ - AccountChangeSet, AccountInfoChangeSet, ExecutionResult, TransactionChangeSet, -}; +use crate::post_state::PostState; use reth_interfaces::executor::Error; use reth_primitives::{ - bloom::logs_bloom, Account, Address, Block, Bloom, ChainSpec, Hardfork, Header, Log, Receipt, - TransactionSigned, H256, U256, + bloom::logs_bloom, Account, Address, Block, Bloom, Bytecode, ChainSpec, Hardfork, Header, Log, + Receipt, TransactionSigned, H256, U256, }; use reth_provider::{BlockExecutor, StateProvider}; use reth_revm::{ @@ -18,7 +16,7 @@ use revm::{ db::AccountState, primitives::{ hash_map::{self, Entry}, - Account as RevmAccount, AccountInfo, Bytecode, ResultAndState, + Account as RevmAccount, AccountInfo, ResultAndState, }, EVM, }; @@ -100,22 +98,16 @@ where ); } - /// Commit change to database and return change diff that is used to update state and create - /// history index - /// - /// ChangeDiff consists of: - /// address->AccountChangeSet (It contains old and new account info,storage wipe flag, and - /// old/new storage) bytecode_hash->bytecodes mapping - /// - /// BTreeMap is used to have sorted values + /// Commit change to the run-time database, and update the given [PostState] with the changes + /// made in the transaction, which can be persisted to the database. fn commit_changes( &mut self, changes: hash_map::HashMap, - ) -> (BTreeMap, BTreeMap) { + has_state_clear_eip: bool, + post_state: &mut PostState, + ) { let db = self.db(); - let mut change = BTreeMap::new(); - let mut new_bytecodes = BTreeMap::new(); // iterate over all changed accounts for (address, account) in changes { if account.is_destroyed { @@ -128,16 +120,8 @@ where }; // Insert into `change` a old account and None for new account // and mark storage to be mapped - change.insert( - address, - AccountChangeSet { - account: AccountInfoChangeSet::Destroyed { - old: to_reth_acc(&db_account.info), - }, - storage: BTreeMap::new(), - wipe_storage: true, - }, - ); + post_state.destroy_account(address, to_reth_acc(&db_account.info)); + // clear cached DB and mark account as not existing db_account.storage.clear(); db_account.account_state = AccountState::NotExisting; @@ -149,53 +133,54 @@ where // does it exist inside cached contracts if it doesn't it is new bytecode that // we are inserting inside `change` if let Some(ref code) = account.info.code { - if !code.is_empty() { - match db.contracts.entry(account.info.code_hash) { - Entry::Vacant(entry) => { - entry.insert(code.clone()); - new_bytecodes.insert(H256(account.info.code_hash.0), code.clone()); - } - Entry::Occupied(mut entry) => { - entry.insert(code.clone()); - } - } + if !code.is_empty() && !db.contracts.contains_key(&account.info.code_hash) { + db.contracts.insert(account.info.code_hash, code.clone()); + post_state.add_bytecode(account.info.code_hash, Bytecode(code.clone())); } } // get old account that is going to be overwritten or none if it does not exist // and get new account that was just inserted. new account mut ref is used for // inserting storage - let (account_info_changeset, new_account) = match db.accounts.entry(address) { + let cached_account = match db.accounts.entry(address) { Entry::Vacant(entry) => { let entry = entry.insert(Default::default()); entry.info = account.info.clone(); - // account was not existing, so this means new account is created - (AccountInfoChangeSet::Created { new: to_reth_acc(&entry.info) }, entry) + + let account = to_reth_acc(&entry.info); + if !(has_state_clear_eip && account.is_empty()) { + post_state.create_account(address, account); + } + entry } Entry::Occupied(entry) => { let entry = entry.into_mut(); - // account is present inside cache but is marked as NotExisting. - let account_changeset = - if matches!(entry.account_state, AccountState::NotExisting) { - AccountInfoChangeSet::Created { new: to_reth_acc(&account.info) } - } else if entry.info != account.info { - AccountInfoChangeSet::Changed { - old: to_reth_acc(&entry.info), - new: to_reth_acc(&account.info), - } - } else { - AccountInfoChangeSet::NoChange { is_empty: account.is_empty() } - }; + if matches!(entry.account_state, AccountState::NotExisting) { + let account = to_reth_acc(&account.info); + if !(has_state_clear_eip && account.is_empty()) { + post_state.create_account(address, account); + } + } else if entry.info != account.info { + post_state.change_account( + address, + to_reth_acc(&entry.info), + to_reth_acc(&account.info), + ); + } else if has_state_clear_eip && account.is_empty() { + // The account was touched, but it is empty, so it should be deleted. + post_state.destroy_account(address, to_reth_acc(&account.info)); + } + entry.info = account.info.clone(); - (account_changeset, entry) + entry } }; - new_account.account_state = if account.storage_cleared { - new_account.storage.clear(); + cached_account.account_state = if account.storage_cleared { + cached_account.storage.clear(); AccountState::StorageCleared - } else if new_account.account_state.is_storage_cleared() { + } else if cached_account.account_state.is_storage_cleared() { // the account already exists and its storage was cleared, preserve its previous // state AccountState::StorageCleared @@ -204,32 +189,28 @@ where }; // Insert storage. - let mut storage = BTreeMap::new(); + let mut storage_changeset = BTreeMap::new(); // insert storage into new db account. - new_account.storage.extend(account.storage.into_iter().map(|(key, value)| { - storage.insert(key, (value.original_value(), value.present_value())); + cached_account.storage.extend(account.storage.into_iter().map(|(key, value)| { + storage_changeset.insert(key, (value.original_value(), value.present_value())); (key, value.present_value()) })); // Insert into change. - change.insert( - address, - AccountChangeSet { - account: account_info_changeset, - storage, - wipe_storage: false, - }, - ); + if !storage_changeset.is_empty() { + post_state.change_storage(address, storage_changeset); + } } } - (change, new_bytecodes) } - /// Collect all balance changes at the end of the block. Balance changes might include block - /// reward, uncle rewards, withdrawals or irregular state changes (DAO fork). + /// Collect all balance changes at the end of the block. + /// + /// Balance changes might include the block reward, uncle rewards, withdrawals, or irregular + /// state changes (DAO fork). fn post_block_balance_increments( - &mut self, + &self, block: &Block, td: U256, ) -> Result, Error> { @@ -291,48 +272,36 @@ where } /// Irregular state change at Ethereum DAO hardfork - fn dao_fork_changeset(&mut self) -> Result, Error> { + fn apply_dao_fork_changes(&mut self, post_state: &mut PostState) -> Result<(), Error> { let db = self.db(); let mut drained_balance = U256::ZERO; // drain all accounts ether - let mut changesets = crate::eth_dao_fork::DAO_HARDKFORK_ACCOUNTS - .iter() - .map(|&address| { - let db_account = db.load_account(address).map_err(|_| Error::ProviderError)?; - let old = to_reth_acc(&db_account.info); - // drain balance - drained_balance += core::mem::take(&mut db_account.info.balance); - let new = to_reth_acc(&db_account.info); - // assume it is changeset as it is irregular state change - Ok((address, AccountInfoChangeSet::Changed { new, old })) - }) - .collect::, _>>()?; + for address in crate::eth_dao_fork::DAO_HARDKFORK_ACCOUNTS { + let db_account = db.load_account(address).map_err(|_| Error::ProviderError)?; + let old = to_reth_acc(&db_account.info); + // drain balance + drained_balance += core::mem::take(&mut db_account.info.balance); + let new = to_reth_acc(&db_account.info); + // assume it is changeset as it is irregular state change + post_state.change_account(address, old, new); + } // add drained ether to beneficiary. let beneficiary = crate::eth_dao_fork::DAO_HARDFORK_BENEFICIARY; + self.increment_account_balance(beneficiary, drained_balance, post_state)?; - let beneficiary_db_account = - db.load_account(beneficiary).map_err(|_| Error::ProviderError)?; - let old = to_reth_acc(&beneficiary_db_account.info); - beneficiary_db_account.info.balance += drained_balance; - let new = to_reth_acc(&beneficiary_db_account.info); - - let beneficiary_changeset = AccountInfoChangeSet::Changed { new, old }; - - // insert changeset - changesets.insert(beneficiary, beneficiary_changeset); - - Ok(changesets) + Ok(()) } - /// Generate balance increment account changeset and mutate account database entry in place. - fn account_balance_increment_changeset( + /// Increment the balance for the given account in the [PostState]. + fn increment_account_balance( &mut self, address: Address, increment: U256, - ) -> Result { + post_state: &mut PostState, + ) -> Result<(), Error> { let db = self.db(); let beneficiary = db.load_account(address).map_err(|_| Error::ProviderError)?; let old = to_reth_acc(&beneficiary.info); @@ -346,9 +315,10 @@ where beneficiary.account_state = AccountState::StorageCleared; // if account was not present append `Created` changeset - Ok(AccountInfoChangeSet::Created { - new: Account { nonce: 0, balance: new.balance, bytecode_hash: None }, - }) + post_state.create_account( + address, + Account { nonce: 0, balance: new.balance, bytecode_hash: None }, + ) } AccountState::StorageCleared | AccountState::Touched | AccountState::None => { @@ -359,9 +329,11 @@ where beneficiary.account_state = AccountState::Touched; } // if account was present, append changed changeset. - Ok(AccountInfoChangeSet::Changed { new, old }) + post_state.change_account(address, old, new); } } + + Ok(()) } /// Runs a single transaction in the configured environment and proceeds @@ -393,23 +365,26 @@ where out.map_err(|e| Error::EVM { hash, message: format!("{e:?}") }) } - /// Runs the provided transactions and commits their state. Will proceed - /// to return the total gas used by this batch of transaction as well as the - /// changesets generated by each tx. + /// Runs the provided transactions and commits their state to the run-time database. + /// + /// The returned [PostState] can be used to persist the changes to disk, and contains the + /// changes made by each transaction. + /// + /// The changes in [PostState] have a transition ID associated with them: there is one + /// transition ID for each transaction (with the first executed tx having transition ID 0, and + /// so on). pub fn execute_transactions( &mut self, block: &Block, total_difficulty: U256, senders: Option>, - ) -> Result<(Vec, u64), Error> { + ) -> Result<(PostState, u64), Error> { let senders = self.recover_senders(&block.body, senders)?; self.init_env(&block.header, total_difficulty); let mut cumulative_gas_used = 0; - // output of execution - let mut tx_changesets = Vec::with_capacity(block.body.len()); - + let mut post_state = PostState::with_tx_capacity(block.body.len()); for (transaction, sender) in block.body.iter().zip(senders.into_iter()) { // The sum of the transaction’s gas limit, Tg, and the gas utilised in this block prior, // must be no greater than the block’s gasLimit. @@ -424,7 +399,11 @@ where let ResultAndState { result, state } = self.transact(transaction, sender)?; // commit changes - let (changeset, new_bytecodes) = self.commit_changes(state); + self.commit_changes( + state, + self.chain_spec.fork(Hardfork::SpuriousDragon).active_at_block(block.number), + &mut post_state, + ); // append gas used cumulative_gas_used += result.gas_used(); @@ -433,22 +412,19 @@ where let logs: Vec = result.logs().into_iter().map(into_reth_log).collect(); // Push transaction changeset and calculate header bloom filter for receipt. - tx_changesets.push(TransactionChangeSet { - receipt: Receipt { - tx_type: transaction.tx_type(), - // Success flag was added in `EIP-658: Embedding transaction status code in - // receipts`. - success: result.is_success(), - cumulative_gas_used, - bloom: logs_bloom(logs.iter()), - logs, - }, - changeset, - new_bytecodes, + post_state.add_receipt(Receipt { + tx_type: transaction.tx_type(), + // Success flag was added in `EIP-658: Embedding transaction status code in + // receipts`. + success: result.is_success(), + cumulative_gas_used, + bloom: logs_bloom(logs.iter()), + logs, }); + post_state.finish_transition(); } - Ok((tx_changesets, cumulative_gas_used)) + Ok((post_state, cumulative_gas_used)) } } @@ -461,8 +437,8 @@ where block: &Block, total_difficulty: U256, senders: Option>, - ) -> Result { - let (tx_changesets, cumulative_gas_used) = + ) -> Result { + let (mut post_state, cumulative_gas_used) = self.execute_transactions(block, total_difficulty, senders)?; // Check if gas used matches the value set in header. @@ -470,21 +446,22 @@ where return Err(Error::BlockGasUsed { got: cumulative_gas_used, expected: block.gas_used }) } - let mut block_changesets = BTreeMap::default(); let balance_increments = self.post_block_balance_increments(block, total_difficulty)?; - for (address, increment) in balance_increments { - let changeset = self.account_balance_increment_changeset(address, increment)?; - block_changesets.insert(address, changeset); + let mut includes_block_transition = !balance_increments.is_empty(); + for (address, increment) in balance_increments.into_iter() { + self.increment_account_balance(address, increment, &mut post_state)?; } if self.chain_spec.fork(Hardfork::Dao).transitions_at_block(block.number) { - for (address, changeset) in self.dao_fork_changeset()? { - // No account collision between rewarded accounts and DAO fork related accounts. - block_changesets.insert(address, changeset); - } + includes_block_transition = true; + self.apply_dao_fork_changes(&mut post_state)?; + } + + if includes_block_transition { + post_state.finish_transition(); } - Ok(ExecutionResult { tx_changesets, block_changesets }) + Ok(post_state) } fn execute_and_verify_receipt( @@ -492,14 +469,15 @@ where block: &Block, total_difficulty: U256, senders: Option>, - ) -> Result { - let execution_result = self.execute(block, total_difficulty, senders)?; - - let receipts_iter = - execution_result.tx_changesets.iter().map(|changeset| &changeset.receipt); + ) -> Result { + let post_state = self.execute(block, total_difficulty, senders)?; if self.chain_spec.fork(Hardfork::Byzantium).active_at_block(block.header.number) { - verify_receipt(block.header.receipts_root, block.header.logs_bloom, receipts_iter)?; + verify_receipt( + block.header.receipts_root, + block.header.logs_bloom, + post_state.receipts().iter(), + )?; } // TODO Before Byzantium, receipts contained state root that would mean that expensive @@ -507,7 +485,7 @@ where // transaction This was replaced with is_success flag. // See more about EIP here: https://eips.ethereum.org/EIPS/eip-658 - Ok(execution_result) + Ok(post_state) } } @@ -541,7 +519,10 @@ mod tests { hex_literal::hex, keccak256, Account, Address, Bytecode, Bytes, ChainSpecBuilder, ForkCondition, StorageKey, H256, MAINNET, U256, }; - use reth_provider::{AccountProvider, BlockHashProvider, StateProvider}; + use reth_provider::{ + post_state::{Change, Storage}, + AccountProvider, BlockHashProvider, StateProvider, + }; use reth_revm::database::State; use reth_rlp::Decodable; use std::{collections::HashMap, str::FromStr}; @@ -661,17 +642,21 @@ mod tests { // execute chain and verify receipts let mut executor = Executor::new(chain_spec, db); - let out = executor.execute_and_verify_receipt(&block, U256::ZERO, None).unwrap(); + let post_state = executor.execute_and_verify_receipt(&block, U256::ZERO, None).unwrap(); - assert_eq!(out.tx_changesets.len(), 1, "Should executed one transaction"); + assert_eq!(post_state.transitions_count(), 2, "Should executed one transaction"); - let changesets = out.tx_changesets[0].clone(); - assert_eq!(changesets.new_bytecodes.len(), 0, "Should have zero new bytecodes"); + let block_reward = U256::from(WEI_2ETH + (WEI_2ETH >> 5)); let account1_info = Account { balance: U256::ZERO, nonce: 0x00, bytecode_hash: None }; let account2_info = Account { - balance: U256::from(0x1bc16d674ece94bau128 - 0x1bc16d674ec80000u128), /* decrease for - * block reward */ + // Block reward decrease + balance: U256::from(0x1bc16d674ece94bau128 - 0x1bc16d674ec80000u128), + nonce: 0x00, + bytecode_hash: None, + }; + let account2_info_with_block_reward = Account { + balance: account2_info.balance + block_reward, nonce: 0x00, bytecode_hash: None, }; @@ -680,8 +665,8 @@ mod tests { nonce: 0x01, bytecode_hash: None, }; - - let block_reward = U256::from(WEI_2ETH + (WEI_2ETH >> 5)); + let ommer_beneficiary_info = + Account { nonce: 0, balance: U256::from((8 * WEI_2ETH) >> 3), bytecode_hash: None }; // Check if cache is set // account1 @@ -707,60 +692,88 @@ mod tests { assert_eq!(cached_acc3.account_state, AccountState::Touched); assert_eq!(cached_acc3.storage.len(), 0); - assert_eq!( - changesets.changeset.get(&account1).unwrap().account, - AccountInfoChangeSet::NoChange { is_empty: false }, - "No change to account" + assert!( + post_state.accounts().get(&account1).is_none(), + "Account should not be present in post-state since it was not changed" ); + + // Check changes + const TX_TRANSITION_ID: u64 = 0; + const BLOCK_TRANSITION_ID: u64 = 1; + + // Clone and sort to make the test deterministic + let mut changes = post_state.changes().to_vec(); + changes.sort_by_key(|change| (change.transition_id(), change.address())); assert_eq!( - changesets.changeset.get(&account2).unwrap().account, - AccountInfoChangeSet::Created { new: account2_info }, - "New account" + changes, + &[ + // Storage changes on account 1 + Change::StorageChanged { + id: TX_TRANSITION_ID, + address: account1, + changeset: [(U256::from(1), (U256::ZERO, U256::from(2)))].into() + }, + // New account + Change::AccountCreated { + id: TX_TRANSITION_ID, + address: account2, + account: account2_info + }, + // Changed account + Change::AccountChanged { + id: TX_TRANSITION_ID, + address: account3, + old: account3_old_info, + new: account3_info + }, + // Block reward + Change::AccountChanged { + id: BLOCK_TRANSITION_ID, + address: account2, + old: account2_info, + new: account2_info_with_block_reward + }, + // Ommer reward + Change::AccountCreated { + id: BLOCK_TRANSITION_ID, + address: ommer_beneficiary, + account: ommer_beneficiary_info + }, + ], + "Changeset did not match" ); + + // Check final post-state assert_eq!( - changesets.changeset.get(&account3).unwrap().account, - AccountInfoChangeSet::Changed { old: account3_old_info, new: account3_info }, - "Change to account state" + post_state.storage(), + &BTreeMap::from([( + account1, + Storage { wiped: false, storage: BTreeMap::from([(U256::from(1), U256::from(2))]) } + )]), + "Should have changed 1 storage slot" ); + assert_eq!(post_state.bytecodes().len(), 0, "Should have zero new bytecodes"); - // check block rewards changeset. - let mut block_rewarded_acc_info = account2_info; - // add Blocks 2 eth reward and 2>>5 for one ommer - block_rewarded_acc_info.balance += block_reward; - - // check block reward changeset + let accounts = post_state.accounts(); assert_eq!( - out.block_changesets, - BTreeMap::from([ - ( - account2, - AccountInfoChangeSet::Changed { - new: block_rewarded_acc_info, - old: account2_info - } - ), - ( - ommer_beneficiary, - AccountInfoChangeSet::Created { - new: Account { - nonce: 0, - balance: U256::from((8 * WEI_2ETH) >> 3), - bytecode_hash: None - } - } - ) - ]) + accounts.len(), + 3, + "Should have 4 accounts (account 2, 3 and the ommer beneficiary)" ); - - assert_eq!(changesets.new_bytecodes.len(), 0, "No new bytecodes"); - - // check storage - let storage = &changesets.changeset.get(&account1).unwrap().storage; - assert_eq!(storage.len(), 1, "Only one storage change"); assert_eq!( - storage.get(&U256::from(1)), - Some(&(U256::ZERO, U256::from(2))), - "Storage change from 0 to 2 on slot 1" + accounts.get(&account2).unwrap(), + &Some(account2_info_with_block_reward), + "Account 2 state is wrong" + ); + assert_eq!( + accounts.get(&account3).unwrap(), + &Some(account3_info), + "Account 3 state is wrong" + ); + assert_eq!( + accounts.get(&ommer_beneficiary).unwrap(), + &Some(ommer_beneficiary_info), + "Ommer beneficiary state is wrong" ); } @@ -798,7 +811,11 @@ mod tests { None, ) .unwrap(); - assert_eq!(out.tx_changesets.len(), 0, "No tx"); + assert_eq!( + out.transitions_count(), + 1, + "Should only have 1 transition (the block transition)" + ); // Check if cache is set // beneficiary @@ -813,24 +830,15 @@ mod tests { } // check changesets - let change_set = - out.block_changesets.get(&crate::eth_dao_fork::DAO_HARDFORK_BENEFICIARY).unwrap(); + let beneficiary_state = + out.accounts().get(&crate::eth_dao_fork::DAO_HARDFORK_BENEFICIARY).unwrap().unwrap(); assert_eq!( - *change_set, - AccountInfoChangeSet::Changed { - new: Account { balance: U256::from(beneficiary_balance), ..Default::default() }, - old: Account { balance: U256::ZERO, ..Default::default() } - } + beneficiary_state, + Account { balance: U256::from(beneficiary_balance), ..Default::default() }, ); - for (i, address) in crate::eth_dao_fork::DAO_HARDKFORK_ACCOUNTS.iter().enumerate() { - let change_set = out.block_changesets.get(address).unwrap(); - assert_eq!( - *change_set, - AccountInfoChangeSet::Changed { - new: Account { balance: U256::ZERO, ..Default::default() }, - old: Account { balance: U256::from(i), ..Default::default() } - } - ); + for address in crate::eth_dao_fork::DAO_HARDKFORK_ACCOUNTS.iter() { + let updated_account = out.accounts().get(address).unwrap().unwrap(); + assert_eq!(updated_account, Account { balance: U256::ZERO, ..Default::default() }); } } @@ -885,10 +893,12 @@ mod tests { let mut executor = Executor::new(chain_spec, db); let out = executor.execute_and_verify_receipt(&block, U256::ZERO, None).unwrap(); - assert_eq!(out.tx_changesets.len(), 1, "Should executed one transaction"); - - let changesets = out.tx_changesets[0].clone(); - assert_eq!(changesets.new_bytecodes.len(), 0, "Should have zero new bytecodes"); + assert_eq!( + out.transitions_count(), + 2, + "Should only have two transitions (the transaction and the block)" + ); + assert_eq!(out.bytecodes().len(), 0, "Should have zero new bytecodes"); let post_account_caller = Account { balance: U256::from(0x0de0b6b3a761cf60u64), @@ -897,21 +907,20 @@ mod tests { }; assert_eq!( - changesets.changeset.get(&address_caller).unwrap().account, - AccountInfoChangeSet::Changed { new: post_account_caller, old: pre_account_caller }, + out.accounts().get(&address_caller).unwrap().unwrap(), + post_account_caller, "Caller account has changed and fee is deduced" ); - let selfdestroyer_changeset = changesets.changeset.get(&address_selfdestruct).unwrap(); - - // check account assert_eq!( - selfdestroyer_changeset.account, - AccountInfoChangeSet::Destroyed { old: pre_account_selfdestroyed }, - "Selfdestroyed account" + out.accounts().get(&address_selfdestruct).unwrap(), + &None, + "Selfdestructed account should have been deleted" + ); + assert!( + out.storage().get(&address_selfdestruct).unwrap().wiped, + "Selfdestructed account should have its storage wiped" ); - - assert!(selfdestroyer_changeset.wipe_storage); } // Test vector from https://github.com/ethereum/tests/blob/3156db5389921125bb9e04142d18e0e7b0cf8d64/BlockchainTests/EIPTests/bc4895-withdrawals/twoIdenticalIndexDifferentValidator.json @@ -933,7 +942,7 @@ mod tests { // execute chain and verify receipts let mut executor = Executor::new(chain_spec, db); let out = executor.execute_and_verify_receipt(&block, U256::ZERO, None).unwrap(); - assert_eq!(out.tx_changesets.len(), 0, "No tx"); + assert_eq!(out.transitions_count(), 1, "Only one transition (the block transition)"); let withdrawal_sum = withdrawals.iter().fold(U256::ZERO, |sum, w| sum + w.amount_wei()); let beneficiary_account = executor.db().accounts.get(&withdrawal_beneficiary).unwrap(); @@ -941,29 +950,28 @@ mod tests { assert_eq!(beneficiary_account.info.nonce, 0); assert_eq!(beneficiary_account.account_state, AccountState::StorageCleared); - assert_eq!(out.block_changesets.len(), 1); assert_eq!( - out.block_changesets.get(&withdrawal_beneficiary), - Some(&AccountInfoChangeSet::Created { - new: Account { nonce: 0, balance: withdrawal_sum, bytecode_hash: None }, - }) + out.accounts().get(&withdrawal_beneficiary).unwrap(), + &Some(Account { nonce: 0, balance: withdrawal_sum, bytecode_hash: None }), + "Withdrawal account should have gotten its balance set" ); // Execute same block again let out = executor.execute_and_verify_receipt(&block, U256::ZERO, None).unwrap(); - assert_eq!(out.tx_changesets.len(), 0, "No tx"); + assert_eq!( + out.transitions_count(), + 1, + "Should only have one transition (the block transition)" + ); - assert_eq!(out.block_changesets.len(), 1); assert_eq!( - out.block_changesets.get(&withdrawal_beneficiary), - Some(&AccountInfoChangeSet::Changed { - old: Account { nonce: 0, balance: withdrawal_sum, bytecode_hash: None }, - new: Account { - nonce: 0, - balance: withdrawal_sum + withdrawal_sum, - bytecode_hash: None - }, - }) + out.accounts().get(&withdrawal_beneficiary).unwrap(), + &Some(Account { + nonce: 0, + balance: withdrawal_sum + withdrawal_sum, + bytecode_hash: None + }), + "Withdrawal account should have gotten its balance set" ); } @@ -987,23 +995,35 @@ mod tests { }; let mut executor = Executor::new(chain_spec, db); // touch account - executor.commit_changes(hash_map::HashMap::from([( - account, - RevmAccount { ..default_acc.clone() }, - )])); + executor.commit_changes( + hash_map::HashMap::from([(account, RevmAccount { ..default_acc.clone() })]), + true, + &mut PostState::default(), + ); // destroy account - executor.commit_changes(hash_map::HashMap::from([( - account, - RevmAccount { is_destroyed: true, is_touched: true, ..default_acc.clone() }, - )])); + executor.commit_changes( + hash_map::HashMap::from([( + account, + RevmAccount { is_destroyed: true, is_touched: true, ..default_acc.clone() }, + )]), + true, + &mut PostState::default(), + ); // re-create account - executor.commit_changes(hash_map::HashMap::from([( - account, - RevmAccount { is_touched: true, storage_cleared: true, ..default_acc.clone() }, - )])); + executor.commit_changes( + hash_map::HashMap::from([( + account, + RevmAccount { is_touched: true, storage_cleared: true, ..default_acc.clone() }, + )]), + true, + &mut PostState::default(), + ); // touch account - executor - .commit_changes(hash_map::HashMap::from([(account, RevmAccount { ..default_acc })])); + executor.commit_changes( + hash_map::HashMap::from([(account, RevmAccount { ..default_acc })]), + true, + &mut PostState::default(), + ); let db = executor.db(); diff --git a/crates/executor/src/lib.rs b/crates/executor/src/lib.rs index d1b17d73972..5bbd34a3b89 100644 --- a/crates/executor/src/lib.rs +++ b/crates/executor/src/lib.rs @@ -11,8 +11,10 @@ pub mod eth_dao_fork; pub mod substate; /// Execution result types. -pub use reth_provider::execution_result; +pub use reth_provider::post_state; + pub mod blockchain_tree; + /// Executor pub mod executor; diff --git a/crates/executor/src/substate.rs b/crates/executor/src/substate.rs index 41b5fe6483c..7da7acbc5e4 100644 --- a/crates/executor/src/substate.rs +++ b/crates/executor/src/substate.rs @@ -2,249 +2,35 @@ use reth_interfaces::{provider::ProviderError, Result}; use reth_primitives::{Account, Address, BlockHash, BlockNumber, Bytecode, Bytes, H256, U256}; -use reth_provider::{AccountProvider, BlockHashProvider, StateProvider}; -use std::collections::{hash_map::Entry, BTreeMap, HashMap}; - -use crate::execution_result::{AccountInfoChangeSet, ExecutionResult}; - -/// Memory backend, storing all state values in a `Map` in memory. -#[derive(Debug, Default, Clone, PartialEq, Eq)] -pub struct SubStateData { - /// Account info where None means it is not existing. Not existing state is needed for Pre - /// TANGERINE forks. `code` is always `None`, and bytecode can be found in `contracts`. - pub accounts: HashMap, - /// New bytecodes - pub bytecodes: HashMap, -} - -impl SubStateData { - /// Apply changesets to substate. - pub fn apply(&mut self, changesets: &[ExecutionResult]) { - for changeset in changesets { - self.apply_one(changeset) - } - } - - /// Apply one changeset to substate. - pub fn apply_one(&mut self, changeset: &ExecutionResult) { - for tx_changeset in changeset.tx_changesets.iter() { - // apply accounts - for (address, account_change) in tx_changeset.changeset.iter() { - // revert account - self.apply_account(address, &account_change.account); - // revert its storage - self.apply_storage(address, &account_change.storage); - } - // apply bytecodes - for (hash, bytecode) in tx_changeset.new_bytecodes.iter() { - self.bytecodes.entry(*hash).or_insert((0, Bytecode(bytecode.clone()))).0 += 1; - } - } - // apply block reward - for (address, change) in changeset.block_changesets.iter() { - self.apply_account(address, change) - } - } - - /// Apply account changeset to substate - fn apply_account(&mut self, address: &Address, change: &AccountInfoChangeSet) { - match change { - AccountInfoChangeSet::Created { new } => match self.accounts.entry(*address) { - Entry::Vacant(entry) => { - entry.insert(AccountSubState::created_account(*new)); - } - Entry::Occupied(mut entry) => { - let account = entry.get_mut(); - // increment counter - account.inc_storage_counter(); - account.info = *new; - } - }, - AccountInfoChangeSet::Destroyed { .. } => { - // set selfdestructed account - let account = self.accounts.entry(*address).or_default(); - account.inc_storage_counter(); - account.info = Default::default(); - account.storage.clear(); - } - AccountInfoChangeSet::Changed { old, .. } => { - self.accounts.entry(*address).or_default().info = *old; - } - AccountInfoChangeSet::NoChange { is_empty } => { - if *is_empty { - self.accounts.entry(*address).or_default(); - } - } - } - } - - /// Apply storage changeset to substate - fn apply_storage(&mut self, address: &Address, storage: &BTreeMap) { - if let Entry::Occupied(mut entry) = self.accounts.entry(*address) { - let account_storage = &mut entry.get_mut().storage; - for (key, (_, new_value)) in storage { - let key = H256(key.to_be_bytes()); - account_storage.insert(key, *new_value); - } - } - } - - /// Revert to old state in substate. Changesets will be reverted in reverse order, - pub fn revert(&mut self, changesets: &[ExecutionResult]) { - for changeset in changesets.iter().rev() { - // revert block changeset - for (address, change) in changeset.block_changesets.iter() { - self.revert_account(address, change) - } - - for tx_changeset in changeset.tx_changesets.iter() { - // revert bytecodes - for (hash, _) in tx_changeset.new_bytecodes.iter() { - match self.bytecodes.entry(*hash) { - Entry::Vacant(_) => panic!("Bytecode should be present"), - Entry::Occupied(mut entry) => { - let (cnt, _) = entry.get_mut(); - *cnt -= 1; - if *cnt == 0 { - entry.remove_entry(); - } - } - } - } - // revert accounts - for (address, account_change) in tx_changeset.changeset.iter() { - // revert account - self.revert_account(address, &account_change.account); - // revert its storage - self.revert_storage(address, &account_change.storage); - } - } - } - } - - /// Revert storage - fn revert_storage(&mut self, address: &Address, storage: &BTreeMap) { - if let Entry::Occupied(mut entry) = self.accounts.entry(*address) { - let account_storage = &mut entry.get_mut().storage; - for (key, (old_value, _)) in storage { - let key = H256(key.to_be_bytes()); - account_storage.insert(key, *old_value); - } - } - } - - /// Revert account - fn revert_account(&mut self, address: &Address, change: &AccountInfoChangeSet) { - match change { - AccountInfoChangeSet::Created { .. } => { - match self.accounts.entry(*address) { - Entry::Vacant(_) => { - // We inserted this account in apply fn. - panic!("It should be present, something is broken"); - } - Entry::Occupied(mut entry) => { - let val = entry.get_mut(); - if val.decr_storage_counter() { - // remove account that we didn't change from substate - - entry.remove_entry(); - return - } - val.info = Account::default(); - val.storage.clear(); - } - }; - } - AccountInfoChangeSet::Destroyed { old } => match self.accounts.entry(*address) { - Entry::Vacant(_) => { - // We inserted this account in apply fn. - panic!("It should be present, something is broken"); - } - Entry::Occupied(mut entry) => { - let val = entry.get_mut(); - - // Contrary to Created we are not removing this account as we dont know if - // this account was changer or not by `Changed` changeset. - val.decr_storage_counter(); - val.info = *old; - } - }, - AccountInfoChangeSet::Changed { old, .. } => { - self.accounts.entry(*address).or_default().info = *old; - } - AccountInfoChangeSet::NoChange { is_empty: _ } => { - // do nothing - } - } - } -} -/// Account changes in substate -#[derive(Debug, Clone, Default, Eq, PartialEq)] -pub struct AccountSubState { - /// New account state - pub info: Account, - /// If account is selfdestructed or newly created, storage will be cleared. - /// and we dont need to ask the provider for data. - /// As we need to have as - pub storage_is_clear: Option, - /// storage slots - pub storage: HashMap, -} - -impl AccountSubState { - /// Increment storage counter to mark this storage was cleared - pub fn inc_storage_counter(&mut self) { - self.storage_is_clear = Some(self.storage_is_clear.unwrap_or_default() + 1); - } - - /// Decrement storage counter to represent that changeset that cleared storage was reverted. - pub fn decr_storage_counter(&mut self) -> bool { - let Some(cnt) = self.storage_is_clear else { return false}; - - if cnt == 1 { - self.storage_is_clear = None; - return true - } - false - } - /// Account is created - pub fn created_account(info: Account) -> Self { - Self { info, storage_is_clear: Some(1), storage: HashMap::new() } - } - /// Should we ask the provider for storage data - pub fn ask_provider(&self) -> bool { - self.storage_is_clear.is_none() - } -} - -/// Wrapper around substate and provider, it decouples the database that can be Latest or historical -/// with substate changes that happened previously. -pub struct SubStateWithProvider<'a, SP: StateProvider> { - /// Substate - substate: &'a SubStateData, - /// Provider +use reth_provider::{post_state::PostState, AccountProvider, BlockHashProvider, StateProvider}; +use std::collections::BTreeMap; + +/// A state provider that either resolves to data in a wrapped [`PostState`], or an underlying state +/// provider. +pub struct PostStateProvider<'a, SP: StateProvider> { + /// The wrapped state after execution of one or more transactions and/or blocks. + state: &'a PostState, + /// The inner state provider. provider: SP, - /// side chain block hashes + /// The blocks in the sidechain. sidechain_block_hashes: &'a BTreeMap, - /// Last N canonical hashes, + /// The blocks in the canonical chain. canonical_block_hashes: &'a BTreeMap, } -impl<'a, SP: StateProvider> SubStateWithProvider<'a, SP> { - /// Create new substate with provider +impl<'a, SP: StateProvider> PostStateProvider<'a, SP> { + /// Create new post-state provider pub fn new( - substate: &'a SubStateData, + state: &'a PostState, provider: SP, sidechain_block_hashes: &'a BTreeMap, canonical_block_hashes: &'a BTreeMap, ) -> Self { - Self { substate, provider, sidechain_block_hashes, canonical_block_hashes } + Self { state, provider, sidechain_block_hashes, canonical_block_hashes } } } -/* Implement StateProvider traits */ - -impl<'a, SP: StateProvider> BlockHashProvider for SubStateWithProvider<'a, SP> { +impl<'a, SP: StateProvider> BlockHashProvider for PostStateProvider<'a, SP> { fn block_hash(&self, number: U256) -> Result> { // All block numbers fit inside u64 and revm checks if it is last 256 block numbers. let block_number = number.as_limbs()[0]; @@ -262,33 +48,45 @@ impl<'a, SP: StateProvider> BlockHashProvider for SubStateWithProvider<'a, SP> { } } -impl<'a, SP: StateProvider> AccountProvider for SubStateWithProvider<'a, SP> { +impl<'a, SP: StateProvider> AccountProvider for PostStateProvider<'a, SP> { fn basic_account(&self, address: Address) -> Result> { - if let Some(account) = self.substate.accounts.get(&address).map(|acc| acc.info) { - return Ok(Some(account)) + if let Some(account) = self.state.account(&address) { + Ok(*account) + } else { + self.provider.basic_account(address) } - self.provider.basic_account(address) } } -impl<'a, SP: StateProvider> StateProvider for SubStateWithProvider<'a, SP> { +impl<'a, SP: StateProvider> StateProvider for PostStateProvider<'a, SP> { fn storage( &self, account: Address, storage_key: reth_primitives::StorageKey, ) -> Result> { - if let Some(substate_account) = self.substate.accounts.get(&account) { - if let Some(storage) = substate_account.storage.get(&storage_key) { - return Ok(Some(*storage)) - } - if !substate_account.ask_provider() { + if let Some(storage) = self.state.account_storage(&account) { + if storage.wiped { return Ok(Some(U256::ZERO)) } + + if let Some(value) = + storage.storage.get(&U256::from_be_bytes(storage_key.to_fixed_bytes())) + { + return Ok(Some(*value)) + } } + self.provider.storage(account, storage_key) } - /// Get account and storage proofs. + fn bytecode_by_hash(&self, code_hash: H256) -> Result> { + if let Some(bytecode) = self.state.bytecode(&code_hash).cloned() { + return Ok(Some(bytecode)) + } + + self.provider.bytecode_by_hash(code_hash) + } + fn proof( &self, _address: Address, @@ -296,11 +94,4 @@ impl<'a, SP: StateProvider> StateProvider for SubStateWithProvider<'a, SP> { ) -> Result<(Vec, H256, Vec>)> { Err(ProviderError::HistoryStateRoot.into()) } - - fn bytecode_by_hash(&self, code_hash: H256) -> Result> { - if let Some((_, bytecode)) = self.substate.bytecodes.get(&code_hash).cloned() { - return Ok(Some(bytecode)) - } - self.provider.bytecode_by_hash(code_hash) - } } diff --git a/crates/stages/src/stages/execution.rs b/crates/stages/src/stages/execution.rs index e08caa69d7a..d0d44d8967d 100644 --- a/crates/stages/src/stages/execution.rs +++ b/crates/stages/src/stages/execution.rs @@ -11,7 +11,9 @@ use reth_db::{ }; use reth_interfaces::provider::ProviderError; use reth_primitives::{Address, Block, U256}; -use reth_provider::{BlockExecutor, ExecutorFactory, LatestStateProviderRef, Transaction}; +use reth_provider::{ + post_state::PostState, BlockExecutor, ExecutorFactory, LatestStateProviderRef, Transaction, +}; use tracing::*; /// The [`StageId`] of the execution stage. @@ -112,7 +114,7 @@ impl ExecutionStage { let mut executor = self.executor_factory.with_sp(LatestStateProviderRef::new(&**tx)); // Fetch transactions, execute them and generate results - let mut changesets = Vec::with_capacity(block_batch.len()); + let mut changesets = PostState::default(); for (header, td, body, ommers, withdrawals) in block_batch.into_iter() { let block_number = header.number; tracing::trace!(target: "sync::stages::execution", ?block_number, "Execute block."); @@ -154,11 +156,12 @@ impl ExecutionStage { Some(signers), ) .map_err(|error| StageError::ExecutionError { block: block_number, error })?; - changesets.push(changeset); + changesets.extend(changeset); } // put execution results to database - tx.insert_execution_result(changesets, self.executor_factory.chain_spec(), last_block)?; + let first_transition_id = tx.get_block_transition(last_block)?; + changesets.write_to_db(&**tx, first_transition_id)?; let done = !capped; info!(target: "sync::stages::execution", stage_progress = end_block, done, "Sync iteration finished"); diff --git a/crates/storage/provider/src/execution_result.rs b/crates/storage/provider/src/execution_result.rs deleted file mode 100644 index 5a5d3cc744d..00000000000 --- a/crates/storage/provider/src/execution_result.rs +++ /dev/null @@ -1,206 +0,0 @@ -//! Output of execution. - -use reth_db::{models::AccountBeforeTx, tables, transaction::DbTxMut, Error as DbError}; -use reth_primitives::{Account, Address, Receipt, H256, U256}; -use revm_primitives::Bytecode; -use std::collections::BTreeMap; - -/// Execution Result containing vector of transaction changesets -/// and block reward if present -#[derive(Debug, Default, Eq, PartialEq, Clone)] -pub struct ExecutionResult { - /// Transaction changeset containing [Receipt], changed [Accounts][Account] and Storages. - pub tx_changesets: Vec, - /// Post block account changesets. This might include block reward, uncle rewards, withdrawals - /// or irregular state changes (DAO fork). - pub block_changesets: BTreeMap, -} - -/// After transaction is executed this structure contain -/// transaction [Receipt] every change to state ([Account], Storage, [Bytecode]) -/// that this transaction made and its old values -/// so that history account table can be updated. -#[derive(Debug, Eq, PartialEq, Clone)] -pub struct TransactionChangeSet { - /// Transaction receipt - pub receipt: Receipt, - /// State change that this transaction made on state. - pub changeset: BTreeMap, - /// new bytecode created as result of transaction execution. - pub new_bytecodes: BTreeMap, -} - -/// Contains old/new account changes -#[derive(Debug, Clone, Eq, PartialEq)] -pub enum AccountInfoChangeSet { - /// The account is newly created. Account can be created by just by sending balance, - /// - /// Revert of this changeset is empty account, - Created { - /// The newly created account. - new: Account, - }, - /// An account was deleted (selfdestructed) or we have touched - /// an empty account and we need to remove/destroy it. - /// (Look at state clearing [EIP-158](https://eips.ethereum.org/EIPS/eip-158)) - /// - /// Revert of this changeset is old account - Destroyed { - /// The account that was destroyed. - old: Account, - }, - /// The account was changed. - /// - /// revert of this changeset is old account - Changed { - /// The account after the change. - new: Account, - /// The account prior to the change. - old: Account, - }, - /// Nothing was changed for the account (nonce/balance). - NoChange { - /// Used to clear existing empty accounts pre-EIP-161. - is_empty: bool, - }, -} - -impl Default for AccountInfoChangeSet { - fn default() -> Self { - AccountInfoChangeSet::NoChange { is_empty: false } - } -} - -impl AccountInfoChangeSet { - /// Create new account info changeset - pub fn new(old: Option, new: Option) -> Self { - match (old, new) { - (Some(old), Some(new)) => { - if new != old { - Self::Changed { new, old } - } else { - if new.is_empty() {} - Self::NoChange { is_empty: true } - } - } - (None, Some(new)) => Self::Created { new }, - (Some(old), None) => Self::Destroyed { old }, - (None, None) => Self::NoChange { is_empty: false }, - } - } - /// Apply the changes from the changeset to a database transaction. - pub fn apply_to_db<'a, TX: DbTxMut<'a>>( - self, - tx: &TX, - address: Address, - tx_index: u64, - has_state_clear_eip: bool, - ) -> Result<(), DbError> { - match self { - AccountInfoChangeSet::Changed { old, new } => { - // insert old account in AccountChangeSet - // check for old != new was already done - tx.put::( - tx_index, - AccountBeforeTx { address, info: Some(old) }, - )?; - tx.put::(address, new)?; - } - AccountInfoChangeSet::Created { new } => { - // Ignore account that are created empty and state clear (SpuriousDragon) hardfork - // is activated. - if has_state_clear_eip && new.is_empty() { - return Ok(()) - } - tx.put::( - tx_index, - AccountBeforeTx { address, info: None }, - )?; - tx.put::(address, new)?; - } - AccountInfoChangeSet::Destroyed { old } => { - tx.delete::(address, None)?; - tx.put::( - tx_index, - AccountBeforeTx { address, info: Some(old) }, - )?; - } - AccountInfoChangeSet::NoChange { is_empty } => { - if has_state_clear_eip && is_empty { - tx.delete::(address, None)?; - } - } - } - Ok(()) - } -} - -/// Diff change set that is needed for creating history index and updating current world state. -#[derive(Debug, Default, Eq, PartialEq, Clone)] -pub struct AccountChangeSet { - /// Old and New account account change. - pub account: AccountInfoChangeSet, - /// Storage containing key -> (OldValue,NewValue). in case that old value is not existing - /// we can expect to have U256::ZERO, same with new value. - pub storage: BTreeMap, - /// Just to make sure that we are taking selfdestruct cleaning we have this field that wipes - /// storage. There are instances where storage is changed but account is not touched, so we - /// can't take into account that if new account is None that it is selfdestruct. - pub wipe_storage: bool, -} - -#[cfg(test)] -mod tests { - use std::sync::Arc; - - use reth_db::{ - database::Database, - mdbx::{test_utils, Env, EnvKind, WriteMap}, - transaction::DbTx, - }; - use reth_primitives::H160; - - use super::*; - - #[test] - fn apply_account_info_changeset() { - let db: Arc> = test_utils::create_test_db(EnvKind::RW); - let address = H160::zero(); - let tx_num = 0; - let acc1 = Account { balance: U256::from(1), nonce: 2, bytecode_hash: Some(H256::zero()) }; - let acc2 = Account { balance: U256::from(3), nonce: 4, bytecode_hash: Some(H256::zero()) }; - - let tx = db.tx_mut().unwrap(); - - // check Changed changeset - AccountInfoChangeSet::Changed { new: acc1, old: acc2 } - .apply_to_db(&tx, address, tx_num, true) - .unwrap(); - assert_eq!( - tx.get::(tx_num), - Ok(Some(AccountBeforeTx { address, info: Some(acc2) })) - ); - assert_eq!(tx.get::(address), Ok(Some(acc1))); - - AccountInfoChangeSet::Created { new: acc1 } - .apply_to_db(&tx, address, tx_num, true) - .unwrap(); - assert_eq!( - tx.get::(tx_num), - Ok(Some(AccountBeforeTx { address, info: None })) - ); - assert_eq!(tx.get::(address), Ok(Some(acc1))); - - // delete old value, as it is dupsorted - tx.delete::(tx_num, None).unwrap(); - - AccountInfoChangeSet::Destroyed { old: acc2 } - .apply_to_db(&tx, address, tx_num, true) - .unwrap(); - assert_eq!(tx.get::(address), Ok(None)); - assert_eq!( - tx.get::(tx_num), - Ok(Some(AccountBeforeTx { address, info: Some(acc2) })) - ); - } -} diff --git a/crates/storage/provider/src/lib.rs b/crates/storage/provider/src/lib.rs index 77882f42b4b..6acc1306037 100644 --- a/crates/storage/provider/src/lib.rs +++ b/crates/storage/provider/src/lib.rs @@ -27,7 +27,7 @@ pub use providers::{ pub mod trie; /// Execution result -pub mod execution_result; +pub mod post_state; /// Helper types for interacting with the database mod transaction; diff --git a/crates/storage/provider/src/post_state.rs b/crates/storage/provider/src/post_state.rs new file mode 100644 index 00000000000..4870c4fb9a6 --- /dev/null +++ b/crates/storage/provider/src/post_state.rs @@ -0,0 +1,828 @@ +//! Output of execution. +use reth_db::{ + cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO, DbDupCursorRW}, + models::{AccountBeforeTx, TransitionIdAddress}, + tables, + transaction::{DbTx, DbTxMut}, + Error as DbError, +}; +use reth_primitives::{ + Account, Address, Bytecode, Receipt, StorageEntry, TransitionId, H256, U256, +}; +use std::collections::BTreeMap; + +/// Storage for an account. +/// +/// # Wiped Storage +/// +/// The field `wiped` denotes whether any of the values contained in storage are valid or not; if +/// `wiped` is `true`, the storage should be considered empty. +#[derive(Debug, Default, Clone, Eq, PartialEq)] +pub struct Storage { + /// Whether the storage was wiped or not. + pub wiped: bool, + /// The storage slots. + pub storage: BTreeMap, +} + +/// Storage for an account with the old and new values for each slot. +/// TODO: Do we actually need (old, new) anymore, or is (old) sufficient? (Check the writes) +/// If we don't, we can unify this and [Storage]. +pub type StorageChangeset = BTreeMap; + +/// A change to the state of accounts or storage. +#[derive(Debug, Clone, Eq, PartialEq)] +pub enum Change { + /// A new account was created. + AccountCreated { + /// The ID of the transition this change is a part of. + id: TransitionId, + /// The address of the account that was created. + address: Address, + /// The account. + account: Account, + }, + /// An existing account was changed. + AccountChanged { + /// The ID of the transition this change is a part of. + id: TransitionId, + /// The address of the account that was changed. + address: Address, + /// The account before the change. + old: Account, + /// The account after the change. + new: Account, + }, + /// Storage slots for an account were changed. + StorageChanged { + /// The ID of the transition this change is a part of. + id: TransitionId, + /// The address of the account associated with the storage slots. + address: Address, + /// The storage changeset. + changeset: StorageChangeset, + }, + /// Storage was wiped + StorageWiped { + /// The ID of the transition this change is a part of. + id: TransitionId, + /// The address of the account whose storage was wiped. + address: Address, + }, + /// An account was destroyed. + /// + /// This removes all of the information associated with the account. An accompanying + /// [Change::StorageWiped] will also be present to mark the deletion of storage. + /// + /// If a change to an account satisfies the conditions for EIP-158, this change variant is also + /// applied instead of the change that would otherwise have happened. + AccountDestroyed { + /// The ID of the transition this change is a part of. + id: TransitionId, + /// The address of the destroyed account. + address: Address, + /// The account before it was destroyed. + old: Account, + }, +} + +impl Change { + /// Get the transition ID for the change + pub fn transition_id(&self) -> TransitionId { + match self { + Change::AccountChanged { id, .. } | + Change::AccountCreated { id, .. } | + Change::StorageChanged { id, .. } | + Change::StorageWiped { id, .. } | + Change::AccountDestroyed { id, .. } => *id, + } + } + + /// Get the address of the account this change operates on. + pub fn address(&self) -> Address { + match self { + Change::AccountChanged { address, .. } | + Change::AccountCreated { address, .. } | + Change::StorageChanged { address, .. } | + Change::StorageWiped { address, .. } | + Change::AccountDestroyed { address, .. } => *address, + } + } + + /// Set the transition ID of this change. + pub fn set_transition_id(&mut self, new_id: TransitionId) { + match self { + Change::AccountChanged { ref mut id, .. } | + Change::AccountCreated { ref mut id, .. } | + Change::StorageChanged { ref mut id, .. } | + Change::StorageWiped { ref mut id, .. } | + Change::AccountDestroyed { ref mut id, .. } => { + *id = new_id; + } + } + } +} + +/// The state of accounts after execution of one or more transactions, including receipts and new +/// bytecode. +/// +/// The latest state can be found in `accounts`, `storage`, and `bytecode`. The receipts for the +/// transactions that lead to these changes can be found in `receipts`, and each change leading to +/// this state can be found in `changes`. +/// +/// # Wiped Storage +/// +/// The [Storage] type has a field, `wiped`, which denotes whether any of the values contained +/// in storage are valid or not; if `wiped` is `true`, the storage for the account should be +/// considered empty. +/// +/// # Transitions +/// +/// Each [Change] has an `id` field that marks what transition it is part of. Each transaction is +/// its own transition, but there may be 0 or 1 transitions associated with the block. +/// +/// The block level transition includes: +/// +/// - Block rewards +/// - Ommer rewards +/// - Withdrawals +/// - The irregular state change for the DAO hardfork +/// +/// [PostState::finish_transition] should be called after every transaction, and after every block. +/// +/// The first transaction executed and added to the [PostState] has a transition ID of 0, the next +/// one a transition ID of 1, and so on. If the [PostState] is for a single block, and the number of +/// transitions ([PostState::transitions_count]) is greater than the number of transactions in the +/// block, then the last transition is the block transition. +/// +/// For multi-block [PostState]s it is not possible to figure out what transition ID maps on to a +/// transaction or a block. +/// +/// # Shaving Allocations +/// +/// Since most [PostState]s in reth are for multiple blocks it is better to pre-allocate capacity +/// for receipts and changes, which [PostState::new] does, and thus it (or +/// [PostState::with_tx_capacity]) should be preferred to using the [Default] implementation. +#[derive(Debug, Default, Clone, Eq, PartialEq)] +pub struct PostState { + /// The ID of the current transition. + current_transition_id: TransitionId, + /// The state of all modified accounts after execution. + /// + /// If the value contained is `None`, then the account should be deleted. + accounts: BTreeMap>, + /// The state of all modified storage after execution + /// + /// If the contained [Storage] is marked as wiped, then all storage values should be cleared + /// from the database. + storage: BTreeMap, + /// The changes to state that happened during execution + changes: Vec, + /// New code created during the execution + bytecode: BTreeMap, + /// The receipt(s) of the executed transaction(s). + receipts: Vec, +} + +/// Used to determine preallocation sizes of [PostState]'s internal [Vec]s. It denotes the number of +/// best-guess changes each transaction causes to state. +const BEST_GUESS_CHANGES_PER_TX: usize = 8; + +/// How many [Change]s to preallocate for in [PostState]. +/// +/// This is just a guesstimate based on: +/// +/// - Each block having ~200-300 transactions +/// - Each transaction having some amount of changes +const PREALLOC_CHANGES_SIZE: usize = 256 * BEST_GUESS_CHANGES_PER_TX; + +impl PostState { + /// Create an empty [PostState]. + pub fn new() -> Self { + Self { changes: Vec::with_capacity(PREALLOC_CHANGES_SIZE), ..Default::default() } + } + + /// Create an empty [PostState] with pre-allocated space for a certain amount of transactions. + pub fn with_tx_capacity(txs: usize) -> Self { + Self { + changes: Vec::with_capacity(txs * BEST_GUESS_CHANGES_PER_TX), + receipts: Vec::with_capacity(txs), + ..Default::default() + } + } + + /// Get the latest state of accounts. + pub fn accounts(&self) -> &BTreeMap> { + &self.accounts + } + + /// Get the latest state for a specific account. + /// + /// # Returns + /// + /// - `None` if the account does not exist + /// - `Some(&None)` if the account existed, but has since been deleted. + /// - `Some(..)` if the account currently exists + pub fn account(&self, address: &Address) -> Option<&Option> { + self.accounts.get(address) + } + + /// Get the latest state of storage. + pub fn storage(&self) -> &BTreeMap { + &self.storage + } + + /// Get the storage for an account. + pub fn account_storage(&self, address: &Address) -> Option<&Storage> { + self.storage.get(address) + } + + /// Get the changes causing this [PostState]. + pub fn changes(&self) -> &[Change] { + &self.changes + } + + /// Get the newly created bytecodes + pub fn bytecodes(&self) -> &BTreeMap { + &self.bytecode + } + + /// Get a bytecode in the post-state. + pub fn bytecode(&self, code_hash: &H256) -> Option<&Bytecode> { + self.bytecode.get(code_hash) + } + + /// Get the receipts for the transactions executed to form this [PostState]. + pub fn receipts(&self) -> &[Receipt] { + &self.receipts + } + + /// Get the number of transitions causing this [PostState] + pub fn transitions_count(&self) -> usize { + self.current_transition_id as usize + } + + /// Extend this [PostState] with the changes in another [PostState]. + pub fn extend(&mut self, other: PostState) { + self.changes.reserve(other.changes.len()); + + let mut next_transition_id = self.current_transition_id; + for mut change in other.changes.into_iter() { + next_transition_id = self.current_transition_id + change.transition_id(); + change.set_transition_id(next_transition_id); + self.add_and_apply(change); + } + self.receipts.extend(other.receipts); + self.bytecode.extend(other.bytecode); + self.current_transition_id = next_transition_id + 1; + } + + /// Reverts each change up to and including any change that is part of `transition_id`. + /// + /// The reverted changes are removed from this post-state, and their effects are reverted. + /// + /// The reverted changes are returned. + pub fn revert_to(&mut self, transition_id: usize) -> Vec { + let mut changes_to_revert = Vec::new(); + self.changes.retain(|change| { + if change.transition_id() >= transition_id as u64 { + changes_to_revert.push(change.clone()); + false + } else { + true + } + }); + + for change in changes_to_revert.iter_mut().rev() { + change.set_transition_id(change.transition_id() - transition_id as TransitionId); + self.revert(change.clone()); + } + self.current_transition_id = transition_id as TransitionId; + changes_to_revert + } + + /// Reverts each change up to and including any change that is part of `transition_id`. + /// + /// The reverted changes are removed from this post-state, and their effects are reverted. + /// + /// A new post-state containing the pre-revert state, as well as the reverted changes *only* is + /// returned. + /// + /// This effectively splits the post state in two: + /// + /// 1. This post-state has the changes reverted + /// 2. The returned post-state does *not* have the changes reverted, but only contains the + /// descriptions of the changes that were reverted in the first post-state. + pub fn split_at(&mut self, transition_id: usize) -> Self { + // Clone ourselves + let mut non_reverted_state = self.clone(); + + // Revert the desired changes + let reverted_changes = self.revert_to(transition_id); + + // Compute the new `current_transition_id` for `non_reverted_state`. + let new_transition_id = + reverted_changes.last().map(|c| c.transition_id()).unwrap_or_default(); + non_reverted_state.changes = reverted_changes; + non_reverted_state.current_transition_id = new_transition_id + 1; + + non_reverted_state + } + + /// Add a newly created account to the post-state. + pub fn create_account(&mut self, address: Address, account: Account) { + self.add_and_apply(Change::AccountCreated { + id: self.current_transition_id, + address, + account, + }); + } + + /// Add a changed account to the post-state. + /// + /// If the account also has changed storage values, [PostState::change_storage] should also be + /// called. + pub fn change_account(&mut self, address: Address, old: Account, new: Account) { + self.add_and_apply(Change::AccountChanged { + id: self.current_transition_id, + address, + old, + new, + }); + } + + /// Mark an account as destroyed. + pub fn destroy_account(&mut self, address: Address, account: Account) { + self.add_and_apply(Change::AccountDestroyed { + id: self.current_transition_id, + address, + old: account, + }); + self.add_and_apply(Change::StorageWiped { id: self.current_transition_id, address }); + } + + /// Add changed storage values to the post-state. + pub fn change_storage(&mut self, address: Address, changeset: StorageChangeset) { + self.add_and_apply(Change::StorageChanged { + id: self.current_transition_id, + address, + changeset, + }); + } + + /// Add new bytecode to the post-state. + pub fn add_bytecode(&mut self, code_hash: H256, bytecode: Bytecode) { + // TODO: Is this faster than just doing `.insert`? + // Assumption: `insert` will override the value if present, but since the code hash for a + // given bytecode will always be the same, we are overriding with the same value. + // + // In other words: if this entry already exists, replacing the bytecode will replace with + // the same value, which is wasteful. + self.bytecode.entry(code_hash).or_insert(bytecode); + } + + /// Add a transaction receipt to the post-state. + /// + /// Transactions should always include their receipts in the post-state. + pub fn add_receipt(&mut self, receipt: Receipt) { + self.receipts.push(receipt); + } + + /// Mark all prior changes as being part of one transition, and start a new one. + pub fn finish_transition(&mut self) { + self.current_transition_id += 1; + } + + /// Add a new change, and apply its transformations to the current state + pub fn add_and_apply(&mut self, change: Change) { + match &change { + Change::AccountCreated { address, account, .. } | + Change::AccountChanged { address, new: account, .. } => { + self.accounts.insert(*address, Some(*account)); + } + Change::AccountDestroyed { address, .. } => { + self.accounts.insert(*address, None); + } + Change::StorageChanged { address, changeset, .. } => { + let storage = self.storage.entry(*address).or_default(); + storage.wiped = false; + for (slot, (_, current_value)) in changeset { + storage.storage.insert(*slot, *current_value); + } + } + Change::StorageWiped { address, .. } => { + let storage = self.storage.entry(*address).or_default(); + storage.wiped = true; + } + } + + self.changes.push(change); + } + + /// Revert a change, applying the inverse of its transformations to the current state. + fn revert(&mut self, change: Change) { + match &change { + Change::AccountCreated { address, .. } => { + self.accounts.remove(address); + } + Change::AccountChanged { address, old, .. } => { + self.accounts.insert(*address, Some(*old)); + } + Change::AccountDestroyed { address, old, .. } => { + self.accounts.insert(*address, Some(*old)); + } + Change::StorageChanged { address, changeset, .. } => { + let storage = self.storage.entry(*address).or_default(); + storage.wiped = false; + for (slot, (old_value, _)) in changeset { + storage.storage.insert(*slot, *old_value); + } + } + Change::StorageWiped { address, .. } => { + let storage = self.storage.entry(*address).or_default(); + storage.wiped = false; + } + } + } + + /// Write the post state to the database. + pub fn write_to_db<'a, TX: DbTxMut<'a> + DbTx<'a>>( + mut self, + tx: &TX, + first_transition_id: TransitionId, + ) -> Result<(), DbError> { + // Collect and sort changesets by their key to improve write performance + let mut changesets = std::mem::take(&mut self.changes); + changesets + .sort_unstable_by_key(|changeset| (changeset.transition_id(), changeset.address())); + + // Partition changesets into account and storage changes + let (account_changes, storage_changes): (Vec, Vec) = + changesets.into_iter().partition(|changeset| { + matches!( + changeset, + Change::AccountChanged { .. } | + Change::AccountCreated { .. } | + Change::AccountDestroyed { .. } + ) + }); + + // Write account changes + let mut account_changeset_cursor = tx.cursor_dup_write::()?; + for changeset in account_changes.into_iter() { + match changeset { + Change::AccountDestroyed { id, address, old } | + Change::AccountChanged { id, address, old, .. } => { + account_changeset_cursor.append_dup( + first_transition_id + id, + AccountBeforeTx { address, info: Some(old) }, + )?; + } + Change::AccountCreated { id, address, .. } => { + account_changeset_cursor.append_dup( + first_transition_id + id, + AccountBeforeTx { address, info: None }, + )?; + } + _ => unreachable!(), + } + } + + // Write storage changes + let mut storages_cursor = tx.cursor_dup_write::()?; + let mut storage_changeset_cursor = tx.cursor_dup_write::()?; + for changeset in storage_changes.into_iter() { + match changeset { + Change::StorageChanged { id, address, changeset } => { + let storage_id = TransitionIdAddress((first_transition_id + id, address)); + + for (key, (old_value, _)) in changeset { + storage_changeset_cursor.append_dup( + storage_id, + StorageEntry { key: H256(key.to_be_bytes()), value: old_value }, + )?; + } + } + Change::StorageWiped { id, address } => { + let storage_id = TransitionIdAddress((first_transition_id + id, address)); + + if let Some((_, entry)) = storages_cursor.seek_exact(address)? { + storage_changeset_cursor.append_dup(storage_id, entry)?; + + while let Some(entry) = storages_cursor.next_dup_val()? { + storage_changeset_cursor.append_dup(storage_id, entry)?; + } + } + } + _ => unreachable!(), + } + } + + // Write new storage state + for (address, storage) in self.storage.into_iter() { + if storage.wiped { + if storages_cursor.seek_exact(address)?.is_some() { + storages_cursor.delete_current_duplicates()?; + } + + // If the storage is marked as wiped, it might still contain values. This is to + // avoid deallocating where possible, but these values should not be written to the + // database. + continue + } + + for (key, value) in storage.storage { + let key = H256(key.to_be_bytes()); + if let Some(entry) = storages_cursor.seek_by_key_subkey(address, key)? { + if entry.key == key { + storages_cursor.delete_current()?; + } + } + + if value != U256::ZERO { + storages_cursor.upsert(address, StorageEntry { key, value })?; + } + } + } + + // Write new account state + let mut accounts_cursor = tx.cursor_write::()?; + for (address, account) in self.accounts.into_iter() { + if let Some(account) = account { + accounts_cursor.upsert(address, account)?; + } else if accounts_cursor.seek_exact(address)?.is_some() { + accounts_cursor.delete_current()?; + } + } + + // Write bytecode + let mut bytecodes_cursor = tx.cursor_write::()?; + for (hash, bytecode) in self.bytecode.into_iter() { + bytecodes_cursor.upsert(hash, bytecode)?; + } + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use reth_db::{ + database::Database, + mdbx::{test_utils, Env, EnvKind, WriteMap}, + transaction::DbTx, + }; + use std::sync::Arc; + + #[test] + fn extend() { + let mut a = PostState::new(); + a.create_account(Address::zero(), Account::default()); + a.destroy_account(Address::zero(), Account::default()); + a.finish_transition(); + + assert_eq!(a.transitions_count(), 1); + assert_eq!(a.changes().len(), 3); + + let mut b = PostState::new(); + b.create_account(Address::repeat_byte(0xff), Account::default()); + b.finish_transition(); + + assert_eq!(b.transitions_count(), 1); + assert_eq!(b.changes.len(), 1); + + let mut c = a.clone(); + c.extend(b.clone()); + + assert_eq!(c.transitions_count(), 2); + assert_eq!(c.changes.len(), a.changes.len() + b.changes.len()); + } + + #[test] + fn write_to_db_account_info() { + let db: Arc> = test_utils::create_test_db(EnvKind::RW); + let tx = db.tx_mut().expect("Could not get database tx"); + + let mut post_state = PostState::new(); + + let address_a = Address::zero(); + let address_b = Address::repeat_byte(0xff); + + let account_a = Account { balance: U256::from(1), nonce: 1, bytecode_hash: None }; + let account_b = Account { balance: U256::from(2), nonce: 2, bytecode_hash: None }; + let account_b_changed = Account { balance: U256::from(3), nonce: 3, bytecode_hash: None }; + + // 0x00.. is created + post_state.create_account(address_a, account_a); + // 0x11.. is changed (balance + 1, nonce + 1) + post_state.change_account(address_b, account_b, account_b_changed); + post_state.write_to_db(&tx, 0).expect("Could not write post state to DB"); + + // Check plain state + assert_eq!( + tx.get::(address_a).expect("Could not read account state"), + Some(account_a), + "Account A state is wrong" + ); + assert_eq!( + tx.get::(address_b).expect("Could not read account state"), + Some(account_b_changed), + "Account B state is wrong" + ); + + // Check change set + let mut changeset_cursor = tx + .cursor_dup_read::() + .expect("Could not open changeset cursor"); + assert_eq!( + changeset_cursor.seek_exact(0).expect("Could not read account change set"), + Some((0, AccountBeforeTx { address: address_a, info: None })), + "Account A changeset is wrong" + ); + assert_eq!( + changeset_cursor.next_dup().expect("Changeset table is malformed"), + Some((0, AccountBeforeTx { address: address_b, info: Some(account_b) })), + "Account B changeset is wrong" + ); + + let mut post_state = PostState::new(); + // 0x11.. is destroyed + post_state.destroy_account(address_b, account_b_changed); + post_state.write_to_db(&tx, 1).expect("Could not write second post state to DB"); + + // Check new plain state for account B + assert_eq!( + tx.get::(address_b).expect("Could not read account state"), + None, + "Account B should be deleted" + ); + + // Check change set + assert_eq!( + changeset_cursor.seek_exact(1).expect("Could not read account change set"), + Some((1, AccountBeforeTx { address: address_b, info: Some(account_b_changed) })), + "Account B changeset is wrong after deletion" + ); + } + + #[test] + fn write_to_db_storage() { + let db: Arc> = test_utils::create_test_db(EnvKind::RW); + let tx = db.tx_mut().expect("Could not get database tx"); + + let mut post_state = PostState::new(); + + let address_a = Address::zero(); + let address_b = Address::repeat_byte(0xff); + + // 0x00 => 0 => 1 + // 0x01 => 0 => 2 + let storage_a_changeset = BTreeMap::from([ + (U256::from(0), (U256::from(0), U256::from(1))), + (U256::from(1), (U256::from(0), U256::from(2))), + ]); + + // 0x01 => 1 => 2 + let storage_b_changeset = BTreeMap::from([(U256::from(1), (U256::from(1), U256::from(2)))]); + + post_state.change_storage(address_a, storage_a_changeset); + post_state.change_storage(address_b, storage_b_changeset); + post_state.write_to_db(&tx, 0).expect("Could not write post state to DB"); + + // Check plain storage state + let mut storage_cursor = tx + .cursor_dup_read::() + .expect("Could not open plain storage state cursor"); + + assert_eq!( + storage_cursor.seek_exact(address_a).unwrap(), + Some((address_a, StorageEntry { key: H256::zero(), value: U256::from(1) })), + "Slot 0 for account A should be 1" + ); + assert_eq!( + storage_cursor.next_dup().unwrap(), + Some(( + address_a, + StorageEntry { key: H256::from(U256::from(1).to_be_bytes()), value: U256::from(2) } + )), + "Slot 1 for account A should be 2" + ); + assert_eq!( + storage_cursor.next_dup().unwrap(), + None, + "Account A should only have 2 storage slots" + ); + + assert_eq!( + storage_cursor.seek_exact(address_b).unwrap(), + Some(( + address_b, + StorageEntry { key: H256::from(U256::from(1).to_be_bytes()), value: U256::from(2) } + )), + "Slot 1 for account B should be 2" + ); + assert_eq!( + storage_cursor.next_dup().unwrap(), + None, + "Account B should only have 1 storage slot" + ); + + // Check change set + let mut changeset_cursor = tx + .cursor_dup_read::() + .expect("Could not open storage changeset cursor"); + assert_eq!( + changeset_cursor.seek_exact(TransitionIdAddress((0, address_a))).unwrap(), + Some(( + TransitionIdAddress((0, address_a)), + StorageEntry { key: H256::zero(), value: U256::from(0) } + )), + "Slot 0 for account A should have changed from 0" + ); + assert_eq!( + changeset_cursor.next_dup().unwrap(), + Some(( + TransitionIdAddress((0, address_a)), + StorageEntry { key: H256::from(U256::from(1).to_be_bytes()), value: U256::from(0) } + )), + "Slot 1 for account A should have changed from 0" + ); + assert_eq!( + changeset_cursor.next_dup().unwrap(), + None, + "Account A should only be in the changeset 2 times" + ); + + assert_eq!( + changeset_cursor.seek_exact(TransitionIdAddress((0, address_b))).unwrap(), + Some(( + TransitionIdAddress((0, address_b)), + StorageEntry { key: H256::from(U256::from(1).to_be_bytes()), value: U256::from(1) } + )), + "Slot 1 for account B should have changed from 1" + ); + assert_eq!( + changeset_cursor.next_dup().unwrap(), + None, + "Account B should only be in the changeset 1 time" + ); + + // Delete account A + let mut post_state = PostState::new(); + post_state.destroy_account(address_a, Account::default()); + post_state.write_to_db(&tx, 1).expect("Could not write post state to DB"); + + assert_eq!( + storage_cursor.seek_exact(address_a).unwrap(), + None, + "Account A should have no storage slots after deletion" + ); + + assert_eq!( + changeset_cursor.seek_exact(TransitionIdAddress((1, address_a))).unwrap(), + Some(( + TransitionIdAddress((1, address_a)), + StorageEntry { key: H256::zero(), value: U256::from(1) } + )), + "Slot 0 for account A should have changed from 1 on deletion" + ); + assert_eq!( + changeset_cursor.next_dup().unwrap(), + Some(( + TransitionIdAddress((1, address_a)), + StorageEntry { key: H256::from(U256::from(1).to_be_bytes()), value: U256::from(2) } + )), + "Slot 1 for account A should have changed from 2 on deletion" + ); + assert_eq!( + changeset_cursor.next_dup().unwrap(), + None, + "Account A should only be in the changeset 2 times on deletion" + ); + } + + #[test] + fn revert_to() { + let mut state = PostState::new(); + state.create_account( + Address::repeat_byte(0), + Account { nonce: 1, balance: U256::from(1), bytecode_hash: None }, + ); + state.finish_transition(); + let revert_to = state.current_transition_id; + state.create_account( + Address::repeat_byte(0xff), + Account { nonce: 2, balance: U256::from(2), bytecode_hash: None }, + ); + state.finish_transition(); + + assert_eq!(state.transitions_count(), 2); + assert_eq!(state.accounts().len(), 2); + + let reverted_changes = state.revert_to(revert_to as usize); + assert_eq!(state.accounts().len(), 1); + assert_eq!(state.transitions_count(), 1); + assert_eq!(reverted_changes.len(), 1); + } +} diff --git a/crates/storage/provider/src/test_utils/blocks.rs b/crates/storage/provider/src/test_utils/blocks.rs index 9beffb33949..19d4545eacd 100644 --- a/crates/storage/provider/src/test_utils/blocks.rs +++ b/crates/storage/provider/src/test_utils/blocks.rs @@ -1,15 +1,10 @@ //! Dummy blocks and data for tests -use crate::{ - execution_result::{ - AccountChangeSet, AccountInfoChangeSet, ExecutionResult, TransactionChangeSet, - }, - Transaction, -}; +use crate::{post_state::PostState, Transaction}; use reth_db::{database::Database, models::StoredBlockBody, tables}; use reth_primitives::{ - hex_literal::hex, proofs::EMPTY_ROOT, Account, Header, Receipt, SealedBlock, - SealedBlockWithSenders, Withdrawal, H160, H256, U256, + hex_literal::hex, proofs::EMPTY_ROOT, Account, Header, SealedBlock, SealedBlockWithSenders, + Withdrawal, H160, H256, U256, }; use reth_rlp::Decodable; use std::collections::BTreeMap; @@ -54,7 +49,7 @@ pub struct BlockChainTestData { /// Genesis pub genesis: SealedBlock, /// Blocks with its execution result - pub blocks: Vec<(SealedBlockWithSenders, ExecutionResult)>, + pub blocks: Vec<(SealedBlockWithSenders, PostState)>, } impl Default for BlockChainTestData { @@ -75,7 +70,7 @@ pub fn genesis() -> SealedBlock { } /// Block one that points to genesis -fn block1() -> (SealedBlockWithSenders, ExecutionResult) { +fn block1() -> (SealedBlockWithSenders, PostState) { let mut block_rlp = hex!("f9025ff901f7a0c86e8cc0310ae7c531c758678ddbfd16fc51c8cef8cec650b032de9869e8b94fa01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347942adc25665018aa1fe0e6bc666dac8fc2697ff9baa050554882fbbda2c2fd93fdc466db9946ea262a67f7a76cc169e714f105ab583da00967f09ef1dfed20c0eacfaa94d5cd4002eda3242ac47eae68972d07b106d192a0e3c8b47fbfc94667ef4cceb17e5cc21e3b1eebd442cebb27f07562b33836290db90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008302000001830f42408238108203e800a00000000000000000000000000000000000000000000000000000000000000000880000000000000000f862f860800a83061a8094095e7baea6a6c7c4c2dfeb977efac326af552d8780801ba072ed817487b84ba367d15d2f039b5fc5f087d0a8882fbdf73e8cb49357e1ce30a0403d800545b8fc544f92ce8124e2255f8c3c6af93f28243a120585d4c4c6a2a3c0").as_slice(); let mut block = SealedBlock::decode(&mut block_rlp).unwrap(); block.withdrawals = Some(vec![Withdrawal::default()]); @@ -86,28 +81,29 @@ fn block1() -> (SealedBlockWithSenders, ExecutionResult) { header.parent_hash = H256::zero(); block.header = header.seal_slow(); - let mut account_changeset = AccountChangeSet { - account: AccountInfoChangeSet::Created { - new: Account { nonce: 1, balance: U256::from(10), bytecode_hash: None }, - }, - ..Default::default() - }; - account_changeset.storage.insert(U256::from(5), (U256::ZERO, U256::from(10))); - - let exec_res = ExecutionResult { - tx_changesets: vec![TransactionChangeSet { - receipt: Receipt::default(), /* receipts are not saved. */ - changeset: BTreeMap::from([(H160([0x60; 20]), account_changeset.clone())]), - new_bytecodes: BTreeMap::from([]), - }], - block_changesets: BTreeMap::from([(H160([0x61; 20]), account_changeset.account)]), - }; + let mut post_state = PostState::default(); + // Transaction changes + post_state.create_account( + H160([0x60; 20]), + Account { nonce: 1, balance: U256::from(10), bytecode_hash: None }, + ); + post_state.change_storage( + H160([0x60; 20]), + BTreeMap::from([(U256::from(5), (U256::ZERO, U256::from(10)))]), + ); + post_state.finish_transition(); + // Block changes + post_state.create_account( + H160([0x61; 20]), + Account { nonce: 1, balance: U256::from(10), bytecode_hash: None }, + ); + post_state.finish_transition(); - (SealedBlockWithSenders { block, senders: vec![H160([0x30; 20])] }, exec_res) + (SealedBlockWithSenders { block, senders: vec![H160([0x30; 20])] }, post_state) } /// Block two that points to block 1 -fn block2() -> (SealedBlockWithSenders, ExecutionResult) { +fn block2() -> (SealedBlockWithSenders, PostState) { let mut block_rlp = hex!("f9025ff901f7a0c86e8cc0310ae7c531c758678ddbfd16fc51c8cef8cec650b032de9869e8b94fa01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347942adc25665018aa1fe0e6bc666dac8fc2697ff9baa050554882fbbda2c2fd93fdc466db9946ea262a67f7a76cc169e714f105ab583da00967f09ef1dfed20c0eacfaa94d5cd4002eda3242ac47eae68972d07b106d192a0e3c8b47fbfc94667ef4cceb17e5cc21e3b1eebd442cebb27f07562b33836290db90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008302000001830f42408238108203e800a00000000000000000000000000000000000000000000000000000000000000000880000000000000000f862f860800a83061a8094095e7baea6a6c7c4c2dfeb977efac326af552d8780801ba072ed817487b84ba367d15d2f039b5fc5f087d0a8882fbdf73e8cb49357e1ce30a0403d800545b8fc544f92ce8124e2255f8c3c6af93f28243a120585d4c4c6a2a3c0").as_slice(); let mut block = SealedBlock::decode(&mut block_rlp).unwrap(); block.withdrawals = Some(vec![Withdrawal::default()]); @@ -120,27 +116,25 @@ fn block2() -> (SealedBlockWithSenders, ExecutionResult) { H256(hex!("d846db2ab174c492cfe985c18fa75b154e20572bc33bb1c67cf5d2995791bdb7")); block.header = header.seal_slow(); - let mut account_changeset = AccountChangeSet::default(); - // storage will be moved - let info_changeset = AccountInfoChangeSet::Changed { - old: Account { nonce: 1, balance: U256::from(10), bytecode_hash: None }, - new: Account { nonce: 2, balance: U256::from(15), bytecode_hash: None }, - }; - account_changeset.account = info_changeset; - account_changeset.storage.insert(U256::from(5), (U256::from(10), U256::from(15))); - - let block_changeset = AccountInfoChangeSet::Changed { - old: Account { nonce: 2, balance: U256::from(15), bytecode_hash: None }, - new: Account { nonce: 3, balance: U256::from(20), bytecode_hash: None }, - }; - let exec_res = ExecutionResult { - tx_changesets: vec![TransactionChangeSet { - receipt: Receipt::default(), /* receipts are not saved. */ - changeset: BTreeMap::from([(H160([0x60; 20]), account_changeset.clone())]), - new_bytecodes: BTreeMap::from([]), - }], - block_changesets: BTreeMap::from([(H160([0x60; 20]), block_changeset)]), - }; + let mut post_state = PostState::default(); + // Transaction changes + post_state.change_account( + H160([0x60; 20]), + Account { nonce: 1, balance: U256::from(10), bytecode_hash: None }, + Account { nonce: 2, balance: U256::from(15), bytecode_hash: None }, + ); + post_state.change_storage( + H160([0x60; 20]), + BTreeMap::from([(U256::from(5), (U256::from(10), U256::from(15)))]), + ); + post_state.finish_transition(); + // Block changes + post_state.change_account( + H160([0x60; 20]), + Account { nonce: 2, balance: U256::from(15), bytecode_hash: None }, + Account { nonce: 3, balance: U256::from(20), bytecode_hash: None }, + ); + post_state.finish_transition(); - (SealedBlockWithSenders { block, senders: vec![H160([0x31; 20])] }, exec_res) + (SealedBlockWithSenders { block, senders: vec![H160([0x31; 20])] }, post_state) } diff --git a/crates/storage/provider/src/traits/executor.rs b/crates/storage/provider/src/traits/executor.rs index 98150d506f8..e25ab2c5649 100644 --- a/crates/storage/provider/src/traits/executor.rs +++ b/crates/storage/provider/src/traits/executor.rs @@ -1,6 +1,6 @@ //! Executor Factory -use crate::{execution_result::ExecutionResult, StateProvider}; +use crate::{post_state::PostState, StateProvider}; use reth_interfaces::executor::Error; use reth_primitives::{Address, Block, ChainSpec, U256}; @@ -33,7 +33,7 @@ pub trait BlockExecutor { block: &Block, total_difficulty: U256, senders: Option>, - ) -> Result; + ) -> Result; /// Executes the block and checks receipts fn execute_and_verify_receipt( @@ -41,5 +41,5 @@ pub trait BlockExecutor { block: &Block, total_difficulty: U256, senders: Option>, - ) -> Result; + ) -> Result; } diff --git a/crates/storage/provider/src/transaction.rs b/crates/storage/provider/src/transaction.rs index bf60a31b154..a4268c8df98 100644 --- a/crates/storage/provider/src/transaction.rs +++ b/crates/storage/provider/src/transaction.rs @@ -1,3 +1,8 @@ +use crate::{ + insert_canonical_block, + post_state::{Change, PostState, StorageChangeset}, + trie::{DBTrieLoader, TrieError}, +}; use itertools::{izip, Itertools}; use reth_db::{ common::KeyValue, @@ -6,7 +11,7 @@ use reth_db::{ models::{ sharded_key, storage_sharded_key::{self, StorageShardedKey}, - ShardedKey, StoredBlockBody, TransitionIdAddress, + AccountBeforeTx, ShardedKey, StoredBlockBody, TransitionIdAddress, }, table::Table, tables, @@ -15,25 +20,16 @@ use reth_db::{ }; use reth_interfaces::{db::Error as DbError, provider::ProviderError}; use reth_primitives::{ - keccak256, proofs::EMPTY_ROOT, Account, Address, BlockHash, BlockNumber, Bytecode, ChainSpec, - Hardfork, Header, Receipt, SealedBlock, SealedBlockWithSenders, StorageEntry, - TransactionSignedEcRecovered, TransitionId, TxNumber, H256, U256, + keccak256, proofs::EMPTY_ROOT, Account, Address, BlockHash, BlockNumber, ChainSpec, Hardfork, + Header, SealedBlock, SealedBlockWithSenders, StorageEntry, TransactionSignedEcRecovered, + TransitionId, TxNumber, H256, U256, }; -use reth_tracing::tracing::{info, trace}; use std::{ collections::{btree_map::Entry, BTreeMap, BTreeSet}, fmt::Debug, ops::{Bound, Deref, DerefMut, Range, RangeBounds}, }; -use crate::{ - execution_result::{AccountInfoChangeSet, TransactionChangeSet}, - insert_canonical_block, - trie::{DBTrieLoader, TrieError}, -}; - -use crate::execution_result::{AccountChangeSet, ExecutionResult}; - /// A container for any DB transaction that will open a new inner transaction when the current /// one is committed. // NOTE: This container is needed since `Transaction::commit` takes `mut self`, so methods in @@ -317,7 +313,7 @@ where pub fn get_block_execution_result_range( &self, range: impl RangeBounds + Clone, - ) -> Result, TransactionError> { + ) -> Result, TransactionError> { self.get_take_block_execution_result_range::(range) } @@ -327,7 +323,7 @@ where pub fn take_block_execution_result_range( &self, range: impl RangeBounds + Clone, - ) -> Result, TransactionError> { + ) -> Result, TransactionError> { self.get_take_block_execution_result_range::(range) } @@ -336,7 +332,7 @@ where &self, chain_spec: &ChainSpec, range: impl RangeBounds + Clone, - ) -> Result, TransactionError> { + ) -> Result, TransactionError> { self.get_take_block_and_execution_range::(chain_spec, range) } @@ -345,7 +341,7 @@ where &self, chain_spec: &ChainSpec, range: impl RangeBounds + Clone, - ) -> Result, TransactionError> { + ) -> Result, TransactionError> { self.get_take_block_and_execution_range::(chain_spec, range) } @@ -514,69 +510,87 @@ where Ok(()) } - /// Insert full block and make it canonical + /// Insert full block and make it canonical. /// - /// This is atomic operation and transaction will do one commit at the end of the function. - pub fn insert_block( - &mut self, - block: SealedBlockWithSenders, - chain_spec: &ChainSpec, - changeset: ExecutionResult, - ) -> Result<(), TransactionError> { + /// This inserts the block and builds history related indexes. Once all blocks in a chain have + /// been committed, the state root needs to be inserted separately with + /// [`Transaction::insert_hashes`]. + /// + /// # Note + /// + /// This assumes that we are using beacon consensus and that the block is post-merge, which + /// means that the block will have no block reward. + pub fn insert_block(&mut self, block: SealedBlockWithSenders) -> Result<(), TransactionError> { // Header, Body, SenderRecovery, TD, TxLookup stages let (block, senders) = block.into_components(); - let block_number = block.number; - let block_state_root = block.state_root; - let block_hash = block.hash(); - let parent_block_number = block.number.saturating_sub(1); let (from, to) = insert_canonical_block(self.deref_mut(), block, Some(senders), false).unwrap(); - // execution stage - self.insert_execution_result(vec![changeset], chain_spec, parent_block_number)?; + // account history stage + { + let indices = self.get_account_transition_ids_from_changeset(from, to)?; + self.insert_account_history_index(indices)?; + } + + // storage history stage + { + let indices = self.get_storage_transition_ids_from_changeset(from, to)?; + self.insert_storage_history_index(indices)?; + } + + Ok(()) + } + /// Calculate the hashes of all changed accounts and storages, and finally calculate the state + /// root. + /// + /// The chain goes from `fork_block_number + 1` to `current_block_number`, and hashes are + /// calculated from `from_transition_id` to `to_transition_id`. + /// + /// The resulting state root is compared with `expected_state_root`. + pub fn insert_hashes( + &mut self, + fork_block_number: BlockNumber, + from_transition_id: TransitionId, + to_transition_id: TransitionId, + current_block_number: BlockNumber, + current_block_hash: H256, + expected_state_root: H256, + ) -> Result<(), TransactionError> { // storage hashing stage { - let lists = self.get_addresses_and_keys_of_changed_storages(from, to)?; + let lists = self + .get_addresses_and_keys_of_changed_storages(from_transition_id, to_transition_id)?; let storages = self.get_plainstate_storages(lists.into_iter())?; self.insert_storage_for_hashing(storages.into_iter())?; } // account hashing stage { - let lists = self.get_addresses_of_changed_accounts(from, to)?; + let lists = + self.get_addresses_of_changed_accounts(from_transition_id, to_transition_id)?; let accounts = self.get_plainstate_accounts(lists.into_iter())?; self.insert_account_for_hashing(accounts.into_iter())?; } // merkle tree { - let current_root = self.get_header(parent_block_number)?.state_root; + let current_root = self.get_header(fork_block_number)?.state_root; let mut loader = DBTrieLoader::new(self.deref_mut()); - let root = loader.update_root(current_root, from..to).and_then(|e| e.root())?; - if root != block_state_root { + let root = loader + .update_root(current_root, from_transition_id..to_transition_id) + .and_then(|e| e.root())?; + if root != expected_state_root { return Err(TransactionError::StateTrieRootMismatch { got: root, - expected: block_state_root, - block_number, - block_hash, + expected: expected_state_root, + block_number: current_block_number, + block_hash: current_block_hash, }) } } - // account history stage - { - let indices = self.get_account_transition_ids_from_changeset(from, to)?; - self.insert_account_history_index(indices)?; - } - - // storage history stage - { - let indices = self.get_storage_transition_ids_from_changeset(from, to)?; - self.insert_storage_history_index(indices)?; - } - Ok(()) } @@ -764,27 +778,32 @@ where Ok(blocks) } - /// Transverse over changesets and plain state and recreated the execution results. - /// - /// Iterate over [tables::BlockTransitionIndex] and take all transitions. - /// Then iterate over all [tables::StorageChangeSet] and [tables::AccountChangeSet] in reverse - /// order and populate all changesets. To be able to populate changesets correctly and to - /// have both, new and old value of account/storage, we needs to have local state and access - /// to [tables::PlainAccountState] [tables::PlainStorageState]. - /// While iteration over acocunt/storage changesets. - /// At first instance of account/storage we are taking old value from changeset, - /// new value from plain state and saving old value to local state. - /// As second accounter of same account/storage we are again taking old value from changeset, - /// but new value is taken from local state and old value is again updated to local state. + /// Traverse over changesets and plain state and recreate the [`PostState`]s for the given range + /// of blocks. /// - /// Now if TAKE is true we will use local state and update all old values to plain state tables. + /// 1. Iterate over the [BlockTransitionIndex][tables::BlockTransitionIndex] table to get all + /// the transitions + /// 2. Iterate over the [StorageChangeSet][tables::StorageChangeSet] table + /// and the [AccountChangeSet][tables::AccountChangeSet] tables in reverse order to reconstruct + /// the changesets. + /// - In order to have both the old and new values in the changesets, we also access the + /// plain state tables. + /// 3. While iterating over the changeset tables, if we encounter a new account or storage slot, + /// we: + /// 1. Take the old value from the changeset + /// 2. Take the new value from the plain state + /// 3. Save the old value to the local state + /// 4. While iterating over the changeset tables, if we encounter an account/storage slot we + /// have seen before we: + /// 1. Take the old value from the changeset + /// 2. Take the new value from the local state + /// 3. Set the local state to the value in the changeset /// - /// After that, iterate over [`tables::BlockBodies`] and pack created changesets in block chunks - /// taking care if block has block changesets or not. + /// If `TAKE` is `true`, the local state will be written to the plain state tables. fn get_take_block_execution_result_range( &self, range: impl RangeBounds + Clone, - ) -> Result, TransactionError> { + ) -> Result, TransactionError> { let block_transition = self.get_or_take::(range.clone())?; @@ -815,10 +834,7 @@ where // Double option around Account represent if Account state is know (first option) and // account is removed (Second Option) type LocalPlainState = BTreeMap>, BTreeMap)>; - type Changesets = BTreeMap< - TransitionId, - BTreeMap)>, - >; + type Changesets = BTreeMap>; let mut local_plain_state: LocalPlainState = BTreeMap::new(); @@ -832,33 +848,51 @@ where // add account changeset changes for (transition_id, account_before) in account_changeset.into_iter().rev() { - let new_info = match local_plain_state.entry(account_before.address) { + let AccountBeforeTx { info: old_info, address } = account_before; + let new_info = match local_plain_state.entry(address) { Entry::Vacant(entry) => { - let new_account = - plain_accounts_cursor.seek(account_before.address)?.map(|(_s, i)| i); - entry.insert((Some(account_before.info), BTreeMap::new())); + let new_account = plain_accounts_cursor.seek_exact(address)?.map(|kv| kv.1); + entry.insert((Some(old_info), BTreeMap::new())); new_account } Entry::Occupied(mut entry) => { - let new_account = - std::mem::replace(&mut entry.get_mut().0, Some(account_before.info)); + let new_account = std::mem::replace(&mut entry.get_mut().0, Some(old_info)); new_account.expect("As we are stacking account first, account would always be Some(Some) or Some(None)") } }; - let account_info_changeset = AccountInfoChangeSet::new(account_before.info, new_info); - // insert changeset to transition id. Multiple account for same transition Id are not - // possible. - all_changesets - .entry(transition_id) - .or_default() - .entry(account_before.address) - .or_default() - .0 = account_info_changeset + + let change = match (old_info, new_info) { + (Some(old), Some(new)) => { + if new != old { + Change::AccountChanged { + id: transition_id, + address, + old, + new, + } + } else { + unreachable!("Junk data in database: an account changeset did not represent any change"); + } + } + (None, Some(account)) => Change::AccountCreated { + id: transition_id, + address, + account + }, + (Some(old), None) => Change::AccountDestroyed { + id: transition_id, + address, + old + }, + (None, None) => unreachable!("Junk data in database: an account changeset transitioned from no account to no account"), + }; + all_changesets.entry(transition_id).or_default().push(change); } // add storage changeset changes + let mut storage_changes: BTreeMap = BTreeMap::new(); for (transition_and_address, storage_entry) in storage_changeset.into_iter().rev() { - let TransitionIdAddress((transition_id, address)) = transition_and_address; + let TransitionIdAddress((_, address)) = transition_and_address; let new_storage = match local_plain_state.entry(address).or_default().1.entry(storage_entry.key) { Entry::Vacant(entry) => { @@ -873,13 +907,20 @@ where std::mem::replace(entry.get_mut(), storage_entry.value) } }; - all_changesets - .entry(transition_id) - .or_default() - .entry(address) - .or_default() - .1 - .insert(storage_entry.key, (storage_entry.value, new_storage)); + storage_changes.entry(transition_and_address).or_default().insert( + U256::from_be_bytes(storage_entry.key.0), + (storage_entry.value, new_storage), + ); + } + + for (TransitionIdAddress((transition_id, address)), storage_changeset) in + storage_changes.into_iter() + { + all_changesets.entry(transition_id).or_default().push(Change::StorageChanged { + id: transition_id, + address, + changeset: storage_changeset, + }); } if TAKE { @@ -894,10 +935,12 @@ where plain_accounts_cursor.delete_current()?; } } + // revert storages for (storage_key, storage_value) in storage.into_iter() { let storage_entry = StorageEntry { key: storage_key, value: storage_value }; // delete previous value + // TODO: This does not use dupsort features if plain_storage_cursor .seek_by_key_subkey(address, storage_key)? .filter(|s| s.key == storage_key) @@ -905,6 +948,8 @@ where { plain_storage_cursor.delete_current()? } + + // TODO: This does not use dupsort features // insert value if needed if storage_value != U256::ZERO { plain_storage_cursor.insert(address, storage_entry)?; @@ -913,73 +958,39 @@ where } } - // NOTE: Some storage changesets can be empty, - // all account changeset have at least beneficiary fee transfer. - // iterate over block body and create ExecutionResult let mut block_exec_results = Vec::new(); - - let mut changeset_iter = all_changesets.into_iter(); let mut block_transition_iter = block_transition.into_iter(); let mut next_transition_id = from; - let mut next_changeset = changeset_iter.next().unwrap_or_default(); // loop break if we are at the end of the blocks. for (_, block_body) in block_bodies.into_iter() { - let mut block_exec_res = ExecutionResult::default(); + let mut block_exec_res = PostState::new(); for _ in 0..block_body.tx_count { - // only if next_changeset - let changeset = if next_transition_id == next_changeset.0 { - let changeset = next_changeset - .1 - .into_iter() - .map(|(address, (account, storage))| { - ( - address, - AccountChangeSet { - account, - storage: storage - .into_iter() - .map(|(key, val)| (U256::from_be_bytes(key.0), val)) - .collect(), - wipe_storage: false, /* it is always false as all storage - * changesets for selfdestruct are - * already accounted. */ - }, - ) - }) - .collect(); - next_changeset = changeset_iter.next().unwrap_or_default(); - changeset - } else { - BTreeMap::new() - }; - + if let Some(changes) = all_changesets.remove(&next_transition_id) { + for mut change in changes.into_iter() { + change + .set_transition_id(block_exec_res.transitions_count() as TransitionId); + block_exec_res.add_and_apply(change); + } + } + block_exec_res.finish_transition(); next_transition_id += 1; - block_exec_res.tx_changesets.push(TransactionChangeSet { - receipt: Receipt::default(), /* TODO(receipt) when they are saved, load them - * from db */ - changeset, - new_bytecodes: Default::default(), /* TODO(bytecode), bytecode is not cleared - * so it is same sa previous. */ - }); } let Some((_,block_transition)) = block_transition_iter.next() else { break}; // if block transition points to 1+next transition id it means that there is block // changeset. if block_transition == next_transition_id + 1 { - // assert last_transition_id == block_transition - if next_transition_id == next_changeset.0 { - // take block changeset - block_exec_res.block_changesets = next_changeset - .1 - .into_iter() - .map(|(address, (account, _))| (address, account)) - .collect(); - next_changeset = changeset_iter.next().unwrap_or_default(); + if let Some(changes) = all_changesets.remove(&next_transition_id) { + for mut change in changes.into_iter() { + change + .set_transition_id(block_exec_res.transitions_count() as TransitionId); + block_exec_res.add_and_apply(change); + } + block_exec_res.finish_transition(); + next_transition_id += 1; } - next_transition_id += 1; } block_exec_results.push(block_exec_res) } @@ -991,7 +1002,7 @@ where &self, chain_spec: &ChainSpec, range: impl RangeBounds + Clone, - ) -> Result, TransactionError> { + ) -> Result, TransactionError> { if TAKE { let (from_transition, parent_number, parent_state_root) = match range.start_bound() { Bound::Included(n) => { @@ -1355,134 +1366,6 @@ where Ok(()) } - /// Used inside execution stage to commit created account storage changesets for transaction or - /// block state change. - pub fn insert_execution_result( - &self, - changesets: Vec, - chain_spec: &ChainSpec, - parent_block_number: u64, - ) -> Result<(), TransactionError> { - // Get last tx count so that we can know amount of transaction in the block. - let mut current_transition_id = self - .get::(parent_block_number)? - .ok_or(ProviderError::BlockTransition { block_number: parent_block_number })?; - - info!(target: "sync::stages::execution", current_transition_id, blocks = changesets.len(), "Inserting execution results"); - - // apply changes to plain database. - let mut block_number = parent_block_number; - for results in changesets.into_iter() { - block_number += 1; - let spurious_dragon_active = - chain_spec.fork(Hardfork::SpuriousDragon).active_at_block(block_number); - // insert state change set - for result in results.tx_changesets.into_iter() { - for (address, account_change_set) in result.changeset.into_iter() { - let AccountChangeSet { account, wipe_storage, storage } = account_change_set; - // apply account change to db. Updates AccountChangeSet and PlainAccountState - // tables. - trace!(target: "sync::stages::execution", ?address, current_transition_id, ?account, wipe_storage, "Applying account changeset"); - account.apply_to_db( - &**self, - address, - current_transition_id, - spurious_dragon_active, - )?; - - let storage_id = TransitionIdAddress((current_transition_id, address)); - - // cast key to H256 and trace the change - let storage = storage - .into_iter() - .map(|(key, (old_value,new_value))| { - let hkey = H256(key.to_be_bytes()); - trace!(target: "sync::stages::execution", ?address, current_transition_id, ?hkey, ?old_value, ?new_value, "Applying storage changeset"); - (hkey, old_value,new_value) - }) - .collect::>(); - - let mut cursor_storage_changeset = - self.cursor_write::()?; - cursor_storage_changeset.seek_exact(storage_id)?; - - if wipe_storage { - // iterate over storage and save them before entry is deleted. - self.cursor_read::()? - .walk(Some(address))? - .take_while(|res| { - res.as_ref().map(|(k, _)| *k == address).unwrap_or_default() - }) - .try_for_each(|entry| { - let (_, old_value) = entry?; - cursor_storage_changeset.append(storage_id, old_value) - })?; - - // delete all entries - self.delete::(address, None)?; - - // insert storage changeset - for (key, _, new_value) in storage { - // old values are already cleared. - if new_value != U256::ZERO { - self.put::( - address, - StorageEntry { key, value: new_value }, - )?; - } - } - } else { - // insert storage changeset - for (key, old_value, new_value) in storage { - let old_entry = StorageEntry { key, value: old_value }; - let new_entry = StorageEntry { key, value: new_value }; - // insert into StorageChangeSet - cursor_storage_changeset.append(storage_id, old_entry)?; - - // Always delete old value as duplicate table, put will not override it - self.delete::(address, Some(old_entry))?; - if new_value != U256::ZERO { - self.put::(address, new_entry)?; - } - } - } - } - // insert bytecode - for (hash, bytecode) in result.new_bytecodes.into_iter() { - // make different types of bytecode. Checked and maybe even analyzed (needs to - // be packed). Currently save only raw bytes. - let bytes = bytecode.bytes(); - trace!(target: "sync::stages::execution", ?hash, ?bytes, len = bytes.len(), "Inserting bytecode"); - self.put::(hash, Bytecode(bytecode))?; - // NOTE: bytecode bytes are not inserted in change set and can be found in - // separate table - } - current_transition_id += 1; - } - - let have_block_changeset = !results.block_changesets.is_empty(); - - // If there are any post block changes, we will add account changesets to db. - for (address, changeset) in results.block_changesets.into_iter() { - trace!(target: "sync::stages::execution", ?address, current_transition_id, "Applying block reward"); - changeset.apply_to_db( - &**self, - address, - current_transition_id, - spurious_dragon_active, - )?; - } - - // Transition is incremeneted every time before Paris hardfork and after - // Shanghai only if there are Withdrawals in the block. So it is correct to - // to increment transition id every time there is a block changeset present. - if have_block_changeset { - current_transition_id += 1; - } - } - Ok(()) - } - /// Return full table as Vec pub fn table(&self) -> Result>, DbError> where @@ -1598,7 +1481,7 @@ pub enum TransactionError { mod test { use crate::{insert_canonical_block, test_utils::blocks::*, Transaction}; use reth_db::{mdbx::test_utils::create_test_rw_db, tables, transaction::DbTxMut}; - use reth_primitives::{proofs::EMPTY_ROOT, ChainSpecBuilder, MAINNET}; + use reth_primitives::{proofs::EMPTY_ROOT, ChainSpecBuilder, TransitionId, MAINNET}; use std::ops::DerefMut; #[test] @@ -1623,7 +1506,17 @@ mod test { tx.put::(EMPTY_ROOT, vec![0x80]).unwrap(); assert_genesis_block(&tx, data.genesis); - tx.insert_block(block1.clone(), &chain_spec, exec_res1.clone()).unwrap(); + exec_res1.clone().write_to_db(tx.deref_mut(), 0).unwrap(); + tx.insert_block(block1.clone()).unwrap(); + tx.insert_hashes( + genesis.number, + 0, + exec_res1.transitions_count() as TransitionId, + block1.number, + block1.hash, + block1.state_root, + ) + .unwrap(); // get one block let get = tx.get_block_and_execution_range(&chain_spec, 1..=1).unwrap(); @@ -1634,8 +1527,32 @@ mod test { assert_eq!(take, vec![(block1.clone(), exec_res1.clone())]); assert_genesis_block(&tx, genesis.clone()); - tx.insert_block(block1.clone(), &chain_spec, exec_res1.clone()).unwrap(); - tx.insert_block(block2.clone(), &chain_spec, exec_res2.clone()).unwrap(); + exec_res1.clone().write_to_db(tx.deref_mut(), 0).unwrap(); + tx.insert_block(block1.clone()).unwrap(); + tx.insert_hashes( + genesis.number, + 0, + exec_res1.transitions_count() as TransitionId, + block1.number, + block1.hash, + block1.state_root, + ) + .unwrap(); + + exec_res2 + .clone() + .write_to_db(tx.deref_mut(), exec_res1.transitions_count() as TransitionId) + .unwrap(); + tx.insert_block(block2.clone()).unwrap(); + tx.insert_hashes( + block1.number, + exec_res1.transitions_count() as TransitionId, + exec_res2.transitions_count() as TransitionId, + 2, + block2.hash, + block2.state_root, + ) + .unwrap(); // get second block let get = tx.get_block_and_execution_range(&chain_spec, 2..=2).unwrap(); From 090bbe7836c5dd4d9b81ac96e2cd6c7725ced660 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 17 Mar 2023 00:50:33 +0100 Subject: [PATCH 154/191] fix(rpc): fallback to basefee when calculating fee fields (#1800) Co-authored-by: Georgios Konstantopoulos --- crates/rpc/rpc-types/src/eth/call.rs | 8 +++---- crates/rpc/rpc/src/eth/error.rs | 12 +++++----- crates/rpc/rpc/src/eth/revm_utils.rs | 35 ++++++++++++++++------------ 3 files changed, 30 insertions(+), 25 deletions(-) diff --git a/crates/rpc/rpc-types/src/eth/call.rs b/crates/rpc/rpc-types/src/eth/call.rs index dd69d809346..6076d073340 100644 --- a/crates/rpc/rpc-types/src/eth/call.rs +++ b/crates/rpc/rpc-types/src/eth/call.rs @@ -1,4 +1,4 @@ -use reth_primitives::{AccessList, Address, Bytes, U128, U256, U64}; +use reth_primitives::{AccessList, Address, Bytes, U256, U64}; use serde::{Deserialize, Serialize}; /// Call request @@ -10,11 +10,11 @@ pub struct CallRequest { /// To pub to: Option
, /// Gas Price - pub gas_price: Option, + pub gas_price: Option, /// EIP-1559 Max base fee the caller is willing to pay - pub max_fee_per_gas: Option, + pub max_fee_per_gas: Option, /// EIP-1559 Priority fee the caller is paying to the block author - pub max_priority_fee_per_gas: Option, + pub max_priority_fee_per_gas: Option, /// Gas pub gas: Option, /// Value diff --git a/crates/rpc/rpc/src/eth/error.rs b/crates/rpc/rpc/src/eth/error.rs index 149c3c84506..8260db82a86 100644 --- a/crates/rpc/rpc/src/eth/error.rs +++ b/crates/rpc/rpc/src/eth/error.rs @@ -2,7 +2,7 @@ use crate::result::{internal_rpc_err, rpc_err}; use jsonrpsee::{core::Error as RpcError, types::error::INVALID_PARAMS_CODE}; -use reth_primitives::{constants::SELECTOR_LEN, Address, Bytes, U128, U256}; +use reth_primitives::{constants::SELECTOR_LEN, Address, Bytes, U256}; use reth_rpc_types::{error::EthRpcErrorCode, BlockError}; use reth_transaction_pool::error::{InvalidPoolTransactionError, PoolError}; use revm::primitives::{EVMError, ExecutionResult, Halt, OutOfGasError, Output}; @@ -31,15 +31,15 @@ pub enum EthApiError { #[error("Prevrandao not in th EVM's environment after merge")] PrevrandaoNotSet, #[error("Conflicting fee values in request. Both legacy gasPrice {gas_price} and maxFeePerGas {max_fee_per_gas} set")] - ConflictingRequestGasPrice { gas_price: U128, max_fee_per_gas: U128 }, + ConflictingRequestGasPrice { gas_price: U256, max_fee_per_gas: U256 }, #[error("Conflicting fee values in request. Both legacy gasPrice {gas_price} maxFeePerGas {max_fee_per_gas} and maxPriorityFeePerGas {max_priority_fee_per_gas} set")] ConflictingRequestGasPriceAndTipSet { - gas_price: U128, - max_fee_per_gas: U128, - max_priority_fee_per_gas: U128, + gas_price: U256, + max_fee_per_gas: U256, + max_priority_fee_per_gas: U256, }, #[error("Conflicting fee values in request. Legacy gasPrice {gas_price} and maxPriorityFeePerGas {max_priority_fee_per_gas} set")] - RequestLegacyGasPriceAndTipSet { gas_price: U128, max_priority_fee_per_gas: U128 }, + RequestLegacyGasPriceAndTipSet { gas_price: U256, max_priority_fee_per_gas: U256 }, #[error(transparent)] InvalidTransaction(#[from] InvalidTransactionError), /// Thrown when constructing an RPC block from a primitive block data failed. diff --git a/crates/rpc/rpc/src/eth/revm_utils.rs b/crates/rpc/rpc/src/eth/revm_utils.rs index 63b7ac9b8ff..09fcb6c6da5 100644 --- a/crates/rpc/rpc/src/eth/revm_utils.rs +++ b/crates/rpc/rpc/src/eth/revm_utils.rs @@ -1,7 +1,7 @@ //! utilities for working with revm use crate::eth::error::{EthApiError, EthResult, InvalidTransactionError}; -use reth_primitives::{AccessList, Address, U128, U256}; +use reth_primitives::{AccessList, Address, U256}; use reth_rpc_types::CallRequest; use revm::{ precompile::{Precompiles, SpecId as PrecompilesSpecId}, @@ -83,8 +83,12 @@ pub(crate) fn create_txn_env(block_env: &BlockEnv, request: CallRequest) -> EthR chain_id, } = request; - let CallFees { max_priority_fee_per_gas, gas_price } = - CallFees::ensure_fees(gas_price, max_fee_per_gas, max_priority_fee_per_gas)?; + let CallFees { max_priority_fee_per_gas, gas_price } = CallFees::ensure_fees( + gas_price, + max_fee_per_gas, + max_priority_fee_per_gas, + block_env.basefee, + )?; let gas_limit = gas.unwrap_or(block_env.gas_limit.min(U256::from(u64::MAX))); @@ -112,7 +116,7 @@ pub(crate) struct CallFees { max_priority_fee_per_gas: Option, /// Unified gas price setting /// - /// Will be `0` if unset in request + /// Will be the configured `basefee` if unset in the request /// /// `gasPrice` for legacy, /// `maxFeePerGas` for EIP-1559 @@ -122,22 +126,26 @@ pub(crate) struct CallFees { // === impl CallFees === impl CallFees { - /// Ensures the fields of a [CallRequest] are not conflicting + /// Ensures the fields of a [CallRequest] are not conflicting. + /// + /// If no `gasPrice` or `maxFeePerGas` is set, then the `gas_price` in the response will + /// fallback to the given `basefee`. fn ensure_fees( - call_gas_price: Option, - call_max_fee: Option, - call_priority_fee: Option, + call_gas_price: Option, + call_max_fee: Option, + call_priority_fee: Option, + base_fee: U256, ) -> EthResult { match (call_gas_price, call_max_fee, call_priority_fee) { (gas_price, None, None) => { // request for a legacy transaction // set everything to zero - let gas_price = gas_price.unwrap_or_default(); - Ok(CallFees { gas_price: U256::from(gas_price), max_priority_fee_per_gas: None }) + let gas_price = gas_price.unwrap_or(base_fee); + Ok(CallFees { gas_price, max_priority_fee_per_gas: None }) } (None, max_fee_per_gas, max_priority_fee_per_gas) => { // request for eip-1559 transaction - let max_fee = max_fee_per_gas.unwrap_or_default(); + let max_fee = max_fee_per_gas.unwrap_or(base_fee); if let Some(max_priority) = max_priority_fee_per_gas { if max_priority > max_fee { @@ -148,10 +156,7 @@ impl CallFees { ) } } - Ok(CallFees { - gas_price: U256::from(max_fee), - max_priority_fee_per_gas: max_priority_fee_per_gas.map(U256::from), - }) + Ok(CallFees { gas_price: max_fee, max_priority_fee_per_gas }) } (Some(gas_price), Some(max_fee_per_gas), Some(max_priority_fee_per_gas)) => { Err(EthApiError::ConflictingRequestGasPriceAndTipSet { From 57894f7b98419199cbd5949569525927f446e9c0 Mon Sep 17 00:00:00 2001 From: his-dudenesss <109000751+his-dudenesss@users.noreply.github.com> Date: Fri, 17 Mar 2023 00:59:29 +0100 Subject: [PATCH 155/191] Issue 1770 move rlp impl for enr to discv4 crate (#1804) Co-authored-by: Georgios Konstantopoulos --- crates/net/discv4/Cargo.toml | 2 +- crates/net/discv4/src/lib.rs | 4 +- crates/net/discv4/src/proto.rs | 190 +++++++++++++++++++++++++++++++-- crates/rlp/Cargo.toml | 3 - crates/rlp/src/decode.rs | 97 ----------------- crates/rlp/src/encode.rs | 65 ----------- 6 files changed, 187 insertions(+), 174 deletions(-) diff --git a/crates/net/discv4/Cargo.toml b/crates/net/discv4/Cargo.toml index fdc0421cd01..93fcb95cbea 100644 --- a/crates/net/discv4/Cargo.toml +++ b/crates/net/discv4/Cargo.toml @@ -12,7 +12,7 @@ Ethereum network discovery [dependencies] # reth reth-primitives = { path = "../../primitives" } -reth-rlp = { path = "../../rlp", features = ["enr"] } +reth-rlp = { path = "../../rlp" } reth-rlp-derive = { path = "../../rlp/rlp-derive" } reth-net-common = { path = "../common" } reth-net-nat = { path = "../nat" } diff --git a/crates/net/discv4/src/lib.rs b/crates/net/discv4/src/lib.rs index c5bee30030c..51997d2ccde 100644 --- a/crates/net/discv4/src/lib.rs +++ b/crates/net/discv4/src/lib.rs @@ -30,7 +30,7 @@ use discv5::{ ConnectionDirection, ConnectionState, }; use enr::{Enr, EnrBuilder}; -use proto::{EnrRequest, EnrResponse}; +use proto::{EnrRequest, EnrResponse, EnrWrapper}; use reth_primitives::{ bytes::{Bytes, BytesMut}, ForkId, PeerId, H256, @@ -1125,7 +1125,7 @@ impl Discv4Service { self.send_packet( Message::EnrResponse(EnrResponse { request_hash, - enr: self.local_eip_868_enr.clone(), + enr: EnrWrapper::new(self.local_eip_868_enr.clone()), }), remote_addr, ); diff --git a/crates/net/discv4/src/proto.rs b/crates/net/discv4/src/proto.rs index e4e8f28594a..ad52267ca6d 100644 --- a/crates/net/discv4/src/proto.rs +++ b/crates/net/discv4/src/proto.rs @@ -1,12 +1,14 @@ #![allow(missing_docs)] use crate::{error::DecodePacketError, PeerId, MAX_PACKET_SIZE, MIN_PACKET_SIZE}; -use enr::Enr; +use enr::{Enr, EnrKey}; use reth_primitives::{ bytes::{Buf, BufMut, Bytes, BytesMut}, - keccak256, ForkId, NodeRecord, H256, + keccak256, + rpc_utils::rlp, + ForkId, NodeRecord, H256, }; -use reth_rlp::{Decodable, DecodeError, Encodable, Header}; +use reth_rlp::{length_of_length, Decodable, DecodeError, Encodable, Header}; use reth_rlp_derive::{RlpDecodable, RlpEncodable}; use secp256k1::{ ecdsa::{RecoverableSignature, RecoveryId}, @@ -209,6 +211,80 @@ pub struct Neighbours { pub expire: u64, } +/// Passthrough newtype to [`Enr`]. +/// +/// We need to wrap the ENR type because of Rust's orphan rules not allowing +/// implementing a foreign trait on a foreign type. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct EnrWrapper(Enr); + +impl EnrWrapper { + pub fn new(enr: Enr) -> Self { + EnrWrapper(enr) + } +} + +impl Encodable for EnrWrapper +where + K: EnrKey, +{ + fn encode(&self, out: &mut dyn BufMut) { + let payload_length = self.0.signature().length() + + self.0.seq().length() + + self.0.iter().fold(0, |acc, (k, v)| acc + k.as_slice().length() + v.len()); + + let header = Header { list: true, payload_length }; + header.encode(out); + + self.0.signature().encode(out); + self.0.seq().encode(out); + + for (k, v) in self.0.iter() { + // Keys are byte data + k.as_slice().encode(out); + // Values are raw RLP encoded data + out.put_slice(v); + } + } + + fn length(&self) -> usize { + let payload_length = self.0.signature().length() + + self.0.seq().length() + + self.0.iter().fold(0, |acc, (k, v)| acc + k.as_slice().length() + v.len()); + payload_length + length_of_length(payload_length) + } +} + +impl Decodable for EnrWrapper { + fn decode(buf: &mut &[u8]) -> Result { + let enr = as rlp::Decodable>::decode(&rlp::Rlp::new(buf)) + .map_err(|e| match e { + rlp::DecoderError::RlpIsTooShort => DecodeError::InputTooShort, + rlp::DecoderError::RlpInvalidLength => DecodeError::Overflow, + rlp::DecoderError::RlpExpectedToBeList => DecodeError::UnexpectedString, + rlp::DecoderError::RlpExpectedToBeData => DecodeError::UnexpectedList, + rlp::DecoderError::RlpDataLenWithZeroPrefix | + rlp::DecoderError::RlpListLenWithZeroPrefix => DecodeError::LeadingZero, + rlp::DecoderError::RlpInvalidIndirection => DecodeError::NonCanonicalSize, + rlp::DecoderError::RlpIncorrectListLen => { + DecodeError::Custom("incorrect list length when decoding rlp") + } + rlp::DecoderError::RlpIsTooBig => DecodeError::Custom("rlp is too big"), + rlp::DecoderError::RlpInconsistentLengthAndData => { + DecodeError::Custom("inconsistent length and data when decoding rlp") + } + rlp::DecoderError::Custom(s) => DecodeError::Custom(s), + }) + .map(EnrWrapper::new); + if enr.is_ok() { + // Decode was successful, advance buffer + let header = Header::decode(buf)?; + buf.advance(header.payload_length); + } + enr + } +} + /// A [ENRRequest packet](https://github.com/ethereum/devp2p/blob/master/discv4.md#enrrequest-packet-0x05). #[derive(Clone, Copy, Debug, Eq, PartialEq, RlpEncodable, RlpDecodable)] pub struct EnrRequest { @@ -219,7 +295,7 @@ pub struct EnrRequest { #[derive(Clone, Debug, Eq, PartialEq, RlpEncodable)] pub struct EnrResponse { pub request_hash: H256, - pub enr: Enr, + pub enr: EnrWrapper, } // === impl EnrResponse === @@ -229,7 +305,7 @@ impl EnrResponse { /// /// See also pub fn eth_fork_id(&self) -> Option { - let mut maybe_fork_id = self.enr.get(b"eth")?; + let mut maybe_fork_id = self.enr.0.get(b"eth")?; ForkId::decode(&mut maybe_fork_id).ok() } } @@ -244,7 +320,7 @@ impl Decodable for EnrResponse { // let started_len = b.len(); let this = Self { request_hash: reth_rlp::Decodable::decode(b)?, - enr: reth_rlp::Decodable::decode(b)?, + enr: EnrWrapper::::decode(b)?, }; // TODO: `Decodable` can be derived once we have native reth_rlp decoding for ENR: // Skipping the size check here is fine since the `buf` is the UDP datagram @@ -422,7 +498,9 @@ mod tests { test_utils::{rng_endpoint, rng_ipv4_record, rng_ipv6_record, rng_message}, SAFE_MAX_DATAGRAM_NEIGHBOUR_RECORDS, }; + use enr::{EnrBuilder, EnrPublicKey}; use rand::{thread_rng, Rng, RngCore}; + use reth_primitives::hex_literal::hex; #[test] fn test_endpoint_ipv_v4() { @@ -635,4 +713,104 @@ mod tests { let data = hex::decode(packet).unwrap(); Message::decode(&data).unwrap(); } + + // test vector from the enr library rlp encoding tests + // + + #[test] + fn encode_known_rlp_enr() { + use self::EnrWrapper; + use enr::{secp256k1::SecretKey, EnrPublicKey}; + use reth_rlp::Decodable; + use std::net::Ipv4Addr; + + let valid_record = + hex!("f884b8407098ad865b00a582051940cb9cf36836572411a47278783077011599ed5cd16b76f2635f4e234738f30813a89eb9137e3e3df5266e3a1f11df72ecf1145ccb9c01826964827634826970847f00000189736563703235366b31a103ca634cae0d49acb401d8a4c6b6fe8c55b70d115bf400769cc1400f3258cd31388375647082765f" + ); + let signature = + hex!("7098ad865b00a582051940cb9cf36836572411a47278783077011599ed5cd16b76f2635f4e234738f30813a89eb9137e3e3df5266e3a1f11df72ecf1145ccb9c" + ); + let expected_pubkey = + hex!("03ca634cae0d49acb401d8a4c6b6fe8c55b70d115bf400769cc1400f3258cd3138"); + + let enr = EnrWrapper::::decode(&mut &valid_record[..]).unwrap(); + let pubkey = enr.0.public_key().encode(); + + assert_eq!(enr.0.ip4(), Some(Ipv4Addr::new(127, 0, 0, 1))); + assert_eq!(enr.0.id(), Some(String::from("v4"))); + assert_eq!(enr.0.udp4(), Some(30303)); + assert_eq!(enr.0.tcp4(), None); + assert_eq!(enr.0.signature(), &signature[..]); + assert_eq!(pubkey.to_vec(), expected_pubkey); + assert!(enr.0.verify()); + + let mut encoded = BytesMut::new(); + enr.encode(&mut encoded); + assert_eq!(&encoded[..], &valid_record[..]); + + // ensure the length is equal + assert_eq!(enr.length(), valid_record.len()); + } + + // test vector from the enr library rlp encoding tests + // + #[test] + fn decode_enr_rlp() { + use enr::secp256k1::SecretKey; + use std::net::Ipv4Addr; + + let valid_record = hex!("f884b8407098ad865b00a582051940cb9cf36836572411a47278783077011599ed5cd16b76f2635f4e234738f30813a89eb9137e3e3df5266e3a1f11df72ecf1145ccb9c01826964827634826970847f00000189736563703235366b31a103ca634cae0d49acb401d8a4c6b6fe8c55b70d115bf400769cc1400f3258cd31388375647082765f"); + let signature = hex!("7098ad865b00a582051940cb9cf36836572411a47278783077011599ed5cd16b76f2635f4e234738f30813a89eb9137e3e3df5266e3a1f11df72ecf1145ccb9c"); + let expected_pubkey = + hex!("03ca634cae0d49acb401d8a4c6b6fe8c55b70d115bf400769cc1400f3258cd3138"); + + let mut valid_record_buf = valid_record.as_slice(); + let enr = EnrWrapper::::decode(&mut valid_record_buf).unwrap(); + let pubkey = enr.0.public_key().encode(); + + // Byte array must be consumed after enr has finished decoding + assert!(valid_record_buf.is_empty()); + + assert_eq!(enr.0.ip4(), Some(Ipv4Addr::new(127, 0, 0, 1))); + assert_eq!(enr.0.id(), Some(String::from("v4"))); + assert_eq!(enr.0.udp4(), Some(30303)); + assert_eq!(enr.0.tcp4(), None); + assert_eq!(enr.0.signature(), &signature[..]); + assert_eq!(pubkey.to_vec(), expected_pubkey); + assert!(enr.0.verify()); + } + + // test vector from the enr library rlp encoding tests + // + #[test] + fn encode_decode_enr_rlp() { + use enr::{secp256k1::SecretKey, EnrPublicKey}; + use std::net::Ipv4Addr; + + let key = SecretKey::new(&mut rand::rngs::OsRng); + let ip = Ipv4Addr::new(127, 0, 0, 1); + let tcp = 3000; + + let enr = { + let mut builder = EnrBuilder::new("v4"); + builder.ip(ip.into()); + builder.tcp4(tcp); + EnrWrapper::new(builder.build(&key).unwrap()) + }; + + let mut encoded = BytesMut::new(); + enr.encode(&mut encoded); + let mut encoded_bytes = &encoded[..]; + let decoded_enr = EnrWrapper::::decode(&mut encoded_bytes).unwrap(); + + // Byte array must be consumed after enr has finished decoding + assert!(encoded_bytes.is_empty()); + + assert_eq!(decoded_enr, enr); + assert_eq!(decoded_enr.0.id(), Some("v4".into())); + assert_eq!(decoded_enr.0.ip4(), Some(ip)); + assert_eq!(decoded_enr.0.tcp4(), Some(tcp)); + assert_eq!(decoded_enr.0.public_key().encode(), key.public().encode()); + assert!(decoded_enr.0.verify()); + } } diff --git a/crates/rlp/Cargo.toml b/crates/rlp/Cargo.toml index cf6ebe9d5b8..b050e1f09a1 100644 --- a/crates/rlp/Cargo.toml +++ b/crates/rlp/Cargo.toml @@ -12,7 +12,6 @@ auto_impl = "1" bytes = { version = "1", default-features = false } ethnum = { version = "1", default-features = false, optional = true } smol_str = { version = "0.1", default-features = false, optional = true } -enr = { version = "0.8.0", default-features = false, optional = true } rlp = { version = "0.5.2", default-features = false, optional = true } ethereum-types = { version = "0.14", features = ["codec"], optional = true } revm-primitives = {version = "1.0.0", features = ["serde"] } @@ -24,7 +23,6 @@ reth-rlp = { path = ".", package = "reth-rlp", features = [ "std", "ethnum", "ethereum-types", - "enr", "smol_str" ] } criterion = "0.4.0" @@ -39,7 +37,6 @@ pprof = { version = "0.11", features = ["flamegraph", "frame-pointer", "criterio alloc = [] derive = ["reth-rlp-derive"] std = ["alloc"] -enr = ["dep:enr", "dep:rlp", "enr/rust-secp256k1", "derive"] [[bench]] name = "bench" diff --git a/crates/rlp/src/decode.rs b/crates/rlp/src/decode.rs index d152d87087e..94d466612bf 100644 --- a/crates/rlp/src/decode.rs +++ b/crates/rlp/src/decode.rs @@ -448,39 +448,6 @@ impl Decodable for smol_str::SmolStr { } } -#[cfg(feature = "enr")] -impl Decodable for enr::Enr -where - K: enr::EnrKey, -{ - fn decode(buf: &mut &[u8]) -> Result { - // currently the only way to build an enr is to decode it using the rlp::Decodable trait - let enr = ::decode(&rlp::Rlp::new(buf)).map_err(|e| match e { - rlp::DecoderError::RlpIsTooShort => DecodeError::InputTooShort, - rlp::DecoderError::RlpInvalidLength => DecodeError::Overflow, - rlp::DecoderError::RlpExpectedToBeList => DecodeError::UnexpectedString, - rlp::DecoderError::RlpExpectedToBeData => DecodeError::UnexpectedList, - rlp::DecoderError::RlpDataLenWithZeroPrefix | - rlp::DecoderError::RlpListLenWithZeroPrefix => DecodeError::LeadingZero, - rlp::DecoderError::RlpInvalidIndirection => DecodeError::NonCanonicalSize, - rlp::DecoderError::RlpIncorrectListLen => { - DecodeError::Custom("incorrect list length when decoding rlp") - } - rlp::DecoderError::RlpIsTooBig => DecodeError::Custom("rlp is too big"), - rlp::DecoderError::RlpInconsistentLengthAndData => { - DecodeError::Custom("inconsistent length and data when decoding rlp") - } - rlp::DecoderError::Custom(s) => DecodeError::Custom(s), - }); - if enr.is_ok() { - // Decode was successful, advance buffer - let header = Header::decode(buf)?; - buf.advance(header.payload_length); - } - enr - } -} - #[cfg(test)] mod tests { extern crate alloc; @@ -699,68 +666,4 @@ mod tests { (Err(DecodeError::UnexpectedList), &hex!("C0")[..]), ]) } - - // test vector from the enr library rlp encoding tests - // - #[cfg(feature = "enr")] - #[test] - fn decode_enr_rlp() { - use enr::{secp256k1::SecretKey, Enr, EnrPublicKey}; - use std::net::Ipv4Addr; - - let valid_record = hex!("f884b8407098ad865b00a582051940cb9cf36836572411a47278783077011599ed5cd16b76f2635f4e234738f30813a89eb9137e3e3df5266e3a1f11df72ecf1145ccb9c01826964827634826970847f00000189736563703235366b31a103ca634cae0d49acb401d8a4c6b6fe8c55b70d115bf400769cc1400f3258cd31388375647082765f"); - let signature = hex!("7098ad865b00a582051940cb9cf36836572411a47278783077011599ed5cd16b76f2635f4e234738f30813a89eb9137e3e3df5266e3a1f11df72ecf1145ccb9c"); - let expected_pubkey = - hex!("03ca634cae0d49acb401d8a4c6b6fe8c55b70d115bf400769cc1400f3258cd3138"); - - let mut valid_record_buf = valid_record.as_slice(); - let enr = Enr::::decode(&mut valid_record_buf).unwrap(); - let pubkey = enr.public_key().encode(); - - // Byte array must be consumed after enr has finished decoding - assert!(valid_record_buf.is_empty()); - - assert_eq!(enr.ip4(), Some(Ipv4Addr::new(127, 0, 0, 1))); - assert_eq!(enr.id(), Some(String::from("v4"))); - assert_eq!(enr.udp4(), Some(30303)); - assert_eq!(enr.tcp4(), None); - assert_eq!(enr.signature(), &signature[..]); - assert_eq!(pubkey.to_vec(), expected_pubkey); - assert!(enr.verify()); - } - - // test vector from the enr library rlp encoding tests - // - #[cfg(feature = "enr")] - #[test] - fn encode_decode_enr_rlp() { - use enr::{secp256k1::SecretKey, Enr, EnrBuilder, EnrKey, EnrPublicKey}; - use std::net::Ipv4Addr; - - let key = SecretKey::new(&mut rand::rngs::OsRng); - let ip = Ipv4Addr::new(127, 0, 0, 1); - let tcp = 3000; - - let enr = { - let mut builder = EnrBuilder::new("v4"); - builder.ip(ip.into()); - builder.tcp4(tcp); - builder.build(&key).unwrap() - }; - - let mut encoded = BytesMut::new(); - enr.encode(&mut encoded); - let mut encoded_bytes = &encoded[..]; - let decoded_enr = Enr::::decode(&mut encoded_bytes).unwrap(); - - // Byte array must be consumed after enr has finished decoding - assert!(encoded_bytes.is_empty()); - - assert_eq!(decoded_enr, enr); - assert_eq!(decoded_enr.id(), Some("v4".into())); - assert_eq!(decoded_enr.ip4(), Some(ip)); - assert_eq!(decoded_enr.tcp4(), Some(tcp)); - assert_eq!(decoded_enr.public_key().encode(), key.public().encode()); - assert!(decoded_enr.verify()); - } } diff --git a/crates/rlp/src/encode.rs b/crates/rlp/src/encode.rs index 0c11b5f8c01..73f62c01afa 100644 --- a/crates/rlp/src/encode.rs +++ b/crates/rlp/src/encode.rs @@ -190,38 +190,6 @@ impl Encodable for smol_str::SmolStr { } } -#[cfg(feature = "enr")] -impl Encodable for enr::Enr -where - K: enr::EnrKey, -{ - fn encode(&self, out: &mut dyn BufMut) { - let payload_length = self.signature().length() + - self.seq().length() + - self.iter().fold(0, |acc, (k, v)| acc + k.as_slice().length() + v.len()); - - let header = Header { list: true, payload_length }; - header.encode(out); - - self.signature().encode(out); - self.seq().encode(out); - - for (k, v) in self.iter() { - // Keys are byte data - k.as_slice().encode(out); - // Values are raw RLP encoded data - out.put_slice(v); - } - } - - fn length(&self) -> usize { - let payload_length = self.signature().length() + - self.seq().length() + - self.iter().fold(0, |acc, (k, v)| acc + k.as_slice().length() + v.len()); - payload_length + length_of_length(payload_length) - } -} - #[cfg(feature = "std")] impl Encodable for std::net::IpAddr { fn encode(&self, out: &mut dyn BufMut) { @@ -649,37 +617,4 @@ mod tests { "abcdefgh".to_string().encode(&mut b); assert_eq!(&encoded(SmolStr::new("abcdefgh"))[..], b.as_ref()); } - - // test vector from the enr library rlp encoding tests - // - #[cfg(feature = "enr")] - #[test] - fn encode_known_rlp_enr() { - use crate::Decodable; - use enr::{secp256k1::SecretKey, Enr, EnrPublicKey}; - use std::net::Ipv4Addr; - - let valid_record = hex!("f884b8407098ad865b00a582051940cb9cf36836572411a47278783077011599ed5cd16b76f2635f4e234738f30813a89eb9137e3e3df5266e3a1f11df72ecf1145ccb9c01826964827634826970847f00000189736563703235366b31a103ca634cae0d49acb401d8a4c6b6fe8c55b70d115bf400769cc1400f3258cd31388375647082765f"); - let signature = hex!("7098ad865b00a582051940cb9cf36836572411a47278783077011599ed5cd16b76f2635f4e234738f30813a89eb9137e3e3df5266e3a1f11df72ecf1145ccb9c"); - let expected_pubkey = - hex!("03ca634cae0d49acb401d8a4c6b6fe8c55b70d115bf400769cc1400f3258cd3138"); - - let enr = Enr::::decode(&mut &valid_record[..]).unwrap(); - let pubkey = enr.public_key().encode(); - - assert_eq!(enr.ip4(), Some(Ipv4Addr::new(127, 0, 0, 1))); - assert_eq!(enr.id(), Some(String::from("v4"))); - assert_eq!(enr.udp4(), Some(30303)); - assert_eq!(enr.tcp4(), None); - assert_eq!(enr.signature(), &signature[..]); - assert_eq!(pubkey.to_vec(), expected_pubkey); - assert!(enr.verify()); - - let mut encoded = BytesMut::new(); - enr.encode(&mut encoded); - assert_eq!(&encoded[..], &valid_record[..]); - - // ensure the length is equal - assert_eq!(enr.length(), valid_record.len()); - } } From 241ec32abf8d393a6bb1fd2a393ce6a0aa6ca85c Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 17 Mar 2023 01:06:32 +0100 Subject: [PATCH 156/191] feat: add canonical_headers_range (#1795) Co-authored-by: Georgios Konstantopoulos --- Cargo.lock | 1 - crates/executor/src/executor.rs | 21 ++++++++++--- crates/executor/src/substate.rs | 10 +++++-- crates/net/network/src/eth_requests.rs | 4 +-- crates/revm/src/database.rs | 3 +- crates/rpc/rpc-engine-api/src/engine_api.rs | 2 +- crates/rpc/rpc/src/eth/api/server.rs | 3 +- crates/rpc/rpc/src/eth/filter.rs | 7 ++--- crates/storage/provider/src/providers/mod.rs | 18 ++++++++--- .../src/providers/state/historical.rs | 19 ++++++++++-- .../provider/src/providers/state/latest.rs | 21 ++++++++++--- .../provider/src/providers/state/macros.rs | 7 +++-- .../storage/provider/src/test_utils/mock.rs | 30 ++++++++++++------- .../storage/provider/src/test_utils/noop.rs | 6 +++- .../storage/provider/src/traits/block_hash.rs | 7 +++-- .../storage/provider/src/traits/block_id.rs | 4 +-- 16 files changed, 114 insertions(+), 49 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6a29470793e..8926959e519 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5109,7 +5109,6 @@ dependencies = [ "auto_impl", "bytes", "criterion", - "enr 0.8.0", "ethereum-types", "ethnum", "hex-literal", diff --git a/crates/executor/src/executor.rs b/crates/executor/src/executor.rs index 80edf42a863..f292a297f89 100644 --- a/crates/executor/src/executor.rs +++ b/crates/executor/src/executor.rs @@ -516,8 +516,8 @@ pub fn verify_receipt<'a>( mod tests { use super::*; use reth_primitives::{ - hex_literal::hex, keccak256, Account, Address, Bytecode, Bytes, ChainSpecBuilder, - ForkCondition, StorageKey, H256, MAINNET, U256, + hex_literal::hex, keccak256, Account, Address, BlockNumber, Bytecode, Bytes, + ChainSpecBuilder, ForkCondition, StorageKey, H256, MAINNET, U256, }; use reth_provider::{ post_state::{Change, Storage}, @@ -531,7 +531,7 @@ mod tests { struct StateProviderTest { accounts: HashMap, Account)>, contracts: HashMap, - block_hash: HashMap, + block_hash: HashMap, } impl StateProviderTest { @@ -560,9 +560,22 @@ mod tests { } impl BlockHashProvider for StateProviderTest { - fn block_hash(&self, number: U256) -> reth_interfaces::Result> { + fn block_hash(&self, number: u64) -> reth_interfaces::Result> { Ok(self.block_hash.get(&number).cloned()) } + + fn canonical_hashes_range( + &self, + start: BlockNumber, + end: BlockNumber, + ) -> reth_interfaces::Result> { + let range = start..end; + Ok(self + .block_hash + .iter() + .filter_map(|(block, hash)| if range.contains(block) { Some(*hash) } else { None }) + .collect()) + } } impl StateProvider for StateProviderTest { diff --git a/crates/executor/src/substate.rs b/crates/executor/src/substate.rs index 7da7acbc5e4..082ce140b6b 100644 --- a/crates/executor/src/substate.rs +++ b/crates/executor/src/substate.rs @@ -30,10 +30,10 @@ impl<'a, SP: StateProvider> PostStateProvider<'a, SP> { } } +/* Implement StateProvider traits */ + impl<'a, SP: StateProvider> BlockHashProvider for PostStateProvider<'a, SP> { - fn block_hash(&self, number: U256) -> Result> { - // All block numbers fit inside u64 and revm checks if it is last 256 block numbers. - let block_number = number.as_limbs()[0]; + fn block_hash(&self, block_number: BlockNumber) -> Result> { if let Some(sidechain_block_hash) = self.sidechain_block_hashes.get(&block_number).cloned() { return Ok(Some(sidechain_block_hash)) @@ -46,6 +46,10 @@ impl<'a, SP: StateProvider> BlockHashProvider for PostStateProvider<'a, SP> { .ok_or(ProviderError::BlockchainTreeBlockHash { block_number })?, )) } + + fn canonical_hashes_range(&self, _start: BlockNumber, _end: BlockNumber) -> Result> { + unimplemented!() + } } impl<'a, SP: StateProvider> AccountProvider for PostStateProvider<'a, SP> { diff --git a/crates/net/network/src/eth_requests.rs b/crates/net/network/src/eth_requests.rs index 324419be45d..0972ac84979 100644 --- a/crates/net/network/src/eth_requests.rs +++ b/crates/net/network/src/eth_requests.rs @@ -7,7 +7,7 @@ use reth_eth_wire::{ GetReceipts, NodeData, Receipts, }; use reth_interfaces::p2p::error::RequestResult; -use reth_primitives::{BlockHashOrNumber, Header, HeadersDirection, PeerId, U256}; +use reth_primitives::{BlockHashOrNumber, Header, HeadersDirection, PeerId}; use reth_provider::{BlockProvider, HeaderProvider}; use std::{ borrow::Borrow, @@ -82,7 +82,7 @@ where let mut block: BlockHashOrNumber = match start_block { BlockHashOrNumber::Hash(start) => start.into(), BlockHashOrNumber::Number(num) => { - if let Some(hash) = self.client.block_hash(U256::from(num)).unwrap_or_default() { + if let Some(hash) = self.client.block_hash(num).unwrap_or_default() { hash.into() } else { return headers diff --git a/crates/revm/src/database.rs b/crates/revm/src/database.rs index d18d872f2cc..1698b5500d6 100644 --- a/crates/revm/src/database.rs +++ b/crates/revm/src/database.rs @@ -63,6 +63,7 @@ impl DatabaseRef for State { } fn block_hash(&self, number: U256) -> Result { - Ok(self.0.block_hash(number)?.unwrap_or_default()) + // Note: this unwrap is potentially unsafe + Ok(self.0.block_hash(number.try_into().unwrap())?.unwrap_or_default()) } } diff --git a/crates/rpc/rpc-engine-api/src/engine_api.rs b/crates/rpc/rpc-engine-api/src/engine_api.rs index 31044a96d7f..8ae27ffbc87 100644 --- a/crates/rpc/rpc-engine-api/src/engine_api.rs +++ b/crates/rpc/rpc-engine-api/src/engine_api.rs @@ -462,7 +462,7 @@ impl HeaderProvider for ShareableDatabase { } impl BlockHashProvider for ShareableDatabase { - fn block_hash(&self, number: U256) -> Result> { - // TODO: This unwrap is potentially unsafe + fn block_hash(&self, number: u64) -> Result> { + self.db.view(|tx| tx.get::(number))?.map_err(Into::into) + } + + fn canonical_hashes_range(&self, start: BlockNumber, end: BlockNumber) -> Result> { + let range = start..end; self.db - .view(|tx| tx.get::(number.try_into().unwrap()))? + .view(|tx| { + let mut cursor = tx.cursor_read::()?; + cursor + .walk_range(range)? + .map(|result| result.map(|(_, hash)| hash).map_err(Into::into)) + .collect::>>() + })? .map_err(Into::into) } } @@ -109,7 +119,7 @@ impl BlockIdProvider for ShareableDatabase { .view(|tx| tx.get::("Finish".to_string()))? .map_err(Into::::into)? .unwrap_or_default(); - let best_hash = self.block_hash(U256::from(best_number))?.unwrap_or_default(); + let best_hash = self.block_hash(best_number)?.unwrap_or_default(); Ok(ChainInfo { best_hash, best_number, last_finalized: None, safe_finalized: None }) } diff --git a/crates/storage/provider/src/providers/state/historical.rs b/crates/storage/provider/src/providers/state/historical.rs index 99819caebc1..e8776aee404 100644 --- a/crates/storage/provider/src/providers/state/historical.rs +++ b/crates/storage/provider/src/providers/state/historical.rs @@ -10,7 +10,7 @@ use reth_db::{ }; use reth_interfaces::Result; use reth_primitives::{ - Account, Address, Bytecode, Bytes, StorageKey, StorageValue, TransitionId, H256, U256, + Account, Address, BlockNumber, Bytecode, Bytes, StorageKey, StorageValue, TransitionId, H256, }; use std::marker::PhantomData; @@ -72,8 +72,21 @@ impl<'a, 'b, TX: DbTx<'a>> AccountProvider for HistoricalStateProviderRef<'a, 'b impl<'a, 'b, TX: DbTx<'a>> BlockHashProvider for HistoricalStateProviderRef<'a, 'b, TX> { /// Get block hash by number. - fn block_hash(&self, number: U256) -> Result> { - self.tx.get::(number.to::()).map_err(Into::into) + fn block_hash(&self, number: u64) -> Result> { + self.tx.get::(number).map_err(Into::into) + } + + fn canonical_hashes_range(&self, start: BlockNumber, end: BlockNumber) -> Result> { + let range = start..end; + self.tx + .cursor_read::() + .map(|mut cursor| { + cursor + .walk_range(range)? + .map(|result| result.map(|(_, hash)| hash).map_err(Into::into)) + .collect::>>() + })? + .map_err(Into::into) } } diff --git a/crates/storage/provider/src/providers/state/latest.rs b/crates/storage/provider/src/providers/state/latest.rs index cf20585beee..56d59250829 100644 --- a/crates/storage/provider/src/providers/state/latest.rs +++ b/crates/storage/provider/src/providers/state/latest.rs @@ -9,8 +9,8 @@ use reth_db::{ }; use reth_interfaces::{provider::ProviderError, Result}; use reth_primitives::{ - keccak256, Account, Address, Bytecode, Bytes, StorageKey, StorageValue, H256, KECCAK_EMPTY, - U256, + keccak256, Account, Address, BlockNumber, Bytecode, Bytes, StorageKey, StorageValue, H256, + KECCAK_EMPTY, }; use std::marker::PhantomData; @@ -38,8 +38,21 @@ impl<'a, 'b, TX: DbTx<'a>> AccountProvider for LatestStateProviderRef<'a, 'b, TX impl<'a, 'b, TX: DbTx<'a>> BlockHashProvider for LatestStateProviderRef<'a, 'b, TX> { /// Get block hash by number. - fn block_hash(&self, number: U256) -> Result> { - self.db.get::(number.to::()).map_err(Into::into) + fn block_hash(&self, number: u64) -> Result> { + self.db.get::(number).map_err(Into::into) + } + + fn canonical_hashes_range(&self, start: BlockNumber, end: BlockNumber) -> Result> { + let range = start..end; + self.db + .cursor_read::() + .map(|mut cursor| { + cursor + .walk_range(range)? + .map(|result| result.map(|(_, hash)| hash).map_err(Into::into)) + .collect::>>() + })? + .map_err(Into::into) } } diff --git a/crates/storage/provider/src/providers/state/macros.rs b/crates/storage/provider/src/providers/state/macros.rs index ab4ae7b7bfb..d6d3f60ec83 100644 --- a/crates/storage/provider/src/providers/state/macros.rs +++ b/crates/storage/provider/src/providers/state/macros.rs @@ -5,12 +5,12 @@ /// /// Used to implement provider traits. macro_rules! delegate_impls_to_as_ref { - (for $target:ty => $($trait:ident $(where [$($generics:tt)*])? { $(fn $func:ident(&self, $($arg:ident: $argty:ty),*) -> $ret:path;)* })* ) => { + (for $target:ty => $($trait:ident $(where [$($generics:tt)*])? { $(fn $func:ident$(<$($generic_arg:ident: $generic_arg_ty:path),*>)?(&self, $($arg:ident: $argty:ty),*) -> $ret:path;)* })* ) => { $( impl<'a, $($($generics)*)?> $trait for $target { $( - fn $func(&self, $($arg: $argty),*) -> $ret { + fn $func$(<$($generic_arg: $generic_arg_ty),*>)?(&self, $($arg: $argty),*) -> $ret { self.as_ref().$func($($arg),*) } )* @@ -34,7 +34,8 @@ macro_rules! delegate_provider_impls { fn basic_account(&self, address: reth_primitives::Address) -> reth_interfaces::Result>; } BlockHashProvider $(where [$($generics)*])? { - fn block_hash(&self, number: reth_primitives::U256) -> reth_interfaces::Result>; + fn block_hash(&self, number: u64) -> reth_interfaces::Result>; + fn canonical_hashes_range(&self, start: reth_primitives::BlockNumber, end: reth_primitives::BlockNumber) -> reth_interfaces::Result>; } StateProvider $(where [$($generics)*])?{ fn storage(&self, account: reth_primitives::Address, storage_key: reth_primitives::StorageKey) -> reth_interfaces::Result>; diff --git a/crates/storage/provider/src/test_utils/mock.rs b/crates/storage/provider/src/test_utils/mock.rs index 83e5885ea29..0c976f27e87 100644 --- a/crates/storage/provider/src/test_utils/mock.rs +++ b/crates/storage/provider/src/test_utils/mock.rs @@ -124,7 +124,12 @@ impl HeaderProvider for MockEthProvider { range: impl RangeBounds, ) -> Result> { let lock = self.headers.lock(); - Ok(lock.values().filter(|header| range.contains(&header.number)).cloned().collect()) + + let mut headers: Vec<_> = + lock.values().filter(|header| range.contains(&header.number)).cloned().collect(); + headers.sort_by_key(|header| header.number); + + Ok(headers) } } @@ -171,21 +176,24 @@ impl ReceiptProvider for MockEthProvider { } impl BlockHashProvider for MockEthProvider { - fn block_hash(&self, number: U256) -> Result> { + fn block_hash(&self, number: u64) -> Result> { let lock = self.blocks.lock(); let hash = - lock.iter().find_map( - |(hash, b)| { - if b.number == number.to::() { - Some(*hash) - } else { - None - } - }, - ); + lock.iter().find_map(|(hash, b)| if b.number == number { Some(*hash) } else { None }); Ok(hash) } + + fn canonical_hashes_range(&self, start: BlockNumber, end: BlockNumber) -> Result> { + let range = start..end; + let lock = self.blocks.lock(); + + let mut hashes: Vec<_> = + lock.iter().filter(|(_, block)| range.contains(&block.number)).collect(); + hashes.sort_by_key(|(_, block)| block.number); + + Ok(hashes.into_iter().map(|(hash, _)| *hash).collect()) + } } impl BlockIdProvider for MockEthProvider { diff --git a/crates/storage/provider/src/test_utils/noop.rs b/crates/storage/provider/src/test_utils/noop.rs index d79970eaa71..b8f151e42e0 100644 --- a/crates/storage/provider/src/test_utils/noop.rs +++ b/crates/storage/provider/src/test_utils/noop.rs @@ -18,9 +18,13 @@ pub struct NoopProvider; /// Noop implementation for testing purposes impl BlockHashProvider for NoopProvider { - fn block_hash(&self, _number: U256) -> Result> { + fn block_hash(&self, _number: u64) -> Result> { Ok(None) } + + fn canonical_hashes_range(&self, _start: BlockNumber, _end: BlockNumber) -> Result> { + Ok(vec![]) + } } impl BlockIdProvider for NoopProvider { diff --git a/crates/storage/provider/src/traits/block_hash.rs b/crates/storage/provider/src/traits/block_hash.rs index 3bf3d9dfa85..855c77a6151 100644 --- a/crates/storage/provider/src/traits/block_hash.rs +++ b/crates/storage/provider/src/traits/block_hash.rs @@ -1,11 +1,14 @@ use auto_impl::auto_impl; use reth_interfaces::Result; -use reth_primitives::{H256, U256}; +use reth_primitives::{BlockNumber, H256}; /// Client trait for fetching block hashes by number. #[auto_impl(&, Arc, Box)] pub trait BlockHashProvider: Send + Sync { /// Get the hash of the block with the given number. Returns `None` if no block with this number /// exists. - fn block_hash(&self, number: U256) -> Result>; + fn block_hash(&self, number: BlockNumber) -> Result>; + + /// Get headers in range of block hashes or numbers + fn canonical_hashes_range(&self, start: BlockNumber, end: BlockNumber) -> Result>; } diff --git a/crates/storage/provider/src/traits/block_id.rs b/crates/storage/provider/src/traits/block_id.rs index fb393f35db8..42a9fa89add 100644 --- a/crates/storage/provider/src/traits/block_id.rs +++ b/crates/storage/provider/src/traits/block_id.rs @@ -1,6 +1,6 @@ use super::BlockHashProvider; use reth_interfaces::Result; -use reth_primitives::{BlockId, BlockNumberOrTag, ChainInfo, H256, U256}; +use reth_primitives::{BlockId, BlockNumberOrTag, ChainInfo, H256}; /// Client trait for transforming [BlockId]. #[auto_impl::auto_impl(&, Arc)] @@ -33,7 +33,7 @@ pub trait BlockIdProvider: BlockHashProvider + Send + Sync { return Ok(Some(self.chain_info()?.best_hash)) } self.convert_block_number(num)? - .map(|num| self.block_hash(U256::from(num))) + .map(|num| self.block_hash(num)) .transpose() .map(|maybe_hash| maybe_hash.flatten()) } From 1711d801af42d44619ae932fc30e05cae5646cdc Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Thu, 16 Mar 2023 22:33:11 -0400 Subject: [PATCH 157/191] feat: continuous download (#1744) --- bin/reth/src/node/mod.rs | 103 +++++-- crates/interfaces/src/p2p/error.rs | 8 + .../interfaces/src/p2p/headers/downloader.rs | 11 +- .../src/headers/reverse_headers.rs | 282 +++++++++++++++--- crates/stages/src/sets.rs | 52 +++- crates/stages/src/stages/headers.rs | 31 +- 6 files changed, 404 insertions(+), 83 deletions(-) diff --git a/bin/reth/src/node/mod.rs b/bin/reth/src/node/mod.rs index bb99109720e..4e784335f33 100644 --- a/bin/reth/src/node/mod.rs +++ b/bin/reth/src/node/mod.rs @@ -37,7 +37,7 @@ use reth_network::{ error::NetworkError, FetchClient, NetworkConfig, NetworkHandle, NetworkManager, }; use reth_network_api::NetworkInfo; -use reth_primitives::{BlockHashOrNumber, ChainSpec, Head, H256}; +use reth_primitives::{BlockHashOrNumber, ChainSpec, Head, SealedHeader, H256}; use reth_provider::{BlockProvider, HeaderProvider, ShareableDatabase}; use reth_rpc_engine_api::{EngineApi, EngineApiHandle}; use reth_staged_sync::{ @@ -50,7 +50,7 @@ use reth_staged_sync::{ }; use reth_stages::{ prelude::*, - stages::{ExecutionStage, SenderRecoveryStage, TotalDifficultyStage, FINISH}, + stages::{ExecutionStage, HeaderStage, SenderRecoveryStage, TotalDifficultyStage, FINISH}, }; use reth_tasks::TaskExecutor; use std::{ @@ -106,6 +106,12 @@ pub struct Command { #[clap(flatten)] network: NetworkArgs, + /// Prompt the downloader to download blocks one at a time. + /// + /// NOTE: This is for testing purposes only. + #[arg(long = "debug.continuous", help_heading = "Debug")] + continuous: bool, + /// Set the chain tip manually for testing purposes. /// /// NOTE: This is a temporary flag @@ -173,6 +179,10 @@ impl Command { .await?; info!(target: "reth::cli", "Started RPC server"); + if self.continuous { + info!(target: "reth::cli", "Continuous sync mode enabled"); + } + let engine_api_handle = self.init_engine_api(Arc::clone(&db), forkchoice_state_tx, &ctx.task_executor); info!(target: "reth::cli", "Engine API handler initialized"); @@ -259,6 +269,7 @@ impl Command { network.clone(), consensus, max_block, + self.continuous, ) .await?; @@ -391,17 +402,38 @@ impl Command { fetch_client: FetchClient, tip: H256, ) -> Result { - if let Some(number) = db.view(|tx| tx.get::(tip))?? { - info!(target: "reth::cli", ?tip, number, "Successfully looked up tip block number in the database"); - return Ok(number) + Ok(self.fetch_tip(db, fetch_client, BlockHashOrNumber::Hash(tip)).await?.number) + } + + /// Attempt to look up the block with the given number and return the header. + /// + /// NOTE: The download is attempted with infinite retries. + async fn fetch_tip( + &self, + db: Arc>, + fetch_client: FetchClient, + tip: BlockHashOrNumber, + ) -> Result { + let tip_num = match tip { + BlockHashOrNumber::Hash(hash) => { + info!(target: "reth::cli", ?hash, "Fetching tip block from the network."); + db.view(|tx| tx.get::(hash))??.unwrap() + } + BlockHashOrNumber::Number(number) => number, + }; + + // try to look up the header in the database + if let Some(header) = db.view(|tx| tx.get::(tip_num))?? { + info!(target: "reth::cli", ?tip, "Successfully looked up tip block in the database"); + return Ok(header.seal_slow()) } - info!(target: "reth::cli", ?tip, "Fetching tip block number from the network."); + info!(target: "reth::cli", ?tip, "Fetching tip block from the network."); loop { - match get_single_header(fetch_client.clone(), BlockHashOrNumber::Hash(tip)).await { + match get_single_header(fetch_client.clone(), tip).await { Ok(tip_header) => { - info!(target: "reth::cli", ?tip, number = tip_header.number, "Successfully fetched tip block number"); - return Ok(tip_header.number) + info!(target: "reth::cli", ?tip, "Successfully fetched tip"); + return Ok(tip_header) } Err(error) => { error!(target: "reth::cli", %error, "Failed to fetch the tip. Retrying..."); @@ -429,6 +461,7 @@ impl Command { .build(ShareableDatabase::new(db, self.chain.clone())) } + #[allow(clippy::too_many_arguments)] async fn build_pipeline( &self, config: &Config, @@ -437,6 +470,7 @@ impl Command { updater: U, consensus: &Arc, max_block: Option, + continuous: bool, ) -> eyre::Result, U>> where H: HeaderDownloader + 'static, @@ -453,24 +487,43 @@ impl Command { } let factory = reth_executor::Factory::new(self.chain.clone()); + + let default_stages = if continuous { + let continuous_headers = + HeaderStage::new(header_downloader, consensus.clone()).continuous(); + let online_builder = OnlineStages::builder_with_headers( + continuous_headers, + consensus.clone(), + body_downloader, + ); + DefaultStages::::add_offline_stages( + online_builder, + updater.clone(), + factory.clone(), + ) + } else { + DefaultStages::new( + consensus.clone(), + header_downloader, + body_downloader, + updater.clone(), + factory.clone(), + ) + .builder() + }; + let pipeline = builder - .with_sync_state_updater(updater.clone()) + .with_sync_state_updater(updater) .add_stages( - DefaultStages::new( - consensus.clone(), - header_downloader, - body_downloader, - updater, - factory.clone(), - ) - .set( - TotalDifficultyStage::new(consensus.clone()) - .with_commit_threshold(stage_conf.total_difficulty.commit_threshold), - ) - .set(SenderRecoveryStage { - commit_threshold: stage_conf.sender_recovery.commit_threshold, - }) - .set(ExecutionStage::new(factory, stage_conf.execution.commit_threshold)), + default_stages + .set( + TotalDifficultyStage::new(consensus.clone()) + .with_commit_threshold(stage_conf.total_difficulty.commit_threshold), + ) + .set(SenderRecoveryStage { + commit_threshold: stage_conf.sender_recovery.commit_threshold, + }) + .set(ExecutionStage::new(factory, stage_conf.execution.commit_threshold)), ) .build(); diff --git a/crates/interfaces/src/p2p/error.rs b/crates/interfaces/src/p2p/error.rs index 78623328a62..70e22f6cd01 100644 --- a/crates/interfaces/src/p2p/error.rs +++ b/crates/interfaces/src/p2p/error.rs @@ -148,6 +148,14 @@ pub enum DownloadError { /// The hash of the expected tip expected: H256, }, + /// Received a tip with an invalid tip number + #[error("Received invalid tip number: {received:?}. Expected {expected:?}.")] + InvalidTipNumber { + /// The block number of the received tip + received: u64, + /// The block number of the expected tip + expected: u64, + }, /// Received a response to a request with unexpected start block #[error("Headers response starts at unexpected block: {received:?}. Expected {expected:?}.")] HeadersResponseStartBlockMismatch { diff --git a/crates/interfaces/src/p2p/headers/downloader.rs b/crates/interfaces/src/p2p/headers/downloader.rs index 7b129e29d80..ac10d2be79e 100644 --- a/crates/interfaces/src/p2p/headers/downloader.rs +++ b/crates/interfaces/src/p2p/headers/downloader.rs @@ -3,7 +3,7 @@ use crate::{ p2p::error::{DownloadError, DownloadResult}, }; use futures::Stream; -use reth_primitives::{SealedHeader, H256}; +use reth_primitives::{BlockHashOrNumber, SealedHeader, H256}; /// A downloader capable of fetching and yielding block headers. /// @@ -48,6 +48,8 @@ pub enum SyncTarget { /// The benefit of this variant is, that this already provides the block number of the highest /// missing block. Gap(SealedHeader), + /// This represents a tip by block number + TipNum(u64), } // === impl SyncTarget === @@ -57,10 +59,11 @@ impl SyncTarget { /// /// This returns the hash if the target is [SyncTarget::Tip] or the `parent_hash` of the given /// header in [SyncTarget::Gap] - pub fn tip(&self) -> H256 { + pub fn tip(&self) -> BlockHashOrNumber { match self { - SyncTarget::Tip(tip) => *tip, - SyncTarget::Gap(gap) => gap.parent_hash, + SyncTarget::Tip(tip) => (*tip).into(), + SyncTarget::Gap(gap) => gap.parent_hash.into(), + SyncTarget::TipNum(num) => (*num).into(), } } } diff --git a/crates/net/downloaders/src/headers/reverse_headers.rs b/crates/net/downloaders/src/headers/reverse_headers.rs index 4d00ce0edfa..241bf3ded5a 100644 --- a/crates/net/downloaders/src/headers/reverse_headers.rs +++ b/crates/net/downloaders/src/headers/reverse_headers.rs @@ -16,7 +16,9 @@ use reth_interfaces::{ priority::Priority, }, }; -use reth_primitives::{BlockNumber, Header, HeadersDirection, PeerId, SealedHeader, H256}; +use reth_primitives::{ + BlockHashOrNumber, BlockNumber, Header, HeadersDirection, PeerId, SealedHeader, H256, +}; use reth_tasks::{TaskSpawner, TokioTaskExecutor}; use std::{ cmp::{Ordering, Reverse}, @@ -116,14 +118,14 @@ where self.local_head.as_ref().expect("is initialized").number } - /// Returns the existing sync target hash. + /// Returns the existing sync target. /// /// # Panics /// /// If the sync target has never been set. #[inline] - fn existing_sync_target_hash(&self) -> H256 { - self.sync_target.as_ref().expect("is initialized").hash + fn existing_sync_target(&self) -> SyncTargetBlock { + self.sync_target.as_ref().expect("is initialized").clone() } /// Max requests to handle at the same time @@ -198,7 +200,7 @@ where headers: Vec
, peer_id: PeerId, ) -> Result<(), HeadersResponseError> { - let sync_target_hash = self.existing_sync_target_hash(); + let sync_target = self.existing_sync_target(); let mut validated = Vec::with_capacity(headers.len()); let sealed_headers = headers.into_par_iter().map(|h| h.seal_slow()).collect::>(); @@ -211,15 +213,45 @@ where trace!(target: "downloaders::headers", ?error ,"Failed to validate header"); return Err(HeadersResponseError { request, peer_id: Some(peer_id), error }) } - } else if parent.hash() != sync_target_hash { - return Err(HeadersResponseError { - request, - peer_id: Some(peer_id), - error: DownloadError::InvalidTip { - received: parent.hash(), - expected: sync_target_hash, - }, - }) + } else { + match sync_target { + SyncTargetBlock::Hash(hash) => { + if parent.hash() != hash { + return Err(HeadersResponseError { + request, + peer_id: Some(peer_id), + error: DownloadError::InvalidTip { + received: parent.hash(), + expected: hash, + }, + }) + } + } + SyncTargetBlock::Number(number) => { + if parent.number != number { + return Err(HeadersResponseError { + request, + peer_id: Some(peer_id), + error: DownloadError::InvalidTipNumber { + received: parent.number, + expected: number, + }, + }) + } + } + SyncTargetBlock::HashAndNumber { hash, .. } => { + if parent.hash() != hash { + return Err(HeadersResponseError { + request, + peer_id: Some(peer_id), + error: DownloadError::InvalidTip { + received: parent.hash(), + expected: hash, + }, + }) + } + } + } } validated.push(parent); @@ -246,7 +278,7 @@ where fn on_block_number_update(&mut self, target_block_number: u64, next_block: u64) { // Update the trackers if let Some(old_target) = - self.sync_target.as_mut().and_then(|t| t.number.replace(target_block_number)) + self.sync_target.as_mut().and_then(|t| t.replace_number(target_block_number)) { if target_block_number > old_target { // the new target is higher than the old target we need to update the @@ -277,7 +309,7 @@ where &mut self, response: HeadersRequestOutcome, ) -> Result<(), HeadersResponseError> { - let sync_target_hash = self.existing_sync_target_hash(); + let sync_target = self.existing_sync_target(); let HeadersRequestOutcome { request, outcome } = response; match outcome { Ok(res) => { @@ -299,15 +331,43 @@ where let target = headers.remove(0).seal_slow(); - if target.hash() != sync_target_hash { - return Err(HeadersResponseError { - request, - peer_id: Some(peer_id), - error: DownloadError::InvalidTip { - received: target.hash(), - expected: sync_target_hash, - }, - }) + match sync_target { + SyncTargetBlock::Hash(hash) => { + if target.hash() != hash { + return Err(HeadersResponseError { + request, + peer_id: Some(peer_id), + error: DownloadError::InvalidTip { + received: target.hash(), + expected: hash, + }, + }) + } + } + SyncTargetBlock::Number(number) => { + if target.number != number { + return Err(HeadersResponseError { + request, + peer_id: Some(peer_id), + error: DownloadError::InvalidTipNumber { + received: target.number, + expected: number, + }, + }) + } + } + SyncTargetBlock::HashAndNumber { hash, .. } => { + if target.hash() != hash { + return Err(HeadersResponseError { + request, + peer_id: Some(peer_id), + error: DownloadError::InvalidTip { + received: target.hash(), + expected: hash, + }, + }) + } + } } trace!(target: "downloaders::headers", head=?self.local_block_number(), hash=?target.hash(), number=%target.number, "Received sync target"); @@ -463,8 +523,8 @@ where } /// Returns the request for the `sync_target` header. - fn get_sync_target_request(&self, start: H256) -> HeadersRequest { - HeadersRequest { start: start.into(), limit: 1, direction: HeadersDirection::Falling } + fn get_sync_target_request(&self, start: BlockHashOrNumber) -> HeadersRequest { + HeadersRequest { start, limit: 1, direction: HeadersDirection::Falling } } /// Starts a request future @@ -552,7 +612,7 @@ where /// If the given target is different from the current target, we need to update the sync target fn update_sync_target(&mut self, target: SyncTarget) { - let current_tip = self.sync_target.as_ref().map(|t| t.hash); + let current_tip = self.sync_target.as_ref().and_then(|t| t.hash()); match target { SyncTarget::Tip(tip) => { if Some(tip) != current_tip { @@ -574,8 +634,9 @@ where trace!(target: "downloaders::headers", new=?target, "Request new sync target"); self.metrics.out_of_order_requests.increment(1); self.sync_target = Some(new_sync_target); - self.sync_target_request = - Some(self.request_fut(self.get_sync_target_request(tip), Priority::High)); + self.sync_target_request = Some( + self.request_fut(self.get_sync_target_request(tip.into()), Priority::High), + ); } } SyncTarget::Gap(existing) => { @@ -591,15 +652,23 @@ where // Update the sync target hash self.sync_target = match self.sync_target.take() { - Some(mut sync_target) => { - sync_target.hash = target; - Some(sync_target) - } + Some(sync_target) => Some(sync_target.with_hash(target)), None => Some(SyncTargetBlock::from_hash(target)), }; self.on_block_number_update(parent_block_number, parent_block_number); } } + SyncTarget::TipNum(num) => { + let current_tip_num = self.sync_target.as_ref().and_then(|t| t.number()); + if Some(num) != current_tip_num { + trace!(target: "downloaders::headers", %num, "Updating sync target based on num"); + // just update the sync target + self.sync_target = Some(SyncTargetBlock::from_number(num)); + self.sync_target_request = Some( + self.request_fut(self.get_sync_target_request(num.into()), Priority::High), + ); + } + } } } @@ -823,24 +892,90 @@ impl HeadersResponseError { } /// The block to which we want to close the gap: (local head...sync target] -#[derive(Debug, Default)] -struct SyncTargetBlock { +/// This tracks the sync target block, so this could be either a block number or hash. +#[derive(Clone, Debug)] +pub enum SyncTargetBlock { /// Block hash of the targeted block - hash: H256, - /// This is an `Option` because we don't know the block number at first - number: Option, + Hash(H256), + /// Block number of the targeted block + Number(u64), + /// Both the block hash and number of the targeted block + HashAndNumber { + /// Block hash of the targeted block + hash: H256, + /// Block number of the targeted block + number: u64, + }, } impl SyncTargetBlock { /// Create new instance from hash. fn from_hash(hash: H256) -> Self { - Self { hash, number: None } + Self::Hash(hash) + } + + /// Create new instance from number. + fn from_number(num: u64) -> Self { + Self::Number(num) + } + + /// Set the hash for the sync target. + fn with_hash(self, hash: H256) -> Self { + match self { + Self::Hash(_) => Self::Hash(hash), + Self::Number(number) => Self::HashAndNumber { hash, number }, + Self::HashAndNumber { number, .. } => Self::HashAndNumber { hash, number }, + } } /// Set a number on the instance. - fn with_number(mut self, number: u64) -> Self { - self.number = Some(number); - self + fn with_number(self, number: u64) -> Self { + match self { + Self::Hash(hash) => Self::HashAndNumber { hash, number }, + Self::Number(_) => Self::Number(number), + Self::HashAndNumber { hash, .. } => Self::HashAndNumber { hash, number }, + } + } + + /// Replace the target block number, and return the old block number, if it was set. + /// + /// If the target block is a hash, this be converted into a `HashAndNumber`, but return `None`. + /// The semantics should be equivalent to that of `Option::replace`. + fn replace_number(&mut self, number: u64) -> Option { + match self { + Self::Hash(hash) => { + *self = Self::HashAndNumber { hash: *hash, number }; + None + } + Self::Number(old_number) => { + let res = Some(*old_number); + *self = Self::Number(number); + res + } + Self::HashAndNumber { number: old_number, hash } => { + let res = Some(*old_number); + *self = Self::HashAndNumber { hash: *hash, number }; + res + } + } + } + + /// Return the hash of the target block, if it is set. + fn hash(&self) -> Option { + match self { + Self::Hash(hash) => Some(*hash), + Self::Number(_) => None, + Self::HashAndNumber { hash, .. } => Some(*hash), + } + } + + /// Return the block number of the sync target, if it is set. + fn number(&self) -> Option { + match self { + Self::Hash(_) => None, + Self::Number(number) => Some(*number), + Self::HashAndNumber { number, .. } => Some(*number), + } } } @@ -989,6 +1124,65 @@ mod tests { use reth_interfaces::test_utils::{TestConsensus, TestHeadersClient}; use reth_primitives::SealedHeader; + /// Tests that `replace_number` works the same way as Option::replace + #[test] + fn test_replace_number_semantics() { + struct Fixture { + // input fields (both SyncTargetBlock and Option) + sync_target_block: SyncTargetBlock, + sync_target_option: Option, + + // option to replace + replace_number: u64, + + // expected method result + expected_result: Option, + + // output state + new_number: u64, + } + + let fixtures = vec![ + Fixture { + sync_target_block: SyncTargetBlock::Hash(H256::random()), + // Hash maps to None here, all other variants map to Some + sync_target_option: None, + replace_number: 1, + expected_result: None, + new_number: 1, + }, + Fixture { + sync_target_block: SyncTargetBlock::Number(1), + sync_target_option: Some(1), + replace_number: 2, + expected_result: Some(1), + new_number: 2, + }, + Fixture { + sync_target_block: SyncTargetBlock::HashAndNumber { + hash: H256::random(), + number: 1, + }, + sync_target_option: Some(1), + replace_number: 2, + expected_result: Some(1), + new_number: 2, + }, + ]; + + for fixture in fixtures { + let mut sync_target_block = fixture.sync_target_block; + let result = sync_target_block.replace_number(fixture.replace_number); + assert_eq!(result, fixture.expected_result); + assert_eq!(sync_target_block.number(), Some(fixture.new_number)); + + let mut sync_target_option = fixture.sync_target_option; + let option_result = sync_target_option.replace(fixture.replace_number); + assert_eq!(option_result, fixture.expected_result); + assert_eq!(sync_target_option, Some(fixture.new_number)); + } + } + /// Tests that request calc works #[test] fn test_sync_target_update() { @@ -1013,7 +1207,7 @@ mod tests { assert!(downloader.sync_target_request.is_none()); assert_matches!( downloader.sync_target, - Some(target) => target.number.is_some() + Some(target) => target.number().is_some() ); } diff --git a/crates/stages/src/sets.rs b/crates/stages/src/sets.rs index aca8bca593d..501cd0fdde3 100644 --- a/crates/stages/src/sets.rs +++ b/crates/stages/src/sets.rs @@ -94,6 +94,23 @@ impl DefaultStages { } } +impl DefaultStages +where + S: StatusUpdater + 'static, + EF: ExecutorFactory, +{ + /// Appends the default offline stages and default finish stage to the given builder. + pub fn add_offline_stages( + default_offline: StageSetBuilder, + status_updater: S, + executor_factory: EF, + ) -> StageSetBuilder { + default_offline + .add_set(OfflineStages::new(executor_factory)) + .add_stage(FinishStage::new(status_updater)) + } +} + impl StageSet for DefaultStages where DB: Database, @@ -103,10 +120,7 @@ where EF: ExecutorFactory, { fn builder(self) -> StageSetBuilder { - self.online - .builder() - .add_set(OfflineStages::new(self.executor_factory)) - .add_stage(FinishStage::new(self.status_updater)) + Self::add_offline_stages(self.online.builder(), self.status_updater, self.executor_factory) } } @@ -131,6 +145,36 @@ impl OnlineStages { } } +impl OnlineStages +where + H: HeaderDownloader + 'static, + B: BodyDownloader + 'static, +{ + /// Create a new builder using the given headers stage. + pub fn builder_with_headers( + headers: HeaderStage, + consensus: Arc, + body_downloader: B, + ) -> StageSetBuilder { + StageSetBuilder::default() + .add_stage(headers) + .add_stage(TotalDifficultyStage::new(consensus.clone())) + .add_stage(BodyStage { downloader: body_downloader, consensus }) + } + + /// Create a new builder using the given bodies stage. + pub fn builder_with_bodies( + bodies: BodyStage, + consensus: Arc, + header_downloader: H, + ) -> StageSetBuilder { + StageSetBuilder::default() + .add_stage(HeaderStage::new(header_downloader, consensus.clone())) + .add_stage(TotalDifficultyStage::new(consensus.clone())) + .add_stage(bodies) + } +} + impl StageSet for OnlineStages where DB: Database, diff --git a/crates/stages/src/stages/headers.rs b/crates/stages/src/stages/headers.rs index b656652131e..b33ae869a8e 100644 --- a/crates/stages/src/stages/headers.rs +++ b/crates/stages/src/stages/headers.rs @@ -11,7 +11,7 @@ use reth_interfaces::{ p2p::headers::downloader::{HeaderDownloader, SyncTarget}, provider::ProviderError, }; -use reth_primitives::{BlockNumber, SealedHeader}; +use reth_primitives::{BlockHashOrNumber, BlockNumber, SealedHeader}; use reth_provider::Transaction; use std::sync::Arc; use tracing::*; @@ -38,6 +38,8 @@ pub struct HeaderStage { downloader: D, /// Consensus client implementation consensus: Arc, + /// Whether or not the stage should download continuously, or wait for the fork choice state + continuous: bool, } // === impl HeaderStage === @@ -48,7 +50,7 @@ where { /// Create a new header stage pub fn new(downloader: D, consensus: Arc) -> Self { - Self { downloader, consensus } + Self { downloader, consensus, continuous: false } } fn is_stage_done( @@ -64,6 +66,12 @@ where Ok(header_cursor.next()?.map(|(next_num, _)| head_num + 1 == next_num).unwrap_or_default()) } + /// Set the stage to download continuously + pub fn continuous(mut self) -> Self { + self.continuous = true; + self + } + /// Get the head and tip of the range we need to sync /// /// See also [SyncTarget] @@ -104,7 +112,14 @@ where // reverse from there. Else, it should use whatever the forkchoice state reports. let target = match next_header { Some(header) if stage_progress + 1 != header.number => SyncTarget::Gap(header), - None => SyncTarget::Tip(self.next_fork_choice_state().await.head_block_hash), + None => { + if self.continuous { + tracing::trace!(target: "sync::stages::headers", ?head_num, "No next header found, using continuous sync strategy"); + SyncTarget::TipNum(head_num + 1) + } else { + SyncTarget::Tip(self.next_fork_choice_state().await.head_block_hash) + } + } _ => return Err(StageError::StageProgress(stage_progress)), }; @@ -250,7 +265,10 @@ impl SyncGap { /// Returns `true` if the gap from the head to the target was closed #[inline] pub fn is_closed(&self) -> bool { - self.local_head.hash() == self.target.tip() + match self.target.tip() { + BlockHashOrNumber::Hash(hash) => self.local_head.hash() == hash, + BlockHashOrNumber::Number(num) => self.local_head.number == num, + } } } @@ -326,6 +344,7 @@ mod tests { HeaderStage { consensus: self.consensus.clone(), downloader: (*self.downloader_factory)(), + continuous: false, } } } @@ -510,7 +529,7 @@ mod tests { let gap = stage.get_sync_gap(&tx, stage_progress).await.unwrap(); assert_eq!(gap.local_head, head); - assert_eq!(gap.target.tip(), consensus_tip); + assert_eq!(gap.target.tip(), consensus_tip.into()); // Checkpoint and gap tx.put::(gap_tip.number, gap_tip.hash()) @@ -520,7 +539,7 @@ mod tests { let gap = stage.get_sync_gap(&tx, stage_progress).await.unwrap(); assert_eq!(gap.local_head, head); - assert_eq!(gap.target.tip(), gap_tip.parent_hash); + assert_eq!(gap.target.tip(), gap_tip.parent_hash.into()); // Checkpoint and gap closed tx.put::(gap_fill.number, gap_fill.hash()) From 8485d7b209f577753ca2baeb441e61ac6b50093c Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Fri, 17 Mar 2023 08:36:27 +0200 Subject: [PATCH 158/191] ci: revert nightly downgrade (#1807) --- .github/workflows/ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d56f7da792e..5a171f1177c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,9 +23,8 @@ jobs: - name: Checkout sources uses: actions/checkout@v3 - name: Install toolchain - uses: dtolnay/rust-toolchain@master + uses: dtolnay/rust-toolchain@nightly with: - toolchain: nightly-2023-03-14 components: rustfmt, clippy - uses: Swatinem/rust-cache@v2 with: From 8f60f7d75a28357d5d581bc8fb879ad07b700412 Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Fri, 17 Mar 2023 13:39:08 +0200 Subject: [PATCH 159/191] fix(cli): tip number lookup (#1808) --- bin/reth/src/node/mod.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/bin/reth/src/node/mod.rs b/bin/reth/src/node/mod.rs index 4e784335f33..39aa20f7f92 100644 --- a/bin/reth/src/node/mod.rs +++ b/bin/reth/src/node/mod.rs @@ -37,7 +37,7 @@ use reth_network::{ error::NetworkError, FetchClient, NetworkConfig, NetworkHandle, NetworkManager, }; use reth_network_api::NetworkInfo; -use reth_primitives::{BlockHashOrNumber, ChainSpec, Head, SealedHeader, H256}; +use reth_primitives::{BlockHashOrNumber, ChainSpec, Head, Header, SealedHeader, H256}; use reth_provider::{BlockProvider, HeaderProvider, ShareableDatabase}; use reth_rpc_engine_api::{EngineApi, EngineApiHandle}; use reth_staged_sync::{ @@ -414,16 +414,16 @@ impl Command { fetch_client: FetchClient, tip: BlockHashOrNumber, ) -> Result { - let tip_num = match tip { - BlockHashOrNumber::Hash(hash) => { - info!(target: "reth::cli", ?hash, "Fetching tip block from the network."); - db.view(|tx| tx.get::(hash))??.unwrap() - } - BlockHashOrNumber::Number(number) => number, - }; + let header = db.view(|tx| -> Result, reth_db::Error> { + let number = match tip { + BlockHashOrNumber::Hash(hash) => tx.get::(hash)?, + BlockHashOrNumber::Number(number) => Some(number), + }; + Ok(number.map(|number| tx.get::(number)).transpose()?.flatten()) + })??; // try to look up the header in the database - if let Some(header) = db.view(|tx| tx.get::(tip_num))?? { + if let Some(header) = header { info!(target: "reth::cli", ?tip, "Successfully looked up tip block in the database"); return Ok(header.seal_slow()) } From 376bfa2b5116119899888d6ed4254c23090b65a6 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 17 Mar 2023 12:43:13 +0100 Subject: [PATCH 160/191] chore: spawn engine api task as critical (#1813) --- bin/reth/src/node/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/reth/src/node/mod.rs b/bin/reth/src/node/mod.rs index 39aa20f7f92..94888d82f84 100644 --- a/bin/reth/src/node/mod.rs +++ b/bin/reth/src/node/mod.rs @@ -337,7 +337,7 @@ impl Command { message_rx, forkchoice_state_tx, ); - task_executor.spawn(engine_api); + task_executor.spawn_critical("engine API task", engine_api); message_tx } From 3a01ad9cd7ca0c5f0e0d57b1af1b89308c17f9a8 Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Fri, 17 Mar 2023 14:19:12 +0200 Subject: [PATCH 161/191] fix(cli): create net dir if non existent (#1809) --- bin/reth/src/args/network_args.rs | 3 +-- bin/reth/src/node/mod.rs | 3 ++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/reth/src/args/network_args.rs b/bin/reth/src/args/network_args.rs index ff99f419ca2..3d06cd38185 100644 --- a/bin/reth/src/args/network_args.rs +++ b/bin/reth/src/args/network_args.rs @@ -55,9 +55,8 @@ impl NetworkArgs { config: &Config, chain_spec: Arc, ) -> NetworkConfigBuilder { - let peers_file = (!self.no_persist_peers).then_some(&self.peers_file); let network_config_builder = config - .network_config(self.nat, peers_file.map(|f| f.as_ref().to_path_buf())) + .network_config(self.nat, self.persistent_peers_file()) .boot_nodes(self.bootnodes.clone().unwrap_or_else(mainnet_nodes)) .chain_spec(chain_spec); diff --git a/bin/reth/src/node/mod.rs b/bin/reth/src/node/mod.rs index 94888d82f84..f16fdc9e550 100644 --- a/bin/reth/src/node/mod.rs +++ b/bin/reth/src/node/mod.rs @@ -551,7 +551,8 @@ async fn run_network_until_shutdown( let known_peers = network.all_peers().collect::>(); if let Ok(known_peers) = serde_json::to_string_pretty(&known_peers) { trace!(target : "reth::cli", peers_file =?file_path, num_peers=%known_peers.len(), "Saving current peers"); - match std::fs::write(&file_path, known_peers) { + let parent_dir = file_path.parent().map(std::fs::create_dir_all).transpose(); + match parent_dir.and_then(|_| std::fs::write(&file_path, known_peers)) { Ok(_) => { info!(target: "reth::cli", peers_file=?file_path, "Wrote network peers to file"); } From f08a70dad88d8714d184bead6199a807f2d3217b Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 17 Mar 2023 13:19:21 +0100 Subject: [PATCH 162/191] feat: add int or hex serde parsers (#1811) --- crates/primitives/src/serde_helper/mod.rs | 2 + crates/primitives/src/serde_helper/num.rs | 88 +++++++++++++++++++ .../src/eth/trace/{geth.rs => geth/mod.rs} | 0 3 files changed, 90 insertions(+) create mode 100644 crates/primitives/src/serde_helper/num.rs rename crates/rpc/rpc-types/src/eth/trace/{geth.rs => geth/mod.rs} (100%) diff --git a/crates/primitives/src/serde_helper/mod.rs b/crates/primitives/src/serde_helper/mod.rs index 7efbed1daca..b2d44797875 100644 --- a/crates/primitives/src/serde_helper/mod.rs +++ b/crates/primitives/src/serde_helper/mod.rs @@ -3,6 +3,8 @@ mod jsonu256; pub use jsonu256::*; +pub mod num; + /// serde functions for handling primitive `u64` as [U64](crate::U64) pub mod u64_hex { use crate::U64; diff --git a/crates/primitives/src/serde_helper/num.rs b/crates/primitives/src/serde_helper/num.rs new file mode 100644 index 00000000000..beab749ab3b --- /dev/null +++ b/crates/primitives/src/serde_helper/num.rs @@ -0,0 +1,88 @@ +//! Numeric helpers + +use crate::U256; +use serde::{de, Deserialize, Deserializer}; +use std::str::FromStr; + +/// Deserializes the input into an `Option`, using [`from_int_or_hex`] to deserialize the +/// inner value. +pub fn from_int_or_hex_opt<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + match Option::::deserialize(deserializer)? { + Some(val) => val.try_into_u256().map(Some), + None => Ok(None), + } +} + +/// Deserializes the input into a U256, accepting both 0x-prefixed hex and decimal strings with +/// arbitrary precision, defined by serde_json's [`Number`](serde_json::Number). +pub fn from_int_or_hex<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + NumberOrHexU256::deserialize(deserializer)?.try_into_u256() +} + +#[derive(Deserialize)] +#[serde(untagged)] +enum NumberOrHexU256 { + Int(serde_json::Number), + Hex(U256), +} + +impl NumberOrHexU256 { + fn try_into_u256(self) -> Result { + match self { + NumberOrHexU256::Int(num) => { + U256::from_str(num.to_string().as_str()).map_err(E::custom) + } + NumberOrHexU256::Hex(val) => Ok(val), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_u256_int_or_hex() { + #[derive(Debug, Deserialize, PartialEq, Eq)] + struct V(#[serde(deserialize_with = "from_int_or_hex")] U256); + + proptest::proptest!(|(value: u64)| { + let u256_val = U256::from(value); + + let num_obj = serde_json::to_string(&value).unwrap(); + let hex_obj = serde_json::to_string(&u256_val).unwrap(); + + let int_val:V = serde_json::from_str(&num_obj).unwrap(); + let hex_val = serde_json::from_str(&hex_obj).unwrap(); + assert_eq!(int_val, hex_val); + }); + } + + #[test] + fn test_u256_int_or_hex_opt() { + #[derive(Debug, Deserialize, PartialEq, Eq)] + struct V(#[serde(deserialize_with = "from_int_or_hex_opt")] Option); + + let null = serde_json::to_string(&None::).unwrap(); + let val: V = serde_json::from_str(&null).unwrap(); + assert!(val.0.is_none()); + + proptest::proptest!(|(value: u64)| { + let u256_val = U256::from(value); + + let num_obj = serde_json::to_string(&value).unwrap(); + let hex_obj = serde_json::to_string(&u256_val).unwrap(); + + let int_val:V = serde_json::from_str(&num_obj).unwrap(); + let hex_val = serde_json::from_str(&hex_obj).unwrap(); + assert_eq!(int_val, hex_val); + assert_eq!(int_val.0, Some(u256_val)); + }); + } +} diff --git a/crates/rpc/rpc-types/src/eth/trace/geth.rs b/crates/rpc/rpc-types/src/eth/trace/geth/mod.rs similarity index 100% rename from crates/rpc/rpc-types/src/eth/trace/geth.rs rename to crates/rpc/rpc-types/src/eth/trace/geth/mod.rs From 1a317f20e4d6f89577d5951c6eac2409e991dee8 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 17 Mar 2023 13:20:00 +0100 Subject: [PATCH 163/191] refactor: split consensus crate (#1814) --- Cargo.lock | 15 ++++++++++--- Cargo.toml | 3 ++- bin/reth/Cargo.toml | 10 ++++----- bin/reth/src/chain/import.rs | 2 +- bin/reth/src/node/mod.rs | 2 +- bin/reth/src/stage/mod.rs | 2 +- crates/consensus/Cargo.toml | 22 ------------------- crates/consensus/beacon/Cargo.toml | 19 ++++++++++++++++ .../beacon => beacon/src}/beacon_consensus.rs | 5 ++--- .../{src/beacon => beacon/src}/builder.rs | 0 crates/consensus/beacon/src/lib.rs | 13 +++++++++++ crates/consensus/common/Cargo.toml | 20 +++++++++++++++++ crates/consensus/{ => common}/src/lib.rs | 4 +--- .../consensus/{ => common}/src/validation.rs | 6 ++--- crates/consensus/src/beacon/mod.rs | 7 ------ 15 files changed, 79 insertions(+), 51 deletions(-) delete mode 100644 crates/consensus/Cargo.toml create mode 100644 crates/consensus/beacon/Cargo.toml rename crates/consensus/{src/beacon => beacon/src}/beacon_consensus.rs (98%) rename crates/consensus/{src/beacon => beacon/src}/builder.rs (100%) create mode 100644 crates/consensus/beacon/src/lib.rs create mode 100644 crates/consensus/common/Cargo.toml rename crates/consensus/{ => common}/src/lib.rs (76%) rename crates/consensus/{ => common}/src/validation.rs (99%) delete mode 100644 crates/consensus/src/beacon/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 8926959e519..09fd1b2f003 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4569,7 +4569,7 @@ dependencies = [ "metrics-exporter-prometheus", "metrics-util", "proptest", - "reth-consensus", + "reth-beacon-consensus", "reth-db", "reth-discv4", "reth-downloaders", @@ -4598,6 +4598,16 @@ dependencies = [ "tui", ] +[[package]] +name = "reth-beacon-consensus" +version = "0.1.0" +dependencies = [ + "reth-consensus-common", + "reth-interfaces", + "reth-primitives", + "tokio", +] + [[package]] name = "reth-codecs" version = "0.1.0" @@ -4614,7 +4624,7 @@ dependencies = [ ] [[package]] -name = "reth-consensus" +name = "reth-consensus-common" version = "0.1.0" dependencies = [ "assert_matches", @@ -4622,7 +4632,6 @@ dependencies = [ "reth-interfaces", "reth-primitives", "reth-provider", - "tokio", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 62904d8309d..0481e20bd9e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,8 @@ [workspace] members = [ "bin/reth", - "crates/consensus", + "crates/consensus/beacon", + "crates/consensus/common", "crates/executor", "crates/interfaces", "crates/metrics/metrics-derive", diff --git a/bin/reth/Cargo.toml b/bin/reth/Cargo.toml index 0e220277567..ff1aa0aed3a 100644 --- a/bin/reth/Cargo.toml +++ b/bin/reth/Cargo.toml @@ -9,22 +9,22 @@ readme = "README.md" [dependencies] # reth reth-primitives = { path = "../../crates/primitives", features = ["arbitrary"] } -reth-db = {path = "../../crates/storage/db", features = ["mdbx", "test-utils"] } +reth-db = { path = "../../crates/storage/db", features = ["mdbx", "test-utils"] } # TODO: Temporary use of the test-utils feature reth-provider = { path = "../../crates/storage/provider", features = ["test-utils"] } reth-staged-sync = { path = "../../crates/staged-sync" } reth-stages = { path = "../../crates/stages"} reth-interfaces = { path = "../../crates/interfaces", features = ["test-utils"] } reth-transaction-pool = { path = "../../crates/transaction-pool", features = ["test-utils"] } -reth-consensus = { path = "../../crates/consensus" } +reth-beacon-consensus = { path = "../../crates/consensus/beacon" } reth-executor = { path = "../../crates/executor" } reth-rpc-engine-api = { path = "../../crates/rpc/rpc-engine-api" } reth-rpc-builder = { path = "../../crates/rpc/rpc-builder" } reth-rpc = { path = "../../crates/rpc/rpc" } reth-rlp = { path = "../../crates/rlp" } -reth-network = {path = "../../crates/net/network", features = ["serde"] } -reth-network-api = {path = "../../crates/net/network-api" } -reth-downloaders = {path = "../../crates/net/downloaders", features = ["test-utils"] } +reth-network = { path = "../../crates/net/network", features = ["serde"] } +reth-network-api = { path = "../../crates/net/network-api" } +reth-downloaders = { path = "../../crates/net/downloaders", features = ["test-utils"] } reth-tracing = { path = "../../crates/tracing" } reth-tasks = { path = "../../crates/tasks" } reth-net-nat = { path = "../../crates/net/nat" } diff --git a/bin/reth/src/chain/import.rs b/bin/reth/src/chain/import.rs index 40c382b278d..2a6ad2490da 100644 --- a/bin/reth/src/chain/import.rs +++ b/bin/reth/src/chain/import.rs @@ -5,7 +5,7 @@ use crate::{ use clap::{crate_version, Parser}; use eyre::Context; use futures::{Stream, StreamExt}; -use reth_consensus::beacon::BeaconConsensus; +use reth_beacon_consensus::BeaconConsensus; use reth_db::mdbx::{Env, WriteMap}; use reth_downloaders::{ bodies::bodies::BodiesDownloaderBuilder, diff --git a/bin/reth/src/node/mod.rs b/bin/reth/src/node/mod.rs index f16fdc9e550..225676bd7fd 100644 --- a/bin/reth/src/node/mod.rs +++ b/bin/reth/src/node/mod.rs @@ -13,7 +13,7 @@ use events::NodeEvent; use eyre::Context; use fdlimit::raise_fd_limit; use futures::{pin_mut, stream::select as stream_select, Stream, StreamExt}; -use reth_consensus::beacon::BeaconConsensus; +use reth_beacon_consensus::BeaconConsensus; use reth_db::{ database::Database, mdbx::{Env, WriteMap}, diff --git a/bin/reth/src/stage/mod.rs b/bin/reth/src/stage/mod.rs index bcb079abf21..ca0c4fb366d 100644 --- a/bin/reth/src/stage/mod.rs +++ b/bin/reth/src/stage/mod.rs @@ -7,7 +7,7 @@ use crate::{ prometheus_exporter, }; use clap::{Parser, ValueEnum}; -use reth_consensus::beacon::BeaconConsensus; +use reth_beacon_consensus::BeaconConsensus; use reth_downloaders::bodies::bodies::BodiesDownloaderBuilder; use reth_primitives::ChainSpec; use reth_provider::{ShareableDatabase, Transaction}; diff --git a/crates/consensus/Cargo.toml b/crates/consensus/Cargo.toml deleted file mode 100644 index fbd98504de5..00000000000 --- a/crates/consensus/Cargo.toml +++ /dev/null @@ -1,22 +0,0 @@ -[package] -name = "reth-consensus" -version = "0.1.0" -edition = "2021" -license = "MIT OR Apache-2.0" -repository = "https://github.com/paradigmxyz/reth" -readme = "README.md" - -[dependencies] -# reth -reth-primitives = { path = "../primitives" } -reth-interfaces = { path = "../interfaces" } -reth-provider = { path = "../storage/provider" } - -# async -tokio = { version = "1", features = ["sync"] } - -[dev-dependencies] -reth-interfaces = { path = "../interfaces", features = ["test-utils"] } -reth-provider = { path = "../storage/provider", features = ["test-utils"] } -assert_matches = "1.5.0" -mockall = "0.11.3" diff --git a/crates/consensus/beacon/Cargo.toml b/crates/consensus/beacon/Cargo.toml new file mode 100644 index 00000000000..90a304372eb --- /dev/null +++ b/crates/consensus/beacon/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "reth-beacon-consensus" +version = "0.1.0" +edition = "2021" +license = "MIT OR Apache-2.0" +repository = "https://github.com/paradigmxyz/reth" +readme = "README.md" + +[dependencies] +# reth +reth-consensus-common = { path = "../common" } +reth-primitives = { path = "../../primitives" } +reth-interfaces = { path = "../../interfaces" } + +# async +tokio = { version = "1", features = ["sync"] } + +[dev-dependencies] +reth-interfaces = { path = "../../interfaces", features = ["test-utils"] } diff --git a/crates/consensus/src/beacon/beacon_consensus.rs b/crates/consensus/beacon/src/beacon_consensus.rs similarity index 98% rename from crates/consensus/src/beacon/beacon_consensus.rs rename to crates/consensus/beacon/src/beacon_consensus.rs index d39895c3795..5f7d2260230 100644 --- a/crates/consensus/src/beacon/beacon_consensus.rs +++ b/crates/consensus/beacon/src/beacon_consensus.rs @@ -1,9 +1,8 @@ //! Consensus for ethereum network -use std::sync::Arc; - -use crate::validation; +use reth_consensus_common::validation; use reth_interfaces::consensus::{Consensus, ConsensusError, ForkchoiceState}; use reth_primitives::{ChainSpec, Hardfork, SealedBlock, SealedHeader, EMPTY_OMMER_ROOT, U256}; +use std::sync::Arc; use tokio::sync::watch; use super::BeaconConsensusBuilder; diff --git a/crates/consensus/src/beacon/builder.rs b/crates/consensus/beacon/src/builder.rs similarity index 100% rename from crates/consensus/src/beacon/builder.rs rename to crates/consensus/beacon/src/builder.rs diff --git a/crates/consensus/beacon/src/lib.rs b/crates/consensus/beacon/src/lib.rs new file mode 100644 index 00000000000..5ecb1a1e05f --- /dev/null +++ b/crates/consensus/beacon/src/lib.rs @@ -0,0 +1,13 @@ +#![warn(missing_docs, unreachable_pub, unused_crate_dependencies)] +#![deny(unused_must_use, rust_2018_idioms)] +#![doc(test( + no_crate_inject, + attr(deny(warnings, rust_2018_idioms), allow(dead_code, unused_variables)) +))] +//! Beacon consensus implementation. + +mod beacon_consensus; +mod builder; + +pub use beacon_consensus::BeaconConsensus; +pub use builder::BeaconConsensusBuilder; diff --git a/crates/consensus/common/Cargo.toml b/crates/consensus/common/Cargo.toml new file mode 100644 index 00000000000..627f896657d --- /dev/null +++ b/crates/consensus/common/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "reth-consensus-common" +version = "0.1.0" +edition = "2021" +license = "MIT OR Apache-2.0" +repository = "https://github.com/paradigmxyz/reth" +readme = "README.md" + +[dependencies] +# reth +reth-primitives = { path = "../../primitives" } +reth-interfaces = { path = "../../interfaces" } +reth-provider = { path = "../../storage/provider" } + + +[dev-dependencies] +reth-interfaces = { path = "../../interfaces", features = ["test-utils"] } +reth-provider = { path = "../../storage/provider", features = ["test-utils"] } +assert_matches = "1.5.0" +mockall = "0.11.3" diff --git a/crates/consensus/src/lib.rs b/crates/consensus/common/src/lib.rs similarity index 76% rename from crates/consensus/src/lib.rs rename to crates/consensus/common/src/lib.rs index 35bf33b0334..4b1917d612d 100644 --- a/crates/consensus/src/lib.rs +++ b/crates/consensus/common/src/lib.rs @@ -4,10 +4,8 @@ no_crate_inject, attr(deny(warnings, rust_2018_idioms), allow(dead_code, unused_variables)) ))] -//! Consensus algorithms for Ethereum. -/// Beacon consensus implementation. -pub mod beacon; +//! Commonly used consensus methods. /// Collection of consensus validation methods. pub mod validation; diff --git a/crates/consensus/src/validation.rs b/crates/consensus/common/src/validation.rs similarity index 99% rename from crates/consensus/src/validation.rs rename to crates/consensus/common/src/validation.rs index af283908597..8a0d8f733ac 100644 --- a/crates/consensus/src/validation.rs +++ b/crates/consensus/common/src/validation.rs @@ -1,8 +1,8 @@ //! Collection of methods for block validation. use reth_interfaces::{consensus::ConsensusError, Result as RethResult}; use reth_primitives::{ - BlockNumber, ChainSpec, Hardfork, Header, InvalidTransactionError, SealedBlock, SealedHeader, - Transaction, TransactionSignedEcRecovered, TxEip1559, TxEip2930, TxLegacy, + constants, BlockNumber, ChainSpec, Hardfork, Header, InvalidTransactionError, SealedBlock, + SealedHeader, Transaction, TransactionSignedEcRecovered, TxEip1559, TxEip2930, TxLegacy, }; use reth_provider::{AccountProvider, HeaderProvider, WithdrawalsProvider}; use std::{ @@ -10,8 +10,6 @@ use std::{ time::SystemTime, }; -use reth_primitives::constants; - /// Validate header standalone pub fn validate_header_standalone( header: &SealedHeader, diff --git a/crates/consensus/src/beacon/mod.rs b/crates/consensus/src/beacon/mod.rs deleted file mode 100644 index ec2d4ab5c15..00000000000 --- a/crates/consensus/src/beacon/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -//! Beacon consensus implementation. - -mod beacon_consensus; -mod builder; - -pub use beacon_consensus::BeaconConsensus; -pub use builder::BeaconConsensusBuilder; From 7c877cb9213fdf78ac6621973247f1a1763575ab Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 17 Mar 2023 13:20:11 +0100 Subject: [PATCH 164/191] chore: cleanup some staged-sync deps (#1815) --- Cargo.lock | 41 --------------------- crates/staged-sync/Cargo.toml | 21 ++--------- crates/staged-sync/src/lib.rs | 9 +++++ crates/staged-sync/src/test_utils/clique.rs | 4 +- crates/staged-sync/src/test_utils/mod.rs | 2 - crates/staged-sync/src/utils/init.rs | 12 +++--- 6 files changed, 22 insertions(+), 67 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 09fd1b2f003..455fb0ddc9d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -199,28 +199,6 @@ dependencies = [ "event-listener", ] -[[package]] -name = "async-stream" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad445822218ce64be7a341abfb0b1ea43b5c23aa83902542a4542e78309d8e5e" -dependencies = [ - "async-stream-impl", - "futures-core", - "pin-project-lite", -] - -[[package]] -name = "async-stream-impl" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4655ae1a7b0cdf149156f780c5bf3f1352bc53cbd9e0a361a7ef7b22947e965" -dependencies = [ - "proc-macro2 1.0.52", - "quote 1.0.23", - "syn 1.0.109", -] - [[package]] name = "async-trait" version = "0.1.65" @@ -5269,15 +5247,11 @@ dependencies = [ "reth-db", "reth-discv4", "reth-downloaders", - "reth-interfaces", "reth-net-nat", "reth-network", "reth-network-api", "reth-primitives", "reth-provider", - "reth-staged-sync", - "reth-stages", - "reth-tasks", "reth-tracing", "secp256k1", "serde", @@ -5286,8 +5260,6 @@ dependencies = [ "tempfile", "thiserror", "tokio", - "tokio-stream", - "tokio-test", "tracing", "walkdir", ] @@ -6592,19 +6564,6 @@ dependencies = [ "tokio-util 0.7.7", ] -[[package]] -name = "tokio-test" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53474327ae5e166530d17f2d956afcb4f8a004de581b3cae10f12006bc8163e3" -dependencies = [ - "async-stream", - "bytes", - "futures-core", - "tokio", - "tokio-stream", -] - [[package]] name = "tokio-tungstenite" version = "0.18.0" diff --git a/crates/staged-sync/Cargo.toml b/crates/staged-sync/Cargo.toml index 5ab9d6c58aa..d81750ff669 100644 --- a/crates/staged-sync/Cargo.toml +++ b/crates/staged-sync/Cargo.toml @@ -7,12 +7,6 @@ repository = "https://github.com/paradigmxyz/reth" readme = "README.md" description = "Puts together all the Reth stages in a unified abstraction" -[package.metadata.cargo-udeps.ignore] -normal = [ - # Used for config loading - "confy" -] - [dependencies] # reth reth-db = {path = "../../crates/storage/db", features = ["mdbx", "test-utils"] } @@ -23,13 +17,10 @@ reth-downloaders = { path = "../../crates/net/downloaders" } reth-primitives = { path = "../../crates/primitives" } reth-provider = { path = "../../crates/storage/provider", features = ["test-utils"] } reth-net-nat = { path = "../../crates/net/nat" } -reth-interfaces = { path = "../interfaces", optional = true } -reth-tasks = { path = "../../crates/tasks" } # io serde = "1.0" serde_json = "1.0.91" -confy = "0.5" # misc walkdir = "2.3.2" @@ -55,23 +46,18 @@ ethers-signers = { git = "https://github.com/gakonst/ethers-rs", default-feature # async / futures async-trait = { version = "0.1", optional = true } tokio = { version = "1", features = ["io-util", "net", "macros", "rt-multi-thread", "time"], optional = true } -tokio-test = { version = "0.4", optional = true } # misc -tempfile = { version = "3.3", optional = true } hex = { version = "0.4", optional = true } [dev-dependencies] # reth crates reth-tracing = { path = "../tracing" } -reth-stages = { path = "../stages" } reth-downloaders = { path = "../net/downloaders" } -reth-staged-sync = { path = ".", features = ["test-utils"] } # async/futures futures = "0.3" tokio = { version = "1", features = ["io-util", "net", "macros", "rt-multi-thread", "time"] } -tokio-stream = "0.1" # crypto secp256k1 = { version = "0.26.0", features = [ @@ -80,19 +66,20 @@ secp256k1 = { version = "0.26.0", features = [ "recovery", ] } +confy = "0.5" + +tempfile = "3.4" + [features] test-utils = [ "reth-network/test-utils", - "reth-interfaces/test-utils", "reth-network/test-utils", "reth-provider/test-utils", "dep:enr", "dep:ethers-core", - "dep:tempfile", "dep:hex", "dep:rand", "dep:tokio", - "dep:tokio-test", "dep:ethers-signers", "dep:ethers-providers", "dep:ethers-middleware", diff --git a/crates/staged-sync/src/lib.rs b/crates/staged-sync/src/lib.rs index 781199fb5e6..a0ea405be98 100644 --- a/crates/staged-sync/src/lib.rs +++ b/crates/staged-sync/src/lib.rs @@ -1,3 +1,12 @@ +#![warn(missing_docs, unreachable_pub)] +#![deny(unused_must_use, rust_2018_idioms)] +#![doc(test( + no_crate_inject, + attr(deny(warnings, rust_2018_idioms), allow(dead_code, unused_variables)) +))] + +//! Puts together all the Reth stages in a unified abstraction + pub mod config; pub use config::Config; diff --git a/crates/staged-sync/src/test_utils/clique.rs b/crates/staged-sync/src/test_utils/clique.rs index 8a75e600982..c053d8ee03a 100644 --- a/crates/staged-sync/src/test_utils/clique.rs +++ b/crates/staged-sync/src/test_utils/clique.rs @@ -20,7 +20,7 @@ use std::{ /// ```no_run /// # use ethers_core::utils::Geth; /// # use reth_staged_sync::test_utils::CliqueGethInstance; -/// # tokio_test::block_on(async { +/// # let clique = async { /// /// // this creates a funded geth /// let clique_geth = Geth::new() @@ -32,7 +32,7 @@ use std::{ /// /// // don't print logs, but drain the stderr /// clique.prevent_blocking().await; -/// # }); +/// # }; /// ``` pub struct CliqueGethInstance { /// The spawned [`GethInstance`](ethers_core::utils::GethInstance). diff --git a/crates/staged-sync/src/test_utils/mod.rs b/crates/staged-sync/src/test_utils/mod.rs index 51dc7cd5798..ff1105afade 100644 --- a/crates/staged-sync/src/test_utils/mod.rs +++ b/crates/staged-sync/src/test_utils/mod.rs @@ -1,5 +1,3 @@ -#![warn(missing_docs, unreachable_pub)] - //! Common helpers for staged sync integration testing. pub mod clique; diff --git a/crates/staged-sync/src/utils/init.rs b/crates/staged-sync/src/utils/init.rs index d911d5d2297..7ee1d76382c 100644 --- a/crates/staged-sync/src/utils/init.rs +++ b/crates/staged-sync/src/utils/init.rs @@ -12,10 +12,7 @@ use tracing::debug; /// Opens up an existing database or creates a new one at the specified path. pub fn init_db>(path: P) -> eyre::Result> { std::fs::create_dir_all(path.as_ref())?; - let db = reth_db::mdbx::Env::::open( - path.as_ref(), - reth_db::mdbx::EnvKind::RW, - )?; + let db = Env::::open(path.as_ref(), reth_db::mdbx::EnvKind::RW)?; db.create_tables()?; Ok(db) @@ -26,7 +23,12 @@ pub fn init_db>(path: P) -> eyre::Result> { pub enum InitDatabaseError { /// Attempted to reinitialize database with inconsistent genesis block #[error("Genesis hash mismatch: expected {expected}, got {actual}")] - GenesisHashMismatch { expected: H256, actual: H256 }, + GenesisHashMismatch { + /// Expected genesis hash. + expected: H256, + /// Actual genesis hash. + actual: H256, + }, /// Low-level database error. #[error(transparent)] From ef51f566fe1565d81692590424b8bba9fdfd6302 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 17 Mar 2023 14:16:09 +0100 Subject: [PATCH 165/191] chore(deps): bump revm and use new utils (#1816) --- Cargo.lock | 8 ++++---- Cargo.toml | 4 ++-- crates/rpc/rpc/src/eth/error.rs | 10 ++-------- 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 455fb0ddc9d..d5bd3c910fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5347,7 +5347,7 @@ dependencies = [ [[package]] name = "revm" version = "3.0.0" -source = "git+https://github.com/bluealloy/revm?rev=afc3066#afc30663270f77df9b4399ad9d4cfb0ad2b814ec" +source = "git+https://github.com/bluealloy/revm#3d8ca6641d2e72448c23f4596f769c8fd1c784d1" dependencies = [ "auto_impl", "revm-interpreter", @@ -5357,7 +5357,7 @@ dependencies = [ [[package]] name = "revm-interpreter" version = "1.0.0" -source = "git+https://github.com/bluealloy/revm?rev=afc3066#afc30663270f77df9b4399ad9d4cfb0ad2b814ec" +source = "git+https://github.com/bluealloy/revm#3d8ca6641d2e72448c23f4596f769c8fd1c784d1" dependencies = [ "derive_more", "enumn", @@ -5368,7 +5368,7 @@ dependencies = [ [[package]] name = "revm-precompile" version = "2.0.0" -source = "git+https://github.com/bluealloy/revm?rev=afc3066#afc30663270f77df9b4399ad9d4cfb0ad2b814ec" +source = "git+https://github.com/bluealloy/revm#3d8ca6641d2e72448c23f4596f769c8fd1c784d1" dependencies = [ "k256 0.11.6", "num", @@ -5384,7 +5384,7 @@ dependencies = [ [[package]] name = "revm-primitives" version = "1.0.0" -source = "git+https://github.com/bluealloy/revm?rev=afc3066#afc30663270f77df9b4399ad9d4cfb0ad2b814ec" +source = "git+https://github.com/bluealloy/revm#3d8ca6641d2e72448c23f4596f769c8fd1c784d1" dependencies = [ "arbitrary", "auto_impl", diff --git a/Cargo.toml b/Cargo.toml index 0481e20bd9e..0da5e418d6e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,7 +48,7 @@ inherits = "release" debug = true [patch.crates-io] -revm = { git = "https://github.com/bluealloy/revm", rev = "afc3066" } -revm-primitives = { git = "https://github.com/bluealloy/revm", rev = "afc3066" } +revm = { git = "https://github.com/bluealloy/revm" } +revm-primitives = { git = "https://github.com/bluealloy/revm" } # patched for quantity U256 responses ruint = { git = "https://github.com/paradigmxyz/uint" } diff --git a/crates/rpc/rpc/src/eth/error.rs b/crates/rpc/rpc/src/eth/error.rs index 8260db82a86..0615a39df07 100644 --- a/crates/rpc/rpc/src/eth/error.rs +++ b/crates/rpc/rpc/src/eth/error.rs @@ -5,7 +5,7 @@ use jsonrpsee::{core::Error as RpcError, types::error::INVALID_PARAMS_CODE}; use reth_primitives::{constants::SELECTOR_LEN, Address, Bytes, U256}; use reth_rpc_types::{error::EthRpcErrorCode, BlockError}; use reth_transaction_pool::error::{InvalidPoolTransactionError, PoolError}; -use revm::primitives::{EVMError, ExecutionResult, Halt, OutOfGasError, Output}; +use revm::primitives::{EVMError, ExecutionResult, Halt, OutOfGasError}; /// Result alias pub type EthResult = Result; @@ -368,13 +368,7 @@ pub enum SignError { /// [ExecutionResult::Success]. pub(crate) fn ensure_success(result: ExecutionResult) -> EthResult { match result { - ExecutionResult::Success { output, .. } => { - let data = match output { - Output::Call(data) => data, - Output::Create(data, _) => data, - }; - Ok(data.into()) - } + ExecutionResult::Success { output, .. } => Ok(output.into_data().into()), ExecutionResult::Revert { output, .. } => { Err(InvalidTransactionError::Revert(RevertError::new(output)).into()) } From 9036f5ea9f5a3dfe34972a69569a30864be3416c Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Fri, 17 Mar 2023 15:16:29 +0200 Subject: [PATCH 166/191] style: prefer `then_some` over `if else` (#1810) --- bin/reth/src/test_eth_chain/runner.rs | 2 +- crates/executor/src/executor.rs | 2 +- crates/metrics/metrics-derive/tests/metrics.rs | 6 +++--- crates/revm/revm-primitives/src/compat.rs | 2 +- crates/rpc/rpc/src/eth/api/server.rs | 3 +-- crates/rpc/rpc/src/eth/filter.rs | 3 +-- crates/storage/provider/src/test_utils/mock.rs | 5 ++--- 7 files changed, 10 insertions(+), 13 deletions(-) diff --git a/bin/reth/src/test_eth_chain/runner.rs b/bin/reth/src/test_eth_chain/runner.rs index d65760fb55a..277c22d74bf 100644 --- a/bin/reth/src/test_eth_chain/runner.rs +++ b/bin/reth/src/test_eth_chain/runner.rs @@ -152,7 +152,7 @@ pub async fn run_test(path: PathBuf) -> eyre::Result { pre_state.into_iter().try_for_each(|(address, account)| -> eyre::Result<()> { let has_code = !account.code.is_empty(); - let code_hash = if has_code { Some(keccak256(&account.code)) } else { None }; + let code_hash = has_code.then(|| keccak256(&account.code)); tx.put::( address, RethAccount { diff --git a/crates/executor/src/executor.rs b/crates/executor/src/executor.rs index f292a297f89..6a8acf4209d 100644 --- a/crates/executor/src/executor.rs +++ b/crates/executor/src/executor.rs @@ -573,7 +573,7 @@ mod tests { Ok(self .block_hash .iter() - .filter_map(|(block, hash)| if range.contains(block) { Some(*hash) } else { None }) + .filter_map(|(block, hash)| range.contains(block).then_some(*hash)) .collect()) } } diff --git a/crates/metrics/metrics-derive/tests/metrics.rs b/crates/metrics/metrics-derive/tests/metrics.rs index 8f260761301..cca5ef83ca5 100644 --- a/crates/metrics/metrics-derive/tests/metrics.rs +++ b/crates/metrics/metrics-derive/tests/metrics.rs @@ -303,21 +303,21 @@ impl Recorder for TestRecorder { fn register_counter(&self, key: &Key) -> Counter { let labels_vec: Vec